copied cosmostat web dashboard for init dashboard work

README.md

@@ -1,10 +1,14 @@
-# Xiaomii BLE Sensor API
+# Xiaomii BLE Sensor Dashboard
 
 ## Summary
 
-This is an API for monitoring Temperature and Humidity data for Xiaomii LYWSD03MMC BLE sensors. The app will scan for any of these devices by looking for the Bluetooth MAC octet and then filtering by model. Once it has a list of MACs, it will then attempt to read the Temp/Humidity data for each MAC and record it. This data can be interacted with through an API. The app has a Redis publishing function for use with websockets. The comment field for sensors can be configured using the appropriate variable in the `config.py` as well. This app uses *Pickle* to persist Sensor object data between App launches. This doesn't keep history like Telegraf would, but it will allow for faster API startup. 
+This is a web dashboard for monitoring Temperature and Humidity data for Xiaomii LYWSD03MMC BLE sensors. This app has a Python API to discover and monitor the sensors, and will have a website that interfaces with it. The website will load the intial static content with the API, and has a Node.js server for handling live web socket data using the Redis server.
 
-Some product info can be found here at my Knowledgebase:
+The Python API will scan for any of these devices by looking for the Bluetooth MAC octet and then filtering by model. Once it has a list of MACs, it will then attempt to read the Temp/Humidity data for each MAC and record it. This data can be interacted with through an API. The app has a Redis publishing function for use with websockets. The comment field for sensors can be configured using the appropriate variable in the `config.py` as well. This app uses *Pickle* to persist Sensor object data between App launches. This doesn't keep history like Telegraf would, but it will allow for faster API startup. 
+
+The web dashboard lives in a Docker container based on PHP with Node and Redis added, and is managed by Supervisord. The static page is rendered with PHP and live data will be updated with Node.js based on the data received by Redis. The tech stack is Apache+PHP for the HTML, and Node.js for Websockets. The page loads statically with the API initially and will get updates from the Node server when Redis data is received. 
+
+Some product info for these sensors can be found here at my Knowledgebase:
 
 https://kb.matt-cloud.com/books/07-saved-articles/page/xiaomii-temperaturehumidity-sensor
 

api/app.py

@@ -0,0 +1,61 @@
+import redis
+import atexit
+from flask import Flask, jsonify
+from flask_apscheduler import APScheduler
+from datetime import datetime, timedelta
+from scanner import *
+from config import *
+from ble_redis import *
+import logging
+logging.getLogger('apscheduler').setLevel(logging.ERROR)  
+
+
+app = Flask(__name__)
+scheduler = APScheduler()
+scheduler.init_app(app)
+scheduler.start()
+
+
+scan_macs()
+# --- Scheduler job ----------------------------------------------------
+@scheduler.task(
+    'interval',
+    id='ble_scan',
+    minutes=SCAN_INTERVAL_MINUTES,
+    misfire_grace_time=30
+)
+def ble_scan_job():
+    """Runs in the background, calls the async scanner."""
+    update_cache()
+
+
+# --- API -------------------------------------------------------------
+@app.route('/macs', methods=['GET'])
+def list_macs():
+    """Return the current set of MACs as JSON."""
+    macs = sorted(get_cache_snapshot())
+    return jsonify({"count": len(macs), "macs": macs})
+
+@app.route('/details', methods=['GET'])
+def sensor_detail():
+    """Return the current set of MACs as JSON."""
+    sensors = get_sensor_details()
+    return jsonify(sensors)
+
+@app.route('/summary', methods=['GET'])
+def sensor_summary():
+    """Return the current set of MACs as JSON."""
+    sensors = get_sensor_summary()
+    return jsonify(sensors)
+    
+# --- Convenience route -----------------------------------------------
+@app.route('/', methods=['GET'])
+def hello():
+    return "BLE Scanner API - see /macs for the list."
+
+# --- Run --------------------------------------------------------------
+if __name__ == '__main__':
+    # The scheduler runs in its own thread; the main thread serves Flask.
+    app.run(host=bind_ip, port=api_port, debug=False)
+    # Register a graceful shutdown hook so the scanner state is persisted
+    atexit.register(save_state)

api/ble.py

@@ -0,0 +1,124 @@
+import asyncio
+from bleak import BleakClient, BleakScanner
+import warnings, logging
+warnings.filterwarnings("ignore")
+logging.getLogger('apscheduler').setLevel(logging.ERROR)
+
+# Mi sensor UUIDs
+MI_SERVICE_UUID = "ebe0ccb0-7a0a-4b0c-8a1a-6ff2997da3a6"
+TEMP_CHAR_UUID = "ebe0ccc1-7a0a-4b0c-8a1a-6ff2997da3a6"
+
+# scanner constants
+full_timeout = 60
+attempts = 6
+
+async def read_data(mac_address):
+
+    # Scan for the device
+    ratio = full_timeout / attempts
+    fail_count = full_timeout / ratio
+    device = None
+    print(f"Beginning read_data for {mac_address} - {full_timeout} timeout - {ratio} ratio - please stand by...")
+    while fail_count > 0 and not device:
+        # print(f"Starting discover loop {fail_count}")
+        devices = await BleakScanner.discover(timeout=ratio)
+        # print("BleakScanner discover timeout pass")
+        device = next((d for d in devices if d.address.lower() == mac_address.lower()), None)
+        if device:
+            # print(f"BleakScanner device variable loaded - {device.name}")
+            print(f"..  Found device: {device.name} ({device.address}) - connecting...")
+            fail_count = 0
+        # else:
+        #     print("BleakScanner device not loaded yet")
+        fail_count = fail_count - 1
+
+    if not device:
+        print(f"!!  Device with MAC {mac_address} not found")
+        return None
+
+    # Connect to the device
+    mi_service = None
+    async with BleakClient(device) as client:
+
+        success = False
+        # Check if Mi service is available
+
+        #print("..  Waiting on client to connect...")
+        while not client.is_connected:
+            await asyncio.sleep(0.1)
+        while not success:
+            try:
+                #print("..  Client connected, getting services")
+                services = client.services
+                #print("..  Services acquired, loading uuid matches")
+                mi_service = next((s for s in services if s.uuid.lower() == MI_SERVICE_UUID.lower()), None)
+                success = True
+                #print("uuid matches loaded")
+            except Exception as e:
+                print(f"Mi service unavailable, infinite loop looping RIP - {e}")
+                            
+            # Read temperature data
+            sensor_data = await client.read_gatt_char(TEMP_CHAR_UUID)
+
+            # Parse the temperature value (same logic as in JavaScript)
+            sign = sensor_data[1] & (1 << 7)
+            temp = ((sensor_data[1] & 0x7F) << 8 | sensor_data[0])
+            if sign:
+                temp = temp - 32767
+            temp = temp / 100
+            hum = sensor_data[2]
+
+            print(f"--  Temperature: {temp:.1f}°C, Humidity: {hum}%")
+            return temp, hum
+
+def sensor_data_query(mac_list):
+    result = []
+    for mac in mac_list:
+        try:
+            this_result = asyncio.run(read_data(mac))
+        except Exception as e:
+            print(f"Error reading BLE data - {e}")
+        result.append({"name": mac, "temp": this_result[0], "humidity": this_result[1]})
+    print(f"Data query complete - returning result")
+    return result
+
+def test():    
+    test_result = False
+    sensors = []
+    mac_list = [
+        "A4:C1:38:11:F4:2A",
+        "A4:C1:38:24:15:B0",
+        "A4:C1:38:D7:8D:26"
+    ]
+
+    print("Starting BLE test...")
+    print("  Scanning MACs:")
+    for mac in mac_list:
+        print(f"    {mac}")
+    test_list = mac_list
+    while not test_result:
+        print("  Test loop start. Initializing run...")
+        test_run_results = sensor_data_query(test_list)
+        for test_run_result in test_run_results:
+            sensors.append(test_run_result)
+        passed_list =  [device['name'] for device in sensors]
+        print(f"    Passed list - {passed_list}")
+        if len(mac_list) == len(passed_list):
+            print("  Test run passed, yee-haw")
+            test_result = True
+        else:
+            print(f"  Test run incomplete/failed, re-running, updating test list")
+            mac_list_new = []
+            for mac in mac_list:
+                if mac not in passed_list:
+                    mac_list_new.append(mac)
+            test_list = mac_list_new
+            print(f"  Re-do list - {test_list}")
+
+if __name__ == "__main__":
+    #print("Welp, this will keep going forever. Have fun :)")
+    #while True:
+    #try:
+    test()
+    #except Exception as e:
+    #    print(f"Test failed brah - {e}")

api/ble_redis.py

@@ -0,0 +1,19 @@
+
+import redis
+import json
+from config import *
+
+####################################################
+### Redis Functions
+####################################################
+r = redis.Redis(host=bind_ip, port=redis_port)
+
+def update_redis(data):
+    # push data to redis
+    # Publish to the Redis channel that the WS server is listening on
+    r.publish('temp_hum', json.dumps(data))
+    if debug_output:
+        print("=== Stats Redis Update ===")
+        print(json.dumps(data, indent=2))
+    return True
+

api/config.py

@@ -0,0 +1,29 @@
+# --- BLE & app config -------------------------------------------------
+
+MAC_PREFIX = "A4:C1:38"
+SCAN_INTERVAL_MINUTES = 1
+debug_output = False
+bind_ip = "0.0.0.0"
+api_port = 5000
+redis_port = 6379
+enable_redis = True
+
+# --- This will store friendly names for MAC addresses -----------------
+comment_lookup = [
+    {
+        "mac": "A4:C1:38:11:F4:2A",
+        "comment": "Kitchen - 0"
+    },
+    {
+        "mac": "A4:C1:38:D7:8D:26",
+        "comment": "Table - 1"
+    },
+    {
+        "mac": "A4:C1:38:24:15:B0",
+        "comment": "Hallway 2"
+    },
+    {
+        "mac": "A4:C1:38:96:53:30",
+        "comment": "Bedroom 3"
+    }
+]

api/requirements.txt

@@ -0,0 +1,9 @@
+anyio
+bleak
+cryptography
+click
+setuptools
+Flask-APScheduler
+Flask
+APScheduler
+redis

api/run.sh

@@ -0,0 +1,11 @@
+
+# 3. Create a venv (optional but recommended)
+python3 -m venv .venv
+source .venv/bin/activate
+
+# 4. Install dependencies
+pip install -r requirements.txt --quiet
+
+# 5. Start
+#python ble.py
+python app.py

api/scanner.py

@@ -0,0 +1,189 @@
+import asyncio
+import threading
+import time
+import redis
+import pickle
+import os
+from datetime import datetime, timezone
+from zoneinfo import ZoneInfo   # Python 3.9+
+from bleak import BleakScanner, BleakClient
+from typing import Set, Iterable
+from config import *
+from sensors import *
+from ble import *
+from ble_redis import *
+
+# In-memory cache + thread-safe lock
+_mac_cache: Set[str] = set()
+_lock = threading.Lock()
+_sensors_cache = []
+
+# ----------------------------------------------------------------------
+# Pickle persistence helpers
+# ----------------------------------------------------------------------
+STATE_FILE = "scanner_state.pkl"
+
+def save_state() -> None:
+    """
+    Persist the MAC cache and sensor objects to disk.
+    Called automatically after each cache refresh and at program exit.
+    """
+    with _lock:
+        state = {
+            "mac_cache": _mac_cache,
+            "sensors": _sensors_cache,
+        }
+    try:
+        with open(STATE_FILE, "wb") as f:
+            pickle.dump(state, f, protocol=pickle.HIGHEST_PROTOCOL)
+        print("[Scanner] State persisted to", STATE_FILE)
+    except Exception as exc:
+        # Don't crash the app – just log the error
+        print("[Scanner] Warning: could not persist state:", exc)
+
+def load_state() -> None:
+    """
+    Load the MAC cache and sensor objects from disk if the file exists.
+    Called automatically on import.
+    """
+    if not os.path.exists(STATE_FILE):
+        return
+    try:
+        with open(STATE_FILE, "rb") as f:
+            state = pickle.load(f)
+        with _lock:
+            # NOTE: we replace the lists/sets to avoid mutating old references
+            _mac_cache.clear()
+            _mac_cache.update(state.get("mac_cache", []))
+            global _sensors_cache
+            _sensors_cache = state.get("sensors", [])
+        print("[Scanner] Loaded state from", STATE_FILE)
+        for cached_mac in _mac_cache:
+            print(f"Found {cached_mac}")
+    except Exception as exc:
+        print("[Scanner] Warning: could not load state:", exc)
+
+# Load any persisted state on module import
+load_state()
+
+# Redis Functions
+r = redis.Redis(host=bind_ip, port=redis_port)
+
+def update_redis(data):
+    # push data to redis
+    # Publish to the Redis channel that the WS server is listening on
+    r.publish('temp_hum', json.dumps(data))
+    if debug_output:
+        print("=== Stats Redis Update ===")
+        print(json.dumps(data, indent=2))
+    return True
+
+# this function filters MACs
+def _is_match(mac: str, name: str) -> bool:
+    """Return True if the MAC starts with the configured prefix and is correct model."""
+    if name == "LYWSD03MMC":
+        return mac.upper().startswith(MAC_PREFIX.upper())
+
+# this function scans for MACs
+async def _scan_once() -> Iterable[str]:
+    """Do one scan cycle and return the matching MACs."""
+    devices = await BleakScanner.discover(timeout=5.0)  # 5s scan
+    return [dev.address for dev in devices if _is_match(dev.address, dev.name)]
+
+# this very important function updates the MAC cache and the sensor objects
+def update_cache() -> None:
+    """
+    Called by the scheduler (in a separate thread).  It runs a tiny
+    asyncio event loop to call the Bleak scan, then updates the cache.
+    """
+    # Run the async scan in a fresh loop; Bleak requires this.
+    # Gets the MACs
+    print("")
+    print("--- Starting update_cache loop ---")
+    new_macs = scan_macs()
+    # now that we have MACs, let's get sensor data
+    print("Updating sensor data...")
+    sensor_data = None
+    try:
+        sensor_data = sensor_data_query(new_macs)
+        # print(sensor_data)
+    except Exception as e:
+        print(f"Aww fuck, no data :( - {e}")
+    # now we update the data in the sensor objects
+    print("Sensor data collected:")
+    for sensor_datum in sensor_data:
+        print(f"MAC - {sensor_datum['name']} | temp - {sensor_datum['temp']} | humidity - {sensor_datum['humidity']}")
+        for sensor in _sensors_cache:
+            if sensor.name == sensor_datum['name']:
+                sensor.add_or_update(key="temp", value=sensor_datum['temp'])
+                sensor.add_or_update(key="humidity", value=sensor_datum['humidity'])
+    # if redis is enabled, push full details to the redis server
+    if enable_redis:
+        update_redis(get_sensor_details())
+    save_state()
+
+# this helper function scans for MACs
+def scan_macs():
+    print("Scanning for MACs:")
+    new_macs = asyncio.run(_scan_once())
+    with _lock:        
+        for new_mac in new_macs:
+            print(f"{new_mac}")
+            new_sensor = Sensor(name=f"{new_mac}")
+            if new_mac not in set(_mac_cache):
+                _sensors_cache.append(new_sensor)
+        _mac_cache.update(new_macs)
+    print(f"Found {len(new_macs)} new MACs, total {len(_mac_cache)} MACs in cache.")
+    return new_macs
+
+# return just the MACs
+def get_cache_snapshot() -> Set[str]:
+    """Return a thread-safe snapshot of the current cache."""
+    with _lock:
+        return set(_mac_cache)
+
+# return sensor data and timestamps, can return int or string
+def get_sensor_details(friendly_date = False):
+    sensor_list = []
+    for sensor in _sensors_cache:
+        first_seen = sensor.first_seen
+        last_seen = sensor.last_seen
+        if friendly_date:
+            first_seen = ts_to_pst_string(sensor.first_seen)
+            last_seen = ts_to_pst_string(sensor.last_seen)
+        # print(f"Summary - {sensor.name}")
+        new_sensor = {
+            "name": sensor.name,
+            "first_seen": first_seen,
+            "last_seen": last_seen,
+            "comment": sensor.comment,
+            "data": sensor.to_dict()
+
+        }
+        sensor_list.append(new_sensor)
+    result = {
+        "sensors": sensor_list
+    }
+    return result
+
+# return just sensor data without timestamps
+def get_sensor_summary():
+    sensor_list = []
+    for sensor in _sensors_cache:
+        # print(f"Summary - {sensor.name}")
+        new_sensor = {
+            "name": sensor.name,
+            "comment": sensor.comment,
+            "data": sensor.to_dict(simple_return=True)
+        }
+        sensor_list.append(new_sensor)
+    result = {
+        "sensors": sensor_list
+    }
+    return result
+
+# helper function for ts int to string
+def ts_to_pst_string(ts_str):
+    utc_dt = datetime.fromtimestamp(ts_str, tz=timezone.utc)
+    pacific_dt = utc_dt.astimezone(ZoneInfo("America/Los_Angeles"))
+    return pacific_dt.strftime("%Y-%m-%d %H:%M:%S")

api/sensors.py

@@ -0,0 +1,207 @@
+from __future__ import annotations
+
+import re, redis
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional, Union
+from config import *
+
+# --------------------------------------------------------------------------- #
+#  Helper utilities
+# --------------------------------------------------------------------------- #
+
+def _current_timestamp() -> int:
+    """Return the current UTC timestamp (seconds)."""
+    return int(datetime.now(timezone.utc).timestamp())
+
+def _ensure_timestamp(ts: Optional[Union[int, datetime]]) -> int:
+    """
+    Normalise a timestamp that can be an int (Unix seconds) or a datetime.
+    Raises ValueError if the value is not recognised.
+    """
+    if ts is None:
+        return _current_timestamp()
+    if isinstance(ts, int):
+        return ts
+    if isinstance(ts, datetime):
+        return int(ts.replace(tzinfo=timezone.utc).timestamp())
+    raise ValueError(f"Unsupported timestamp type: {type(ts)}")
+
+# --------------------------------------------------------------------------- #
+#  Sensor implementation
+# --------------------------------------------------------------------------- #
+
+@dataclass
+class Sensor:
+    """
+    Represents a sensor that records timestamped values under a set of keys.
+
+    Attributes
+    ----------
+    name : str
+        The unique identifier for the sensor itself.
+    first_seen : int
+        Unix timestamp of when the sensor was first instantiated.
+    last_seen : int
+        Unix timestamp of the most recent data update (or creation time).
+    data : List[Dict[str, Any]]
+        A list of data points, each with keys:
+            - name   (str)
+            - value  (Any)
+            - first_seen (int)
+            - last_seen (int)
+    """
+    name: str
+    first_seen: int = field(init=False)
+    last_seen: int = field(init=False)
+    data: List[Dict[str, Any]] = field(default_factory=list)
+
+    # ----------------------------------------------------------------------- #
+    #  Validity rules
+    # ----------------------------------------------------------------------- #
+
+    # Current rule: key must be a "word" (letters, digits, underscore)
+    VALID_NAME_RE = re.compile(r"^[A-Za-z0-9_]+$")
+
+    def _is_valid_key(self, key: str) -> bool:
+        """Return True if the key meets current validity criteria."""
+        is_valid = False
+        if bool(self.VALID_NAME_RE.match(key)):
+            is_valid = True
+        return is_valid
+
+    # ----------------------------------------------------------------------- #
+    #  Life-cycle hooks
+    # ----------------------------------------------------------------------- #
+
+    def __post_init__(self) -> None:
+        """Initialise timestamps when the object is created."""
+        # print(f"Instantiating new sensor object - {self.name}")
+        ts = _current_timestamp()
+        self.first_seen = ts
+        self.last_seen = ts
+        self.comment = self.name
+        for sensor in comment_lookup:
+            if sensor['mac'] == self.name:
+                self.comment = sensor['comment']
+
+    # ----------------------------------------------------------------------- #
+    #  Core API
+    # ----------------------------------------------------------------------- #
+
+    def add_or_update(self,
+                      key: str,
+                      value: Any,
+                      timestamp: Optional[Union[int, datetime]] = None) -> None:
+        """
+        Add a new data point or update an existing one.
+
+        Parameters
+        ----------
+        key : str
+            The data key. Must pass :meth:`_is_valid_key`.
+        value : Any
+            The data value to store.
+        timestamp : int | datetime | None
+            When the data was recorded. If omitted, current time is used.
+
+        Raises
+        ------
+        ValueError
+            If the key is not valid.
+        """
+        if not self._is_valid_key(key):
+            raise ValueError(f"Invalid key '{key}'. Must contain only "
+                             "letters, digits, or underscores.")
+
+        ts = _ensure_timestamp(timestamp)
+
+        # Update sensor-level timestamps
+        self.last_seen = max(self.last_seen, ts)
+
+        # Search for existing entry
+        for entry in self.data:
+            if entry["name"] == key:
+                # Update existing entry
+                entry["value"] = value
+                entry["last_seen"] = ts
+                if ts < entry["first_seen"]:
+                    entry["first_seen"] = ts
+                return
+
+        # If we get here, it's a new key
+        new_entry: Dict[str, Any] = {
+            "name": key,
+            "value": value,
+            "first_seen": ts,
+            "last_seen": ts
+        }
+        self.data.append(new_entry)
+
+    def get_value(self, key: str) -> Optional[Any]:
+        """Return the most recent value for a key, or None if it doesn't exist."""
+        for entry in self.data:
+            if entry["name"] == key:
+                return entry["value"]
+        return None
+
+    def update_comment(self, new_comment: str):
+        self.comment = new_comment
+
+    def to_dict(self, simple_return = False) -> Dict[str, List[Dict[str, Any]]]:
+        """
+        Return the data in the exact structure requested.
+
+        Example
+        -------
+        >>> sensor.to_dict()
+        {
+            "data": [
+                {"name": "temp", "value": 23, "first_seen": 1710000000, "last_seen": 1710000005},
+                {"name": "hum",  "value": 45, "first_seen": 1710000010, "last_seen": 1710000010}
+            ]
+        }
+        """
+        if simple_return:
+            simple_return = {
+                "temp": self.get_value(key="temp"),
+                "humidity": self.get_value(key="humidity")
+            }
+            return simple_return
+        
+        else:
+            return {"data": self.data.copy()}
+
+    # ----------------------------------------------------------------------- #
+    #  Convenience & debugging
+    # ----------------------------------------------------------------------- #
+
+    def __repr__(self) -> str:
+        return (f"{self.__class__.__name__}(name={self.name!r}, "
+                f"first_seen={self.first_seen}, last_seen={self.last_seen}, "
+                f"data={self.data!r})")
+
+# --------------------------------------------------------------------------- #
+#  Example usage
+# --------------------------------------------------------------------------- #
+
+if __name__ == "__main__":
+    # Instantiate a sensor
+    sensor = Sensor(name="weather_station_01")
+
+    # Add some data
+    sensor.add_or_update("temperature", 23.5)
+    sensor.add_or_update("humidity", 45)
+
+    # Update an existing key
+    sensor.add_or_update("temperature", 24.0)
+
+    # Try an invalid key (will raise ValueError)
+    try:
+        sensor.add_or_update("bad key!", 0)
+    except ValueError as e:
+        print(e)
+
+    # Print the structured data
+    import json
+    print(json.dumps(sensor.to_dict(), indent=4))

web_dash/Dockerfile

@@ -0,0 +1,60 @@
+
+# Base image
+FROM php:8.1-apache
+
+# Install system packages
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    # Services
+    redis-server nginx \
+    # Python
+    python3 python3-venv python3-pip \
+    # Process supervisor
+    supervisor \
+    # Others
+    net-tools curl \
+    # Clean up
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Node.js LTS 18.x
+RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - \
+    && apt-get update && apt-get install -y nodejs \
+    && rm -rf /var/lib/apt/lists/*
+
+# copy the config file
+COPY cosmostat_settings.yaml /app/cosmostat_settings.yaml
+
+# Node on 3000
+WORKDIR /usr/src/app
+COPY web/node_server/ .
+#COPY cosmostat_settings.yaml /usr/src/app/cosmostat_settings.yaml
+RUN npm install --only=production
+
+# Apache on 8080
+COPY apache_ports.conf /etc/apache2/ports.conf
+COPY apache_vhost.conf /etc/apache2/sites-available/000-default.conf
+COPY web/html/ /var/www/html/
+
+# nginx on 80
+RUN rm -rf /etc/nginx/sites-enabled/default 
+COPY web/proxy/nginx.conf /etc/nginx/conf.d/default.conf
+
+# DriveHealth on 5001
+RUN mkdir -p /opt/DriveHealth
+RUN python3 -m venv /opt/DriveHealth/venv
+COPY apis/StorageSummary/requirements.txt /opt/DriveHealth/
+RUN /opt/DriveHealth/venv/bin/pip3 install --no-cache-dir -r /opt/DriveHealth/requirements.txt
+# Copy the actual app code after installing deps
+COPY apis/StorageSummary/ /opt/DriveHealth/
+
+# Add supervisord configuration
+COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+# Expose ports
+EXPOSE 80
+EXPOSE 6379
+
+# healthcheck looks for apache
+HEALTHCHECK CMD netstat -ltn | grep -c ":8080" > /dev/null; if [ 0 != $? ]; then exit 1; fi;
+
+# Start supervisord
+CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]

web_dash/apache_ports.conf

@@ -0,0 +1,5 @@
+# Listen on 8080 inside the container
+Listen 8080
+
+# If you still want Apache to listen on 80 (rare), add it back:
+# Listen 80

web_dash/apache_vhost.conf

@@ -0,0 +1,14 @@
+<VirtualHost *:8080>
+    ServerAdmin webmaster@localhost
+    DocumentRoot /var/www/html
+
+    # Log files
+    ErrorLog ${APACHE_LOG_DIR}/error.log
+    CustomLog ${APACHE_LOG_DIR}/access.log combined
+
+    # If you need PHP processing
+    <Directory /var/www/html>
+        AllowOverride All
+        Require all granted
+    </Directory>
+</VirtualHost>

web_dash/apis/StorageSummary/Helpers.py

@@ -0,0 +1,66 @@
+
+import base64, hashlib
+import subprocess
+import ipaddress
+from typing import Dict, Any, List
+
+# pickle subroutines
+import pickle
+from pathlib import Path
+
+print("Importing Helpers")
+# subnet helper app
+def is_ip_in_subnets(ip, subnet):
+    try:
+        ip_obj = ipaddress.IPv4Address(ip)
+        subnet_obj = ipaddress.IPv4Network(subnet)
+        if ip_obj in subnet_obj:
+            return True
+        return False
+    except ValueError as e:
+        # If the IP address is not valid, raise an error
+        return False
+
+# subroutine to run a command, return stdout as array unless zero_only then return [0]
+def run_command(cmd, zero_only=False, use_shell=True, req_check = True):
+    # Run the command and capture the output
+    result = subprocess.run(cmd, shell=use_shell, check=req_check, stdout=subprocess.PIPE, stderr=subprocess.PIPE)    
+    # Decode the byte output to a string
+    output = result.stdout.decode('utf-8')
+    # Split the output into lines and store it in an array
+    output_lines = [line for line in output.split('\n') if line]
+    # Return result
+    try:
+        return output_lines[0] if zero_only else output_lines
+    except:
+        return output_lines
+
+def short_uuid(value: str, length=8):
+    hasher = hashlib.md5()
+    hasher.update(value.encode('utf-8'))
+    full_hex = hasher.hexdigest()
+    return full_hex[:length]
+
+# test subroutine
+def get_hostname():
+    hostname_command = "hostname"
+    return run_command(hostname_command, zero_only = True)
+
+
+# pickle helpers
+# Where the pickled state will live
+STATE_FILE = Path(__file__).parent / "storage_api_state.pkl"
+
+def save_state(obj: object, path: Path | str = STATE_FILE) -> None:
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("wb") as f:
+        pickle.dump(obj, f, protocol=pickle.HIGHEST_PROTOCOL)
+    print("Pickle saved")
+
+def load_state(path: Path | str = STATE_FILE) -> object | None:
+    path = Path(path)
+    if path.is_file():
+        with path.open("rb") as f:
+            return pickle.load(f)
+    return None

web_dash/apis/StorageSummary/Routes.py

@@ -0,0 +1,157 @@
+# flask routes for storage summary API
+
+# import external libraries
+from flask import Flask, jsonify, request, Response, abort
+#from flask_apscheduler import APScheduler
+from typing import Dict, Union
+import json, time, redis, yaml, datetime
+import secrets, string
+import requests
+from requests import RequestException, Response
+
+# import needed Class Libraries
+from Storage import *
+from Helpers import *
+
+SummaryServer = load_state()
+
+if SummaryServer is None:
+    SummaryServer = DriveHealthServer(get_hostname())
+    print("Created new SummaryServer")
+
+# declare flask app
+app = Flask(__name__)
+
+############################
+# Flask routes
+############################
+
+# client update
+@app.route('/storage_client_update', methods=['POST'])
+def storage_client_update():
+    payload = request.get_json(silent=False)
+    if payload is None:
+        abort(400, description="Request body must be valid JSON")
+    payload["IP Address"] = request.remote_addr
+    # offload processing to helper
+    processed_payload = client_update_helper(payload)
+    return jsonify(processed_payload), 200
+
+# remove client
+@app.route('/storage_client_delete', methods=['POST'])
+def storage_client_delete():
+    payload = request.get_json(silent=False)
+    print(payload)
+    if payload is None:
+        abort(400, description="Request body must be valid JSON")
+    result = client_remove_helper(payload)
+    print(result)
+    return jsonify(result)
+
+# client details
+@app.route('/client_details', methods=['GET'])
+def client_details():
+    result = []
+    for client in SummaryServer.clients:
+        result.append(client.get_details())
+    return jsonify(result)
+
+# client summary
+@app.route('/client_summary', methods=['GET'])
+def client_summary():
+    result = []
+    for client in SummaryServer.clients:
+        result.append(client.get_summary())
+    return jsonify(result)
+
+# client brief summary
+@app.route('/brief_summary', methods=['GET'])
+def brief_summary():
+    result = []
+    for client in SummaryServer.clients:
+        result.append(f"{client.name} at {client.ip} - {len(client.drives)} drives")
+    return jsonify({
+        "message": "Brief Summary",
+        "result": result
+        })
+
+# test route
+@app.route('/test', methods=['GET'])
+def test_route():
+    return jsonify({
+        "message": "Hello world!",
+        "hostname": get_hostname(),
+        "DriveHealthServer": f"{SummaryServer}"
+    })
+
+# test route 2
+@app.route('/test_storage_summary', methods=['GET'])
+def test_storage_summary():
+    return test_route()
+
+
+############################
+# Route Helpers
+############################
+
+# helper function for client_update route
+# handles the submission data from the flask route
+def client_update_helper(payload: dict):
+    result = None
+    required_keys = {"hostname", "API_KEY", "drives", "IP Address"}
+    # check json structure and API key
+    processed_payload = post_processor(payload, required_keys)
+    # add or update the client
+    result = client_processor(processed_payload)
+    return result
+
+# client processing function, add/update logic in Class Methods
+def client_processor(client_dict: dict):
+    result = SummaryServer.process_client_data(client_dict)
+    save_state(SummaryServer)
+    return result
+
+# handle submission from remove route
+def client_remove_helper(payload: dict):
+    result = None
+    required_keys = {"remove_hosts", "API_KEY"}
+    # check the submission data
+    processed_payload = post_processor(payload, required_keys)
+    result = SummaryServer.remove_client(processed_payload["remove_hosts"])
+    return result
+
+# this function takes the raw POST input from client_update and makes sure it is valid and returns it if so
+def post_processor(client_dict: dict, required_keys: dict):
+    payload_safe = False
+    keys_present = False
+    api_valid = False
+    api_key = "deadbeef"
+    # check for keys
+    missing = required_keys - client_dict.keys()
+    if not missing:
+        keys_present = True
+    else:
+        return {
+            "message": f"error - {missing} keys missing"
+        }
+    # check API
+    if client_dict["API_KEY"] == api_key:
+        api_valid = True
+    
+    # if both then safe
+    if keys_present and api_valid:
+        payload_safe = True
+
+    # add a key to indicate this was processed
+    client_dict["processed_at"] = time.time()
+    return client_dict
+
+############################
+# Main function
+############################
+
+def run_main():
+
+    # Flask API
+    app.run(debug=False, host='0.0.0.0', port=5001)
+        

web_dash/apis/StorageSummary/Storage.py

@@ -0,0 +1,178 @@
+# Class definitions for storage summary
+# Classes needed:
+### DriveHealthServer - this will be the list of remote server objects and functions for interacting with them
+### DriveHealthClient - this will be the remote client class where all the drives are
+
+from typing import List, Mapping, Any, Sequence, Dict
+from Helpers import *
+print("Importing Storage Class")
+
+#################################################################
+### DriveHealthServer Class
+### This is the server objext
+#################################################################
+
+class DriveHealthServer:
+    
+    # create server object for local cache of remote clients
+    def __init__(self, hostname: str):
+        # the system needs a name, should be equal to the uuid of the client
+        self.name = hostname
+        self.short_id = short_uuid(self.name)
+        self.hostname = hostname
+        # system contains an array of CosmostatClient Objects
+        self.clients = [] 
+
+    def __str__(self):
+        self_string = f"DriveHealthServer {self.name} - {self.short_id}"
+        return self_string
+
+    def __repr__(self):
+        self_string = f"DriveHealthServer {self.name} - {self.short_id}"
+    
+    def __del__(self): 
+        print("Deleting Server") 
+
+    # either add or update client
+    def process_client_data(self, client_data: dict):
+        result = None
+        if self.check_for_uuid(self.calculate_uuid(client_data)):
+            result = {
+                "message": "Updating client.",
+                "summary": self.update_client(client_data)
+                }
+        else:
+            result = {
+                "message": "Creating new client",
+                "summary": self.add_client(client_data)
+                }
+        return result
+    
+    def add_client(self, client_data: dict):
+        new_client = DriveHealthClient(client_data)
+        self.clients.append(new_client)
+        return new_client.get_summary()
+    
+    def update_client(self, client_data):
+        result = self.get_client(self.calculate_uuid(client_data))
+        result.update_client(client_data)
+        return result.get_summary()        
+
+    def get_client(self, client_uuid: str):
+        result = None
+        for client in self.clients:
+            if client.short_id == client_uuid:
+                result = client
+        return result 
+
+    def remove_client(self, client_uuid: str | list[str]):
+        result = None
+        old_clients = self.clients
+        temp_clients = []
+        purged_clients = []
+        for client in old_clients:
+            if client.short_id in client_uuid:
+                purged_clients.append(client)
+            else:
+                temp_clients.append(client)
+        self.clients = temp_clients
+        result = {
+            "message": "client removal complete",
+            #"clients_removed": purged_clients,
+            #"new_client_count": len(self.clients),
+            #"old_client_count": len(old_clients)
+        }
+        return result
+
+    def check_for_uuid(self, uuid: str):
+        result = False
+        for client in self.clients:
+            if client.short_id == uuid:
+                result = True
+        return result
+    
+    # calculate uuid based on same parameters
+    def calculate_uuid(self, client_data):
+        unique_string = f"{client_data["hostname"]} - {client_data["IP Address"]}"
+        return short_uuid(unique_string)
+
+
+#################################################################
+### DriveHealthClient Class
+### These are the actual remote clients
+#################################################################
+
+class DriveHealthClient:
+
+    ############################################################
+    # instantiate new DriveHealthClient
+    ############################################################
+    
+    def __init__(self, client_data: dict):
+        # the system needs a name, should be equal to the uuid of the client
+        self.client_data = client_data
+        self.ip = self.client_data["IP Address"]
+        self.name = self.client_data["hostname"]
+        self.data_timestamp = self.client_data["processed_at"]
+        self._unique_string = f"{self.name} - {self.ip}"
+        self.short_id = short_uuid(self._unique_string)
+        self.drives = self.client_data["drives"]
+        
+
+    def __str__(self):
+        self_string = f"DriveHealthClient Server {self.name} - {self.short_id}"
+        return self_string
+
+    def __repr__(self):
+        self_string = f"DriveHealthClient Server {self.name} - {self.short_id}"
+    
+    def __del__(self): 
+        print("Deleting Client") 
+        
+    def get_summary(self):
+        drives_brief = []
+        for drive in self.drives:
+            drives_brief.append({
+                "serial": drive["Serial Number"],
+                "model": drive["Model"],
+                "capacity": drive["Disk Size"]
+            })
+        result = {
+            "name": self.name,
+            "hostname": self.ip,
+            "uuid": self.short_id,
+            "drives": drives_brief
+        }
+        return result
+
+    def get_details(self):
+        drive_details = []
+        for drive in self.drives:
+            drive_details.append({
+                "disk_id": drive["Disk ID"],
+                "serial": drive["Serial Number"],
+                "health_status": drive["Health Status"],
+                "model": drive["Model"],
+                "capacity": drive["Disk Size"],
+                "power_on_hours": drive["Power On Hours"],
+                "power_on_count": drive["Power On Count"],
+                "host_writes": drive["Host Writes"],
+                "wear_level": drive["Wear Level Count"],
+                "drive_letter": drive["Drive Letter"],
+                "drive_interface": drive["Interface"],
+                "transfer_mode" : drive["Transfer Mode"]
+            })
+        result = {
+            "name": self.name,
+            "ip": self.ip,
+            "uuid": self.short_id,
+            "timestamp": self.data_timestamp,
+            "drives": drive_details
+        }
+        return result
+        
+    def update_client(self, client_data: dict):
+        result = None
+        self.data_timestamp = client_data["processed_at"]
+        self.drives = client_data["drives"]
+        return result

web_dash/apis/StorageSummary/app.py

@@ -0,0 +1,14 @@
+# main function for storage API
+# this is the barest of bones to load the api
+
+# import class libraries
+from Routes import *
+
+#######################################################################
+#######################################################################
+### Main Subroutine
+#######################################################################
+#######################################################################
+
+if __name__ == '__main__':
+    run_main()

web_dash/apis/StorageSummary/requirements.txt

@@ -0,0 +1,7 @@
+flask
+pytz
+requests
+opencv-python
+redis
+flask_apscheduler
+pyyaml

web_dash/supervisord.conf

@@ -0,0 +1,59 @@
+[supervisord]
+nodaemon=true
+logfile=/dev/stdout
+logfile_maxbytes=0
+
+# ------------------------------------------------------------------
+# 1. Apache (from the base image)
+# ------------------------------------------------------------------
+[program:apache2]
+command=/usr/sbin/apache2ctl -D FOREGROUND
+stdout_logfile=/dev/stdout
+stderr_logfile=/dev/stderr
+autorestart=true
+priority=1
+
+# ------------------------------------------------------------------
+# 2. Redis
+# ------------------------------------------------------------------
+[program:redis]
+command=/usr/bin/redis-server --daemonize no --protected-mode no
+stdout_logfile=/dev/stdout
+stderr_logfile=/dev/stderr
+autorestart=true
+priority=2
+
+# ------------------------------------------------------------------
+# 3. Nginx (will listen on 8080 by default)
+# ------------------------------------------------------------------
+[program:nginx]
+command=/usr/sbin/nginx -g 'daemon off;'
+stdout_logfile=/dev/stdout
+stderr_logfile=/dev/stderr
+autorestart=true
+priority=3
+
+# ------------------------------------------------------------------
+# 4. Node.js
+# ------------------------------------------------------------------
+# NOTE: Adjust the command/path to match your app
+[program:node]
+command=sh -c "npm install && node server.js"
+#command=npm start
+directory=/usr/src/app
+stdout_logfile=/dev/stdout
+stderr_logfile=/dev/stderr
+autorestart=true
+priority=4
+
+
+# ----------------------------------------------------------
+# 5. DriveHealth app
+# ----------------------------------------------------------
+[program:DriveHealth]
+command=/opt/DriveHealth/venv/bin/python3 /opt/DriveHealth/app.py
+directory=/opt/DriveHealth
+stdout_logfile=/dev/stdout
+stderr_logfile=/dev/stderr
+autorestart=true
+priority=5

web_dash/web/html/index.php

@@ -0,0 +1,512 @@
+<?php
+
+declare(strict_types=1);
+
+/* ---------- Utility functions ---------- */
+date_default_timezone_set('America/Los_Angeles');
+
+# for drive_health page, removal handler
+$remove_hosts = [];
+if ($_SERVER['REQUEST_METHOD'] === 'POST' && isset($_POST['action'])) {
+    if ($_POST['action'] === 'remove') {
+        // The Remove form sends a comma-separated string of short_id's
+        if (!empty($_POST['remove_hosts'])) {
+            $remove_hosts = array_filter(
+                explode(',', $_POST['remove_hosts']),
+                fn($s) => $s !== ''
+            );
+        } else {
+            $remove_hosts = [];
+        }
+        if (!empty($remove_hosts)){
+            foreach ($remove_hosts as $host) {
+                echo "remove ".$host."<br>";
+            }
+            removeClient($remove_hosts);
+        }
+    }
+}
+
+# authelia user handler
+
+$authelia_user = "not-set";
+if (isset($_SERVER['HTTP_REMOTE_USER'])) {
+  $authelia_user = $_SERVER['HTTP_REMOTE_USER'];
+}
+
+/* ---------- Helper: remove client details ---------- */
+function removeClient($clientList)
+{
+    
+    $url = "http://0.0.0.0:5001/storage_client_delete";
+    $payload = [
+        'API_KEY'      => "deadbeef",
+        'remove_hosts' => $clientList,
+    ];
+    $json = json_encode($payload, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
+    if ($json === false) {
+        throw new RuntimeException('JSON encoding failed: ' . json_last_error_msg());
+    }
+    $ch = curl_init($url);
+    if ($ch === false) {
+        throw new RuntimeException('Unable to initialise cURL.');
+    }
+    curl_setopt_array($ch, [
+        CURLOPT_POST          => true,
+        CURLOPT_POSTFIELDS    => $json,
+        CURLOPT_HTTPHEADER    => [
+            'Content-Type: application/json',
+            'Content-Length: ' . strlen($json),
+            'Accept: application/json',
+        ],
+        CURLOPT_RETURNTRANSFER=> true,
+        CURLOPT_TIMEOUT       => 2,
+        CURLOPT_FOLLOWLOCATION=> true,  // follow redirects if any
+    ]);
+    // Execute curl request
+    $response = curl_exec($ch);
+    
+
+    // cURL error handling
+    if ($response === false) {
+        $error   = curl_error($ch);
+        $errno   = curl_errno($ch);
+        curl_close($ch);
+        throw new RuntimeException("cURL error ({$errno}): {$error}");
+    }
+
+    // Grab HTTP status code
+    $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+    curl_close($ch);
+
+    if ($httpCode < 200 || $httpCode >= 300) {
+        throw new RuntimeException("API returned HTTP {$httpCode}: {$response}");
+    }
+
+    // Decode the JSON response
+    $decoded = json_decode($response, true);
+    if ($decoded === null && json_last_error() !== JSON_ERROR_NONE) {
+        throw new RuntimeException('Failed to decode JSON response: ' . json_last_error_msg());
+    }
+
+    return $decoded;
+
+}
+
+/**
+ * Escape HTML
+ */
+function h(string $s): string
+{
+    return htmlspecialchars($s, ENT_QUOTES, 'UTF-8');
+}
+
+/**
+ * Load simple key/value pairs from a YAML file.
+ * Lines starting with '#' are ignored.
+ * The function returns an associative array.
+ */
+function loadYaml(string $path): array
+{
+    if (!file_exists($path)) {
+        return [];
+    }
+
+    $lines = file($path, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
+    $data  = [];
+
+    foreach ($lines as $line) {
+        $line = trim($line);
+        if ($line === '' || $line[0] === '#') {
+            continue;
+        }
+        $pos = strpos($line, ':');
+        if ($pos === false) {
+            continue;
+        }
+        $key   = trim(substr($line, 0, $pos));
+        $value = trim(substr($line, $pos + 1));
+        $value = trim($value, "\"'"); // remove surrounding quotes
+        if ($value === '') {
+            $value = null;
+        }
+        $data[$key] = $value;
+    }
+
+    return $data;
+}
+
+/* ---------- Load settings ---------- */
+
+$settingsPath = '/app/cosmostat_settings.yaml';
+$settings     = loadYaml($settingsPath);
+
+
+/* ---------- Page mode handling ---------- */
+
+$mode = $_GET['mode'] ?? 'cosmostat';          // default mode
+$validModes = ['cosmostat', 'drive_health']; // extend as needed
+// 'gali', 
+if (!in_array($mode, $validModes, true)) {
+    $mode = 'cosmostat';
+}
+
+
+/* ---------- API configuration per mode ---------- */
+$apiConfig = [
+    'cosmostat'    => ['bind' => '10.200.27.20', 'port' => '5000'],
+    /*'gali'         => ['bind' => '10.200.27.20', 'port' => '5000'],   // space filler */
+    'drive_health' => ['bind' => '0.0.0.0', 'port' => '5001'],      // drive health API
+];
+
+/* ---------- Helper: fetch client details ---------- */
+function fetchClientDetails(string $bindIp, string $port, string $path = '/client_details'): array
+{
+    $url = "http://{$bindIp}:{$port}{$path}";
+    $ctx = stream_context_create([
+        'http' => [
+            'timeout' => 2,
+            'header'  => "User-Agent: PHP/" . PHP_VERSION . "\r\n"
+        ]
+    ]);
+    $json = @file_get_contents($url, false, $ctx);
+    if ($json === false) {
+        return [];   // caller will handle empty case
+    }
+    $data = json_decode($json, true);
+    if (!is_array($data)) {
+        return ['fail'];
+    }
+    return $data;
+}
+
+/* ---------- Fetch client details ---------- */
+$apiInfo = $apiConfig[$mode];
+$clients = fetchClientDetails($apiInfo['bind'], $apiInfo['port']);
+
+/* ---------- Ensure each client has a short_id ---------- */
+foreach ($clients as &$client) {
+    if (!isset($client['short_id'])) {
+        $client['short_id'] = substr($client['uuid'] ?? '', 0, 8);
+    }
+}
+unset($client);
+
+
+/* ---------- Determine selected hosts (Drive Health only) ---------- */
+$selectedHosts = $_GET['hosts'] ?? [];
+
+if ($mode === 'drive_health') {
+    if (isset($_GET['action'])) {
+        switch ($_GET['action']) {
+            case 'all':
+                $selectedHosts = array_column($clients, 'short_id');
+                break;
+            case 'none':
+                $selectedHosts = [];
+                break;
+            // 'apply' - nothing to do; $selectedHosts already contains the posted hosts
+        }
+    }
+}
+if ($mode === 'drive_health' && empty($selectedHosts) && !isset($_GET['action'])) {
+    $selectedHosts = array_column($clients, 'short_id');
+}
+/* ---- Determine selected host ---- */
+
+$selectedId  = $_GET['host'] ?? '';
+$selectedIdx = null;
+
+foreach ($clients as $idx => $client) {
+    if (isset($client['short_id']) && $client['short_id'] === $selectedId) {
+        $selectedIdx = $idx;
+        break;
+    }
+}
+if ($selectedIdx === null) {
+    // Default to the first client (if any)
+    $selectedIdx = 0;
+    $selectedId   = $clients[$selectedIdx]['short_id'] ?? '';
+}
+#global $clients, $client, $properties, $systemProperties, $systemComponents, $selectedHost, $selectedHosts, $selectedId, $selectedIdx;
+
+$client            = $clients[$selectedIdx] ?? null;
+$properties        = $client['client_properties'][0] ?? [];
+$systemProperties  = $properties['system_properties'] ?? [];
+$systemComponents  = $properties['system_components'] ?? [];
+$selectedHost      = $clients[$selectedIdx]['hostname'] ?? 'Unknown';
+
+
+/* ---- Sidebar Renderer ---- */
+
+function renderSidebar(string $mode){
+    global $clients, $client, $properties, $systemProperties, $systemComponents, $selectedHost, $selectedHosts, $selectedId, $selectedIdx, $remove_hosts;
+
+    $modes = [
+        'cosmostat'    => 'Cosmostat',
+      /*  'gali'         => 'Shuttle Gali',*/
+        'drive_health' => 'Drive Health',
+    ];
+
+    ?>
+    <nav class="sidebar">
+        <form method="get" id="modeForm">
+            <label for="modeSelect"></label>
+            <select class="select-dark" name="mode" id="modeSelect" onchange="this.form.submit()">
+                <?php foreach ($modes as $key => $label): ?><option value="<?= h($key) ?>" <?= $mode === $key ? 'selected' : '' ?>><?= h($label) ?></option>
+            <?php endforeach; ?></select>
+        </form>
+        <p>
+    <?php if ($mode === 'drive_health'): ?>
+        <?php
+        if ( !is_array($clients) || empty($clients) ) {
+            // Graceful “no data” handling
+            echo '<p style="margin:1rem 0; font-style:italic;">'
+                . 'No hosts are available to manage at this time.'
+                . '</p>';
+            }
+        ?>
+        <?php if (is_array($clients) &&  !empty($clients)): ?>
+
+        <h3>Hosts</h3>
+            <form method="get" id="driveHealthForm">
+            <input type="hidden" name="mode" value="drive_health">
+            <ul>
+                <?php foreach ($clients as $c): ?>
+                    <?php $id = $c['short_id']; ?>
+
+                    <li>
+                        <label>
+                            <input type="checkbox" name="hosts[]" value="<?= h($id) ?>" <?= in_array($id, $selectedHosts, true) ? 'checked' : '' ?>><?= h($c['name']) ?>
+
+                        </label>
+                    </li>
+                <?php endforeach; ?>
+            </ul><p>
+            <button class="btn btn-primary" type="submit" name="action" value="apply">Apply</button><p>
+            <button class="btn btn-primary" type="submit" name="action" value="all">Select All</button><p>
+            <button class="btn btn-primary" type="submit" name="action" value="none">Select None</button><p>
+        </form>
+        <!-- Remove host button (POST) -->
+        <form method="post" id="removeForm">
+            <input type="hidden" name="mode" value="drive_health">
+            <input type="hidden" name="action" value="remove">
+            <input type="hidden" name="remove_hosts" id="remove_hosts_input" value="">
+            <button class="btn btn-primary" type="submit">Remove</button>
+        </form>                
+        <?php endif; ?>   
+    <?php endif; ?>
+ 
+    <?php if ($mode == 'cosmostat'): ?>
+ 
+    <input type="hidden" name="host" value="<?= h($selectedId) ?>">
+    <h3>Endpoints</h3>
+    <ol id="endpointList"></ol>
+    <?php endif; ?>
+
+    <?php if ($mode == 'gali'): ?>
+    <h3>Shuttle Gali</h3>
+    <?php endif; ?>
+
+    </nav>
+
+    <?php
+    }
+
+/* ---- Main Content Renderer ---- */
+
+function renderMainContent(string $mode){
+    global $clients, $client, $properties, $systemProperties, $systemComponents, $selectedHost, $selectedHosts, $remove_hosts, $selectedId, $selectedIdx;
+    /* ---------- Handle empty / error ---------- */
+    if ($clients === ['fail']) {
+        die('<div class="card"><p style="color:red;">Could not retrieve data from the API for mode "' . h($mode) . '".</p></div>');
+    }
+
+    if ($mode === 'drive_health') {
+        // If nothing is selected, show a friendly message
+        if (empty($selectedHosts)) {
+            echo '<div class="card"><p>No hosts selected.</p></div>';
+            return;
+        }
+        echo '
+    <div class="storage_client">';
+        foreach ($selectedHosts as $sid) {
+            // Find the client that matches this short_id
+            $c = null;
+            foreach ($clients as $cl) {
+                if ($cl['short_id'] === $sid) {
+                    $c = $cl;
+                    break;
+                }
+            }
+            if ($c === null) continue;   // safety
+
+            $hostname = $c['name'] ?? 'Unknown';
+            echo '
+        <div class="card">
+        <h2>Drive Health - ' . h($hostname) . '</h2>
+        <h3>IP: '.h($c['ip']).'<br>
+        Timestamp: '.date('F j, Y g:i a', (int) $c['timestamp']).' </h3>
+        ';
+
+            if (isset($c['drives']) && is_array($c['drives']) && count($c['drives']) > 0) {
+                echo '
+        <table id="host_metrics_table">
+            <thead><tr>
+                <th>Drive Letter</th><th>Disk ID</th><th>Health Metrics</th><th>Model</th><th>Serial</th><th>Capacity</th>
+            </tr></thead>
+            <tbody>';
+                foreach ($c['drives'] as $drive) {
+                    $wear_gb = '';
+                    if (isset($drive['host_writes'])){
+                        $wear_gb = $drive['host_writes'].' wear';
+                    }
+                    echo '
+                <tr>
+                    <td>' . h($drive['drive_letter'] ?? '') . '</td>
+                    <td>' . h("".$drive['disk_id'] ?? '') . '</td>
+                    <td>' .                     
+                    h($drive['health_status'] ?? '').'<br>'.
+                    h($drive['power_on_hours'] ?? '').'<br>'.
+                    h($wear_gb ?? '').
+                    '</td>
+                    <td>' . h($drive['model'] ?? '') . '</td>
+                    <td>' . h($drive['serial'] ?? '') . '</td>
+                    <td>' . h($drive['capacity'] ?? '') . '</td>
+                </tr>';
+                }
+                echo '
+            </tbody></table> ';
+            } else {
+                echo '<p>No drive data available for this host.</p>';
+            }
+            echo '
+        </div>';
+        }
+        echo '
+    </div>';
+        return;
+    }
+    ?>
+    <div class="main">
+
+        <?php if ($mode == 'cosmostat'): ?><!-- Header Card -->
+        <div class="card">
+            <h2>Matt-Cloud Cosmostat Dashboard</h2>
+            <p>This dashboard shows the local Matt-Cloud system stats.</p>
+            <div class="help-link" id="helpToggle">API</div>
+        </div> <!-- / Header Card -->
+        <!-- Hidden API Card -->
+        <div id="helpText" class="card">
+            <strong>Component Desriptor</strong>
+            <p>To view the component descriptor, you may <br>
+            <code>curl -s https://<?= h($_SERVER['SERVER_NAME']) ?>/descriptor</code></p>
+            This will return the entire JSON descriptor variable.<br>
+            The endpoint agent uses this descriptor to build out its local System Object.<br>
+            The agent then reports back to the Cosmostat Server with all the data found in the descriptor.<br>
+            Full Source Code can be found at its <a target="_blank" rel="noopener noreferrer" href="https://gitea.matt-cloud.com/matt/cosmoserver">Gitea</a> page. 
+        </div> <!-- / Header Card -->
+        <!-- summary card -->
+        <div class="card">
+            <?php if (!empty($systemProperties)): ?>
+                <h2>System Properties</h2>
+                <table><tr>
+                    <td>
+                        <ul class="system-list">
+                            <?php foreach ($systemProperties as $prop): ?>
+                                <li><?= h($prop['Property']) ?></li>
+                            <?php endforeach; ?>
+                        </ul>
+                    </td>
+                    <td>
+                        <h2>Live System Metrics</h2>
+                        <div id="host_metrics" class="column">Connecting...</div>
+                    </td>
+                </tr></table>
+            <?php endif; ?><br>
+            <div class="componentDetail-link" id="componentDetailToggle">Toggle Component Details</div>
+        </div> <!--/summary card --> 
+        
+        <!-- hidden detail card -->
+        <div id="componentDetailText" class="card">
+            <?php if (!empty($systemComponents)): ?>
+                <h2>Components</h2>
+                <div class="components">
+                    <?php foreach ($systemComponents as $comp): ?>
+                        <div class="component">
+                            <h3><?= h($comp['component_name']) ?></h3>
+                            <ul class="info-list">
+                                <?php foreach ($comp['info_strings'] as $info): ?>
+                                    <li><?= h($info) ?></li>
+                                <?php endforeach; ?>
+                            </ul>
+                        </div>
+                    <?php endforeach; ?>
+                </div>
+            <?php endif; ?>
+        </div> <!--/hidden detail card --> 
+        <?php endif; ?>
+
+
+        <?php if ($mode == 'gali'): ?>
+
+        <h3>Shuttle Gali</h3>
+        <?php endif; ?>
+
+
+    </div> <!-- /main -->
+<?php
+}
+
+/* ---- Render server dashboard ---- */
+?>
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>Cosmostat - <?= h($selectedHost) ?></title>
+<link rel="stylesheet" href="src/styles.css">
+</head>
+<body>
+
+<div class="wrapper">
+
+<!-- Sidebar -->
+<?php renderSidebar($mode); ?>
+
+<!-- Main content -->
+<?php renderMainContent($mode); ?>
+
+
+</div> <!-- /wrapper -->
+
+<!-- cosmostat javascript -->
+<script src="socket.io/socket.io.js"></script>
+<script src="src/system_metrics.js"></script>
+<script>
+    // Toggles for hidden cards
+    document.getElementById('helpToggle').addEventListener('click', function () {
+        const help = document.getElementById('helpText');
+        help.style.display = help.style.display === 'none' || help.style.display === '' ? 'block' : 'none';
+    });
+    document.getElementById('componentDetailToggle').addEventListener('click', function () {
+        const help = document.getElementById('componentDetailText');
+        help.style.display = help.style.display === 'none' || help.style.display === '' ? 'block' : 'none';
+    });
+</script>
+
+<!-- drive health javascript -->
+<script>
+    // Removal Handler
+    document.getElementById('removeForm').addEventListener('submit', function (e) {
+        // Grab all checked checkboxes from the Drive-Health form
+        const checked = document.querySelectorAll('#driveHealthForm input[type="checkbox"][name="hosts[]"]:checked');
+        const ids = Array.from(checked).map(cb => cb.value);
+
+        // Pass the comma-separated list to the hidden input
+        document.getElementById('remove_hosts_input').value = ids.join(',');
+    });    
+</script>
+
+</body>
+</html>

web_dash/web/html/src/redis.js

@@ -0,0 +1,145 @@
+/* ------------------------------------------------------------
+   1.  Socket-IO connection & helper functions
+   ------------------------------------------------------------ */
+const socket = io();
+
+socket.on('host_metrics', renderStatsTable);
+socket.on('connect_error', err => {
+  safeSetText('host_metrics', `Could not connect to server - ${err.message}`);
+});
+socket.on('reconnect', attempt => {
+  safeSetText('host_metrics', `Re-connected (attempt ${attempt})`);
+});
+
+function safeSetText(id, txt) {
+  const el = document.getElementById(id);
+  if (el) el.textContent = txt;
+}
+
+/* ------------------------------------------------------------------
+   2.  Table rendering - now orders rows before building the table
+   ------------------------------------------------------------------ */
+// helper function for table row ordering   
+function renderStatsTable(data) {
+  socket.emit('tableRendered');
+  renderGenericTable('host_metrics', data, 'No Stats available');
+}
+
+function renderGenericTable(containerId, data, emptyMsg) {
+  const container = document.getElementById(containerId);
+  if (!Array.isArray(data) || !data.length) {
+    container.textContent = emptyMsg;
+    return;
+  }
+
+  /* Merge rows by name (new logic) */
+  const mergedData = mergeRowsByName(data);
+
+  /* Order the merged rows – priority first */
+  const orderedData = orderRows(mergedData);
+
+  /* Build the table from the ordered data */
+  const table = buildTable(orderedData);
+  table.id = 'host_metrics_table';
+  container.innerHTML = '';
+  container.appendChild(table);
+}
+
+/* ------------------------------------------------------------
+   3.  Merge rows by name
+   ------------------------------------------------------------ */
+function mergeRowsByName(data) {
+  const groups = {}; // { source: { ... } }
+  data.forEach(row => {
+    const source = row.Source;      // <-- changed
+    if (!source) return;
+    if (!groups[source]) {
+      groups[source] = { Metric: [], Data: [] };
+    }
+    if ('Metric' in row && 'Data' in row) {
+      groups[source].Metric.push(row.Metric);
+      groups[source].Data.push(row.Data);
+    } 
+  });
+
+  const merged = [];
+  Object.entries(groups).forEach(([source, grp]) => {
+    merged.push({
+      Source: source,          // <-- keep the original key
+      Metric: grp.Metric,
+      Data: grp.Data,
+      Property: grp.Property,
+      Value: grp.Value
+    });
+  });
+  return merged;
+}
+
+// 3b. Order rows – put “System”, “CPU”, “RAM” first
+function orderRows(rows) {
+  // Priority list – can be updated later
+  const priority = ['System', 'CPU', 'RAM'];
+
+  // Map source → priority index
+  const priorityMap = {};
+  priority.forEach((src, idx) => {
+    priorityMap[src] = idx;
+  });
+
+  // Stable sort: keep original position if priorities are equal
+  return [...rows].sort((a, b) => {
+    const aIdx = priorityMap.hasOwnProperty(a.Source)
+      ? priorityMap[a.Source]
+      : Infinity;               // anything not in priority goes to the end
+    const bIdx = priorityMap.hasOwnProperty(b.Source)
+      ? priorityMap[b.Source]
+      : Infinity;
+
+    // If both have the same priority (or both Infinity), keep original order
+    return aIdx - bIdx;
+  });
+}
+
+/* ------------------------------------------------------------
+   4.  Build an HTML table from an array of objects
+   ------------------------------------------------------------ */
+function buildTable(data) {
+  const cols = ['Source', 'Metric', 'Data']; // explicit order
+  const table = document.createElement('table');
+
+  // Header
+  const thead = table.createTHead();
+  const headerRow = thead.insertRow();
+  cols.forEach(col => {
+    const th = document.createElement('th');
+    th.textContent = col;
+    headerRow.appendChild(th);
+  });
+
+  // Body
+  const tbody = table.createTBody();
+  data.forEach(item => {
+    const tr = tbody.insertRow();
+    cols.forEach(col => {
+      const td = tr.insertCell();
+       
+      const val = item[col];
+      if (Array.isArray(val)) {
+        // Create a <span> for each item
+        val.forEach((v, idx) => {
+          td.id = 'host_metrics_column';
+          const span = document.createElement('span');
+          span.textContent = v;
+          td.appendChild(span);
+
+          // Insert a line break after every item except the last
+          if (idx < val.length - 1) td.appendChild(document.createElement('br'));
+        });
+      } else {
+        td.textContent = val !== undefined ? val : '';
+      }
+    });
+  });
+
+  return table;
+}

web_dash/web/html/src/styles.css

@@ -0,0 +1,401 @@
+/* -------------------------------------------------
+   1. Global settings & color palette
+   ------------------------------------------------- */
+:root {
+    /* Dark theme - body & card backgrounds */
+    --bg-body:     #2c3e50;   /* main page background */
+    --bg-card:     #34495e;   /* card / panel background */
+    --bg-sidebar:  #3d566e;   /* sidebar background (slightly lighter) */
+    /* Accent / link color */
+    --clr-accent:  #3498db;   /* blue accent for links */
+    /* Text color */
+    --clr-text:    #ecf0f1;   /* light whiteish text */
+    /* Borders / accents */
+    --clr-border:  #1f2b38;   /* dark border color */
+    /* button colors*/
+    --clr-btn-bg:       var(--bg-card);          /* default button background */
+    --clr-btn-text:     var(--clr-text);         /* button text color */
+    --clr-btn-border:   var(--clr-border);       /* border color */
+    --clr-btn-accent:   var(--clr-accent);       /* hover/focus color */
+    --clr-btn-shadow:   rgba(0,0,0,0.3);
+    /* select colors */
+    --clr-select-bg:        var(--bg-card);   /* dropdown background */
+    --clr-select-text:      var(--clr-text);  /* option text colour */
+    --clr-select-border:    var(--clr-border);
+    --clr-select-accent:    var(--clr-accent);/* focus / hover border */
+    --clr-select-shadow:    rgba(0,0,0,.3);
+
+}
+
+* { box-sizing: border-box; }
+
+/* Body */
+body {
+    margin: 0;
+    padding: 0;
+    background: var(--bg-body);
+    color: var(--clr-text);
+    font-family: Arial, Helvetica, sans-serif;
+}
+
+/* Links */
+a { color: var(--clr-accent); text-decoration: none; }
+a:hover { text-decoration: underline; }
+
+/* -------------------------------------------------
+   2. Layout - wrapper, sidebar, main
+   ------------------------------------------------- */
+.wrapper { display: flex; }
+
+.sidebar {
+    position: fixed;          /* keep sidebar visible during scroll */
+    top: 0;                   /* stick to the top of the viewport */
+    left: 0;                  /* align to the left edge */
+    height: 100vh;            /* full viewport height */
+    /* ---- size & spacing ------------------------------------------- */
+    width: 200px;             /* same as before */
+    padding: 1rem;
+    overflow-y: auto;         /* allow sidebar content to scroll if needed */
+    /* ---- look ------------------------------------------------------- */
+    background: var(--bg-sidebar);
+    /* optional: keep it above other content */
+    z-index: 1000;
+}
+.sidebar h3 { margin: 0 0 .4rem 0; font-size: 1.3rem; }
+.sidebar ul { list-style: none; padding: 0; margin: 0; }
+.sidebar ol { list-style: none; padding: 0; margin: 0; }
+.sidebar li { margin-bottom: .4rem; font-size: 1.1rem; }
+.sidebar a { color: var(--clr-accent); }
+.sidebar a.active { font-weight: bold; }
+/*
+.endpointList {
+
+}
+*/
+.main{
+    flex: 1;
+    padding: 1rem;
+    padding-left: 200px;   /* space for the fixed sidebar */
+    /* optional: avoid accidental horizontal overflow */
+    overflow-x: hidden;
+}
+/* -------------------------------------------------
+   3. Card component
+   ------------------------------------------------- */
+.card {
+    max-width: 950px;
+    margin: 20px auto 1rem auto;
+    padding: 20px;
+    background: var(--bg-card);
+    border: 1px solid var(--clr-border);
+    border-radius: 8px;
+    box-shadow: 0 2px 4px rgba(0,0,0,.3);
+}
+
+/* -------------------------------------------------
+   4. Tables
+   ------------------------------------------------- */
+table, th, td {
+    border: 2px solid var(--clr-border);
+    border-collapse: collapse;
+}
+th, td { padding: 10px; }
+
+/* Alternate row color for metrics table */
+#host_metrics_table tbody tr td:nth-of-type(even) {
+    background: #3e5c78;   /* slight contrast */
+}
+
+/* -------------------------------------------------
+   5. Lists & headings
+   ------------------------------------------------- */
+h1, h2, h3, h4 { color: var(--clr-text); margin: 0 0 .4rem 0; }
+ul { list-style: none; padding: 0; }
+ol { list-style: none; padding: 0; }
+li { margin-bottom: 10px; color: var(--clr-text); }
+
+/* System & component lists */
+.system-list, .info-list {
+    list-style: none; padding: 0; margin: 0;
+}
+.system-list li, .info-list li { margin-bottom: 5px; color: var(--clr-text); }
+
+/* -------------------------------------------------
+   6. Components grid
+   ------------------------------------------------- */
+.components {
+    display: grid;
+    grid-template-columns: repeat(auto-fill, minmax(280px, 1fr));
+    gap: 1rem;
+}
+.component {
+    padding: 10px;
+    border: 1px solid var(--clr-border);
+    border-radius: 4px;
+}
+.component h3 { margin: 0 0 5px; }
+
+/* -------------------------------------------------
+   7. Panel toggles / modal
+   ------------------------------------------------- */
+.help-link {
+    cursor: pointer;
+    user-select: none;
+    color: var(--clr-accent);
+    text-align: right;
+}
+.help-link:hover { text-decoration: underline; }
+#helpText { display: none; }
+
+.componentDetail-link {
+    cursor: pointer;
+    user-select: none;
+    color: var(--clr-accent);
+    text-align: left;
+}
+.componentDetail-link:hover { text-decoration: underline; }
+#componentDetailText { display: none; }
+
+
+/* -------------------------------------------------
+   8. Misc helpers
+   ------------------------------------------------- */
+/* Hide numeric markers in metric columns (if any) */
+#host_metrics_column td { list-style: none; padding-left: 0; margin-left: 0; }
+
+.host-status {
+    display: inline-block;
+    width: 10px;
+    height: 10px;
+    border-radius: 50%;
+    margin-left: 6px;
+    margin-right: 8px;
+    vertical-align: middle;
+    background: #808080;          /* default - unknown / no timestamp */
+    transition: background-color 1s linear;   /* smooth fade */
+}
+
+
+/* -------------------------------------------------
+   9. Mobile adjustments
+   ------------------------------------------------- */
+@media (max-width: 768px) {
+  /* 1. Make the whole page a column */
+  .wrapper {
+    flex-direction: column;
+  }
+
+  /* 2. Hide the sidebar initially */
+  .sidebar {
+    position: relative;          /* take it out of the flow */
+    width: 100%;
+    max-height: 0;               /* collapsed */
+    overflow: hidden;
+    transition: max-height 0.3s ease-out;
+    background: var(--bg-sidebar);
+    padding: 0;                  /* remove padding */
+  }
+
+  .sidebar.show {
+    max-height: 500px;           /* enough for all items */
+    padding: 1rem;
+  }
+
+  /* 3. Move the toggle button into the header */
+  .mobile-toggle {
+    display: block;
+    font-size: 1.5rem;
+    background: transparent;
+    border: none;
+    color: var(--clr-accent);
+    padding: 0.5rem;
+    margin-bottom: 0.5rem;
+  }
+
+  /* 4. Main content no left padding */
+  .main {
+    padding-left: 0;
+    padding-right: 1rem;
+  }
+
+  /* 5. Table scroll on small screens */
+  .card table {
+    width: 100%;
+    table-layout: fixed;
+  }
+
+  .card table thead,
+  .card table tbody,
+  .card table tr,
+  .card table td,
+  .card table th {
+    display: block;
+  }
+
+  .card table tbody {
+    overflow-x: auto;
+    white-space: nowrap;
+  }
+
+  .card table td,
+  .card table th {
+    display: inline-block;
+    vertical-align: top;
+    width: 30%;
+  }
+
+  /* 6. Adjust the host status dot positioning */
+  .host-status {
+    margin-left: 2px;
+    margin-right: 12px;
+  }
+}
+
+/* Optional: style the drawer indicator */
+.sidebar.show::before {
+  content: "✕ Close";
+  display: block;
+  padding: 0.5rem 1rem;
+  background: var(--bg-sidebar);
+  color: var(--clr-accent);
+  font-weight: bold;
+}
+
+
+/* for stacking storage clients  */
+.storage_client{
+
+    flex: 1;
+    padding: 1rem;
+    padding-left: 200px;  /* space for the fixed sidebar */
+    overflow-x: hidden;
+    display: flex;        /* make it a flex container */
+    flex-direction: column; /* stack its children (cards) vertically */
+    align-items: center;  /* center the cards horizontally */
+
+}
+
+/* lets make buttons nice*/
+
+/* 2. Base button style */
+.btn {
+  display: inline-block;
+  padding: .55rem 1.25rem;
+  font-size: .9rem;
+  font-weight: 500;
+  line-height: 1.4;
+  color: var(--clr-btn-text);
+  background: var(--clr-btn-bg);
+  border: 1px solid var(--clr-btn-border);
+  border-radius: .25rem;
+  cursor: pointer;
+  transition: background .15s ease, box-shadow .15s ease, transform .075s ease;
+  text-align: center;
+  user-select: none;
+  vertical-align: middle;
+}
+
+/* 3. Primary button - accent color */
+.btn-primary {
+  background: var(--clr-btn-accent);
+  border-color: var(--clr-btn-accent);
+  color: #fff;          /* brighter text for dark button */
+}
+
+/* 4. Secondary button - subtle background */
+.btn-secondary {
+  background: transparent;
+  border-color: var(--clr-btn-accent);
+  color: var(--clr-btn-accent);
+}
+
+/* 5. Hover & focus states */
+.btn:hover,
+.btn:focus-visible {
+  background: #2b3f4c;   /* a slightly darker shade of card bg */
+  box-shadow: 0 3px 8px var(--clr-btn-shadow);
+  transform: translateY(-1px);
+  outline: 0;
+}
+
+/* 6. Disabled state */
+.btn:disabled,
+.btn[aria-disabled="true"] {
+  opacity: .55;
+  cursor: not-allowed;
+  background: var(--bg-card);
+  border-color: var(--clr-border);
+}
+
+/* 7. Extra - block‑level button */
+.btn-block {
+  display: block;
+  width: 100%;
+  text-align: center;
+}
+
+/* let's also make the <select> element not suck*/
+
+/* Base <select> style */
+.select-dark {
+  appearance: none;          /* hide native arrow in Chrome/Edge/FF */
+  -webkit-appearance: none;
+  -moz-appearance: none;
+
+  display: inline-block;
+  padding: .45rem 1.25rem .45rem .75rem;
+  font-size: .9rem;
+  line-height: 1.4;
+  color: var(--clr-select-text);
+  background: var(--clr-select-bg);
+  border: 1px solid var(--clr-select-border);
+  border-radius: .25rem;
+  cursor: pointer;
+  position: relative;
+  min-width: 150px;          /* optional - forces a decent width */
+
+  /* Custom arrow - positioned after the element */
+  background-image: linear-gradient(45deg, transparent 50%, var(--clr-select-accent) 50%),
+                    linear-gradient(135deg, var(--clr-select-accent) 50%, transparent 50%),
+                    linear-gradient(to right, var(--clr-select-border), var(--clr-select-border));
+  background-position:
+    calc(100% - 20px) calc(50% + 2px),
+    calc(100% - 15px) calc(50% + 2px),
+    calc(100% - 3px) 50%;
+  background-size:
+    5px 5px,
+    5px 5px,
+    1px 1.5rem;
+  background-repeat: no-repeat;
+}
+
+/* Hover & focus - subtle accent border */
+.select-dark:hover,
+.select-dark:focus-visible {
+  border-color: var(--clr-select-accent);
+  box-shadow: 0 0 3px var(--clr-select-shadow);
+  outline: none; /* rely on the box-shadow for focus */
+}
+
+/* Disabled state - greys out the control */
+.select-dark:disabled,
+.select-dark[aria-disabled="true"] {
+  opacity: .55;
+  cursor: not-allowed;
+  background: var(--bg-card);
+  border-color: var(--clr-border);
+}
+
+/* <option> styling - works in most browsers */
+.select-dark option {
+  background: var(--bg-card);
+  color: var(--clr-select-text);
+}
+
+/* Optional - make the arrow bigger on hover for visibility */
+.select-dark:hover {
+  background-position:
+    calc(100% - 18px) calc(50% + 2px),
+    calc(100% - 13px) calc(50% + 2px),
+    calc(100% - 3px) 50%;
+}

web_dash/web/html/src/system_metrics.js

@@ -0,0 +1,284 @@
+/* system_metrics.js */
+
+(() => {
+  /* Socket.IO setup */
+  const socket = io({
+    transports: ['websocket', 'polling'],
+    reconnection: true,
+    reconnectionAttempts: Infinity,
+    reconnectionDelay: 3000,
+    reconnectionDelayMax: 60000,
+    timeout: 60000,
+    pingTimeout: 5000,
+    pingInterval: 25000,
+  });
+  /* Color constants */
+  const GREEN  = [ 39, 174,  96];  // #27ae60
+  const YELLOW = [243, 156,  18];  // #f39c12
+  const RED    = [192,  57,  43];  // #c0392b
+  /* Helpers */
+  const hostTimestamps = {};          // keyed by short_id
+  const toRgb = (r, g, b) => `rgb(${r},${g},${b})`;
+  const T20  = 20 * 1000;
+  const T40  = 40 * 1000;
+  const T60  = 60 * 1000;
+  function getFreshnessColor(ageMs) {
+    if (ageMs <= T20) {
+      return toRgb(...GREEN);
+    }
+    if (ageMs <= T40) {
+      const t = (ageMs - T20) / (T40 - T20);
+      const r = Math.round(GREEN[0] + t * (YELLOW[0] - GREEN[0]));
+      const g = Math.round(GREEN[1] + t * (YELLOW[1] - GREEN[1]));
+      const b = Math.round(GREEN[2] + t * (YELLOW[2] - GREEN[2]));
+      return toRgb(r, g, b);
+    }
+    if (ageMs <= T60) {
+      const t = (ageMs - T40) / (T60 - T40);
+      const r = Math.round(YELLOW[0] + t * (RED[0] - YELLOW[0]));
+      const g = Math.round(YELLOW[1] + t * (RED[1] - YELLOW[1]));
+      const b = Math.round(YELLOW[2] + t * (RED[2] - YELLOW[2]));
+      return toRgb(r, g, b);
+    }
+    return toRgb(...RED);
+  }
+  function safeSetText(id, txt) {
+    const el = document.getElementById(id);
+    if (el) el.textContent = txt;
+  }
+  /* Get the short_id from the query string */
+  function getSelectedId() {
+    return new URLSearchParams(window.location.search).get('host') || '';
+  }
+  /* Sidebar building - uses short_id for status key */
+
+  function buildList(systemList) {
+    const ul = document.getElementById('endpointList');
+
+    if (!Array.isArray(systemList)) {
+      ul.innerHTML = '';        // nothing to show
+      return;
+    }
+
+    /* Sort: servers first, then by IP */
+    const toInt = ip => {
+      // guard against undefined / empty string
+      if (!ip) return 0;
+      return ip.split('.').reduce((acc, octet) => (acc << 8) + parseInt(octet, 10), 0);
+    };
+
+    const sorted = [...systemList].sort((a, b) => {
+      // a.  Servers go before non-servers
+      const aServer = !!a.is_server;
+      const bServer = !!b.is_server;
+      if (aServer !== bServer) return aServer ? -1 : 1;   // true < false
+
+      // b.  Same “is_server” status - fall back to IP sorting
+      const aIp = a.active_ip ?? '';
+      const bIp = b.active_ip ?? '';
+
+      if (!aIp) return 1;          // push empty IPs to the end
+      if (!bIp) return -1;
+
+      return toInt(aIp) - toInt(bIp);
+    });
+
+    /* Bail if nothing actually changed */
+    const current = Array.from(ul.children).map(li => li.dataset.id);
+    const newIds   = sorted.map(s => s.short_id);
+    if (arraysEqual(current, newIds)) return;   // no visual change needed
+
+    /* Build the DOM */
+    const selected = getSelectedId().toLowerCase();
+    ul.innerHTML = ''; // reset
+
+    sorted.forEach(item => {
+      const li   = document.createElement('li');
+
+      // • Status dot
+      const status = document.createElement('span');
+      status.className = 'host-status';
+      status.dataset.id = item.short_id;
+
+      // • Link - display hostname, encode short_id in URL
+      const a = document.createElement('a');
+      a.href = '?host=' + encodeURIComponent(item.short_id);
+      a.textContent = item.hostname;
+      a.title = item.active_ip ? `Active IP: ${item.active_ip}` : '';
+      if (item.short_id.toLowerCase() === selected) a.classList.add('active');
+
+      li.appendChild(status);
+      li.appendChild(a);
+      ul.appendChild(li);
+    });
+  }
+
+  /* Update status colors every second */
+  function updateStatusColors() {
+    const nowSec = Date.now() / 1000;
+    Object.entries(hostTimestamps).forEach(([id, ts]) => {
+      const ageMs = (nowSec - ts) * 1000;
+      const color = getFreshnessColor(ageMs);
+      const span = document.querySelector(
+        `.host-status[data-id="${id}"]`
+      );
+      if (span) span.style.backgroundColor = color;
+    });
+  }
+  setInterval(updateStatusColors, 1000);
+  /* Utility helpers */
+  function arraysEqual(a, b) {
+    if (a.length !== b.length) return false;
+    for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
+    return true;
+  }
+  function renderGenericTable(containerId, data, emptyMsg) {
+    const container = document.getElementById(containerId);
+    if (!Array.isArray(data) || !data.length) {
+      container.textContent = emptyMsg;
+      return;
+    }
+    const merged = mergeRowsByName(data);
+    const ordered = orderRows(merged);
+    const table = buildTable(ordered);
+    table.id = 'host_metrics_table';
+    container.innerHTML = '';
+    container.appendChild(table);
+  }
+  function mergeRowsByName(rows) {
+    const groups = {};          // { Source: { Metric: [], Data: [] } }
+    rows.forEach(r => {
+      const src = r.Source;
+      if (!src) return;
+      if (!groups[src]) groups[src] = { Metric: [], Data: [] };
+      if ('Metric' in r && 'Data' in r) {
+        groups[src].Metric.push(r.Metric);
+        groups[src].Data.push(r.Data);
+      }
+    });
+    return Object.entries(groups).map(([src, g]) => ({
+      Source: src,
+      Metric: g.Metric,
+      Data:   g.Data,
+    }));
+  }
+  function orderRows(rows) {
+    const priority = ['System', 'CPU', 'RAM'];
+    const map = {};
+    priority.forEach((s, i) => map[s] = i);
+    return [...rows].sort((a, b) => {
+      const ai = map.hasOwnProperty(a.Source) ? map[a.Source] : Infinity;
+      const bi = map.hasOwnProperty(b.Source) ? map[b.Source] : Infinity;
+      return ai - bi;
+    });
+  }
+  function buildTable(rows) {
+    const cols = ['Source', 'Metric', 'Data'];
+    const table = document.createElement('table');
+    // Header
+    const thead = table.createTHead();
+    const headerRow = thead.insertRow();
+    cols.forEach(col => {
+      const th = document.createElement('th');
+      th.textContent = col;
+      headerRow.appendChild(th);
+    });
+    // Body
+    const tbody = table.createTBody();
+    rows.forEach(item => {
+      const tr = tbody.insertRow();
+      cols.forEach(col => {
+        const td = tr.insertCell();
+        const val = item[col];
+        if (Array.isArray(val)) {
+          val.forEach((v, i) => {
+            const span = document.createElement('span');
+            span.textContent = v;
+            td.appendChild(span);
+            if (i < val.length - 1) td.appendChild(document.createElement('br'));
+          });
+        } else {
+          td.textContent = val !== undefined ? val : '';
+        }
+      });
+    });
+    return table;
+  }
+  /* Handle incoming data */
+  let lastUpdate = Date.now();
+  function handleSummary(raw) {
+    lastUpdate = Date.now();                // reset watchdog
+    let payload;
+    if (typeof raw === 'string') {
+      try { payload = JSON.parse(raw); } catch (e) {
+        safeSetText('client_summary', 'Invalid data received');
+        return;
+      }
+    } else payload = raw;
+    if (!Array.isArray(payload) || !payload.length) {
+      safeSetText('client_summary', 'No data available');
+      return;
+    }
+    // Build the list first (so <span> elements exist)
+    buildList(payload);
+    // Store the timestamp for every short_id
+    payload.forEach(hostObj => {
+      if (hostObj.short_id && hostObj.data_timestamp) {
+        hostTimestamps[hostObj.short_id] = hostObj.data_timestamp; // seconds
+      }
+    });
+    // Immediately update colors for the current view
+    updateStatusColors();
+    // Metric table for selected host
+    const selectedId = getSelectedId();
+    const hostObj = payload.find(h => h.short_id === selectedId) || payload[0];
+    const hostData = hostObj && Array.isArray(hostObj.redis_data)
+                     ? hostObj.redis_data
+                     : [];
+    renderGenericTable('host_metrics', hostData, 'No Stats available');
+  }
+  /* Socket event wiring */
+  socket.on('client_summary', handleSummary);
+  socket.on('connect', () => {
+    safeSetText('client_summary', 'Connected');
+    requestSummary();
+  });
+  socket.on('disconnect', () => {
+    safeSetText('client_summary', 'Disconnected - retrying...');
+  });
+  socket.on('reconnect', attempt => {
+    safeSetText('client_summary', `Re-connected (attempt ${attempt})`);
+    requestSummary();
+  });
+  /* Request logic */
+  function requestSummary() {
+    if (!socket.connected) return;          // guard against stale emits
+    socket.emit('get_client_summary');      // server will reply via client_summary
+  }
+  /* Recursive polling */
+  let pollTimer = null;
+  function pollLoop() {
+    if (!socket.connected) return;
+    requestSummary();
+    pollTimer = setTimeout(pollLoop, 5000);
+  }
+  socket.on('connect', () => {
+    if (!pollTimer) pollLoop();
+  });
+  /* Watchdog - force reconnect if no data for 15 s */
+  function watchdog() {
+    if (Date.now() - lastUpdate > 15000 && socket.connected) {
+      safeSetText('client_summary', 'No updates - reconnecting...');
+      socket.disconnect();                 // forces a reconnect cycle
+    }
+    setTimeout(watchdog, 5000);
+  }
+  watchdog();
+  /* Keep the 'active' link in sync when the URL changes */
+  window.addEventListener('popstate', () => {
+    const selected = getSelectedId().toLowerCase();
+    document.querySelectorAll('#endpointList a').forEach(a =>
+      a.classList.toggle('active', a.href.includes('host=' + encodeURIComponent(selected)))
+    );
+  });
+})();

web_dash/web/node_server/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "redis-table-demo",
+  "version": "1.0.0",
+  "main": "server.js",
+  "scripts": {
+    "start": "node server.js"
+  },
+  "dependencies": {
+    "express": "^4.18.2",
+    "socket.io": "^4.7.2",
+    "redis": "^4.6.7",
+    "node-fetch": "^2.6.7",
+    "js-yaml": "^4.1.0" 
+  }
+}

web_dash/web/node_server/server.js

@@ -0,0 +1,153 @@
+// server.js
+const http   = require('http');
+const express = require('express');
+const { createClient } = require('redis');
+const { Server } = require('socket.io');
+const fetch  = require('node-fetch');   // npm i node-fetch@2
+const fs     = require('fs');
+const yaml   = require('js-yaml');      // npm i js-yaml
+const path   = require('path');
+
+const app    = express();
+const server = http.createServer(app);
+const io     = new Server(server);
+
+/* --------------------------------------------------------------------- */
+/* ---------- 1. Load the YAML configuration file ---------------------- */
+/* --------------------------------------------------------------------- */
+let config = {};
+try {
+  const filePath = '/app/cosmostat_settings.yaml';
+  const file     = fs.readFileSync(filePath, 'utf8');
+  config = yaml.load(file);
+} catch (e) {
+  console.error('Failed to load config.yaml:', e);
+  process.exit(1);
+}
+const API_PORT   = config.custom_api_port   || 5000;   // fallback to 5000
+const API_HOST   = config.api_bind_ip   || '192.168.37.1'; // fallback IP
+const API_BASE   = `http://${API_HOST}:${API_PORT}`;
+console.log('API URL:', API_BASE);
+
+/* --------------------------------------------------------------------- */
+/* ---------- 2. Socket.io -------------------------------------------- */
+/* --------------------------------------------------------------------- */
+io.on('connection', async socket => {
+  console.log('client connected:', socket.id);
+
+  /* -------------  send cached client_summary  ------------- */
+  if (clientSummaryCache.last) {
+    socket.emit('client_summary', clientSummaryCache.last);
+    console.log('sent cached client_summary to', socket.id);
+  }
+
+  /* ----------------------------------------------------------------- */
+  /* Call the external API every time a client connects                */
+  /* ----------------------------------------------------------------- */
+  try {
+    const resp = await fetch(`${API_BASE}/start_timer`, { method: 'GET' });
+    const data = await resp.json();
+    console.log('API responded to connect:', data);
+  } catch (err) {
+    console.error('Failed to hit start_timer endpoint:', err);
+  }
+
+  /* ----------------------------------------------------------------- */
+  /* Listen for tableRendered event from the client                    */
+  /* ----------------------------------------------------------------- */
+  socket.on('tableRendered', async () => {
+    console.log('Client reported table rendered - starting timer');
+    try {
+      const resp = await fetch(`${API_BASE}/start_timer`, { method: 'GET' });
+      const text = await resp.text();
+      console.log('Timer endpoint responded:', text);
+    } catch (err) {
+      console.error('Failed to hit start_timer:', err);
+    }
+  });
+});
+
+/* --------------------------------------------------------------------- */
+/* ---------- 3. Serve static files ----------------------------------- */
+/* --------------------------------------------------------------------- */
+app.use(express.static('public'));
+
+/* --------------------------------------------------------------------- */
+/* ---------- 4. Redis subscriber ------------------------------------- */
+/* --------------------------------------------------------------------- */
+const redisClient = createClient({
+  url:      'redis://0.0.0.0:6379',
+  socket: { keepAlive: 60000, // 60 s TCP keep-alive
+            reconnectStrategy: attempts => Math.min(attempts * 100, 3000) } // back-off
+});
+redisClient.on('error', err => console.error('Redis error', err));
+
+/* --- local cache for client_summary -------------------------------- */
+const clientSummaryCache = {};   // { last: <payload> }
+
+/* --------------------------------------------------------------------- */
+(async () => {
+  await redisClient.connect();
+  const sub = redisClient.duplicate();
+  await sub.connect();
+
+  /* --------------------------------------------------------------------- */
+  /* Helper that re-subscribes to a channel (and re-sends the handler)     */
+  /* --------------------------------------------------------------------- */
+  async function safeSubscribe(channel, handler) {
+    try {
+      await sub.subscribe(channel, handler);
+      console.log(`Subscribed to ${channel}`);
+    } catch (e) {
+      console.error(`Failed to subscribe to ${channel}`, e);
+    }
+  }
+
+  /* --------------------------------------------------------------------- */
+  /* Subscribe to all required channels                                   */
+  /* --------------------------------------------------------------------- */
+  await safeSubscribe('host_metrics',   (msg) => forward('host_metrics',   msg));
+  await safeSubscribe('client_summary', (msg) => forward('client_summary', msg));
+
+  /* --------------------------------------------------------------------- */
+  /* Forward messages to Socket.io                                         */
+  /* --------------------------------------------------------------------- */
+  function forward(channel, message) {
+    try {
+      const payload = JSON.parse(message);
+
+      /* -----  update cache on client_summary  ----- */
+      if (channel === 'client_summary') {
+        clientSummaryCache.last = payload;
+      }
+
+      io.emit(channel, payload);
+    } catch (e) {
+      console.error(`Failed to parse message from ${channel}`, e);
+    }
+  }
+
+  /* --------------------------------------------------------------------- */
+  /* Re-subscribe automatically when the Redis connection reconnects     */
+  /* --------------------------------------------------------------------- */
+  sub.on('reconnecting', () => console.log('Redis reconnecting…'));
+  sub.on('ready', async () => {
+    console.log('Redis ready - re-subscribing to all channels');
+    await safeSubscribe('host_metrics',   (msg) => forward('host_metrics',   msg));
+    await safeSubscribe('client_summary', (msg) => forward('client_summary', msg));
+  });
+
+  /* Optional: if the connection ends for any reason, close the process */
+  sub.on('end', () => {
+    console.error('Redis connection closed - exiting');
+    process.exit(1);
+  });
+})();
+
+/* --------------------------------------------------------------------- */
+/* ---------- 5. Start the HTTP server --------------------------------- */
+/* --------------------------------------------------------------------- */
+const PORT = process.env.PORT || 3000;
+server.listen(PORT, () => {
+  console.log(`Server listening on http://localhost:${PORT}`);
+});

web_dash/web/proxy/nginx.conf

@@ -0,0 +1,78 @@
+# nginx.conf
+# This file will be mounted into /etc/nginx/conf.d/default.conf inside the container
+
+# Enable proxy buffers (optional but recommended)
+proxy_buffering on;
+proxy_buffers 16 16k;
+proxy_buffer_size 32k;
+
+server {
+listen 80;
+server_name localhost;
+
+    # ---------------------------------------
+    # API Endpoints
+    # ---------------------------------------
+    location = /descriptor {
+    proxy_pass http://192.168.37.1:5000/descriptor;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    }
+    location = /update_client {
+    proxy_pass http://192.168.37.1:5000/update_client;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    }
+    location = /create_client {
+    proxy_pass http://192.168.37.1:5000/create_client;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    }
+    location = /storage_client_update {
+    proxy_pass http://0.0.0.0:5001/storage_client_update;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    }
+    location = /test_storage_summary {
+    proxy_pass http://0.0.0.0:5001/test_storage_summary;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    }
+    
+
+    # ---------------------------------------
+    # WebSocket endpoint
+    # ---------------------------------------
+    location /socket.io/ {
+        proxy_pass http://0.0.0.0:3000/socket.io/;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+    # ---------------------------------------
+    # All other paths → Apache (PHP)
+    # ---------------------------------------
+    location / {
+        proxy_pass http://0.0.0.0:8080;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
+}
+