purging old files

This commit is contained in:
matt committed 2026-10-14 00:53:35 -07:00
1 parent ce011bd236
commit ac2b142b7a
9 files changed
+1 -806

No files matched your search

-61
View File
@@ -1,61 +0,0 @@
import redis
import atexit
from flask import Flask, jsonify
from flask_apscheduler import APScheduler
from datetime import datetime, timedelta
from scanner import *
from config import *
from ble_redis import *
import logging
logging.getLogger('apscheduler').setLevel(logging.ERROR)
app = Flask(__name__)
scheduler = APScheduler()
scheduler.init_app(app)
scheduler.start()
scan_macs()
# --- Scheduler job ----------------------------------------------------
@scheduler.task(
'interval',
id='ble_scan',
minutes=SCAN_INTERVAL_MINUTES,
misfire_grace_time=30
)
def ble_scan_job():
"""Runs in the background, calls the async scanner."""
update_cache()
# --- API -------------------------------------------------------------
@app.route('/macs', methods=['GET'])
def list_macs():
"""Return the current set of MACs as JSON."""
macs = sorted(get_cache_snapshot())
return jsonify({"count": len(macs), "macs": macs})
@app.route('/details', methods=['GET'])
def sensor_detail():
"""Return the current set of MACs as JSON."""
sensors = get_sensor_details()
return jsonify(sensors)
@app.route('/summary', methods=['GET'])
def sensor_summary():
"""Return the current set of MACs as JSON."""
sensors = get_sensor_summary()
return jsonify(sensors)
# --- Convenience route -----------------------------------------------
@app.route('/', methods=['GET'])
def hello():
return "BLE Scanner API - see /macs for the list."
# --- Run --------------------------------------------------------------
if __name__ == '__main__':
# The scheduler runs in its own thread; the main thread serves Flask.
app.run(host=bind_ip, port=api_port, debug=False)
# Register a graceful shutdown hook so the scanner state is persisted
atexit.register(save_state)
-124
View File
@@ -1,124 +0,0 @@
import asyncio
from bleak import BleakClient, BleakScanner
import warnings, logging
warnings.filterwarnings("ignore")
logging.getLogger('apscheduler').setLevel(logging.ERROR)
# Mi sensor UUIDs
MI_SERVICE_UUID = "ebe0ccb0-7a0a-4b0c-8a1a-6ff2997da3a6"
TEMP_CHAR_UUID = "ebe0ccc1-7a0a-4b0c-8a1a-6ff2997da3a6"
# scanner constants
full_timeout = 60
attempts = 6
async def read_data(mac_address):
# Scan for the device
ratio = full_timeout / attempts
fail_count = full_timeout / ratio
device = None
print(f"Beginning read_data for {mac_address} - {full_timeout} timeout - {ratio} ratio - please stand by...")
while fail_count > 0 and not device:
# print(f"Starting discover loop {fail_count}")
devices = await BleakScanner.discover(timeout=ratio)
# print("BleakScanner discover timeout pass")
device = next((d for d in devices if d.address.lower() == mac_address.lower()), None)
if device:
# print(f"BleakScanner device variable loaded - {device.name}")
print(f".. Found device: {device.name} ({device.address}) - connecting...")
fail_count = 0
# else:
# print("BleakScanner device not loaded yet")
fail_count = fail_count - 1
if not device:
print(f"!! Device with MAC {mac_address} not found")
return None
# Connect to the device
mi_service = None
async with BleakClient(device) as client:
success = False
# Check if Mi service is available
#print(".. Waiting on client to connect...")
while not client.is_connected:
await asyncio.sleep(0.1)
while not success:
try:
#print(".. Client connected, getting services")
services = client.services
#print(".. Services acquired, loading uuid matches")
mi_service = next((s for s in services if s.uuid.lower() == MI_SERVICE_UUID.lower()), None)
success = True
#print("uuid matches loaded")
except Exception as e:
print(f"Mi service unavailable, infinite loop looping RIP - {e}")
# Read temperature data
sensor_data = await client.read_gatt_char(TEMP_CHAR_UUID)
# Parse the temperature value (same logic as in JavaScript)
sign = sensor_data[1] & (1 << 7)
temp = ((sensor_data[1] & 0x7F) << 8 | sensor_data[0])
if sign:
temp = temp - 32767
temp = temp / 100
hum = sensor_data[2]
print(f"-- Temperature: {temp:.1f}°C, Humidity: {hum}%")
return temp, hum
def sensor_data_query(mac_list):
result = []
for mac in mac_list:
try:
this_result = asyncio.run(read_data(mac))
except Exception as e:
print(f"Error reading BLE data - {e}")
result.append({"name": mac, "temp": this_result[0], "humidity": this_result[1]})
print(f"Data query complete - returning result")
return result
def test():
test_result = False
sensors = []
mac_list = [
"A4:C1:38:11:F4:2A",
"A4:C1:38:24:15:B0",
"A4:C1:38:D7:8D:26"
]
print("Starting BLE test...")
print(" Scanning MACs:")
for mac in mac_list:
print(f" {mac}")
test_list = mac_list
while not test_result:
print(" Test loop start. Initializing run...")
test_run_results = sensor_data_query(test_list)
for test_run_result in test_run_results:
sensors.append(test_run_result)
passed_list = [device['name'] for device in sensors]
print(f" Passed list - {passed_list}")
if len(mac_list) == len(passed_list):
print(" Test run passed, yee-haw")
test_result = True
else:
print(f" Test run incomplete/failed, re-running, updating test list")
mac_list_new = []
for mac in mac_list:
if mac not in passed_list:
mac_list_new.append(mac)
test_list = mac_list_new
print(f" Re-do list - {test_list}")
if __name__ == "__main__":
#print("Welp, this will keep going forever. Have fun :)")
#while True:
#try:
test()
#except Exception as e:
# print(f"Test failed brah - {e}")
-19
View File
@@ -1,19 +0,0 @@
import redis
import json
from config import *
####################################################
### Redis Functions
####################################################
r = redis.Redis(host=bind_ip, port=redis_port)
def update_redis(data):
# push data to redis
# Publish to the Redis channel that the WS server is listening on
r.publish('temp_hum', json.dumps(data))
if debug_output:
print("=== Stats Redis Update ===")
print(json.dumps(data, indent=2))
return True
-29
View File
@@ -1,29 +0,0 @@
# --- BLE & app config -------------------------------------------------
MAC_PREFIX = "A4:C1:38"
SCAN_INTERVAL_MINUTES = 1
debug_output = False
bind_ip = "0.0.0.0"
api_port = 5000
redis_port = 6379
enable_redis = True
# --- This will store friendly names for MAC addresses -----------------
comment_lookup = [
{
"mac": "A4:C1:38:11:F4:2A",
"comment": "Kitchen - 0"
},
{
"mac": "A4:C1:38:D7:8D:26",
"comment": "Table - 1"
},
{
"mac": "A4:C1:38:24:15:B0",
"comment": "Hallway 2"
},
{
"mac": "A4:C1:38:96:53:30",
"comment": "Bedroom 3"
}
]
-9
View File
@@ -1,9 +0,0 @@
anyio
bleak
cryptography
click
setuptools
Flask-APScheduler
Flask
APScheduler
redis
-11
View File
@@ -1,11 +0,0 @@
# 3. Create a venv (optional but recommended)
python3 -m venv .venv
source .venv/bin/activate
# 4. Install dependencies
pip install -r requirements.txt --quiet
# 5. Start
#python ble.py
python app.py
-189
View File
@@ -1,189 +0,0 @@
import asyncio
import threading
import time
import redis
import pickle
import os
from datetime import datetime, timezone
from zoneinfo import ZoneInfo # Python 3.9+
from bleak import BleakScanner, BleakClient
from typing import Set, Iterable
from config import *
from sensors import *
from ble import *
from ble_redis import *
# In-memory cache + thread-safe lock
_mac_cache: Set[str] = set()
_lock = threading.Lock()
_sensors_cache = []
# ----------------------------------------------------------------------
# Pickle persistence helpers
# ----------------------------------------------------------------------
STATE_FILE = "scanner_state.pkl"
def save_state() -> None:
"""
Persist the MAC cache and sensor objects to disk.
Called automatically after each cache refresh and at program exit.
"""
with _lock:
state = {
"mac_cache": _mac_cache,
"sensors": _sensors_cache,
}
try:
with open(STATE_FILE, "wb") as f:
pickle.dump(state, f, protocol=pickle.HIGHEST_PROTOCOL)
print("[Scanner] State persisted to", STATE_FILE)
except Exception as exc:
# Don't crash the app – just log the error
print("[Scanner] Warning: could not persist state:", exc)
def load_state() -> None:
"""
Load the MAC cache and sensor objects from disk if the file exists.
Called automatically on import.
"""
if not os.path.exists(STATE_FILE):
return
try:
with open(STATE_FILE, "rb") as f:
state = pickle.load(f)
with _lock:
# NOTE: we replace the lists/sets to avoid mutating old references
_mac_cache.clear()
_mac_cache.update(state.get("mac_cache", []))
global _sensors_cache
_sensors_cache = state.get("sensors", [])
print("[Scanner] Loaded state from", STATE_FILE)
for cached_mac in _mac_cache:
print(f"Found {cached_mac}")
except Exception as exc:
print("[Scanner] Warning: could not load state:", exc)
# Load any persisted state on module import
load_state()
# Redis Functions
r = redis.Redis(host=bind_ip, port=redis_port)
def update_redis(data):
# push data to redis
# Publish to the Redis channel that the WS server is listening on
r.publish('temp_hum', json.dumps(data))
if debug_output:
print("=== Stats Redis Update ===")
print(json.dumps(data, indent=2))
return True
# this function filters MACs
def _is_match(mac: str, name: str) -> bool:
"""Return True if the MAC starts with the configured prefix and is correct model."""
if name == "LYWSD03MMC":
return mac.upper().startswith(MAC_PREFIX.upper())
# this function scans for MACs
async def _scan_once() -> Iterable[str]:
"""Do one scan cycle and return the matching MACs."""
devices = await BleakScanner.discover(timeout=5.0) # 5s scan
return [dev.address for dev in devices if _is_match(dev.address, dev.name)]
# this very important function updates the MAC cache and the sensor objects
def update_cache() -> None:
"""
Called by the scheduler (in a separate thread). It runs a tiny
asyncio event loop to call the Bleak scan, then updates the cache.
"""
# Run the async scan in a fresh loop; Bleak requires this.
# Gets the MACs
print("")
print("--- Starting update_cache loop ---")
new_macs = scan_macs()
# now that we have MACs, let's get sensor data
print("Updating sensor data...")
sensor_data = None
try:
sensor_data = sensor_data_query(new_macs)
# print(sensor_data)
except Exception as e:
print(f"Aww fuck, no data :( - {e}")
# now we update the data in the sensor objects
print("Sensor data collected:")
for sensor_datum in sensor_data:
print(f"MAC - {sensor_datum['name']} | temp - {sensor_datum['temp']} | humidity - {sensor_datum['humidity']}")
for sensor in _sensors_cache:
if sensor.name == sensor_datum['name']:
sensor.add_or_update(key="temp", value=sensor_datum['temp'])
sensor.add_or_update(key="humidity", value=sensor_datum['humidity'])
# if redis is enabled, push full details to the redis server
if enable_redis:
update_redis(get_sensor_details())
save_state()
# this helper function scans for MACs
def scan_macs():
print("Scanning for MACs:")
new_macs = asyncio.run(_scan_once())
with _lock:
for new_mac in new_macs:
print(f"{new_mac}")
new_sensor = Sensor(name=f"{new_mac}")
if new_mac not in set(_mac_cache):
_sensors_cache.append(new_sensor)
_mac_cache.update(new_macs)
print(f"Found {len(new_macs)} new MACs, total {len(_mac_cache)} MACs in cache.")
return new_macs
# return just the MACs
def get_cache_snapshot() -> Set[str]:
"""Return a thread-safe snapshot of the current cache."""
with _lock:
return set(_mac_cache)
# return sensor data and timestamps, can return int or string
def get_sensor_details(friendly_date = False):
sensor_list = []
for sensor in _sensors_cache:
first_seen = sensor.first_seen
last_seen = sensor.last_seen
if friendly_date:
first_seen = ts_to_pst_string(sensor.first_seen)
last_seen = ts_to_pst_string(sensor.last_seen)
# print(f"Summary - {sensor.name}")
new_sensor = {
"name": sensor.name,
"first_seen": first_seen,
"last_seen": last_seen,
"comment": sensor.comment,
"data": sensor.to_dict()
}
sensor_list.append(new_sensor)
result = {
"sensors": sensor_list
}
return result
# return just sensor data without timestamps
def get_sensor_summary():
sensor_list = []
for sensor in _sensors_cache:
# print(f"Summary - {sensor.name}")
new_sensor = {
"name": sensor.name,
"comment": sensor.comment,
"data": sensor.to_dict(simple_return=True)
}
sensor_list.append(new_sensor)
result = {
"sensors": sensor_list
}
return result
# helper function for ts int to string
def ts_to_pst_string(ts_str):
utc_dt = datetime.fromtimestamp(ts_str, tz=timezone.utc)
pacific_dt = utc_dt.astimezone(ZoneInfo("America/Los_Angeles"))
return pacific_dt.strftime("%Y-%m-%d %H:%M:%S")
-207
View File
@@ -1,207 +0,0 @@
from __future__ import annotations
import re, redis
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Union
from config import *
# --------------------------------------------------------------------------- #
# Helper utilities
# --------------------------------------------------------------------------- #
def _current_timestamp() -> int:
"""Return the current UTC timestamp (seconds)."""
return int(datetime.now(timezone.utc).timestamp())
def _ensure_timestamp(ts: Optional[Union[int, datetime]]) -> int:
"""
Normalise a timestamp that can be an int (Unix seconds) or a datetime.
Raises ValueError if the value is not recognised.
"""
if ts is None:
return _current_timestamp()
if isinstance(ts, int):
return ts
if isinstance(ts, datetime):
return int(ts.replace(tzinfo=timezone.utc).timestamp())
raise ValueError(f"Unsupported timestamp type: {type(ts)}")
# --------------------------------------------------------------------------- #
# Sensor implementation
# --------------------------------------------------------------------------- #
@dataclass
class Sensor:
"""
Represents a sensor that records timestamped values under a set of keys.
Attributes
----------
name : str
The unique identifier for the sensor itself.
first_seen : int
Unix timestamp of when the sensor was first instantiated.
last_seen : int
Unix timestamp of the most recent data update (or creation time).
data : List[Dict[str, Any]]
A list of data points, each with keys:
- name (str)
- value (Any)
- first_seen (int)
- last_seen (int)
"""
name: str
first_seen: int = field(init=False)
last_seen: int = field(init=False)
data: List[Dict[str, Any]] = field(default_factory=list)
# ----------------------------------------------------------------------- #
# Validity rules
# ----------------------------------------------------------------------- #
# Current rule: key must be a "word" (letters, digits, underscore)
VALID_NAME_RE = re.compile(r"^[A-Za-z0-9_]+$")
def _is_valid_key(self, key: str) -> bool:
"""Return True if the key meets current validity criteria."""
is_valid = False
if bool(self.VALID_NAME_RE.match(key)):
is_valid = True
return is_valid
# ----------------------------------------------------------------------- #
# Life-cycle hooks
# ----------------------------------------------------------------------- #
def __post_init__(self) -> None:
"""Initialise timestamps when the object is created."""
# print(f"Instantiating new sensor object - {self.name}")
ts = _current_timestamp()
self.first_seen = ts
self.last_seen = ts
self.comment = self.name
for sensor in comment_lookup:
if sensor['mac'] == self.name:
self.comment = sensor['comment']
# ----------------------------------------------------------------------- #
# Core API
# ----------------------------------------------------------------------- #
def add_or_update(self,
key: str,
value: Any,
timestamp: Optional[Union[int, datetime]] = None) -> None:
"""
Add a new data point or update an existing one.
Parameters
----------
key : str
The data key. Must pass :meth:`_is_valid_key`.
value : Any
The data value to store.
timestamp : int | datetime | None
When the data was recorded. If omitted, current time is used.
Raises
------
ValueError
If the key is not valid.
"""
if not self._is_valid_key(key):
raise ValueError(f"Invalid key '{key}'. Must contain only "
"letters, digits, or underscores.")
ts = _ensure_timestamp(timestamp)
# Update sensor-level timestamps
self.last_seen = max(self.last_seen, ts)
# Search for existing entry
for entry in self.data:
if entry["name"] == key:
# Update existing entry
entry["value"] = value
entry["last_seen"] = ts
if ts < entry["first_seen"]:
entry["first_seen"] = ts
return
# If we get here, it's a new key
new_entry: Dict[str, Any] = {
"name": key,
"value": value,
"first_seen": ts,
"last_seen": ts
}
self.data.append(new_entry)
def get_value(self, key: str) -> Optional[Any]:
"""Return the most recent value for a key, or None if it doesn't exist."""
for entry in self.data:
if entry["name"] == key:
return entry["value"]
return None
def update_comment(self, new_comment: str):
self.comment = new_comment
def to_dict(self, simple_return = False) -> Dict[str, List[Dict[str, Any]]]:
"""
Return the data in the exact structure requested.
Example
-------
>>> sensor.to_dict()
{
"data": [
{"name": "temp", "value": 23, "first_seen": 1710000000, "last_seen": 1710000005},
{"name": "hum", "value": 45, "first_seen": 1710000010, "last_seen": 1710000010}
]
}
"""
if simple_return:
simple_return = {
"temp": self.get_value(key="temp"),
"humidity": self.get_value(key="humidity")
}
return simple_return
else:
return {"data": self.data.copy()}
# ----------------------------------------------------------------------- #
# Convenience & debugging
# ----------------------------------------------------------------------- #
def __repr__(self) -> str:
return (f"{self.__class__.__name__}(name={self.name!r}, "
f"first_seen={self.first_seen}, last_seen={self.last_seen}, "
f"data={self.data!r})")
# --------------------------------------------------------------------------- #
# Example usage
# --------------------------------------------------------------------------- #
if __name__ == "__main__":
# Instantiate a sensor
sensor = Sensor(name="weather_station_01")
# Add some data
sensor.add_or_update("temperature", 23.5)
sensor.add_or_update("humidity", 45)
# Update an existing key
sensor.add_or_update("temperature", 24.0)
# Try an invalid key (will raise ValueError)
try:
sensor.add_or_update("bad key!", 0)
except ValueError as e:
print(e)
# Print the structured data
import json
print(json.dumps(sensor.to_dict(), indent=4))
+1 -157
View File
@@ -237,82 +237,6 @@ $systemComponents = $properties['system_components'] ?? [];
$selectedHost = $clients[$selectedIdx]['hostname'] ?? 'Unknown';
/* ---- Sidebar Renderer ---- */
function renderSidebar(string $mode){
global $clients, $client, $properties, $systemProperties, $systemComponents, $selectedHost, $selectedHosts, $selectedId, $selectedIdx, $remove_hosts;
$modes = [
'cosmostat' => 'Cosmostat',
/* 'gali' => 'Shuttle Gali',*/
'drive_health' => 'Drive Health',
];
?>
<nav class="sidebar">
<form method="get" id="modeForm">
<label for="modeSelect"></label>
<select class="select-dark" name="mode" id="modeSelect" onchange="this.form.submit()">
<?php foreach ($modes as $key => $label): ?><option value="<?= h($key) ?>" <?= $mode === $key ? 'selected' : '' ?>><?= h($label) ?></option>
<?php endforeach; ?></select>
</form>
<p>
<?php if ($mode === 'drive_health'): ?>
<?php
if ( !is_array($clients) || empty($clients) ) {
// Graceful “no data” handling
echo '<p style="margin:1rem 0; font-style:italic;">'
. 'No hosts are available to manage at this time.'
. '</p>';
}
?>
<?php if (is_array($clients) && !empty($clients)): ?>
<h3>Hosts</h3>
<form method="get" id="driveHealthForm">
<input type="hidden" name="mode" value="drive_health">
<ul>
<?php foreach ($clients as $c): ?>
<?php $id = $c['short_id']; ?>
<li>
<label>
<input type="checkbox" name="hosts[]" value="<?= h($id) ?>" <?= in_array($id, $selectedHosts, true) ? 'checked' : '' ?>><?= h($c['name']) ?>
</label>
</li>
<?php endforeach; ?>
</ul><p>
<button class="btn btn-primary" type="submit" name="action" value="apply">Apply</button><p>
<button class="btn btn-primary" type="submit" name="action" value="all">Select All</button><p>
<button class="btn btn-primary" type="submit" name="action" value="none">Select None</button><p>
</form>
<!-- Remove host button (POST) -->
<form method="post" id="removeForm">
<input type="hidden" name="mode" value="drive_health">
<input type="hidden" name="action" value="remove">
<input type="hidden" name="remove_hosts" id="remove_hosts_input" value="">
<button class="btn btn-primary" type="submit">Remove</button>
</form>
<?php endif; ?>
<?php endif; ?>
<?php if ($mode == 'cosmostat'): ?>
<input type="hidden" name="host" value="<?= h($selectedId) ?>">
<h3>Endpoints</h3>
<ol id="endpointList"></ol>
<?php endif; ?>
<?php if ($mode == 'gali'): ?>
<h3>Shuttle Gali</h3>
<?php endif; ?>
</nav>
<?php
}
/* ---- Main Content Renderer ---- */
function renderMainContent(string $mode){
@@ -322,71 +246,6 @@ function renderMainContent(string $mode){
die('<div class="card"><p style="color:red;">Could not retrieve data from the API for mode "' . h($mode) . '".</p></div>');
}
if ($mode === 'drive_health') {
// If nothing is selected, show a friendly message
if (empty($selectedHosts)) {
echo '<div class="card"><p>No hosts selected.</p></div>';
return;
}
echo '
<div class="storage_client">';
foreach ($selectedHosts as $sid) {
// Find the client that matches this short_id
$c = null;
foreach ($clients as $cl) {
if ($cl['short_id'] === $sid) {
$c = $cl;
break;
}
}
if ($c === null) continue; // safety
$hostname = $c['name'] ?? 'Unknown';
echo '
<div class="card">
<h2>Drive Health - ' . h($hostname) . '</h2>
<h3>IP: '.h($c['ip']).'<br>
Timestamp: '.date('F j, Y g:i a', (int) $c['timestamp']).' </h3>
';
if (isset($c['drives']) && is_array($c['drives']) && count($c['drives']) > 0) {
echo '
<table id="host_metrics_table">
<thead><tr>
<th>Drive Letter</th><th>Disk ID</th><th>Health Metrics</th><th>Model</th><th>Serial</th><th>Capacity</th>
</tr></thead>
<tbody>';
foreach ($c['drives'] as $drive) {
$wear_gb = '';
if (isset($drive['host_writes'])){
$wear_gb = $drive['host_writes'].' wear';
}
echo '
<tr>
<td>' . h($drive['drive_letter'] ?? '') . '</td>
<td>' . h("".$drive['disk_id'] ?? '') . '</td>
<td>' .
h($drive['health_status'] ?? '').'<br>'.
h($drive['power_on_hours'] ?? '').'<br>'.
h($wear_gb ?? '').
'</td>
<td>' . h($drive['model'] ?? '') . '</td>
<td>' . h($drive['serial'] ?? '') . '</td>
<td>' . h($drive['capacity'] ?? '') . '</td>
</tr>';
}
echo '
</tbody></table> ';
} else {
echo '<p>No drive data available for this host.</p>';
}
echo '
</div>';
}
echo '
</div>';
return;
}
?>
<div class="main">
@@ -471,16 +330,13 @@ function renderMainContent(string $mode){
<div class="wrapper">
<!-- Sidebar -->
<?php renderSidebar($mode); ?>
<!-- Main content -->
<?php renderMainContent($mode); ?>
</div> <!-- /wrapper -->
<!-- cosmostat javascript -->
<!-- javascript -->
<script src="socket.io/socket.io.js"></script>
<script src="src/system_metrics.js"></script>
<script>
@@ -495,18 +351,6 @@ function renderMainContent(string $mode){
});
</script>
<!-- drive health javascript -->
<script>
// Removal Handler
document.getElementById('removeForm').addEventListener('submit', function (e) {
// Grab all checked checkboxes from the Drive-Health form
const checked = document.querySelectorAll('#driveHealthForm input[type="checkbox"][name="hosts[]"]:checked');
const ids = Array.from(checked).map(cb => cb.value);
// Pass the comma-separated list to the hidden input
document.getElementById('remove_hosts_input').value = ids.join(',');
});
</script>
</body>
</html>