matt/cosmoserver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add string returns for metrics and properties

Browse Source
This commit is contained in:
Matt
2026-03-16 09:17:44 -07:00
parent adb1387693
commit 61421305ed
16 changed files with 524 additions and 390 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
defaults/main.yaml
View File

@ -2,6 +2,7 @@
# required system packages
cosmostat_packages:
- docker
- docker.io
- docker-compose
- python3
@ -27,6 +28,8 @@ cosmostat_venv_packages: |
cosmostat_sudoers_content: |
cosmos ALL=(root) NOPASSWD: /usr/bin/lshw
cosmos ALL=(root) NOPASSWD: /usr/sbin/smartctl
cosmos ALL=(root) NOPASSWD: /usr/bin/dmesg
cosmos ALL=(root) NOPASSWD: /usr/sbin/dmidecode
# subnet for service
docker_subnet: "192.168.37.0/24"
@ -47,16 +50,17 @@ api_service_exe: "{{ venv_folder }}/bin/python -u {{ api_service_folder }}/app.p
# dashboard vars
service_control_web_folder: "{{ service_folder }}/web"
public_dashboard: true
# will skip init when true
quick_refresh: false
# cosmostat_settings
noisy_test: false
debug_output: false
debug_output: true
secure_api: true
push_redis: true
run_background : true
log_output: true
update_frequency: "2"
update_frequency: "1"
...

422
files/api/Components.py
View File

@ -1,6 +1,7 @@
# this class file is for the cosmostat service
import subprocess
import json
import time
from typing import Dict, Any, List
# Global Class Vars
@ -8,8 +9,6 @@ global_max_length = 500
debug_output = False
# import the component descriptor
# this outlines how the component class works
# each type of component has a "type"
try:
with open("component_descriptors.json", encoding="utf-8") as f:
component_class_tree: List[Dict] = json.load(f)
@ -18,18 +17,25 @@ except FileNotFoundError as exc:
component_types = [{"name": entry["name"], "multi_check": entry["multi_check"] == "True"} for entry in component_class_tree]
#################################################################
# Component Class
#################################################################
class Component:
############################################################
# instantiate new component
############################################################
def __init__(self, name: str, comp_type: str, this_device="None"):
self.name = name
self.type = comp_type
self.this_device = this_device
print(f"This device - {self.this_device}")
# build the component descriptor dictionary
for component in component_class_tree:
if component["name"] == self.type:
COMPONENT_DESCRIPTORS = component
# Load component type descriptor from class tree
# COMPONENT_DESCRIPTORS = {d['type']: d for d in component_class_tree}
descriptor = COMPONENT_DESCRIPTORS
self._descriptor = descriptor
if descriptor is None:
@ -39,10 +45,10 @@ class Component:
)
# store static properties
self.multi_check = self.is_multi()
self.virt_ignore = self._descriptor.get('virt_ignore', [])
self._properties: Dict[str, str] = {}
for key, command in descriptor.get('properties', {}).items():
if self.this_device != "None":
print(f"command - {command}; this_device - {self.this_device}")
formatted_command = command.format(this_device=self.this_device)
self._properties[key] = run_command(formatted_command, True)
else:
@ -64,6 +70,10 @@ class Component:
f"{self.description}")
return self_string
############################################################
# Class Functions
############################################################
def update_metrics(self):
for key, command in self._descriptor.get('metrics', {}).items():
if self.this_device != "None":
@ -74,20 +84,85 @@ class Component:
else:
self._metrics[key] = run_command(command, True)
def get_property(self, type):
return self._properties[type]
def is_multi(self):
for component_type in component_types:
if self.type == component_type["name"]:
return component_type["multi_check"]
return False
########################################################
# redis data functions
########################################################
def get_properties_keys(self):
result = []
for name, value in self._properties.items():
this_property = {
"Source": self.name,
"Property": name,
"Value": value
}
if name not in self.virt_ignore:
result.append(this_property)
return result
def get_properties_strings(self):
result = []
for name, value in self._properties.items():
this_property = {
"Source": self.name,
"Property": f"{name}: {value}"
}
if name not in self.virt_ignore:
result.append(this_property)
return result
def get_metrics_keys(self):
result = []
empty_value = ["", "null", None, []]
for name, value in self._metrics.items():
this_metric = {
"Source": self.name,
"Metric": name,
"Data": value
}
if value not in empty_value and name not in self.virt_ignore:
result.append(this_metric)
return result
def get_metrics_strings(self):
result = []
empty_value = ["", "null", None, []]
for name, value in self._metrics.items():
this_metric = {
"Source": self.name,
"Metric": f"{name}:{value}"
}
if value not in empty_value and name not in self.virt_ignore:
result.append(this_metric)
return result
########################################################
# random data functions
########################################################
# complex data type return
def get_metrics(self, type = None):
these_metrics = []
if type == None:
for name, value in self._metrics:
these_metrics.append({"name": name, "value": value})
these_metrics.append({"Metric": name, "Data": value})
else:
for name, value in self._metrics:
if name == type:
these_metrics.append({"name": name, "value": value})
these_metrics.append({"Metric": name, "Data": value})
result = {
"name": self.name,
"type": self.type,
"metrics": these_metrics
"Source": self.name,
"Component Type": self.type,
"Metrics": these_metrics
}
return result
@ -96,89 +171,69 @@ class Component:
these_properties = []
if type == None:
for name, value in self._properties.items():
these_properties.append({"name": name, "value": value})
these_properties.append({"Property": name, "Value": value})
else:
for name, value in self._properties.items():
if name == type:
these_properties.append({"name": name, "value": value})
these_properties.append({"Property": name, "Value": value})
result = {
"name": self.name,
"type": self.type,
"properties": these_properties
"Source": self.name,
"Component Type": self.type,
"Properties": these_properties
}
return result
# this gets the value of a specified property, type required
def get_property(self, type):
return self._properties[type]
# returns array of dicts for redis
def get_metrics_keys(self):
result = []
empty_value = ["", "null", None, []]
for name, value in self._metrics.items():
this_metric = {
"name": self.name,
"type": name,
"metric": value
}
if value not in empty_value:
result.append(this_metric)
return result
def get_properties_keys(self):
result = []
for name, value in self._properties.items():
this_property = {
"name": self.name,
"property": name,
"value": value
}
result.append(this_property)
return result
# full data return
def get_description(self):
these_properties = []
for name, value in self._metrics.items():
these_properties.append({"name": name, "value": value})
for name, value in self._properties.items():
these_properties.append({"Property": name, "Value": value})
these_metrics = []
for name, value in self._metrics.items():
these_metrics.append({"name": name, "value": value})
these_metrics.append({"Metric": name, "Data": value})
result = {
"name": self.name,
"type": self.type,
"properties": these_properties,
"metrics": these_metrics
"Source": self.name,
"Type": self.type,
"Properties": these_properties,
"Metrics": these_metrics
}
return result
def is_multi(self):
for component_type in component_types:
if self.type == component_type["name"]:
return component_type["multi_check"]
return False
############################################################
# System Class
# this is a big one...
############################################################
class System:
########################################################
# system variable declarations
# keys to add: model and serial number
########################################################
static_key_variables = [
{"name": "hostname", "command": "hostname"},
{"name": "virt_string", "command": "systemd-detect-virt"}
{"name": "Hostname", "command": "hostname"},
{"name": "Virtual Machine", "command": "echo $([[ \"$(systemd-detect-virt)\" == none ]] && echo False || echo True)"},
{"name": "CPU Architecture:", "command": "lscpu --json | jq -r '.lscpu[] | select(.field==\"Architecture:\") | .data'"},
{"name": "OS Kernel", "command": "uname -r"},
{"name": "OS Name", "command": "cat /etc/os-release | grep PRETTY | cut -d\\\" -f2"},
{"name": "Manufacturer", "command": "sudo dmidecode --type 1 | grep Manufacturer: | cut -d: -f2 | sed -e 's/^[ \\t]*//'"},
{"name": "Product Name", "command": "sudo dmidecode --type 2 | grep 'Product Name:' | cut -d: -f2 | sed -e 's/^[ \\t]*//'"},
{"name": "Serial Number", "command": "sudo dmidecode --type 2 | grep 'Serial Number: '| cut -d: -f2 | sed -e 's/^[ \\t]*//'"},
]
dynamic_key_variables = [
{"name": "uptime", "command": "uptime -p"},
{"name": "timestamp", "command": "date '+%D %r'"},
{"name": "System Uptime", "command": "uptime -p"},
{"name": "Current Date", "command": "date '+%D %r'"},
]
# add components based on the class tree
# component_types = [{"name": entry["name"], "multi_check": entry["multi_check"] == "True"} for entry in component_class_tree]
virt_ignore = [
"Product Name",
"Serial Number"
]
########################################################
# instantiate new system
########################################################
def __init__(self, name: str):
# the system needs a name
self.name = name
@ -189,18 +244,29 @@ class System:
# initialize system properties and metrics dicts
self._properties: Dict[str, str] = {}
self._metrics: Dict[str, str] = {}
# timekeeping for websocket
self.recent_check = int(time.time())
# load static keys
for static_key in self.static_key_variables:
command = static_key["command"]
result = run_command(command, True)
if debug_output:
print(f"Static key [{static_key["name"]}] - command [{command}] - output [{result}]")
self._properties[static_key["name"]] = result
if static_key["name"] not in self.virt_ignore:
command = static_key["command"]
result = run_command(command, True)
if debug_output:
print(f'Static key [{static_key["name"]}] - command [{command}] - output [{result}]')
self._properties[static_key["name"]] = result
# initialize live keys
self.update_live_keys()
# initialze components
self.load_components()
def __str__(self):
components_str = "\n".join(f" - {c}" for c in self.components)
return f"System hostname: {self.name}\nComponent Count: {self.get_component_count()}\n{components_str}"
########################################################
# critical class functions
########################################################
# update only system dynamic keys
def update_live_keys(self):
for live_key in self.dynamic_key_variables:
@ -209,7 +275,7 @@ class System:
result = run_command(command, True)
self._metrics[live_key['name']] = result
if debug_output:
print(f"Command {live_key["name"]} - [{command}] Result - [{result}]")
print(f'Command {live_key["name"]} - [{command}] Result - [{result}]')
# update all dynamic keys, including components
def update_system_state(self):
@ -236,7 +302,7 @@ class System:
else:
if debug_output:
print(f"Creating component {component["name"]}")
print(f'Creating component {component["name"]}')
self.add_components(Component(component_name, component_name))
# Add a component to the system
@ -244,6 +310,10 @@ class System:
if debug_output:
print(f"Component description: {component.description}")
self.components.append(component)
########################################################
# helper class functions
########################################################
# Get all components, optionally filtered by type
def get_components(self, component_type: type = None):
@ -259,31 +329,123 @@ class System:
else:
return result[0]
def get_component_strings(self, component_type: type = None):
if component_type is None:
result = []
for component in self.components:
result.append(component.description)
return result
else:
result = []
for component in self.components:
if component.type == component_type:
result.append(component.description)
if component.is_multi():
return result
else:
return result[0]
# get component count
def get_component_count(self):
result = int(len(self.components))
return result
def __str__(self):
components_str = "\n".join(f" - {c}" for c in self.components)
return f"System hostname: {self.name}\nComponent Count: {self.get_component_count()}\n{components_str}"
def is_virtual(self):
virt_check = self._properties.get('virt_ignore', {}).items()
def check_system_timer(self):
time_lapsed = time.time() - float(self.recent_check)
return time_lapsed < 30.0
########################################################
# static metrics redis data functions
########################################################
# return list of all static metrics from system and properties
def get_static_metrics(self, human_readable = False):
result = []
for component_property in self.get_component_properties(human_readable):
result.append(component_property)
for system_property in self.get_system_properties(human_readable):
result.append(system_property)
return result
def get_component_properties(self, human_readable = False):
result = []
for component in self.components:
if human_readable:
for metric in component.get_properties_strings():
result.append(metric)
else:
for metric in component.get_properties_keys():
result.append(metric)
return result
def get_system_properties(self, human_readable = False):
result = []
for name, value in self._properties.items():
if human_readable:
result.append({
"Source": "System",
"Property": f"{name}: {value}"
})
else:
result.append({
"Source": "System",
"Property": name,
"Value": value
})
return result
########################################################
# live metrics redis data functions
########################################################
# return list of all live metrics from system and properties
def get_live_metrics(self, human_readable = False):
result = []
for component_metric in self.get_component_metrics(human_readable):
result.append(component_metric)
for system_metric in self.get_system_metrics(human_readable):
result.append(system_metric)
return result
def get_component_metrics(self, human_readable = False):
result = []
for component in self.components:
if human_readable:
metrics_keys = component.get_metrics_strings()
else:
metrics_keys = component.get_metrics_keys()
for metric in metrics_keys:
result.append(metric)
return result
def get_system_metrics(self, human_readable = False):
if human_readable:
return self.get_system_metric_strings()
else:
return self.get_system_metric_keys()
def get_system_metric_keys(self):
result = []
for name, value in self._metrics.items():
thisvar = {
"Source": "System",
"Metric": name,
"Data": value
}
result.append(thisvar)
# add internal dynamic metrics
result.append({
"Source": "System",
"Metric": "component_count",
"Data": self.get_component_count()
})
return result
def get_system_metric_strings(self):
result = []
for name, value in self._metrics.items():
thisvar = {
"Source": "System",
"Metric": f"{name}: {value}"
}
result.append(thisvar)
# add internal dynamic metrics
result.append({
"Source": "System",
"Metric": f"component_count: {self.get_component_count()}"
})
return result
# straggler functions, might cut them
# return both static and dynamic data
def get_sysvars_summary_keys(self):
result = []
@ -303,70 +465,24 @@ class System:
result.append(thisvar)
return result
# return list of all live metrics from system and properties
def get_live_metrics(self):
result = []
for component_metric in self.get_component_metrics():
result.append(component_metric)
for system_metric in self.get_system_metrics():
result.append(system_metric)
return result
# return array of all component metrics
def get_component_metrics(self):
result = []
for component in self.components:
for metric in component.get_metrics_keys():
result.append(metric)
return result
# return array of all component metrics
def get_component_properties(self):
result = []
for component in self.components:
for metric in component.get_properties_keys():
result.append(metric)
return result
# return array of all system metrics
def get_system_metrics(self):
result = []
for name, value in self._metrics.items():
thisvar = {
"name": "System",
"type": name,
"metric": value
}
result.append(thisvar)
# add component count
result.append({
"name": "System",
"type": "component_count",
"metric": self.get_component_count()
})
return result
def get_system_properties(self):
result = []
for name, value in self._properties.items():
if name == "virt_string":
thisvar = {
"name": "System",
"property": name,
"value": value == "none"
}
def get_component_strings(self, component_type: type = None):
if component_type is None:
result = []
for component in self.components:
result.append(component.description)
return result
else:
result = []
for component in self.components:
if component.type == component_type:
result.append(component.description)
if component.is_multi():
return result
else:
thisvar = {
"name": "System",
"property": name,
"value": value
}
result.append(thisvar)
return result
return result[0]
############################################################
# Helper Functions
# Non-class Helper Functions
############################################################
# subroutine to run a command, return stdout as array unless zero_only then return [0]

67
files/api/app.py
View File

@ -76,7 +76,8 @@ def update_redis_channel(redis_channel, data):
def update_redis_server():
# Update Stats Redis Channel
update_redis_channel("host_metrics", get_redis_data())
if cosmostat_system.check_system_timer():
update_redis_channel("host_metrics", get_redis_data(human_readable = False))
# Update history_stats Redis Channel
# update_redis_channel("history_stats", get_component_list())
@ -98,7 +99,12 @@ def static_data():
# redis data
@app.route('/redis_data', methods=['GET'])
def redis_data():
return jsonify(get_redis_data())
return jsonify(get_redis_data(human_readable = False))
# redis strings
@app.route('/redis_strings', methods=['GET'])
def redis_strings():
return jsonify(get_redis_data(human_readable = True))
# full summary
@app.route('/full_summary', methods=['GET'])
@ -110,15 +116,43 @@ def full_summary():
def info():
return jsonify(get_info())
# socket timer
@app.route('/start_timer', methods=['GET'])
def start_timer():
current_timestamp = int(time.time())
cosmostat_system.recent_check = current_timestamp
if app_settings["noisy_test"]:
print(f"Timestamp updated to {cosmostat_system.recent_check}")
return jsonify(
{
"message": "websocket timer reset",
"new_timestamp": cosmostat_system.recent_check
}
)
# socket timer data
@app.route('/timer_data', methods=['GET'])
def timer_data():
time_now = time.time()
time_lapsed = time_now - float(cosmostat_system.recent_check)
result = {
"Time Lapsed": time_lapsed,
"Current Time Value": time_now,
"Last Update Value": float(cosmostat_system.recent_check),
"System Updating": cosmostat_system.check_system_timer()
}
return jsonify(result)
# test route
@app.route('/test', methods=['GET'])
def test():
this_cpu = cosmostat_system.get_components(component_type="CPU")
return jsonify(
{
"component_count:": len(cosmostat_system.components),
"user": jenkins_user_settings(),
"hostname": jenkins_hostname_settings(),
"cpu_model": cosmostat_system.get_components(component_type="CPU").description
"cpu_model": this_cpu[0].description
}
)
@ -128,22 +162,18 @@ def test():
# needs to return array of {name: name, type: type, metrics: metrics}
# for redis table generation, includes system and component metrics
def get_dynamic_data():
return cosmostat_system.get_live_metrics()
def get_dynamic_data(human_readable = False):
return cosmostat_system.get_live_metrics(human_readable)
def get_static_data():
def get_static_data(human_readable = False):
result = []
for metric in cosmostat_system.get_system_properties():
result.append(metric)
for metric in cosmostat_system.get_component_properties():
result.append(metric)
return result
return cosmostat_system.get_static_metrics(human_readable)
def get_redis_data():
def get_redis_data(human_readable = False):
result = []
for metric in get_dynamic_data():
for metric in get_dynamic_data(human_readable):
result.append(metric)
for metric in get_static_data():
for metric in get_static_data(human_readable):
result.append(metric)
return result
@ -171,11 +201,11 @@ def get_info():
component_strings = []
for component in cosmostat_system.get_components():
component_strings.append({"name": component.name, "description": component.description})
system_strings = []
result = {
"hostname": jenkins_hostname_settings(),
"component_strings": component_strings
"component_strings": component_strings,
"system_strings": cosmostat_system.get_sysvars_summary_keys()
}
#for component_string in component_strings:
# for name, description in component_string.items():
@ -199,7 +229,8 @@ def new_cosmos_system():
# Background Loop Function
def background_loop():
# Update all data on the System object
cosmostat_system.update_system_state()
if cosmostat_system.check_system_timer():
cosmostat_system.update_system_state()
if app_settings["push_redis"]:
update_redis_server()
@ -241,7 +272,7 @@ if __name__ == '__main__':
print("Skipping flask background task")
# Flask API
app.run(debug=True, host=service_gateway_ip(), port=5000)
app.run(debug=False, host=service_gateway_ip(), port=5000)

48
files/api/component_descriptors.json
View File

@ -1,46 +1,56 @@
[
{
"name": "CPU",
"description": "{model_name} with {core_count} cores.",
"description": "{CPU Model} with {Core Count} cores.",
"multi_check": "False",
"properties": {
"core_count": "lscpu --json | jq -r '.lscpu[] | select(.field==\"CPU(s):\") | .data'",
"model_name": "lscpu --json | jq -r '.lscpu[] | select(.field==\"Model name:\") | .data'"
"Core Count": "lscpu --json | jq -r '.lscpu[] | select(.field==\"CPU(s):\") | .data'",
"CPU Model": "lscpu --json | jq -r '.lscpu[] | select(.field==\"Model name:\") | .data'",
"Clock Speed": "sudo dmesg | grep MHz | grep tsc | cut -d: -f2 | awk '{print $2 \" \" $3}'"
},
"metrics": {
"1m_load": "cat /proc/loadavg | awk '{print $1}'",
"5m_load": "cat /proc/loadavg | awk '{print $2}'",
"15m_load": "cat /proc/loadavg | awk '{print $3}'"
"15m_load": "cat /proc/loadavg | awk '{print $3}'",
"current_mhz": "less /proc/cpuinfo | grep MHz | cut -d: -f2 | awk '{sum += $1} END {print sum/NR}'"
}
},
{
"name": "RAM",
"description": "Total {bytes_total}GB in {module_count} modules.",
"description": "Total {Total GB}GB in {RAM Module Count} modules.",
"multi_check": "False",
"properties": {
"bytes_total": "sudo lshw -json -c memory | jq -r '.[] | select(.description==\"System Memory\").size' | awk '{printf \"%.2f\\n\", $1/1073741824}'",
"module_count": "sudo lshw -json -c memory | jq -r '.[] | select(.id | contains(\"bank\")) | .id ' | wc -l"
"Total GB": "sudo /usr/bin/lshw -json -c memory | jq -r '.[] | select(.description==\"System Memory\").size' | awk '{printf \"%.2f\\n\", $1/1073741824}'",
"RAM Module Count": "sudo /usr/bin/lshw -json -c memory | jq -r '.[] | select(.id | contains(\"bank\")) | .id ' | wc -l",
"RAM Type": "/usr/sbin/dmidecode --type 17 | grep Type: | sort -u | cut -d: -f2 | xargs",
"RAM Speed": "/usr/sbin/dmidecode --type 17 | grep Speed: | grep -v Configured | sort -u | cut -d: -f2 | xargs",
"RAM Voltage": "/usr/sbin/dmidecode --type 17 | grep 'Configured Voltage' | sort -u | cut -d: -f2 | xargs"
},
"metrics": {
"used_capacity_mb": "free -m | grep Mem | awk '{print $3}'",
"free_capacity_mb": "free -m | grep Mem | awk '{print $4}'"
}
"MB Used": "free -m | grep Mem | awk '{print $3}'",
"MB Free": "free -m | grep Mem | awk '{print $4}'"
},
"virt_ignore": [
"RAM Type",
"RAM Speed",
"RAM Voltage"
]
},
{
"name": "Block Storage",
"description": "{device_id} is of type {drive_type} with capacity of {drive_capacity}.",
"name": "STOR",
"description": "{Device Path} is of type {Drive Type} with capacity of {Total Capacity}.",
"multi_check": "True",
"device_list": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME | awk '{print $1}'",
"properties": {
"device_name": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME | awk '{{print $1}}' | grep {this_device}",
"device_id": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME | awk '{{print \"/dev/\"$1}}' | grep {this_device}",
"drive_type": "lsblk -d -o NAME,TRAN | grep {this_device} | awk '{{print $2}}'",
"drive_capacity": "lsblk -d -o NAME,SIZE | grep {this_device} | awk '{{print $2}}'"
"Device Name": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME | awk '{{print $1}}' | grep {this_device}",
"Device Path": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME | awk '{{print \"/dev/\"$1}}' | grep {this_device}",
"Drive Type": "lsblk -d -o NAME,TRAN | grep {this_device} | awk '{{print $2}}'",
"Total Capacity": "lsblk -d -o NAME,SIZE | grep {this_device} | awk '{{print $2}}'"
},
"metrics": {
"smart_status": "sudo smartctl -x --json /dev/{this_device} | jq -r .smart_status.passed",
"ssd_endurance_string": "sudo smartctl -x --json /dev/{this_device} | jq -r '.physical_block_size as $block |.ata_device_statistics.pages[] | select(.name == \"General Statistics\") | .table[] | select(.name == \"Logical Sectors Written\") | .value as $sectors | ($sectors * $block) / 1073741824 ' | awk '{{printf \"%.2f GiB Written\\n\", $0}}' || true",
"nvme_endurance_string": "sudo smartctl -x --json /dev/{this_device} | jq -r ' .nvme_smart_health_information_log.data_units_written as $dw | .logical_block_size as $ls | ($dw * $ls) / 1073741824 ' | awk '{{printf \"%.2f GiB Written\\n\", $0}}' || true"
"SMART Check": "/usr/sbin/smartctl -x --json /dev/{this_device} | jq -r .smart_status.passed",
"SATA GBW": "/usr/sbin/smartctl -x --json /dev/{this_device} | jq -r '.physical_block_size as $block |.ata_device_statistics.pages[] | select(.name == \"General Statistics\") | .table[] | select(.name == \"Logical Sectors Written\") | .value as $sectors | ($sectors * $block) / 1073741824 ' | awk '{{printf \"%.2f GiB Written\\n\", $0}}' || true",
"NVMe GBW": "/usr/sbin/smartctl -x --json /dev/{this_device} | jq -r ' .nvme_smart_health_information_log.data_units_written as $dw | .logical_block_size as $ls | ($dw * $ls) / 1073741824 ' | awk '{{printf \"%.2f GiB Written\\n\", $0}}' || true"
}
}
]

4
files/web/html/index.php
View File

@ -8,11 +8,11 @@
<link rel="stylesheet" href="src/styles.css">
</head>
<body>
<div class="container">
<div class="card">
<h2>Matt-Cloud Cosmostat Dashboard</h2>
This dashboard shows the local Matt-Cloud system stats.<p>
</div>
<div class="container">
<div class="card">
<h2>Live System Metrics</h2>
<div id="host_metrics" class="column">Connecting…</div>
</div>

72
files/web/html/src/redis.js
View File

@ -21,6 +21,7 @@ function safeSetText(id, txt) {
------------------------------------------------------------------ */
// helper function for table row ordering
function renderStatsTable(data) {
socket.emit('tableRendered');
renderGenericTable('host_metrics', data, 'No Stats available');
}
@ -48,59 +49,56 @@ function renderGenericTable(containerId, data, emptyMsg) {
3. Merge rows by name
------------------------------------------------------------ */
function mergeRowsByName(data) {
const groups = {}; // { name: { types: [], metrics: [], props: [], values: [] } }
const groups = {}; // { source: { ... } }
data.forEach(row => {
const name = row.name;
if (!name) return; // ignore rows without a name
if (!groups[name]) {
groups[name] = { types: [], metrics: [], props: [], values: [] };
const source = row.Source; // <-- changed
if (!source) return;
if (!groups[source]) {
groups[source] = { Metric: [], Data: [], Property: [], Value: [] };
}
// Metric rows - contain type + metric
if ('type' in row && 'metric' in row) {
groups[name].types.push(row.type);
groups[name].metrics.push(row.metric);
}
// Property rows - contain property + value
else if ('property' in row && 'value' in row) {
groups[name].props.push(row.property);
groups[name].values.push(row.value);
if ('Metric' in row && 'Data' in row) {
groups[source].Metric.push(row.Metric);
groups[source].Data.push(row.Data);
} else if ('Property' in row && 'Value' in row) {
groups[source].Property.push(row.Property);
groups[source].Value.push(row.Value);
}
});
// Convert each group into a single row object
const merged = [];
Object.entries(groups).forEach(([name, grp]) => {
Object.entries(groups).forEach(([source, grp]) => {
merged.push({
name,
type: grp.types, // array of types
metric: grp.metrics, // array of metrics
property: grp.props, // array of property names
value: grp.values, // array of property values
Source: source, // <-- keep the original key
Metric: grp.Metric,
Data: grp.Data,
Property: grp.Property,
Value: grp.Value
});
});
return merged;
}
/* ------------------------------------------------------------------
3b. Order rows - put “System”, “CPU”, “RAM” first
------------------------------------------------------------------ */
// 3b. Order rows – put “System”, “CPU”, “RAM” first
function orderRows(rows) {
// this should be updatable if i want
// Priority list – can be updated later
const priority = ['System', 'CPU', 'RAM'];
const priorityMap = {};
priority.forEach((name, idx) => (priorityMap[name] = idx));
// Map source → priority index
const priorityMap = {};
priority.forEach((src, idx) => {
priorityMap[src] = idx;
});
// Stable sort: keep original position if priorities are equal
return [...rows].sort((a, b) => {
const aIdx = priorityMap.hasOwnProperty(a.name)
? priorityMap[a.name]
: Infinity; // anything not in priority goes to the end
const bIdx = priorityMap.hasOwnProperty(b.name)
? priorityMap[b.name]
const aIdx = priorityMap.hasOwnProperty(a.Source)
? priorityMap[a.Source]
: Infinity; // anything not in priority goes to the end
const bIdx = priorityMap.hasOwnProperty(b.Source)
? priorityMap[b.Source]
: Infinity;
// If both have the same priority (or both Infinity), keep original order
return aIdx - bIdx;
});
}
@ -109,7 +107,7 @@ function orderRows(rows) {
4. Build an HTML table from an array of objects
------------------------------------------------------------ */
function buildTable(data) {
const cols = ['name', 'type', 'metric', 'property', 'value']; // explicit order
const cols = ['Source', 'Property', 'Value', 'Metric', 'Data']; // explicit order
const table = document.createElement('table');
// Header

76
files/web/html/src/styles.css
View File

@ -8,33 +8,15 @@ body {
color: #bdc3c7; /* Dimmer text color */
}
.hidden-info {
display: none;
}
.title-button {
background-color: #34495e;
border: none;
color: white;
padding: 15px 32px;
text-align: center;
text-decoration: none;
display: inline-block;
font-size: 16px;
margin: 4px 2px;
cursor: pointer;
}
table, th, td {
border: 1px solid black;
border: 2px solid #182939;
border-collapse: collapse;
}
th, td {
padding: 10px;
}
.container {
.card {
max-width: 950px;
margin: 0 auto;
padding: 20px;
@ -43,15 +25,6 @@ th, td {
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.3); /* Slightly darker shadow */
margin-top: 20px;
}
.container-small {
max-width: 550px;
margin: 0 auto;
padding: 20px;
background-color: #34495e; /* Darker background for container */
border-radius: 8px;
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.3); /* Slightly darker shadow */
margin-top: 20px;
}
h1, h2, h3, h4 {
color: #bdc3c7; /* Dimmer text color */
@ -67,55 +40,12 @@ li {
color: #bdc3c7; /* Dimmer text color */
}
.group-columns {
display: flex;
}
.group-rows {
display: flex;
flex-wrap: wrap;
justify-content: flex-start; /* Left justification */
margin-top: 10px;
}
.group-column {
flex: 0 0 calc(33% - 10px); /* Adjust width of each column */
}
.column {
flex: 1;
padding: 0 10px; /* Adjust spacing between columns */
}
.subcolumn {
margin-left: 10px;
}
.grid {
display: flex;
flex-wrap: wrap;
justify-content: space-between;
margin-top: 5px;
}
.meter {
width: calc(90% - 5px);
max-width: calc(45% - 5px);
margin-bottom: 5px;
border: 1px solid #7f8c8d; /* Light border color */
border-radius: 5px;
padding: 5px;
text-align: center;
background-color: #2c3e50; /* Dark background for meter */
}
#host_metrics_column td {
list-style: none; /* removes the numeric markers */
padding-left: 0; /* remove the default left indent */
margin-left: 0; /* remove the default left margin */
}
#host_metrics_table tbody tr td :nth-of-type(even) {
background-color: #2c3e50;
background-color: #3e5c78;
}

3
files/web/node_server/package.json
View File

@ -8,6 +8,7 @@
"dependencies": {
"express": "^4.18.2",
"socket.io": "^4.7.2",
"redis": "^4.6.7"
"redis": "^4.6.7",
"node-fetch": "^2.6.7"
}
}

60
files/web/node_server/server.js
View File

@ -1,32 +1,63 @@
// server.js
const http = require('http');
const http = require('http');
const express = require('express');
const { createClient } = require('redis');
const { Server } = require('socket.io');
const fetch = require('node-fetch'); // npm i node-fetch@2
const app = express();
const app = express();
const server = http.createServer(app);
const io = new Server(server);
const io = new Server(server);
// Serve static files (index.html)
// ---------- Socket.io ----------
io.on('connection', async socket => {
console.log('client connected:', socket.id);
// Call the external API every time a client connects
try {
const resp = await fetch('http://192.168.37.1:5000/start_timer', {
method: 'GET'
});
const data = await resp.json();
console.log('API responded to connect:', data);
} catch (err) {
console.error('Failed to hit start_timer endpoint:', err);
}
// Listen for tableRendered event from the client
socket.on('tableRendered', async () => {
console.log('Client reported table rendered - starting timer');
try {
const resp = await fetch('http://192.168.37.1:5000/start_timer', {
method: 'GET'
});
const text = await resp.text();
console.log('Timer endpoint responded:', text);
} catch (err) {
console.error('Failed to hit start_timer:', err);
}
});
});
// Serve static files (index.html, etc.)
app.use(express.static('public'));
// ---------- Redis subscriber ----------
const redisClient = createClient({
url: 'redis://192.168.37.1:6379'
});
const redisClient = createClient({ url: 'redis://192.168.37.1:6379' });
redisClient.on('error', err => console.error('Redis error', err));
(async () => {
await redisClient.connect();
const sub = redisClient.duplicate(); // duplicate to keep separate pub/sub
await sub.connect();
// Subscribe to the channel that sends host stats
// Subscribe to the channel that sends host stats
await sub.subscribe(
['host_metrics'],
(message, channel) => { // <-- single handler
(message, channel) => {
let payload;
try {
payload = JSON.parse(message); // message is a JSON string
@ -34,20 +65,13 @@ redisClient.on('error', err => console.error('Redis error', err));
console.error(`Failed to parse ${channel}`, e);
return;
}
io.emit(channel, payload);
}
);
sub.on('error', err => console.error('Subscriber error', err));
sub.on('error', err => console.error('Subscriber error', err));
})();
// ---------- Socket.io ----------
io.on('connection', socket => {
console.log('client connected:', socket.id);
// Optional: send the current state on connect if you keep it cached
});
// ---------- Start ----------
const PORT = process.env.PORT || 3000;
server.listen(PORT, () => {

0
files/proxy/nginx.conf → files/web/proxy/nginx.conf
View File

20
tasks/init.yaml
View File

@ -1,5 +1,5 @@
---
# package handler
- name: Cosmostat - Init - Get installed package list
when: dpkg_output is undefined
shell: "dpkg --list | grep ii | awk '{print $2}'"
@ -46,7 +46,7 @@
mode: '0755'
# create user service folder
- name: Cosmostat - Init - create cosmostat service folder
- name: Cosmostat - Init - create cosmostat user service folder
file:
path: "{{ user_service_folder }}"
state: directory
@ -85,6 +85,7 @@
# create node.js docker container for web dashboard
- name: node.js server container handler
when: false
block:
- name: Cosmostat - Init - node.js - copy server files
@ -96,13 +97,16 @@
group: "{{ service_user }}"
- name: Cosmostat - Init - node.js - build docker container
community.docker.docker_image_build:
community.docker.docker_image:
name: ws_node
tag: latest
rebuild: always
path: "{{ service_control_web_folder }}/node_server"
dockerfile: Dockerfile
labels:
ws_node: "true"
source: local
build:
path: "{{ service_control_web_folder }}/node_server"
dockerfile: Dockerfile
force_tag: true
state: present
force_source: true
...

14
tasks/main.yaml
View File

@ -1,17 +1,21 @@
---
# initializa environment
# set up API
# set up web stack
- name: Initialize Environment
when: not quick_refresh | bool
include_tasks: init.yaml
# set up API
- name: Build API
include_tasks: api.yaml
# set up web stack
- name: Build Web Dashboard
include_tasks: web.yaml
#- name: Purge Old Containers
# when: not quick_refresh | bool
# include_tasks: purge.yaml
...

34
tasks/purge.yaml Normal file
View File

@ -0,0 +1,34 @@
- name: Cosmostat - Clean up old ws_node image tags
block:
# Grab a list of all tags the image has
- name: Get all ws_node image tags
command: |
docker images --format "{{.Repository}}:{{.Tag}}" \
--filter=reference="ws_node:*"
register: all_tags_raw
changed_when: false
# Turn that raw string into a list of just the tag names
- name: Parse tag names out of the list
set_fact:
all_tags: >-
{{ all_tags_raw.stdout_lines |
map('regex_replace', '^ws_node:', '') |
list }}
# Keep everything *except* the one that ends with “:latest”
- name: Build list of tags that should be removed
set_fact:
tags_to_remove: "{{ all_tags | difference(['latest']) }}"
# Remove each old tag
- name: Delete old ws_node image tags
community.docker.docker_image:
name: ws_node
tag: "{{ item }}"
state: absent
loop: "{{ tags_to_remove }}"
when: tags_to_remove | length > 0
when: tags_to_remove | length > 0
tags:
- cleanup

54
tasks/web.yaml
View File

@ -12,53 +12,25 @@
owner: "{{ service_user }}"
group: "{{ service_user }}"
- name: Cosmostat - Init - copy dashboard web files
- name: Cosmostat - Web - copy docker files
copy:
src: "web/html"
dest: "{{ service_control_web_folder }}/"
src: "web/"
dest: "{{ service_control_web_folder }}"
mode: 0755
owner: "{{ service_user }}"
group: "{{ service_user }}"
# These are not needed unless there is a stack
#- name: Cosmostat - Web - copy files for history dashboard
# copy:
# src: "dashboard/"
# dest: "{{ service_control_web_folder }}/html"
# mode: 0755
# owner: "{{ service_user }}"
# group: "{{ service_user }}"
- name: Cosmostat - Web - template docker-compose.yaml
template:
src: docker-compose-php.yaml
dest: "{{ service_control_web_folder }}/docker-compose.yaml"
mode: 0644
- name: Cosmostat - Web - copy files for proxy container
copy:
src: "proxy/"
dest: "{{ service_control_web_folder }}/proxy"
mode: 0755
owner: "{{ service_user }}"
group: "{{ service_user }}"
- name: docker container handler
block:
- name: Cosmostat - Web - template docker-compose.yaml
template:
src: docker-compose-php.yaml
dest: "{{ service_control_web_folder }}/docker-compose.yaml"
mode: 0644
- name: Cosmostat - Web - Start containers
shell: "docker-compose -f {{ service_control_web_folder }}/docker-compose.yaml up -d"
register: docker_output
- debug: |
msg="{{ docker_output.stdout_lines }}"
- name: Cosmostat - Web - Start containers
shell: "docker-compose -f {{ service_control_web_folder }}/docker-compose.yaml up -d"
register: docker_output
- debug: |
msg="{{ docker_output.stdout_lines }}"
msg="{{ docker_output.stderr_lines }}"
- name: Cosmostat - Web - Prune old containers
community.docker.docker_prune:
containers: true
containers_filters:
label:
ws_node: "true"
...

22
templates/docker-compose-php.yaml
View File

@ -8,37 +8,41 @@ services:
networks:
- cosmostat_net
restart: always
cosmostat_ws_node:
image: node:18-alpine
working_dir: /app
command: sh -c "npm install && node server.js"
container_name: cosmostat_ws_node
build:
context: "{{ service_control_web_folder }}/node_server"
dockerfile: Dockerfile
image: ws_node:latest
volumes:
- "{{ service_control_web_folder }}/html:/usr/src/app/public"
- "{{ service_control_web_folder }}/node_server:/app"
- /app/node_modules
ports:
- "{{ (docker_gateway + ':') if secure_api else '' }}3000:3000"
- "{{ docker_gateway }}:3000:3000"
networks:
- cosmostat_net
restart: always
depends_on:
- cosmostat_redis
- cosmostat_redis
cosmostat_web_dash:
container_name: cosmostat_web_dash
image: php:8.0-apache
ports:
- "{{ (docker_gateway + ':') if secure_api else '' }}8080:80"
- "{{ docker_gateway }}:8080:80"
volumes:
- ./html:/var/www/html/
networks:
- cosmostat_net
restart: always
# public_dashboard: {{ public_dashboard }}
cosmostat_nginx_proxy:
container_name: cosmostat_nginx_proxy
image: nginx:latest
ports:
- "{{ (docker_gateway + ':') if secure_api else '' }}80:80"
- "{{ (docker_gateway + ':') if not public_dashboard | bool else '' }}80:80"
volumes:
- ./proxy/nginx.conf:/etc/nginx/conf.d/default.conf
networks:

10
templates/docker-compose.yaml
View File

@ -14,13 +14,15 @@ services:
restart: always
ws_node:
image: node:18-alpine
working_dir: /app
command: sh -c "npm install && node server.js"
container_name: ws_node
build:
context: {{ service_control_web_folder }}/node_server
dockerfile: Dockerfile
image: ws_node:latest
volumes:
- {{ service_control_web_folder }}/html:/usr/src/app/public
- {{ service_control_web_folder }}/node_server:/app
- /app/node_modules
ports:
# put back to 3000 if the stack is needed
- {{ (docker_gateway + ':') if secure_api else '' }}80:3000