From 61421305eda8021c15677345c65c203378730350 Mon Sep 17 00:00:00 2001 From: Matt Date: Mon, 16 Mar 2026 09:17:44 -0700 Subject: [PATCH] add string returns for metrics and properties --- defaults/main.yaml | 8 +- files/api/Components.py | 422 +++++++++++++++++---------- files/api/app.py | 67 +++-- files/api/component_descriptors.json | 48 +-- files/web/html/index.php | 4 +- files/web/html/src/redis.js | 72 +++-- files/web/html/src/styles.css | 76 +---- files/web/node_server/package.json | 3 +- files/web/node_server/server.js | 60 ++-- files/{ => web}/proxy/nginx.conf | 0 tasks/init.yaml | 20 +- tasks/main.yaml | 14 +- tasks/purge.yaml | 34 +++ tasks/web.yaml | 54 +--- templates/docker-compose-php.yaml | 22 +- templates/docker-compose.yaml | 10 +- 16 files changed, 524 insertions(+), 390 deletions(-) rename files/{ => web}/proxy/nginx.conf (100%) create mode 100644 tasks/purge.yaml diff --git a/defaults/main.yaml b/defaults/main.yaml index e5afa37..a1bbc74 100644 --- a/defaults/main.yaml +++ b/defaults/main.yaml @@ -2,6 +2,7 @@ # required system packages cosmostat_packages: + - docker - docker.io - docker-compose - python3 @@ -27,6 +28,8 @@ cosmostat_venv_packages: | cosmostat_sudoers_content: | cosmos ALL=(root) NOPASSWD: /usr/bin/lshw cosmos ALL=(root) NOPASSWD: /usr/sbin/smartctl + cosmos ALL=(root) NOPASSWD: /usr/bin/dmesg + cosmos ALL=(root) NOPASSWD: /usr/sbin/dmidecode # subnet for service docker_subnet: "192.168.37.0/24" @@ -47,16 +50,17 @@ api_service_exe: "{{ venv_folder }}/bin/python -u {{ api_service_folder }}/app.p # dashboard vars service_control_web_folder: "{{ service_folder }}/web" +public_dashboard: true # will skip init when true quick_refresh: false # cosmostat_settings noisy_test: false -debug_output: false +debug_output: true secure_api: true push_redis: true run_background : true log_output: true -update_frequency: "2" +update_frequency: "1" ... \ No newline at end of file diff --git a/files/api/Components.py b/files/api/Components.py index 65d223b..a32c0c3 100644 --- a/files/api/Components.py +++ b/files/api/Components.py @@ -1,6 +1,7 @@ # this class file is for the cosmostat service import subprocess import json +import time from typing import Dict, Any, List # Global Class Vars @@ -8,8 +9,6 @@ global_max_length = 500 debug_output = False # import the component descriptor -# this outlines how the component class works -# each type of component has a "type" try: with open("component_descriptors.json", encoding="utf-8") as f: component_class_tree: List[Dict] = json.load(f) @@ -18,18 +17,25 @@ except FileNotFoundError as exc: component_types = [{"name": entry["name"], "multi_check": entry["multi_check"] == "True"} for entry in component_class_tree] +################################################################# +# Component Class +################################################################# + class Component: + ############################################################ + # instantiate new component + ############################################################ + def __init__(self, name: str, comp_type: str, this_device="None"): self.name = name self.type = comp_type self.this_device = this_device print(f"This device - {self.this_device}") + # build the component descriptor dictionary for component in component_class_tree: if component["name"] == self.type: COMPONENT_DESCRIPTORS = component - # Load component type descriptor from class tree - # COMPONENT_DESCRIPTORS = {d['type']: d for d in component_class_tree} descriptor = COMPONENT_DESCRIPTORS self._descriptor = descriptor if descriptor is None: @@ -39,10 +45,10 @@ class Component: ) # store static properties self.multi_check = self.is_multi() + self.virt_ignore = self._descriptor.get('virt_ignore', []) self._properties: Dict[str, str] = {} for key, command in descriptor.get('properties', {}).items(): if self.this_device != "None": - print(f"command - {command}; this_device - {self.this_device}") formatted_command = command.format(this_device=self.this_device) self._properties[key] = run_command(formatted_command, True) else: @@ -64,6 +70,10 @@ class Component: f"{self.description}") return self_string + ############################################################ + # Class Functions + ############################################################ + def update_metrics(self): for key, command in self._descriptor.get('metrics', {}).items(): if self.this_device != "None": @@ -74,20 +84,85 @@ class Component: else: self._metrics[key] = run_command(command, True) + def get_property(self, type): + return self._properties[type] + + def is_multi(self): + for component_type in component_types: + if self.type == component_type["name"]: + return component_type["multi_check"] + return False + + ######################################################## + # redis data functions + ######################################################## + + def get_properties_keys(self): + result = [] + for name, value in self._properties.items(): + this_property = { + "Source": self.name, + "Property": name, + "Value": value + } + if name not in self.virt_ignore: + result.append(this_property) + return result + + def get_properties_strings(self): + result = [] + for name, value in self._properties.items(): + this_property = { + "Source": self.name, + "Property": f"{name}: {value}" + } + if name not in self.virt_ignore: + result.append(this_property) + return result + + def get_metrics_keys(self): + result = [] + empty_value = ["", "null", None, []] + for name, value in self._metrics.items(): + this_metric = { + "Source": self.name, + "Metric": name, + "Data": value + } + if value not in empty_value and name not in self.virt_ignore: + result.append(this_metric) + return result + + def get_metrics_strings(self): + result = [] + empty_value = ["", "null", None, []] + for name, value in self._metrics.items(): + this_metric = { + "Source": self.name, + "Metric": f"{name}:{value}" + } + if value not in empty_value and name not in self.virt_ignore: + result.append(this_metric) + return result + + ######################################################## + # random data functions + ######################################################## + # complex data type return def get_metrics(self, type = None): these_metrics = [] if type == None: for name, value in self._metrics: - these_metrics.append({"name": name, "value": value}) + these_metrics.append({"Metric": name, "Data": value}) else: for name, value in self._metrics: if name == type: - these_metrics.append({"name": name, "value": value}) + these_metrics.append({"Metric": name, "Data": value}) result = { - "name": self.name, - "type": self.type, - "metrics": these_metrics + "Source": self.name, + "Component Type": self.type, + "Metrics": these_metrics } return result @@ -96,89 +171,69 @@ class Component: these_properties = [] if type == None: for name, value in self._properties.items(): - these_properties.append({"name": name, "value": value}) + these_properties.append({"Property": name, "Value": value}) else: for name, value in self._properties.items(): if name == type: - these_properties.append({"name": name, "value": value}) + these_properties.append({"Property": name, "Value": value}) result = { - "name": self.name, - "type": self.type, - "properties": these_properties + "Source": self.name, + "Component Type": self.type, + "Properties": these_properties } return result - # this gets the value of a specified property, type required - def get_property(self, type): - return self._properties[type] - - # returns array of dicts for redis - def get_metrics_keys(self): - result = [] - empty_value = ["", "null", None, []] - for name, value in self._metrics.items(): - this_metric = { - "name": self.name, - "type": name, - "metric": value - } - if value not in empty_value: - result.append(this_metric) - return result - - def get_properties_keys(self): - result = [] - for name, value in self._properties.items(): - this_property = { - "name": self.name, - "property": name, - "value": value - } - result.append(this_property) - return result - # full data return def get_description(self): these_properties = [] - for name, value in self._metrics.items(): - these_properties.append({"name": name, "value": value}) + for name, value in self._properties.items(): + these_properties.append({"Property": name, "Value": value}) these_metrics = [] for name, value in self._metrics.items(): - these_metrics.append({"name": name, "value": value}) + these_metrics.append({"Metric": name, "Data": value}) result = { - "name": self.name, - "type": self.type, - "properties": these_properties, - "metrics": these_metrics + "Source": self.name, + "Type": self.type, + "Properties": these_properties, + "Metrics": these_metrics } return result - - def is_multi(self): - for component_type in component_types: - if self.type == component_type["name"]: - return component_type["multi_check"] - return False - ############################################################ # System Class +# this is a big one... ############################################################ class System: + + ######################################################## # system variable declarations - # keys to add: model and serial number + ######################################################## + static_key_variables = [ - {"name": "hostname", "command": "hostname"}, - {"name": "virt_string", "command": "systemd-detect-virt"} + {"name": "Hostname", "command": "hostname"}, + {"name": "Virtual Machine", "command": "echo $([[ \"$(systemd-detect-virt)\" == none ]] && echo False || echo True)"}, + {"name": "CPU Architecture:", "command": "lscpu --json | jq -r '.lscpu[] | select(.field==\"Architecture:\") | .data'"}, + {"name": "OS Kernel", "command": "uname -r"}, + {"name": "OS Name", "command": "cat /etc/os-release | grep PRETTY | cut -d\\\" -f2"}, + {"name": "Manufacturer", "command": "sudo dmidecode --type 1 | grep Manufacturer: | cut -d: -f2 | sed -e 's/^[ \\t]*//'"}, + {"name": "Product Name", "command": "sudo dmidecode --type 2 | grep 'Product Name:' | cut -d: -f2 | sed -e 's/^[ \\t]*//'"}, + {"name": "Serial Number", "command": "sudo dmidecode --type 2 | grep 'Serial Number: '| cut -d: -f2 | sed -e 's/^[ \\t]*//'"}, ] dynamic_key_variables = [ - {"name": "uptime", "command": "uptime -p"}, - {"name": "timestamp", "command": "date '+%D %r'"}, + {"name": "System Uptime", "command": "uptime -p"}, + {"name": "Current Date", "command": "date '+%D %r'"}, ] - # add components based on the class tree - # component_types = [{"name": entry["name"], "multi_check": entry["multi_check"] == "True"} for entry in component_class_tree] + virt_ignore = [ + "Product Name", + "Serial Number" + ] + + ######################################################## # instantiate new system + ######################################################## + def __init__(self, name: str): # the system needs a name self.name = name @@ -189,18 +244,29 @@ class System: # initialize system properties and metrics dicts self._properties: Dict[str, str] = {} self._metrics: Dict[str, str] = {} + # timekeeping for websocket + self.recent_check = int(time.time()) # load static keys for static_key in self.static_key_variables: - command = static_key["command"] - result = run_command(command, True) - if debug_output: - print(f"Static key [{static_key["name"]}] - command [{command}] - output [{result}]") - self._properties[static_key["name"]] = result + if static_key["name"] not in self.virt_ignore: + command = static_key["command"] + result = run_command(command, True) + if debug_output: + print(f'Static key [{static_key["name"]}] - command [{command}] - output [{result}]') + self._properties[static_key["name"]] = result # initialize live keys self.update_live_keys() # initialze components self.load_components() + def __str__(self): + components_str = "\n".join(f" - {c}" for c in self.components) + return f"System hostname: {self.name}\nComponent Count: {self.get_component_count()}\n{components_str}" + + ######################################################## + # critical class functions + ######################################################## + # update only system dynamic keys def update_live_keys(self): for live_key in self.dynamic_key_variables: @@ -209,7 +275,7 @@ class System: result = run_command(command, True) self._metrics[live_key['name']] = result if debug_output: - print(f"Command {live_key["name"]} - [{command}] Result - [{result}]") + print(f'Command {live_key["name"]} - [{command}] Result - [{result}]') # update all dynamic keys, including components def update_system_state(self): @@ -236,7 +302,7 @@ class System: else: if debug_output: - print(f"Creating component {component["name"]}") + print(f'Creating component {component["name"]}') self.add_components(Component(component_name, component_name)) # Add a component to the system @@ -244,6 +310,10 @@ class System: if debug_output: print(f"Component description: {component.description}") self.components.append(component) + + ######################################################## + # helper class functions + ######################################################## # Get all components, optionally filtered by type def get_components(self, component_type: type = None): @@ -259,31 +329,123 @@ class System: else: return result[0] - def get_component_strings(self, component_type: type = None): - if component_type is None: - result = [] - for component in self.components: - result.append(component.description) - return result - else: - result = [] - for component in self.components: - if component.type == component_type: - result.append(component.description) - if component.is_multi(): - return result - else: - return result[0] - - # get component count def get_component_count(self): result = int(len(self.components)) return result - def __str__(self): - components_str = "\n".join(f" - {c}" for c in self.components) - return f"System hostname: {self.name}\nComponent Count: {self.get_component_count()}\n{components_str}" + def is_virtual(self): + virt_check = self._properties.get('virt_ignore', {}).items() + + def check_system_timer(self): + time_lapsed = time.time() - float(self.recent_check) + return time_lapsed < 30.0 + + ######################################################## + # static metrics redis data functions + ######################################################## + + # return list of all static metrics from system and properties + def get_static_metrics(self, human_readable = False): + result = [] + for component_property in self.get_component_properties(human_readable): + result.append(component_property) + for system_property in self.get_system_properties(human_readable): + result.append(system_property) + return result + + def get_component_properties(self, human_readable = False): + result = [] + for component in self.components: + if human_readable: + for metric in component.get_properties_strings(): + result.append(metric) + else: + for metric in component.get_properties_keys(): + result.append(metric) + return result + def get_system_properties(self, human_readable = False): + result = [] + for name, value in self._properties.items(): + if human_readable: + result.append({ + "Source": "System", + "Property": f"{name}: {value}" + }) + else: + result.append({ + "Source": "System", + "Property": name, + "Value": value + }) + return result + + ######################################################## + # live metrics redis data functions + ######################################################## + + # return list of all live metrics from system and properties + def get_live_metrics(self, human_readable = False): + result = [] + for component_metric in self.get_component_metrics(human_readable): + result.append(component_metric) + for system_metric in self.get_system_metrics(human_readable): + result.append(system_metric) + return result + + def get_component_metrics(self, human_readable = False): + result = [] + for component in self.components: + if human_readable: + metrics_keys = component.get_metrics_strings() + else: + metrics_keys = component.get_metrics_keys() + for metric in metrics_keys: + result.append(metric) + return result + + def get_system_metrics(self, human_readable = False): + if human_readable: + return self.get_system_metric_strings() + else: + return self.get_system_metric_keys() + + def get_system_metric_keys(self): + result = [] + for name, value in self._metrics.items(): + thisvar = { + "Source": "System", + "Metric": name, + "Data": value + } + + result.append(thisvar) + # add internal dynamic metrics + result.append({ + "Source": "System", + "Metric": "component_count", + "Data": self.get_component_count() + }) + return result + + def get_system_metric_strings(self): + result = [] + for name, value in self._metrics.items(): + thisvar = { + "Source": "System", + "Metric": f"{name}: {value}" + } + + result.append(thisvar) + # add internal dynamic metrics + result.append({ + "Source": "System", + "Metric": f"component_count: {self.get_component_count()}" + }) + return result + + # straggler functions, might cut them + # return both static and dynamic data def get_sysvars_summary_keys(self): result = [] @@ -303,70 +465,24 @@ class System: result.append(thisvar) return result - # return list of all live metrics from system and properties - def get_live_metrics(self): - result = [] - for component_metric in self.get_component_metrics(): - result.append(component_metric) - for system_metric in self.get_system_metrics(): - result.append(system_metric) - return result - - # return array of all component metrics - def get_component_metrics(self): - result = [] - for component in self.components: - for metric in component.get_metrics_keys(): - result.append(metric) - return result - - # return array of all component metrics - def get_component_properties(self): - result = [] - for component in self.components: - for metric in component.get_properties_keys(): - result.append(metric) - return result - - # return array of all system metrics - def get_system_metrics(self): - result = [] - for name, value in self._metrics.items(): - thisvar = { - "name": "System", - "type": name, - "metric": value - } - result.append(thisvar) - # add component count - result.append({ - "name": "System", - "type": "component_count", - "metric": self.get_component_count() - }) - return result - - def get_system_properties(self): - result = [] - for name, value in self._properties.items(): - if name == "virt_string": - thisvar = { - "name": "System", - "property": name, - "value": value == "none" - } + def get_component_strings(self, component_type: type = None): + if component_type is None: + result = [] + for component in self.components: + result.append(component.description) + return result + else: + result = [] + for component in self.components: + if component.type == component_type: + result.append(component.description) + if component.is_multi(): + return result else: - thisvar = { - "name": "System", - "property": name, - "value": value - } - result.append(thisvar) - return result - + return result[0] ############################################################ -# Helper Functions +# Non-class Helper Functions ############################################################ # subroutine to run a command, return stdout as array unless zero_only then return [0] diff --git a/files/api/app.py b/files/api/app.py index d01b678..3317f7e 100644 --- a/files/api/app.py +++ b/files/api/app.py @@ -76,7 +76,8 @@ def update_redis_channel(redis_channel, data): def update_redis_server(): # Update Stats Redis Channel - update_redis_channel("host_metrics", get_redis_data()) + if cosmostat_system.check_system_timer(): + update_redis_channel("host_metrics", get_redis_data(human_readable = False)) # Update history_stats Redis Channel # update_redis_channel("history_stats", get_component_list()) @@ -98,7 +99,12 @@ def static_data(): # redis data @app.route('/redis_data', methods=['GET']) def redis_data(): - return jsonify(get_redis_data()) + return jsonify(get_redis_data(human_readable = False)) + +# redis strings +@app.route('/redis_strings', methods=['GET']) +def redis_strings(): + return jsonify(get_redis_data(human_readable = True)) # full summary @app.route('/full_summary', methods=['GET']) @@ -110,15 +116,43 @@ def full_summary(): def info(): return jsonify(get_info()) +# socket timer +@app.route('/start_timer', methods=['GET']) +def start_timer(): + current_timestamp = int(time.time()) + cosmostat_system.recent_check = current_timestamp + if app_settings["noisy_test"]: + print(f"Timestamp updated to {cosmostat_system.recent_check}") + return jsonify( + { + "message": "websocket timer reset", + "new_timestamp": cosmostat_system.recent_check + } + ) + +# socket timer data +@app.route('/timer_data', methods=['GET']) +def timer_data(): + time_now = time.time() + time_lapsed = time_now - float(cosmostat_system.recent_check) + result = { + "Time Lapsed": time_lapsed, + "Current Time Value": time_now, + "Last Update Value": float(cosmostat_system.recent_check), + "System Updating": cosmostat_system.check_system_timer() + } + return jsonify(result) + # test route @app.route('/test', methods=['GET']) def test(): + this_cpu = cosmostat_system.get_components(component_type="CPU") return jsonify( { "component_count:": len(cosmostat_system.components), "user": jenkins_user_settings(), "hostname": jenkins_hostname_settings(), - "cpu_model": cosmostat_system.get_components(component_type="CPU").description + "cpu_model": this_cpu[0].description } ) @@ -128,22 +162,18 @@ def test(): # needs to return array of {name: name, type: type, metrics: metrics} # for redis table generation, includes system and component metrics -def get_dynamic_data(): - return cosmostat_system.get_live_metrics() +def get_dynamic_data(human_readable = False): + return cosmostat_system.get_live_metrics(human_readable) -def get_static_data(): +def get_static_data(human_readable = False): result = [] - for metric in cosmostat_system.get_system_properties(): - result.append(metric) - for metric in cosmostat_system.get_component_properties(): - result.append(metric) - return result + return cosmostat_system.get_static_metrics(human_readable) -def get_redis_data(): +def get_redis_data(human_readable = False): result = [] - for metric in get_dynamic_data(): + for metric in get_dynamic_data(human_readable): result.append(metric) - for metric in get_static_data(): + for metric in get_static_data(human_readable): result.append(metric) return result @@ -171,11 +201,11 @@ def get_info(): component_strings = [] for component in cosmostat_system.get_components(): component_strings.append({"name": component.name, "description": component.description}) - system_strings = [] result = { "hostname": jenkins_hostname_settings(), - "component_strings": component_strings + "component_strings": component_strings, + "system_strings": cosmostat_system.get_sysvars_summary_keys() } #for component_string in component_strings: # for name, description in component_string.items(): @@ -199,7 +229,8 @@ def new_cosmos_system(): # Background Loop Function def background_loop(): # Update all data on the System object - cosmostat_system.update_system_state() + if cosmostat_system.check_system_timer(): + cosmostat_system.update_system_state() if app_settings["push_redis"]: update_redis_server() @@ -241,7 +272,7 @@ if __name__ == '__main__': print("Skipping flask background task") # Flask API - app.run(debug=True, host=service_gateway_ip(), port=5000) + app.run(debug=False, host=service_gateway_ip(), port=5000) diff --git a/files/api/component_descriptors.json b/files/api/component_descriptors.json index 6ec1667..8b5914b 100644 --- a/files/api/component_descriptors.json +++ b/files/api/component_descriptors.json @@ -1,46 +1,56 @@ [ { "name": "CPU", - "description": "{model_name} with {core_count} cores.", + "description": "{CPU Model} with {Core Count} cores.", "multi_check": "False", "properties": { - "core_count": "lscpu --json | jq -r '.lscpu[] | select(.field==\"CPU(s):\") | .data'", - "model_name": "lscpu --json | jq -r '.lscpu[] | select(.field==\"Model name:\") | .data'" + "Core Count": "lscpu --json | jq -r '.lscpu[] | select(.field==\"CPU(s):\") | .data'", + "CPU Model": "lscpu --json | jq -r '.lscpu[] | select(.field==\"Model name:\") | .data'", + "Clock Speed": "sudo dmesg | grep MHz | grep tsc | cut -d: -f2 | awk '{print $2 \" \" $3}'" }, "metrics": { "1m_load": "cat /proc/loadavg | awk '{print $1}'", "5m_load": "cat /proc/loadavg | awk '{print $2}'", - "15m_load": "cat /proc/loadavg | awk '{print $3}'" + "15m_load": "cat /proc/loadavg | awk '{print $3}'", + "current_mhz": "less /proc/cpuinfo | grep MHz | cut -d: -f2 | awk '{sum += $1} END {print sum/NR}'" } }, { "name": "RAM", - "description": "Total {bytes_total}GB in {module_count} modules.", + "description": "Total {Total GB}GB in {RAM Module Count} modules.", "multi_check": "False", "properties": { - "bytes_total": "sudo lshw -json -c memory | jq -r '.[] | select(.description==\"System Memory\").size' | awk '{printf \"%.2f\\n\", $1/1073741824}'", - "module_count": "sudo lshw -json -c memory | jq -r '.[] | select(.id | contains(\"bank\")) | .id ' | wc -l" + "Total GB": "sudo /usr/bin/lshw -json -c memory | jq -r '.[] | select(.description==\"System Memory\").size' | awk '{printf \"%.2f\\n\", $1/1073741824}'", + "RAM Module Count": "sudo /usr/bin/lshw -json -c memory | jq -r '.[] | select(.id | contains(\"bank\")) | .id ' | wc -l", + "RAM Type": "/usr/sbin/dmidecode --type 17 | grep Type: | sort -u | cut -d: -f2 | xargs", + "RAM Speed": "/usr/sbin/dmidecode --type 17 | grep Speed: | grep -v Configured | sort -u | cut -d: -f2 | xargs", + "RAM Voltage": "/usr/sbin/dmidecode --type 17 | grep 'Configured Voltage' | sort -u | cut -d: -f2 | xargs" }, "metrics": { - "used_capacity_mb": "free -m | grep Mem | awk '{print $3}'", - "free_capacity_mb": "free -m | grep Mem | awk '{print $4}'" - } + "MB Used": "free -m | grep Mem | awk '{print $3}'", + "MB Free": "free -m | grep Mem | awk '{print $4}'" + }, + "virt_ignore": [ + "RAM Type", + "RAM Speed", + "RAM Voltage" + ] }, { - "name": "Block Storage", - "description": "{device_id} is of type {drive_type} with capacity of {drive_capacity}.", + "name": "STOR", + "description": "{Device Path} is of type {Drive Type} with capacity of {Total Capacity}.", "multi_check": "True", "device_list": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME | awk '{print $1}'", "properties": { - "device_name": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME | awk '{{print $1}}' | grep {this_device}", - "device_id": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME | awk '{{print \"/dev/\"$1}}' | grep {this_device}", - "drive_type": "lsblk -d -o NAME,TRAN | grep {this_device} | awk '{{print $2}}'", - "drive_capacity": "lsblk -d -o NAME,SIZE | grep {this_device} | awk '{{print $2}}'" + "Device Name": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME | awk '{{print $1}}' | grep {this_device}", + "Device Path": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME | awk '{{print \"/dev/\"$1}}' | grep {this_device}", + "Drive Type": "lsblk -d -o NAME,TRAN | grep {this_device} | awk '{{print $2}}'", + "Total Capacity": "lsblk -d -o NAME,SIZE | grep {this_device} | awk '{{print $2}}'" }, "metrics": { - "smart_status": "sudo smartctl -x --json /dev/{this_device} | jq -r .smart_status.passed", - "ssd_endurance_string": "sudo smartctl -x --json /dev/{this_device} | jq -r '.physical_block_size as $block |.ata_device_statistics.pages[] | select(.name == \"General Statistics\") | .table[] | select(.name == \"Logical Sectors Written\") | .value as $sectors | ($sectors * $block) / 1073741824 ' | awk '{{printf \"%.2f GiB Written\\n\", $0}}' || true", - "nvme_endurance_string": "sudo smartctl -x --json /dev/{this_device} | jq -r ' .nvme_smart_health_information_log.data_units_written as $dw | .logical_block_size as $ls | ($dw * $ls) / 1073741824 ' | awk '{{printf \"%.2f GiB Written\\n\", $0}}' || true" + "SMART Check": "/usr/sbin/smartctl -x --json /dev/{this_device} | jq -r .smart_status.passed", + "SATA GBW": "/usr/sbin/smartctl -x --json /dev/{this_device} | jq -r '.physical_block_size as $block |.ata_device_statistics.pages[] | select(.name == \"General Statistics\") | .table[] | select(.name == \"Logical Sectors Written\") | .value as $sectors | ($sectors * $block) / 1073741824 ' | awk '{{printf \"%.2f GiB Written\\n\", $0}}' || true", + "NVMe GBW": "/usr/sbin/smartctl -x --json /dev/{this_device} | jq -r ' .nvme_smart_health_information_log.data_units_written as $dw | .logical_block_size as $ls | ($dw * $ls) / 1073741824 ' | awk '{{printf \"%.2f GiB Written\\n\", $0}}' || true" } } ] \ No newline at end of file diff --git a/files/web/html/index.php b/files/web/html/index.php index 1ad2ee5..4f315b8 100644 --- a/files/web/html/index.php +++ b/files/web/html/index.php @@ -8,11 +8,11 @@ -
+

Matt-Cloud Cosmostat Dashboard

This dashboard shows the local Matt-Cloud system stats.

-
+

Live System Metrics

Connecting…
diff --git a/files/web/html/src/redis.js b/files/web/html/src/redis.js index d5d1ed0..bc77270 100644 --- a/files/web/html/src/redis.js +++ b/files/web/html/src/redis.js @@ -21,6 +21,7 @@ function safeSetText(id, txt) { ------------------------------------------------------------------ */ // helper function for table row ordering function renderStatsTable(data) { + socket.emit('tableRendered'); renderGenericTable('host_metrics', data, 'No Stats available'); } @@ -48,59 +49,56 @@ function renderGenericTable(containerId, data, emptyMsg) { 3. Merge rows by name ------------------------------------------------------------ */ function mergeRowsByName(data) { - const groups = {}; // { name: { types: [], metrics: [], props: [], values: [] } } - + const groups = {}; // { source: { ... } } data.forEach(row => { - const name = row.name; - if (!name) return; // ignore rows without a name - - if (!groups[name]) { - groups[name] = { types: [], metrics: [], props: [], values: [] }; + const source = row.Source; // <-- changed + if (!source) return; + if (!groups[source]) { + groups[source] = { Metric: [], Data: [], Property: [], Value: [] }; } - - // Metric rows - contain type + metric - if ('type' in row && 'metric' in row) { - groups[name].types.push(row.type); - groups[name].metrics.push(row.metric); - } - // Property rows - contain property + value - else if ('property' in row && 'value' in row) { - groups[name].props.push(row.property); - groups[name].values.push(row.value); + if ('Metric' in row && 'Data' in row) { + groups[source].Metric.push(row.Metric); + groups[source].Data.push(row.Data); + } else if ('Property' in row && 'Value' in row) { + groups[source].Property.push(row.Property); + groups[source].Value.push(row.Value); } }); - // Convert each group into a single row object const merged = []; - Object.entries(groups).forEach(([name, grp]) => { + Object.entries(groups).forEach(([source, grp]) => { merged.push({ - name, - type: grp.types, // array of types - metric: grp.metrics, // array of metrics - property: grp.props, // array of property names - value: grp.values, // array of property values + Source: source, // <-- keep the original key + Metric: grp.Metric, + Data: grp.Data, + Property: grp.Property, + Value: grp.Value }); }); - return merged; } -/* ------------------------------------------------------------------ - 3b. Order rows - put “System”, “CPU”, “RAM” first - ------------------------------------------------------------------ */ +// 3b. Order rows – put “System”, “CPU”, “RAM” first function orderRows(rows) { - // this should be updatable if i want + // Priority list – can be updated later const priority = ['System', 'CPU', 'RAM']; - const priorityMap = {}; - priority.forEach((name, idx) => (priorityMap[name] = idx)); + // Map source → priority index + const priorityMap = {}; + priority.forEach((src, idx) => { + priorityMap[src] = idx; + }); + + // Stable sort: keep original position if priorities are equal return [...rows].sort((a, b) => { - const aIdx = priorityMap.hasOwnProperty(a.name) - ? priorityMap[a.name] - : Infinity; // anything not in priority goes to the end - const bIdx = priorityMap.hasOwnProperty(b.name) - ? priorityMap[b.name] + const aIdx = priorityMap.hasOwnProperty(a.Source) + ? priorityMap[a.Source] + : Infinity; // anything not in priority goes to the end + const bIdx = priorityMap.hasOwnProperty(b.Source) + ? priorityMap[b.Source] : Infinity; + + // If both have the same priority (or both Infinity), keep original order return aIdx - bIdx; }); } @@ -109,7 +107,7 @@ function orderRows(rows) { 4. Build an HTML table from an array of objects ------------------------------------------------------------ */ function buildTable(data) { - const cols = ['name', 'type', 'metric', 'property', 'value']; // explicit order + const cols = ['Source', 'Property', 'Value', 'Metric', 'Data']; // explicit order const table = document.createElement('table'); // Header diff --git a/files/web/html/src/styles.css b/files/web/html/src/styles.css index b938f79..6584592 100644 --- a/files/web/html/src/styles.css +++ b/files/web/html/src/styles.css @@ -8,33 +8,15 @@ body { color: #bdc3c7; /* Dimmer text color */ } -.hidden-info { - display: none; -} - -.title-button { - background-color: #34495e; - border: none; - color: white; - padding: 15px 32px; - text-align: center; - text-decoration: none; - display: inline-block; - font-size: 16px; - margin: 4px 2px; - cursor: pointer; -} - - table, th, td { - border: 1px solid black; + border: 2px solid #182939; border-collapse: collapse; } th, td { padding: 10px; } -.container { +.card { max-width: 950px; margin: 0 auto; padding: 20px; @@ -43,15 +25,6 @@ th, td { box-shadow: 0 2px 4px rgba(0, 0, 0, 0.3); /* Slightly darker shadow */ margin-top: 20px; } -.container-small { - max-width: 550px; - margin: 0 auto; - padding: 20px; - background-color: #34495e; /* Darker background for container */ - border-radius: 8px; - box-shadow: 0 2px 4px rgba(0, 0, 0, 0.3); /* Slightly darker shadow */ - margin-top: 20px; -} h1, h2, h3, h4 { color: #bdc3c7; /* Dimmer text color */ @@ -67,55 +40,12 @@ li { color: #bdc3c7; /* Dimmer text color */ } -.group-columns { - display: flex; -} - -.group-rows { - display: flex; - flex-wrap: wrap; - justify-content: flex-start; /* Left justification */ - margin-top: 10px; -} - -.group-column { - flex: 0 0 calc(33% - 10px); /* Adjust width of each column */ -} - -.column { - flex: 1; - padding: 0 10px; /* Adjust spacing between columns */ -} - -.subcolumn { - margin-left: 10px; -} - -.grid { - display: flex; - flex-wrap: wrap; - justify-content: space-between; - margin-top: 5px; -} - -.meter { - width: calc(90% - 5px); - max-width: calc(45% - 5px); - margin-bottom: 5px; - border: 1px solid #7f8c8d; /* Light border color */ - border-radius: 5px; - padding: 5px; - text-align: center; - background-color: #2c3e50; /* Dark background for meter */ -} - #host_metrics_column td { list-style: none; /* removes the numeric markers */ padding-left: 0; /* remove the default left indent */ margin-left: 0; /* remove the default left margin */ } - #host_metrics_table tbody tr td :nth-of-type(even) { - background-color: #2c3e50; + background-color: #3e5c78; } diff --git a/files/web/node_server/package.json b/files/web/node_server/package.json index 3b2246c..bc2fb35 100644 --- a/files/web/node_server/package.json +++ b/files/web/node_server/package.json @@ -8,6 +8,7 @@ "dependencies": { "express": "^4.18.2", "socket.io": "^4.7.2", - "redis": "^4.6.7" + "redis": "^4.6.7", + "node-fetch": "^2.6.7" } } \ No newline at end of file diff --git a/files/web/node_server/server.js b/files/web/node_server/server.js index e0ec3d4..d548a36 100644 --- a/files/web/node_server/server.js +++ b/files/web/node_server/server.js @@ -1,32 +1,63 @@ // server.js -const http = require('http'); +const http = require('http'); const express = require('express'); const { createClient } = require('redis'); const { Server } = require('socket.io'); +const fetch = require('node-fetch'); // npm i node-fetch@2 -const app = express(); +const app = express(); const server = http.createServer(app); -const io = new Server(server); +const io = new Server(server); -// Serve static files (index.html) +// ---------- Socket.io ---------- +io.on('connection', async socket => { + console.log('client connected:', socket.id); + + // Call the external API every time a client connects + try { + const resp = await fetch('http://192.168.37.1:5000/start_timer', { + method: 'GET' + }); + + const data = await resp.json(); + console.log('API responded to connect:', data); + } catch (err) { + console.error('Failed to hit start_timer endpoint:', err); + } + + // Listen for tableRendered event from the client + socket.on('tableRendered', async () => { + console.log('Client reported table rendered - starting timer'); + try { + const resp = await fetch('http://192.168.37.1:5000/start_timer', { + method: 'GET' + }); + const text = await resp.text(); + console.log('Timer endpoint responded:', text); + } catch (err) { + console.error('Failed to hit start_timer:', err); + } + }); +}); + +// Serve static files (index.html, etc.) app.use(express.static('public')); // ---------- Redis subscriber ---------- -const redisClient = createClient({ - url: 'redis://192.168.37.1:6379' -}); +const redisClient = createClient({ url: 'redis://192.168.37.1:6379' }); + redisClient.on('error', err => console.error('Redis error', err)); (async () => { await redisClient.connect(); - const sub = redisClient.duplicate(); // duplicate to keep separate pub/sub await sub.connect(); -// Subscribe to the channel that sends host stats + + // Subscribe to the channel that sends host stats await sub.subscribe( ['host_metrics'], - (message, channel) => { // <-- single handler + (message, channel) => { let payload; try { payload = JSON.parse(message); // message is a JSON string @@ -34,20 +65,13 @@ redisClient.on('error', err => console.error('Redis error', err)); console.error(`Failed to parse ${channel}`, e); return; } - io.emit(channel, payload); } ); - sub.on('error', err => console.error('Subscriber error', err)); + sub.on('error', err => console.error('Subscriber error', err)); })(); -// ---------- Socket.io ---------- -io.on('connection', socket => { - console.log('client connected:', socket.id); - // Optional: send the current state on connect if you keep it cached -}); - // ---------- Start ---------- const PORT = process.env.PORT || 3000; server.listen(PORT, () => { diff --git a/files/proxy/nginx.conf b/files/web/proxy/nginx.conf similarity index 100% rename from files/proxy/nginx.conf rename to files/web/proxy/nginx.conf diff --git a/tasks/init.yaml b/tasks/init.yaml index ae3f2fe..1543a41 100644 --- a/tasks/init.yaml +++ b/tasks/init.yaml @@ -1,5 +1,5 @@ --- - +# package handler - name: Cosmostat - Init - Get installed package list when: dpkg_output is undefined shell: "dpkg --list | grep ii | awk '{print $2}'" @@ -46,7 +46,7 @@ mode: '0755' # create user service folder -- name: Cosmostat - Init - create cosmostat service folder +- name: Cosmostat - Init - create cosmostat user service folder file: path: "{{ user_service_folder }}" state: directory @@ -85,6 +85,7 @@ # create node.js docker container for web dashboard - name: node.js server container handler + when: false block: - name: Cosmostat - Init - node.js - copy server files @@ -96,13 +97,16 @@ group: "{{ service_user }}" - name: Cosmostat - Init - node.js - build docker container - community.docker.docker_image_build: + community.docker.docker_image: name: ws_node tag: latest - rebuild: always - path: "{{ service_control_web_folder }}/node_server" - dockerfile: Dockerfile - labels: - ws_node: "true" + source: local + build: + path: "{{ service_control_web_folder }}/node_server" + dockerfile: Dockerfile + force_tag: true + state: present + force_source: true + ... \ No newline at end of file diff --git a/tasks/main.yaml b/tasks/main.yaml index 78adbf7..9e57daf 100644 --- a/tasks/main.yaml +++ b/tasks/main.yaml @@ -1,17 +1,21 @@ --- + + + # initializa environment - -# set up API - -# set up web stack - - name: Initialize Environment when: not quick_refresh | bool include_tasks: init.yaml +# set up API - name: Build API include_tasks: api.yaml +# set up web stack - name: Build Web Dashboard include_tasks: web.yaml + +#- name: Purge Old Containers +# when: not quick_refresh | bool +# include_tasks: purge.yaml ... \ No newline at end of file diff --git a/tasks/purge.yaml b/tasks/purge.yaml new file mode 100644 index 0000000..85df7f8 --- /dev/null +++ b/tasks/purge.yaml @@ -0,0 +1,34 @@ +- name: Cosmostat - Clean up old ws_node image tags + block: + # Grab a list of all tags the image has + - name: Get all ws_node image tags + command: | + docker images --format "{{.Repository}}:{{.Tag}}" \ + --filter=reference="ws_node:*" + register: all_tags_raw + changed_when: false + + # Turn that raw string into a list of just the tag names + - name: Parse tag names out of the list + set_fact: + all_tags: >- + {{ all_tags_raw.stdout_lines | + map('regex_replace', '^ws_node:', '') | + list }} + + # Keep everything *except* the one that ends with “:latest” + - name: Build list of tags that should be removed + set_fact: + tags_to_remove: "{{ all_tags | difference(['latest']) }}" + + # Remove each old tag + - name: Delete old ws_node image tags + community.docker.docker_image: + name: ws_node + tag: "{{ item }}" + state: absent + loop: "{{ tags_to_remove }}" + when: tags_to_remove | length > 0 + when: tags_to_remove | length > 0 + tags: + - cleanup \ No newline at end of file diff --git a/tasks/web.yaml b/tasks/web.yaml index be935da..d8067ee 100644 --- a/tasks/web.yaml +++ b/tasks/web.yaml @@ -12,53 +12,25 @@ owner: "{{ service_user }}" group: "{{ service_user }}" -- name: Cosmostat - Init - copy dashboard web files +- name: Cosmostat - Web - copy docker files copy: - src: "web/html" - dest: "{{ service_control_web_folder }}/" + src: "web/" + dest: "{{ service_control_web_folder }}" mode: 0755 owner: "{{ service_user }}" group: "{{ service_user }}" -# These are not needed unless there is a stack -#- name: Cosmostat - Web - copy files for history dashboard -# copy: -# src: "dashboard/" -# dest: "{{ service_control_web_folder }}/html" -# mode: 0755 -# owner: "{{ service_user }}" -# group: "{{ service_user }}" +- name: Cosmostat - Web - template docker-compose.yaml + template: + src: docker-compose-php.yaml + dest: "{{ service_control_web_folder }}/docker-compose.yaml" + mode: 0644 -- name: Cosmostat - Web - copy files for proxy container - copy: - src: "proxy/" - dest: "{{ service_control_web_folder }}/proxy" - mode: 0755 - owner: "{{ service_user }}" - group: "{{ service_user }}" - -- name: docker container handler - block: - - - name: Cosmostat - Web - template docker-compose.yaml - template: - src: docker-compose-php.yaml - dest: "{{ service_control_web_folder }}/docker-compose.yaml" - mode: 0644 - - - name: Cosmostat - Web - Start containers - shell: "docker-compose -f {{ service_control_web_folder }}/docker-compose.yaml up -d" - register: docker_output - - debug: | - msg="{{ docker_output.stdout_lines }}" +- name: Cosmostat - Web - Start containers + shell: "docker-compose -f {{ service_control_web_folder }}/docker-compose.yaml up -d" + register: docker_output +- debug: | + msg="{{ docker_output.stdout_lines }}" msg="{{ docker_output.stderr_lines }}" - - name: Cosmostat - Web - Prune old containers - community.docker.docker_prune: - containers: true - containers_filters: - label: - ws_node: "true" - - ... \ No newline at end of file diff --git a/templates/docker-compose-php.yaml b/templates/docker-compose-php.yaml index c843456..8df80c2 100644 --- a/templates/docker-compose-php.yaml +++ b/templates/docker-compose-php.yaml @@ -8,37 +8,41 @@ services: networks: - cosmostat_net restart: always - + cosmostat_ws_node: + image: node:18-alpine + working_dir: /app + command: sh -c "npm install && node server.js" container_name: cosmostat_ws_node - build: - context: "{{ service_control_web_folder }}/node_server" - dockerfile: Dockerfile - image: ws_node:latest + volumes: + - "{{ service_control_web_folder }}/html:/usr/src/app/public" + - "{{ service_control_web_folder }}/node_server:/app" + - /app/node_modules ports: - - "{{ (docker_gateway + ':') if secure_api else '' }}3000:3000" + - "{{ docker_gateway }}:3000:3000" networks: - cosmostat_net restart: always depends_on: - - cosmostat_redis + - cosmostat_redis cosmostat_web_dash: container_name: cosmostat_web_dash image: php:8.0-apache ports: - - "{{ (docker_gateway + ':') if secure_api else '' }}8080:80" + - "{{ docker_gateway }}:8080:80" volumes: - ./html:/var/www/html/ networks: - cosmostat_net restart: always + # public_dashboard: {{ public_dashboard }} cosmostat_nginx_proxy: container_name: cosmostat_nginx_proxy image: nginx:latest ports: - - "{{ (docker_gateway + ':') if secure_api else '' }}80:80" + - "{{ (docker_gateway + ':') if not public_dashboard | bool else '' }}80:80" volumes: - ./proxy/nginx.conf:/etc/nginx/conf.d/default.conf networks: diff --git a/templates/docker-compose.yaml b/templates/docker-compose.yaml index 77fd7e3..f387124 100644 --- a/templates/docker-compose.yaml +++ b/templates/docker-compose.yaml @@ -14,13 +14,15 @@ services: restart: always ws_node: + + image: node:18-alpine + working_dir: /app + command: sh -c "npm install && node server.js" container_name: ws_node - build: - context: {{ service_control_web_folder }}/node_server - dockerfile: Dockerfile - image: ws_node:latest volumes: - {{ service_control_web_folder }}/html:/usr/src/app/public + - {{ service_control_web_folder }}/node_server:/app + - /app/node_modules ports: # put back to 3000 if the stack is needed - {{ (docker_gateway + ':') if secure_api else '' }}80:3000