Matt-Cloud Cosmostat Dashboard
This dashboard shows the local Matt-Cloud system stats.
diff --git a/defaults/main.yaml b/defaults/main.yaml index 52c5640..30b28e2 100644 --- a/defaults/main.yaml +++ b/defaults/main.yaml @@ -14,6 +14,7 @@ cosmostat_packages: - jc - smartmontools - inxi + - easy-rsa # python venv packages cosmostat_venv_packages: | @@ -38,7 +39,6 @@ docker_gateway: "192.168.37.1" cosmostat_server_ip: "10.200.27.20" api_bind_ip: "{{ docker_gateway }}" - # cosmostat service folder root service_folder: "/opt/cosmostat" @@ -52,6 +52,7 @@ api_service_folder: "{{ service_folder }}/api" venv_folder: "{{ service_folder }}/venv" api_service_exe: "{{ venv_folder }}/bin/python -u {{ api_service_folder }}/app.py" custom_api_port: "5000" +REAL_API_KEY: "DEADBEEF" # dashboard vars service_control_web_folder: "{{ service_folder }}/web" @@ -67,11 +68,14 @@ x64_arch: true noisy_test: false debug_output: true secure_api: true -push_redis: true +push_redis: false run_background : true log_output: true update_frequency: "1" cosmostat_server: true -cosmostat_server_api: "http://{{ cosmostat_server_ip }}:{{ custom_api_port }}/" +cosmostat_server_api: "https://cosmostat.testy-cal.com/" +local_api_address: "http://{{ cosmostat_server_ip }}:{{ custom_api_port }}/" cosmostat_server_reporter: false +# setting this to true for default install +disable_local_api: true ... \ No newline at end of file diff --git a/files/api/Components.py b/files/api/Components.py index d6eb14e..81120cb 100644 --- a/files/api/Components.py +++ b/files/api/Components.py @@ -69,6 +69,7 @@ class Component: self.virt_ignore = self._descriptor.get('virt_ignore', []) self.multi_metrics = self._descriptor.get('multi_metrics', []) self.arch_check = self._descriptor.get('arch_check', []) + self.php_extra_list = self._descriptor.get('php_extra', []) if self.is_virtual: self.virt_ignore = [] @@ -173,6 +174,13 @@ class Component: log_data(log_output = f"result - {result_command}", log_level = "debug_output") return result_command + # check if this property should show in the System Properties box + def check_php_extra(self, property_name): + result = False + if property_name in self.php_extra_list: + result = True + return result + ######################################################## # keyed data functions ######################################################## @@ -407,7 +415,6 @@ class System: if multi_check: log_data(log_output = f"Creating one component of type {component_name} for each one found", log_level = "log_output") component_type_device_list = get_device_list(component_name) - component_id = 0 for this_device in component_type_device_list: this_component_ID = component_type_device_list.index(this_device) this_component_name = f"{component_name} {this_component_ID}" @@ -486,20 +493,36 @@ class System: result.append(metric) return result - def get_system_properties(self, human_readable = False): + def get_system_properties(self, human_readable = False, php_extra = False): result = [] for name, value in self._properties.items(): if human_readable: result.append({ "Source": "System", "Property": f"{name}: {value}" - }) + }) else: result.append({ "Source": "System", "Property": name, "Value": value }) + if php_extra and human_readable: + for component_result in self.php_component_data(): + result.append(component_result) + + return result + + def php_component_data(self): + result = [] + for component in self.components: + for this_property in component._properties: + if component.check_php_extra(this_property): + result_string = f"{this_property}: {component._properties[this_property]}" + result.append({ + "Source": "System", + "Property": result_string + }) return result ######################################################## @@ -584,7 +607,12 @@ def run_command(cmd, zero_only=False, use_shell=True, req_check = True): except: return output_lines +# need to add a archticture checker for this +# i also want to make the loop cleaner +# i don't need to iterate over the component class tree +# to get what I want, i think def get_device_list(device_type_name: str): + result = [] for component in component_class_tree: precheck_value = 1 @@ -594,6 +622,7 @@ def get_device_list(device_type_name: str): precheck_value = int(precheck_value_output) log_data(log_output = f"Precheck found - {precheck_command} - {precheck_value}", log_level = "log_output") if component["name"] == device_type_name and precheck_value != 0: + device_list_command = component["device_list"] device_list_result = run_command(device_list_command) result = device_list_result diff --git a/files/api/Cosmos_Settings.py b/files/api/Cosmos_Settings.py index bab2c72..21c8d75 100644 --- a/files/api/Cosmos_Settings.py +++ b/files/api/Cosmos_Settings.py @@ -1,5 +1,6 @@ import yaml from urllib.parse import urlparse +import secrets, string ####################################################################### ### Settings Handler Functions ####################################################################### @@ -16,7 +17,11 @@ app_settings = { "cosmostat_server_reporter": False, "update_frequency": 1, "custom_api_port": "5000", - "cosmostat_server_api": "http://10.200.27.20:5000/" + "cosmostat_server_api": "http://10.200.27.20:5000/", + "REAL_API_KEY": ''.join(secrets.choice(string.ascii_letters + string.digits) for _ in range(256)), + "disable_local_api": False, + "local_api_address": "http://10.200.27.20:5000/", + "cosmostat_server_ip": "10.200.27.20" } with open('cosmostat_settings.yaml', 'r') as f: @@ -61,9 +66,7 @@ def run_cosmostat_reporter(): def service_gateway_ip(): result = "0.0.0.0" - if cosmostat_settings["cosmostat_server"]: - result = urlparse(cosmostat_settings["cosmostat_server_api"]).hostname - elif cosmostat_settings["secure_api"]: + if cosmostat_settings["secure_api"] and not cosmostat_settings["cosmostat_server"]: result = cosmostat_bind_ip() return result diff --git a/files/api/Cosmostat.py b/files/api/Cosmostat.py index 8ddcb26..1744dcf 100644 --- a/files/api/Cosmostat.py +++ b/files/api/Cosmostat.py @@ -93,15 +93,33 @@ class CosmostatServer: def get_client_hostname(self, system_uuid: str): client = self.get_system(system_uuid) return client.hostname + + def get_client_timestamp(self, system_hostname: str): + client = self.get_system(get_uuid_from_hostname(system_hostname)) + return client.data_timestamp - def get_client_hostnames(self, send_age = False): - result = [] + def get_uuid_from_hostname(self, system_hostname): + result = "" for system in self.systems: - data_age = time.time() - system.data_timestamp - if int(data_age) > 60: - self.systems.remove(system) - else: - result.append(system.hostname) + if system.hostname == system_hostname: + result = system.uuid + return result + + def get_client_hostnames(self, send_age = False): + now = time.time() + fresh_systems = [] + result = [] + + for system in self.systems: + age = now - system.data_timestamp + if age <= 60: # keep only fresh servers + fresh_systems.append(system) + if send_age: + result.append({"hostname": system.hostname, "data_age": age}) + else: + result.append(system.hostname) + + self.systems = fresh_systems # replace the old list return result diff --git a/files/api/app.py b/files/api/app.py index 6ea1886..cad0222 100644 --- a/files/api/app.py +++ b/files/api/app.py @@ -4,6 +4,7 @@ from typing import Dict, Union import json, time, redis, yaml import base64, hashlib +import secrets, string import requests from requests import RequestException, Response @@ -35,7 +36,7 @@ def update_redis_server(): if run_cosmostat_server(): update_redis_channel("client_summary", get_server_redis_data()) - update_redis_channel("client_hostnames", get_server_hostnames()) + #update_redis_channel("client_hostnames", get_server_hostnames()) # History Redis Tree # Update history_stats Redis Channel @@ -54,6 +55,7 @@ def get_server_redis_data(): for client in cosmostat_server.systems: this_client_key = { "hostname": client.hostname, + "data_timestamp": client.data_timestamp, "uuid": client.uuid, "short_id": client.name, "redis_data": client.redis_data @@ -64,6 +66,7 @@ def get_server_redis_data(): def get_server_hostnames(): return cosmostat_server.get_client_hostnames() + ####################################################################### ### Client Flask Routes ####################################################################### @@ -152,7 +155,7 @@ def get_static_data(human_readable = False): return cosmostat_client.get_static_metrics(human_readable) def get_php_summary(): - system_properties = cosmostat_client.get_system_properties(human_readable = True) + system_properties = cosmostat_client.get_system_properties(human_readable = True, php_extra = True) system_components = [] for component in cosmostat_client.get_components(): this_component = { @@ -161,6 +164,18 @@ def get_php_summary(): } system_components.append(this_component) + if run_cosmostat_server(): + print(cosmostat_client.name) + client_uuid = cosmostat_server.get_uuid_from_hostname(cosmostat_client.name) + print(client_uuid) + data_timestamp = cosmostat_server.get_system(client_uuid) + print(data_timestamp) + component_age = { + "component_name": "Data Timestamp", + "info_strings": f"Data is {data_timestamp} seconds old" + } + system_components.append(component_age) + result = [{ "system_properties": system_properties, "system_components": system_components @@ -229,7 +244,17 @@ def client_details(): def client_hostnames(): result = [] if run_cosmostat_server(): - result = cosmostat_server.get_client_hostnames() + result = cosmostat_server.get_client_hostnames(send_age = True) + else: + result = {"message": "server not running on this endpoint"} + return jsonify(result) + +# api to get server redis data +@app.route('/get_server_redis', methods=['GET']) +def get_server_redis(): + result = [] + if run_cosmostat_server(): + result = get_server_redis_data() else: result = {"message": "server not running on this endpoint"} return jsonify(result) @@ -241,30 +266,48 @@ def client_hostnames(): # update client on server def run_update_client(this_client): - if not cosmostat_server.check_uuid(this_client["uuid"]): - return { "message": "client not found" } - else: - timestamp_update = cosmostat_server.update_system(system_dictionary = this_client["redis_data"], system_uuid = this_client["uuid"]) - update_status = f'updated client {this_client["short_id"]}' + if public_api_check(this_client): + if not cosmostat_server.check_uuid(this_client["uuid"]): + return { "message": "client not found" } + else: + timestamp_update = cosmostat_server.update_system(system_dictionary = this_client["redis_data"], system_uuid = this_client["uuid"]) + update_status = f'updated client {this_client["short_id"]}' - return { - "status": update_status, - "uuid": this_client["uuid"], - "redis_data": this_client, - "timestamp_update": timestamp_update - } + return { + "status": update_status, + "uuid": this_client["uuid"], + "redis_data": this_client, + "timestamp_update": timestamp_update + } + else: + return{ + "status": "api failure" + } # create client on server def run_create_client(this_client): - timestamp_update = cosmostat_server.add_system(system_dictionary = this_client) - update_status = f'created client {this_client["short_id"]}' - return { - "status": update_status, - "uuid": this_client["uuid"], - "client_properties": this_client, - "timestamp_update": timestamp_update + if public_api_check(this_client): + timestamp_update = cosmostat_server.add_system(system_dictionary = this_client) + update_status = f'created client {this_client["short_id"]}' + return { + "status": update_status, + "uuid": this_client["uuid"], + "client_properties": this_client, + "timestamp_update": timestamp_update + } + else: + return{ + "status": "api failure" } +def public_api_check(this_client): + result = False + default_key = ''.join(secrets.choice(string.ascii_letters + string.digits) for _ in range(256)) + api_key = this_client.get('API_KEY', default_key) + if api_key == app_settings["REAL_API_KEY"]: + result = True + return result + # flask submission check function def client_submit_check(request, dict_name: str): payload = {} @@ -324,6 +367,7 @@ def get_client_details(): # Cosmostat Client Reporter def client_update(): api_url = f"{cosmostat_server_api()}update_client" + print(api_url) payload = get_client_payload(get_client_redis_data(human_readable = False), "redis_data") log_data(log_output = "client_update - redis data from local client:", log_level = "noisy_test") log_data(log_output = payload, log_level = "noisy_test") @@ -366,7 +410,8 @@ def get_client_payload(system_dictionary: dict, dictionary_name: str): "uuid": this_uuid, "short_id": this_short_id, "hostname": this_hostname, - dictionary_name: system_dictionary + dictionary_name: system_dictionary, + "API_KEY": app_settings["REAL_API_KEY"] } return payload @@ -401,25 +446,26 @@ if __name__ == '__main__': # Background Loop Function def background_loop(): - # Update all data on the System object - if cosmostat_client.check_system_timer(): + # Update all data on the System object unless this is the server + if cosmostat_client.check_system_timer() and not run_cosmostat_server(): cosmostat_client.update_system_state() - if app_settings["push_redis"]: + if app_settings["push_redis"] and not app_settings["disable_local_api"]: update_redis_server() if run_cosmostat_reporter(): if int(time.time()) % 5 == 0 and not cosmostat_client.check_system_timer(): cosmostat_client.update_system_state() - client_update() + client_update() if run_cosmostat_server(): + # update the client state since that was skipped + cosmostat_client.update_system_state() this_client = get_client_payload(get_client_redis_data(human_readable = False), "redis_data") + if app_settings["noisy_test"]: + print(this_client) run_update_client(this_client) - - - time.sleep(0.5) ###################################### @@ -443,14 +489,14 @@ if __name__ == '__main__': # send initial stats update to redis ###################################### - if app_settings["push_redis"]: + if app_settings["push_redis"] and not app_settings["disable_local_api"]: update_redis_server() ###################################### # Flask scheduler for scanner ###################################### - if app_settings["run_background"]: + if app_settings["run_background"] and not app_settings["disable_local_api"]: log_data(log_output = "Loading flask background subroutine...", log_level = "log_output") scheduler.add_job(id='background_loop', @@ -467,11 +513,15 @@ if __name__ == '__main__': ###################################### # Flask API ###################################### - - app.run(debug=False, host=service_gateway_ip(), port=service_api_port()) - - - - + print(f"gateway: {service_gateway_ip()} - port: {service_api_port()}") + if not app_settings["disable_local_api"]: + app.run(debug=False, host=service_gateway_ip(), port=service_api_port()) + else: + print("Internal API Disabled.") + while True: + if int(time.time()) % 5 == 0 and not cosmostat_client.check_system_timer(): + cosmostat_client.update_system_state() + client_update() + time.sleep(0.5) \ No newline at end of file diff --git a/files/api/descriptors.json b/files/api/descriptors.json index 9dbd9ab..7e451da 100644 --- a/files/api/descriptors.json +++ b/files/api/descriptors.json @@ -89,6 +89,9 @@ "arch_variance": [ "current_mhz", "Clock Speed" + ], + "php_extra" :[ + "CPU Model" ] }, { @@ -132,20 +135,23 @@ "RAM Type", "RAM Speed", "RAM Voltage" + ], + "php_extra" :[ + "Total GB" ] }, { "name": "LAN", - "description": "{Device Name} - {Device ID} - {MAC Address}", + "description": "{Device Name} - {Device ID}", "multi_check": "True", "device_list": "ip link | grep default | grep -v -e docker -e 127.0.0.1 -e br- -e veth -e lo -e tun | cut -d ':' -f 2 | awk '{{print $1}}' ", "properties": { - "MAC Address": "ip link | grep -A1 ' {this_device}' | grep ether | awk '{{print $2}}'", + "MAC Address": "ip link | grep -A1 ' {this_device}' | grep ether | awk '{{print $2}}' || echo MAC missing", "Device Name": "echo {this_device}", "Device ID": "( udevadm info -q property -p $(ls -l /sys/class/net/ | grep {this_device} | cut -d '>' -f2 | cut -b 8-) | grep ID_MODEL_FROM_DATABASE || echo 'ID_MODEL_FROM_DATABASE=missing' ) | cut -d '=' -f2" }, "metrics": { - "IP Address": "ip -o -4 ad | grep -v -e docker -e 127.0.0.1 -e br- | grep {this_device} | awk '{{print $4}}'", + "IP Address": "ip -o -4 ad | grep -v -e docker -e 127.0.0.1 -e br- -e tun | grep {this_device} | awk '{{print $4}}'", "Data Transmitted": "ifconfig {this_device} | grep RX | grep bytes | cut -d '(' -f2 | tr -d ')'", "Data Received": "ifconfig {this_device} | grep TX | grep bytes | cut -d '(' -f2 | tr -d ')'", "Link State": "cat /sys/class/net/{this_device}/operstate", @@ -155,13 +161,28 @@ "IP Address" ] }, + { + "name": "VPN", + "description": "{Device Name} - VPN Tunnel", + "multi_check": "True", + "precheck": "ip link | grep tun | wc -l", + "device_list": "ip link | grep default | grep tun | cut -d ':' -f 2 | awk '{{print $1}}' ", + "properties": { + "Device Name": "echo {this_device}" + }, + "metrics": { + "IP Address": "ip -o -4 ad | grep -v -e docker -e 127.0.0.1 -e br- | grep {this_device} | awk '{{print $4}}'", + "Data Transmitted": "ifconfig {this_device} | grep RX | grep bytes | cut -d '(' -f2 | tr -d ')'", + "Data Received": "ifconfig {this_device} | grep TX | grep bytes | cut -d '(' -f2 | tr -d ')'" + } + }, { "name": "NVGPU", - "description": "NVGPU{Device ID} - {Device Model} with {Memory Size}, Max Power {Maximum Power}", + "description": "NVGPU{Device ID} - {GPU Model} with {Memory Size}, Max Power {Maximum Power}", "multi_check": "True", "device_list": "nvidia-smi --query-gpu=index --format=csv,noheader,nounits", "properties": { - "Device Model": "nvidia-smi --id={this_device} --query-gpu=name --format=csv,noheader,nounits", + "GPU Model": "nvidia-smi --id={this_device} --query-gpu=name --format=csv,noheader,nounits", "Device ID": "echo NVGPU{this_device}", "Driver Version": "nvidia-smi --id={this_device} --query-gpu=driver_version --format=csv,noheader,nounits", "Maximum Power": "nvidia-smi --id={this_device} --query-gpu=power.limit --format=csv,noheader,nounits", @@ -175,16 +196,19 @@ "GPU Load": "nvidia-smi --id={this_device} --query-gpu=utilization.gpu --format=csv,noheader,nounits" }, - "precheck": "lspci | grep NVIDIA | wc -l" + "precheck": "lspci | grep NVIDIA | wc -l", + "php_extra" :[ + "GPU Model" + ] }, { "name": "STOR", "description": "{Device Path} is of type {Drive Type} with capacity of {Total Capacity}.", "multi_check": "True", - "device_list": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME | awk '{print $1}'", + "device_list": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME -e sr0| awk '{print $1}'", "properties": { - "Device Name": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME | awk '{{print $1}}' | grep {this_device}", - "Device Path": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME | awk '{{print \"/dev/\"$1}}' | grep {this_device}", + "Device Name": "echo {this_device}", + "Device Path": "echo /dev/{this_device}", "Drive Type": "lsblk -d -o NAME,TRAN | grep {this_device} | awk '{{print ($2 != \"\" ? $2 : \"missing\")}}'", "Total Capacity": "lsblk -d -o NAME,SIZE | grep {this_device} | awk '{{print $2}}'", "SMART Check": "sudo /usr/sbin/smartctl -x --json /dev/{this_device} | jq -r .smart_status.passed" @@ -192,5 +216,34 @@ "metrics": { "placeholder": "" } + }, + { + "name": "MOUNT", + "description": "Storage device {Device Location} mounted at {Storage Path} with {Total Space} total space", + "multi_check": "True", + "device_list": "df -h | grep -v -e 'Use%' -e tmpfs -e overlay -e efi -e udev | awk '{{print $1}}'", + "properties": { + "Device Location": "echo {this_device}", + "Storage Path": "df -h | grep '{this_device} ' | awk '{{print $6}}'", + "Total Space": "df -h | grep '{this_device} ' | awk '{{print $2}}'" + }, + "metrics": { + "Free Space": "df -h | grep '{this_device} ' | awk '{{print $4}}' ", + "Used Space": "df -h | grep '{this_device} ' | awk '{{print $3}}' " + } + }, + { + "name": "DVD", + "description": "{Device Path} is a DVD or Virtual DVD drive.", + "multi_check": "True", + "device_list": "lsblk -d -o NAME,SIZE | grep sr0| awk '{print $1}'", + "properties": { + "Device Name": "echo {this_device}", + "Device Path": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME | awk '{{print \"/dev/\"$1}}' | grep {this_device}", + "Total Capacity": "lsblk -d -o NAME,SIZE | grep {this_device} | awk '{{print $2}}'" + }, + "metrics": { + "placeholder": "" + } } ] \ No newline at end of file diff --git a/files/server/server.php b/files/server/server.php index 21125ff..adb2f4e 100644 --- a/files/server/server.php +++ b/files/server/server.php @@ -1,14 +1,11 @@ Malformed JSON returned from the API.
'); } -/* --------------------- 2. Resolve selected host ------------- */ -$selectedHost = $_GET['host'] ?? ''; -$selectedIdx = null; +// hostname get handler +$selectedId = $_GET['host'] ?? ''; // the value passed in ?host= +$selectedIdx = null; foreach ($clients as $idx => $client) { - if (strtolower($client['hostname']) === strtolower($selectedHost)) { + if (isset($client['short_id']) && $client['short_id'] === $selectedId) { $selectedIdx = $idx; break; } } if ($selectedIdx === null) { - // no match - default to the first host (or none) + // No match – fall back to the first client (or none) $selectedIdx = 0; - $selectedHost = $clients[$selectedIdx]['hostname'] ?? ''; + $selectedId = $clients[$selectedIdx]['short_id'] ?? ''; } $client = $clients[$selectedIdx] ?? null; $properties = $client['client_properties'][0] ?? []; $systemProperties = $properties['system_properties'] ?? []; $systemComponents = $properties['system_components'] ?? []; +$selectedHost = $clients[$selectedIdx]['hostname']; + ?> @@ -76,86 +75,82 @@ $systemComponents = $properties['system_components'] ?? [];This dashboard shows the local Matt-Cloud system stats.
To view the component descriptor, you may
curl -s https://= h($_SERVER['SERVER_NAME']) ?>/descriptor
This will return the entire JSON descriptor variable
-
-
|
-
- Live System Metrics-Connecting...
- |
-
+
|
+ Live System Metrics+ +
+ Connecting...
+
+ |
+