matt/cosmoserver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cosmostat working

Browse Source
This commit is contained in:
Matt
2026-03-29 09:39:43 -07:00
parent 97fdb3d5d8
commit 4c4d9e4d6f
19 changed files with 813 additions and 491 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
files/api/Components.py
View File

@ -69,6 +69,7 @@ class Component:
self.virt_ignore = self._descriptor.get('virt_ignore', [])
self.multi_metrics = self._descriptor.get('multi_metrics', [])
self.arch_check = self._descriptor.get('arch_check', [])
self.php_extra_list = self._descriptor.get('php_extra', [])
if self.is_virtual:
self.virt_ignore = []
@ -173,6 +174,13 @@ class Component:
log_data(log_output = f"result - {result_command}", log_level = "debug_output")
return result_command
# check if this property should show in the System Properties box
def check_php_extra(self, property_name):
result = False
if property_name in self.php_extra_list:
result = True
return result
########################################################
# keyed data functions
########################################################
@ -407,7 +415,6 @@ class System:
if multi_check:
log_data(log_output = f"Creating one component of type {component_name} for each one found", log_level = "log_output")
component_type_device_list = get_device_list(component_name)
component_id = 0
for this_device in component_type_device_list:
this_component_ID = component_type_device_list.index(this_device)
this_component_name = f"{component_name} {this_component_ID}"
@ -486,20 +493,36 @@ class System:
result.append(metric)
return result
def get_system_properties(self, human_readable = False):
def get_system_properties(self, human_readable = False, php_extra = False):
result = []
for name, value in self._properties.items():
if human_readable:
result.append({
"Source": "System",
"Property": f"{name}: {value}"
})
})
else:
result.append({
"Source": "System",
"Property": name,
"Value": value
})
if php_extra and human_readable:
for component_result in self.php_component_data():
result.append(component_result)
return result
def php_component_data(self):
result = []
for component in self.components:
for this_property in component._properties:
if component.check_php_extra(this_property):
result_string = f"{this_property}: {component._properties[this_property]}"
result.append({
"Source": "System",
"Property": result_string
})
return result
########################################################
@ -584,7 +607,12 @@ def run_command(cmd, zero_only=False, use_shell=True, req_check = True):
except:
return output_lines
# need to add a archticture checker for this
# i also want to make the loop cleaner
# i don't need to iterate over the component class tree
# to get what I want, i think
def get_device_list(device_type_name: str):
result = []
for component in component_class_tree:
precheck_value = 1
@ -594,6 +622,7 @@ def get_device_list(device_type_name: str):
precheck_value = int(precheck_value_output)
log_data(log_output = f"Precheck found - {precheck_command} - {precheck_value}", log_level = "log_output")
if component["name"] == device_type_name and precheck_value != 0:
device_list_command = component["device_list"]
device_list_result = run_command(device_list_command)
result = device_list_result

11
files/api/Cosmos_Settings.py
View File

@ -1,5 +1,6 @@
import yaml
from urllib.parse import urlparse
import secrets, string
#######################################################################
### Settings Handler Functions
#######################################################################
@ -16,7 +17,11 @@ app_settings = {
"cosmostat_server_reporter": False,
"update_frequency": 1,
"custom_api_port": "5000",
"cosmostat_server_api": "http://10.200.27.20:5000/"
"cosmostat_server_api": "http://10.200.27.20:5000/",
"REAL_API_KEY": ''.join(secrets.choice(string.ascii_letters + string.digits) for _ in range(256)),
"disable_local_api": False,
"local_api_address": "http://10.200.27.20:5000/",
"cosmostat_server_ip": "10.200.27.20"
}
with open('cosmostat_settings.yaml', 'r') as f:
@ -61,9 +66,7 @@ def run_cosmostat_reporter():
def service_gateway_ip():
result = "0.0.0.0"
if cosmostat_settings["cosmostat_server"]:
result = urlparse(cosmostat_settings["cosmostat_server_api"]).hostname
elif cosmostat_settings["secure_api"]:
if cosmostat_settings["secure_api"] and not cosmostat_settings["cosmostat_server"]:
result = cosmostat_bind_ip()
return result

32
files/api/Cosmostat.py
View File

@ -93,15 +93,33 @@ class CosmostatServer:
def get_client_hostname(self, system_uuid: str):
client = self.get_system(system_uuid)
return client.hostname
def get_client_timestamp(self, system_hostname: str):
client = self.get_system(get_uuid_from_hostname(system_hostname))
return client.data_timestamp
def get_client_hostnames(self, send_age = False):
result = []
def get_uuid_from_hostname(self, system_hostname):
result = ""
for system in self.systems:
data_age = time.time() - system.data_timestamp
if int(data_age) > 60:
self.systems.remove(system)
else:
result.append(system.hostname)
if system.hostname == system_hostname:
result = system.uuid
return result
def get_client_hostnames(self, send_age = False):
now = time.time()
fresh_systems = []
result = []
for system in self.systems:
age = now - system.data_timestamp
if age <= 60: # keep only fresh servers
fresh_systems.append(system)
if send_age:
result.append({"hostname": system.hostname, "data_age": age})
else:
result.append(system.hostname)
self.systems = fresh_systems # replace the old list
return result

124
files/api/app.py
View File

@ -4,6 +4,7 @@ from typing import Dict, Union
import json, time, redis, yaml
import base64, hashlib
import secrets, string
import requests
from requests import RequestException, Response
@ -35,7 +36,7 @@ def update_redis_server():
if run_cosmostat_server():
update_redis_channel("client_summary", get_server_redis_data())
update_redis_channel("client_hostnames", get_server_hostnames())
#update_redis_channel("client_hostnames", get_server_hostnames())
# History Redis Tree
# Update history_stats Redis Channel
@ -54,6 +55,7 @@ def get_server_redis_data():
for client in cosmostat_server.systems:
this_client_key = {
"hostname": client.hostname,
"data_timestamp": client.data_timestamp,
"uuid": client.uuid,
"short_id": client.name,
"redis_data": client.redis_data
@ -64,6 +66,7 @@ def get_server_redis_data():
def get_server_hostnames():
return cosmostat_server.get_client_hostnames()
#######################################################################
### Client Flask Routes
#######################################################################
@ -152,7 +155,7 @@ def get_static_data(human_readable = False):
return cosmostat_client.get_static_metrics(human_readable)
def get_php_summary():
system_properties = cosmostat_client.get_system_properties(human_readable = True)
system_properties = cosmostat_client.get_system_properties(human_readable = True, php_extra = True)
system_components = []
for component in cosmostat_client.get_components():
this_component = {
@ -161,6 +164,18 @@ def get_php_summary():
}
system_components.append(this_component)
if run_cosmostat_server():
print(cosmostat_client.name)
client_uuid = cosmostat_server.get_uuid_from_hostname(cosmostat_client.name)
print(client_uuid)
data_timestamp = cosmostat_server.get_system(client_uuid)
print(data_timestamp)
component_age = {
"component_name": "Data Timestamp",
"info_strings": f"Data is {data_timestamp} seconds old"
}
system_components.append(component_age)
result = [{
"system_properties": system_properties,
"system_components": system_components
@ -229,7 +244,17 @@ def client_details():
def client_hostnames():
result = []
if run_cosmostat_server():
result = cosmostat_server.get_client_hostnames()
result = cosmostat_server.get_client_hostnames(send_age = True)
else:
result = {"message": "server not running on this endpoint"}
return jsonify(result)
# api to get server redis data
@app.route('/get_server_redis', methods=['GET'])
def get_server_redis():
result = []
if run_cosmostat_server():
result = get_server_redis_data()
else:
result = {"message": "server not running on this endpoint"}
return jsonify(result)
@ -241,30 +266,48 @@ def client_hostnames():
# update client on server
def run_update_client(this_client):
if not cosmostat_server.check_uuid(this_client["uuid"]):
return { "message": "client not found" }
else:
timestamp_update = cosmostat_server.update_system(system_dictionary = this_client["redis_data"], system_uuid = this_client["uuid"])
update_status = f'updated client {this_client["short_id"]}'
if public_api_check(this_client):
if not cosmostat_server.check_uuid(this_client["uuid"]):
return { "message": "client not found" }
else:
timestamp_update = cosmostat_server.update_system(system_dictionary = this_client["redis_data"], system_uuid = this_client["uuid"])
update_status = f'updated client {this_client["short_id"]}'
return {
"status": update_status,
"uuid": this_client["uuid"],
"redis_data": this_client,
"timestamp_update": timestamp_update
}
return {
"status": update_status,
"uuid": this_client["uuid"],
"redis_data": this_client,
"timestamp_update": timestamp_update
}
else:
return{
"status": "api failure"
}
# create client on server
def run_create_client(this_client):
timestamp_update = cosmostat_server.add_system(system_dictionary = this_client)
update_status = f'created client {this_client["short_id"]}'
return {
"status": update_status,
"uuid": this_client["uuid"],
"client_properties": this_client,
"timestamp_update": timestamp_update
if public_api_check(this_client):
timestamp_update = cosmostat_server.add_system(system_dictionary = this_client)
update_status = f'created client {this_client["short_id"]}'
return {
"status": update_status,
"uuid": this_client["uuid"],
"client_properties": this_client,
"timestamp_update": timestamp_update
}
else:
return{
"status": "api failure"
}
def public_api_check(this_client):
result = False
default_key = ''.join(secrets.choice(string.ascii_letters + string.digits) for _ in range(256))
api_key = this_client.get('API_KEY', default_key)
if api_key == app_settings["REAL_API_KEY"]:
result = True
return result
# flask submission check function
def client_submit_check(request, dict_name: str):
payload = {}
@ -324,6 +367,7 @@ def get_client_details():
# Cosmostat Client Reporter
def client_update():
api_url = f"{cosmostat_server_api()}update_client"
print(api_url)
payload = get_client_payload(get_client_redis_data(human_readable = False), "redis_data")
log_data(log_output = "client_update - redis data from local client:", log_level = "noisy_test")
log_data(log_output = payload, log_level = "noisy_test")
@ -366,7 +410,8 @@ def get_client_payload(system_dictionary: dict, dictionary_name: str):
"uuid": this_uuid,
"short_id": this_short_id,
"hostname": this_hostname,
dictionary_name: system_dictionary
dictionary_name: system_dictionary,
"API_KEY": app_settings["REAL_API_KEY"]
}
return payload
@ -401,25 +446,26 @@ if __name__ == '__main__':
# Background Loop Function
def background_loop():
# Update all data on the System object
if cosmostat_client.check_system_timer():
# Update all data on the System object unless this is the server
if cosmostat_client.check_system_timer() and not run_cosmostat_server():
cosmostat_client.update_system_state()
if app_settings["push_redis"]:
if app_settings["push_redis"] and not app_settings["disable_local_api"]:
update_redis_server()
if run_cosmostat_reporter():
if int(time.time()) % 5 == 0 and not cosmostat_client.check_system_timer():
cosmostat_client.update_system_state()
client_update()
client_update()
if run_cosmostat_server():
# update the client state since that was skipped
cosmostat_client.update_system_state()
this_client = get_client_payload(get_client_redis_data(human_readable = False), "redis_data")
if app_settings["noisy_test"]:
print(this_client)
run_update_client(this_client)
time.sleep(0.5)
######################################
@ -443,14 +489,14 @@ if __name__ == '__main__':
# send initial stats update to redis
######################################
if app_settings["push_redis"]:
if app_settings["push_redis"] and not app_settings["disable_local_api"]:
update_redis_server()
######################################
# Flask scheduler for scanner
######################################
if app_settings["run_background"]:
if app_settings["run_background"] and not app_settings["disable_local_api"]:
log_data(log_output = "Loading flask background subroutine...", log_level = "log_output")
scheduler.add_job(id='background_loop',
@ -467,11 +513,15 @@ if __name__ == '__main__':
######################################
# Flask API
######################################
app.run(debug=False, host=service_gateway_ip(), port=service_api_port())
print(f"gateway: {service_gateway_ip()} - port: {service_api_port()}")
if not app_settings["disable_local_api"]:
app.run(debug=False, host=service_gateway_ip(), port=service_api_port())
else:
print("Internal API Disabled.")
while True:
if int(time.time()) % 5 == 0 and not cosmostat_client.check_system_timer():
cosmostat_client.update_system_state()
client_update()
time.sleep(0.5)

71
files/api/descriptors.json
View File

@ -89,6 +89,9 @@
"arch_variance": [
"current_mhz",
"Clock Speed"
],
"php_extra" :[
"CPU Model"
]
},
{
@ -132,20 +135,23 @@
"RAM Type",
"RAM Speed",
"RAM Voltage"
],
"php_extra" :[
"Total GB"
]
},
{
"name": "LAN",
"description": "{Device Name} - {Device ID} - {MAC Address}",
"description": "{Device Name} - {Device ID}",
"multi_check": "True",
"device_list": "ip link | grep default | grep -v -e docker -e 127.0.0.1 -e br- -e veth -e lo -e tun | cut -d ':' -f 2 | awk '{{print $1}}' ",
"properties": {
"MAC Address": "ip link | grep -A1 ' {this_device}' | grep ether | awk '{{print $2}}'",
"MAC Address": "ip link | grep -A1 ' {this_device}' | grep ether | awk '{{print $2}}' || echo MAC missing",
"Device Name": "echo {this_device}",
"Device ID": "( udevadm info -q property -p $(ls -l /sys/class/net/ | grep {this_device} | cut -d '>' -f2 | cut -b 8-) | grep ID_MODEL_FROM_DATABASE || echo 'ID_MODEL_FROM_DATABASE=missing' ) | cut -d '=' -f2"
},
"metrics": {
"IP Address": "ip -o -4 ad | grep -v -e docker -e 127.0.0.1 -e br- | grep {this_device} | awk '{{print $4}}'",
"IP Address": "ip -o -4 ad | grep -v -e docker -e 127.0.0.1 -e br- -e tun | grep {this_device} | awk '{{print $4}}'",
"Data Transmitted": "ifconfig {this_device} | grep RX | grep bytes | cut -d '(' -f2 | tr -d ')'",
"Data Received": "ifconfig {this_device} | grep TX | grep bytes | cut -d '(' -f2 | tr -d ')'",
"Link State": "cat /sys/class/net/{this_device}/operstate",
@ -155,13 +161,28 @@
"IP Address"
]
},
{
"name": "VPN",
"description": "{Device Name} - VPN Tunnel",
"multi_check": "True",
"precheck": "ip link | grep tun | wc -l",
"device_list": "ip link | grep default | grep tun | cut -d ':' -f 2 | awk '{{print $1}}' ",
"properties": {
"Device Name": "echo {this_device}"
},
"metrics": {
"IP Address": "ip -o -4 ad | grep -v -e docker -e 127.0.0.1 -e br- | grep {this_device} | awk '{{print $4}}'",
"Data Transmitted": "ifconfig {this_device} | grep RX | grep bytes | cut -d '(' -f2 | tr -d ')'",
"Data Received": "ifconfig {this_device} | grep TX | grep bytes | cut -d '(' -f2 | tr -d ')'"
}
},
{
"name": "NVGPU",
"description": "NVGPU{Device ID} - {Device Model} with {Memory Size}, Max Power {Maximum Power}",
"description": "NVGPU{Device ID} - {GPU Model} with {Memory Size}, Max Power {Maximum Power}",
"multi_check": "True",
"device_list": "nvidia-smi --query-gpu=index --format=csv,noheader,nounits",
"properties": {
"Device Model": "nvidia-smi --id={this_device} --query-gpu=name --format=csv,noheader,nounits",
"GPU Model": "nvidia-smi --id={this_device} --query-gpu=name --format=csv,noheader,nounits",
"Device ID": "echo NVGPU{this_device}",
"Driver Version": "nvidia-smi --id={this_device} --query-gpu=driver_version --format=csv,noheader,nounits",
"Maximum Power": "nvidia-smi --id={this_device} --query-gpu=power.limit --format=csv,noheader,nounits",
@ -175,16 +196,19 @@
"GPU Load": "nvidia-smi --id={this_device} --query-gpu=utilization.gpu --format=csv,noheader,nounits"
},
"precheck": "lspci | grep NVIDIA | wc -l"
"precheck": "lspci | grep NVIDIA | wc -l",
"php_extra" :[
"GPU Model"
]
},
{
"name": "STOR",
"description": "{Device Path} is of type {Drive Type} with capacity of {Total Capacity}.",
"multi_check": "True",
"device_list": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME | awk '{print $1}'",
"device_list": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME -e sr0| awk '{print $1}'",
"properties": {
"Device Name": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME | awk '{{print $1}}' | grep {this_device}",
"Device Path": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME | awk '{{print \"/dev/\"$1}}' | grep {this_device}",
"Device Name": "echo {this_device}",
"Device Path": "echo /dev/{this_device}",
"Drive Type": "lsblk -d -o NAME,TRAN | grep {this_device} | awk '{{print ($2 != \"\" ? $2 : \"missing\")}}'",
"Total Capacity": "lsblk -d -o NAME,SIZE | grep {this_device} | awk '{{print $2}}'",
"SMART Check": "sudo /usr/sbin/smartctl -x --json /dev/{this_device} | jq -r .smart_status.passed"
@ -192,5 +216,34 @@
"metrics": {
"placeholder": ""
}
},
{
"name": "MOUNT",
"description": "Storage device {Device Location} mounted at {Storage Path} with {Total Space} total space",
"multi_check": "True",
"device_list": "df -h | grep -v -e 'Use%' -e tmpfs -e overlay -e efi -e udev | awk '{{print $1}}'",
"properties": {
"Device Location": "echo {this_device}",
"Storage Path": "df -h | grep '{this_device} ' | awk '{{print $6}}'",
"Total Space": "df -h | grep '{this_device} ' | awk '{{print $2}}'"
},
"metrics": {
"Free Space": "df -h | grep '{this_device} ' | awk '{{print $4}}' ",
"Used Space": "df -h | grep '{this_device} ' | awk '{{print $3}}' "
}
},
{
"name": "DVD",
"description": "{Device Path} is a DVD or Virtual DVD drive.",
"multi_check": "True",
"device_list": "lsblk -d -o NAME,SIZE | grep sr0| awk '{print $1}'",
"properties": {
"Device Name": "echo {this_device}",
"Device Path": "lsblk -d -o NAME,SIZE | grep -v -e 0B -e NAME | awk '{{print \"/dev/\"$1}}' | grep {this_device}",
"Total Capacity": "lsblk -d -o NAME,SIZE | grep {this_device} | awk '{{print $2}}'"
},
"metrics": {
"placeholder": ""
}
}
]