matt/cosmoserver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cosmostat python comment enrichment

Browse Source
This commit is contained in:
Matt
2026-03-29 10:38:07 -07:00
parent 9646ee92fd
commit 6453a839a9
14 changed files with 95 additions and 791 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
files/api/Components.py
View File

@ -1,10 +1,16 @@
# this class file is for the cosmostat service
#################################################################
#################################################################
### Cosmostat Component and System Class
#################################################################
#################################################################
import subprocess
import json
import time
import weakref
import base64, hashlib
from typing import Dict, Any, List
# Import Cosmos Settings
from Cosmos_Settings import *
# Global Class Vars
@ -33,7 +39,13 @@ for entry in component_class_tree:
#################################################################
#################################################################
# Component Class
### Component Class
### Each Component type is defined by the descriptor and built
### as part of the System Class Instantiation
### Each Component Object contains static and dynamic data
### The static data is declared once at instantiation
### THe dynamic data is periodically updated by the application
### where the System Class Object is instantiated
#################################################################
#################################################################
@ -42,8 +54,7 @@ class Component:
############################################################
# instantiate new component
# this_device is set when the component has multiple instances
############################################################
############################################################
def __init__(self, name: str, comp_type: str, parent_system, this_device=None):
# begin init
@ -246,7 +257,7 @@ class Component:
return result
########################################################
# random data functions
# various data functions
########################################################
# complex data type return
@ -301,7 +312,15 @@ class Component:
############################################################
############################################################
# System Class
### System Class
### The System Class uses the Descriptor to build a List
### of Components and interact with the data in a
### useful manner. The System Object is similar to a
### Component Object in that it has Static and Dynamic
### properties, which are populated in a similar manner
### to the Components. In fact, this is designed for the
### System object to update Component Dymanic Metrics
### as part of the same subroutine that updates its own
############################################################
############################################################

56
files/api/Cosmostat.py
View File

@ -1,9 +1,16 @@
# This will be a class definitation for the cosmostat server
# On the server, there will be a Cosmostat Class Object
# This will have an array of System Class Objects
# These will be created based on API input from remote systems
# The remote systems will submit a json of their state to a private API
# this will define the System Class
#################################################################
#################################################################
### Cosmostat Classes
### The Cosmostat Server is a Class for the API running on the
### dashboard. This keeps track of all active systems that are
### actively reporting back to the Cosmostat Server Dashboard
### The static and active data is maintained in a single
### Cosmostat Server Object, and this Object contains a List
### of Cosmostat Client Objects. This is where the data actually
### lives
#################################################################
#################################################################
import subprocess
import json
@ -11,14 +18,15 @@ import time
import weakref
import base64, hashlib
from typing import Dict, Any, List
# Import Cosmos Settings
from Cosmos_Settings import *
#################################################################
#################################################################
# Cosmostat Class
#################################################################
### Cosmostat Server Class
### This Class is for maintaining a list of Client Objects.
### Each Object is a remote System reporting back
### The Class Functions are for the main application to interact
### with client Object data.
#################################################################
class CosmostatServer:
@ -106,22 +114,30 @@ class CosmostatServer:
return result
def get_client_hostnames(self, send_age = False):
self.purge_stale_hostnames()
result = []
for system in self.systems:
if send_age:
result.append({"hostname": system.hostname, "data_age": age})
else:
result.append(system.hostname)
return result
def purge_stale_hostnames(self):
now = time.time()
fresh_systems = []
result = []
for system in self.systems:
age = now - system.data_timestamp
if age <= 60: # keep only fresh servers
fresh_systems.append(system)
if send_age:
result.append({"hostname": system.hostname, "data_age": age})
else:
result.append(system.hostname)
self.systems = fresh_systems # replace the old list
return result
self.systems = fresh_systems # replace the old list
#################################################################
### Cosmostat Client Class
### Each Class Object contains static and active data as well as
### the hostname and uuid/short_id.
### The timestamp is for removing stale Clients
#################################################################
class CosmostatClient:

43
files/api/app.py
View File

@ -1,16 +1,21 @@
#######################################################################
### app.py
### cosmostat service handler
#######################################################################
from flask import Flask, jsonify, request, Response
from flask_apscheduler import APScheduler
from typing import Dict, Union
import json, time, redis, yaml
import base64, hashlib
import secrets, string
import requests
from requests import RequestException, Response
from Components import *
# Import Cosmos Settings
from Cosmos_Settings import *
# System and Component Classes
from Components import *
# Cosmostat server Classes
from Cosmostat import *
# declare flask apps
@ -373,11 +378,11 @@ def client_update():
log_data(log_output = payload, log_level = "noisy_test")
# execute API call
result = client_submission_handler(api_url, payload)
client_initialize()
client_api_initialize()
return result
# Cosmostat Client Initializer
def client_initialize():
def client_api_initialize():
api_url = f"{cosmostat_server_api()}create_client"
# generate payload
payload = get_client_payload(get_php_summary(), "client_properties")
@ -446,34 +451,39 @@ if __name__ == '__main__':
# Background Loop Function
def background_loop():
# Update all data on the System object unless this is the server
if cosmostat_client.check_system_timer() and not run_cosmostat_server():
# Update all data on the local System object
if cosmostat_client.check_system_timer() or run_cosmostat_server():
cosmostat_client.update_system_state()
# publish to redis if the web dashboard is active locally
if app_settings["push_redis"] and not app_settings["disable_local_api"]:
update_redis_server()
# report data to the server if configured
if run_cosmostat_reporter():
if int(time.time()) % 5 == 0 and not cosmostat_client.check_system_timer():
cosmostat_client.update_system_state()
client_update()
# if this is the server, do this stuff
if run_cosmostat_server():
# update the client state since that was skipped
cosmostat_client.update_system_state()
this_client = get_client_payload(get_client_redis_data(human_readable = False), "redis_data")
if app_settings["noisy_test"]:
print(this_client)
run_update_client(this_client)
# purge stale client systems
cosmostat_server.purge_stale_hostnames()
# report the server's own client object to itself
run_update_client(get_client_payload(get_client_redis_data(human_readable = False), "redis_data"))
log_data(log_output = f"{this_client}", log_level = "noisy_test")
time.sleep(0.5)
######################################
# instantiate client
# instantiate client
######################################
# local client System Class Object
cosmostat_client = new_cosmos_client()
# remote client reporter
if app_settings["cosmostat_server_reporter"] and not app_settings["cosmostat_server"]:
client_initialize()
client_api_initialize()
######################################
# instantiate server
@ -517,6 +527,7 @@ if __name__ == '__main__':
if not app_settings["disable_local_api"]:
app.run(debug=False, host=service_gateway_ip(), port=service_api_port())
else:
# if local API disabled, phone home if configured
print("Internal API Disabled.")
while True:
if int(time.time()) % 5 == 0 and not cosmostat_client.check_system_timer():

85
files/api/new_descriptors.json
View File

@ -1,4 +1,7 @@
[
{
"notes:": "this is both a scratch file and a reference for new component descriptors"
},
{
"name": "",
"description": "",
@ -37,88 +40,6 @@
"which have variance"
]
},
{
"static_key_variables": [
{"name": "Hostname", "command": "hostname"},
{"name": "Virtual Machine", "command": "echo $( [ \"$(systemd-detect-virt)\" = none ] && echo False || echo True )", "req_check": "False"},
{"name": "CPU Architecture", "command": "lscpu --json | jq -r '.lscpu[] | select(.field==\"Architecture:\") | .data'"},
{"name": "OS Kernel", "command": "uname -r"},
{"name": "OS Name", "command": "cat /etc/os-release | grep PRETTY | cut -d\\\" -f2"},
{"name": "Manufacturer", "command": "sudo dmidecode --type 1 | grep Manufacturer: | cut -d: -f2 | sed -e 's/^[ \\t]*//'"},
{"name": "Product Name", "command": "sudo dmidecode --type 2 | grep 'Product Name:' | cut -d: -f2 | sed -e 's/^[ \\t]*//'"},
{"name": "Serial Number", "command": "sudo dmidecode --type 2 | grep 'Serial Number: '| cut -d: -f2 | sed -e 's/^[ \\t]*//'"}
],
"dynamic_key_variables": [
{"name": "System Uptime", "command": "uptime -p"},
{"name": "Current Date", "command": "date '+%D %r'"}
],
"virt_ignore": [
"Product Name",
"Serial Number"
]
},
{
"name:": "System",
"static_key_variables": [
{
"name": "Hostname",
"command": "hostname"
},
{
"name": "Virtual Machine",
"command": "echo $( [ \"$(systemd-detect-virt)\" = none ] && echo False || echo True )",
"req_check": "False"
},
{
"name": "CPU Architecture",
"command": "lscpu --json | jq -r '.lscpu[] | select(.field==\"Architecture:\") | .data'"
},
{
"name": "OS Kernel",
"command": "uname -r"
},
{
"name": "OS Name",
"command": "cat /etc/os-release | grep PRETTY | cut -d\\\" -f2"
},
{
"name": "Manufacturer",
"command":{
"x86_64": "sudo dmidecode --type 1 | grep Manufacturer: | cut -d: -f2 | sed -e 's/^[ \\t]*//'"
},
"arch_check": "true"
},
{
"name": "Product Name",
"command": {
"x86_64": "sudo dmidecode --type 2 | grep 'Product Name:' | cut -d: -f2 | sed -e 's/^[ \\t]*//'"
},
"arch_check": "true"
},
{
"name": "Serial Number",
"command": {
"x86_64": "sudo dmidecode --type 2 | grep 'Serial Number: '| cut -d: -f2 | sed -e 's/^[ \\t]*//'"
},
"arch_check": "true"
}
],
"dynamic_key_variables": [
{
"name": "System Uptime",
"command": "uptime -p"
},
{
"name": "Current Date",
"command": "date '+%D %r'"
}
],
"virt_ignore": [
"Product Name",
"Serial Number"
]
},
{
"SATA GBW": "sudo /usr/sbin/smartctl -x --json /dev/{this_device} | jq -r '.physical_block_size as $block |.ata_device_statistics.pages[] | select(.name == \"General Statistics\") | .table[] | select(.name == \"Logical Sectors Written\") | .value as $sectors | ($sectors * $block) / 1073741824 ' | awk '{{printf \"%.2f GiB Written\\n\", $0}}' || true",

46
files/api/shrink.py
View File

@ -1,46 +0,0 @@
def get_properties_keys(self, component = None):
component_properties = []
if component == None:
component_properties = self._properties.items()
else:
component_properties = self.get_property(component)
result = self.process_key_list(key_items = component_properties, key_name = "Property", return_type = "key" key_value = "Value")
return result
def get_metrics_keys(self):
result = self.process_key_list(key_items = self._metrics.items(), key_name = "Metric", key_value = "Data", return_type = "key")
return result
def get_properties_strings(self, return_simple = True):
result = self.process_key_list(key_items = self._properties.items(), key_name = "Property", return_type = "string", return_simple = return_simple)
return result
def get_metrics_strings(self, return_simple = True):
result = self.process_key_list(key_items = self._metrics.items(), key_name = "Metric", return_type = "string", return_simple = return_simple)
return result
def process_key_list(self, key_items: str, key_name: str, return_type: str, key_value = "none"):
result = []
empty_value = ["", "null", None, []]
for name, values in key_items:
for value in (values if isinstance(values, list) else [values]):
if value not in empty_value and name not in self.virt_ignore:
this_key_string = f"{name}: {value}"
if return_simple:
result.append(this_key_string)
elif return_keys:
this_key_value = {
"Source": self.name,
key_name: name,
key_value: value
}
result.append(this_key_value)
else:
complex_key_string = {
"Source": self.name,
key_name: this_key_string
}
result.append(complex_key_string)
return result