matt/cosmoserver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

web dashboard in single container

Browse Source
This commit is contained in:
Matt
2026-04-05 01:50:41 -07:00
parent a89703c420
commit c6007d9c33
26 changed files with 460 additions and 365 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
defaults/main.yaml
View File

@ -68,10 +68,12 @@ refresh_special: false
special_server: "none"
# cosmostat_settings, will be for special_server defaults
# the special_server install will be the full stack with public visibility
# regular installs will be handled through specific pipelines
noisy_test: false
debug_output: false
secure_api: true
push_redis: false
secure_api: false
push_redis: true
run_background : true
log_output: true
update_frequency: "1"
@ -79,6 +81,6 @@ cosmostat_server: false
cosmostat_server_api: "https://cosmostat.matt-cloud.com/"
local_api_address: "http://{{ cosmostat_server_ip }}:{{ custom_api_port }}/"
cosmostat_server_reporter: true
# setting this to true for default install
disable_local_dashboard: true
# setting this to false for default install
disable_local_dashboard: false
...

164
files/api/Components.py
View File

@ -13,6 +13,7 @@ import ipaddress
from typing import Dict, Any, List
# Import Cosmos Settings
from Cosmos_Settings import *
from Helpers import *
# Global Class Vars
global_max_length = 500
@ -158,6 +159,7 @@ class Component:
# iterate over all properties to process descriptor
def _process_properties(self):
for key, command in self._descriptor.get('properties', {}).items():
# if this is a component that can be multi, then it might need to return a list
return_string = True
if key in self.multi_metrics:
return_string = False
@ -168,6 +170,8 @@ class Component:
self._properties[key] = result
# helper function to parse command key
# this is for substituting variables in descriptor commands when there are multi things
# think needing to grep eth0 or grep eth1 to get the same metric for a different component
def _parse_command(self, key: str, command: str | list[str], return_string = True):
result_command = command
log_data(log_output = f"_parse_command - {command}", log_level = "debug_output")
@ -187,6 +191,8 @@ class Component:
return result_command
# check if this property should show in the System Properties box
# these are intended to provide a summary of critical things that all computers have
# like memory and cpu and cores
def check_php_extra(self, property_name):
result = False
if property_name in self.php_extra_list:
@ -195,6 +201,7 @@ class Component:
########################################################
# keyed data functions
# various ways to query the component data
########################################################
def get_properties_keys(self, component = None):
@ -270,73 +277,6 @@ class Component:
else:
return result
########################################################
# various data functions
########################################################
# complex data type return
def get_metrics(self, type = None):
these_metrics = []
if type == None:
for name, value in self._metrics:
these_metrics.append({"Metric": name, "Data": value})
else:
for name, value in self._metrics:
if name == type:
these_metrics.append({"Metric": name, "Data": value})
result = {
"Source": self.name,
"Component Type": self.type,
"Metrics": these_metrics
}
return result
# complex data type return
def get_property_summary(self, type = None):
these_properties = []
if type == None:
for name, value in self._properties.items():
these_properties.append({"Property": name, "Value": value})
else:
for name, value in self._properties.items():
if type in name:
these_properties.append({"Property": name, "Value": value})
result = {
"Source": self.name,
"Component Type": self.type,
"Properties": these_properties
}
return result
# simple data value return
def get_property_value(self, type = None):
result = []
print(f"Component type: {type}")
for name, value in self._properties.items():
print(f"Component Property: {name}")
if type in name:
result.append(value)
if len(result) == 1:
return result[0]
else:
return result
# full data return
def get_description(self):
these_properties = []
for name, value in self._properties.items():
these_properties.append({"Property": name, "Value": value})
these_metrics = []
for name, value in self._metrics.items():
these_metrics.append({"Metric": name, "Data": value})
result = {
"Source": self.name,
"Type": self.type,
"Properties": these_properties,
"Metrics": these_metrics
}
return result
############################################################
############################################################
### System Class
@ -396,7 +336,7 @@ class System:
# load static keys
for static_key in self.static_key_variables:
if static_key["name"] not in self._virt_ignore:
self.process_property(static_key = static_key)
self._process_property(static_key = static_key)
# initialize live keys
self.update_live_keys()
# initialze components
@ -422,8 +362,8 @@ class System:
# critical class functions
########################################################
# process static keys
def process_property(self, static_key):
# process static properties, done once at instantiation
def _process_property(self, static_key):
command = static_key["command"]
if "arch_check" in static_key:
arch_string = run_command("lscpu --json | jq -r '.lscpu[] | select(.field==\"Architecture:\") | .data'", zero_only = True)
@ -439,7 +379,7 @@ class System:
if result not in null_result:
self._properties[static_key["name"]] = result
# update only system dynamic keys
# update only system dynamic keys, done periodically
def update_live_keys(self):
for live_key in self.dynamic_key_variables:
if live_key['command'] is not None:
@ -457,25 +397,53 @@ class System:
# component creation helper
def create_component(self, component):
# this is the type, i.e. CPU MEM LAN etc
component_name = component["name"]
# if there can be multiples, i.e.LAN STOR GPU etc
multi_check = component["multi_check"]
# if multi, note that the command in device_list creates the list of things to pipe into this_device
if multi_check:
log_data(log_output = f"Creating one component of type {component_name} for each one found", log_level = "log_output")
component_type_device_list = get_device_list(component_name)
# build list of components of this type to initialize
component_type_device_list = self.get_device_list(component_name)
for this_device in component_type_device_list:
this_component_ID = component_type_device_list.index(this_device)
this_component_name = f"{component_name} {this_component_ID}"
log_data(log_output = f"{this_component_name} - {component_name} - {this_device}", log_level = "debug_output")
new_component = Component(name = this_component_name, comp_type = component_name, this_device = this_device, parent_system = self)
# add new component to system
self.components.append(new_component)
# if it's just a single thing like CPU RAM then just do one device
else:
log_data(log_output = f'Creating component {component["name"]}', log_level = "debug_output")
new_component = Component(name = component_name, comp_type = component_name, parent_system = self)
# add new component to system
self.components.append(new_component)
# helper function - component type list builder for multi check items
def get_device_list(self, device_type_name: str):
result = []
for component in component_class_tree:
# pre-set to 1 for true value by default
precheck_value = 1
# the precheck is critical, i.e there can be 0 or 1 or 2 GPU
if "precheck" in component:
precheck_command = component["precheck"]
precheck_value_output = run_command(precheck_command, zero_only = True)
precheck_value = int(precheck_value_output)
log_data(log_output = f"Precheck found - {precheck_command} - {precheck_value}", log_level = "log_output")
# build a list of devices if precheck passes, this can be a list of 1 like a single GPU system
if component["name"] == device_type_name and precheck_value != 0:
device_list_command = component["device_list"]
device_list_result = run_command(device_list_command)
result = device_list_result
return result
# return the IP of the interface with the gateway
# remember the IP is stored like this 192.168.1.0/24
# this data is pulled from the system properties class data
# this was built for the update inventory generation
# but is not the complete picture
def get_primary_ip(self):
primary_ip = None
interfaces = self.get_components(component_type = "LAN")
@ -577,6 +545,7 @@ class System:
return result
# helper function for system properties for dashboard rendering
def php_component_data(self):
result = []
for component in self.components:
@ -653,57 +622,6 @@ class System:
})
return result
############################################################
# Non-class Helper Functions
############################################################
# subroutine to run a command, return stdout as array unless zero_only then return [0]
def run_command(cmd, zero_only=False, use_shell=True, req_check = True):
# Run the command and capture the output
result = subprocess.run(cmd, shell=use_shell, check=req_check, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# Decode the byte output to a string
output = result.stdout.decode('utf-8')
# Split the output into lines and store it in an array
output_lines = [line for line in output.split('\n') if line]
# Return result
try:
return output_lines[0] if zero_only else output_lines
except:
return output_lines
# need to add a archticture checker for this
# i also want to make the loop cleaner
# i don't need to iterate over the component class tree
# to get what I want, i think
def get_device_list(device_type_name: str):
result = []
for component in component_class_tree:
precheck_value = 1
if "precheck" in component:
precheck_command = component["precheck"]
precheck_value_output = run_command(precheck_command, zero_only = True)
precheck_value = int(precheck_value_output)
log_data(log_output = f"Precheck found - {precheck_command} - {precheck_value}", log_level = "log_output")
if component["name"] == device_type_name and precheck_value != 0:
device_list_command = component["device_list"]
device_list_result = run_command(device_list_command)
result = device_list_result
return result
# subnet helper app
def is_ip_in_subnets(ip, subnet):
try:
ip_obj = ipaddress.IPv4Address(ip)
subnet_obj = ipaddress.IPv4Network(subnet)
if ip_obj in subnet_obj:
return True
return False
except ValueError as e:
# If the IP address is not valid, raise an error
return False

14
files/api/Cosmostat.py
View File

@ -21,6 +21,7 @@ import base64, hashlib
from typing import Dict, Any, List
# Import Cosmos Settings
from Cosmos_Settings import *
from Helpers import *
#################################################################
### Cosmostat Server Class
@ -197,16 +198,3 @@ class CosmostatClient:
def get_redis(self):
return self.redis_data
# subnet helper app
def is_ip_in_subnets(ip, subnet):
try:
ip_obj = ipaddress.IPv4Address(ip)
subnet_obj = ipaddress.IPv4Network(subnet)
if ip_obj in subnet_obj:
return True
return False
except ValueError as e:
# If the IP address is not valid, raise an error
return False

30
files/api/Helpers.py Normal file
View File

@ -0,0 +1,30 @@
import subprocess
import ipaddress
from typing import Dict, Any, List
# subnet helper app
def is_ip_in_subnets(ip, subnet):
try:
ip_obj = ipaddress.IPv4Address(ip)
subnet_obj = ipaddress.IPv4Network(subnet)
if ip_obj in subnet_obj:
return True
return False
except ValueError as e:
# If the IP address is not valid, raise an error
return False
# subroutine to run a command, return stdout as array unless zero_only then return [0]
def run_command(cmd, zero_only=False, use_shell=True, req_check = True):
# Run the command and capture the output
result = subprocess.run(cmd, shell=use_shell, check=req_check, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# Decode the byte output to a string
output = result.stdout.decode('utf-8')
# Split the output into lines and store it in an array
output_lines = [line for line in output.split('\n') if line]
# Return result
try:
return output_lines[0] if zero_only else output_lines
except:
return output_lines

349
files/api/app.py
View File

@ -29,11 +29,12 @@ scheduler = APScheduler()
# Redis client - will publish updates
r = redis.Redis(host=redis_gateway_ip(), port=6379)
def update_redis_channel(redis_channel, data):
# Publish to the specified Redis channel
def update_redis_channel(redis_channel, data):
r.publish(redis_channel, json.dumps(data))
log_data(log_output = data, log_level = "noisy_test")
# update redis server for both modes
def update_redis_server():
# Client Redis Tree
if cosmostat_client.check_system_timer():
@ -41,18 +42,12 @@ def update_redis_server():
if run_cosmostat_server():
update_redis_channel("client_summary", get_server_redis_data())
#update_redis_channel("client_hostnames", get_server_hostnames())
# History Redis Tree
# Update history_stats Redis Channel
# update_redis_channel("history_stats", get_component_list())
# function helper for both modes
def get_client_redis_data(human_readable = False):
result = []
for metric in get_dynamic_data(human_readable):
result.append(metric)
#for metric in get_static_data(human_readable):
# result.append(metric)
return result
def get_server_redis_data():
@ -70,70 +65,25 @@ def get_server_redis_data():
result.append(this_client_key)
return result
def get_server_hostnames():
return cosmostat_server.get_client_hostnames()
#######################################################################
### Client Flask Routes
#######################################################################
# dynamic data
# this will go to the redis server
@app.route('/dynamic_data', methods=['GET'])
def dynamic_data():
return jsonify(get_dynamic_data())
# static data
@app.route('/static_data', methods=['GET'])
def static_data():
return jsonify(get_static_data())
# redis data
@app.route('/redis_data', methods=['GET'])
def redis_data():
return jsonify(get_client_redis_data(human_readable = False))
# redis strings
@app.route('/redis_strings', methods=['GET'])
def redis_strings():
return jsonify(get_client_redis_data(human_readable = True))
# php summary
@app.route('/php_summary', methods=['GET'])
def php_summary():
return jsonify(get_php_summary())
# return full descriptor
@app.route('/descriptor', methods=['GET'])
def descriptor():
return jsonify(get_descriptor())
# socket timer handler
@app.route('/start_timer', methods=['GET'])
def start_timer():
current_timestamp = int(time.time())
cosmostat_client.recent_check = current_timestamp
log_data(log_output = f"Timestamp updated to {cosmostat_client.recent_check}", log_level = "noisy_test")
return jsonify(
{
return jsonify({
"message": "websocket timer reset",
"new_timestamp": cosmostat_client.recent_check
}
)
# socket timer data
@app.route('/timer_data', methods=['GET'])
def timer_data():
time_now = time.time()
time_lapsed = time_now - float(cosmostat_client.recent_check)
result = {
"Time Lapsed": time_lapsed,
"Current Time Value": time_now,
"Last Update Value": float(cosmostat_client.recent_check),
"System Updating": cosmostat_client.check_system_timer()
}
return jsonify(result)
})
# test route
@app.route('/test', methods=['GET'])
@ -157,70 +107,63 @@ def test():
def get_dynamic_data(human_readable = False):
return cosmostat_client.get_live_metrics(human_readable)
def get_static_data(human_readable = False):
result = []
return cosmostat_client.get_static_metrics(human_readable)
# this generates the static data used by the client dashboard and the CosmostatServer properties
def get_php_summary():
# separate lists for system properties and components
system_properties = cosmostat_client.get_system_properties(human_readable = True, php_extra = True)
system_components = []
# build a list of Component property lists
for component in cosmostat_client.get_components():
this_component = {
"component_name": component.name,
"info_strings": component.get_properties_strings(return_simple = True)
"info_strings": component.get_properties_strings(return_simple = True) # this is a list of strings from the Class
}
system_components.append(this_component)
this_primary_ip = cosmostat_client.primary_ip
# if this is the server there are extra steps here to add all client data to the return
# this can make for a big json
if run_cosmostat_server():
# get uuid for reference
client_uuid = cosmostat_server.get_uuid_from_hostname(cosmostat_client.name)
# get timestamp for bubble
data_timestamp = cosmostat_server.get_system(client_uuid)
log_string = f"Cosmostat Client Name: {cosmostat_client.name} - UUID: {client_uuid}- Timestamp: {data_timestamp}"
log_data(log_output = log_string, log_level = "log_output")
component_age = {
"component_name": "Data Timestamp",
"info_strings": f"Data is {data_timestamp} seconds old"
}
# get the primary IP for inventory file generation
this_primary_ip = cosmostat_settings["cosmostat_server_ip"]
system_components.append(component_age)
# build and return dict of all these lists
result = [{
"system_properties": system_properties,
"system_components": system_components,
"active_interface": this_primary_ip
}]
return result
def get_descriptor():
return cosmostat_client.get_component_class_tree()
def generate_state_definition():
result = {
"uuid": cosmostat_client.uuid,
"state_definition": get_php_summary()
}
return result
#######################################################################
### Server Flask Routes
#######################################################################
# update client on server
# return full descriptor - public route
@app.route('/descriptor', methods=['GET'])
def descriptor():
return jsonify(cosmostat_client.get_component_class_tree())
# update client on server - public route
@app.route('/update_client', methods=['POST'])
def update_client():
result = {}
# check the request and return payload dict {} if all good
payload = client_submit_check(request = request, dict_name = "redis_data")
# sanitize and validate POST data
payload = client_submit_check(request = request)
# process client update request
result = run_update_client(payload)
return jsonify(result), 200
# create client on server
# create client on server - public route
@app.route('/create_client', methods=['POST'])
def create_client():
result = {}
# check the request and return payload dict {} if all good
payload = client_submit_check(request = request, dict_name = "client_properties")
# sanitize and validate POST data
payload = client_submit_check(request = request)
# if the client does not exist, create it
if not cosmostat_server.check_uuid(payload["uuid"]):
result = run_create_client(payload)
@ -231,17 +174,7 @@ def create_client():
}
return jsonify(result), 200
# api to validate Cosmostat Class
@app.route('/client_summary', methods=['GET'])
def client_summary():
result = []
if run_cosmostat_server():
result = get_client_summary()
else:
result = {"message": "server not running on this endpoint"}
return jsonify(result)
# api to pull all data
# api to pull all data for server dashboard rendering
@app.route('/client_details', methods=['GET'])
def client_details():
result = []
@ -251,36 +184,6 @@ def client_details():
result = {"message": "server not running on this endpoint"}
return jsonify(result)
# api to get all hostnames
@app.route('/client_hostnames', methods=['GET'])
def client_hostnames():
result = []
if run_cosmostat_server():
result = cosmostat_server.get_client_hostnames(send_age = True)
else:
result = {"message": "server not running on this endpoint"}
return jsonify(result)
# api to get server redis data
@app.route('/get_server_redis', methods=['GET'])
def get_server_redis():
result = []
if run_cosmostat_server():
result = get_server_redis_data()
else:
result = {"message": "server not running on this endpoint"}
return jsonify(result)
# api to get server redis data
@app.route('/client_ip_summary', methods=['GET'])
def client_ip_summary():
result = []
if run_cosmostat_server():
result = get_client_ip_summary()
else:
result = {"message": "server not running on this endpoint"}
return jsonify(result)
# return inventory file of all clients for update
@app.route("/client_inventory", methods=["GET"])
def client_inventory():
@ -290,15 +193,27 @@ def client_inventory():
### Server Flask Helpers
#######################################################################
# update client on server
# this is the helper function for processing client submission requests
# this only updates the dynamic metrics
# the output of this function is returned to the flask function
# This function is also used to report the cosmostat server's CosmostatClient
def run_update_client(this_client):
# this is where the API check is done
if public_api_check(this_client):
# return if object does not exist
if not cosmostat_server.check_uuid(this_client["uuid"]):
return { "message": "client not found" }
else:
timestamp_update = cosmostat_server.update_system(system_dictionary = this_client["redis_data"], system_uuid = this_client["uuid"])
# this is where the CosmostatClient object is updated
# done by passing the UUID and redis data
# that Class Function returns the timestamp
timestamp_update = cosmostat_server.update_system(
system_dictionary = this_client["redis_data"],
system_uuid = this_client["uuid"]
)
# after this happens, record a successful status
update_status = f'updated client {this_client["short_id"]}'
# build a return string and send it
return {
"status": update_status,
"client_updated": "True",
@ -306,17 +221,25 @@ def run_update_client(this_client):
"redis_data": this_client,
"timestamp_update": timestamp_update
}
# if the API check fails, return as much
else:
return{
"status": "api failure"
}
# create client on server
# this is the helper function for processing client creation requests
# this creates a new CosmostatClient object on the 'cosmostat_server' Object
# the output of this function is returned to the flask function
def run_create_client(this_client):
# this is where the API check is done
if public_api_check(this_client):
#this_client["active_interface"] = cosmostat_client.primary_ip
# this is where the CosmostatClient object is inialized
# done by passing the UUID and all static data
# that Class Function returns the timestamp
timestamp_update = cosmostat_server.add_system(system_dictionary = this_client)
# after this happens, record a successful status
update_status = f'created client {this_client["short_id"]}'
# build a return string and send it
return {
"status": update_status,
"client_created": "True",
@ -324,11 +247,13 @@ def run_create_client(this_client):
"client_properties": this_client,
"timestamp_update": timestamp_update
}
# if the API check fails, return as much
else:
return{
"status": "api failure"
}
# API check helper
def public_api_check(this_client):
result = False
default_key = ''.join(secrets.choice(string.ascii_letters + string.digits) for _ in range(256))
@ -338,9 +263,12 @@ def public_api_check(this_client):
return result
# flask submission check function
def client_submit_check(request, dict_name: str):
# all this does is take the POST data from
# remote client submissions and sanitize and validate
# it returns the jsonified submission after that
def client_submit_check(request):
payload = {}
required_keys = {"uuid", "short_id", "hostname", dict_name}
required_keys = {"uuid", "short_id", "hostname"}
if not request.is_json:
logging.warning("Received non-JSON request")
return jsonify({"error": "Content-type must be application/json"}), 400
@ -351,25 +279,8 @@ def client_submit_check(request, dict_name: str):
missing = required_keys - payload.keys()
if missing:
raise ValueError(f"Missing required keys: {', '.join(sorted(missing))}")
return payload
# generate cosmostat server summary
def get_client_summary():
result = []
for client in cosmostat_server.systems:
data_age = time.time() - client.data_timestamp
this_client = {
"uuid": client.uuid,
"short_id": client.name,
"data_age": data_age,
"hostname": client.hostname
}
result.append(this_client)
if result == []:
result = {"message": "no clients reporting"}
return result
# no redis data needed here
def get_client_details():
result = []
@ -387,27 +298,29 @@ def get_client_details():
result = {"message": "no clients reporting"}
return result
# get client IP summary
# get client IP summary either in a Dict or List
def get_client_ip_summary(return_list = False):
result = []
for client in cosmostat_server.systems:
# build the Dict
device_info = {
"small_id": client.name,
"uuid": client.uuid,
"hostname": client.hostname,
"lan_ip": client.active_ip,
}
# build the result depending on mode
if client.hostname != cosmostat_server.hostname:
if return_list:
result.append(client.active_ip)
result.append(client.active_ip) # only the list
else:
result.append(device_info)
result.append(device_info) # the full dictionary
return result
# build client ansible inventory file
def build_inventory():
all_ips = get_client_ip_summary(return_list = True)
# subnets reachable from the cosmostat server
cosmos_subnets = [
"172.25.1.0/24",
"172.20.0.0/16",
@ -416,9 +329,10 @@ def build_inventory():
"10.200.27.0/24",
"192.168.60.0/24",
]
# inialize lists
ips = []
bad_ips = []
# build list of reachable IPs
# build list of active IPs
for ip in all_ips:
for subnet in cosmos_subnets:
if is_ip_in_subnets(ip, subnet) and cosmostat_server.get_vpn_ip(ip, just_check = True):
@ -427,7 +341,7 @@ def build_inventory():
for ip in all_ips:
if ip not in ips:
bad_ips.append(ip)
# add any VPN IPs for bad IPs to the main IP list
# any system with a VPN to jenkins, add to inventory
for ip in bad_ips:
ips.append(cosmostat_server.get_vpn_ip(ip))
hosts = {ip: {"ansible_host": ip} for ip in ips}
@ -470,52 +384,42 @@ def build_inventory():
#######################################################################
### Cosmostat Client Subroutines
### These are only for the clients
### The cosmostat server has its own functions
#######################################################################
# Cosmostat Client Reporter Handler
# this handles updating the active data for the local client instance
# to the remote cosmostat server
def client_update():
# update client API path
api_url = f"{cosmostat_server_api()}update_client"
payload = get_client_payload(get_client_redis_data(human_readable = False), "redis_data")
log_data(log_output = f"API Update Called - {api_url}", log_level = "debug_output")
log_data(log_output = "client_update - redis data from local client:", log_level = "noisy_test")
log_data(log_output = payload, log_level = "noisy_test")
# execute API call
# execute API call, we do need the result
result = client_submission_handler(api_url, payload)
# if this client does not exist on the server, we need to try to initialize it
if not result or not result.get('client_updated'):
log_data(log_output = f"Client not updated, initializing", log_level = "log_output")
result = client_api_initialize()
# this result does not matter and is not used anywhere
return result
client_api_initialize()
# Cosmostat Client Initializer
# this handles inializing the CosmostatClient object on the remote cosmostat server
def client_api_initialize():
# create client API path
api_url = f"{cosmostat_server_api()}create_client"
# generate payload
# generate payload for the remote server
payload = get_client_payload(get_php_summary(), "client_properties")
# execute API call
# add extra init vars
payload["active_interface"] = cosmostat_client.primary_ip
payload["is_server"] = False
# execute API call on remote server
client_submission_handler(api_url, payload)
result = client_submission_handler(api_url, payload)
return result
# Cosmostat Client API Reporting Handler
def client_submission_handler(api_url: str, payload: dict):
result = None
try:
# `json=` automatically sets Content-Type to application/json
response: Response = requests.post(api_url, json=payload, timeout=4)
response.raise_for_status() # raise HTTPError for 4xx/5xx
except RequestException as exc:
# Wrap the low-level exception in a more descriptive one
log_data(log_output = f"Failed to POST to {api_url!r}: {exc}", log_level = "log_output")
# process reply from API
try:
result = response.json()
except ValueError as exc:
log_data(log_output = f"Server responded with non-JSON payload: {response.text!r}", log_level = "log_output")
return result
# this returns the payload in the format that the API expects with extra necessary data
# used for both initializing and updating; dictionary name is part of the function
def get_client_payload(system_dictionary: dict, dictionary_name: str):
this_uuid = cosmostat_client.uuid
this_short_id = cosmostat_client.short_id
@ -530,6 +434,26 @@ def get_client_payload(system_dictionary: dict, dictionary_name: str):
}
return payload
# Cosmostat Client API Reporting Handler
# this helper makes the POST request to the remote cosmostat server
# the payload and URL are passed to this function
def client_submission_handler(api_url: str, payload: dict):
result = None
try:
# this is the actual submission to the server
response: Response = requests.post(api_url, json=payload, timeout=4)
# raise HTTPError for 4xx/5xx if failure
response.raise_for_status()
except RequestException as exc:
# Wrap the low-level exception in a more descriptive one
log_data(log_output = f"Failed to POST to {api_url!r}: {exc}", log_level = "log_output")
# process reply from API
try:
result = response.json()
except ValueError as exc:
log_data(log_output = f"Server responded with non-JSON payload: {response.text!r}", log_level = "log_output")
return result
#######################################################################
#######################################################################
### Main Subroutine
@ -538,12 +462,15 @@ def get_client_payload(system_dictionary: dict, dictionary_name: str):
if __name__ == '__main__':
######################################
### Main Functions
######################################
log_data(log_output = f"Main Function Start", log_level = "log_output")
######################################
# Main function helpers
#
# instantiate and return the Client System object
def new_cosmos_client():
new_client = System(f"{jenkins_hostname_settings()}")
@ -552,12 +479,13 @@ if __name__ == '__main__':
log_data(log_output = component.description, log_level = "log_output")
return new_client
# instantiate and return the Cosmoserver System object
# instantiate and return the CosmostatServer System object
def new_cosmostat_server():
new_server = CosmostatServer(name = cosmostat_client.uuid, hostname = jenkins_hostname_settings())
log_data(log_output = f"New Cosmostat serverobject name: {new_server.name}", log_level = "log_output")
return new_server
# helper function for checking if we should update the local System Object this time
def update_local_system():
result = False
if cosmostat_client.check_system_timer():
@ -569,8 +497,10 @@ if __name__ == '__main__':
return result
# Background Loop Function
# That makes this the service loop
def background_loop():
# Update all data on the local System object
# This is done on both client and server
if update_local_system():
cosmostat_client.update_system_state()
@ -579,6 +509,7 @@ if __name__ == '__main__':
update_redis_server()
# report data to the server if configured
# this is the client-only update function
if run_cosmostat_reporter():
if cosmostat_client.check_system_timer or int(time.time()) % 5 == 0:
client_update()
@ -588,53 +519,64 @@ if __name__ == '__main__':
# purge stale client systems
cosmostat_server.purge_stale_hostnames()
# report the server's own client object to itself
# uses the helper function upstream of the API route
# since the route is already here
run_update_client(get_client_payload(get_client_redis_data(human_readable = False), "redis_data"))
log_data(log_output = f"{this_client}", log_level = "noisy_test")
# short sleep time to keep things chill
time.sleep(0.2)
######################################
# instantiate client
#
# End Main Functions
######################################
# local client System Class Object
#######################################################################
#######################################################################
###
### Here is where this whole thing actually starts doing something
###
#######################################################################
#######################################################################
# First thing is to create the local Cosmostat System
log_data(log_output = f"Cosmostat Client Start", log_level = "log_output")
# Here is the Object Instatiation Command
cosmostat_client = new_cosmos_client()
# Update the system state on new object
cosmostat_client.update_system_state()
# remote client reporter
# If the option was set to phone home, initialize the remote CosmostatClient Object
if app_settings["cosmostat_server_reporter"] and not app_settings["cosmostat_server"]:
log_data(log_output = f"Initialize Client Reporter", log_level = "log_output")
client_api_initialize()
######################################
# instantiate server
######################################
# If this is the server, this will make it the server
# call the object outside the if
cosmostat_server = None
if run_cosmostat_server():
log_data(log_output = f"Cosmostat Server Start", log_level = "log_output")
# instantiate it if so
cosmostat_server = new_cosmostat_server()
this_client = get_client_payload(get_php_summary(), "client_properties")
# extra initial vars
# any extra data that should be associated with a CosmostatClient Object should be added here
this_client["active_interface"] = cosmostat_settings["cosmostat_server_ip"]
this_client["is_server"] = True
# add the server itself as a CosmostatClient object
# use the same upstream function as the API endpoint
timestamp_update = cosmostat_server.add_system(system_dictionary = this_client)
# if not a server, update client on the server
else:
client_update()
######################################
# send initial stats update to redis
######################################
if app_settings["push_redis"] and not app_settings["disable_local_dashboard"]:
log_data(log_output = f"Initial Redis Push", log_level = "log_output")
update_redis_server()
######################################
# Flask scheduler for scanner
######################################
# Flask scheduler for background loop, run if requested
# this is not needed for the API routes
if app_settings["run_background"] and not app_settings["disable_local_dashboard"]:
log_data(log_output = f"Background Function Initializing", log_level = "log_output")
log_data(log_output = "Loading flask background subroutine...", log_level = "log_output")
@ -645,26 +587,23 @@ if __name__ == '__main__':
seconds=app_settings["update_frequency"])
scheduler.init_app(app)
scheduler.start()
log_data(log_output = "...Done", log_level = "log_output")
else:
log_data(log_output = "Skipping flask background task", log_level = "log_output")
######################################
# Flask API
######################################
# Flask API, run if requested
log_data(log_output = f"gateway: {service_gateway_ip()} - port: {service_api_port()}", log_level = "log_output")
if not app_settings["disable_local_dashboard"]:
log_data(log_output = f"Main API Start", log_level = "log_output")
app.run(debug=False, host=service_gateway_ip(), port=service_api_port())
else:
# if local API disabled, phone home if configured
# if the flask app is not ran, then the background scheduler also does not run
log_data(log_output = f"Internal API Disabled", log_level = "log_output")
cosmostat_client.update_system_state()
client_update()
while True:
if int(time.time()) % 5 == 0:
cosmostat_client.update_system_state()
client_update()
time.sleep(5)
time.sleep(0.5)

183
files/api/archive.py Normal file
View File

@ -0,0 +1,183 @@
# dynamic data
# this will go to the redis server
@app.route('/dynamic_data', methods=['GET'])
def dynamic_data():
return jsonify(get_dynamic_data())
# static data
@app.route('/static_data', methods=['GET'])
def static_data():
return jsonify(get_static_data())
# redis data
@app.route('/redis_data', methods=['GET'])
def redis_data():
return jsonify(get_client_redis_data(human_readable = False))
# redis strings
@app.route('/redis_strings', methods=['GET'])
def redis_strings():
return jsonify(get_client_redis_data(human_readable = True))
# socket timer data
@app.route('/timer_data', methods=['GET'])
def timer_data():
time_now = time.time()
time_lapsed = time_now - float(cosmostat_client.recent_check)
result = {
"Time Lapsed": time_lapsed,
"Current Time Value": time_now,
"Last Update Value": float(cosmostat_client.recent_check),
"System Updating": cosmostat_client.check_system_timer()
}
return jsonify(result)
def get_static_data(human_readable = False):
result = []
return cosmostat_client.get_static_metrics(human_readable)
def generate_state_definition():
result = {
"uuid": cosmostat_client.uuid,
"state_definition": get_php_summary()
}
return result
# api to validate Cosmostat Class
@app.route('/client_summary', methods=['GET'])
def client_summary():
result = []
if run_cosmostat_server():
result = get_client_summary()
else:
result = {"message": "server not running on this endpoint"}
return jsonify(result)
# api to get all hostnames
@app.route('/client_hostnames', methods=['GET'])
def client_hostnames():
result = []
if run_cosmostat_server():
result = cosmostat_server.get_client_hostnames(send_age = True)
else:
result = {"message": "server not running on this endpoint"}
return jsonify(result)
# api to get server redis data
@app.route('/get_server_redis', methods=['GET'])
def get_server_redis():
result = []
if run_cosmostat_server():
result = get_server_redis_data()
else:
result = {"message": "server not running on this endpoint"}
return jsonify(result)
# api to get server redis data
@app.route('/client_ip_summary', methods=['GET'])
def client_ip_summary():
result = []
if run_cosmostat_server():
result = get_client_ip_summary()
else:
result = {"message": "server not running on this endpoint"}
return jsonify(result)
# generate cosmostat server summary
def get_client_summary():
result = []
for client in cosmostat_server.systems:
data_age = time.time() - client.data_timestamp
this_client = {
"uuid": client.uuid,
"short_id": client.name,
"data_age": data_age,
"hostname": client.hostname
}
result.append(this_client)
if result == []:
result = {"message": "no clients reporting"}
return result
########################################################
# various data functions
########################################################
# complex data type return
def get_property_summary(self, type = None):
these_properties = []
if type == None:
for name, value in self._properties.items():
these_properties.append({"Property": name, "Value": value})
else:
for name, value in self._properties.items():
if type in name:
these_properties.append({"Property": name, "Value": value})
result = {
"Source": self.name,
"Component Type": self.type,
"Properties": these_properties
}
return result
# simple data value return
def get_property_value(self, type = None):
result = []
print(f"Component type: {type}")
for name, value in self._properties.items():
print(f"Component Property: {name}")
if type in name:
result.append(value)
if len(result) == 1:
return result[0]
else:
return result
# full data return
def get_description(self):
these_properties = []
for name, value in self._properties.items():
these_properties.append({"Property": name, "Value": value})
these_metrics = []
for name, value in self._metrics.items():
these_metrics.append({"Metric": name, "Data": value})
result = {
"Source": self.name,
"Type": self.type,
"Properties": these_properties,
"Metrics": these_metrics
}
return result
# complex data type return
def get_metrics(self, type = None):
these_metrics = []
if type == None:
for name, value in self._metrics:
these_metrics.append({"Metric": name, "Data": value})
else:
for name, value in self._metrics:
if name == type:
these_metrics.append({"Metric": name, "Data": value})
result = {
"Source": self.name,
"Component Type": self.type,
"Metrics": these_metrics
}
return result

0
templates/docker-compose-php.yaml → files/archive/docker-compose-php.yaml
View File

0
templates/docker-compose-single.yaml → files/archive/docker-compose-single.yaml
View File

0
templates/vpn_client.conf → files/archive/vpn_client.conf
View File

0
templates/vpn_server.conf → files/archive/vpn_server.conf
View File

2
tasks/web.yaml → files/archive/web.yaml
View File

@ -7,7 +7,7 @@
when: not quick_refresh | bool
community.docker.docker_compose_v2:
project_src: "{{ service_control_web_folder }}"
state: stopped
state: "{{ 'stopped' if quick_refresh | bool else 'absent' }}"
ignore_errors: yes
# Create web Folder

0
files/web/html/index.php → files/archive/web/html/index.php
View File

0
files/web/html/src/redis.js → files/archive/web/html/src/redis.js
View File

0
files/web/html/src/styles.css → files/archive/web/html/src/styles.css
View File

0
files/web/node_server/package.json → files/archive/web/node_server/package.json
View File

0
files/web/node_server/server.js → files/archive/web/node_server/server.js
View File

0
files/web/proxy/nginx.conf → files/archive/web/proxy/nginx.conf
View File

7
files/docker/Dockerfile
View File

@ -8,6 +8,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
redis-server nginx \
# Process supervisor
supervisor \
# Others
net-tools \
# Clean up
&& rm -rf /var/lib/apt/lists/*
@ -22,11 +24,12 @@ COPY cosmostat_settings.yaml /app/cosmostat_settings.yaml
# Node on 3000
WORKDIR /usr/src/app
COPY web/node_server/ .
#COPY cosmostat_settings.yaml /usr/src/app/cosmostat_settings.yaml
RUN npm install --only=production
# Apache on 8080
RUN sed -i 's/^Listen .*/Listen 8080/' /etc/apache2/ports.conf && \
sed -i 's/<VirtualHost \*:80>/<VirtualHost \*:8080>/' /etc/apache2/sites-enabled/000-default.conf
COPY apache_ports.conf /etc/apache2/ports.conf
COPY apache_vhost.conf /etc/apache2/sites-available/000-default.conf
COPY web/html/ /var/www/html/
# nginx on 80

5
files/docker/apache_ports.conf Normal file
View File

@ -0,0 +1,5 @@
# Listen on 8080 inside the container
Listen 8080
# If you still want Apache to listen on 80 (rare), add it back:
# Listen 80

14
files/docker/apache_vhost.conf Normal file
View File

@ -0,0 +1,14 @@
<VirtualHost *:8080>
ServerAdmin webmaster@localhost
DocumentRoot /var/www/html
# Log files
ErrorLog ${APACHE_LOG_DIR}/error.log
CustomLog ${APACHE_LOG_DIR}/access.log combined
# If you need PHP processing
<Directory /var/www/html>
AllowOverride All
Require all granted
</Directory>
</VirtualHost>

3
files/docker/supervisord.conf
View File

@ -38,7 +38,8 @@ priority=3
# ------------------------------------------------------------------
# NOTE: Adjust the command/path to match your app
[program:node]
command=npm start
command=sh -c "npm install && node server.js"
#command=npm start
directory=/usr/src/app
stdout_logfile=/dev/stdout
stderr_logfile=/dev/stderr

3
files/docker/web/node_server/server.js
View File

@ -17,7 +17,8 @@ const io = new Server(server);
/* --------------------------------------------------------------------- */
let config = {};
try {
const file = fs.readFileSync(path.resolve(__dirname, 'cosmostat_settings.yaml'), 'utf8');
const filePath = '/app/cosmostat_settings.yaml';
const file = fs.readFileSync(filePath, 'utf8');
config = yaml.load(file);
} catch (e) {
console.error('Failed to load config.yaml:', e);

4
files/docker/web/proxy/nginx.conf
View File

@ -39,7 +39,7 @@ server_name localhost;
# WebSocket endpoint
# ---------------------------------------
location /socket.io/ {
proxy_pass http://192.168.37.1:3000/socket.io/;
proxy_pass http://0.0.0.0:3000/socket.io/;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
@ -53,7 +53,7 @@ server_name localhost;
# All other paths → Apache (PHP)
# ---------------------------------------
location / {
proxy_pass http://192.168.37.1:8080;
proxy_pass http://0.0.0.0:8080;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

10
tasks/docker.yaml
View File

@ -6,9 +6,16 @@
- name: Cosmostat - Web - stop containers
community.docker.docker_compose_v2:
project_src: "{{ service_control_docker_folder }}"
state: stopped
state: "{{ 'stopped' if quick_refresh | bool else 'absent' }}"
ignore_errors: yes
- name: Cosmostat - Web - Remove Cosmostat Image
when: not quick_refresh | bool
community.docker.docker_image:
name: "cosmostat-dash"
tag: latest
state: absent
# Create web Folder
- name: "Cosmostat - Web - create {{ service_control_docker_folder }}"
file:
@ -71,6 +78,7 @@
community.docker.docker_compose_v2:
project_src: "{{ service_control_docker_folder }}"
state: present
build: "{{ 'always' if not quick_refresh | bool else 'never' }}"
register: docker_output
- debug: |
msg="{{ docker_output.actions }}"

2
tasks/main.yaml
View File

@ -18,6 +18,6 @@
# set up web stack
- name: Build Web Dashboard
when: not disable_local_dashboard | bool
include_tasks: web.yaml
include_tasks: docker.yaml
...

13
templates/docker-compose.yaml
View File

@ -3,21 +3,24 @@
services:
cosmostat-dash:
container_name: cosmostat-dash
image: cosmostat-dash:latest
restart: always
build:
context: .
dockerfile: Dockerfile
container_name: cosmostat-dash
image: cosmostat-dash:latest
restart: always
networks:
- cosmostat_net
ports:
- "{{ docker_gateway }}:6379:6379"
- "{{ (docker_gateway + ':') if not public_dashboard | bool else '' }}{{ custom_port }}:80"
volumes:
# When the container is built in Ansible, these are all copied
# if any changes are made manually on the endpoint, uncomment as needed
#volumes:
# - "/opt/cosmostat/api/cosmostat_settings.yaml:/app/cosmostat_settings.yaml:ro"
# - "/opt/cosmostat/api/cosmostat_settings.yaml:/usr/src/app/cosmostat_settings.yaml:ro"
# - "/opt/cosmostat/docker/web/html:/var/www/html"
# - "/opt/cosmostat/docker/web/node_server:/app"
- "/opt/cosmostat/api/cosmostat_settings.yaml:/app/cosmostat_settings.yaml:ro"
networks:
cosmostat_net: