web dashboard in single container

defaults/main.yaml

@@ -68,10 +68,12 @@ refresh_special: false
 special_server: "none"
 
 # cosmostat_settings, will be for special_server defaults
+# the special_server install will be the full stack with public visibility
+# regular installs will be handled through specific pipelines
 noisy_test: false
 debug_output: false
-secure_api: true
+secure_api: false
-push_redis: false
+push_redis: true
 run_background : true
 log_output: true
 update_frequency: "1"
@@ -79,6 +81,6 @@ cosmostat_server: false
 cosmostat_server_api: "https://cosmostat.matt-cloud.com/"
 local_api_address: "http://{{ cosmostat_server_ip }}:{{ custom_api_port }}/"
 cosmostat_server_reporter: true
-# setting this to true for default install
+# setting this to false for default install
-disable_local_dashboard: true
+disable_local_dashboard: false
 ...

files/api/Components.py

@@ -13,6 +13,7 @@ import ipaddress
 from typing import Dict, Any, List
 # Import Cosmos Settings
 from Cosmos_Settings import *
+from Helpers import *
 
 # Global Class Vars
 global_max_length = 500
@@ -158,6 +159,7 @@ class Component:
     # iterate over all properties to process descriptor
     def _process_properties(self):
         for key, command in self._descriptor.get('properties', {}).items():
+            # if this is a component that can be multi, then it might need to return a list
             return_string = True
             if key in self.multi_metrics:
                 return_string = False
@@ -168,6 +170,8 @@ class Component:
                 self._properties[key] = result
 
     # helper function to parse command key
+    # this is for substituting variables in descriptor commands when there are multi things
+    # think needing to grep eth0 or grep eth1 to get the same metric for a different component
     def _parse_command(self, key: str, command: str | list[str], return_string = True):
         result_command = command
         log_data(log_output = f"_parse_command - {command}", log_level = "debug_output")
@@ -187,6 +191,8 @@ class Component:
         return result_command
 
     # check if this property should show in the System Properties box
+    # these are intended to provide a summary of critical things that all computers have
+    # like memory and cpu and cores
     def check_php_extra(self, property_name):
         result = False
         if property_name in self.php_extra_list:
@@ -195,6 +201,7 @@ class Component:
 
     ########################################################
     # keyed data functions
+    # various ways to query the component data
     ########################################################
 
     def get_properties_keys(self, component = None):
@@ -270,73 +277,6 @@ class Component:
         else:
             return result
     
-    ########################################################
-    # various data functions
-    ########################################################
-
-    # complex data type return
-    def get_metrics(self, type = None):
-        these_metrics = []
-        if type == None:
-            for name, value in self._metrics:
-                these_metrics.append({"Metric": name, "Data": value})
-        else:
-            for name, value in self._metrics:
-                if name == type: 
-                    these_metrics.append({"Metric": name, "Data": value})
-        result = {
-            "Source": self.name,
-            "Component Type": self.type,
-            "Metrics": these_metrics
-        }
-        return result
-    
-    # complex data type return
-    def get_property_summary(self, type = None):
-        these_properties = []
-        if type == None:
-            for name, value in self._properties.items():
-                these_properties.append({"Property": name, "Value": value})
-        else:
-            for name, value in self._properties.items():
-                if type in name: 
-                    these_properties.append({"Property": name, "Value": value})
-        result = {
-            "Source": self.name,
-            "Component Type": self.type,
-            "Properties": these_properties
-        }
-        return result
-    
-    # simple data value return
-    def get_property_value(self, type = None):
-        result = []
-        print(f"Component type: {type}")
-        for name, value in self._properties.items():
-            print(f"Component Property: {name}")
-            if type in name: 
-                result.append(value)
-        if len(result) == 1:
-            return result[0]
-        else:
-            return result
-    
-    # full data return
-    def get_description(self):
-        these_properties = []
-        for name, value in self._properties.items():
-            these_properties.append({"Property": name, "Value": value})
-        these_metrics = []
-        for name, value in self._metrics.items():
-            these_metrics.append({"Metric": name, "Data": value})
-        result = {
-            "Source": self.name,
-            "Type": self.type,
-            "Properties": these_properties,
-            "Metrics": these_metrics
-        }
-        return result
-    
 ############################################################
 ############################################################
 ### System Class
@@ -396,7 +336,7 @@ class System:
         # load static keys
         for static_key in self.static_key_variables:
             if static_key["name"] not in self._virt_ignore:
-                self.process_property(static_key = static_key)
+                self._process_property(static_key = static_key)
         # initialize live keys
         self.update_live_keys()
         # initialze components
@@ -422,8 +362,8 @@ class System:
     # critical class functions
     ########################################################
 
-    # process static keys
+    # process static properties, done once at instantiation
-    def process_property(self, static_key):
+    def _process_property(self, static_key):
         command = static_key["command"]
         if "arch_check" in static_key:
             arch_string = run_command("lscpu --json | jq -r '.lscpu[] | select(.field==\"Architecture:\") | .data'", zero_only = True)
@@ -439,7 +379,7 @@ class System:
         if result not in null_result:
             self._properties[static_key["name"]] = result
 
-    # update only system dynamic keys
+    # update only system dynamic keys, done periodically
     def update_live_keys(self):
         for live_key in self.dynamic_key_variables:
             if live_key['command'] is not None:
@@ -457,25 +397,53 @@ class System:
 
     # component creation helper
     def create_component(self, component):
+        # this is the type, i.e. CPU MEM LAN etc
         component_name = component["name"]
+        # if there can be multiples, i.e.LAN STOR GPU etc
         multi_check = component["multi_check"]
         # if multi, note that the command in device_list creates the list of things to pipe into this_device
         if multi_check:
             log_data(log_output = f"Creating one component of type {component_name} for each one found", log_level = "log_output")
-            component_type_device_list = get_device_list(component_name)
+            # build list of components of this type to initialize
+            component_type_device_list = self.get_device_list(component_name)
             for this_device in component_type_device_list:
                 this_component_ID = component_type_device_list.index(this_device)
                 this_component_name = f"{component_name} {this_component_ID}"
                 log_data(log_output = f"{this_component_name} - {component_name} - {this_device}", log_level = "debug_output")
                 new_component = Component(name = this_component_name, comp_type = component_name, this_device = this_device, parent_system = self)
+                # add new component to system
                 self.components.append(new_component)
+        # if it's just a single thing like CPU RAM then just do one device
         else:
             log_data(log_output = f'Creating component {component["name"]}', log_level = "debug_output")
             new_component = Component(name = component_name, comp_type = component_name, parent_system = self)
+            # add new component to system
             self.components.append(new_component)      
 
+    # helper function - component type list builder for multi check items
+    def get_device_list(self, device_type_name: str):
+        result = []
+        for component in component_class_tree:
+            # pre-set to 1 for true value by default
+            precheck_value = 1
+            # the precheck is critical, i.e there can be 0 or 1 or 2 GPU
+            if "precheck" in component:
+                precheck_command = component["precheck"]
+                precheck_value_output = run_command(precheck_command, zero_only = True)
+                precheck_value = int(precheck_value_output)
+                log_data(log_output = f"Precheck found - {precheck_command} - {precheck_value}", log_level = "log_output")
+            # build a list of devices if precheck passes, this can be a list of 1 like a single GPU system 
+            if component["name"] == device_type_name and precheck_value != 0:
+                device_list_command = component["device_list"]
+                device_list_result = run_command(device_list_command)
+                result = device_list_result
+        return result
+
     # return the IP of the interface with the gateway
     # remember the IP is stored like this 192.168.1.0/24
+    # this data is pulled from the system properties class data
+    # this was built for the update inventory generation
+    # but is not the complete picture
     def get_primary_ip(self):
         primary_ip = None
         interfaces = self.get_components(component_type = "LAN")
@@ -577,6 +545,7 @@ class System:
 
         return result
 
+    # helper function for system properties for dashboard rendering
     def php_component_data(self):
         result = []
         for component in self.components:
@@ -653,57 +622,6 @@ class System:
         })
         return result
 
-############################################################
-# Non-class Helper Functions
-############################################################
-
-# subroutine to run a command, return stdout as array unless zero_only then return [0]
-def run_command(cmd, zero_only=False, use_shell=True, req_check = True):
-    # Run the command and capture the output
-    result = subprocess.run(cmd, shell=use_shell, check=req_check, stdout=subprocess.PIPE, stderr=subprocess.PIPE)    
-    # Decode the byte output to a string
-    output = result.stdout.decode('utf-8')
-    # Split the output into lines and store it in an array
-    output_lines = [line for line in output.split('\n') if line]
-    # Return result
-    try:
-        return output_lines[0] if zero_only else output_lines
-    except:
-        return output_lines
-
-# need to add a archticture checker for this
-# i also want to make the loop cleaner
-# i don't need to iterate over the component class tree
-# to get what I want, i think
-def get_device_list(device_type_name: str):
-
-    result = []
-    for component in component_class_tree:
-        precheck_value = 1
-        if "precheck" in component:
-            precheck_command = component["precheck"]
-            precheck_value_output = run_command(precheck_command, zero_only = True)
-            precheck_value = int(precheck_value_output)
-            log_data(log_output = f"Precheck found - {precheck_command} - {precheck_value}", log_level = "log_output")
-        if component["name"] == device_type_name and precheck_value != 0:
-
-            device_list_command = component["device_list"]
-            device_list_result = run_command(device_list_command)
-            result = device_list_result
-
-    return result
-
-# subnet helper app
-def is_ip_in_subnets(ip, subnet):
-    try:
-        ip_obj = ipaddress.IPv4Address(ip)
-        subnet_obj = ipaddress.IPv4Network(subnet)
-        if ip_obj in subnet_obj:
-            return True
-        return False
-    except ValueError as e:
-        # If the IP address is not valid, raise an error
-        return False
 
 
 

files/api/Cosmostat.py

@@ -21,6 +21,7 @@ import base64, hashlib
 from typing import Dict, Any, List
 # Import Cosmos Settings
 from Cosmos_Settings import *
+from Helpers import *
 
 #################################################################
 ### Cosmostat Server Class
@@ -197,16 +198,3 @@ class CosmostatClient:
 
     def get_redis(self):
         return self.redis_data
-
-# subnet helper app
-def is_ip_in_subnets(ip, subnet):
-    try:
-        ip_obj = ipaddress.IPv4Address(ip)
-        subnet_obj = ipaddress.IPv4Network(subnet)
-        if ip_obj in subnet_obj:
-            return True
-        return False
-    except ValueError as e:
-        # If the IP address is not valid, raise an error
-        return False
-

files/api/Helpers.py

@@ -0,0 +1,30 @@
+
+import subprocess
+import ipaddress
+from typing import Dict, Any, List
+
+# subnet helper app
+def is_ip_in_subnets(ip, subnet):
+    try:
+        ip_obj = ipaddress.IPv4Address(ip)
+        subnet_obj = ipaddress.IPv4Network(subnet)
+        if ip_obj in subnet_obj:
+            return True
+        return False
+    except ValueError as e:
+        # If the IP address is not valid, raise an error
+        return False
+
+# subroutine to run a command, return stdout as array unless zero_only then return [0]
+def run_command(cmd, zero_only=False, use_shell=True, req_check = True):
+    # Run the command and capture the output
+    result = subprocess.run(cmd, shell=use_shell, check=req_check, stdout=subprocess.PIPE, stderr=subprocess.PIPE)    
+    # Decode the byte output to a string
+    output = result.stdout.decode('utf-8')
+    # Split the output into lines and store it in an array
+    output_lines = [line for line in output.split('\n') if line]
+    # Return result
+    try:
+        return output_lines[0] if zero_only else output_lines
+    except:
+        return output_lines

files/api/app.py

@@ -29,11 +29,12 @@ scheduler = APScheduler()
 # Redis client - will publish updates
 r = redis.Redis(host=redis_gateway_ip(), port=6379)
 
+# Publish to the specified Redis channel
 def update_redis_channel(redis_channel, data):
-    # Publish to the specified Redis channel
     r.publish(redis_channel, json.dumps(data))
     log_data(log_output = data, log_level = "noisy_test")
 
+# update redis server for both modes
 def update_redis_server():
     # Client Redis Tree
     if cosmostat_client.check_system_timer():
@@ -41,18 +42,12 @@ def update_redis_server():
     
     if run_cosmostat_server():
         update_redis_channel("client_summary", get_server_redis_data())
-        #update_redis_channel("client_hostnames", get_server_hostnames())
-
-    # History Redis Tree
-    # Update history_stats Redis Channel
-    # update_redis_channel("history_stats", get_component_list())
 
+# function helper for both modes
 def get_client_redis_data(human_readable = False):
     result = []
     for metric in get_dynamic_data(human_readable):
         result.append(metric)
-    #for metric in get_static_data(human_readable):
-    #    result.append(metric)
     return result
 
 def get_server_redis_data():
@@ -70,70 +65,25 @@ def get_server_redis_data():
         result.append(this_client_key)
     return result
 
-def get_server_hostnames():
-    return cosmostat_server.get_client_hostnames()
-
-
 #######################################################################
 ### Client Flask Routes
 #######################################################################
 
-# dynamic data
-# this will go to the redis server
-@app.route('/dynamic_data', methods=['GET'])
-def dynamic_data():
-    return jsonify(get_dynamic_data())
-
-# static data
-@app.route('/static_data', methods=['GET'])
-def static_data():
-    return jsonify(get_static_data())
-
-# redis data
-@app.route('/redis_data', methods=['GET'])
-def redis_data():
-    return jsonify(get_client_redis_data(human_readable = False))
-
-# redis strings
-@app.route('/redis_strings', methods=['GET'])
-def redis_strings():
-    return jsonify(get_client_redis_data(human_readable = True))
-
 # php summary
 @app.route('/php_summary', methods=['GET'])
 def php_summary():
     return jsonify(get_php_summary())
 
-# return full descriptor
-@app.route('/descriptor', methods=['GET'])
-def descriptor():
-    return jsonify(get_descriptor())
-
 # socket timer handler
 @app.route('/start_timer', methods=['GET'])
 def start_timer():
     current_timestamp = int(time.time())
     cosmostat_client.recent_check = current_timestamp
     log_data(log_output = f"Timestamp updated to {cosmostat_client.recent_check}", log_level = "noisy_test")
-    return jsonify(
+    return jsonify({
-        {
             "message": "websocket timer reset",
             "new_timestamp": cosmostat_client.recent_check
-        }
+        })
-    )
-
-# socket timer data
-@app.route('/timer_data', methods=['GET'])
-def timer_data():
-    time_now = time.time()
-    time_lapsed = time_now - float(cosmostat_client.recent_check)
-    result = {
-        "Time Lapsed": time_lapsed,
-        "Current Time Value": time_now,
-        "Last Update Value": float(cosmostat_client.recent_check),
-        "System Updating": cosmostat_client.check_system_timer()
-    }
-    return jsonify(result)
 
 # test route
 @app.route('/test', methods=['GET'])
@@ -157,70 +107,63 @@ def test():
 def get_dynamic_data(human_readable = False):
     return cosmostat_client.get_live_metrics(human_readable)
 
-def get_static_data(human_readable = False):
+# this generates the static data used by the client dashboard and the CosmostatServer properties
-    result = []
-    return cosmostat_client.get_static_metrics(human_readable)
-
 def get_php_summary():
+    # separate lists for system properties and components
     system_properties = cosmostat_client.get_system_properties(human_readable = True, php_extra = True)
     system_components = []
+    # build a list of Component property lists
     for component in cosmostat_client.get_components():
         this_component = {
             "component_name": component.name,
-            "info_strings": component.get_properties_strings(return_simple = True)
+            "info_strings": component.get_properties_strings(return_simple = True)        # this is a list of strings from the Class
         }
         system_components.append(this_component)
     this_primary_ip = cosmostat_client.primary_ip
+    # if this is the server there are extra steps here to add all client data to the return
+    # this can make for a big json
     if run_cosmostat_server():
+        # get uuid for reference
         client_uuid = cosmostat_server.get_uuid_from_hostname(cosmostat_client.name)
+        # get timestamp for bubble 
         data_timestamp = cosmostat_server.get_system(client_uuid)
         log_string = f"Cosmostat Client Name: {cosmostat_client.name} - UUID: {client_uuid}- Timestamp: {data_timestamp}"
         log_data(log_output = log_string, log_level = "log_output")
-        component_age = {
+        # get the primary IP for inventory file generation
-            "component_name": "Data Timestamp",
-            "info_strings": f"Data is {data_timestamp} seconds old"
-        }
         this_primary_ip = cosmostat_settings["cosmostat_server_ip"]
-        system_components.append(component_age)
+    # build and return dict of all these lists
-
     result = [{
         "system_properties": system_properties,
         "system_components": system_components,
         "active_interface": this_primary_ip
     }]
-
     return result
 
-def get_descriptor():
-    return cosmostat_client.get_component_class_tree()
-
-def generate_state_definition():
-    result = {
-        "uuid": cosmostat_client.uuid,
-        "state_definition": get_php_summary()
-    }
-    return result
-
-
 #######################################################################
 ### Server Flask Routes
 #######################################################################
 
-# update client on server
+# return full descriptor - public route
+@app.route('/descriptor', methods=['GET'])
+def descriptor():
+    return jsonify(cosmostat_client.get_component_class_tree())
+
+# update client on server - public route
 @app.route('/update_client', methods=['POST'])
 def update_client():
     result = {}
-    # check the request and return payload dict {} if all good
+    # sanitize and validate POST data
-    payload = client_submit_check(request = request, dict_name = "redis_data")
+    payload = client_submit_check(request = request)
+    # process client update request
     result = run_update_client(payload)
     return jsonify(result), 200
 
-# create client on server
+# create client on server - public route
 @app.route('/create_client', methods=['POST'])
 def create_client():
     result = {}
-    # check the request and return payload dict {} if all good
+    # sanitize and validate POST data
-    payload = client_submit_check(request = request, dict_name = "client_properties")
+    payload = client_submit_check(request = request)
     # if the client does not exist, create it
     if not cosmostat_server.check_uuid(payload["uuid"]):
         result = run_create_client(payload)
@@ -231,17 +174,7 @@ def create_client():
         }
     return jsonify(result), 200
 
-# api to validate Cosmostat Class
+# api to pull all data for server dashboard rendering
-@app.route('/client_summary', methods=['GET'])
-def client_summary():
-    result = []
-    if run_cosmostat_server():
-        result = get_client_summary()
-    else:
-        result = {"message": "server not running on this endpoint"}
-    return jsonify(result)
-
-# api to pull all data
 @app.route('/client_details', methods=['GET'])
 def client_details():
     result = []
@@ -251,36 +184,6 @@ def client_details():
         result = {"message": "server not running on this endpoint"}
     return jsonify(result)
 
-# api to get all hostnames
-@app.route('/client_hostnames', methods=['GET'])
-def client_hostnames():
-    result = []
-    if run_cosmostat_server():
-        result = cosmostat_server.get_client_hostnames(send_age = True)
-    else:
-        result = {"message": "server not running on this endpoint"}
-    return jsonify(result)
-
-# api to get server redis data
-@app.route('/get_server_redis', methods=['GET'])
-def get_server_redis():
-    result = []
-    if run_cosmostat_server():
-        result = get_server_redis_data()
-    else:
-        result = {"message": "server not running on this endpoint"}
-    return jsonify(result)
-
-# api to get server redis data
-@app.route('/client_ip_summary', methods=['GET'])
-def client_ip_summary():
-    result = []
-    if run_cosmostat_server():
-        result = get_client_ip_summary()
-    else:
-        result = {"message": "server not running on this endpoint"}
-    return jsonify(result)
-
 # return inventory file of all clients for update
 @app.route("/client_inventory", methods=["GET"])
 def client_inventory():
@@ -290,15 +193,27 @@ def client_inventory():
 ### Server Flask Helpers
 #######################################################################
 
-# update client on server
+# this is the helper function for processing client submission requests
+# this only updates the dynamic metrics
+# the output of this function is returned to the flask function
+# This function is also used to report the cosmostat server's CosmostatClient
 def run_update_client(this_client):
+    # this is where the API check is done
     if public_api_check(this_client):
+        # return if object does not exist
         if not cosmostat_server.check_uuid(this_client["uuid"]):
             return { "message": "client not found" }
         else:
-            timestamp_update = cosmostat_server.update_system(system_dictionary = this_client["redis_data"], system_uuid = this_client["uuid"])
+            # this is where the CosmostatClient object is updated
+            # done by passing the UUID and redis data
+            # that Class Function returns the timestamp
+            timestamp_update = cosmostat_server.update_system(
+                system_dictionary = this_client["redis_data"],
+                system_uuid = this_client["uuid"]
+            )
+            # after this happens, record a successful status
             update_status = f'updated client {this_client["short_id"]}'    
-
+            # build a return string and send it
             return {
                 "status": update_status,
                 "client_updated": "True",
@@ -306,17 +221,25 @@ def run_update_client(this_client):
                 "redis_data": this_client,
                 "timestamp_update": timestamp_update
                 }
+    # if the API check fails, return as much
     else:
         return{
             "status": "api failure"
         }
 
-# create client on server
+# this is the helper function for processing client creation requests
+# this creates a new CosmostatClient object on the 'cosmostat_server' Object
+# the output of this function is returned to the flask function
 def run_create_client(this_client):
+    # this is where the API check is done
     if public_api_check(this_client):
-        #this_client["active_interface"] = cosmostat_client.primary_ip
+        # this is where the CosmostatClient object is inialized
+        # done by passing the UUID and all static data
+        # that Class Function returns the timestamp
         timestamp_update = cosmostat_server.add_system(system_dictionary = this_client)
+        # after this happens, record a successful status
         update_status = f'created client {this_client["short_id"]}'    
+        # build a return string and send it
         return {
             "status": update_status,
             "client_created": "True",
@@ -324,11 +247,13 @@ def run_create_client(this_client):
             "client_properties": this_client,
             "timestamp_update": timestamp_update
             }
+    # if the API check fails, return as much
     else:
         return{
             "status": "api failure"
         }
 
+# API check helper
 def public_api_check(this_client):
     result = False
     default_key = ''.join(secrets.choice(string.ascii_letters + string.digits) for _ in range(256))
@@ -338,9 +263,12 @@ def public_api_check(this_client):
     return result
 
 # flask submission check function
-def client_submit_check(request, dict_name: str):
+# all this does is take the POST data from 
+# remote client submissions and sanitize and validate 
+# it returns the jsonified submission after that
+def client_submit_check(request):
     payload = {}
-    required_keys = {"uuid", "short_id", "hostname", dict_name}
+    required_keys = {"uuid", "short_id", "hostname"}
     if not request.is_json:
         logging.warning("Received non-JSON request")
         return jsonify({"error": "Content-type must be application/json"}), 400
@@ -351,25 +279,8 @@ def client_submit_check(request, dict_name: str):
     missing = required_keys - payload.keys()
     if missing:
         raise ValueError(f"Missing required keys: {', '.join(sorted(missing))}")        
-        
     return payload
 
-# generate cosmostat server summary 
-def get_client_summary():
-    result = []
-    for client in cosmostat_server.systems:
-        data_age = time.time() - client.data_timestamp
-        this_client = {
-            "uuid": client.uuid,
-            "short_id": client.name,
-            "data_age": data_age,
-            "hostname": client.hostname
-        }
-        result.append(this_client)
-    if result == []:
-        result = {"message": "no clients reporting"}
-    return result
-
 # no redis data needed here
 def get_client_details():
     result = []
@@ -387,27 +298,29 @@ def get_client_details():
         result = {"message": "no clients reporting"}
     return result
 
-# get client IP summary
+# get client IP summary either in a Dict or List
 def get_client_ip_summary(return_list = False):
     result = []
     for client in cosmostat_server.systems:
-
+        # build the Dict
         device_info = {
             "small_id": client.name,
             "uuid": client.uuid,
             "hostname": client.hostname,
             "lan_ip": client.active_ip,
         }
+        # build the result depending on mode
         if client.hostname != cosmostat_server.hostname:
             if return_list:
-                result.append(client.active_ip)
+                result.append(client.active_ip)      # only the list
             else:
-                result.append(device_info)
+                result.append(device_info)           # the full dictionary
     return result
 
 # build client ansible inventory file
 def build_inventory():
     all_ips = get_client_ip_summary(return_list = True)
+    # subnets reachable from the cosmostat server
     cosmos_subnets = [
         "172.25.1.0/24",
         "172.20.0.0/16",
@@ -416,9 +329,10 @@ def build_inventory():
         "10.200.27.0/24",
         "192.168.60.0/24",
     ]
+    # inialize lists
     ips = []
     bad_ips = []
-    # build list of reachable IPs
+    # build list of active IPs
     for ip in all_ips:
         for subnet in cosmos_subnets:
             if is_ip_in_subnets(ip, subnet) and cosmostat_server.get_vpn_ip(ip, just_check = True):
@@ -427,7 +341,7 @@ def build_inventory():
     for ip in all_ips:
         if ip not in ips:
             bad_ips.append(ip)
-    # add any VPN IPs for bad IPs to the main IP list
+    # any system with a VPN to jenkins, add to inventory
     for ip in bad_ips:
         ips.append(cosmostat_server.get_vpn_ip(ip))
     hosts = {ip: {"ansible_host": ip} for ip in ips}
@@ -470,52 +384,42 @@ def build_inventory():
 
 #######################################################################
 ### Cosmostat Client Subroutines
+### These are only for the clients
+### The cosmostat server has its own functions
 #######################################################################
 
 # Cosmostat Client Reporter Handler
+# this handles updating the active data for the local client instance
+# to the remote cosmostat server
 def client_update():
+    # update client API path
     api_url = f"{cosmostat_server_api()}update_client"
     payload = get_client_payload(get_client_redis_data(human_readable = False), "redis_data")
     log_data(log_output = f"API Update Called - {api_url}", log_level = "debug_output")
     log_data(log_output = "client_update - redis data from local client:", log_level = "noisy_test")
     log_data(log_output = payload, log_level = "noisy_test")
-    # execute API call
+    # execute API call, we do need the result
     result = client_submission_handler(api_url, payload)
+    # if this client does not exist on the server, we need to try to initialize it
     if not result or not result.get('client_updated'):
         log_data(log_output = f"Client not updated, initializing", log_level = "log_output")
-        result = client_api_initialize()
+        client_api_initialize()
-    # this result does not matter and is not used anywhere
-    return result
 
 # Cosmostat Client Initializer
+# this handles inializing the CosmostatClient object on the remote cosmostat server
 def client_api_initialize():
+    # create client API path
     api_url = f"{cosmostat_server_api()}create_client"
-    # generate payload
+    # generate payload for the remote server
     payload = get_client_payload(get_php_summary(), "client_properties")
-    # execute API call
+    # add extra init vars
     payload["active_interface"] = cosmostat_client.primary_ip
     payload["is_server"] = False
+    # execute API call on remote server
+    client_submission_handler(api_url, payload)
 
-    result = client_submission_handler(api_url, payload)
+# this returns the payload in the format that the API expects with extra necessary data
-    return result
+# used for both initializing and updating; dictionary name is part of the function
-
-# Cosmostat Client API Reporting Handler
-def client_submission_handler(api_url: str, payload: dict):
-    result = None
-    try:
-        # `json=` automatically sets Content-Type to application/json
-        response: Response = requests.post(api_url, json=payload, timeout=4)
-        response.raise_for_status()  # raise HTTPError for 4xx/5xx
-    except RequestException as exc:
-        # Wrap the low-level exception in a more descriptive one
-        log_data(log_output = f"Failed to POST to {api_url!r}: {exc}", log_level = "log_output")
-    # process reply from API
-    try:
-        result = response.json()
-    except ValueError as exc:
-        log_data(log_output = f"Server responded with non-JSON payload: {response.text!r}", log_level = "log_output")
-    return result
-
 def get_client_payload(system_dictionary: dict, dictionary_name: str):
     this_uuid = cosmostat_client.uuid
     this_short_id = cosmostat_client.short_id
@@ -530,6 +434,26 @@ def get_client_payload(system_dictionary: dict, dictionary_name: str):
     }
     return payload
 
+# Cosmostat Client API Reporting Handler
+# this helper makes the POST request to the remote cosmostat server
+# the payload and URL are passed to this function
+def client_submission_handler(api_url: str, payload: dict):
+    result = None
+    try:
+        # this is the actual submission to the server
+        response: Response = requests.post(api_url, json=payload, timeout=4)
+        # raise HTTPError for 4xx/5xx if failure
+        response.raise_for_status()  
+    except RequestException as exc:
+        # Wrap the low-level exception in a more descriptive one
+        log_data(log_output = f"Failed to POST to {api_url!r}: {exc}", log_level = "log_output")
+    # process reply from API
+    try:
+        result = response.json()
+    except ValueError as exc:
+        log_data(log_output = f"Server responded with non-JSON payload: {response.text!r}", log_level = "log_output")
+    return result
+
 #######################################################################
 #######################################################################
 ### Main Subroutine
@@ -538,12 +462,15 @@ def get_client_payload(system_dictionary: dict, dictionary_name: str):
 
 if __name__ == '__main__':
 
-
     ######################################
     ### Main Functions
     ######################################
     log_data(log_output = f"Main Function Start", log_level = "log_output")
 
+    ######################################
+    # Main function helpers
+    #
+
     # instantiate and return the Client System object
     def new_cosmos_client():
         new_client = System(f"{jenkins_hostname_settings()}")
@@ -552,12 +479,13 @@ if __name__ == '__main__':
             log_data(log_output = component.description, log_level = "log_output")
         return new_client
 
-    # instantiate and return the Cosmoserver System object
+    # instantiate and return the CosmostatServer System object
     def new_cosmostat_server():
         new_server = CosmostatServer(name = cosmostat_client.uuid, hostname = jenkins_hostname_settings())
         log_data(log_output = f"New Cosmostat serverobject name: {new_server.name}", log_level = "log_output")
         return new_server
 
+    # helper function for checking if we should update the local System Object this time
     def update_local_system():
         result = False
         if cosmostat_client.check_system_timer():
@@ -569,8 +497,10 @@ if __name__ == '__main__':
         return result
 
     # Background Loop Function
+    # That makes this the service loop
     def background_loop():
         # Update all data on the local System object
+        # This is done on both client and server
         if update_local_system():
             cosmostat_client.update_system_state()
 
@@ -579,6 +509,7 @@ if __name__ == '__main__':
             update_redis_server()
         
         # report data to the server if configured
+        # this is the client-only update function
         if run_cosmostat_reporter():
             if cosmostat_client.check_system_timer or int(time.time()) % 5 == 0:
                 client_update()
@@ -588,53 +519,64 @@ if __name__ == '__main__':
             # purge stale client systems
             cosmostat_server.purge_stale_hostnames()
             # report the server's own client object to itself
+            # uses the helper function upstream of the API route
+            # since the route is already here
             run_update_client(get_client_payload(get_client_redis_data(human_readable = False), "redis_data"))
             log_data(log_output = f"{this_client}", log_level = "noisy_test")
 
+        # short sleep time to keep things chill
         time.sleep(0.2)
 
-    ######################################
+    #
-    # instantiate client 
+    # End Main Functions
     ######################################
 
-    # local client System Class Object
+    #######################################################################
+    #######################################################################
+    ### 
+    ### Here is where this whole thing actually starts doing something
+    ### 
+    #######################################################################
+    #######################################################################
+
+    # First thing is to create the local Cosmostat System
     log_data(log_output = f"Cosmostat Client Start", log_level = "log_output")
+    # Here is the Object Instatiation Command
     cosmostat_client = new_cosmos_client()
+    # Update the system state on new object
     cosmostat_client.update_system_state()
 
-    # remote client reporter
+    # If the option was set to phone home, initialize the remote CosmostatClient Object
     if app_settings["cosmostat_server_reporter"] and not app_settings["cosmostat_server"]:
         log_data(log_output = f"Initialize Client Reporter", log_level = "log_output")
         client_api_initialize()
 
-    ######################################
+    # If this is the server, this will make it the server
-    # instantiate server 
+    # call the object outside the if
-    ######################################
-
     cosmostat_server = None
     if run_cosmostat_server():
         log_data(log_output = f"Cosmostat Server Start", log_level = "log_output")
+        # instantiate it if so
         cosmostat_server = new_cosmostat_server()
         this_client = get_client_payload(get_php_summary(), "client_properties")
+        # extra initial vars
+        # any extra data that should be associated with a CosmostatClient Object should be added here
         this_client["active_interface"] = cosmostat_settings["cosmostat_server_ip"]
         this_client["is_server"] = True
+        # add the server itself as a CosmostatClient object
+        # use the same upstream function as the API endpoint
         timestamp_update = cosmostat_server.add_system(system_dictionary = this_client)
     # if not a server, update client on the server
     else:
         client_update()
 
-    ######################################
     # send initial stats update to redis
-    ######################################
-
     if app_settings["push_redis"] and not app_settings["disable_local_dashboard"]:
         log_data(log_output = f"Initial Redis Push", log_level = "log_output")
         update_redis_server()
 
-    ######################################
+    # Flask scheduler for background loop, run if requested
-    # Flask scheduler for scanner
+    # this is not needed for the API routes
-    ######################################
-
     if app_settings["run_background"] and not app_settings["disable_local_dashboard"]:
         log_data(log_output = f"Background Function Initializing", log_level = "log_output")
         log_data(log_output = "Loading flask background subroutine...", log_level = "log_output")
@@ -645,26 +587,23 @@ if __name__ == '__main__':
                         seconds=app_settings["update_frequency"])
         scheduler.init_app(app)
         scheduler.start()   
-
         log_data(log_output = "...Done", log_level = "log_output")
     else:
         log_data(log_output = "Skipping flask background task", log_level = "log_output")
 
-    ######################################
+    # Flask API, run if requested
-    # Flask API
-    ######################################
     log_data(log_output = f"gateway: {service_gateway_ip()} - port: {service_api_port()}", log_level = "log_output")
     if not app_settings["disable_local_dashboard"]:
         log_data(log_output = f"Main API Start", log_level = "log_output")
         app.run(debug=False, host=service_gateway_ip(), port=service_api_port())
     else:
         # if local API disabled, phone home if configured
+        # if the flask app is not ran, then the background scheduler also does not run
         log_data(log_output = f"Internal API Disabled", log_level = "log_output")
         cosmostat_client.update_system_state()
         client_update()
         while True:    
+            if int(time.time()) % 5 == 0:
                 cosmostat_client.update_system_state()
                 client_update()
-            time.sleep(5)
+            time.sleep(0.5)
-
-    

files/api/archive.py

@@ -0,0 +1,183 @@
+
+
+# dynamic data
+# this will go to the redis server
+@app.route('/dynamic_data', methods=['GET'])
+def dynamic_data():
+    return jsonify(get_dynamic_data())
+
+# static data
+@app.route('/static_data', methods=['GET'])
+def static_data():
+    return jsonify(get_static_data())
+
+# redis data
+@app.route('/redis_data', methods=['GET'])
+def redis_data():
+    return jsonify(get_client_redis_data(human_readable = False))
+
+# redis strings
+@app.route('/redis_strings', methods=['GET'])
+def redis_strings():
+    return jsonify(get_client_redis_data(human_readable = True))
+
+# socket timer data
+@app.route('/timer_data', methods=['GET'])
+def timer_data():
+    time_now = time.time()
+    time_lapsed = time_now - float(cosmostat_client.recent_check)
+    result = {
+        "Time Lapsed": time_lapsed,
+        "Current Time Value": time_now,
+        "Last Update Value": float(cosmostat_client.recent_check),
+        "System Updating": cosmostat_client.check_system_timer()
+    }
+    return jsonify(result)
+
+
+def get_static_data(human_readable = False):
+    result = []
+    return cosmostat_client.get_static_metrics(human_readable)
+
+
+
+def generate_state_definition():
+    result = {
+        "uuid": cosmostat_client.uuid,
+        "state_definition": get_php_summary()
+    }
+    return result
+
+
+# api to validate Cosmostat Class
+@app.route('/client_summary', methods=['GET'])
+def client_summary():
+    result = []
+    if run_cosmostat_server():
+        result = get_client_summary()
+    else:
+        result = {"message": "server not running on this endpoint"}
+    return jsonify(result)
+
+
+# api to get all hostnames
+@app.route('/client_hostnames', methods=['GET'])
+def client_hostnames():
+    result = []
+    if run_cosmostat_server():
+        result = cosmostat_server.get_client_hostnames(send_age = True)
+    else:
+        result = {"message": "server not running on this endpoint"}
+    return jsonify(result)
+
+# api to get server redis data
+@app.route('/get_server_redis', methods=['GET'])
+def get_server_redis():
+    result = []
+    if run_cosmostat_server():
+        result = get_server_redis_data()
+    else:
+        result = {"message": "server not running on this endpoint"}
+    return jsonify(result)
+
+# api to get server redis data
+@app.route('/client_ip_summary', methods=['GET'])
+def client_ip_summary():
+    result = []
+    if run_cosmostat_server():
+        result = get_client_ip_summary()
+    else:
+        result = {"message": "server not running on this endpoint"}
+    return jsonify(result)
+
+
+# generate cosmostat server summary 
+def get_client_summary():
+    result = []
+    for client in cosmostat_server.systems:
+        data_age = time.time() - client.data_timestamp
+        this_client = {
+            "uuid": client.uuid,
+            "short_id": client.name,
+            "data_age": data_age,
+            "hostname": client.hostname
+        }
+        result.append(this_client)
+    if result == []:
+        result = {"message": "no clients reporting"}
+    return result
+
+    ########################################################
+    # various data functions
+    ########################################################
+    
+    # complex data type return
+    def get_property_summary(self, type = None):
+        these_properties = []
+        if type == None:
+            for name, value in self._properties.items():
+                these_properties.append({"Property": name, "Value": value})
+        else:
+            for name, value in self._properties.items():
+                if type in name: 
+                    these_properties.append({"Property": name, "Value": value})
+        result = {
+            "Source": self.name,
+            "Component Type": self.type,
+            "Properties": these_properties
+        }
+        return result
+    
+    # simple data value return
+    def get_property_value(self, type = None):
+        result = []
+        print(f"Component type: {type}")
+        for name, value in self._properties.items():
+            print(f"Component Property: {name}")
+            if type in name: 
+                result.append(value)
+        if len(result) == 1:
+            return result[0]
+        else:
+            return result
+    
+    # full data return
+    def get_description(self):
+        these_properties = []
+        for name, value in self._properties.items():
+            these_properties.append({"Property": name, "Value": value})
+        these_metrics = []
+        for name, value in self._metrics.items():
+            these_metrics.append({"Metric": name, "Data": value})
+        result = {
+            "Source": self.name,
+            "Type": self.type,
+            "Properties": these_properties,
+            "Metrics": these_metrics
+        }
+        return result
+        
+    # complex data type return
+    def get_metrics(self, type = None):
+        these_metrics = []
+        if type == None:
+            for name, value in self._metrics:
+                these_metrics.append({"Metric": name, "Data": value})
+        else:
+            for name, value in self._metrics:
+                if name == type: 
+                    these_metrics.append({"Metric": name, "Data": value})
+        result = {
+            "Source": self.name,
+            "Component Type": self.type,
+            "Metrics": these_metrics
+        }
+        return result
+
+
+
+
+
+
+
+

files/archive/web.yaml

@@ -7,7 +7,7 @@
   when: not quick_refresh | bool
   community.docker.docker_compose_v2:
     project_src: "{{ service_control_web_folder }}"
-    state: stopped
+    state: "{{ 'stopped' if quick_refresh | bool else 'absent' }}"
   ignore_errors: yes
 
 # Create web Folder

files/docker/Dockerfile

@@ -8,6 +8,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     redis-server nginx \
     # Process supervisor
     supervisor \
+    # Others
+    net-tools  \
     # Clean up
     && rm -rf /var/lib/apt/lists/*
 
@@ -22,11 +24,12 @@ COPY cosmostat_settings.yaml /app/cosmostat_settings.yaml
 # Node on 3000
 WORKDIR /usr/src/app
 COPY web/node_server/ .
+#COPY cosmostat_settings.yaml /usr/src/app/cosmostat_settings.yaml
 RUN npm install --only=production
 
 # Apache on 8080
-RUN sed -i 's/^Listen .*/Listen 8080/' /etc/apache2/ports.conf && \
+COPY apache_ports.conf /etc/apache2/ports.conf
-    sed -i 's/<VirtualHost \*:80>/<VirtualHost \*:8080>/' /etc/apache2/sites-enabled/000-default.conf
+COPY apache_vhost.conf /etc/apache2/sites-available/000-default.conf
 COPY web/html/ /var/www/html/
 
 # nginx on 80

files/docker/apache_ports.conf

@@ -0,0 +1,5 @@
+# Listen on 8080 inside the container
+Listen 8080
+
+# If you still want Apache to listen on 80 (rare), add it back:
+# Listen 80

files/docker/apache_vhost.conf

@@ -0,0 +1,14 @@
+<VirtualHost *:8080>
+    ServerAdmin webmaster@localhost
+    DocumentRoot /var/www/html
+
+    # Log files
+    ErrorLog ${APACHE_LOG_DIR}/error.log
+    CustomLog ${APACHE_LOG_DIR}/access.log combined
+
+    # If you need PHP processing
+    <Directory /var/www/html>
+        AllowOverride All
+        Require all granted
+    </Directory>
+</VirtualHost>

files/docker/supervisord.conf

@@ -38,7 +38,8 @@ priority=3
 # ------------------------------------------------------------------
 # NOTE: Adjust the command/path to match your app
 [program:node]
-command=npm start
+command=sh -c "npm install && node server.js"
+#command=npm start
 directory=/usr/src/app
 stdout_logfile=/dev/stdout
 stderr_logfile=/dev/stderr

files/docker/web/node_server/server.js

@@ -17,7 +17,8 @@ const io     = new Server(server);
 /* --------------------------------------------------------------------- */
 let config = {};
 try {
-  const file = fs.readFileSync(path.resolve(__dirname, 'cosmostat_settings.yaml'), 'utf8');
+  const filePath = '/app/cosmostat_settings.yaml';
+  const file     = fs.readFileSync(filePath, 'utf8');
   config = yaml.load(file);
 } catch (e) {
   console.error('Failed to load config.yaml:', e);

files/docker/web/proxy/nginx.conf

@@ -39,7 +39,7 @@ server_name localhost;
     # WebSocket endpoint
     # ---------------------------------------
     location /socket.io/ {
-        proxy_pass http://192.168.37.1:3000/socket.io/;
+        proxy_pass http://0.0.0.0:3000/socket.io/;
         proxy_http_version 1.1;
         proxy_set_header Upgrade $http_upgrade;
         proxy_set_header Connection "upgrade";
@@ -53,7 +53,7 @@ server_name localhost;
     # All other paths → Apache (PHP)
     # ---------------------------------------
     location / {
-        proxy_pass http://192.168.37.1:8080;
+        proxy_pass http://0.0.0.0:8080;
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

tasks/docker.yaml

@@ -6,9 +6,16 @@
 - name: Cosmostat - Web - stop containers
   community.docker.docker_compose_v2:
     project_src: "{{ service_control_docker_folder }}"
-    state: stopped
+    state: "{{ 'stopped' if quick_refresh | bool else 'absent' }}"
   ignore_errors: yes
 
+- name: Cosmostat - Web - Remove Cosmostat Image
+  when: not quick_refresh | bool
+  community.docker.docker_image:
+    name: "cosmostat-dash"
+    tag: latest
+    state: absent
+
 # Create web Folder
 - name: "Cosmostat - Web - create {{ service_control_docker_folder }}"
   file:
@@ -71,6 +78,7 @@
   community.docker.docker_compose_v2:
     project_src: "{{ service_control_docker_folder }}"
     state: present
+    build: "{{ 'always' if not quick_refresh | bool else 'never' }}"
   register: docker_output
 - debug: |
           msg="{{ docker_output.actions }}"

tasks/main.yaml

@@ -18,6 +18,6 @@
 # set up web stack 
 - name: Build Web Dashboard
   when: not disable_local_dashboard | bool
-  include_tasks: web.yaml
+  include_tasks: docker.yaml
 
 ...

templates/docker-compose.yaml

@@ -3,21 +3,24 @@
 services:
 
   cosmostat-dash:
-    container_name: cosmostat-dash
-    image: cosmostat-dash:latest
-    restart: always    
     build:
       context: .
       dockerfile: Dockerfile
+    container_name: cosmostat-dash
+    image: cosmostat-dash:latest
+    restart: always    
     networks:
       - cosmostat_net
     ports:
       - "{{ docker_gateway }}:6379:6379"
       - "{{ (docker_gateway + ':') if not public_dashboard | bool else '' }}{{ custom_port }}:80"
-    volumes:
+    # When the container is built in Ansible, these are all copied
-#      - "/opt/cosmostat/docker/web/html:/var/www/html"
+    # if any changes are made manually on the endpoint, uncomment as needed
-#      - "/opt/cosmostat/docker/web/node_server:/app"
+    #volumes:
-      - "/opt/cosmostat/api/cosmostat_settings.yaml:/app/cosmostat_settings.yaml:ro"
+    #  - "/opt/cosmostat/api/cosmostat_settings.yaml:/app/cosmostat_settings.yaml:ro"
+    #  - "/opt/cosmostat/api/cosmostat_settings.yaml:/usr/src/app/cosmostat_settings.yaml:ro"
+    #  - "/opt/cosmostat/docker/web/html:/var/www/html"
+    #  - "/opt/cosmostat/docker/web/node_server:/app"
 
 networks:
   cosmostat_net: