####################################################################### ### app.py ### cosmostat service handler ####################################################################### from flask import Flask, jsonify, request, Response, abort from flask_apscheduler import APScheduler from typing import Dict, Union import json, time, redis, yaml, datetime import secrets, string import requests from requests import RequestException, Response # Import Cosmos Settings from Cosmos_Settings import * # System and Component Classes from Components import * # Cosmostat server Classes from Cosmostat import * # declare flask apps app = Flask(__name__) scheduler = APScheduler() ####################################################################### ### Redis Functions ####################################################################### # Redis client - will publish updates r = redis.Redis(host=redis_gateway_ip(), port=6379) # Publish to the specified Redis channel def update_redis_channel(redis_channel, data): r.publish(redis_channel, json.dumps(data)) log_data(log_output = data, log_level = "noisy_test") # update redis server for both modes def update_redis_server(): # Client Redis Tree if cosmostat_client.check_system_timer(): update_redis_channel("host_metrics", get_client_redis_data(human_readable = False)) if run_cosmostat_server(): update_redis_channel("client_summary", get_server_redis_data()) # function helper for both modes def get_client_redis_data(human_readable = False): result = [] for metric in get_dynamic_data(human_readable): result.append(metric) return result def get_server_redis_data(): result = [] for client in cosmostat_server.systems: this_client_key = { "hostname": client.hostname, "data_timestamp": client.data_timestamp, "uuid": client.uuid, "short_id": client.name, "active_ip": client.active_ip, "is_server": client.is_server, "redis_data": client.redis_data } result.append(this_client_key) return result ####################################################################### ### Client Flask Routes ####################################################################### # php summary @app.route('/php_summary', methods=['GET']) def php_summary(): return jsonify(get_php_summary()) # socket timer handler @app.route('/start_timer', methods=['GET']) def start_timer(): current_timestamp = int(time.time()) cosmostat_client.recent_check = current_timestamp log_data(log_output = f"Timestamp updated to {cosmostat_client.recent_check}", log_level = "noisy_test") return jsonify({ "message": "websocket timer reset", "new_timestamp": cosmostat_client.recent_check }) # test route @app.route('/test', methods=['GET']) def test(): this_cpu = cosmostat_client.get_components(component_type="CPU") return jsonify( { "component_count:": len(cosmostat_client.components), "user": jenkins_user_settings(), "hostname": jenkins_hostname_settings(), "cpu_model": this_cpu[0].description } ) ####################################################################### ### Client Flask Helpers ####################################################################### # needs to return array of {name: name, type: type, metrics: metrics} # for redis table generation, includes system and component metrics def get_dynamic_data(human_readable = False): return cosmostat_client.get_live_metrics(human_readable) # this generates the static data used by the client dashboard and the CosmostatServer properties def get_php_summary(): # separate lists for system properties and components system_properties = cosmostat_client.get_system_properties(human_readable = True, php_extra = True) system_components = [] # build a list of Component property lists for component in cosmostat_client.get_components(): this_component = { "component_name": component.name, "info_strings": component.get_properties_strings(return_simple = True) # this is a list of strings from the Class } system_components.append(this_component) this_primary_ip = cosmostat_client.primary_ip # if this is the server there are extra steps here to add all client data to the return # this can make for a big json if run_cosmostat_server(): # get uuid for reference client_uuid = cosmostat_server.get_uuid_from_hostname(cosmostat_client.name) # get timestamp for bubble data_timestamp = cosmostat_server.get_system(client_uuid) log_string = f"Cosmostat Client Name: {cosmostat_client.name} - UUID: {client_uuid}- Timestamp: {data_timestamp}" log_data(log_output = log_string, log_level = "log_output") # get the primary IP for inventory file generation this_primary_ip = cosmostat_settings["cosmostat_server_ip"] # build and return dict of all these lists result = [{ "system_properties": system_properties, "system_components": system_components, "active_interface": this_primary_ip }] return result ####################################################################### ### Server Flask Routes ####################################################################### # return full descriptor - public route @app.route('/descriptor', methods=['GET']) def descriptor(): return jsonify(cosmostat_client.get_component_class_tree()) # update client on server - public route @app.route('/update_client', methods=['POST']) def update_client(): result = {} # sanitize and validate POST data payload = client_submit_check(request = request) # process client update request result = run_update_client(payload) return jsonify(result), 200 # create client on server - public route @app.route('/create_client', methods=['POST']) def create_client(): result = {} # sanitize and validate POST data payload = client_submit_check(request = request) # if the client does not exist, create it if not cosmostat_server.check_uuid(payload["uuid"]): result = run_create_client(payload) else: result = { "message": "object already exists, skipping creation", "system_exists": "True" } return jsonify(result), 200 # api to pull all data for server dashboard rendering @app.route('/client_details', methods=['GET']) def client_details(): result = [] if run_cosmostat_server(): result = get_client_details() else: result = {"message": "server not running on this endpoint"} return jsonify(result) # return inventory file of all clients for update @app.route("/client_inventory", methods=["GET"]) def client_inventory(): return build_inventory() ####################################################################### ### Server Flask Helpers ####################################################################### # this is the helper function for processing client submission requests # this only updates the dynamic metrics # the output of this function is returned to the flask function # This function is also used to report the cosmostat server's CosmostatClient def run_update_client(this_client): # this is where the API check is done if public_api_check(this_client): # return if object does not exist if not cosmostat_server.check_uuid(this_client["uuid"]): return { "message": "client not found" } else: # this is where the CosmostatClient object is updated # done by passing the UUID and redis data # that Class Function returns the timestamp timestamp_update = cosmostat_server.update_system( system_dictionary = this_client["redis_data"], system_uuid = this_client["uuid"] ) # after this happens, record a successful status update_status = f'updated client {this_client["short_id"]}' # build a return string and send it return { "status": update_status, "client_updated": "True", "uuid": this_client["uuid"], "redis_data": this_client, "timestamp_update": timestamp_update } # if the API check fails, return as much else: return{ "status": "api failure" } # this is the helper function for processing client creation requests # this creates a new CosmostatClient object on the 'cosmostat_server' Object # the output of this function is returned to the flask function def run_create_client(this_client): # this is where the API check is done if public_api_check(this_client): # this is where the CosmostatClient object is inialized # done by passing the UUID and all static data # that Class Function returns the timestamp timestamp_update = cosmostat_server.add_system(system_dictionary = this_client) # after this happens, record a successful status update_status = f'created client {this_client["short_id"]}' # build a return string and send it return { "status": update_status, "client_created": "True", "uuid": this_client["uuid"], "client_properties": this_client, "timestamp_update": timestamp_update } # if the API check fails, return as much else: return{ "status": "api failure" } # API check helper def public_api_check(this_client): result = False default_key = ''.join(secrets.choice(string.ascii_letters + string.digits) for _ in range(256)) api_key = this_client.get('API_KEY', default_key) if api_key == app_settings["REAL_API_KEY"]: result = True return result # flask submission check function # all this does is take the POST data from # remote client submissions and sanitize and validate # it returns the jsonified submission after that def client_submit_check(request): payload = {} required_keys = {"uuid", "short_id", "hostname"} if not request.is_json: logging.warning("Received non-JSON request") return jsonify({"error": "Content-type must be application/json"}), 400 payload = request.get_json(silent=True) if payload is None: logging.warning("Malformed JSON body") return jsonify({"error": "Malformed JSON"}), 400 missing = required_keys - payload.keys() if missing: raise ValueError(f"Missing required keys: {', '.join(sorted(missing))}") return payload # no redis data needed here def get_client_details(): result = [] for client in cosmostat_server.systems: data_age = time.time() - client.data_timestamp this_client = { "uuid": client.uuid, "short_id": client.name, "client_properties": client.client_properties, # "redis_data": client.redis_data, "hostname": client.hostname } result.append(this_client) if result == []: result = {"message": "no clients reporting"} return result # get client IP summary either in a Dict or List def get_client_ip_summary(return_list = False): result = [] for client in cosmostat_server.systems: # build the Dict device_info = { "small_id": client.name, "uuid": client.uuid, "hostname": client.hostname, "lan_ip": client.active_ip, } # build the result depending on mode if client.hostname != cosmostat_server.hostname: if return_list: result.append(client.active_ip) # only the list else: result.append(device_info) # the full dictionary return result # build client ansible inventory file def build_inventory(): all_ips = get_client_ip_summary(return_list = True) # subnets reachable from the cosmostat server cosmos_subnets = [ "172.25.1.0/24", "172.20.0.0/16", "172.19.10.0/24", "10.200.26.0/24", "10.200.27.0/24", "192.168.60.0/24", ] # inialize lists ips = [] bad_ips = [] # build list of active IPs for ip in all_ips: for subnet in cosmos_subnets: if is_ip_in_subnets(ip, subnet) and cosmostat_server.get_vpn_ip(ip, just_check = True): ips.append(ip) # list of unreachable IPs for ip in all_ips: if ip not in ips: bad_ips.append(ip) # any system with a VPN to jenkins, add to inventory for ip in bad_ips: ips.append(cosmostat_server.get_vpn_ip(ip)) hosts = {ip: {"ansible_host": ip} for ip in ips} inventory = { "all": { "hosts": hosts, "vars": { "refresh_only": "true", "ansible_connection": "ssh", "ansible_ssh_private_key_file": "/var/jenkins_home/jenkins_key", "ansible_python_interpreter": "/usr/bin/python3", "jenkins_user": 'automate', "jenkins_group": 'Jenkins-Admin', "subnet_group_check": 'Jenkins-AllSubnets', "SERVER_SUBNET_GROUP": 'Jenkins-AllSubnets', "inventory_generation_timestamp": f"{datetime.datetime.now().isoformat()}", "playbook_file": '/var/jenkins_home/ansible/playbooks/cosmostat.yaml', "quick_refresh": "true", "noisy_test": "false", "debug_output": "false", "push_redis": "false", "run_background": "true", "log_output": "true", "public_dashboard": "false", "custom_port": "80", "custom_api_port": "5000", "cosmostat_server_reporter": "true", "cosmostat_server": "false", "secure_api": "false", "disable_local_dashboard": "true", "REAL_API_KEY": f"{cosmostat_settings['REAL_API_KEY']}" }, } } return yaml.safe_dump( inventory, default_flow_style=False, sort_keys=False, ) ####################################################################### ### Cosmostat Client Subroutines ### These are only for the clients ### The cosmostat server has its own functions ####################################################################### # Cosmostat Client Reporter Handler # this handles updating the active data for the local client instance # to the remote cosmostat server def client_update(): # update client API path api_url = f"{cosmostat_server_api()}update_client" payload = get_client_payload(get_client_redis_data(human_readable = False), "redis_data") log_data(log_output = f"API Update Called - {api_url}", log_level = "debug_output") log_data(log_output = "client_update - redis data from local client:", log_level = "noisy_test") log_data(log_output = payload, log_level = "noisy_test") # execute API call, we do need the result result = client_submission_handler(api_url, payload) # if this client does not exist on the server, we need to try to initialize it if not result or not result.get('client_updated'): log_data(log_output = f"Client not updated, initializing", log_level = "log_output") client_api_initialize() # Cosmostat Client Initializer # this handles inializing the CosmostatClient object on the remote cosmostat server def client_api_initialize(): # create client API path api_url = f"{cosmostat_server_api()}create_client" # generate payload for the remote server payload = get_client_payload(get_php_summary(), "client_properties") # add extra init vars payload["active_interface"] = cosmostat_client.primary_ip payload["is_server"] = False # execute API call on remote server client_submission_handler(api_url, payload) # this returns the payload in the format that the API expects with extra necessary data # used for both initializing and updating; dictionary name is part of the function def get_client_payload(system_dictionary: dict, dictionary_name: str): this_uuid = cosmostat_client.uuid this_short_id = cosmostat_client.short_id this_hostname = cosmostat_client.name payload = { "uuid": this_uuid, "short_id": this_short_id, "hostname": this_hostname, dictionary_name: system_dictionary, "API_KEY": app_settings["REAL_API_KEY"] } return payload # Cosmostat Client API Reporting Handler # this helper makes the POST request to the remote cosmostat server # the payload and URL are passed to this function def client_submission_handler(api_url: str, payload: dict): result = None try: # this is the actual submission to the server response: Response = requests.post(api_url, json=payload, timeout=4) # raise HTTPError for 4xx/5xx if failure response.raise_for_status() except RequestException as exc: # Wrap the low-level exception in a more descriptive one log_data(log_output = f"Failed to POST to {api_url!r}: {exc}", log_level = "log_output") # process reply from API try: result = response.json() except ValueError as exc: log_data(log_output = f"Server responded with non-JSON payload: {response.text!r}", log_level = "log_output") return result ####################################################################### ####################################################################### ### Main Subroutine ####################################################################### ####################################################################### if __name__ == '__main__': ###################################### ### Main Functions ###################################### log_data(log_output = f"Main Function Start", log_level = "log_output") ###################################### # Main function helpers # # instantiate and return the Client System object def new_cosmos_client(): new_client = System(f"{jenkins_hostname_settings()}") log_data(log_output = f"New System object name: {new_client.name} - {new_client.get_component_count()} components:", log_level = "log_output") for component in new_client.components: log_data(log_output = component.description, log_level = "log_output") return new_client # instantiate and return the CosmostatServer System object def new_cosmostat_server(): new_server = CosmostatServer(name = cosmostat_client.uuid, hostname = jenkins_hostname_settings()) log_data(log_output = f"New Cosmostat serverobject name: {new_server.name}", log_level = "log_output") return new_server # helper function for checking if we should update the local System Object this time def update_local_system(): result = False if cosmostat_client.check_system_timer(): result = True if run_cosmostat_server(): result = True if run_cosmostat_reporter() and int(time.time()) % 5 == 0: result = True return result # Background Loop Function # That makes this the service loop def background_loop(): # Update all data on the local System object # This is done on both client and server if update_local_system(): cosmostat_client.update_system_state() # publish to redis if the web dashboard is active locally if app_settings["push_redis"] and not app_settings["disable_local_dashboard"]: update_redis_server() # report data to the server if configured # this is the client-only update function if run_cosmostat_reporter(): if cosmostat_client.check_system_timer or int(time.time()) % 5 == 0: client_update() # if this is the server, do this stuff if run_cosmostat_server(): # purge stale client systems cosmostat_server.purge_stale_hostnames() # report the server's own client object to itself # uses the helper function upstream of the API route # since the route is already here run_update_client(get_client_payload(get_client_redis_data(human_readable = False), "redis_data")) log_data(log_output = f"{this_client}", log_level = "noisy_test") # short sleep time to keep things chill time.sleep(0.2) # # End Main Functions ###################################### ####################################################################### ####################################################################### ### ### Here is where this whole thing actually starts doing something ### ####################################################################### ####################################################################### # First thing is to create the local Cosmostat System log_data(log_output = f"Cosmostat Client Start", log_level = "log_output") # Here is the Object Instatiation Command cosmostat_client = new_cosmos_client() # Update the system state on new object cosmostat_client.update_system_state() # If the option was set to phone home, initialize the remote CosmostatClient Object if app_settings["cosmostat_server_reporter"] and not app_settings["cosmostat_server"]: log_data(log_output = f"Initialize Client Reporter", log_level = "log_output") client_api_initialize() # If this is the server, this will make it the server # call the object outside the if cosmostat_server = None if run_cosmostat_server(): log_data(log_output = f"Cosmostat Server Start", log_level = "log_output") # instantiate it if so cosmostat_server = new_cosmostat_server() this_client = get_client_payload(get_php_summary(), "client_properties") # extra initial vars # any extra data that should be associated with a CosmostatClient Object should be added here this_client["active_interface"] = cosmostat_settings["cosmostat_server_ip"] this_client["is_server"] = True # add the server itself as a CosmostatClient object # use the same upstream function as the API endpoint timestamp_update = cosmostat_server.add_system(system_dictionary = this_client) # if not a server, update client on the server else: client_update() # send initial stats update to redis if app_settings["push_redis"] and not app_settings["disable_local_dashboard"]: log_data(log_output = f"Initial Redis Push", log_level = "log_output") update_redis_server() # Flask scheduler for background loop, run if requested # this is not needed for the API routes if app_settings["run_background"] and not app_settings["disable_local_dashboard"]: log_data(log_output = f"Background Function Initializing", log_level = "log_output") log_data(log_output = "Loading flask background subroutine...", log_level = "log_output") scheduler.add_job(id='background_loop', func=background_loop, trigger='interval', seconds=app_settings["update_frequency"]) scheduler.init_app(app) scheduler.start() log_data(log_output = "...Done", log_level = "log_output") else: log_data(log_output = "Skipping flask background task", log_level = "log_output") # Flask API, run if requested log_data(log_output = f"gateway: {service_gateway_ip()} - port: {service_api_port()}", log_level = "log_output") if not app_settings["disable_local_dashboard"]: log_data(log_output = f"Main API Start", log_level = "log_output") app.run(debug=False, host=service_gateway_ip(), port=service_api_port()) else: # if local API disabled, phone home if configured # if the flask app is not ran, then the background scheduler also does not run log_data(log_output = f"Internal API Disabled", log_level = "log_output") cosmostat_client.update_system_state() client_update() while True: if int(time.time()) % 5 == 0: cosmostat_client.update_system_state() client_update() time.sleep(0.5)