611 lines
24 KiB
Python
611 lines
24 KiB
Python
#######################################################################
|
|
### app.py
|
|
### cosmostat service handler
|
|
#######################################################################
|
|
|
|
from flask import Flask, jsonify, request, Response, abort
|
|
from flask_apscheduler import APScheduler
|
|
from typing import Dict, Union
|
|
import json, time, redis, yaml, datetime
|
|
import secrets, string
|
|
import requests
|
|
from requests import RequestException, Response
|
|
|
|
# Import Cosmos Settings
|
|
from Cosmos_Settings import *
|
|
# System and Component Classes
|
|
from Components import *
|
|
# Cosmostat server Classes
|
|
from Cosmostat import *
|
|
|
|
# declare flask apps
|
|
app = Flask(__name__)
|
|
scheduler = APScheduler()
|
|
|
|
#######################################################################
|
|
### Redis Functions
|
|
#######################################################################
|
|
|
|
# Redis client - will publish updates
|
|
r = redis.Redis(host=redis_gateway_ip(), port=6379)
|
|
|
|
# Publish to the specified Redis channel
|
|
def update_redis_channel(redis_channel, data):
|
|
r.publish(redis_channel, json.dumps(data))
|
|
log_data(log_output = data, log_level = "noisy_test")
|
|
|
|
# update redis server for both modes
|
|
def update_redis_server():
|
|
# Client Redis Tree
|
|
if cosmostat_client.check_system_timer():
|
|
update_redis_channel("host_metrics", get_client_redis_data(human_readable = False))
|
|
|
|
if run_cosmostat_server():
|
|
update_redis_channel("client_summary", get_server_redis_data())
|
|
|
|
# function helper for both modes
|
|
def get_client_redis_data(human_readable = False):
|
|
result = []
|
|
for metric in get_dynamic_data(human_readable):
|
|
result.append(metric)
|
|
return result
|
|
|
|
def get_server_redis_data():
|
|
result = []
|
|
for client in cosmostat_server.systems:
|
|
this_client_key = {
|
|
"hostname": client.hostname,
|
|
"data_timestamp": client.data_timestamp,
|
|
"uuid": client.uuid,
|
|
"short_id": client.name,
|
|
"active_ip": client.active_ip,
|
|
"is_server": client.is_server,
|
|
"redis_data": client.redis_data
|
|
}
|
|
result.append(this_client_key)
|
|
return result
|
|
|
|
#######################################################################
|
|
### Client Flask Routes
|
|
#######################################################################
|
|
|
|
# php summary
|
|
@app.route('/php_summary', methods=['GET'])
|
|
def php_summary():
|
|
return jsonify(get_php_summary())
|
|
|
|
# socket timer handler
|
|
@app.route('/start_timer', methods=['GET'])
|
|
def start_timer():
|
|
current_timestamp = int(time.time())
|
|
cosmostat_client.recent_check = current_timestamp
|
|
log_data(log_output = f"Timestamp updated to {cosmostat_client.recent_check}", log_level = "noisy_test")
|
|
return jsonify({
|
|
"message": "websocket timer reset",
|
|
"new_timestamp": cosmostat_client.recent_check
|
|
})
|
|
|
|
# test route
|
|
@app.route('/test', methods=['GET'])
|
|
def test():
|
|
this_cpu = cosmostat_client.get_components(component_type="CPU")
|
|
return jsonify(
|
|
{
|
|
"component_count:": len(cosmostat_client.components),
|
|
"user": jenkins_user_settings(),
|
|
"hostname": jenkins_hostname_settings(),
|
|
"cpu_model": this_cpu[0].description
|
|
}
|
|
)
|
|
|
|
#######################################################################
|
|
### Client Flask Helpers
|
|
#######################################################################
|
|
|
|
# needs to return array of {name: name, type: type, metrics: metrics}
|
|
# for redis table generation, includes system and component metrics
|
|
def get_dynamic_data(human_readable = False):
|
|
return cosmostat_client.get_live_metrics(human_readable)
|
|
|
|
# this generates the static data used by the client dashboard and the CosmostatServer properties
|
|
def get_php_summary():
|
|
# separate lists for system properties and components
|
|
system_properties = cosmostat_client.get_system_properties(human_readable = True, php_extra = True)
|
|
system_components = []
|
|
# build a list of Component property lists
|
|
for component in cosmostat_client.get_components():
|
|
this_component = {
|
|
"component_name": component.name,
|
|
"info_strings": component.get_properties_strings(return_simple = True) # this is a list of strings from the Class
|
|
}
|
|
system_components.append(this_component)
|
|
this_primary_ip = cosmostat_client.primary_ip
|
|
# if this is the server there are extra steps here to add all client data to the return
|
|
# this can make for a big json
|
|
if run_cosmostat_server():
|
|
# get uuid for reference
|
|
client_uuid = cosmostat_server.get_uuid_from_hostname(cosmostat_client.name)
|
|
# get timestamp for bubble
|
|
data_timestamp = cosmostat_server.get_system(client_uuid)
|
|
log_string = f"Cosmostat Client Name: {cosmostat_client.name} - UUID: {client_uuid}- Timestamp: {data_timestamp}"
|
|
log_data(log_output = log_string, log_level = "log_output")
|
|
# get the primary IP for inventory file generation
|
|
this_primary_ip = cosmostat_settings["cosmostat_server_ip"]
|
|
# build and return dict of all these lists
|
|
result = [{
|
|
"system_properties": system_properties,
|
|
"system_components": system_components,
|
|
"active_interface": this_primary_ip
|
|
}]
|
|
return result
|
|
|
|
#######################################################################
|
|
### Server Flask Routes
|
|
#######################################################################
|
|
|
|
# return full descriptor - public route
|
|
@app.route('/descriptor', methods=['GET'])
|
|
def descriptor():
|
|
return jsonify(cosmostat_client.get_component_class_tree())
|
|
|
|
# update client on server - public route
|
|
@app.route('/update_client', methods=['POST'])
|
|
def update_client():
|
|
result = {}
|
|
# sanitize and validate POST data
|
|
payload = client_submit_check(request = request)
|
|
# process client update request
|
|
result = run_update_client(payload)
|
|
return jsonify(result), 200
|
|
|
|
# create client on server - public route
|
|
@app.route('/create_client', methods=['POST'])
|
|
def create_client():
|
|
result = {}
|
|
# sanitize and validate POST data
|
|
payload = client_submit_check(request = request)
|
|
# if the client does not exist, create it
|
|
if not cosmostat_server.check_uuid(payload["uuid"]):
|
|
result = run_create_client(payload)
|
|
else:
|
|
result = {
|
|
"message": "object already exists, skipping creation",
|
|
"system_exists": "True"
|
|
}
|
|
return jsonify(result), 200
|
|
|
|
# api to pull all data for server dashboard rendering
|
|
@app.route('/client_details', methods=['GET'])
|
|
def client_details():
|
|
result = []
|
|
if run_cosmostat_server():
|
|
result = get_client_details()
|
|
else:
|
|
result = {"message": "server not running on this endpoint"}
|
|
return jsonify(result)
|
|
|
|
# return inventory file of all clients for update
|
|
@app.route("/client_inventory", methods=["GET"])
|
|
def client_inventory():
|
|
return build_inventory()
|
|
|
|
#######################################################################
|
|
### Server Flask Helpers
|
|
#######################################################################
|
|
|
|
# this is the helper function for processing client submission requests
|
|
# this only updates the dynamic metrics
|
|
# the output of this function is returned to the flask function
|
|
# This function is also used to report the cosmostat server's CosmostatClient
|
|
def run_update_client(this_client):
|
|
# this is where the API check is done
|
|
if public_api_check(this_client):
|
|
# return if object does not exist
|
|
if not cosmostat_server.check_uuid(this_client["uuid"]):
|
|
return { "message": "client not found" }
|
|
else:
|
|
# this is where the CosmostatClient object is updated
|
|
# done by passing the UUID and redis data
|
|
# that Class Function returns the timestamp
|
|
timestamp_update = cosmostat_server.update_system(
|
|
system_dictionary = this_client["redis_data"],
|
|
system_uuid = this_client["uuid"]
|
|
)
|
|
# after this happens, record a successful status
|
|
update_status = f'updated client {this_client["short_id"]}'
|
|
# build a return string and send it
|
|
return {
|
|
"status": update_status,
|
|
"client_updated": "True",
|
|
"uuid": this_client["uuid"],
|
|
"redis_data": this_client,
|
|
"timestamp_update": timestamp_update
|
|
}
|
|
# if the API check fails, return as much
|
|
else:
|
|
return{
|
|
"status": "api failure"
|
|
}
|
|
|
|
# this is the helper function for processing client creation requests
|
|
# this creates a new CosmostatClient object on the 'cosmostat_server' Object
|
|
# the output of this function is returned to the flask function
|
|
def run_create_client(this_client):
|
|
# this is where the API check is done
|
|
if public_api_check(this_client):
|
|
# this is where the CosmostatClient object is inialized
|
|
# done by passing the UUID and all static data
|
|
# that Class Function returns the timestamp
|
|
timestamp_update = cosmostat_server.add_system(system_dictionary = this_client)
|
|
# after this happens, record a successful status
|
|
update_status = f'created client {this_client["short_id"]}'
|
|
# build a return string and send it
|
|
return {
|
|
"status": update_status,
|
|
"client_created": "True",
|
|
"uuid": this_client["uuid"],
|
|
"client_properties": this_client,
|
|
"timestamp_update": timestamp_update
|
|
}
|
|
# if the API check fails, return as much
|
|
else:
|
|
return{
|
|
"status": "api failure"
|
|
}
|
|
|
|
# API check helper
|
|
def public_api_check(this_client):
|
|
result = False
|
|
default_key = ''.join(secrets.choice(string.ascii_letters + string.digits) for _ in range(256))
|
|
api_key = this_client.get('API_KEY', default_key)
|
|
if api_key == app_settings["REAL_API_KEY"]:
|
|
result = True
|
|
return result
|
|
|
|
# flask submission check function
|
|
# all this does is take the POST data from
|
|
# remote client submissions and sanitize and validate
|
|
# it returns the jsonified submission after that
|
|
def client_submit_check(request):
|
|
payload = {}
|
|
required_keys = {"uuid", "short_id", "hostname"}
|
|
if not request.is_json:
|
|
logging.warning("Received non-JSON request")
|
|
return jsonify({"error": "Content-type must be application/json"}), 400
|
|
payload = request.get_json(silent=True)
|
|
if payload is None:
|
|
logging.warning("Malformed JSON body")
|
|
return jsonify({"error": "Malformed JSON"}), 400
|
|
missing = required_keys - payload.keys()
|
|
if missing:
|
|
raise ValueError(f"Missing required keys: {', '.join(sorted(missing))}")
|
|
return payload
|
|
|
|
# no redis data needed here
|
|
def get_client_details():
|
|
result = []
|
|
for client in cosmostat_server.systems:
|
|
data_age = time.time() - client.data_timestamp
|
|
this_client = {
|
|
"uuid": client.uuid,
|
|
"short_id": client.name,
|
|
"client_properties": client.client_properties,
|
|
# "redis_data": client.redis_data,
|
|
"hostname": client.hostname
|
|
}
|
|
result.append(this_client)
|
|
if result == []:
|
|
result = {"message": "no clients reporting"}
|
|
return result
|
|
|
|
# get client IP summary either in a Dict or List
|
|
def get_client_ip_summary(return_list = False):
|
|
result = []
|
|
for client in cosmostat_server.systems:
|
|
# build the Dict
|
|
device_info = {
|
|
"small_id": client.name,
|
|
"uuid": client.uuid,
|
|
"hostname": client.hostname,
|
|
"lan_ip": client.active_ip,
|
|
}
|
|
# build the result depending on mode
|
|
if client.hostname != cosmostat_server.hostname:
|
|
if return_list:
|
|
result.append(client.active_ip) # only the list
|
|
else:
|
|
result.append(device_info) # the full dictionary
|
|
return result
|
|
|
|
# build client ansible inventory file
|
|
def build_inventory():
|
|
all_ips = get_client_ip_summary(return_list = True)
|
|
# subnets reachable from the cosmostat server
|
|
cosmos_subnets = [
|
|
"172.25.1.0/24",
|
|
"172.20.0.0/16",
|
|
"172.19.10.0/24",
|
|
"10.200.26.0/24",
|
|
"10.200.27.0/24",
|
|
"192.168.60.0/24",
|
|
]
|
|
# inialize lists
|
|
ips = []
|
|
bad_ips = []
|
|
# build list of active IPs
|
|
for ip in all_ips:
|
|
for subnet in cosmos_subnets:
|
|
if is_ip_in_subnets(ip, subnet) and cosmostat_server.get_vpn_ip(ip, just_check = True):
|
|
ips.append(ip)
|
|
# list of unreachable IPs
|
|
for ip in all_ips:
|
|
if ip not in ips:
|
|
bad_ips.append(ip)
|
|
# any system with a VPN to jenkins, add to inventory
|
|
for ip in bad_ips:
|
|
ips.append(cosmostat_server.get_vpn_ip(ip))
|
|
hosts = {ip: {"ansible_host": ip} for ip in ips}
|
|
inventory = {
|
|
"all": {
|
|
"hosts": hosts,
|
|
"vars": {
|
|
"refresh_only": "true",
|
|
"ansible_connection": "ssh",
|
|
"ansible_ssh_private_key_file": "/var/jenkins_home/jenkins_key",
|
|
"ansible_python_interpreter": "/usr/bin/python3",
|
|
"jenkins_user": 'automate',
|
|
"jenkins_group": 'Jenkins-Admin',
|
|
"subnet_group_check": 'Jenkins-AllSubnets',
|
|
"SERVER_SUBNET_GROUP": 'Jenkins-AllSubnets',
|
|
"inventory_generation_timestamp": f"{datetime.datetime.now().isoformat()}",
|
|
"playbook_file": '/var/jenkins_home/ansible/playbooks/cosmostat.yaml',
|
|
"quick_refresh": "true",
|
|
"noisy_test": "false",
|
|
"debug_output": "false",
|
|
"push_redis": "false",
|
|
"run_background": "true",
|
|
"log_output": "true",
|
|
"public_dashboard": "false",
|
|
"custom_port": "80",
|
|
"custom_api_port": "5000",
|
|
"cosmostat_server_reporter": "true",
|
|
"cosmostat_server": "false",
|
|
"secure_api": "false",
|
|
"disable_local_dashboard": "false",
|
|
"update_pipeline": "true",
|
|
"REAL_API_KEY": f"{cosmostat_settings['REAL_API_KEY']}"
|
|
},
|
|
}
|
|
}
|
|
return yaml.safe_dump(
|
|
inventory,
|
|
default_flow_style=False,
|
|
sort_keys=False,
|
|
)
|
|
|
|
#######################################################################
|
|
### Cosmostat Client Subroutines
|
|
### These are only for the clients
|
|
### The cosmostat server has its own functions
|
|
#######################################################################
|
|
|
|
# Cosmostat Client Reporter Handler
|
|
# this handles updating the active data for the local client instance
|
|
# to the remote cosmostat server
|
|
def client_update():
|
|
# update client API path
|
|
api_url = f"{cosmostat_server_api()}update_client"
|
|
payload = get_client_payload(get_client_redis_data(human_readable = False), "redis_data")
|
|
log_data(log_output = f"API Update Called - {api_url}", log_level = "debug_output")
|
|
log_data(log_output = "client_update - redis data from local client:", log_level = "noisy_test")
|
|
log_data(log_output = payload, log_level = "noisy_test")
|
|
# execute API call, we do need the result
|
|
result = client_submission_handler(api_url, payload)
|
|
# if this client does not exist on the server, we need to try to initialize it
|
|
if not result or not result.get('client_updated'):
|
|
log_data(log_output = f"Client not updated, initializing", log_level = "log_output")
|
|
client_api_initialize()
|
|
|
|
# Cosmostat Client Initializer
|
|
# this handles inializing the CosmostatClient object on the remote cosmostat server
|
|
def client_api_initialize():
|
|
# create client API path
|
|
api_url = f"{cosmostat_server_api()}create_client"
|
|
# generate payload for the remote server
|
|
payload = get_client_payload(get_php_summary(), "client_properties")
|
|
# add extra init vars
|
|
payload["active_interface"] = cosmostat_client.primary_ip
|
|
payload["is_server"] = False
|
|
# execute API call on remote server
|
|
client_submission_handler(api_url, payload)
|
|
|
|
# this returns the payload in the format that the API expects with extra necessary data
|
|
# used for both initializing and updating; dictionary name is part of the function
|
|
def get_client_payload(system_dictionary: dict, dictionary_name: str):
|
|
this_uuid = cosmostat_client.uuid
|
|
this_short_id = cosmostat_client.short_id
|
|
this_hostname = cosmostat_client.name
|
|
payload = {
|
|
"uuid": this_uuid,
|
|
"short_id": this_short_id,
|
|
"hostname": this_hostname,
|
|
dictionary_name: system_dictionary,
|
|
"API_KEY": app_settings["REAL_API_KEY"]
|
|
|
|
}
|
|
return payload
|
|
|
|
# Cosmostat Client API Reporting Handler
|
|
# this helper makes the POST request to the remote cosmostat server
|
|
# the payload and URL are passed to this function
|
|
def client_submission_handler(api_url: str, payload: dict):
|
|
result = None
|
|
try:
|
|
# this is the actual submission to the server
|
|
response: Response = requests.post(api_url, json=payload, timeout=4)
|
|
# raise HTTPError for 4xx/5xx if failure
|
|
response.raise_for_status()
|
|
except RequestException as exc:
|
|
# Wrap the low-level exception in a more descriptive one
|
|
log_data(log_output = f"Failed to POST to {api_url!r}: {exc}", log_level = "log_output")
|
|
# process reply from API
|
|
try:
|
|
result = response.json()
|
|
except ValueError as exc:
|
|
log_data(log_output = f"Server responded with non-JSON payload: {response.text!r}", log_level = "log_output")
|
|
return result
|
|
|
|
#######################################################################
|
|
#######################################################################
|
|
### Main Subroutine
|
|
#######################################################################
|
|
#######################################################################
|
|
|
|
if __name__ == '__main__':
|
|
|
|
######################################
|
|
### Main Functions
|
|
######################################
|
|
log_data(log_output = f"Main Function Start", log_level = "log_output")
|
|
|
|
######################################
|
|
# Main function helpers
|
|
#
|
|
|
|
# instantiate and return the Client System object
|
|
def new_cosmos_client():
|
|
new_client = System(f"{jenkins_hostname_settings()}")
|
|
log_data(log_output = f"New System object name: {new_client.name} - {new_client.get_component_count()} components:", log_level = "log_output")
|
|
for component in new_client.components:
|
|
log_data(log_output = component.description, log_level = "log_output")
|
|
return new_client
|
|
|
|
# instantiate and return the CosmostatServer System object
|
|
def new_cosmostat_server():
|
|
new_server = CosmostatServer(name = cosmostat_client.uuid, hostname = jenkins_hostname_settings())
|
|
log_data(log_output = f"New Cosmostat serverobject name: {new_server.name}", log_level = "log_output")
|
|
return new_server
|
|
|
|
# helper function for checking if we should update the local System Object this time
|
|
def update_local_system():
|
|
result = False
|
|
if cosmostat_client.check_system_timer():
|
|
result = True
|
|
if run_cosmostat_server():
|
|
result = True
|
|
if run_cosmostat_reporter() and int(time.time()) % 5 == 0:
|
|
result = True
|
|
return result
|
|
|
|
# Background Loop Function
|
|
# That makes this the service loop
|
|
def background_loop():
|
|
# Update all data on the local System object
|
|
# This is done on both client and server
|
|
if update_local_system():
|
|
cosmostat_client.update_system_state()
|
|
|
|
# publish to redis if the web dashboard is active locally
|
|
if app_settings["push_redis"] and not app_settings["disable_local_dashboard"]:
|
|
update_redis_server()
|
|
|
|
# report data to the server if configured
|
|
# this is the client-only update function
|
|
if run_cosmostat_reporter():
|
|
if cosmostat_client.check_system_timer or int(time.time()) % 5 == 0:
|
|
client_update()
|
|
|
|
# if this is the server, do this stuff
|
|
if run_cosmostat_server():
|
|
# purge stale client systems
|
|
cosmostat_server.purge_stale_hostnames()
|
|
# report the server's own client object to itself
|
|
# uses the helper function upstream of the API route
|
|
# since the route is already here
|
|
run_update_client(get_client_payload(get_client_redis_data(human_readable = False), "redis_data"))
|
|
log_data(log_output = f"{this_client}", log_level = "noisy_test")
|
|
|
|
# short sleep time to keep things chill
|
|
time.sleep(0.2)
|
|
|
|
#
|
|
# End Main Functions
|
|
######################################
|
|
|
|
#######################################################################
|
|
#######################################################################
|
|
###
|
|
### Here is where this whole thing actually starts doing something
|
|
###
|
|
#######################################################################
|
|
#######################################################################
|
|
|
|
# First thing is to create the local Cosmostat System
|
|
log_data(log_output = f"Cosmostat Client Start", log_level = "log_output")
|
|
# Here is the Object Instatiation Command
|
|
cosmostat_client = new_cosmos_client()
|
|
# Update the system state on new object
|
|
cosmostat_client.update_system_state()
|
|
|
|
# If the option was set to phone home, initialize the remote CosmostatClient Object
|
|
if app_settings["cosmostat_server_reporter"] and not app_settings["cosmostat_server"]:
|
|
log_data(log_output = f"Initialize Client Reporter", log_level = "log_output")
|
|
client_api_initialize()
|
|
|
|
# If this is the server, this will make it the server
|
|
# call the object outside the if
|
|
cosmostat_server = None
|
|
if run_cosmostat_server():
|
|
log_data(log_output = f"Cosmostat Server Start", log_level = "log_output")
|
|
# instantiate it if so
|
|
cosmostat_server = new_cosmostat_server()
|
|
this_client = get_client_payload(get_php_summary(), "client_properties")
|
|
# extra initial vars
|
|
# any extra data that should be associated with a CosmostatClient Object should be added here
|
|
this_client["active_interface"] = cosmostat_settings["cosmostat_server_ip"]
|
|
this_client["is_server"] = True
|
|
# add the server itself as a CosmostatClient object
|
|
# use the same upstream function as the API endpoint
|
|
timestamp_update = cosmostat_server.add_system(system_dictionary = this_client)
|
|
# if not a server, update client on the server
|
|
else:
|
|
client_update()
|
|
|
|
# send initial stats update to redis
|
|
if app_settings["push_redis"] and not app_settings["disable_local_dashboard"]:
|
|
log_data(log_output = f"Initial Redis Push", log_level = "log_output")
|
|
update_redis_server()
|
|
|
|
# Flask scheduler for background loop, run if requested
|
|
# this is not needed for the API routes
|
|
if app_settings["run_background"] and not app_settings["disable_local_dashboard"]:
|
|
log_data(log_output = f"Background Function Initializing", log_level = "log_output")
|
|
log_data(log_output = "Loading flask background subroutine...", log_level = "log_output")
|
|
|
|
scheduler.add_job(id='background_loop',
|
|
func=background_loop,
|
|
trigger='interval',
|
|
seconds=app_settings["update_frequency"])
|
|
scheduler.init_app(app)
|
|
scheduler.start()
|
|
log_data(log_output = "...Done", log_level = "log_output")
|
|
else:
|
|
log_data(log_output = "Skipping flask background task", log_level = "log_output")
|
|
|
|
# Flask API, run if requested
|
|
log_data(log_output = f"gateway: {service_gateway_ip()} - port: {service_api_port()}", log_level = "log_output")
|
|
if not app_settings["disable_local_dashboard"]:
|
|
log_data(log_output = f"Main API Start", log_level = "log_output")
|
|
app.run(debug=False, host=service_gateway_ip(), port=service_api_port())
|
|
else:
|
|
# if local API disabled, phone home if configured
|
|
# if the flask app is not ran, then the background scheduler also does not run
|
|
log_data(log_output = f"Internal API Disabled", log_level = "log_output")
|
|
cosmostat_client.update_system_state()
|
|
client_update()
|
|
while True:
|
|
if int(time.time()) % 5 == 0:
|
|
cosmostat_client.update_system_state()
|
|
client_update()
|
|
time.sleep(0.5)
|