diff --git a/defaults/main.yaml b/defaults/main.yaml index 9126ac3..a2aa348 100644 --- a/defaults/main.yaml +++ b/defaults/main.yaml @@ -56,4 +56,5 @@ secure_api: false push_redis: true run_background : true log_output: true +update_frequency: "1" ... \ No newline at end of file diff --git a/files/api/Components.py b/files/api/Components.py index 8783d7b..98c35f4 100644 --- a/files/api/Components.py +++ b/files/api/Components.py @@ -1,223 +1,255 @@ # this class file is for the cosmostat service import subprocess -from LinkedList import * +import json +from typing import Dict, Any, List +# Global Class Vars global_max_length = 500 +debug_output = False + +# import the component descriptor +# this outlines how the component class works +# each type of component has a "type" +try: + with open("component_descriptors.json", encoding="utf-8") as f: + component_class_tree: List[Dict] = json.load(f) +except FileNotFoundError as exc: + raise RuntimeError("Descriptor file not found") from exc + +component_types = [{"name": entry["name"], "multi_check": entry["multi_check"] == "True"} for entry in component_class_tree] class Component: - ########################################################################################## - # Base class for all system components. All instantiated objects need a child class - # Class data: - ### name - name of the type of component, declared in the parent class - ### status - ### model_string - string with device info, declared in parent class - ### metric_name - name of the value being measured - ### current_value - ### historical_data - This will be a linked list used to generate a json when calling get_historical_data - ### for this to work, the function using these classes needs to update the values periodically - #### historical_data = [ - #### { - #### "timestamp": timestamp, # seconds since epoch - #### "value": value - #### }, - #### { - #### "timestamp": timestamp, - #### "value": value - #### } - #### ] - def __init__(self, name: str, model_string: str = None): - # fail instantiation if critical data is missing - if self.model_string is None: - raise TypeError("Error - missing component model_string") - if self.metric_name is None: - raise TypeError("Error - missing component metric_name") - if self.metric_value_command is None: - raise TypeError("Error - missing component metric_value_command") - if self.type is None: - raise TypeError("Error - missing component type") - if self.has_temp is None: - raise TypeError("Error - missing temp data check") - - # set up history list - self.history_max_length = global_max_length - self.historical_data = ValueHistory(self.history_max_length) - self.history_start = self.historical_data.get_first_timestamp() - self.update_value() - if self.current_value is None: - raise TypeError("Error - failed to read value") - - # if temp data exists, handle it - if self.has_temp: - self.temp_history_data = ValueHistory(self.history_max_length) - self.temp_history_start = self.temp_history_data.get_first_timestamp() - self.current_temp = self.temp_history_data.get_current_value() - else: - self.temp_history_data = None - - # instantiate other shared class variables + def __init__(self, name: str, comp_type: str ): self.name = name - self.current_value = self.historical_data.get_current_value() - if self.has_temp: - self.current_temp = self.temp_history_data.get_current_value() - else: - self.current_temp = None - self.comment = f"This is a {self.type}, so we are measuring {self.metric_name}, currently at {self.current_value}" - - # if nothing failed, the object is ready - self.status = "ready" + self.type = comp_type + for component in component_class_tree: + if component["name"] == self.type: + COMPONENT_DESCRIPTORS = component + # Load component type descriptor from class tree + # COMPONENT_DESCRIPTORS = {d['type']: d for d in component_class_tree} + descriptor = COMPONENT_DESCRIPTORS + self._descriptor = descriptor + if descriptor is None: + raise ValueError( + f"Component type '{comp_type}' is not defined in the " + f"component descriptor tree." + ) + # store static properties + self.multi_check = self.is_multi() + self._properties: Dict[str, str] = {} + for key, command in descriptor.get('properties', {}).items(): + self._properties[key] = run_command(command, True) + # build the description string + self._description_template: str | None = descriptor.get("description") + self.description = self._description_template.format(**self._properties) + # initialize metrics + self._metrics: Dict[str, str] = {} + self.update_metrics() def __str__(self): - return (f"{self.__class__.__name__}: {self.name} " - f"{self.model_string}") - - def __del__(self): - print(f"Deleting {self.type} component - {self.model_string}") + self_string = (f"Component name: {self.name}, type: {self.type} - " + f"{self.description}") + return self_string + + def __repr__(self): + self_string = (f"Component name: {self.name}, type {self.type} - " + f"{self.description}") + return self_string - def get_info_key(self): + def update_metrics(self): + for key, command in self._descriptor.get('metrics', {}).items(): + self._metrics[key] = run_command(command, True) + + # complex data type return + def get_metrics(self, type = None): + these_metrics = [] + if type == None: + for name, value in self._metrics: + these_metrics.append({"name": name, "value": value}) + else: + for name, value in self._metrics: + if name == type: + these_metrics.append({"name": name, "value": value}) result = { "name": self.name, "type": self.type, - "model_string": self.model_string, - "metric_name": self.metric_name + "metrics": these_metrics } return result - def get_summary_key(self): + # complex data type return + def get_properties(self, type = None): + these_properties = [] + if type == None: + for name, value in self._properties.items(): + these_properties.append({"name": name, "value": value}) + else: + for name, value in self._properties.items(): + if name == type: + these_properties.append({"name": name, "value": value}) result = { + "name": self.name, "type": self.type, - "current_value": self.current_value, - "metric_name": self.metric_name, - "model_string": self.model_string + "properties": these_properties } return result - def update_value(self): - #try: - self.current_value = run_command(self.metric_value_command, True) - self.historical_data.add(self.current_value) - #except: - - def update_temp_value(self): - if has_temp: - #try: - self.current_temp = run_command(self.temp_value_command, True) - self.temp_history_data.add(self.current_value) - #except: - else: - return None + # this gets the value of a specified property, type required + def get_property(self, type): + return self._properties[type] - - def get_history(self, count: int = global_max_length): - if self.has_temp: - result = { - "value_metric": self.metric_name, - "history_count": count, - "history_data": self.historical_data.get_history(count), # reminder this is a LinkedList get_history - "history_temp_data": self.temp_history_data.get_history(count) - } - else: - result = { - "value_metric": self.metric_name, - "history_count": count, - "history_data": self.historical_data.get_history(count) # same reminder here - } + # returns array of dicts for redis + def get_metrics_keys(self): + result = [] + for name, value in self._metrics.items(): + this_metric = { + "name": self.name, + "type": name, + "metric": value + } + result.append(this_metric) return result + + def get_properties_keys(self): + result = [] + for name, value in self._properties.items(): + this_property = { + "name": self.name, + "property": name, + "value": value + } + result.append(this_property) + return result + + # full data return + def get_description(self): + these_properties = [] + for name, value in self._metrics.items(): + these_properties.append({"name": name, "value": value}) + these_metrics = [] + for name, value in self._metrics.items(): + these_metrics.append({"name": name, "value": value}) + result = { + "name": self.name, + "type": self.type, + "properties": these_properties, + "metrics": these_metrics + } + return result + + def is_multi(self): + for component_type in component_types: + if self.type == component_type["name"]: + return component_type["multi_check"] + return False + -############################################################ -# Component Class Types -# There needs to be one of these for each monitored thing -############################################################ -# Need to add: -### temperatures -### network + VPN -### storage + ZFS -### video cards -### virtual machines - -# CPU component class. -class CPU(Component): - - def __init__(self, name: str, is_virtual: bool = False): - # Declare component type - self.type = "CPU" - # deal with temp later - self.has_temp = False - # no temp if VM - #self.has_temp = not is_virtual - #self.temp_value_command = "acpi -V | jc --acpi -p | jq '.[] | select(.type==\"Thermal\") | .temperature '" - self.model_string = self.get_model_string() - - # Initialize value - self.metric_name = "1m_load" - self.metric_value_command = "cat /proc/loadavg | awk '{print $1}'" - self.current_value = run_command(self.metric_value_command, True) - - # Complete instantiation - super().__init__(name, self.model_string) - - def get_model_string(self): - # Get CPU Info - model_string_command = "lscpu --json | jq -r '.lscpu[] | select(.field==\"Model name:\") | .data'" - return run_command(model_string_command, True) - -# RAM component class. -class RAM(Component): - - def __init__(self, name: str): - # Declare component type - self.type = "RAM" - self.has_temp = False - self.model_string = self.get_model_string() - - # Initialize Value - self.metric_name = "used_capacity_mb" - self.metric_value_command = "free -m | grep Mem | awk '{print $3}'" - self.current_value = run_command(self.metric_value_command, True) - - # Complete instantiation - super().__init__(name, self.model_string) - - def get_model_string(self): - # Check total system RAM - bytes_total_command = "sudo lshw -json -c memory | jq -r '.[] | select(.description==\"System Memory\").size' " - bytes_total = float(run_command(bytes_total_command, True)) - gb_total = round(bytes_total / 1073741824, 2) - return f"Total Capacity: {gb_total}GB" - ############################################################ # System Class -# A system is build from components ############################################################ + class System: + # system variable declarations + # keys to add: model and serial number + static_key_variables = [ + {"name": "hostname", "command": "hostname"}, + {"name": "virt_string", "command": "systemd-detect-virt"} + ] + dynamic_key_variables = [ + {"name": "uptime", "command": "uptime -p"}, + {"name": "timestamp", "command": "date '+%D %r'"}, + ] + # add components based on the class tree + # component_types = [{"name": entry["name"], "multi_check": entry["multi_check"] == "True"} for entry in component_class_tree] # instantiate new system def __init__(self, name: str): # the system needs a name self.name = name - # system is built of other component objects + if debug_output: + print(f"System initializing, name {self.name}") + # system contains an array of component objects self.components = [] - # other system properties - self.sysvars = {} - # either i do it here or i do it twice - self.sysvars["is_virtual"] = self.check_for_virtual() - # Let's build a system - self.add_component(CPU("CPU", self.sysvars["is_virtual"])) - self.add_component(RAM("RAM")) + # initialize system properties and metrics dicts + self._properties: Dict[str, str] = {} + self._metrics: Dict[str, str] = {} + # load static keys + for static_key in self.static_key_variables: + command = static_key["command"] + result = run_command(command, True) + if debug_output: + print(f"Static key [{static_key["name"]}] - command [{command}] - output [{result}]") + self._properties[static_key["name"]] = result + # initialize live keys + self.update_live_keys() + # initialze components + self.load_components() - # let's build system values - self.check_values() + # update only system dynamic keys + def update_live_keys(self): + for live_key in self.dynamic_key_variables: + if live_key['command'] is not None: + command = live_key['command'] + result = run_command(command, True) + self._metrics[live_key['name']] = result + if debug_output: + print(f"Command {live_key["name"]} - [{command}] Result - [{result}]") + + # update all dynamic keys, including components + def update_system_state(self): + self.update_live_keys() + for component in self.components: + component.update_metrics() + + # check for components + def load_components(self): + for component in component_types: + component_name = component["name"] + multi_check = component["multi_check"] + if multi_check: + print("placeholder...") + else: + if debug_output: + print(f"Creating component {component["name"]}") + self.add_components(Component(component_name, component_name)) # Add a component to the system - def add_component(self, component: Component): + def add_components(self, component: Component): + if debug_output: + print(f"Component description: {component.description}") self.components.append(component) # Get all components, optionally filtered by type def get_components(self, component_type: type = None): if component_type is None: return self.components - return [c for c in self.components if isinstance(c, component_type)] - + else: + result = [] + for component in self.components: + if component.type == component_type: + result.append(component) + if component.is_multi(): + return result + else: + return result[0] + + def get_component_strings(self, component_type: type = None): + if component_type is None: + result = [] + for component in self.components: + result.append(component.description) + return result + else: + result = [] + for component in self.components: + if component.type == component_type: + result.append(component.description) + if component.is_multi(): + return result + else: + return result[0] + # get component count def get_component_count(self): result = int(len(self.components)) @@ -225,64 +257,93 @@ class System: def __str__(self): components_str = "\n".join(f" - {c}" for c in self.components) - return f"System: {self.name}\n{components_str}" - - # update metrics for all components - def update_values(self): - self.check_values() - for component in self.components: - component.update_value() - - def check_for_virtual(self): - check_if_vm_command = "systemd-detect-virt" - check_if_vm = run_command(check_if_vm_command, True) - if check_if_vm != "none": - return True - else: - return False - - def check_uptime(self): - check_uptime_command = "uptime -p" - system_uptime = run_command(check_uptime_command, True) - return system_uptime - - def check_timestamp(self): - check_timestamp_command = "date '+%D %r'" - system_timestamp = run_command(check_timestamp_command, True) - return system_timestamp - - def check_values(self): - self.sysvars["uptime"] = self.check_uptime() - self.sysvars["name"] = self.name - self.sysvars["component_count"] = self.get_component_count() - self.sysvars["timestamp"] = self.check_timestamp() - - def get_sysvars(self): - result = {} - for sysvar in self.sysvars: - result[f"{sysvar}"] = self.sysvars[f"{sysvar}"] - return result + return f"System hostname: {self.name}\nComponent Count: {self.get_component_count()}\n{components_str}" + # return both static and dynamic data def get_sysvars_summary_keys(self): result = [] - for sysvar in self.sysvars: - system_type_string = f"sysvar['{sysvar}']" + for name, value in self._properties.items(): thisvar = { - "type": "System Class Variable", - "current_value": sysvar, - "metric_name": system_type_string, - "model_string": self.sysvars[sysvar] + "name": "System Class Property", + "type": name, + "value": value + } + result.append(thisvar) + for name, value in self._metrics.items(): + thisvar = { + "name": "System Class Metric", + "type": name, + "value": value } result.append(thisvar) return result + + # return list of all live metrics from system and properties + def get_live_metrics(self): + result = [] + for component_metric in self.get_component_metrics(): + result.append(component_metric) + for system_metric in self.get_system_metrics(): + result.append(system_metric) + return result + # return array of all component metrics + def get_component_metrics(self): + result = [] + for component in self.components: + for metric in component.get_metrics_keys(): + result.append(metric) + return result + # return array of all component metrics + def get_component_properties(self): + result = [] + for component in self.components: + for metric in component.get_properties_keys(): + result.append(metric) + return result + + # return array of all system metrics + def get_system_metrics(self): + result = [] + for name, value in self._metrics.items(): + thisvar = { + "name": "System", + "type": name, + "metric": value + } + result.append(thisvar) + # add component count + result.append({ + "name": "System", + "type": "component_count", + "metric": self.get_component_count() + }) + return result + + def get_system_properties(self): + result = [] + for name, value in self._properties.items(): + if name == "virt_string": + thisvar = { + "name": "System", + "property": name, + "value": value == "none" + } + else: + thisvar = { + "name": "System", + "property": name, + "value": value + } + result.append(thisvar) + return result + ############################################################ # Helper Functions ############################################################ - # subroutine to run a command, return stdout as array unless zero_only then return [0] def run_command(cmd, zero_only=False): # Run the command and capture the output diff --git a/files/api/LinkedList.py b/files/api/LinkedList.py deleted file mode 100644 index 1345bb9..0000000 --- a/files/api/LinkedList.py +++ /dev/null @@ -1,98 +0,0 @@ - -############################## -# linked list classes -# written by the intern -############################## - -import time - -# single node in a singly linked list -class Node: - __slots__ = ("value", "next", "timestamp") - - def __init__(self, value): - self.value = value - self.timestamp = time.time() - self.next = None - -# small, bounded history implemented with a singly linked list -class ValueHistory: - def __init__(self, maxlen: int): - if maxlen <= 0: - raise ValueError("maxlen must be a positive integer") - self.maxlen = maxlen - self.head: Node | None = None # oldest entry - self.tail: Node | None = None # newest entry - self.size = 0 - - # Append a new value to the history, dropping the oldest if needed - def add(self, value): - new_node = Node(value) - - # link it after the current tail - if self.tail is None: # empty list - self.head = self.tail = new_node - else: - self.tail.next = new_node - self.tail = new_node - - self.size += 1 - - # 2. enforce the size bound - if self.size > self.maxlen: - # drop the head (oldest item) - assert self.head is not None # for the type checker - self.head = self.head.next - self.size -= 1 - - # If the list became empty, also reset tail - if self.head is None: - self.tail = None - - # Return the history as a Python dict list (oldest → newest) - def get_history(self, count: int | None = None): - if count is None: - count = self.maxlen - out = [] - cur = self.head - counter = 0 - while cur is not None and counter < count: - counter += 1 - out.append( - { - "timestamp": cur.timestamp, - "value": cur.value - } - ) - cur = cur.next - return out - - # Return oldest timestamp - def get_first_timestamp(self): - if self.head is not None: - return self.head.timestamp - else: - return time.time() - - # Return current data - def get_current_value(self): - if self.tail is not None: - return self.tail.value - else: - return 0 - - # ------------------------------------------------------------------ - # Convenience methods - # ------------------------------------------------------------------ - def __len__(self): - return self.size - - def __iter__(self): - """Iterate over values from oldest to newest.""" - cur = self.head - while cur is not None: - yield cur.value - cur = cur.next - - def __repr__(self): - return f"BoundedHistory(maxlen={self.maxlen}, data={self.get()!r})" diff --git a/files/api/app.py b/files/api/app.py index 99639e1..a8570ee 100644 --- a/files/api/app.py +++ b/files/api/app.py @@ -16,11 +16,12 @@ scheduler = APScheduler() # default application setting variables app_settings = { "noisy_test" : False, - "debug_output" : False, - "log_output" : False, + "debug_output" : True, + "log_output" : True, "secure_api" : True, "push_redis" : False, - "run_background" : True + "run_background" : True, + "update_frequency": 1 } with open('cosmostat_settings.yaml', 'r') as f: @@ -75,93 +76,29 @@ def update_redis_channel(redis_channel, data): def update_redis_server(): # Update Stats Redis Channel - update_redis_channel("host_stats", get_full_summary()) + update_redis_channel("host_metrics", get_redis_data()) # Update history_stats Redis Channel - update_redis_channel("history_stats", get_component_list()) - -####################################################################### -### Other Functions -####################################################################### - -def get_component_summary(): - result = [] - for component in cosmostat_system.components: - result.append(component.get_summary_key()) - return result - -def get_full_summary(): - result = [] - - for component in cosmostat_system.components: - result.append(component.get_summary_key()) - - for sysvar in cosmostat_system.get_sysvars_summary_keys(): - result.append(sysvar) - - return result - -# This will instantiate a System object -def new_cosmos_system(): - new_system = System(f"{jenkins_hostname_settings()}") - if app_settings["log_output"]: - print(f"New system object name: {new_system.name}") - for component in new_system.components: - print(component) - return new_system - -def get_component_list(history_count = None): - result = [] - for component in cosmostat_system.components: - if history_count is not None: - history = component.get_history(history_count) - else: - history = component.get_history() - result.append( - { - "info": component.get_info_key(), - "history": history - } - ) - return result - -def get_info(): - device_summary = [] - for component in cosmostat_system.components: - device_summary.append( - { - "info": component.get_info_key(), - } - ) - result = { - "system_info": - { - "user": jenkins_user_settings(), - "hostname": jenkins_hostname_settings(), - "timestamp": jenkins_inventory_generation_timestamp_settings(), - "component_count:": len(cosmostat_system.components), - "object_name": cosmostat_system.name, - "docker_gateway": docker_gateway_settings() - }, - "device_summary": device_summary - } - return result - -#def get_history_summary(): + # update_redis_channel("history_stats", get_component_list()) ####################################################################### ### Flask Routes ####################################################################### -# full component list -@app.route('/component_list', methods=['GET']) -def component_list(): - count = request.args.get('count', type=int) - return jsonify(get_component_list(count)) +# dynamic data +# this will go to the redis server +@app.route('/dynamic_data', methods=['GET']) +def dynamic_data(): + return jsonify(get_dynamic_data()) -# component summary -@app.route('/component_summary', methods=['GET']) -def component_summary(): - return jsonify(get_component_summary()) +# static data +@app.route('/static_data', methods=['GET']) +def static_data(): + return jsonify(get_static_data()) + +# redis data +@app.route('/redis_data', methods=['GET']) +def redis_data(): + return jsonify(get_redis_data()) # full summary @app.route('/full_summary', methods=['GET']) @@ -180,27 +117,103 @@ def test(): { "component_count:": len(cosmostat_system.components), "user": jenkins_user_settings(), - "hostname": jenkins_hostname_settings() + "hostname": jenkins_hostname_settings(), + "cpu_model": cosmostat_system.get_components(component_type="CPU").description } ) + +####################################################################### +### Flask Helpers +####################################################################### + +# needs to return array of {name: name, type: type, metrics: metrics} +# for redis table generation, includes system and component metrics +def get_dynamic_data(): + return cosmostat_system.get_live_metrics() + +def get_static_data(): + result = [] + for metric in cosmostat_system.get_system_properties(): + result.append(metric) + for metric in cosmostat_system.get_component_properties(): + result.append(metric) + return result + +def get_redis_data(): + result = [] + for metric in get_dynamic_data(): + result.append(metric) + for metric in get_static_data(): + result.append(metric) + return result + +def get_full_summary(): + live_metrics = cosmostat_system.get_live_metrics() + system_components = cosmostat_system.get_component_strings() + system_info = get_info() + result = { + "system_settings": + { + "user": jenkins_user_settings(), + "hostname": jenkins_hostname_settings(), + "timestamp": jenkins_inventory_generation_timestamp_settings(), + "component_count:": len(cosmostat_system.components), + "object_name": cosmostat_system.name, + "docker_gateway": docker_gateway_settings() + }, + "live_metrics": live_metrics, + "system_components": system_components, + "system_info": system_info + } + return result + +def get_info(): + component_strings = [] + for component in cosmostat_system.get_components(): + component_strings.append({"name": component.name, "description": component.description}) + system_strings = [] + result = { + "hostname": jenkins_hostname_settings(), + "component_strings": component_strings + } + #for component_string in component_strings: + # for name, description in component_string.items(): + # result[name] = description + return result + +####################################################################### +### Other Functions +####################################################################### + +# instantiate and return the System object +def new_cosmos_system(): + new_system = System(f"{jenkins_hostname_settings()}") + if app_settings["log_output"]: + print(f"New system object name: {new_system.name} - {new_system.get_component_count()} components:") + for component in new_system.components: + print(component.description) + return new_system + + +# Background Loop Function +def background_loop(): + # Update all data on the System object + cosmostat_system.update_system_state() + + if app_settings["push_redis"]: + update_redis_server() + + if app_settings["noisy_test"]: + print("Sorry about the mess...") + print(f"Blame {jenkins_user_settings()}") + ####################################################################### ### Main Subroutine ####################################################################### if __name__ == '__main__': - # Background Loop Function - def background_loop(): - # Update all data on the System object - cosmostat_system.update_values() - - if app_settings["push_redis"]: - update_redis_server() - - if app_settings["noisy_test"]: - print("Sorry about the mess...") - print(f"Blame {jenkins_user_settings()}") # instantiate system cosmostat_system = new_cosmos_system() @@ -217,7 +230,7 @@ if __name__ == '__main__': scheduler.add_job(id='background_loop', func=background_loop, trigger='interval', - seconds=1) + seconds=app_settings["update_frequency"]) scheduler.init_app(app) scheduler.start() diff --git a/files/api/component_descriptors.json b/files/api/component_descriptors.json new file mode 100644 index 0000000..028f969 --- /dev/null +++ b/files/api/component_descriptors.json @@ -0,0 +1,29 @@ +[ + { + "name": "CPU", + "description": "{model_name} with {core_count} cores.", + "multi_check": "False", + "properties": { + "core_count": "lscpu --json | jq -r '.lscpu[] | select(.field==\"CPU(s):\") | .data'", + "model_name": "lscpu --json | jq -r '.lscpu[] | select(.field==\"Model name:\") | .data'" + }, + "metrics": { + "1m_load": "cat /proc/loadavg | awk '{print $1}'", + "5m_load": "cat /proc/loadavg | awk '{print $2}'", + "15m_load": "cat /proc/loadavg | awk '{print $3}'" + } + }, + { + "name": "RAM", + "description": "Total {bytes_total}GB in {module_count} modules.", + "multi_check": "False", + "properties": { + "bytes_total": "sudo lshw -json -c memory | jq -r '.[] | select(.description==\"System Memory\").size' | awk '{printf \"%.2f\\n\", $1/1073741824}'", + "module_count": "sudo lshw -json -c memory | jq -r '.[] | select(.id | contains(\"bank\")) | .id ' | wc -l" + }, + "metrics": { + "used_capacity_mb": "free -m | grep Mem | awk '{print $3}'", + "free_capacity_mb": "free -m | grep Mem | awk '{print $4}'" + } + } +] \ No newline at end of file diff --git a/files/web/html/index.html b/files/web/html/index.html index 36f6dc6..010ed9a 100644 --- a/files/web/html/index.html +++ b/files/web/html/index.html @@ -13,8 +13,8 @@ This dashboard shows the local Matt-Cloud system stats.

-

System Stats

-
Connecting…
+

Live System Metrics

+
Connecting…