diff --git a/data/junos_parsers/request-pfe-execute-jnh-pool.yaml b/data/junos_parsers/request-pfe-execute-jnh-pool.yaml new file mode 100644 index 0000000..2fa5997 --- /dev/null +++ b/data/junos_parsers/request-pfe-execute-jnh-pool.yaml @@ -0,0 +1,35 @@ +parser: + regex-command: request\s+pfe\s+execute\s+command\s+"show\s+jnh\s+\d\s+pool"\s+target\s+fpc\d + matches: + - + type: single-value + method: regex + regex: \s*GOT:\s+Next\s+Hop\s+\w+\s+\d+\s+\d+\s+(\d+)%\s+\d+\s+\d+%$ + variables: + - + variable-name: $host.pfe.memory.nh-USED-PERCENT + variable-type: integer + - + type: single-value + method: regex + regex: \s*GOT:\s+Next\s+Hop\s+\w+\s+\d+\s+(\d+)\s+\d+%\s+\d+\s+\d+%$ + variables: + - + variable-name: $host.pfe.memory.nh-USED + variable-type: integer + - + type: single-value + method: regex + regex: \s*GOT:\s+Firewall\s+\w+\s+\d+\s+\d+\s+(\d+)%\s+\d+\s+\d+%$ + variables: + - + variable-name: $host.pfe.memory.fw-USED-PERCENT + variable-type: integer + - + type: single-value + method: regex + regex: \s*GOT:\s+Firewall\s+\w+\s+\d+\s+(\d+)\s+\d+%\s+\d+\s+\d+%$ + variables: + - + variable-name: $host.pfe.memory.fw-USED + variable-type: integer diff --git a/data/junos_parsers/request-pfe-execute-show-ifl-summary.yaml b/data/junos_parsers/request-pfe-execute-show-ifl-summary.yaml new file mode 100644 index 0000000..5b4d1fb --- /dev/null +++ b/data/junos_parsers/request-pfe-execute-show-ifl-summary.yaml @@ -0,0 +1,20 @@ +parser: + regex-command: request\s+pfe\s+execute\s+command\s+"show\s+ifl\s+summary"\s+target\s+fpc\d + matches: + - + type: single-value + method: regex + regex: \s*GOT:\s+ifl\s+count\s+=\s+(\d+)$ + variables: + - + variable-name: $host.ifl.count.total + variable-type: integer + - + type: single-value + method: regex + regex: \s*GOT:\s+Next\s+Hop\s+\w+\s+\d+\s+(\d+)\s+\d+%\s+\d+\s+\d+%$ + regex: \s*GOT:\s+ifl\s+up\s+count\s+=\s+(\d+)$ + variables: + - + variable-name: $host.ifl.count.up + variable-type: integer \ No newline at end of file diff --git a/data/junos_parsers/show-chassis-fpc.yaml b/data/junos_parsers/show-chassis-fpc.yaml new file mode 100644 index 0000000..95fdbf7 --- /dev/null +++ b/data/junos_parsers/show-chassis-fpc.yaml @@ -0,0 +1,31 @@ +parser: + regex-command: show\s+chassis\s+fpc\s+\|\s+display\s+xml + matches: + - + type: multi-value + method: xpath + xpath: //fpc + loop: + key: ./slot + sub-matches: + - + xpath: ./state + variable-name: $host.chassis.fpc.$key.state + - + xpath: ./temperature + variable-name: $host.chassis.fpc.$key.temp + - + xpath: ./cpu-total + variable-name: $host.chassis.fpc.$key.cpu-total + - + xpath: ./cpu-interrupt + variable-name: $host.chassis.fpc.$key.cpu-interrupt + - + xpath: ./memory-dram-size + variable-name: $host.chassis.fpc.$key.memory-dram-size + - + xpath: ./memory-heap-utilization + variable-name: $host.chassis.fpc.$key.memory-heap-util + - + xpath: ./memory-buffer-utilization + variable-name: $host.chassis.fpc.$key.memory-buffer-util diff --git a/data/junos_parsers/show-chassis-routing-engine.parser.yaml b/data/junos_parsers/show-chassis-routing-engine.parser.yaml index 329ad75..2dfa565 100644 --- a/data/junos_parsers/show-chassis-routing-engine.parser.yaml +++ b/data/junos_parsers/show-chassis-routing-engine.parser.yaml @@ -14,12 +14,21 @@ parser: - xpath: ./memory-buffer-utilization variable-name: $host.chassis.routing-engine.$key.memory-buffer-utilization -# - -# xpath: ./up-time/@seconds -# variable-name: $host.chassis.routing-engine.$key.up-time-msec - xpath: ./up-time variable-name: $host.chassis.routing-engine.$key.up-time-date - xpath: ./cpu-idle variable-name: $host.chassis.routing-engine.$key.cpu-idle + - + xpath: ./cpu-user + variable-name: $host.chassis.routing-engine.$key.cpu-user + - + xpath: ./cpu-background + variable-name: $host.chassis.routing-engine.$key.cpu-background + - + xpath: ./cpu-system + variable-name: $host.chassis.routing-engine.$key.cpu-system + - + xpath: ./cpu-interrupt + variable-name: $host.chassis.routing-engine.$key.cpu-interrupt \ No newline at end of file diff --git a/data/junos_parsers/show-system-processes-extensive.parser.yaml b/data/junos_parsers/show-system-processes-extensive.parser.yaml index d502c29..c7844ea 100644 --- a/data/junos_parsers/show-system-processes-extensive.parser.yaml +++ b/data/junos_parsers/show-system-processes-extensive.parser.yaml @@ -2,142 +2,18 @@ parser: regex-command: show\s+system\s+processes\s+extensive matches: - - type: single-value + type: multi-value + match_count: 5 method: regex - regex: \s*[0-9]+\s+\w+\s+\S+\s+\S+\s+\S+\s+(\d+[K|M|G]?)\s+(\d+[K|M|G]?)\s+\S+\s*[\d+]?\s+\S+\s+(\S+)%\s+rpd$ + regex: \s*[0-9]+\s+\w+\s+\S+\s+\S+\s+\S+\s+(\d+[K|M|G]?)\s+(\d+[K|M|G]?)\s+\S+\s*[\d+]?\s+\S+\s+(\S+)%\s+([\w\d-]+)$ + key_index: 4 variables: - - variable-name: $host.re.memory.rpd-SIZE + variable-name: $host.re.memory.$key-SIZE variable-type: integer - - variable-name: $host.re.memory.rpd-RES + variable-name: $host.re.memory.$key-RES variable-type: integer - - variable-name: $host.re.memory.rpd-CPU - variable-type: integer - - - type: single-value - method: regex - regex: \s*[0-9]+\s+\w+\s+\S+\s+\S+\s+\S+\s+(\d+[K|M|G]?)\s+(\d+[K|M|G]?)\s+\S+\s*[\d+]?\s+\S+\s+(\S+)%\s+dcd$ - variables: - - - variable-name: $host.re.memory.dcd-SIZE - variable-type: integer - - - variable-name: $host.re.memory.dcd-RES - variable-type: integer - - - variable-name: $host.re.memory.dcd-CPU - variable-type: integer - - - type: single-value - method: regex - regex: \s*[0-9]+\s+\w+\s+\S+\s+\S+\s+\S+\s+(\d+[K|M|G]?)\s+(\d+[K|M|G]?)\s+\S+\s*[\d+]?\s+\S+\s+(\S+)%\s+snmpd$ - variables: - - - variable-name: $host.re.memory.snmpd-SIZE - variable-type: integer - - - variable-name: $host.re.memory.snmpd-RES - variable-type: integer - - - variable-name: $host.re.memory.snmpd-CPU - variable-type: integer - - - type: single-value - method: regex - regex: \\s*[0-9]+\s+\w+\s+\S+\s+\S+\s+\S+\s+(\d+[K|M|G]?)\s+(\d+[K|M|G]?)\s+\S+\s*[\d+]?\s+\S+\s+(\S+)%\s+mib2d$ - variables: - - - variable-name: $host.re.memory.mib2d-SIZE - variable-type: integer - - - variable-name: $host.re.memory.mib2d-RES - variable-type: integer - - - variable-name: $host.re.memory.mib2d-CPU - variable-type: integer - - - type: single-value - method: regex - regex: \s*[0-9]+\s+\w+\s+\S+\s+\S+\s+\S+\s+(\d+[K|M|G]?)\s+(\d+[K|M|G]?)\s+\S+\s*[\d+]?\s+\S+\s+(\S+)%\s+dfwd$ - variables: - - - variable-name: $host.re.memory.dfwd-SIZE - variable-type: integer - - - variable-name: $host.re.memory.dfwd-RES - variable-type: integer - - - variable-name: $host.re.memory.dfwd-CPU - variable-type: integer - - - type: single-value - method: regex - regex: \s*[0-9]+\s+\w+\s+\S+\s+\S+\s+\S+\s+(\d+[K|M|G]?)\s+(\d+[K|M|G]?)\s+\S+\s*[\d+]?\s+\S+\s+(\S+)%\s+cosd$ - variables: - - - variable-name: $host.re.memory.cosd-SIZE - variable-type: integer - - - variable-name: $host.re.memory.cosd-RES - variable-type: integer - - - variable-name: $host.re.memory.cosd-CPU - variable-type: integer - - - type: single-value - method: regex - regex: \s*[0-9]+\s+\w+\s+\S+\s+\S+\s+\S+\s+(\d+[K|M|G]?)\s+(\d+[K|M|G]?)\s+\S+\s*[\d+]?\s+\S+\s+(\S+)%\s+sampled$ - variables: - - - variable-name: $host.re.memory.sampled-SIZE - variable-type: integer - - - variable-name: $host.re.memory.sampled-RES - variable-type: integer - - - variable-name: $host.re.memory.sampled-CPU - variable-type: integer - - - type: single-value - method: regex - regex: \s*[0-9]+\s+\w+\s+\S+\s+\S+\s+\S+\s+(\d+[K|M|G]?)\s+(\d+[K|M|G]?)\s+\S+\s*[\d+]?\s+\S+\s+(\S+)%\s+smid$ - variables: - - - variable-name: $host.re.memory.smid-SIZE - variable-type: integer - - - variable-name: $host.re.memory.smid-RES - variable-type: integer - - - variable-name: $host.re.memory.smid-CPU - variable-type: integer - - - type: single-value - method: regex - regex: \s*[0-9]+\s+\w+\s+\S+\s+\S+\s+\S+\s+(\d+[K|M|G]?)\s+(\d+[K|M|G]?)\s+\S+\s*[\d+]?\s+\S+\s+(\S+)%\s+smihelperd$ - variables: - - - variable-name: $host.re.memory.smihelperd-SIZE - variable-type: integer - - - variable-name: $host.re.memory.smihelperd-RES - variable-type: integer - - - variable-name: $host.re.memory.smihelperd-CPU - variable-type: integer - - - type: single-value - method: regex - regex: \s*[0-9]+\s+\w+\s+\S+\s+\S+\s+\S+\s+(\d+[K|M|G]?)\s+(\d+[K|M|G]?)\s+\S+\s*[\d+]?\s+\S+\s+(\S+)%\s+idle$ - variables: - - - variable-name: $host.re.memory.idle-SIZE - variable-type: integer - - - variable-name: $host.re.memory.idle-RES - variable-type: integer - - - variable-name: $host.re.memory.idle-CPU + variable-name: $host.re.memory.$key-CPU variable-type: integer \ No newline at end of file diff --git a/data/junos_parsers/show-version-no-forwarding.parser.yaml b/data/junos_parsers/show-version-no-forwarding.parser.yaml new file mode 100644 index 0000000..d8b91e4 --- /dev/null +++ b/data/junos_parsers/show-version-no-forwarding.parser.yaml @@ -0,0 +1,13 @@ +parser: + regex-command: show\s+version\s+no-forwarding\s+\|\s+display\s+xml + matches: + - + type: single-value + method: xpath + xpath: //product-model + variable-name: $host.product_model + - + type: single-value + method: xpath + xpath: //package-information[name='junos']/comment + variable-name: $host.version \ No newline at end of file diff --git a/open-nti/open-nti.py b/open-nti/open-nti.py index 7bb8350..206734b 100644 --- a/open-nti/open-nti.py +++ b/open-nti/open-nti.py @@ -31,6 +31,7 @@ import xmltodict import yaml import copy +import collections logging.getLogger("paramiko").setLevel(logging.INFO) logging.getLogger("ncclient").setLevel(logging.WARNING) # In order to remove http request from ssh/paramiko @@ -81,7 +82,7 @@ def get_latest_datapoints(**kwargs): dbclient.switch_database(db_name) results = {} if db_schema == 1: - query = "select * from /%s\./ ORDER BY time DESC limit 1 " % (kwargs['host']) + query = "select * from /%s\./ GROUP BY * ORDER BY time DESC limit 1 " % (kwargs['host']) elif db_schema == 2: query = "select * from \"%s\" WHERE device = '%s' GROUP BY * ORDER BY time DESC limit 1 " % ('jnpr.collector',kwargs['host']) elif db_schema == 3: @@ -168,7 +169,7 @@ def execute_command(jdevice,command): command_result = jdevice.rpc.cli(command_tmp, format="xml") except RpcError as err: rpc_error = err.__repr__() - logger.error("Error found on <%s> executing command: %s, error: %s:", jdevice.hostname, command ,rpc_error) + logger.error("Error found executing command: %s, error: %s:", command ,rpc_error) return False if format == "text": @@ -234,6 +235,7 @@ def eval_variable_name(variable,**kwargs): variable = variable.replace("$"+key,keys[key]) variable = variable.replace("$host",kwargs['host']) # the host replacement should be move it to other place + print variable return variable, variable def eval_tag_name(variable,**kwargs): @@ -456,6 +458,87 @@ def parse_result(host,target_command,result,datapoints,latest_datapoints,kpi_tag logger.error('[%s]: More matches found on regex than variables especified on parser: %s', host, regex_command) else: logger.debug('[%s]: No matches found for regex: %s', host, regex) + elif match["type"] == "multi-value": + ''' + New multi-value statement allowing for multiple identical regex matches parsed from a string. + The idea here is if you want to parse a multiline string using regex and return X number of matches + instead of the usual single match. This is useful for capturing, say, the top 5 highest CPU performers + in the "show system processes extensive" command each time you call the command. Used in blackbox style + testing to see which daemons are contributors to high CPU although you might not know the names, hence are + unable to add them into the YAML one by one. + + Each re.search result will capture more than one set of rows if mcount >1, so you'll need a "key" to figure out + which re.search.group() will be equal to a newline and be able to split the output into the appropriate + sub-outputs for parsing. + + Example psuedo code: + input multiline string from XML: + "10 root 1 155 52 0K 12K RUN 594:20 93.80% idle + + 33794 root 1 40 0 43036K 33116K select 0:00 1.75% mgd" + + match.lastindex = 8 + match.groups() = ('0K', '12K', '93.80', 'idle', '43036K', '33116K', '1.75', 'mgd') + key = match.groups([3]) and match.groups([7]) + Use the Key as the daemon name and create a measurement name with it. All other values + are field values. + + ''' + mcount = match["match_count"] + regex_block = match["regex"] + regex_framework = "{0}" * mcount + regex = regex_framework.format(regex_block) + text_matches = re.search(regex,result,re.MULTILINE) + if text_matches: + '''We want to check the number of variables in the yaml is equal to the number of variables/groups + in the regex minus one (for the group's key). + Example: 20 total matches, divided by mcount 5 gives us 4 matches per group, minus 1 for the key + which tells us there are 5 regex output groups we want, each 4 members long with a key + which we need to subtract out. This gives us 3 variables expected in our yaml list. + ''' + # Total number of regex groups per match + regexmatch_group_count = (text_matches.lastindex / mcount) + # Number of variables should be total number of groups minus the index for the group data. + variable_count = regexmatch_group_count - 1 + if variable_count == len(match["variables"]): + logger.debug('[%s]: We have (%s) matches and match_count of (%s) with this regex %s', host, text_matches.lastindex,mcount,regex) + match_groups ={} + try: + key_index = int(match["key_index"]) + if key_index >= 1: + # Used because regex output is a tuple, and indexes start at 0, not 1. + # Tells is the location of our "index" value (aka- measurement name) within the tuple. + tuple_index = key_index - 1 + else: + logger.info('[%s]: Exception found in yaml for key_index = %s. ' + 'Key_index is less than 1. Please make key index 1 or greater.', host, match["key_index"]) + pass # Perhaps some other solution is better? + except ValueError as e: + logger.info('[%s]: Exception found in yaml for key_index = %s. Key_index not an integer', host, match["key_index"]) + logging.exception(e) + except Exception, e: + logger.info('[%s]: Exception found.', host) + logging.exception(e) + start_index = 0 + for i in range(tuple_index,text_matches.lastindex,regexmatch_group_count): + match_groups[(text_matches.groups()[start_index:i])] = {"key":text_matches.groups()[i]} + start_index += regexmatch_group_count + data_tuples = match_groups.keys() + for tuple in data_tuples: + index_key = match_groups[tuple] + # Add measurements to DB per variable for each group of values. + for i in range(0,variable_count): + variable_name = eval_variable_name(match["variables"][i]["variable-name"],host=host,keys=index_key) + value_tmp = tuple[i].strip() + # Begin function (pero pendiente de ver si variable-type existe y su valor) + if "variable-type" in match["variables"][i]: + value_tmp = eval_variable_value(value_tmp, type=match["variables"][i]["variable-type"]) + get_metadata_and_add_datapoint(datapoints=datapoints,match=match["variables"][i],value_tmp=value_tmp,latest_datapoints=latest_datapoints,host=host,kpi_tags=kpi_tags,keys=index_key) + else: + logger.error('[%s]: More matches found on regex than variables specified on parser: %s', host, regex_command) + + else: + logger.debug('[%s]: No matches found for regex: %s', host, regex) else: logger.error('[%s]: An unkown match-type found in parser with regex: %s', host, regex_command) else: