Source code for dynamite_sdk.objects.baselines
import pandas as pd
from ipaddress import ip_address
from dateutil.parser import parse as date_parse
class InvalidIntervalError(Exception):
"""
Thrown when an unexpected raw interval format is encountered
"""
def __init__(self, message):
"""
:param message: A more specific error message
"""
msg = "An error occurred while parsing an interval: {}".format(message)
super(InvalidIntervalError, self).__init__(msg)
[docs]class Interval:
def __init__(self, raw_interval_document: dict):
self.raw_interval_document = raw_interval_document
self.elasticsearch_index = None #: The ElasticSearch index where the interval was found
self.originating_agent_tag = None #: The friendly name of the agent
self.forwarder_type = None #: Always "zeek"
self.event_type = None #: Always "netbase"
self.node_ip_address = None #: The ip_address of the originating host (collector/agent IP)
self.node_hostname = None #: The hostname of the originating host (collector/agent hostname)
self.address = None #: The monitored IP address
self.start_time = None #: Start time of the observation
self.end_time = None #: End time of the observation
self.interval_size = None #: The time between start_time and end_time
self.internal_ports = None #: Unique ports communicated with internally
self.internal_port_count = None #: Count of unique ports communicated with internally
self.internal_hosts = None #: Unique hosts communicated with internally
self.internal_host_count = None #: Count of unique hosts communicated with internally
self.external_ports = None #: Unique ports communicated with externally
self.external_port_counts = None #: Count of unique ports communicated with externally
self.external_hosts = None #: Unique IP's communicated with externally
self.external_host_count = None #: Count of unique hosts communicated with externally
self.internal_clients = None #: Unique internal clients communicating with this IP
self.internal_client_count = None #: Count of unique internal clients communicating with this IP
self.external_clients = None #: Unique external clients communicating with this IP
self.external_client_count = None #: Count fo unique external clients communicating with this IP
self.connections_count = None #: Total count of connections this IP was involved in
self.originating_connection_count = None #: Total count of external conns originated by this IP
self.successful_originating_connection_count = None #: Count of outbound conns originated by this IP that were successful
self.rejected_originating_connection_count = None #: Count of outbound conns originated by this IP that were rejected
self.originating_to_highport_count = None #: Count of outbound conns originated by this IP to ports >= 1024
self.originating_to_lowport_count = None #: Count of outbound conns originated by this IP to ports < 1024
self.originating_to_service = None #: Count of outbound conns originated by this IP to a recognized service
self.internal_originated_connections = None #: Count of internal connections by this host
self.internal_originated_rejected_connections = None #: Count of internal conns originated by this host that were rejected
self.internal_to_highport_count = None #: Count of internal conns to ports >= 1024
self.internal_to_lowport_count = None #: Count of internal conns to ports < 1024
self.internal_to_service_count = None #: Count of internal conns to recognized server
self.internal_received_connections = None #: Count of internal conns this IP responded to
self.internal_originating_bytes_sent = None #: Sum of bytes sent as originator in internal conns
self.internal_originating_bytes_received = None #: Sum of bytes received as originator in internal conns
self.external_originating_bytes_sent = None #: Sum of bytes sent as originator in external conns
self.external_originating_bytes_received = None #: Sum of bytes received as originator in external conns
self.internal_originating_packets_sent = None #: Count of packets sent in internal conns
self.internal_originating_packets_received = None #: Count of packets recevied in internal conns
self.external_originating_packets_sent = None #: Count of packets sent as originator in outbound conns
self.external_originating_packets_received = None #: Count of packets received as originator in outbound conns
self.smb_client_connection_count = None #: Count of SMB connections as a client
self.smb_server_connection_count = None #: Count of SMB connections as a server
self.smb_producer_consumer_ratio_average = None #: Avg pcr for smb connections
self.smb_producer_consumer_ratio_max = None #: Max pcr for smb connections
self.smb_producer_consumer_ratio_min = None #: Min pcr for smb connections
self.http_client_connection_count = None #: Count of http connections as client
self.http_server_connection_count = None #: Count of http connections as server
self.http_producer_consumer_ratio_average = None #: Avg pcr for http connections
self.http_producer_consumer_ratio_max = None #: Max pcr for http connections
self.http_producer_consumer_ratio_min = None #: Min pcr for http connections
self.dns_client_connection_count = None #: Count of dns connections as client
self.dns_server_connection_count = None #: Count of dns connections as server
self.dns_producer_consumer_ratio_average = None #: Avg pcr for dns connections
self.dns_producer_consumer_ratio_max = None #: Max pcr for dns connections
self.dns_producer_consumer_ratio_min = None #: Min pcr for dns connections
self.ssl_client_connection_count = None #: Count of ssl connections as client
self.ssl_server_connection_count = None #: Count of ssl connections as server
self.ssl_producer_consumer_ratio_average = None #: Avg pcr for ssl connections
self.ssl_producer_consumer_ratio_max = None #: Max pcr for ssl connections
self.ssl_producer_consumer_ratio_min = None #: Min pcr for ssl connections
self.rdp_client_connection_count = None #: Count of rdp connections as client
self.rdp_server_connection_count = None #: Count of rdp connections as server
self.rdp_producer_consumer_ratio_average = None #: Avg pcr for rdp connections
self.rdp_producer_consumer_ratio_max = None #: Max pcr for rdp connections
self.rdp_producer_consumer_ratio_min = None #: Min pcr for rdp connections
self._parse_raw_interval()
delattr(self, 'raw_interval_document')
def _parse_raw_interval(self):
try:
self.elasticsearch_index = self.raw_interval_document['_index']
except KeyError:
raise InvalidIntervalError('Missing index field')
try:
_source = self.raw_interval_document['_source']
except KeyError:
raise InvalidIntervalError('Missing _source section')
try:
self.node_ip_address = ip_address(_source['node']['ipaddr'])
except KeyError:
raise InvalidIntervalError('Missing node_ip_address field')
except ValueError:
raise InvalidIntervalError('Invalid node_ip_address field [{}]'.format(_source['node']['ipaddr']))
try:
self.originating_agent_tag = _source['fields']['originating_agent_tag']
self.forwarder_type = 'zeek'
except KeyError:
raise InvalidIntervalError('Missing originating_agent_tag')
try:
self.node_hostname = _source['node']['hostname']
except KeyError:
raise InvalidIntervalError('Missing node_hostname field')
try:
self.event_type = _source['event_type']
except KeyError:
raise InvalidIntervalError('Missing event_type field')
try:
try:
self.start_time = date_parse(_source['zeek']['starttime'])
except KeyError:
raise InvalidIntervalError('Missing starttime field')
except ValueError:
raise InvalidIntervalError('Invalid start_time field [{}]'.format(_source['zeek']['starttime']))
try:
self.start_time = date_parse(_source['zeek']['endtime'])
except KeyError:
raise InvalidIntervalError('Missing endtime field')
except ValueError:
raise InvalidIntervalError('Invalid end_time field [{}]'.format(_source['zeek']['endtime']))
try:
self.address = ip_address(_source['zeek']['address'])
except KeyError:
raise InvalidIntervalError('Missing address field')
except ValueError:
raise InvalidIntervalError('Invalid address field [{}]'.format(_source['zeek']['address']))
self.interval_size = self.end_time - self.start_time
self.internal_ports = _source['zeek'].get('int_ports')
self.internal_port_count = _source['zeek'].get('int_port_cnt')
self.internal_hosts = _source['zeek'].get('int_hosts')
self.internal_host_count = _source['zeek'].get('int_host_cnt')
self.external_ports = _source['zeek'].get('ext_ports')
self.external_port_counts = _source['zeek'].get('ext_port_cnt')
self.external_hosts = _source['zeek'].get('ext_hosts')
self.external_host_count = _source['zeek'].get('ext_host_cnt')
self.internal_clients = _source['zeek'].get('int_clients')
self.internal_client_count = _source['zeek'].get('int_client_cnt')
self.external_clients = _source['zeek'].get('ext_clients')
self.external_client_count = _source['zeek'].get('ext_client_cnt')
self.connections_count = _source['zeek'].get('total_conns')
self.originating_connection_count = _source['zeek'].get('out_orig_conns')
self.successful_originating_connection_count = _source['zeek'].get('out_succ_conns')
self.rejected_originating_connection_count = _source['zeek'].get('out_rej_conns')
self.originating_to_highport_count = _source['zeek'].get('out_to_highports')
self.originating_to_lowport_count = _source['zeek'].get('out_to_lowports')
self.originating_to_service = _source['zeek'].get('out_to_service')
self.internal_originated_connections = _source['zeek'].get('int_orig_conns')
self.internal_originated_rejected_connections = _source['zeek'].get('int_rej_conns')
self.internal_to_highport_count = _source['zeek'].get('int_to_highports')
self.internal_to_lowport_count = _source['zeek'].get('int_to_lowports')
self.internal_to_service_count = _source['zeek'].get('int_to_service')
self.internal_received_connections = _source['zeek'].get('int_resp_conns')
self.internal_originating_bytes_sent = _source['zeek'].get('int_orig_bytes_sent')
self.internal_originating_bytes_received = _source['zeek'].get('int_orig_bytes_rcvd')
self.external_originating_bytes_sent = _source['zeek'].get('out_orig_bytes_sent')
self.external_originating_bytes_received = _source['zeek'].get('out_orig_bytes_rcvd')
self.internal_originating_packets_sent = _source['zeek'].get('int_orig_pkts_sent')
self.internal_originating_packets_received = _source['zeek'].get('int_orig_pkts_recvd')
self.external_originating_packets_sent = _source['zeek'].get('out_orig_pkts_sent')
self.external_originating_packets_received = _source['zeek'].get('out_orig_pkts_recvd')
self.smb_client_connection_count = _source['zeek'].get('smb_client_conns')
self.smb_server_connection_count = _source['zeek'].get('smb_server_conns')
self.smb_producer_consumer_ratio_average = _source['zeek'].get('pcr_smb_avg')
self.smb_producer_consumer_ratio_max = _source['zeek'].get('pcr_smb_max')
self.smb_producer_consumer_ratio_min = _source['zeek'].get('pcr_smb_min')
self.http_client_connection_count = _source['zeek'].get('http_client_conns')
self.http_server_connection_count = _source['zeek'].get('http_server_conns')
self.http_producer_consumer_ratio_average = _source['zeek'].get('pcr_http_avg')
self.http_producer_consumer_ratio_max = _source['zeek'].get('pcr_http_max')
self.http_producer_consumer_ratio_min = _source['zeek'].get('pcr_http_min')
self.dns_client_connection_count = _source['zeek'].get('dns_client_conns')
self.dns_server_connection_count = _source['zeek'].get('dns_server_conns')
self.dns_producer_consumer_ratio_average = _source['zeek'].get('pcr_dns_avg')
self.dns_producer_consumer_ratio_max = _source['zeek'].get('pcr_dns_max')
self.dns_producer_consumer_ratio_min = _source['zeek'].get('pcr_dns_min')
self.ssl_client_connection_count = _source['zeek'].get('ssl_client_conns')
self.ssl_server_connection_count = _source['zeek'].get('ssl_server_conns')
self.ssl_producer_consumer_ratio_average = _source['zeek'].get('pcr_ssl_avg')
self.ssl_producer_consumer_ratio_max = _source['zeek'].get('pcr_ssl_max')
self.smb_producer_consumer_ratio_min = _source['zeek'].get('pcr_ssl_min')
self.rdp_client_connection_count = _source['zeek'].get('rdp_client_conns')
self.rdp_server_connection_count = _source['zeek'].get('rdp_server_conns')
self.rdp_producer_consumer_ratio_average = _source['zeek'].get('pcr_rdp_avg')
self.rdp_producer_consumer_ratio_max = _source['zeek'].get('pcr_rdp_max')
self.rdp_producer_consumer_ratio_min = _source['zeek'].get('pcr_rdp_min')
except KeyError:
raise InvalidIntervalError('Invalid dns record, missing "zeek" section')
def __str__(self) -> str:
"""
:return: A JSON representation of the Interval
"""
return str(vars(self))
[docs] def to_dataframe(self) -> pd.DataFrame:
"""
:return: DataFrame containng the field headings and single of of values
"""
ignore_vars = ['raw_interval_documen']
headers = [var for var in vars(self) if var not in ignore_vars]
data = [[getattr(self, header) for header in headers]]
return pd.DataFrame(data, columns=headers)