Source code for dynamite_sdk.objects.baselines

import pandas as pd

from ipaddress import ip_address
from dateutil.parser import parse as date_parse


class InvalidIntervalError(Exception):
    """
    Thrown when an unexpected raw interval format is encountered
    """
    def __init__(self, message):
        """
        :param message: A more specific error message
        """
        msg = "An error occurred while parsing an interval: {}".format(message)
        super(InvalidIntervalError, self).__init__(msg)


[docs]class Interval: def __init__(self, raw_interval_document: dict): self.raw_interval_document = raw_interval_document self.elasticsearch_index = None #: The ElasticSearch index where the interval was found self.originating_agent_tag = None #: The friendly name of the agent self.forwarder_type = None #: Always "zeek" self.event_type = None #: Always "netbase" self.node_ip_address = None #: The ip_address of the originating host (collector/agent IP) self.node_hostname = None #: The hostname of the originating host (collector/agent hostname) self.address = None #: The monitored IP address self.start_time = None #: Start time of the observation self.end_time = None #: End time of the observation self.interval_size = None #: The time between start_time and end_time self.internal_ports = None #: Unique ports communicated with internally self.internal_port_count = None #: Count of unique ports communicated with internally self.internal_hosts = None #: Unique hosts communicated with internally self.internal_host_count = None #: Count of unique hosts communicated with internally self.external_ports = None #: Unique ports communicated with externally self.external_port_counts = None #: Count of unique ports communicated with externally self.external_hosts = None #: Unique IP's communicated with externally self.external_host_count = None #: Count of unique hosts communicated with externally self.internal_clients = None #: Unique internal clients communicating with this IP self.internal_client_count = None #: Count of unique internal clients communicating with this IP self.external_clients = None #: Unique external clients communicating with this IP self.external_client_count = None #: Count fo unique external clients communicating with this IP self.connections_count = None #: Total count of connections this IP was involved in self.originating_connection_count = None #: Total count of external conns originated by this IP self.successful_originating_connection_count = None #: Count of outbound conns originated by this IP that were successful self.rejected_originating_connection_count = None #: Count of outbound conns originated by this IP that were rejected self.originating_to_highport_count = None #: Count of outbound conns originated by this IP to ports >= 1024 self.originating_to_lowport_count = None #: Count of outbound conns originated by this IP to ports < 1024 self.originating_to_service = None #: Count of outbound conns originated by this IP to a recognized service self.internal_originated_connections = None #: Count of internal connections by this host self.internal_originated_rejected_connections = None #: Count of internal conns originated by this host that were rejected self.internal_to_highport_count = None #: Count of internal conns to ports >= 1024 self.internal_to_lowport_count = None #: Count of internal conns to ports < 1024 self.internal_to_service_count = None #: Count of internal conns to recognized server self.internal_received_connections = None #: Count of internal conns this IP responded to self.internal_originating_bytes_sent = None #: Sum of bytes sent as originator in internal conns self.internal_originating_bytes_received = None #: Sum of bytes received as originator in internal conns self.external_originating_bytes_sent = None #: Sum of bytes sent as originator in external conns self.external_originating_bytes_received = None #: Sum of bytes received as originator in external conns self.internal_originating_packets_sent = None #: Count of packets sent in internal conns self.internal_originating_packets_received = None #: Count of packets recevied in internal conns self.external_originating_packets_sent = None #: Count of packets sent as originator in outbound conns self.external_originating_packets_received = None #: Count of packets received as originator in outbound conns self.smb_client_connection_count = None #: Count of SMB connections as a client self.smb_server_connection_count = None #: Count of SMB connections as a server self.smb_producer_consumer_ratio_average = None #: Avg pcr for smb connections self.smb_producer_consumer_ratio_max = None #: Max pcr for smb connections self.smb_producer_consumer_ratio_min = None #: Min pcr for smb connections self.http_client_connection_count = None #: Count of http connections as client self.http_server_connection_count = None #: Count of http connections as server self.http_producer_consumer_ratio_average = None #: Avg pcr for http connections self.http_producer_consumer_ratio_max = None #: Max pcr for http connections self.http_producer_consumer_ratio_min = None #: Min pcr for http connections self.dns_client_connection_count = None #: Count of dns connections as client self.dns_server_connection_count = None #: Count of dns connections as server self.dns_producer_consumer_ratio_average = None #: Avg pcr for dns connections self.dns_producer_consumer_ratio_max = None #: Max pcr for dns connections self.dns_producer_consumer_ratio_min = None #: Min pcr for dns connections self.ssl_client_connection_count = None #: Count of ssl connections as client self.ssl_server_connection_count = None #: Count of ssl connections as server self.ssl_producer_consumer_ratio_average = None #: Avg pcr for ssl connections self.ssl_producer_consumer_ratio_max = None #: Max pcr for ssl connections self.ssl_producer_consumer_ratio_min = None #: Min pcr for ssl connections self.rdp_client_connection_count = None #: Count of rdp connections as client self.rdp_server_connection_count = None #: Count of rdp connections as server self.rdp_producer_consumer_ratio_average = None #: Avg pcr for rdp connections self.rdp_producer_consumer_ratio_max = None #: Max pcr for rdp connections self.rdp_producer_consumer_ratio_min = None #: Min pcr for rdp connections self._parse_raw_interval() delattr(self, 'raw_interval_document') def _parse_raw_interval(self): try: self.elasticsearch_index = self.raw_interval_document['_index'] except KeyError: raise InvalidIntervalError('Missing index field') try: _source = self.raw_interval_document['_source'] except KeyError: raise InvalidIntervalError('Missing _source section') try: self.node_ip_address = ip_address(_source['node']['ipaddr']) except KeyError: raise InvalidIntervalError('Missing node_ip_address field') except ValueError: raise InvalidIntervalError('Invalid node_ip_address field [{}]'.format(_source['node']['ipaddr'])) try: self.originating_agent_tag = _source['fields']['originating_agent_tag'] self.forwarder_type = 'zeek' except KeyError: raise InvalidIntervalError('Missing originating_agent_tag') try: self.node_hostname = _source['node']['hostname'] except KeyError: raise InvalidIntervalError('Missing node_hostname field') try: self.event_type = _source['event_type'] except KeyError: raise InvalidIntervalError('Missing event_type field') try: try: self.start_time = date_parse(_source['zeek']['starttime']) except KeyError: raise InvalidIntervalError('Missing starttime field') except ValueError: raise InvalidIntervalError('Invalid start_time field [{}]'.format(_source['zeek']['starttime'])) try: self.start_time = date_parse(_source['zeek']['endtime']) except KeyError: raise InvalidIntervalError('Missing endtime field') except ValueError: raise InvalidIntervalError('Invalid end_time field [{}]'.format(_source['zeek']['endtime'])) try: self.address = ip_address(_source['zeek']['address']) except KeyError: raise InvalidIntervalError('Missing address field') except ValueError: raise InvalidIntervalError('Invalid address field [{}]'.format(_source['zeek']['address'])) self.interval_size = self.end_time - self.start_time self.internal_ports = _source['zeek'].get('int_ports') self.internal_port_count = _source['zeek'].get('int_port_cnt') self.internal_hosts = _source['zeek'].get('int_hosts') self.internal_host_count = _source['zeek'].get('int_host_cnt') self.external_ports = _source['zeek'].get('ext_ports') self.external_port_counts = _source['zeek'].get('ext_port_cnt') self.external_hosts = _source['zeek'].get('ext_hosts') self.external_host_count = _source['zeek'].get('ext_host_cnt') self.internal_clients = _source['zeek'].get('int_clients') self.internal_client_count = _source['zeek'].get('int_client_cnt') self.external_clients = _source['zeek'].get('ext_clients') self.external_client_count = _source['zeek'].get('ext_client_cnt') self.connections_count = _source['zeek'].get('total_conns') self.originating_connection_count = _source['zeek'].get('out_orig_conns') self.successful_originating_connection_count = _source['zeek'].get('out_succ_conns') self.rejected_originating_connection_count = _source['zeek'].get('out_rej_conns') self.originating_to_highport_count = _source['zeek'].get('out_to_highports') self.originating_to_lowport_count = _source['zeek'].get('out_to_lowports') self.originating_to_service = _source['zeek'].get('out_to_service') self.internal_originated_connections = _source['zeek'].get('int_orig_conns') self.internal_originated_rejected_connections = _source['zeek'].get('int_rej_conns') self.internal_to_highport_count = _source['zeek'].get('int_to_highports') self.internal_to_lowport_count = _source['zeek'].get('int_to_lowports') self.internal_to_service_count = _source['zeek'].get('int_to_service') self.internal_received_connections = _source['zeek'].get('int_resp_conns') self.internal_originating_bytes_sent = _source['zeek'].get('int_orig_bytes_sent') self.internal_originating_bytes_received = _source['zeek'].get('int_orig_bytes_rcvd') self.external_originating_bytes_sent = _source['zeek'].get('out_orig_bytes_sent') self.external_originating_bytes_received = _source['zeek'].get('out_orig_bytes_rcvd') self.internal_originating_packets_sent = _source['zeek'].get('int_orig_pkts_sent') self.internal_originating_packets_received = _source['zeek'].get('int_orig_pkts_recvd') self.external_originating_packets_sent = _source['zeek'].get('out_orig_pkts_sent') self.external_originating_packets_received = _source['zeek'].get('out_orig_pkts_recvd') self.smb_client_connection_count = _source['zeek'].get('smb_client_conns') self.smb_server_connection_count = _source['zeek'].get('smb_server_conns') self.smb_producer_consumer_ratio_average = _source['zeek'].get('pcr_smb_avg') self.smb_producer_consumer_ratio_max = _source['zeek'].get('pcr_smb_max') self.smb_producer_consumer_ratio_min = _source['zeek'].get('pcr_smb_min') self.http_client_connection_count = _source['zeek'].get('http_client_conns') self.http_server_connection_count = _source['zeek'].get('http_server_conns') self.http_producer_consumer_ratio_average = _source['zeek'].get('pcr_http_avg') self.http_producer_consumer_ratio_max = _source['zeek'].get('pcr_http_max') self.http_producer_consumer_ratio_min = _source['zeek'].get('pcr_http_min') self.dns_client_connection_count = _source['zeek'].get('dns_client_conns') self.dns_server_connection_count = _source['zeek'].get('dns_server_conns') self.dns_producer_consumer_ratio_average = _source['zeek'].get('pcr_dns_avg') self.dns_producer_consumer_ratio_max = _source['zeek'].get('pcr_dns_max') self.dns_producer_consumer_ratio_min = _source['zeek'].get('pcr_dns_min') self.ssl_client_connection_count = _source['zeek'].get('ssl_client_conns') self.ssl_server_connection_count = _source['zeek'].get('ssl_server_conns') self.ssl_producer_consumer_ratio_average = _source['zeek'].get('pcr_ssl_avg') self.ssl_producer_consumer_ratio_max = _source['zeek'].get('pcr_ssl_max') self.smb_producer_consumer_ratio_min = _source['zeek'].get('pcr_ssl_min') self.rdp_client_connection_count = _source['zeek'].get('rdp_client_conns') self.rdp_server_connection_count = _source['zeek'].get('rdp_server_conns') self.rdp_producer_consumer_ratio_average = _source['zeek'].get('pcr_rdp_avg') self.rdp_producer_consumer_ratio_max = _source['zeek'].get('pcr_rdp_max') self.rdp_producer_consumer_ratio_min = _source['zeek'].get('pcr_rdp_min') except KeyError: raise InvalidIntervalError('Invalid dns record, missing "zeek" section') def __str__(self) -> str: """ :return: A JSON representation of the Interval """ return str(vars(self))
[docs] def to_dataframe(self) -> pd.DataFrame: """ :return: DataFrame containng the field headings and single of of values """ ignore_vars = ['raw_interval_documen'] headers = [var for var in vars(self) if var not in ignore_vars] data = [[getattr(self, header) for header in headers]] return pd.DataFrame(data, columns=headers)