From 2f486fe489e21b668857ebb60eb0af5beb3e105d Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:24:24 +0545 Subject: [PATCH 01/20] Modified redis manager to use prefix as id instead of port --- redis_manager.py | 136 ++++++++++++++++++++++++++++------------------- 1 file changed, 80 insertions(+), 56 deletions(-) diff --git a/redis_manager.py b/redis_manager.py index 147105ef10..82540c2178 100644 --- a/redis_manager.py +++ b/redis_manager.py @@ -5,6 +5,7 @@ import redis import os import time +import uuid class RedisManager: def __init__(self, terminate_slips=None): @@ -78,6 +79,9 @@ def get_random_redis_port(self): "Unable to start slips.\n") return False + def get_random_prefix(self) -> str: + return str(uuid.uuid4()) + def clear_redis_cache_database( self, redis_host='localhost', redis_port=6379 ) -> bool: @@ -94,26 +98,39 @@ def clear_redis_cache_database( rcache.flushdb() return True - def close_all_ports(self): + def close_all_instance(self): """ Closes all the redis ports in logfile and in slips supported range of ports """ - if not hasattr(self, 'open_servers_PIDs'): - self.get_open_redis_servers() + if not hasattr(self, 'open_servers_IDs'): + self.get_open_redis_servers_ID() + unique_ports = set() # close all ports in logfile - for pid in self.open_servers_PIDs: - self.flush_redis_server(pid=pid) + for _id in self.open_servers_IDs: + self.flush_redis_id(_id=_id, port = self.open_servers_IDs[_id]) + unique_ports.add(self.open_servers_IDs[_id]) + + # Kill default port + + for port in unique_ports: + # Skip cache db which is in port 6379 + if str(port) == '6379': + continue + pid = self.get_pid_of_redis_server(port = port) self.kill_redis_server(pid) + # closes all the ports in slips supported range of ports - slips_supported_range = list(range(self.start_port, self.end_port + 1)) - slips_supported_range.append(6379) - for port in slips_supported_range: - if pid := self.get_pid_of_redis_server(port): - self.flush_redis_server(pid=pid) - self.kill_redis_server(pid) + # slips_supported_range = list(range(self.start_port, self.end_port + 1)) + # slips_supported_range.append(6379) + + # for port in slips_supported_range: + # if _id := self.get_pid_of_redis_server(port): + # self.flush_redis_server(_id=id) + # self.kill_redis_server(pid) + # print(f"Successfully closed all redis servers on ports {self.start_port} to {self.end_port}") @@ -138,11 +155,11 @@ def get_pid_of_redis_server(self, port: int) -> str: return pid return False - def get_open_redis_servers(self) -> dict: + def get_open_redis_servers_ID(self) -> dict: """ Returns the dict of PIDs and ports of the redis servers started by slips """ - self.open_servers_PIDs = {} + self.open_servers_IDs = {} try: with open(self.running_logfile, 'r') as f: for line in f.read().splitlines(): @@ -154,20 +171,20 @@ def get_open_redis_servers(self) -> dict: ): continue line = line.split(',') - pid, port = line[3], line[2] - self.open_servers_PIDs[pid] = port - return self.open_servers_PIDs + _id, port = line[3], line[2] + self.open_servers_IDs[_id] = port + return self.open_servers_IDs except FileNotFoundError: # print(f"Error: {self.running_logfile} is not found. Can't kill open servers. Stopping.") return {} - def print_open_redis_servers(self): + def print_open_redis_id(self): """ Returns a dict {counter: (used_port,pid) } """ - open_servers = {} + open_ids = {} to_print = f"Choose which one to kill [0,1,2 etc..]\n" \ - f"[0] Close all Redis servers\n" + f"[0] Close all Redis ids\n" there_are_ports_to_print = False try: with open(self.running_logfile, 'r') as f: @@ -182,20 +199,20 @@ def print_open_redis_servers(self): continue line_number += 1 line = line.split(',') - file, port, pid = line[1], line[2], line[3] + file, port, _id = line[1], line[2], line[3] there_are_ports_to_print = True - to_print += f"[{line_number}] {file} - port {port}\n" - open_servers[line_number] = (port, pid) + to_print += f"[{line_number}] {file} - port {port} - id {_id}\n" + open_ids[line_number] = (port, _id) except FileNotFoundError: - print(f"{self.running_logfile} is not found. Can't get open redis servers. Stopping.") + print(f"{self.running_logfile} is not found. Can't get open redis id. Stopping.") return False if there_are_ports_to_print: print(to_print) else: - print(f"No open redis servers in {self.running_logfile}") + print(f"No open redis id in {self.running_logfile}") - return open_servers + return open_ids def get_port_of_redis_server(self, pid: str): @@ -211,23 +228,24 @@ def get_port_of_redis_server(self, pid: str): return False - def flush_redis_server(self, pid: str='', port: str=''): + def flush_redis_id(self, _id: str='', port: str=''): """ Flush the redis server on this pid, only 1 param should be given, pid or port :param pid: can be False if port is given Gets the pid of the port is not given """ - if not port and not pid: + if not port and not _id: return False # sometimes the redis port is given, no need to get it manually - if not port and pid: - if not hasattr(self, 'open_servers_PIDs'): - self.get_open_redis_servers() - port = self.open_servers_PIDs.get(str(pid), False) + if not port and _id: + if not hasattr(self, 'open_servers_IDs'): + self.get_open_redis_servers_ID() + port = self.open_servers_IDs.get(str(_id), False) if not port: - # try to get the port using a cmd - port = self.get_port_of_redis_server(pid) + # use default port + port = '6379' + port = str(port) # clear the server opened on this port @@ -245,9 +263,15 @@ def flush_redis_server(self, pid: str='', port: str=''): retry_on_timeout=True, health_check_interval=20, ) - r.flushall() - r.flushdb() - r.script_flush() + + # r.flushall() + # r.flushdb() + # r.script_flush() + + # Delete keys with the prefix id + for key in r.scan_iter(_id + "*"): + r.delete(key) + return True except redis.exceptions.ConnectionError: # server already killed! @@ -306,52 +330,52 @@ def remove_old_logline(self, redis_port): os.replace(tmpfile, self.running_logfile) - def remove_server_from_log(self, redis_port): + def remove_server_from_log(self, _id): """ deletes the server running on the given pid from running_slips_logs """ - redis_port = str(redis_port) + _id = str(_id) tmpfile = 'tmp_running_slips_log.txt' with open(self.running_logfile, 'r') as logfile: with open(tmpfile, 'w') as tmp: all_lines = logfile.read().splitlines() # delete the line using that port for line in all_lines: - if redis_port not in line: + if _id not in line: tmp.write(f'{line}\n') # replace file with original name os.replace(tmpfile, self.running_logfile) - def close_open_redis_servers(self): + def close_open_redis_id(self): """ Function to close unused open redis-servers based on what the user chooses """ - if not hasattr(self, 'open_servers_PIDs'): + if not hasattr(self, 'open_servers_IDs'): # fill the dict - self.get_open_redis_servers() + self.get_open_redis_servers_ID() with contextlib.suppress(KeyboardInterrupt): # open_servers {counter: (port,pid),...}} - open_servers:dict = self.print_open_redis_servers() - if not open_servers and self.terminate_slips: + open_ids:dict = self.print_open_redis_id() + if not open_ids and self.terminate_slips: self.terminate_slips() - server_to_close = input() + id_to_close = input() # close all ports in running_slips_logs.txt and in our supported range - if server_to_close == '0': - self.close_all_ports() + if id_to_close == '0': + self.close_all_instance() - elif len(open_servers) > 0: + elif len(open_ids) > 0: # close the given server number try: - pid = open_servers[int(server_to_close)][1] - port = open_servers[int(server_to_close)][0] - if self.flush_redis_server(pid=pid) and self.kill_redis_server(pid): - print(f"Killed redis server on port {port}.") + _id = open_ids[int(id_to_close)][1] + port = open_ids[int(id_to_close)][0] + if self.flush_redis_id(_id=_id, port = port): + print(f"Deleted process of id {_id} on port {port}.") else: - print(f"Redis server running on port {port} " - f"is either already killed or you don't have " - f"enough permission to kill it.") - self.remove_server_from_log(port) + print(f"Redis server running on port {port} with id {_id}" + f"is either already deleted or you don't have " + f"enough permission to delete it.") + self.remove_server_from_log(_id) except (KeyError, ValueError): print(f"Invalid input {server_to_close}") From 713d698b8514c4aa82007fbbbf0cae4ada2d8c37 Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:26:15 +0545 Subject: [PATCH 02/20] Modified to use id as prefix instead of port --- slips.py | 65 +++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/slips.py b/slips.py index a1af5b8798..d70c774e86 100755 --- a/slips.py +++ b/slips.py @@ -186,7 +186,7 @@ def update_local_TI_files(self): # only one instance of slips should be able to update ports and orgs at a time # so this function will only be allowed to run from 1 slips instance. with Lock(name="slips_ports_and_orgs"): - update_manager = UpdateFileManager(self.outputqueue, self.redis_port) + update_manager = UpdateFileManager(self.outputqueue, self.prefix) update_manager.update_ports_info() update_manager.update_org_files() except CannotAcquireLock: @@ -288,7 +288,7 @@ def prepare_output_dir(self): # print(f'[Main] Storing Slips logs in {self.args.output}') - def log_redis_server_PID(self, redis_port, redis_pid): + def log_redis_server_prefix(self, redis_port, prefix): now = utils.convert_format(datetime.now(), utils.alerts_format) try: # used in case we need to remove the line using 6379 from running logfile @@ -298,13 +298,13 @@ def log_redis_server_PID(self, redis_port, redis_pid): f.write( '# This file contains a list of used redis ports.\n' '# Once a server is killed, it will be removed from this file.\n' - 'Date, File or interface, Used port, Server PID,' + 'Date, File or interface, Used port, Unique ID (prefix),' ' Output Zeek Dir, Logs Dir, Slips PID, Is Daemon, Save the DB\n' ) f.write( f'{now},{self.input_information},{redis_port},' - f'{redis_pid},{self.zeek_folder},{self.args.output},' + f'{prefix},{self.zeek_folder},{self.args.output},' f'{os.getpid()},' f'{bool(self.args.daemon)},{self.args.save}\n' ) @@ -312,11 +312,11 @@ def log_redis_server_PID(self, redis_port, redis_pid): # last run was by root, change the file ownership to non-root os.remove(self.redis_man.running_logfile) open(self.redis_man.running_logfile, 'w').close() - self.log_redis_server_PID(redis_port, redis_pid) + self.log_redis_server_prefix(redis_port, prefix) - if redis_port == 6379: - # remove the old logline using this port - self.redis_man.remove_old_logline(6379) + # if redis_port == 6379: + # # remove the old logline using this port + # self.redis_man.remove_old_logline(6379) def set_mode(self, mode, daemon=''): """ @@ -412,7 +412,7 @@ def load_redis_db(self, redis_port): self.input_information = os.path.basename(self.args.db) redis_pid = self.redis_man.get_pid_of_redis_server(redis_port) self.zeek_folder = '""' - self.log_redis_server_PID(redis_port, redis_pid) + self.log_redis_server_prefix(redis_port, redis_pid) self.redis_man.remove_old_logline(redis_port) print( @@ -541,22 +541,35 @@ def start(self): ########################## # get the port that is going to be used for this instance of slips + # if self.args.port: + # self.redis_port = int(self.args.port) + # # close slips if port is in use + # self.metadata_man.check_if_port_is_in_use(self.redis_port) + # elif self.args.multiinstance: + # self.redis_port = self.redis_man.get_random_redis_port() + # if not self.redis_port: + # # all ports are unavailable + # inp = input("Press Enter to close all ports.\n") + # if inp == '': + # self.redis_man.close_all_ports() + # self.terminate_slips() + # else: + # # even if this port is in use, it will be overwritten by slips + # self.redis_port = 6379 + # # self.check_if_port_is_in_use(self.redis_port) + if self.args.port: + # set port to specified if explicitly specified self.redis_port = int(self.args.port) - # close slips if port is in use self.metadata_man.check_if_port_is_in_use(self.redis_port) - elif self.args.multiinstance: - self.redis_port = self.redis_man.get_random_redis_port() - if not self.redis_port: - # all ports are unavailable - inp = input("Press Enter to close all ports.\n") - if inp == '': - self.redis_man.close_all_ports() - self.terminate_slips() else: - # even if this port is in use, it will be overwritten by slips + # default port 6379 self.redis_port = 6379 - # self.check_if_port_is_in_use(self.redis_port) + + if self.args.prefix: + self.prefix = self.args.prefix + else: + self.prefix = self.redis_man.get_random_prefix() # Output thread. outputprocess should be created first because it handles # the output of the rest of the threads. @@ -570,6 +583,7 @@ def start(self): self.args.verbose, self.args.debug, self.redis_port, + self.prefix, stdout=current_stdout, stderr=stderr, slips_logfile=slips_logfile, @@ -587,8 +601,8 @@ def start(self): # log the PID of the started redis-server # should be here after we're sure that the server was started - redis_pid = self.redis_man.get_pid_of_redis_server(self.redis_port) - self.log_redis_server_PID(self.redis_port, redis_pid) + # redis_pid = self.redis_man.get_pid_of_redis_server(self.redis_port) + self.log_redis_server_prefix(self.redis_port, self.prefix) __database__.set_slips_mode(self.mode) @@ -608,7 +622,7 @@ def start(self): __database__.store_std_file(**std_files) - self.print(f'Using redis server on port: {green(self.redis_port)}', 1, 0) + self.print(f'Using redis server on port: {green(self.redis_port)} with id: {green(self.prefix)}', 1, 0) self.print(f'Started {green("Main")} process [PID {green(self.pid)}]', 1, 0) self.print(f'Started {green("Output Process")} [PID {green(output_process.pid)}]', 1, 0) self.print('Starting modules', 1, 0) @@ -638,6 +652,7 @@ def sig_handler(sig, frame): self.args.output, logs_dir, self.redis_port, + self.prefix, ) evidence_process.start() self.print( @@ -660,6 +675,7 @@ def sig_handler(sig, frame): self.args.verbose, self.args.debug, self.redis_port, + self.prefix, ) profiler_process.start() self.print( @@ -684,6 +700,7 @@ def sig_handler(sig, frame): self.zeek_folder, self.line_type, self.redis_port, + self.prefix, ) inputProcess.start() self.print( @@ -701,7 +718,7 @@ def sig_handler(sig, frame): self.print('Warning: P2P is only supported using an interface. Disabled P2P.') # warn about unused open redis servers - open_servers = len(self.redis_man.get_open_redis_servers()) + open_servers = len(self.redis_man.get_open_redis_servers_ID()) if open_servers > 1: self.print( f'Warning: You have {open_servers} ' From ff351d63189e1511100e0b0d983eb6f1ad3d9643 Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:27:05 +0545 Subject: [PATCH 03/20] Changed database to publish and store keys with prefix --- slips_files/core/database/_profile_flow.py | 75 ++--- slips_files/core/database/database.py | 304 +++++++++++---------- 2 files changed, 194 insertions(+), 185 deletions(-) diff --git a/slips_files/core/database/_profile_flow.py b/slips_files/core/database/_profile_flow.py index 5a98abfbe0..d651c753a1 100644 --- a/slips_files/core/database/_profile_flow.py +++ b/slips_files/core/database/_profile_flow.py @@ -11,11 +11,14 @@ def __init__(self): # The name is used to print in the outputprocess self.name = 'DB' self.separator = '_' + self.prefix = '' + def set_prefix(self, prefix): + self.prefix = prefix def publish(self, channel, data): """Publish something""" - self.r.publish(channel, data) + self.r.publish(self.prefix + self.separator + str(channel), data) def getIPData(self, ip: str) -> dict: """ @@ -112,7 +115,7 @@ def update_times_contacted(self, ip, direction, profileid, twid): # Get the DstIPs data for this tw in this profile # The format is {'1.1.1.1' : 3} - ips_contacted = self.r.hget(profileid_twid, f'{direction}IPs') + ips_contacted = self.r.hget(self.prefix + self.separator + profileid_twid, f'{direction}IPs') if not ips_contacted: ips_contacted = {} @@ -125,7 +128,7 @@ def update_times_contacted(self, ip, direction, profileid, twid): ips_contacted[ip] = 1 ips_contacted = json.dumps(ips_contacted) - self.r.hset(profileid_twid, f'{direction}IPs', str(ips_contacted)) + self.r.hset(self.prefix + self.separator + profileid_twid, f'{direction}IPs', str(ips_contacted)) def getFinalStateFromFlags(self, state, pkts): """ @@ -300,7 +303,7 @@ def getDataFromProfileTW( try: key = direction + type_data + role + protocol + state # self.print('Asked Key: {}'.format(key)) - data = self.r.hget(f'{profileid}{self.separator}{twid}', key) + data = self.r.hget(self.prefix + self.separator + f'{profileid}{self.separator}{twid}', key) value = {} if data: portdata = json.loads(data) @@ -414,7 +417,7 @@ def add_ips(self, profileid, twid, ip_as_obj, columns, role: str): ) # Store this data in the profile hash self.r.hset( - f'{profileid}{self.separator}{twid}', + self.prefix + self.separator + f'{profileid}{self.separator}{twid}', key_name, json.dumps(profileid_twid_data) ) @@ -515,7 +518,7 @@ def add_tuple( ) # Get all the InTuples or OutTuples for this profileid in this TW profileid_twid = f'{profileid}{self.separator}{twid}' - tuples = self.r.hget(profileid_twid, direction) + tuples = self.r.hget(self.prefix + self.separator + profileid_twid, direction) # Separate the symbold to add and the previous data (symbol_to_add, previous_two_timestamps) = data_tuple if not tuples: @@ -566,7 +569,7 @@ def add_tuple( # Convet the dictionary to json tuples = json.dumps(tuples) # Store the new data on the db - self.r.hset(profileid_twid, direction, str(tuples)) + self.r.hset(self.prefix + self.separator + profileid_twid, direction, str(tuples)) # Mark the tw as modified self.markProfileTWAsModified(profileid, twid, starttime) @@ -578,7 +581,7 @@ def add_tuple( self.outputqueue.put(f'01|database|[DB] {traceback.format_exc()}') def getSlipsInternalTime(self): - return self.r.get('slips_internal_time') + return self.r.get(self.prefix + self.separator + 'slips_internal_time') def search_tws_for_flow(self, profileid, twid, uid, go_back=False): """ @@ -629,7 +632,7 @@ def check_TW_to_close(self, close_all=False): modification_time = float('inf') profiles_tws_to_close = self.r.zrangebyscore( - 'ModifiedTW', 0, modification_time, withscores=True + self.prefix + self.separator + 'ModifiedTW', 0, modification_time, withscores=True ) for profile_tw_to_close in profiles_tws_to_close: @@ -650,8 +653,8 @@ def markProfileTWAsClosed(self, profileid_tw): """ Mark the TW as closed so tools can work on its data """ - self.r.sadd('ClosedTW', profileid_tw) - self.r.zrem('ModifiedTW', profileid_tw) + self.r.sadd(self.prefix + self.separator + 'ClosedTW', profileid_tw) + self.r.zrem(self.prefix + self.separator + 'ModifiedTW', profileid_tw) self.publish('tw_closed', profileid_tw) def markProfileTWAsModified(self, profileid, twid, timestamp): @@ -668,7 +671,7 @@ def markProfileTWAsModified(self, profileid, twid, timestamp): data = { f'{profileid}{self.separator}{twid}': float(timestamp) } - self.r.zadd('ModifiedTW', data) + self.r.zadd(self.prefix + self.separator + 'ModifiedTW', data) self.publish( 'tw_modified', f'{profileid}:{twid}' @@ -780,7 +783,7 @@ def add_port( data = json.dumps(old_profileid_twid_data) hash_key = f'{profileid}{self.separator}{twid}' key_name = f'{port_type}Ports{role}{proto}{summaryState}' - self.r.hset(hash_key, key_name, str(data)) + self.r.hset(self.prefix + self.separator + hash_key, key_name, str(data)) self.markProfileTWAsModified(profileid, twid, starttime) def add_flow( @@ -838,7 +841,7 @@ def add_flow( flow = json.dumps(flow) # Store in the hash x.x.x.x_timewindowx_flows value = self.r.hset( - f'{profileid}{self.separator}{twid}{self.separator}flows', + self.prefix + self.separator + f'{profileid}{self.separator}{twid}{self.separator}flows', uid, flow, ) @@ -849,7 +852,7 @@ def add_flow( # The key was not there before. So this flow is not repeated # Store the label in our uniq set, and increment it by 1 if label: - self.r.zincrby('labels', 1, label) + self.r.zincrby(self.prefix + self.separator + 'labels', 1, label) flow = {uid: flow} @@ -890,51 +893,51 @@ def set_local_network(self, saddr): return # get the local network of this saddr if network_range := utils.get_cidr_of_ip(saddr): - self.r.set("local_network", network_range) + self.r.set(self.prefix + self.separator + "local_network", network_range) self.is_localnet_set = True def get_local_network(self): - return self.r.get("local_network") + return self.r.get(self.prefix + self.separator + "local_network") def get_label_count(self, label): """ :param label: malicious or normal """ - return self.r.zscore('labels', label) + return self.r.zscore(self.prefix + self.separator + 'labels', label) def get_disabled_modules(self)-> list: - return json.loads(self.r.hget('analysis', 'disabled_modules')) + return json.loads(self.r.hget(self.prefix + self.separator + 'analysis', 'disabled_modules')) def set_input_metadata(self, info:dict): """ sets name, size, analysis dates, and zeek_dir in the db """ for info, val in info.items(): - self.r.hset('analysis', info, val) + self.r.hset(self.prefix + self.separator + 'analysis', info, val) def get_zeek_output_dir(self): """ gets zeek output dir from the db """ - return self.r.hget('analysis', 'zeek_dir') + return self.r.hget(self.prefix + self.separator + 'analysis', 'zeek_dir') def get_total_flows(self): """ gets total flows to process from the db """ - return self.r.hget('analysis', 'total_flows') + return self.r.hget(self.prefix + self.separator + 'analysis', 'total_flows') def get_input_type(self): """ gets input type from the db """ - return self.r.hget('analysis', 'input_type') + return self.r.hget(self.prefix + self.separator + 'analysis', 'input_type') def get_output_dir(self, info:dict): """ returns the currently used output dir """ - return self.r.hget('analysis', 'output_dir') + return self.r.hget(self.prefix + self.separator + 'analysis', 'output_dir') def get_flow(self, profileid, twid, uid): @@ -947,7 +950,7 @@ def get_flow(self, profileid, twid, uid): # outside of home_network when this param is given return {} temp = self.r.hget( - f'{profileid}{self.separator}{twid}{self.separator}flows', uid + self.prefix + self.separator + f'{profileid}{self.separator}{twid}{self.separator}flows', uid ) return {uid: temp} @@ -1005,7 +1008,7 @@ def add_out_ssl( # Convert to json string data = json.dumps(data) self.r.hset( - f'{profileid}{self.separator}{twid}{self.separator}altflows', + self.prefix + self.separator + f'{profileid}{self.separator}{twid}{self.separator}altflows', uid, data, ) @@ -1040,7 +1043,7 @@ def add_out_ssl( if SNI_port not in sni_ipdata: # Verify that the SNI is equal to any of the domains in the DNS resolution # only add this SNI to our db if it has a DNS resolution - if dns_resolutions := self.r.hgetall('DNSresolution'): + if dns_resolutions := self.r.hgetall(self.prefix + self.separator + 'DNSresolution'): # dns_resolutions is a dict with {ip:{'ts'..,'domains':..., 'uid':..}} for ip, resolution in dns_resolutions.items(): resolution = json.loads(resolution) @@ -1078,7 +1081,7 @@ def setInfoForIPs(self, ip: str, ipdata: dict): self.rcache.hset('IPsInfo', ip, json.dumps(data)) if new_key: - self.r.publish('ip_info_change', ip) + self.r.publish(self.prefix + self.separator + 'ip_info_change', ip) def get_p2p_reports_about_ip(self, ip) -> dict: """ @@ -1174,7 +1177,7 @@ def add_out_http( data = json.dumps(data) self.r.hset( - f'{profileid}{ self.separator }{twid}{ self.separator }altflows', + self.prefix + self.separator + f'{profileid}{ self.separator }{twid}{ self.separator }altflows', uid, data, ) @@ -1252,7 +1255,7 @@ def add_out_ssh( data = json.dumps(data) # Set the dns as alternative flow self.r.hset( - f'{profileid}{self.separator}{twid}{self.separator}altflows', + self.prefix + self.separator + f'{profileid}{self.separator}{twid}{self.separator}altflows', uid, data, ) @@ -1310,7 +1313,7 @@ def add_out_notice( } to_send = json.dumps(to_send) self.r.hset( - f'{profileid}{self.separator}{twid}{self.separator}altflows', + self.prefix + self.separator + f'{profileid}{self.separator}{twid}{self.separator}altflows', uid, data, ) @@ -1439,9 +1442,9 @@ def set_dns_resolution( ip_info = json.dumps(ip_info) # we store ALL dns resolutions seen since starting slips # store with the IP as the key - self.r.hset('DNSresolution', answer, ip_info) + self.r.hset(self.prefix + self.separator + 'DNSresolution', answer, ip_info) # store with the domain as the key: - self.r.hset('ResolvedDomains', domains[0], answer) + self.r.hset(self.prefix + self.separator + 'ResolvedDomains', domains[0], answer) # these ips will be associated with the query in our db ips_to_add.append(answer) @@ -1465,7 +1468,7 @@ def set_domain_resolution(self, domain, ips): """ stores all the resolved domains with their ips in the db """ - self.r.hset("DomainsResolved", domain, json.dumps(ips)) + self.r.hset(self.prefix + self.separator + "DomainsResolved", domain, json.dumps(ips)) def getDomainData(self, domain): """ @@ -1570,7 +1573,7 @@ def setInfoForDomains(self, domain: str, info_to_set: dict, mode='leave'): domain_data = json.dumps(domain_data) self.rcache.hset('DomainsInfo', domain, domain_data) # Publish the changes - self.r.publish('dns_info_change', domain) + self.r.publish(self.prefix + self.separator + 'dns_info_change', domain) def add_out_dns( self, @@ -1608,7 +1611,7 @@ def add_out_dns( data = json.dumps(data) # Set the dns as alternative flow self.r.hset( - f'{profileid}{self.separator}{twid}{self.separator}altflows', + self.prefix + self.separator + f'{profileid}{self.separator}{twid}{self.separator}altflows', uid, data, ) diff --git a/slips_files/core/database/database.py b/slips_files/core/database/database.py index 09d98aa9da..0f1d9f9b1b 100644 --- a/slips_files/core/database/database.py +++ b/slips_files/core/database/database.py @@ -193,21 +193,21 @@ def set_slips_mode(self, slips_mode): function to store the current mode (daemonized/interactive) in the db """ - self.r.set("mode", slips_mode) + self.r.set(self.prefix + self.separator + "mode", slips_mode) def get_slips_mode(self): """ function to get the current mode (daemonized/interactive) in the db """ - self.r.get("mode") + self.r.get(self.prefix + self.separator + "mode") def get_modified_ips_in_the_last_tw(self): """ this number is updated in the db every 5s by slips.py used for printing running stats in slips.py or outputprocess """ - if modified_ips := self.r.hget('analysis', 'modified_ips_in_the_last_tw'): + if modified_ips := self.r.hget(self.prefix + self.separator + 'analysis', 'modified_ips_in_the_last_tw'): return modified_ips else: return 0 @@ -239,12 +239,18 @@ def change_redis_limits(self, redis_client): "slave 268435456 67108864 60 " "pubsub 4294967296 2147483648 600") - def start(self, redis_port): + def setPrefix(self,prefix): + super().set_prefix(prefix) + self.prefix = prefix + + def start(self, prefix, redis_port:str = '6379'): """Start the DB. Allow it to read the conf""" self.read_configuration() # Read values from the configuration file try: + super().set_prefix(prefix) + self.prefix = prefix if not hasattr(self, 'r'): self.connect_to_redis_server(redis_port) # Set the memory limits of the output buffer, For normal clients: no limits @@ -266,7 +272,7 @@ def start(self, redis_port): # to fix redis.exceptions.ResponseError MISCONF Redis is configured to save RDB snapshots # configure redis to stop writing to dump.rdb when an error occurs without throwing errors in slips # Even if the DB is not deleted. We need to delete some temp data - self.r.delete('zeekfiles') + self.r.delete(self.prefix + self.separator + 'zeekfiles') # By default the slips internal time is 0 until we receive something @@ -278,14 +284,14 @@ def start(self, redis_port): return False def is_connection_error_logged(self): - return bool(self.r.get('logged_connection_error')) + return bool(self.r.get(self.prefix + self.separator + 'logged_connection_error')) def mark_connection_error_as_logged(self): """ When redis connection error occurs, to prevent every module from logging it to slips.log and the console, set this variable in the db """ - self.r.set('logged_connection_error', 'True') + self.r.set(self.prefix + self.separator + 'logged_connection_error', 'True') def get_message(self, channel, timeout=0.0000001): """ @@ -305,11 +311,11 @@ def get_message(self, channel, timeout=0.0000001): def set_slips_start_time(self): """store the time slips started (datetime obj)""" now = utils.convert_format(datetime.now(), utils.alerts_format) - self.r.set('slips_start_time', now) + self.r.set(self.prefix + self.separator + 'slips_start_time', now) def get_slips_start_time(self): """get the time slips started (datetime obj)""" - if start_time := self.r.get('slips_start_time'): + if start_time := self.r.get(self.prefix + self.separator + 'slips_start_time'): start_time = utils.convert_format(start_time, utils.alerts_format) return start_time @@ -327,7 +333,7 @@ def should_add(self, profileid: str) -> bool: # no home_network is specified return True - ip = profileid.split(self.separator)[1] + ip = profileid.split(self.separator)[-1] ip_obj = ipaddress.ip_address(ip) return any(ip_obj in network for network in self.home_network) @@ -336,13 +342,13 @@ def set_loaded_ti_files(self, number_of_loaded_files: int): """ Stores the number of successfully loaded TI files """ - self.r.set('loaded TI files', number_of_loaded_files) + self.r.set(self.prefix + self.separator +'loaded TI files', number_of_loaded_files) def get_loaded_ti_files(self): """ returns the number of successfully loaded TI files. or 0 if none is loaded """ - return self.r.get('loaded TI files') or 0 + return self.r.get(self.prefix + self.separator +'loaded TI files') or 0 def addProfile(self, profileid, starttime, duration): """ @@ -352,24 +358,24 @@ def addProfile(self, profileid, starttime, duration): """ try: # make sure we don't add public ips if the user specified a home_network - if self.r.sismember('profiles', str(profileid)): + if self.r.sismember(self.prefix + self.separator + 'profiles', str(profileid)): # we already have this profile return False # execlude ips outside of local network is it's set in slips.conf if not self.should_add(profileid): return False - # Add the profile to the index. The index is called 'profiles' - self.r.sadd('profiles', str(profileid)) + # Add the profile to the index. The index is called prefix + separator + 'profiles' + self.r.sadd(self.prefix + self.separator + 'profiles', str(profileid)) # Create the hashmap with the profileid. The hasmap of each profile is named with the profileid # Add the start time of profile - self.r.hset(profileid, 'starttime', starttime) + self.r.hset(self.prefix + self.separator + str(profileid), 'starttime', starttime) # For now duration of the TW is fixed - self.r.hset(profileid, 'duration', duration) + self.r.hset(self.prefix + self.separator + str(profileid), 'duration', duration) # When a new profiled is created assign threat level = 0 and confidence = 0.05 - self.r.hset(profileid, 'threat_level', 0) - self.r.hset(profileid, 'confidence', 0.05) + self.r.hset(self.prefix + self.separator + str(profileid), 'threat_level', 0) + self.r.hset(self.prefix + self.separator + str(profileid), 'confidence', 0.05) # The IP of the profile should also be added as a new IP we know about. - ip = profileid.split(self.separator)[1] + ip = profileid.split(self.separator)[-1] # If the ip is new add it to the list of ips self.setNewIP(ip) # Publish that we have a new profile @@ -391,34 +397,34 @@ def was_ip_seen_in_connlog_before(self, ip) -> bool: # but not in conn.log yet # if the ip's not in the following key, then its the first flow seen of this ip - return self.r.sismember("srcips_seen_in_connlog", ip) + return self.r.sismember(self.prefix + self.separator +"srcips_seen_in_connlog", ip) def mark_srcip_as_seen_in_connlog(self, ip): """ Marks the given ip as seen in conn.log if an ip is not present in this set, it means we may have seen it but not in conn.log """ - self.r.sadd("srcips_seen_in_connlog", ip) + self.r.sadd(self.prefix + self.separator +"srcips_seen_in_connlog", ip) def add_user_agent_to_profile(self, profileid, user_agent: dict): """ Used to associate this profile with it's used user_agent :param user_agent: dict containing user_agent, os_type , os_name and agent_name """ - self.r.hset(profileid, 'User-agent', user_agent) + self.r.hset(self.prefix + self.separator + str(profileid), 'User-agent', user_agent) def add_all_user_agent_to_profile(self, profileid, user_agent: str): """ Used to keep history of past user agents of profile :param user_agent: str of user_agent """ - if not self.r.hexists(profileid ,'past_user_agents'): - self.r.hset(profileid, 'past_user_agents', json.dumps([user_agent])) + if not self.r.hexists(self.prefix + self.separator + str(profileid) ,'past_user_agents'): + self.r.hset(self.prefix + self.separator + str(profileid), 'past_user_agents', json.dumps([user_agent])) else: - user_agents = json.loads(self.r.hget(profileid, 'past_user_agents')) + user_agents = json.loads(self.r.hget(self.prefix + self.separator + str(profileid), 'past_user_agents')) if user_agent not in user_agents: user_agents.append(user_agent) - self.r.hset(profileid, 'past_user_agents', json.dumps(user_agents)) + self.r.hset(self.prefix + self.separator + str(profileid), 'past_user_agents', json.dumps(user_agents)) def add_software_to_profile( self, profileid, software, version_major, version_minor, uid @@ -442,10 +448,10 @@ def add_software_to_profile( return # add this new sw to the list of softwares this profile is using cached_sw.update(sw_dict) - self.r.hset(profileid, 'used_software', json.dumps(cached_sw)) + self.r.hset(self.prefix + self.separator + profileid, 'used_software', json.dumps(cached_sw)) else: # first time for this profile to use a software - self.r.hset(profileid, 'used_software', json.dumps(sw_dict)) + self.r.hset(self.prefix + self.separator + profileid, 'used_software', json.dumps(sw_dict)) def get_software_from_profile(self, profileid): """ @@ -454,7 +460,7 @@ def get_software_from_profile(self, profileid): if not profileid: return False - if used_software := self.r.hmget(profileid, 'used_software')[0]: + if used_software := self.r.hmget(self.prefix + self.separator + str(profileid), 'used_software')[0]: used_software = json.loads(used_software) return used_software @@ -468,7 +474,7 @@ def get_user_agent_from_profile(self, profileid) -> str: # profileid is None if we're dealing with a profile # outside of home_network when this param is given return False - if user_agent := self.r.hmget(profileid, 'User-agent')[0]: + if user_agent := self.r.hmget(self.prefix + self.separator + str(profileid), 'User-agent')[0]: # user agents may be OpenSSH_8.6 , no need to deserialize them if '{' in user_agent: user_agent = json.loads(user_agent) @@ -484,13 +490,13 @@ def mark_profile_as_dhcp(self, profileid): return False # returns a list of dhcp if the profile is in the db - profile_in_db = self.r.hmget(profileid, 'dhcp') + profile_in_db = self.r.hmget(self.prefix + self.separator + str(profileid), 'dhcp') if not profile_in_db: return False is_dhcp_set = profile_in_db[0] # check if it's already marked as dhcp if not is_dhcp_set: - self.r.hset(profileid, 'dhcp', 'true') + self.r.hset(self.prefix + self.separator + str(profileid), 'dhcp', 'true') def mark_profile_as_gateway(self, profileid): @@ -502,14 +508,14 @@ def mark_profile_as_gateway(self, profileid): # outside of home_network when this param is given return False - self.r.hset(profileid, 'gateway', 'true') + self.r.hset(self.prefix + self.separator + str(profileid), 'gateway', 'true') def set_ipv6_of_profile(self, profileid, ip: list): - self.r.hset(profileid, 'IPv6', json.dumps(ip)) + self.r.hset(self.prefix + self.separator + str(profileid), 'IPv6', json.dumps(ip)) def set_ipv4_of_profile(self, profileid, ip): - self.r.hset(profileid, 'IPv4', json.dumps([ip])) + self.r.hset(self.prefix + self.separator + str(profileid), 'IPv4', json.dumps([ip])) def is_gw_mac(self, MAC_info, ip) -> bool: """ @@ -545,7 +551,7 @@ def get_IP_of_MAC(self, MAC): """ Returns the IP associated with the given MAC in our database """ - return self.r.hget('MAC', MAC) + return self.r.hget(self.prefix + self.separator + 'MAC', MAC) def add_mac_addr_to_profile(self, profileid, MAC_info): """ @@ -563,7 +569,7 @@ def add_mac_addr_to_profile(self, profileid, MAC_info): if '0.0.0.0' in profileid: return False - incoming_ip = profileid.split('_')[1] + incoming_ip = profileid.split(self.separator)[-1] # sometimes we create profiles with the mac address. # don't save that in MAC hash @@ -577,13 +583,13 @@ def add_mac_addr_to_profile(self, profileid, MAC_info): # we're trying to assign the gw mac to an ip that isn't the gateway's return False # get the ips that belong to this mac - cached_ip = self.r.hmget('MAC', MAC_info['MAC'])[0] + cached_ip = self.r.hmget(self.prefix + self.separator + 'MAC', MAC_info['MAC'])[0] if not cached_ip: # no mac info stored for profileid ip = json.dumps([incoming_ip]) - self.r.hset('MAC', MAC_info['MAC'], ip) + self.r.hset(self.prefix + self.separator + 'MAC', MAC_info['MAC'], ip) # Add the MAC addr, hostname and vendor to this profile - self.r.hset(profileid, 'MAC', json.dumps(MAC_info)) + self.r.hset(self.prefix + self.separator + str(profileid), 'MAC', json.dumps(MAC_info)) else: # we found another profile that has the same mac as this one # incoming_ip = profileid.split('_')[1] @@ -611,7 +617,7 @@ def add_mac_addr_to_profile(self, profileid, MAC_info): # If 2 IPV6 are claiming to have the same MAC it's fine # a computer is allowed to have many ipv6 # add this found ipv6 to the list of ipv6 of the incoming ip(profileid) - ipv6: str = self.r.hmget(profileid, 'IPv6')[0] + ipv6: str = self.r.hmget(self.prefix + self.separator + str(profileid), 'IPv6')[0] if not ipv6: ipv6 = [found_ip] else: @@ -622,7 +628,7 @@ def add_mac_addr_to_profile(self, profileid, MAC_info): self.set_ipv6_of_profile(profileid, ipv6) # add this incoming ipv6(profileid) to the list of ipv6 of the found ip - ipv6: str = self.r.hmget(f'profile_{found_ip}', 'IPv6')[0] + ipv6: str = self.r.hmget(f'{self.prefix}_profile_{found_ip}', 'IPv6')[0] if not ipv6: ipv6 = [incoming_ip] else: @@ -642,7 +648,7 @@ def add_mac_addr_to_profile(self, profileid, MAC_info): # add the incoming ip to the list of ips that belong to this mac cached_ips.add(incoming_ip) cached_ips = json.dumps(list(cached_ips)) - self.r.hset('MAC', MAC_info['MAC'], cached_ips) + self.r.hset(self.prefix + self.separator + 'MAC', MAC_info['MAC'], cached_ips) return True @@ -654,7 +660,7 @@ def get_mac_addr_from_profile(self, profileid) -> str: # profileid is None if we're dealing with a profile # outside of home_network when this param is given return False - if MAC_info := self.r.hget(profileid, 'MAC'): + if MAC_info := self.r.hget(self.prefix + self.separator + str(profileid), 'MAC'): return json.loads(MAC_info)['MAC'] else: return MAC_info @@ -667,7 +673,7 @@ def get_mac_vendor_from_profile(self, profileid) -> str: # profileid is None if we're dealing with a profile # outside of home_network when this param is given return False - if MAC_info := self.r.hget(profileid, 'MAC'): + if MAC_info := self.r.hget(self.prefix + self.separator + str(profileid), 'MAC'): return json.loads(MAC_info)['Vendor'] else: return MAC_info @@ -680,7 +686,7 @@ def get_hostname_from_profile(self, profileid) -> str: # profileid is None if we're dealing with a profile # outside of home_network when this param is given return False - if MAC_info := self.r.hget(profileid, 'MAC'): + if MAC_info := self.r.hget(self.prefix + self.separator + str(profileid), 'MAC'): return json.loads(MAC_info).get('host_name', False) else: return MAC_info @@ -689,13 +695,13 @@ def get_ipv4_from_profile(self, profileid) -> str: """ Returns ipv4 about a certain profile or None """ - return self.r.hmget(profileid, 'IPv4')[0] if profileid else False + return self.r.hmget(self.prefix + self.separator + str(profileid), 'IPv4')[0] if profileid else False def get_ipv6_from_profile(self, profileid) -> str: """ Returns ipv6 about a certain profile or None """ - return self.r.hmget(profileid, 'IPv6')[0] if profileid else False + return self.r.hmget(self.prefix + self.separator + str(profileid), 'IPv6')[0] if profileid else False def get_the_other_ip_version(self, profileid): """ @@ -706,7 +712,7 @@ def get_the_other_ip_version(self, profileid): # profileid is None if we're dealing with a profile # outside of home_network when this param is given return False - srcip = profileid.split('_')[1] + srcip = profileid.split(self.separator)[-1] ip = False if validators.ipv4(srcip): ip = self.get_ipv6_from_profile(profileid) @@ -718,8 +724,8 @@ def get_the_other_ip_version(self, profileid): def getProfileIdFromIP(self, daddr_as_obj): """Receive an IP and we want the profileid""" try: - profileid = f'profile{self.separator}{str(daddr_as_obj)}' - if data := self.r.sismember('profiles', profileid): + profileid = f'{self.prefix}_profile{self.separator}{str(daddr_as_obj)}' + if data := self.r.sismember(self.prefix + self.separator + 'profiles', profileid): return profileid return False except redis.exceptions.ResponseError as inst: @@ -731,7 +737,7 @@ def getProfileIdFromIP(self, daddr_as_obj): def getProfiles(self): """Get a list of all the profiles""" - profiles = self.r.smembers('profiles') + profiles = self.r.smembers(self.prefix + self.separator + 'profiles') return profiles if profiles != set() else {} @@ -741,7 +747,7 @@ def getTWsfromProfile(self, profileid): Returns a list of tuples (twid, ts) or an empty list """ return ( - self.r.zrange(f'tws{profileid}', 0, -1, withscores=True) + self.r.zrange(self.prefix + self.separator + f'tws{profileid}', 0, -1, withscores=True) if profileid else False ) @@ -756,13 +762,13 @@ def getSrcIPsfromProfileTW(self, profileid, twid): """ Get the src ip for a specific TW for a specific profileid """ - return self.r.hget(profileid + self.separator + twid, 'SrcIPs') + return self.r.hget(self.prefix + self.separator + profileid + self.separator + twid, 'SrcIPs') def getDstIPsfromProfileTW(self, profileid, twid): """ Get the dst ip for a specific TW for a specific profileid """ - return self.r.hget(profileid + self.separator + twid, 'DstIPs') + return self.r.hget(self.prefix + self.separator + profileid + self.separator + twid, 'DstIPs') def getT2ForProfileTW(self, profileid, twid, tupleid, tuple_key: str): """ @@ -770,7 +776,7 @@ def getT2ForProfileTW(self, profileid, twid, tupleid, tuple_key: str): """ try: hash_id = profileid + self.separator + twid - data = self.r.hget(hash_id, tuple_key) + data = self.r.hget(self.prefix + self.separator + str(hash_id), tuple_key) if not data: return False, False data = json.loads(data) @@ -793,16 +799,16 @@ def getT2ForProfileTW(self, profileid, twid, tupleid, tuple_key: str): def has_profile(self, profileid): """Check if we have the given profile""" - return self.r.sismember('profiles', profileid) if profileid else False + return self.r.sismember(self.prefix + self.separator + 'profiles', profileid) if profileid else False def getProfilesLen(self): """Return the amount of profiles. Redis should be faster than python to do this count""" - return self.r.scard('profiles') + return self.r.scard(self.prefix + self.separator + 'profiles') def getLastTWforProfile(self, profileid): """Return the last TW id and the time for the given profile id""" return ( - self.r.zrange(f'tws{profileid}', -1, -1, withscores=True) + self.r.zrange(self.prefix + self.separator + f'tws{profileid}', -1, -1, withscores=True) if profileid else False ) @@ -810,7 +816,7 @@ def getLastTWforProfile(self, profileid): def getFirstTWforProfile(self, profileid): """Return the first TW id and the time for the given profile id""" return ( - self.r.zrange(f'tws{profileid}', 0, 0, withscores=True) + self.r.zrange(self.prefix + self.separator + f'tws{profileid}', 0, 0, withscores=True) if profileid else False ) @@ -825,7 +831,7 @@ def getTWofTime(self, profileid, time): # [-1] so we bring the last TW that matched this time. try: data = self.r.zrangebyscore( - f'tws{profileid}', + self.prefix + self.separator + f'tws{profileid}', float('-inf'), float(time), withscores=True, @@ -836,7 +842,7 @@ def getTWofTime(self, profileid, time): except IndexError: # We dont have any last tw? data = self.r.zrangebyscore( - f'tws{profileid}', + self.prefix + self.separator + f'tws{profileid}', 0, float(time), withscores=True, @@ -867,7 +873,7 @@ def addNewOlderTW(self, profileid, startoftw): pass # Add the new TW to the index of TW data = {str(twid): float(startoftw)} - self.r.zadd(f'tws{profileid}', data) + self.r.zadd(self.prefix + self.separator + f'tws{profileid}', data) self.outputqueue.put( f'04|database|[DB]: Created and added to DB the new older TW with id {twid}. Time: {startoftw} ' ) @@ -904,7 +910,7 @@ def addNewTW(self, profileid, startoftw): twid = 'timewindow1' # Add the new TW to the index of TW data = {twid: float(startoftw)} - self.r.zadd(f'tws{profileid}', data) + self.r.zadd(self.prefix + self.separator + f'tws{profileid}', data) self.outputqueue.put( f'04|database|[DB]: Created and added to DB for profile {profileid} on TW with id {twid}. Time: {startoftw} ' ) @@ -923,16 +929,16 @@ def getTimeTW(self, profileid, twid): """Return the time when this TW in this profile was created""" # Get all the TW for this profile # We need to encode it to 'search' because the data in the sorted set is encoded - return self.r.zscore(f'tws{profileid}', twid.encode('utf-8')) + return self.r.zscore(self.prefix + self.separator + f'tws{profileid}', twid.encode('utf-8')) def getAmountTW(self, profileid): """Return the number of tws for this profile id""" - return self.r.zcard(f'tws{profileid}') if profileid else False + return self.r.zcard(self.prefix + self.separator + f'tws{profileid}') if profileid else False def getModifiedTWSinceTime(self, time): """Return the list of modified timewindows since a certain time""" data = self.r.zrangebyscore( - 'ModifiedTW', time, float('+inf'), withscores=True + self.prefix + self.separator + 'ModifiedTW', time, float('+inf'), withscores=True ) return data or [] @@ -954,16 +960,16 @@ def getModifiedProfilesSince(self, time): def getModifiedTW(self): """Return all the list of modified tw""" - data = self.r.zrange('ModifiedTW', 0, -1, withscores=True) + data = self.r.zrange(self.prefix + self.separator + 'ModifiedTW', 0, -1, withscores=True) return data or [] def wasProfileTWModified(self, profileid, twid): """Retrieve from the db if this TW of this profile was modified""" - data = self.r.zrank('ModifiedTW', profileid + self.separator + twid) + data = self.r.zrank(self.prefix + self.separator + 'ModifiedTW', profileid + self.separator + twid) return bool(data) def setSlipsInternalTime(self, timestamp): - self.r.set('slips_internal_time', timestamp) + self.r.set(self.prefix + self.separator + 'slips_internal_time', timestamp) def get_data_from_profile_tw(self, hash_key: str, key_name: str): try: @@ -971,7 +977,7 @@ def get_data_from_profile_tw(self, hash_key: str, key_name: str): key_name = [Src,Dst] + [Port,IP] + [Client,Server] + [TCP,UDP, ICMP, ICMP6] + [Established, NotEstablihed] Example: key_name = 'SrcPortClientTCPEstablished' """ - data = self.r.hget(hash_key, key_name) + data = self.r.hget(self.prefix + self.separator + str(hash_key), key_name) value = {} if data: portdata = json.loads(data) @@ -986,11 +992,11 @@ def get_data_from_profile_tw(self, hash_key: str, key_name: str): def getOutTuplesfromProfileTW(self, profileid, twid): """Get the out tuples""" - return self.r.hget(profileid + self.separator + twid, 'OutTuples') + return self.r.hget(self.prefix + self.separator + profileid + self.separator + twid, 'OutTuples') def getInTuplesfromProfileTW(self, profileid, twid): """Get the in tuples""" - return self.r.hget(profileid + self.separator + twid, 'InTuples') + return self.r.hget(self.prefix + self.separator + profileid + self.separator + twid, 'InTuples') def getFieldSeparator(self): """Return the field separator""" @@ -1001,7 +1007,7 @@ def get_dhcp_flows(self, profileid, twid) -> dict: """ returns a dict of dhcp flows that happaened in this profileid and twid """ - if flows := self.r.hget('DHCP_flows', f'{profileid}_{twid}'): + if flows := self.r.hget(self.prefix + self.separator + 'DHCP_flows', f'{profileid}_{twid}'): return json.loads(flows) @@ -1013,9 +1019,9 @@ def set_dhcp_flow(self, profileid, twid, requested_addr, uid): if cached_flows := self.get_dhcp_flows(profileid, twid): # we already have flows in this twid, update them cached_flows.update(flow) - self.r.hset('DHCP_flows', f'{profileid}_{twid}', json.dumps(cached_flows)) + self.r.hset(self.prefix + self.separator + 'DHCP_flows', f'{profileid}_{twid}', json.dumps(cached_flows)) else: - self.r.hset('DHCP_flows', f'{profileid}_{twid}', json.dumps(flow)) + self.r.hset(self.prefix + self.separator + 'DHCP_flows', f'{profileid}_{twid}', json.dumps(flow)) def update_threat_level(self, profileid, threat_level: str, confidence): @@ -1024,12 +1030,12 @@ def update_threat_level(self, profileid, threat_level: str, confidence): :param threat_level: available options are 'low', 'medium' 'critical' etc """ - self.r.hset(profileid, 'threat_level', threat_level) + self.r.hset(self.prefix + self.separator + profileid, 'threat_level', threat_level) now = time.time() now = utils.convert_format(now, utils.alerts_format) # keep track of old threat levels confidence = f'confidence: {confidence}' - past_threat_levels = self.r.hget(profileid, 'past_threat_levels') + past_threat_levels = self.r.hget(self.prefix + self.separator + profileid, 'past_threat_levels') # this is what we'll be storing in the db, tl, ts, and confidence threat_level_data = (threat_level, now, confidence) if past_threat_levels: @@ -1052,7 +1058,7 @@ def update_threat_level(self, profileid, threat_level: str, confidence): # threat_levels_update_time = [now] past_threat_levels = json.dumps(past_threat_levels) - self.r.hset(profileid, 'past_threat_levels', past_threat_levels) + self.r.hset(self.prefix + self.separator + profileid, 'past_threat_levels', past_threat_levels) # set the score and confidence of the given ip in the db when it causes an evidence # these 2 values will be needed when sharing with peers @@ -1094,7 +1100,7 @@ def set_evidence_causing_alert(self, profileid, twid, alert_ID, evidence_IDs: li profileid_twid_alerts = json.dumps(alert) - self.r.hset(f'{profileid}{self.separator}{twid}', 'alerts', profileid_twid_alerts) + self.r.hset(self.prefix + self.separator + f'{profileid}{self.separator}{twid}', 'alerts', profileid_twid_alerts) # the structure of alerts key is # alerts { @@ -1106,7 +1112,7 @@ def set_evidence_causing_alert(self, profileid, twid, alert_ID, evidence_IDs: li # } # } - profile_alerts = self.r.hget('alerts', profileid) + profile_alerts = self.r.hget(self.prefix + self.separator + 'alerts', profileid) # alert ids look like this profile_192.168.131.2_timewindow1_92a3b9c2-330b-47ab-b73e-c5380af90439 alert_hash = alert_ID.split('_')[-1] alert = { @@ -1117,7 +1123,7 @@ def set_evidence_causing_alert(self, profileid, twid, alert_ID, evidence_IDs: li if not profile_alerts: # first alert in this profile alert = json.dumps(alert) - self.r.hset('alerts', profileid, alert) + self.r.hset(self.prefix + self.separator + 'alerts', profileid, alert) return # the format of this dict is {twid1: {alert_hash: [evidence_IDs]}, @@ -1135,7 +1141,7 @@ def set_evidence_causing_alert(self, profileid, twid, alert_ID, evidence_IDs: li profile_alerts[twid] = twid_alerts profile_alerts = json.dumps(profile_alerts) - self.r.hset('alerts', profileid, profile_alerts) + self.r.hset(self.prefix + self.separator + 'alerts', profileid, profile_alerts) def get_timewindow(self, flowtime, profileid): @@ -1261,7 +1267,7 @@ def get_profileid_twid_alerts(self, profileid, twid) -> dict: The format for the returned dict is {profile123_twid1_: [ev_uuid1, ev_uuid2, ev_uuid3]} """ - alerts = self.r.hget(f'{profileid}{self.separator}{twid}', 'alerts') + alerts = self.r.hget(self.prefix + self.separator + f'{profileid}{self.separator}{twid}', 'alerts') if not alerts: return {} alerts = json.loads(alerts) @@ -1273,7 +1279,7 @@ def get_evidence_causing_alert(self, profileid, twid, alert_ID) -> list: :param alert_ID: ID of alert to export to warden server for example profile_10.0.2.15_timewindow1_4e4e4774-cdd7-4e10-93a3-e764f73af621 """ - if alerts := self.r.hget(f'{profileid}{self.separator}{twid}', 'alerts'): + if alerts := self.r.hget(self.prefix + self.separator + f'{profileid}{self.separator}{twid}', 'alerts'): alerts = json.loads(alerts) return alerts.get(alert_ID, False) return False @@ -1308,10 +1314,10 @@ def is_detection_disabled(self, evidence_type: str): def set_flow_causing_evidence(self, uids: list, evidence_ID): - self.r.hset("flows_causing_evidence", evidence_ID, json.dumps(uids)) + self.r.hset(self.prefix + self.separator + "flows_causing_evidence", evidence_ID, json.dumps(uids)) def get_flows_causing_evidence(self, evidence_ID) -> list: - uids = self.r.hget("flows_causing_evidence", evidence_ID) + uids = self.r.hget(self.prefix + self.separator + "flows_causing_evidence", evidence_ID) return json.loads(uids) if uids else [] @@ -1426,10 +1432,10 @@ def setEvidence( # Set evidence in the database. current_evidence = json.dumps(current_evidence) self.r.hset( - profileid + self.separator + twid, 'Evidence', current_evidence + self.prefix + self.separator + profileid + self.separator + twid, 'Evidence', current_evidence ) - self.r.hset(f'evidence{profileid}', twid, current_evidence) + self.r.hset(self.prefix + self.separator + f'evidence{profileid}', twid, current_evidence) # This is done to ignore repetition of the same evidence sent. # note that publishing HAS TO be done after updating the 'Evidence' keys @@ -1452,10 +1458,10 @@ def mark_evidence_as_processed(self, evidence_ID): """ If an evidence was processed by the evidenceprocess, mark it in the db """ - self.r.sadd('processed_evidence', evidence_ID) + self.r.sadd(self.prefix + self.separator + 'processed_evidence', evidence_ID) def is_evidence_processed(self, evidence_ID): - return self.r.sismember('processed_evidence', evidence_ID) + return self.r.sismember(self.prefix + self.separator + 'processed_evidence', evidence_ID) def store_tranco_whitelisted_domain(self, domain): """ @@ -1475,7 +1481,7 @@ def set_evidence_for_profileid(self, evidence): Set evidence for the profile in the same format as json in alerts.json """ evidence = json.dumps(evidence) - self.r.sadd('Evidence', evidence) + self.r.sadd(self.prefix + self.separator + 'Evidence', evidence) def deleteEvidence(self, profileid, twid, evidence_ID: str): @@ -1489,13 +1495,13 @@ def deleteEvidence(self, profileid, twid, evidence_ID: str): current_evidence.pop(evidence_ID, None) current_evidence_json = json.dumps(current_evidence) self.r.hset( - profileid + self.separator + twid, + self.prefix + self.separator + profileid + self.separator + twid, 'Evidence', current_evidence_json, ) - self.r.hset(f'evidence{profileid}', twid, current_evidence_json) + self.r.hset(self.prefix + self.separator + f'evidence{profileid}', twid, current_evidence_json) # 2. delete evidence from 'alerts' key - profile_alerts = self.r.hget('alerts', profileid) + profile_alerts = self.r.hget(self.prefix + self.separator + 'alerts', profileid) if not profile_alerts: # this means that this evidence wasn't a part of an alert # give redis time to the save the changes before calling this function again @@ -1538,13 +1544,13 @@ def cache_whitelisted_evidence_ID(self, evidence_ID:str): """ # without this function, slips gets the stored evidence id from the db, # before deleteEvidence is called, so we need to keep track of whitelisted evidence ids - self.r.sadd('whitelisted_evidence', evidence_ID) + self.r.sadd(self.prefix + self.separator + 'whitelisted_evidence', evidence_ID) def is_whitelisted_evidence(self, evidence_ID): """ Check if we have the evidence ID as whitelisted in the db to avoid showing it in alerts """ - return self.r.sismember('whitelisted_evidence', evidence_ID) + return self.r.sismember(self.prefix + self.separator + 'whitelisted_evidence', evidence_ID) def remove_whitelisted_evidence(self, all_evidence:str) -> str: """ @@ -1562,7 +1568,7 @@ def remove_whitelisted_evidence(self, all_evidence:str) -> str: def getEvidenceForTW(self, profileid, twid): """Get the evidence for this TW for this Profile""" - evidence = self.r.hget(profileid + self.separator + twid, 'Evidence') + evidence = self.r.hget(self.prefix + self.separator + profileid + self.separator + twid, 'Evidence') if evidence: evidence = self.remove_whitelisted_evidence(evidence) return evidence @@ -1585,7 +1591,7 @@ def set_first_stage_ensembling_label_to_flow( data['1_ensembling_label'] = ensembling_label data = json.dumps(data) self.r.hset( - profileid + self.separator + twid + self.separator + 'flows', + self.prefix + self.separator + profileid + self.separator + twid + self.separator + 'flows', uid, data, ) @@ -1594,11 +1600,11 @@ def set_growing_zeek_dir(self): """ Mark a dir as growing so it can be treated like the zeek logs generated by an interface """ - self.r.set('growing_zeek_dir', 'yes') + self.r.set(self.prefix + self.separator + 'growing_zeek_dir', 'yes') def is_growing_zeek_dir(self): """ Did slips mark the given dir as growing?""" - return 'yes' in str(self.r.get('growing_zeek_dir')) + return 'yes' in str(self.r.get(self.prefix + self.separator + 'growing_zeek_dir')) def set_module_label_to_flow( self, profileid, twid, uid, module_name, module_label @@ -1613,7 +1619,7 @@ def set_module_label_to_flow( data['module_labels'][module_name] = module_label data = json.dumps(data) self.r.hset( - profileid + self.separator + twid + self.separator + 'flows', + self.prefix + self.separator + profileid + self.separator + twid + self.separator + 'flows', uid, data, ) @@ -1635,15 +1641,15 @@ def markProfileTWAsBlocked(self, profileid, twid): """Add this profile and tw to the list of blocked""" tws = self.getBlockedProfTW(profileid) tws.append(twid) - self.r.hset('BlockedProfTW', profileid, json.dumps(tws)) + self.r.hset(self.prefix + self.separator + 'BlockedProfTW', profileid, json.dumps(tws)) def getAllBlockedProfTW(self): """Return all the list of blocked tws""" - return self.r.hgetall('BlockedProfTW') + return self.r.hgetall(self.prefix + self.separator + 'BlockedProfTW') def getBlockedProfTW(self, profileid): """Return all the list of blocked tws""" - if tws := self.r.hget('BlockedProfTW', profileid): + if tws := self.r.hget(self.prefix + self.separator + 'BlockedProfTW', profileid): return json.loads(tws) return [] @@ -1695,7 +1701,7 @@ def get_multiaddr(self): """ this is can only be called when p2p is enabled, this value is set by p2p pigeon """ - return self.r.get('multiAddress') + return self.r.get(self.prefix + self.separator + 'multiAddress') def getURLData(self, url): """ @@ -1772,7 +1778,7 @@ def subscribe(self, channel: str, ignore_subscribe_messages=False): self.pubsub = self.r.pubsub() self.pubsub.subscribe( - channel, ignore_subscribe_messages=ignore_subscribe_messages + self.prefix + self.separator + str(channel), ignore_subscribe_messages=ignore_subscribe_messages ) return self.pubsub @@ -1784,14 +1790,14 @@ def publish_stop(self): all_channels_list = self.r.pubsub_channels() self.print('Sending the stop signal to all listeners', 0, 3) for channel in all_channels_list: - self.r.publish(channel, 'stop_process') + self.r.publish(self.prefix + self.separator + str(channel), 'stop_process') def get_all_flows_in_profileid_twid(self, profileid, twid): """ Return a list of all the flows in this profileid and twid """ if data := self.r.hgetall( - profileid + self.separator + twid + self.separator + 'flows' + self.prefix + self.separator + profileid + self.separator + twid + self.separator + 'flows' ): return data @@ -1852,13 +1858,13 @@ def get_labels(self): Return the amount of each label so far in the DB Used to know how many labels are available during training """ - return self.r.zrange('labels', 0, -1, withscores=True) + return self.r.zrange(self.prefix + self.separator + 'labels', 0, -1, withscores=True) def get_altflow_from_uid(self, profileid, twid, uid): """ Given a uid, get the alternative flow realted to it """ return ( self.r.hget( - profileid + self.separator + twid + self.separator + 'altflows', + self.prefix + self.separator + profileid + self.separator + twid + self.separator + 'altflows', uid, ) if profileid @@ -1871,9 +1877,9 @@ def add_timeline_line(self, profileid, twid, data, timestamp): # profileid is None if we're dealing with a profile # outside of home_network when this param is given return - self.print(f'Adding timeline for {profileid}, {twid}: {data}', 3, 0) + self.print(f'Adding timeline for {self.prefix}, {profileid}, {twid}: {data}', 3, 0) key = str( - profileid + self.separator + twid + self.separator + 'timeline' + self.prefix + self.separator + profileid + self.separator + twid + self.separator + 'timeline' ) data = json.dumps(data) mapping = {data: timestamp} @@ -1890,7 +1896,7 @@ def get_timeline_last_lines( # outside of home_network when this param is given return [], [] key = str( - profileid + self.separator + twid + self.separator + 'timeline' + self.prefix + self.separator + profileid + self.separator + twid + self.separator + 'timeline' ) # The the amount of lines in this list last_index = self.r.zcard(key) @@ -1917,11 +1923,11 @@ def set_ftp_port(self, port): """ Stores the used ftp port in our main db (not the cache like set_port_info) """ - self.r.lpush('used_ftp_ports', str(port)) + self.r.lpush(self.prefix + self.separator + 'used_ftp_ports', str(port)) def is_ftp_port(self, port): # get all used ftp ports - used_ftp_ports = self.r.lrange('used_ftp_ports', 0, -1) + used_ftp_ports = self.r.lrange(self.prefix + self.separator + 'used_ftp_ports', 0, -1) # check if the given port is used as ftp port return str(port) in used_ftp_ports @@ -1946,20 +1952,20 @@ def get_organization_of_port(self, portproto: str): def add_zeek_file(self, filename): """Add an entry to the list of zeek files""" - self.r.sadd('zeekfiles', filename) + self.r.sadd(self.prefix + self.separator + 'zeekfiles', filename) def get_all_zeek_file(self): """Return all entries from the list of zeek files""" - return self.r.smembers('zeekfiles') + return self.r.smembers(self.prefix + self.separator + 'zeekfiles') def get_gateway_ip(self): - return self.r.hget('default_gateway', 'IP') + return self.r.hget(self.prefix + self.separator + 'default_gateway', 'IP') def get_gateway_MAC(self): - return self.r.hget('default_gateway', 'MAC') + return self.r.hget(self.prefix + self.separator + 'default_gateway', 'MAC') def get_gateway_MAC_Vendor(self): - return self.r.hget('default_gateway', 'Vendor') + return self.r.hget(self.prefix + self.separator + 'default_gateway', 'Vendor') def set_default_gateway(self, address_type:str, address:str): """ @@ -1972,7 +1978,7 @@ def set_default_gateway(self, address_type:str, address:str): or address_type == 'MAC' and not self.get_gateway_MAC() or address_type == 'Vendor' and not self.get_gateway_MAC_Vendor() ): - self.r.hset('default_gateway', address_type, address) + self.r.hset(self.prefix + self.separator + 'default_gateway', address_type, address) def get_ssl_info(self, sha1): info = self.rcache.hmget('IoC_SSL', sha1)[0] @@ -1991,7 +1997,7 @@ def set_profile_module_label(self, profileid, module, label): data = self.get_profile_modules_labels(profileid) data[module] = label data = json.dumps(data) - self.r.hset(profileid, 'modules_labels', data) + self.r.hset(self.prefix + self.separator + str(profileid), 'modules_labels', data) def get_profile_modules_labels(self, profileid): """ @@ -2001,7 +2007,7 @@ def get_profile_modules_labels(self, profileid): # profileid is None if we're dealing with a profile # outside of home_network when this param is given return {} - data = self.r.hget(profileid, 'modules_labels') + data = self.r.hget(self.prefix + self.separator + str(profileid), 'modules_labels') data = json.loads(data) if data else {} return data @@ -2113,7 +2119,7 @@ def set_malicious_ip(self, ip, profileid, twid): ) # add key-pair to the dict if does not exist data = json.dumps(ip_profileid_twid) - self.r.hset('MaliciousIPs', ip, data) + self.r.hset(self.prefix + self.separator + 'MaliciousIPs', ip, data) def set_malicious_domain(self, domain, profileid, twid): """ @@ -2141,14 +2147,14 @@ def set_malicious_domain(self, domain, profileid, twid): ) # add key-pair to the dict if does not exist data = json.dumps(domain_profiled_twid) - self.r.hset('MaliciousDomains', domain, data) + self.r.hset(self.prefix + self.separator + 'MaliciousDomains', domain, data) def get_malicious_ip(self, ip): """ Return malicious IP and its list of presence in the traffic (profileid, twid) """ - data = self.r.hget('MaliciousIPs', ip) + data = self.r.hget(self.prefix + self.separator + 'MaliciousIPs', ip) data = json.loads(data) if data else {} return data @@ -2157,7 +2163,7 @@ def get_malicious_domain(self, domain): Return malicious domain and its list of presence in the traffic (profileid, twid) """ - data = self.r.hget('MaliciousDomains', domain) + data = self.r.hget(self.prefix + self.separator + 'MaliciousDomains', domain) data = json.loads(data) if data else {} return data @@ -2165,11 +2171,11 @@ def get_domain_resolution(self, domain): """ Returns the IPs resolved by this domain """ - ips = self.r.hget("DomainsResolved", domain) + ips = self.r.hget(self.prefix + self.separator + "DomainsResolved", domain) return json.loads(ips) if ips else [] def get_all_dns_resolutions(self): - dns_resolutions = self.r.hgetall('DNSresolution') + dns_resolutions = self.r.hgetall(self.prefix + self.separator +'DNSresolution') return dns_resolutions or [] @@ -2222,7 +2228,7 @@ def getReconnectionsForTW(self, profileid, twid): # profileid is None if we're dealing with a profile # outside of home_network when this param is given return False - data = self.r.hget(profileid + self.separator + twid, 'Reconnections') + data = self.r.hget(self.prefix + self.separator + profileid + self.separator + twid, 'Reconnections') data = json.loads(data) if data else {} return data @@ -2230,7 +2236,7 @@ def setReconnections(self, profileid, twid, data): """Set the reconnections for this TW for this Profile""" data = json.dumps(data) self.r.hset( - profileid + self.separator + twid, 'Reconnections', str(data) + self.prefix + self.separator + profileid + self.separator + twid, 'Reconnections', str(data) ) @@ -2257,15 +2263,15 @@ def is_domain_malicious(self, domain: str) -> tuple: def get_host_ip(self): """Get the IP addresses of the host from a db. There can be more than one""" - return self.r.smembers('hostIP') + return self.r.smembers(self.prefix + self.separator + 'hostIP') def set_host_ip(self, ip): """Store the IP address of the host in a db. There can be more than one""" - self.r.sadd('hostIP', ip) + self.r.sadd(self.prefix + self.separator + 'hostIP', ip) def is_profile_malicious(self, profileid: str) -> str: - return self.r.hget(profileid, 'labeled_as_malicious') if profileid else False + return self.r.hget(self.prefix + self.separator + str(profileid), 'labeled_as_malicious') if profileid else False def set_TI_file_info(self, file, data): """ @@ -2364,11 +2370,11 @@ def store_process_PID(self, process, pid): :param pid: int :param process: str """ - self.r.hset('PIDs', process, pid) + self.r.hset(self.prefix + self.separator + 'PIDs', process, pid) def get_PIDs(self): """returns a dict with module names as keys and pids as values""" - return self.r.hgetall('PIDs') + return self.r.hgetall(self.prefix + self.separator + 'PIDs') def set_org_info(self, org, org_info, info_type): """ @@ -2408,18 +2414,18 @@ def set_whitelist(self, type, whitelist_dict): :param type: supporte types are IPs, domains and organizations :param whitelist_dict: the dict of IPs, domains or orgs to store """ - self.r.hset('whitelist', type, json.dumps(whitelist_dict)) + self.r.hset(self.prefix + self.separator + 'whitelist', type, json.dumps(whitelist_dict)) def get_all_whitelist(self): """Return dict of 3 keys: IPs, domains, organizations or mac""" - return self.r.hgetall('whitelist') + return self.r.hgetall(self.prefix + self.separator + 'whitelist') def get_whitelist(self, key): """ Whitelist supports different keys like : IPs domains and organizations this function is used to check if we have any of the above keys whitelisted """ - if whitelist := self.r.hget('whitelist', key): + if whitelist := self.r.hget(self.prefix + self.separator + 'whitelist', key): return json.loads(whitelist) else: return {} @@ -2435,9 +2441,9 @@ def store_dhcp_server(self, server_addr): # not a valid ip skip return False # make sure the server isn't there before adding - DHCP_servers = self.r.lrange('DHCP_servers', 0, -1) + DHCP_servers = self.r.lrange(self.prefix + self.separator + 'DHCP_servers', 0, -1) if server_addr not in DHCP_servers: - self.r.lpush('DHCP_servers', server_addr) + self.r.lpush(self.prefix + self.separator + 'DHCP_servers', server_addr) def save(self, backup_file): """ @@ -2541,13 +2547,13 @@ def set_last_warden_poll_time(self, time): """ :param time: epoch """ - self.r.hset('Warden', 'poll', time) + self.r.hset(self.prefix + self.separator + 'Warden', 'poll', time) def get_last_warden_poll_time(self): """ returns epoch time of last poll """ - time = self.r.hget('Warden', 'poll') + time = self.r.hget(self.prefix + self.separator + 'Warden', 'poll') time = float(time) if time else float('-inf') return time @@ -2578,11 +2584,11 @@ def store_blame_report(self, ip, network_evaluation): def store_zeek_path(self, path): """used to store the path of zeek log files slips is currently using""" - self.r.set('zeek_path', path) + self.r.set(self.prefix + self.separator + 'zeek_path', path) def get_zeek_path(self) -> str: """return the path of zeek log files slips is currently using""" - return self.r.get('zeek_path') + return self.r.get(self.prefix + self.separator + 'zeek_path') def store_std_file(self, **kwargs): """ @@ -2596,9 +2602,9 @@ def store_std_file(self, **kwargs): } """ for file_type, path in kwargs.items(): - self.r.set(file_type, path) + self.r.set(self.prefix + self.separator + str(file_type), path) def get_stdfile(self, file_type): - return self.r.get(file_type) + return self.r.get(self.prefix + self.separator + str(file_type)) __database__ = Database() From 7ce32602d50f369349d630ac91725b23bb51d1c2 Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:27:44 +0545 Subject: [PATCH 04/20] Changed modules to use prefix instead of port --- process_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/process_manager.py b/process_manager.py index 426efa3370..14d4e68d37 100644 --- a/process_manager.py +++ b/process_manager.py @@ -131,13 +131,13 @@ def load_modules(self): if 'P2P Trust' == module_name: module = module_class( self.main.outputqueue, - self.main.redis_port, + self.main.prefix, output_dir=self.main.args.output ) else: module = module_class( self.main.outputqueue, - self.main.redis_port + self.main.prefix ) module.start() __database__.store_process_PID( From 9cd257365db90cbb8beca1079f82a32b5c6363bc Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:29:52 +0545 Subject: [PATCH 05/20] Remove keys from the default database instead of killing the server --- checker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/checker.py b/checker.py index 28cbdd995b..38f40317b8 100644 --- a/checker.py +++ b/checker.py @@ -97,7 +97,7 @@ def check_given_flags(self): # kill all open unused redis servers if the parameter was included if self.main.args.killall: - self.main.redis_man.close_open_redis_servers() + self.main.redis_man.close_open_redis_id() self.main.terminate_slips() if self.main.args.version: @@ -147,7 +147,7 @@ def clear_redis_cache(self): self.main.redis_man.clear_redis_cache_database() self.main.input_information = '' self.main.zeek_folder = '' - self.main.log_redis_server_PID(6379, self.main.redis_man.get_pid_of_redis_server(6379)) + # self.main.log_redis_server_PID(6379, self.main.redis_man.get_pid_of_redis_server(6379)) self.main.terminate_slips() def check_output_redirection(self) -> tuple: From b5ae49dfbe7645acac03347156e9149aec30e596 Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:30:28 +0545 Subject: [PATCH 06/20] Argument to use custom prefix instead of self generated --- slips_files/common/argparse.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/slips_files/common/argparse.py b/slips_files/common/argparse.py index 64495aa64e..6bcd870ed7 100644 --- a/slips_files/common/argparse.py +++ b/slips_files/common/argparse.py @@ -252,6 +252,14 @@ def parse_arguments(self): required=False, help='The redis-server port to use', ) + self.add_argument( + '-uid', + '--prefix', + metavar='', + action='store', + required=False, + help='Sets the prefix of process in the database.', + ) self.add_argument( '-t', '--testing', From 885f5f9a56206d40259ad20facb0bf3ed4b5d75d Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:31:38 +0545 Subject: [PATCH 07/20] Set prefix on database before processing --- slips_files/core/evidenceProcess.py | 5 +++-- slips_files/core/filemonitor.py | 4 ++-- slips_files/core/inputProcess.py | 6 ++++-- slips_files/core/logsProcess.py | 4 +++- slips_files/core/outputProcess.py | 3 ++- slips_files/core/profilerProcess.py | 12 +++++++++--- slips_files/core/whitelist.py | 4 ++-- 7 files changed, 25 insertions(+), 13 deletions(-) diff --git a/slips_files/core/evidenceProcess.py b/slips_files/core/evidenceProcess.py index abb27d8bed..f98402689e 100644 --- a/slips_files/core/evidenceProcess.py +++ b/slips_files/core/evidenceProcess.py @@ -47,13 +47,14 @@ def __init__( output_folder, logs_folder, redis_port, + prefix, ): self.name = 'Evidence' multiprocessing.Process.__init__(self) self.inputqueue = inputqueue self.outputqueue = outputqueue - self.whitelist = Whitelist(outputqueue, redis_port) - __database__.start(redis_port) + self.whitelist = Whitelist(outputqueue, prefix, redis_port) + __database__.start(prefix, redis_port) self.separator = __database__.separator # Read the configuration self.read_configuration() diff --git a/slips_files/core/filemonitor.py b/slips_files/core/filemonitor.py index 2143d1a561..3481b17a1c 100644 --- a/slips_files/core/filemonitor.py +++ b/slips_files/core/filemonitor.py @@ -27,10 +27,10 @@ class FileEventHandler(RegexMatchingEventHandler): REGEX = [r'.*\.log$', r'.*\.conf$'] - def __init__(self, redis_port, dir_to_monitor, input_type): + def __init__(self, redis_port, prefix, dir_to_monitor, input_type): super().__init__(self.REGEX) self.dir_to_monitor = dir_to_monitor - __database__.start(redis_port) + __database__.start(prefix, redis_port) utils.drop_root_privs() self.input_type = input_type diff --git a/slips_files/core/inputProcess.py b/slips_files/core/inputProcess.py index 3a73992989..9d034c376f 100644 --- a/slips_files/core/inputProcess.py +++ b/slips_files/core/inputProcess.py @@ -46,12 +46,14 @@ def __init__( zeek_folder, line_type, redis_port, + prefix, ): multiprocessing.Process.__init__(self) self.name = 'Input' self.outputqueue = outputqueue self.profilerqueue = profilerqueue - __database__.start(redis_port) + __database__.start(prefix, redis_port) + self.prefix = prefix self.redis_port = redis_port self.input_type = input_type # in case of reading from stdin, the user mst tell slips what type of lines is the input @@ -631,7 +633,7 @@ def start_observer(self): # some process to tell us which files to read in real time when they appear # Get the file eventhandler # We have to set event_handler and event_observer before running zeek. - event_handler = FileEventHandler(self.redis_port, self.zeek_folder, self.input_type) + event_handler = FileEventHandler(self.redis_port, self.prefix, self.zeek_folder, self.input_type) # Create an observer self.event_observer = Observer() # Schedule the observer with the callback on the file handler diff --git a/slips_files/core/logsProcess.py b/slips_files/core/logsProcess.py index 36dd040a73..3a73a97b6a 100644 --- a/slips_files/core/logsProcess.py +++ b/slips_files/core/logsProcess.py @@ -40,12 +40,14 @@ def __init__( debug, mainfoldername, redis_port, + prefix, ): self.name = 'Logs' multiprocessing.Process.__init__(self) self.verbose = verbose self.debug = debug - __database__.start(redis_port) + __database__.start(prefix, redis_port) + self.prefix = prefix self.separator = '_' self.inputqueue = inputqueue self.outputqueue = outputqueue diff --git a/slips_files/core/outputProcess.py b/slips_files/core/outputProcess.py index 1c78491b5f..05099a3109 100644 --- a/slips_files/core/outputProcess.py +++ b/slips_files/core/outputProcess.py @@ -40,6 +40,7 @@ def __init__( verbose, debug, redis_port, + prefix, stdout='', stderr='output/errors.log', slips_logfile='output/slips.log' @@ -65,7 +66,7 @@ def __init__( f'Verbosity: {str(self.verbose)}. Debugging: {str(self.debug)}' ) # Start the DB - __database__.start(redis_port) + __database__.start(prefix, redis_port) # are we in daemon of interactive mode self.slips_mode = __database__.get_slips_mode() # we update the stats printed by slips every 5seconds diff --git a/slips_files/core/profilerProcess.py b/slips_files/core/profilerProcess.py index ebbc703fe7..cea27f58ce 100644 --- a/slips_files/core/profilerProcess.py +++ b/slips_files/core/profilerProcess.py @@ -35,7 +35,13 @@ class ProfilerProcess(multiprocessing.Process): """A class to create the profiles for IPs and the rest of data""" def __init__( - self, inputqueue, outputqueue, verbose, debug, redis_port + self, + inputqueue, + outputqueue, + verbose, + debug, + redis_port, + prefix ): self.name = 'Profiler' multiprocessing.Process.__init__(self) @@ -43,10 +49,10 @@ def __init__( self.outputqueue = outputqueue self.timeformat = None self.input_type = False - self.whitelist = Whitelist(outputqueue, redis_port) + self.whitelist = Whitelist(outputqueue, prefix, redis_port) # Read the configuration self.read_configuration() - __database__.start(redis_port) + __database__.start(prefix, redis_port) # Set the database output queue __database__.setOutputQueue(self.outputqueue) self.verbose = verbose diff --git a/slips_files/core/whitelist.py b/slips_files/core/whitelist.py index 2c70048a55..9302d31a9a 100644 --- a/slips_files/core/whitelist.py +++ b/slips_files/core/whitelist.py @@ -9,13 +9,13 @@ class Whitelist: - def __init__(self, outputqueue, redis_port): + def __init__(self, outputqueue, prefix, redis_port='6379'): self.name = 'whitelist' self.outputqueue = outputqueue self.read_configuration() self.org_info_path = 'slips_files/organizations_info/' self.ignored_flow_types = ('arp') - __database__.start(redis_port) + __database__.start(prefix, redis_port) def print(self, text, verbose=1, debug=0): From 2e8a949041c26b066e36114c3aaa21c04af1e1ee Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:34:56 +0545 Subject: [PATCH 08/20] Modified database tests to use prefix --- tests/test_database.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/test_database.py b/tests/test_database.py index 27d8eaaa32..33b5dd538f 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -4,19 +4,18 @@ import os import json import time - +import uuid # random values for testing profileid = 'profile_192.168.1.1' twid = 'timewindow1' test_ip = '192.168.1.1' - +prefix = str(uuid.uuid4()) def do_nothing(*arg): """Used to override the print function because using the self.print causes broken pipes""" pass - # create another database instance other than the one in # conftest because the port in conftest is used in other test files def create_db_instace(outputQueue): @@ -28,6 +27,7 @@ def create_db_instace(outputQueue): __database__.home_network = utils.home_network_ranges __database__.width = 3600 __database__.connect_to_redis_server(6381) + __database__.setPrefix(prefix) __database__.r.flushdb() __database__.setSlipsInternalTime(0) return __database__ @@ -82,7 +82,7 @@ def test_add_flow(outputQueue): "module_labels": {}} assert add_flow(database) is True assert ( - json.loads(database.r.hget(f'{profileid}_{twid}_flows', uid)) + json.loads(database.r.hget(f'{prefix}_{profileid}_{twid}_flows', uid)) == added_flow ) @@ -95,7 +95,7 @@ def test_getProfileIdFromIP(outputQueue): os.system('./slips.py -c slips.conf -cc') # add a profile - database.addProfile('profile_192.168.1.1', '00:00', '1') + database.addProfile(f'{prefix}_profile_192.168.1.1', '00:00', '1') # try to retrieve it assert database.getProfileIdFromIP(test_ip) is not False @@ -103,7 +103,7 @@ def test_getProfileIdFromIP(outputQueue): def test_timewindows(outputQueue): """unit tests for addNewTW ,getLastTWforProfile and getFirstTWforProfile""" database = create_db_instace(outputQueue) - profileid = 'profile_192.168.1.1' + profileid = f'{prefix}_profile_192.168.1.1' # add a profile database.addProfile(profileid, '00:00', '1') # add a tw to that profile (first tw) @@ -144,7 +144,7 @@ def test_add_ips(outputQueue): assert ( database.add_ips(profileid, twid, ipaddress.ip_address(test_ip), columns, 'Server') is True ) - hash_id = f'{profileid}_{twid}' + hash_id = f'{prefix}_{profileid}_{twid}' stored_dstips = database.r.hget(hash_id, 'SrcIPs') assert stored_dstips == '{"192.168.1.1": 1}' @@ -167,7 +167,7 @@ def test_add_port(outputQueue): 'starttime': '20.0', } database.add_port(profileid, twid, test_ip, columns, 'Server', 'Dst') - hash_key = f'{profileid}_{twid}' + hash_key = f'{prefix}_{profileid}_{twid}' added_ports = database.r.hgetall(hash_key) assert 'DstPortsServerTCPNot Established' in added_ports.keys() assert test_ip in added_ports['DstPortsServerTCPNot Established'] @@ -187,8 +187,8 @@ def test_setEvidence(outputQueue): database.setEvidence(evidence_type, attacker_direction, attacker, threat_level, confidence, description, timestamp, category, profileid=profileid, twid=twid, uid=uid) - added_evidence = database.r.hget(f'evidence{profileid}', twid) - added_evidence2 = database.r.hget(f'{profileid}_{twid}', 'Evidence') + added_evidence = database.r.hget(f'{prefix}_evidence{profileid}', twid) + added_evidence2 = database.r.hget(f'{prefix}_{profileid}_{twid}', 'Evidence') assert added_evidence2 == added_evidence added_evidence = json.loads(added_evidence) @@ -204,9 +204,9 @@ def test_deleteEvidence(outputQueue): database = create_db_instace(outputQueue) description = 'SSH Successful to IP :8.8.8.8. From IP 192.168.1.1' database.deleteEvidence(profileid, twid, description) - added_evidence = json.loads(database.r.hget(f'evidence{profileid}', twid)) + added_evidence = json.loads(database.r.hget(f'{prefix}_evidence{profileid}', twid)) added_evidence2 = json.loads( - database.r.hget(f'{profileid}_{twid}', 'Evidence') + database.r.hget(f'{prefix}_{profileid}_{twid}', 'Evidence') ) assert 'SSHSuccessful-by-192.168.1.1' not in added_evidence assert 'SSHSuccessful-by-192.168.1.1' not in added_evidence2 @@ -266,7 +266,7 @@ def test_add_mac_addr_to_profile(outputQueue): MAC_info = {'MAC': '00:00:5e:00:53:af'} # first associate this ip with some mac assert database.add_mac_addr_to_profile(profileid_ipv4, MAC_info) is True - assert ipv4 in str(database.r.hget('MAC', MAC_info['MAC'])) + assert ipv4 in str(database.r.hget(f'{prefix}_MAC', MAC_info['MAC'])) # now claim that we found another profile # that has the same mac as this one @@ -274,21 +274,21 @@ def test_add_mac_addr_to_profile(outputQueue): profileid = 'profile_192.168.1.6' assert database.add_mac_addr_to_profile(profileid, MAC_info) is False # this ip shouldnt be added to the profile as they're both ipv4 - assert '192.168.1.6' not in database.r.hget('MAC', MAC_info['MAC']) + assert '192.168.1.6' not in database.r.hget(f'{prefix}_MAC', MAC_info['MAC']) # now claim that another ipv6 has this mac ipv6 = '2001:0db8:85a3:0000:0000:8a2e:0370:7334' profileid_ipv6 = f'profile_{ipv6}' database.add_mac_addr_to_profile(profileid_ipv6, MAC_info) # make sure the mac is associated with his ipv6 - assert ipv6 in database.r.hget('MAC', MAC_info['MAC']) + assert ipv6 in database.r.hget(f'{prefix}_MAC', MAC_info['MAC']) # make sure the ipv4 is associated with this # ipv6 profile - assert ipv4 in str(database.r.hmget(profileid_ipv6, 'IPv4')) + assert ipv4 in str(database.r.hmget(prefix + '_' + profileid_ipv6, 'IPv4')) # make sure the ipv6 is associated with the # profile that has the same ipv4 as the mac - assert ipv6 in str(database.r.hmget(profileid_ipv4, 'IPv6')) + assert ipv6 in str(database.r.hmget(prefix + '_' + profileid_ipv4, 'IPv6')) def test_get_the_other_ip_version(outputQueue): From 52d199e1a0a165d425e726735cf438db103ec4f0 Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:24:24 +0545 Subject: [PATCH 09/20] Modified redis manager to use prefix as id instead of port --- redis_manager.py | 136 ++++++++++++++++++++++++++++------------------- 1 file changed, 80 insertions(+), 56 deletions(-) diff --git a/redis_manager.py b/redis_manager.py index 147105ef10..82540c2178 100644 --- a/redis_manager.py +++ b/redis_manager.py @@ -5,6 +5,7 @@ import redis import os import time +import uuid class RedisManager: def __init__(self, terminate_slips=None): @@ -78,6 +79,9 @@ def get_random_redis_port(self): "Unable to start slips.\n") return False + def get_random_prefix(self) -> str: + return str(uuid.uuid4()) + def clear_redis_cache_database( self, redis_host='localhost', redis_port=6379 ) -> bool: @@ -94,26 +98,39 @@ def clear_redis_cache_database( rcache.flushdb() return True - def close_all_ports(self): + def close_all_instance(self): """ Closes all the redis ports in logfile and in slips supported range of ports """ - if not hasattr(self, 'open_servers_PIDs'): - self.get_open_redis_servers() + if not hasattr(self, 'open_servers_IDs'): + self.get_open_redis_servers_ID() + unique_ports = set() # close all ports in logfile - for pid in self.open_servers_PIDs: - self.flush_redis_server(pid=pid) + for _id in self.open_servers_IDs: + self.flush_redis_id(_id=_id, port = self.open_servers_IDs[_id]) + unique_ports.add(self.open_servers_IDs[_id]) + + # Kill default port + + for port in unique_ports: + # Skip cache db which is in port 6379 + if str(port) == '6379': + continue + pid = self.get_pid_of_redis_server(port = port) self.kill_redis_server(pid) + # closes all the ports in slips supported range of ports - slips_supported_range = list(range(self.start_port, self.end_port + 1)) - slips_supported_range.append(6379) - for port in slips_supported_range: - if pid := self.get_pid_of_redis_server(port): - self.flush_redis_server(pid=pid) - self.kill_redis_server(pid) + # slips_supported_range = list(range(self.start_port, self.end_port + 1)) + # slips_supported_range.append(6379) + + # for port in slips_supported_range: + # if _id := self.get_pid_of_redis_server(port): + # self.flush_redis_server(_id=id) + # self.kill_redis_server(pid) + # print(f"Successfully closed all redis servers on ports {self.start_port} to {self.end_port}") @@ -138,11 +155,11 @@ def get_pid_of_redis_server(self, port: int) -> str: return pid return False - def get_open_redis_servers(self) -> dict: + def get_open_redis_servers_ID(self) -> dict: """ Returns the dict of PIDs and ports of the redis servers started by slips """ - self.open_servers_PIDs = {} + self.open_servers_IDs = {} try: with open(self.running_logfile, 'r') as f: for line in f.read().splitlines(): @@ -154,20 +171,20 @@ def get_open_redis_servers(self) -> dict: ): continue line = line.split(',') - pid, port = line[3], line[2] - self.open_servers_PIDs[pid] = port - return self.open_servers_PIDs + _id, port = line[3], line[2] + self.open_servers_IDs[_id] = port + return self.open_servers_IDs except FileNotFoundError: # print(f"Error: {self.running_logfile} is not found. Can't kill open servers. Stopping.") return {} - def print_open_redis_servers(self): + def print_open_redis_id(self): """ Returns a dict {counter: (used_port,pid) } """ - open_servers = {} + open_ids = {} to_print = f"Choose which one to kill [0,1,2 etc..]\n" \ - f"[0] Close all Redis servers\n" + f"[0] Close all Redis ids\n" there_are_ports_to_print = False try: with open(self.running_logfile, 'r') as f: @@ -182,20 +199,20 @@ def print_open_redis_servers(self): continue line_number += 1 line = line.split(',') - file, port, pid = line[1], line[2], line[3] + file, port, _id = line[1], line[2], line[3] there_are_ports_to_print = True - to_print += f"[{line_number}] {file} - port {port}\n" - open_servers[line_number] = (port, pid) + to_print += f"[{line_number}] {file} - port {port} - id {_id}\n" + open_ids[line_number] = (port, _id) except FileNotFoundError: - print(f"{self.running_logfile} is not found. Can't get open redis servers. Stopping.") + print(f"{self.running_logfile} is not found. Can't get open redis id. Stopping.") return False if there_are_ports_to_print: print(to_print) else: - print(f"No open redis servers in {self.running_logfile}") + print(f"No open redis id in {self.running_logfile}") - return open_servers + return open_ids def get_port_of_redis_server(self, pid: str): @@ -211,23 +228,24 @@ def get_port_of_redis_server(self, pid: str): return False - def flush_redis_server(self, pid: str='', port: str=''): + def flush_redis_id(self, _id: str='', port: str=''): """ Flush the redis server on this pid, only 1 param should be given, pid or port :param pid: can be False if port is given Gets the pid of the port is not given """ - if not port and not pid: + if not port and not _id: return False # sometimes the redis port is given, no need to get it manually - if not port and pid: - if not hasattr(self, 'open_servers_PIDs'): - self.get_open_redis_servers() - port = self.open_servers_PIDs.get(str(pid), False) + if not port and _id: + if not hasattr(self, 'open_servers_IDs'): + self.get_open_redis_servers_ID() + port = self.open_servers_IDs.get(str(_id), False) if not port: - # try to get the port using a cmd - port = self.get_port_of_redis_server(pid) + # use default port + port = '6379' + port = str(port) # clear the server opened on this port @@ -245,9 +263,15 @@ def flush_redis_server(self, pid: str='', port: str=''): retry_on_timeout=True, health_check_interval=20, ) - r.flushall() - r.flushdb() - r.script_flush() + + # r.flushall() + # r.flushdb() + # r.script_flush() + + # Delete keys with the prefix id + for key in r.scan_iter(_id + "*"): + r.delete(key) + return True except redis.exceptions.ConnectionError: # server already killed! @@ -306,52 +330,52 @@ def remove_old_logline(self, redis_port): os.replace(tmpfile, self.running_logfile) - def remove_server_from_log(self, redis_port): + def remove_server_from_log(self, _id): """ deletes the server running on the given pid from running_slips_logs """ - redis_port = str(redis_port) + _id = str(_id) tmpfile = 'tmp_running_slips_log.txt' with open(self.running_logfile, 'r') as logfile: with open(tmpfile, 'w') as tmp: all_lines = logfile.read().splitlines() # delete the line using that port for line in all_lines: - if redis_port not in line: + if _id not in line: tmp.write(f'{line}\n') # replace file with original name os.replace(tmpfile, self.running_logfile) - def close_open_redis_servers(self): + def close_open_redis_id(self): """ Function to close unused open redis-servers based on what the user chooses """ - if not hasattr(self, 'open_servers_PIDs'): + if not hasattr(self, 'open_servers_IDs'): # fill the dict - self.get_open_redis_servers() + self.get_open_redis_servers_ID() with contextlib.suppress(KeyboardInterrupt): # open_servers {counter: (port,pid),...}} - open_servers:dict = self.print_open_redis_servers() - if not open_servers and self.terminate_slips: + open_ids:dict = self.print_open_redis_id() + if not open_ids and self.terminate_slips: self.terminate_slips() - server_to_close = input() + id_to_close = input() # close all ports in running_slips_logs.txt and in our supported range - if server_to_close == '0': - self.close_all_ports() + if id_to_close == '0': + self.close_all_instance() - elif len(open_servers) > 0: + elif len(open_ids) > 0: # close the given server number try: - pid = open_servers[int(server_to_close)][1] - port = open_servers[int(server_to_close)][0] - if self.flush_redis_server(pid=pid) and self.kill_redis_server(pid): - print(f"Killed redis server on port {port}.") + _id = open_ids[int(id_to_close)][1] + port = open_ids[int(id_to_close)][0] + if self.flush_redis_id(_id=_id, port = port): + print(f"Deleted process of id {_id} on port {port}.") else: - print(f"Redis server running on port {port} " - f"is either already killed or you don't have " - f"enough permission to kill it.") - self.remove_server_from_log(port) + print(f"Redis server running on port {port} with id {_id}" + f"is either already deleted or you don't have " + f"enough permission to delete it.") + self.remove_server_from_log(_id) except (KeyError, ValueError): print(f"Invalid input {server_to_close}") From 0c71d95f6270aa601970371721bbc8c39324bbd2 Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:26:15 +0545 Subject: [PATCH 10/20] Modified to use id as prefix instead of port --- slips.py | 65 +++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/slips.py b/slips.py index a1af5b8798..d70c774e86 100755 --- a/slips.py +++ b/slips.py @@ -186,7 +186,7 @@ def update_local_TI_files(self): # only one instance of slips should be able to update ports and orgs at a time # so this function will only be allowed to run from 1 slips instance. with Lock(name="slips_ports_and_orgs"): - update_manager = UpdateFileManager(self.outputqueue, self.redis_port) + update_manager = UpdateFileManager(self.outputqueue, self.prefix) update_manager.update_ports_info() update_manager.update_org_files() except CannotAcquireLock: @@ -288,7 +288,7 @@ def prepare_output_dir(self): # print(f'[Main] Storing Slips logs in {self.args.output}') - def log_redis_server_PID(self, redis_port, redis_pid): + def log_redis_server_prefix(self, redis_port, prefix): now = utils.convert_format(datetime.now(), utils.alerts_format) try: # used in case we need to remove the line using 6379 from running logfile @@ -298,13 +298,13 @@ def log_redis_server_PID(self, redis_port, redis_pid): f.write( '# This file contains a list of used redis ports.\n' '# Once a server is killed, it will be removed from this file.\n' - 'Date, File or interface, Used port, Server PID,' + 'Date, File or interface, Used port, Unique ID (prefix),' ' Output Zeek Dir, Logs Dir, Slips PID, Is Daemon, Save the DB\n' ) f.write( f'{now},{self.input_information},{redis_port},' - f'{redis_pid},{self.zeek_folder},{self.args.output},' + f'{prefix},{self.zeek_folder},{self.args.output},' f'{os.getpid()},' f'{bool(self.args.daemon)},{self.args.save}\n' ) @@ -312,11 +312,11 @@ def log_redis_server_PID(self, redis_port, redis_pid): # last run was by root, change the file ownership to non-root os.remove(self.redis_man.running_logfile) open(self.redis_man.running_logfile, 'w').close() - self.log_redis_server_PID(redis_port, redis_pid) + self.log_redis_server_prefix(redis_port, prefix) - if redis_port == 6379: - # remove the old logline using this port - self.redis_man.remove_old_logline(6379) + # if redis_port == 6379: + # # remove the old logline using this port + # self.redis_man.remove_old_logline(6379) def set_mode(self, mode, daemon=''): """ @@ -412,7 +412,7 @@ def load_redis_db(self, redis_port): self.input_information = os.path.basename(self.args.db) redis_pid = self.redis_man.get_pid_of_redis_server(redis_port) self.zeek_folder = '""' - self.log_redis_server_PID(redis_port, redis_pid) + self.log_redis_server_prefix(redis_port, redis_pid) self.redis_man.remove_old_logline(redis_port) print( @@ -541,22 +541,35 @@ def start(self): ########################## # get the port that is going to be used for this instance of slips + # if self.args.port: + # self.redis_port = int(self.args.port) + # # close slips if port is in use + # self.metadata_man.check_if_port_is_in_use(self.redis_port) + # elif self.args.multiinstance: + # self.redis_port = self.redis_man.get_random_redis_port() + # if not self.redis_port: + # # all ports are unavailable + # inp = input("Press Enter to close all ports.\n") + # if inp == '': + # self.redis_man.close_all_ports() + # self.terminate_slips() + # else: + # # even if this port is in use, it will be overwritten by slips + # self.redis_port = 6379 + # # self.check_if_port_is_in_use(self.redis_port) + if self.args.port: + # set port to specified if explicitly specified self.redis_port = int(self.args.port) - # close slips if port is in use self.metadata_man.check_if_port_is_in_use(self.redis_port) - elif self.args.multiinstance: - self.redis_port = self.redis_man.get_random_redis_port() - if not self.redis_port: - # all ports are unavailable - inp = input("Press Enter to close all ports.\n") - if inp == '': - self.redis_man.close_all_ports() - self.terminate_slips() else: - # even if this port is in use, it will be overwritten by slips + # default port 6379 self.redis_port = 6379 - # self.check_if_port_is_in_use(self.redis_port) + + if self.args.prefix: + self.prefix = self.args.prefix + else: + self.prefix = self.redis_man.get_random_prefix() # Output thread. outputprocess should be created first because it handles # the output of the rest of the threads. @@ -570,6 +583,7 @@ def start(self): self.args.verbose, self.args.debug, self.redis_port, + self.prefix, stdout=current_stdout, stderr=stderr, slips_logfile=slips_logfile, @@ -587,8 +601,8 @@ def start(self): # log the PID of the started redis-server # should be here after we're sure that the server was started - redis_pid = self.redis_man.get_pid_of_redis_server(self.redis_port) - self.log_redis_server_PID(self.redis_port, redis_pid) + # redis_pid = self.redis_man.get_pid_of_redis_server(self.redis_port) + self.log_redis_server_prefix(self.redis_port, self.prefix) __database__.set_slips_mode(self.mode) @@ -608,7 +622,7 @@ def start(self): __database__.store_std_file(**std_files) - self.print(f'Using redis server on port: {green(self.redis_port)}', 1, 0) + self.print(f'Using redis server on port: {green(self.redis_port)} with id: {green(self.prefix)}', 1, 0) self.print(f'Started {green("Main")} process [PID {green(self.pid)}]', 1, 0) self.print(f'Started {green("Output Process")} [PID {green(output_process.pid)}]', 1, 0) self.print('Starting modules', 1, 0) @@ -638,6 +652,7 @@ def sig_handler(sig, frame): self.args.output, logs_dir, self.redis_port, + self.prefix, ) evidence_process.start() self.print( @@ -660,6 +675,7 @@ def sig_handler(sig, frame): self.args.verbose, self.args.debug, self.redis_port, + self.prefix, ) profiler_process.start() self.print( @@ -684,6 +700,7 @@ def sig_handler(sig, frame): self.zeek_folder, self.line_type, self.redis_port, + self.prefix, ) inputProcess.start() self.print( @@ -701,7 +718,7 @@ def sig_handler(sig, frame): self.print('Warning: P2P is only supported using an interface. Disabled P2P.') # warn about unused open redis servers - open_servers = len(self.redis_man.get_open_redis_servers()) + open_servers = len(self.redis_man.get_open_redis_servers_ID()) if open_servers > 1: self.print( f'Warning: You have {open_servers} ' From 9702263453c5a8ace4a2ff14a7b82644cc5ac88b Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:27:05 +0545 Subject: [PATCH 11/20] Changed database to publish and store keys with prefix --- slips_files/core/database/_profile_flow.py | 75 ++--- slips_files/core/database/database.py | 304 +++++++++++---------- 2 files changed, 194 insertions(+), 185 deletions(-) diff --git a/slips_files/core/database/_profile_flow.py b/slips_files/core/database/_profile_flow.py index 28f2ef3500..a0142c7f74 100644 --- a/slips_files/core/database/_profile_flow.py +++ b/slips_files/core/database/_profile_flow.py @@ -11,11 +11,14 @@ def __init__(self): # The name is used to print in the outputprocess self.name = 'DB' self.separator = '_' + self.prefix = '' + def set_prefix(self, prefix): + self.prefix = prefix def publish(self, channel, data): """Publish something""" - self.r.publish(channel, data) + self.r.publish(self.prefix + self.separator + str(channel), data) def getIPData(self, ip: str) -> dict: """ @@ -131,7 +134,7 @@ def update_times_contacted(self, ip, direction, profileid, twid): # Get the DstIPs data for this tw in this profile # The format is {'1.1.1.1' : 3} - ips_contacted = self.r.hget(profileid_twid, f'{direction}IPs') + ips_contacted = self.r.hget(self.prefix + self.separator + profileid_twid, f'{direction}IPs') if not ips_contacted: ips_contacted = {} @@ -144,7 +147,7 @@ def update_times_contacted(self, ip, direction, profileid, twid): ips_contacted[ip] = 1 ips_contacted = json.dumps(ips_contacted) - self.r.hset(profileid_twid, f'{direction}IPs', str(ips_contacted)) + self.r.hset(self.prefix + self.separator + profileid_twid, f'{direction}IPs', str(ips_contacted)) def getFinalStateFromFlags(self, state, pkts): """ @@ -319,7 +322,7 @@ def getDataFromProfileTW( try: key = direction + type_data + role + protocol + state # self.print('Asked Key: {}'.format(key)) - data = self.r.hget(f'{profileid}{self.separator}{twid}', key) + data = self.r.hget(self.prefix + self.separator + f'{profileid}{self.separator}{twid}', key) value = {} if data: portdata = json.loads(data) @@ -433,7 +436,7 @@ def add_ips(self, profileid, twid, ip_as_obj, columns, role: str): ) # Store this data in the profile hash self.r.hset( - f'{profileid}{self.separator}{twid}', + self.prefix + self.separator + f'{profileid}{self.separator}{twid}', key_name, json.dumps(profileid_twid_data) ) @@ -534,7 +537,7 @@ def add_tuple( ) # Get all the InTuples or OutTuples for this profileid in this TW profileid_twid = f'{profileid}{self.separator}{twid}' - tuples = self.r.hget(profileid_twid, direction) + tuples = self.r.hget(self.prefix + self.separator + profileid_twid, direction) # Separate the symbold to add and the previous data (symbol_to_add, previous_two_timestamps) = data_tuple if not tuples: @@ -585,7 +588,7 @@ def add_tuple( # Convet the dictionary to json tuples = json.dumps(tuples) # Store the new data on the db - self.r.hset(profileid_twid, direction, str(tuples)) + self.r.hset(self.prefix + self.separator + profileid_twid, direction, str(tuples)) # Mark the tw as modified self.markProfileTWAsModified(profileid, twid, starttime) @@ -597,7 +600,7 @@ def add_tuple( self.outputqueue.put(f'01|database|[DB] {traceback.format_exc()}') def getSlipsInternalTime(self): - return self.r.get('slips_internal_time') + return self.r.get(self.prefix + self.separator + 'slips_internal_time') def search_tws_for_flow(self, profileid, twid, uid, go_back=False): """ @@ -648,7 +651,7 @@ def check_TW_to_close(self, close_all=False): modification_time = float('inf') profiles_tws_to_close = self.r.zrangebyscore( - 'ModifiedTW', 0, modification_time, withscores=True + self.prefix + self.separator + 'ModifiedTW', 0, modification_time, withscores=True ) for profile_tw_to_close in profiles_tws_to_close: @@ -669,8 +672,8 @@ def markProfileTWAsClosed(self, profileid_tw): """ Mark the TW as closed so tools can work on its data """ - self.r.sadd('ClosedTW', profileid_tw) - self.r.zrem('ModifiedTW', profileid_tw) + self.r.sadd(self.prefix + self.separator + 'ClosedTW', profileid_tw) + self.r.zrem(self.prefix + self.separator + 'ModifiedTW', profileid_tw) self.publish('tw_closed', profileid_tw) def markProfileTWAsModified(self, profileid, twid, timestamp): @@ -687,7 +690,7 @@ def markProfileTWAsModified(self, profileid, twid, timestamp): data = { f'{profileid}{self.separator}{twid}': float(timestamp) } - self.r.zadd('ModifiedTW', data) + self.r.zadd(self.prefix + self.separator + 'ModifiedTW', data) self.publish( 'tw_modified', f'{profileid}:{twid}' @@ -799,7 +802,7 @@ def add_port( data = json.dumps(old_profileid_twid_data) hash_key = f'{profileid}{self.separator}{twid}' key_name = f'{port_type}Ports{role}{proto}{summaryState}' - self.r.hset(hash_key, key_name, str(data)) + self.r.hset(self.prefix + self.separator + hash_key, key_name, str(data)) self.markProfileTWAsModified(profileid, twid, starttime) def add_flow( @@ -857,7 +860,7 @@ def add_flow( flow = json.dumps(flow) # Store in the hash x.x.x.x_timewindowx_flows value = self.r.hset( - f'{profileid}{self.separator}{twid}{self.separator}flows', + self.prefix + self.separator + f'{profileid}{self.separator}{twid}{self.separator}flows', uid, flow, ) @@ -868,7 +871,7 @@ def add_flow( # The key was not there before. So this flow is not repeated # Store the label in our uniq set, and increment it by 1 if label: - self.r.zincrby('labels', 1, label) + self.r.zincrby(self.prefix + self.separator + 'labels', 1, label) flow = {uid: flow} @@ -909,51 +912,51 @@ def set_local_network(self, saddr): return # get the local network of this saddr if network_range := utils.get_cidr_of_ip(saddr): - self.r.set("local_network", network_range) + self.r.set(self.prefix + self.separator + "local_network", network_range) self.is_localnet_set = True def get_local_network(self): - return self.r.get("local_network") + return self.r.get(self.prefix + self.separator + "local_network") def get_label_count(self, label): """ :param label: malicious or normal """ - return self.r.zscore('labels', label) + return self.r.zscore(self.prefix + self.separator + 'labels', label) def get_disabled_modules(self) -> list: - return json.loads(self.r.hget('analysis', 'disabled_modules')) + return json.loads(self.r.hget(self.prefix + self.separator + 'analysis', 'disabled_modules')) def set_input_metadata(self, info:dict): """ sets name, size, analysis dates, and zeek_dir in the db """ for info, val in info.items(): - self.r.hset('analysis', info, val) + self.r.hset(self.prefix + self.separator + 'analysis', info, val) def get_zeek_output_dir(self): """ gets zeek output dir from the db """ - return self.r.hget('analysis', 'zeek_dir') + return self.r.hget(self.prefix + self.separator + 'analysis', 'zeek_dir') def get_total_flows(self): """ gets total flows to process from the db """ - return self.r.hget('analysis', 'total_flows') + return self.r.hget(self.prefix + self.separator + 'analysis', 'total_flows') def get_input_type(self): """ gets input type from the db """ - return self.r.hget('analysis', 'input_type') + return self.r.hget(self.prefix + self.separator + 'analysis', 'input_type') def get_output_dir(self, info:dict): """ returns the currently used output dir """ - return self.r.hget('analysis', 'output_dir') + return self.r.hget(self.prefix + self.separator + 'analysis', 'output_dir') def get_flow(self, profileid, twid, uid): @@ -966,7 +969,7 @@ def get_flow(self, profileid, twid, uid): # outside of home_network when this param is given return {} temp = self.r.hget( - f'{profileid}{self.separator}{twid}{self.separator}flows', uid + self.prefix + self.separator + f'{profileid}{self.separator}{twid}{self.separator}flows', uid ) return {uid: temp} @@ -1024,7 +1027,7 @@ def add_out_ssl( # Convert to json string data = json.dumps(data) self.r.hset( - f'{profileid}{self.separator}{twid}{self.separator}altflows', + self.prefix + self.separator + f'{profileid}{self.separator}{twid}{self.separator}altflows', uid, data, ) @@ -1059,7 +1062,7 @@ def add_out_ssl( if SNI_port not in sni_ipdata: # Verify that the SNI is equal to any of the domains in the DNS resolution # only add this SNI to our db if it has a DNS resolution - if dns_resolutions := self.r.hgetall('DNSresolution'): + if dns_resolutions := self.r.hgetall(self.prefix + self.separator + 'DNSresolution'): # dns_resolutions is a dict with {ip:{'ts'..,'domains':..., 'uid':..}} for ip, resolution in dns_resolutions.items(): resolution = json.loads(resolution) @@ -1097,7 +1100,7 @@ def setInfoForIPs(self, ip: str, ipdata: dict): self.rcache.hset('IPsInfo', ip, json.dumps(data)) if new_key: - self.r.publish('ip_info_change', ip) + self.r.publish(self.prefix + self.separator + 'ip_info_change', ip) def get_p2p_reports_about_ip(self, ip) -> dict: """ @@ -1193,7 +1196,7 @@ def add_out_http( data = json.dumps(data) self.r.hset( - f'{profileid}{ self.separator }{twid}{ self.separator }altflows', + self.prefix + self.separator + f'{profileid}{ self.separator }{twid}{ self.separator }altflows', uid, data, ) @@ -1271,7 +1274,7 @@ def add_out_ssh( data = json.dumps(data) # Set the dns as alternative flow self.r.hset( - f'{profileid}{self.separator}{twid}{self.separator}altflows', + self.prefix + self.separator + f'{profileid}{self.separator}{twid}{self.separator}altflows', uid, data, ) @@ -1329,7 +1332,7 @@ def add_out_notice( } to_send = json.dumps(to_send) self.r.hset( - f'{profileid}{self.separator}{twid}{self.separator}altflows', + self.prefix + self.separator + f'{profileid}{self.separator}{twid}{self.separator}altflows', uid, data, ) @@ -1458,9 +1461,9 @@ def set_dns_resolution( ip_info = json.dumps(ip_info) # we store ALL dns resolutions seen since starting slips # store with the IP as the key - self.r.hset('DNSresolution', answer, ip_info) + self.r.hset(self.prefix + self.separator + 'DNSresolution', answer, ip_info) # store with the domain as the key: - self.r.hset('ResolvedDomains', domains[0], answer) + self.r.hset(self.prefix + self.separator + 'ResolvedDomains', domains[0], answer) # these ips will be associated with the query in our db ips_to_add.append(answer) @@ -1484,7 +1487,7 @@ def set_domain_resolution(self, domain, ips): """ stores all the resolved domains with their ips in the db """ - self.r.hset("DomainsResolved", domain, json.dumps(ips)) + self.r.hset(self.prefix + self.separator + "DomainsResolved", domain, json.dumps(ips)) def getDomainData(self, domain): """ @@ -1589,7 +1592,7 @@ def setInfoForDomains(self, domain: str, info_to_set: dict, mode='leave'): domain_data = json.dumps(domain_data) self.rcache.hset('DomainsInfo', domain, domain_data) # Publish the changes - self.r.publish('dns_info_change', domain) + self.r.publish(self.prefix + self.separator + 'dns_info_change', domain) def add_out_dns( self, @@ -1627,7 +1630,7 @@ def add_out_dns( data = json.dumps(data) # Set the dns as alternative flow self.r.hset( - f'{profileid}{self.separator}{twid}{self.separator}altflows', + self.prefix + self.separator + f'{profileid}{self.separator}{twid}{self.separator}altflows', uid, data, ) diff --git a/slips_files/core/database/database.py b/slips_files/core/database/database.py index 93333b4288..02631171b6 100644 --- a/slips_files/core/database/database.py +++ b/slips_files/core/database/database.py @@ -193,21 +193,21 @@ def set_slips_mode(self, slips_mode): function to store the current mode (daemonized/interactive) in the db """ - self.r.set("mode", slips_mode) + self.r.set(self.prefix + self.separator + "mode", slips_mode) def get_slips_mode(self): """ function to get the current mode (daemonized/interactive) in the db """ - self.r.get("mode") + self.r.get(self.prefix + self.separator + "mode") def get_modified_ips_in_the_last_tw(self): """ this number is updated in the db every 5s by slips.py used for printing running stats in slips.py or outputprocess """ - if modified_ips := self.r.hget('analysis', 'modified_ips_in_the_last_tw'): + if modified_ips := self.r.hget(self.prefix + self.separator + 'analysis', 'modified_ips_in_the_last_tw'): return modified_ips else: return 0 @@ -239,12 +239,18 @@ def change_redis_limits(self, redis_client): "slave 268435456 67108864 60 " "pubsub 4294967296 2147483648 600") - def start(self, redis_port): + def setPrefix(self,prefix): + super().set_prefix(prefix) + self.prefix = prefix + + def start(self, prefix, redis_port:str = '6379'): """Start the DB. Allow it to read the conf""" self.read_configuration() # Read values from the configuration file try: + super().set_prefix(prefix) + self.prefix = prefix if not hasattr(self, 'r'): self.connect_to_redis_server(redis_port) # Set the memory limits of the output buffer, For normal clients: no limits @@ -266,7 +272,7 @@ def start(self, redis_port): # to fix redis.exceptions.ResponseError MISCONF Redis is configured to save RDB snapshots # configure redis to stop writing to dump.rdb when an error occurs without throwing errors in slips # Even if the DB is not deleted. We need to delete some temp data - self.r.delete('zeekfiles') + self.r.delete(self.prefix + self.separator + 'zeekfiles') # By default the slips internal time is 0 until we receive something @@ -278,14 +284,14 @@ def start(self, redis_port): return False def is_connection_error_logged(self): - return bool(self.r.get('logged_connection_error')) + return bool(self.r.get(self.prefix + self.separator + 'logged_connection_error')) def mark_connection_error_as_logged(self): """ When redis connection error occurs, to prevent every module from logging it to slips.log and the console, set this variable in the db """ - self.r.set('logged_connection_error', 'True') + self.r.set(self.prefix + self.separator + 'logged_connection_error', 'True') def get_message(self, channel, timeout=0.0000001): """ @@ -305,11 +311,11 @@ def get_message(self, channel, timeout=0.0000001): def set_slips_start_time(self): """store the time slips started (datetime obj)""" now = utils.convert_format(datetime.now(), utils.alerts_format) - self.r.set('slips_start_time', now) + self.r.set(self.prefix + self.separator + 'slips_start_time', now) def get_slips_start_time(self): """get the time slips started (datetime obj)""" - if start_time := self.r.get('slips_start_time'): + if start_time := self.r.get(self.prefix + self.separator + 'slips_start_time'): start_time = utils.convert_format(start_time, utils.alerts_format) return start_time @@ -327,7 +333,7 @@ def should_add(self, profileid: str) -> bool: # no home_network is specified return True - ip = profileid.split(self.separator)[1] + ip = profileid.split(self.separator)[-1] ip_obj = ipaddress.ip_address(ip) return any(ip_obj in network for network in self.home_network) @@ -336,13 +342,13 @@ def set_loaded_ti_files(self, number_of_loaded_files: int): """ Stores the number of successfully loaded TI files """ - self.r.set('loaded TI files', number_of_loaded_files) + self.r.set(self.prefix + self.separator +'loaded TI files', number_of_loaded_files) def get_loaded_ti_files(self): """ returns the number of successfully loaded TI files. or 0 if none is loaded """ - return self.r.get('loaded TI files') or 0 + return self.r.get(self.prefix + self.separator +'loaded TI files') or 0 def addProfile(self, profileid, starttime, duration): """ @@ -352,24 +358,24 @@ def addProfile(self, profileid, starttime, duration): """ try: # make sure we don't add public ips if the user specified a home_network - if self.r.sismember('profiles', str(profileid)): + if self.r.sismember(self.prefix + self.separator + 'profiles', str(profileid)): # we already have this profile return False # execlude ips outside of local network is it's set in slips.conf if not self.should_add(profileid): return False - # Add the profile to the index. The index is called 'profiles' - self.r.sadd('profiles', str(profileid)) + # Add the profile to the index. The index is called prefix + separator + 'profiles' + self.r.sadd(self.prefix + self.separator + 'profiles', str(profileid)) # Create the hashmap with the profileid. The hasmap of each profile is named with the profileid # Add the start time of profile - self.r.hset(profileid, 'starttime', starttime) + self.r.hset(self.prefix + self.separator + str(profileid), 'starttime', starttime) # For now duration of the TW is fixed - self.r.hset(profileid, 'duration', duration) + self.r.hset(self.prefix + self.separator + str(profileid), 'duration', duration) # When a new profiled is created assign threat level = 0 and confidence = 0.05 - self.r.hset(profileid, 'threat_level', 0) - self.r.hset(profileid, 'confidence', 0.05) + self.r.hset(self.prefix + self.separator + str(profileid), 'threat_level', 0) + self.r.hset(self.prefix + self.separator + str(profileid), 'confidence', 0.05) # The IP of the profile should also be added as a new IP we know about. - ip = profileid.split(self.separator)[1] + ip = profileid.split(self.separator)[-1] # If the ip is new add it to the list of ips self.setNewIP(ip) # Publish that we have a new profile @@ -391,34 +397,34 @@ def was_ip_seen_in_connlog_before(self, ip) -> bool: # but not in conn.log yet # if the ip's not in the following key, then its the first flow seen of this ip - return self.r.sismember("srcips_seen_in_connlog", ip) + return self.r.sismember(self.prefix + self.separator +"srcips_seen_in_connlog", ip) def mark_srcip_as_seen_in_connlog(self, ip): """ Marks the given ip as seen in conn.log if an ip is not present in this set, it means we may have seen it but not in conn.log """ - self.r.sadd("srcips_seen_in_connlog", ip) + self.r.sadd(self.prefix + self.separator +"srcips_seen_in_connlog", ip) def add_user_agent_to_profile(self, profileid, user_agent: dict): """ Used to associate this profile with it's used user_agent :param user_agent: dict containing user_agent, os_type , os_name and agent_name """ - self.r.hset(profileid, 'User-agent', user_agent) + self.r.hset(self.prefix + self.separator + str(profileid), 'User-agent', user_agent) def add_all_user_agent_to_profile(self, profileid, user_agent: str): """ Used to keep history of past user agents of profile :param user_agent: str of user_agent """ - if not self.r.hexists(profileid ,'past_user_agents'): - self.r.hset(profileid, 'past_user_agents', json.dumps([user_agent])) + if not self.r.hexists(self.prefix + self.separator + str(profileid) ,'past_user_agents'): + self.r.hset(self.prefix + self.separator + str(profileid), 'past_user_agents', json.dumps([user_agent])) else: - user_agents = json.loads(self.r.hget(profileid, 'past_user_agents')) + user_agents = json.loads(self.r.hget(self.prefix + self.separator + str(profileid), 'past_user_agents')) if user_agent not in user_agents: user_agents.append(user_agent) - self.r.hset(profileid, 'past_user_agents', json.dumps(user_agents)) + self.r.hset(self.prefix + self.separator + str(profileid), 'past_user_agents', json.dumps(user_agents)) def add_software_to_profile( self, profileid, software, version_major, version_minor, uid @@ -442,10 +448,10 @@ def add_software_to_profile( return # add this new sw to the list of softwares this profile is using cached_sw.update(sw_dict) - self.r.hset(profileid, 'used_software', json.dumps(cached_sw)) + self.r.hset(self.prefix + self.separator + profileid, 'used_software', json.dumps(cached_sw)) else: # first time for this profile to use a software - self.r.hset(profileid, 'used_software', json.dumps(sw_dict)) + self.r.hset(self.prefix + self.separator + profileid, 'used_software', json.dumps(sw_dict)) def get_software_from_profile(self, profileid): """ @@ -454,7 +460,7 @@ def get_software_from_profile(self, profileid): if not profileid: return False - if used_software := self.r.hmget(profileid, 'used_software')[0]: + if used_software := self.r.hmget(self.prefix + self.separator + str(profileid), 'used_software')[0]: used_software = json.loads(used_software) return used_software @@ -468,7 +474,7 @@ def get_user_agent_from_profile(self, profileid) -> str: # profileid is None if we're dealing with a profile # outside of home_network when this param is given return False - if user_agent := self.r.hmget(profileid, 'User-agent')[0]: + if user_agent := self.r.hmget(self.prefix + self.separator + str(profileid), 'User-agent')[0]: # user agents may be OpenSSH_8.6 , no need to deserialize them if '{' in user_agent: user_agent = json.loads(user_agent) @@ -484,13 +490,13 @@ def mark_profile_as_dhcp(self, profileid): return False # returns a list of dhcp if the profile is in the db - profile_in_db = self.r.hmget(profileid, 'dhcp') + profile_in_db = self.r.hmget(self.prefix + self.separator + str(profileid), 'dhcp') if not profile_in_db: return False is_dhcp_set = profile_in_db[0] # check if it's already marked as dhcp if not is_dhcp_set: - self.r.hset(profileid, 'dhcp', 'true') + self.r.hset(self.prefix + self.separator + str(profileid), 'dhcp', 'true') def mark_profile_as_gateway(self, profileid): @@ -502,14 +508,14 @@ def mark_profile_as_gateway(self, profileid): # outside of home_network when this param is given return False - self.r.hset(profileid, 'gateway', 'true') + self.r.hset(self.prefix + self.separator + str(profileid), 'gateway', 'true') def set_ipv6_of_profile(self, profileid, ip: list): - self.r.hset(profileid, 'IPv6', json.dumps(ip)) + self.r.hset(self.prefix + self.separator + str(profileid), 'IPv6', json.dumps(ip)) def set_ipv4_of_profile(self, profileid, ip): - self.r.hset(profileid, 'IPv4', json.dumps([ip])) + self.r.hset(self.prefix + self.separator + str(profileid), 'IPv4', json.dumps([ip])) def is_gw_mac(self, MAC_info, ip) -> bool: """ @@ -545,7 +551,7 @@ def get_IP_of_MAC(self, MAC): """ Returns the IP associated with the given MAC in our database """ - return self.r.hget('MAC', MAC) + return self.r.hget(self.prefix + self.separator + 'MAC', MAC) def add_mac_addr_to_profile(self, profileid, MAC_info): """ @@ -563,7 +569,7 @@ def add_mac_addr_to_profile(self, profileid, MAC_info): if '0.0.0.0' in profileid: return False - incoming_ip = profileid.split('_')[1] + incoming_ip = profileid.split(self.separator)[-1] # sometimes we create profiles with the mac address. # don't save that in MAC hash @@ -577,13 +583,13 @@ def add_mac_addr_to_profile(self, profileid, MAC_info): # we're trying to assign the gw mac to an ip that isn't the gateway's return False # get the ips that belong to this mac - cached_ip = self.r.hmget('MAC', MAC_info['MAC'])[0] + cached_ip = self.r.hmget(self.prefix + self.separator + 'MAC', MAC_info['MAC'])[0] if not cached_ip: # no mac info stored for profileid ip = json.dumps([incoming_ip]) - self.r.hset('MAC', MAC_info['MAC'], ip) + self.r.hset(self.prefix + self.separator + 'MAC', MAC_info['MAC'], ip) # Add the MAC addr, hostname and vendor to this profile - self.r.hset(profileid, 'MAC', json.dumps(MAC_info)) + self.r.hset(self.prefix + self.separator + str(profileid), 'MAC', json.dumps(MAC_info)) else: # we found another profile that has the same mac as this one # incoming_ip = profileid.split('_')[1] @@ -611,7 +617,7 @@ def add_mac_addr_to_profile(self, profileid, MAC_info): # If 2 IPV6 are claiming to have the same MAC it's fine # a computer is allowed to have many ipv6 # add this found ipv6 to the list of ipv6 of the incoming ip(profileid) - ipv6: str = self.r.hmget(profileid, 'IPv6')[0] + ipv6: str = self.r.hmget(self.prefix + self.separator + str(profileid), 'IPv6')[0] if not ipv6: ipv6 = [found_ip] else: @@ -622,7 +628,7 @@ def add_mac_addr_to_profile(self, profileid, MAC_info): self.set_ipv6_of_profile(profileid, ipv6) # add this incoming ipv6(profileid) to the list of ipv6 of the found ip - ipv6: str = self.r.hmget(f'profile_{found_ip}', 'IPv6')[0] + ipv6: str = self.r.hmget(f'{self.prefix}_profile_{found_ip}', 'IPv6')[0] if not ipv6: ipv6 = [incoming_ip] else: @@ -642,7 +648,7 @@ def add_mac_addr_to_profile(self, profileid, MAC_info): # add the incoming ip to the list of ips that belong to this mac cached_ips.add(incoming_ip) cached_ips = json.dumps(list(cached_ips)) - self.r.hset('MAC', MAC_info['MAC'], cached_ips) + self.r.hset(self.prefix + self.separator + 'MAC', MAC_info['MAC'], cached_ips) return True @@ -654,7 +660,7 @@ def get_mac_addr_from_profile(self, profileid) -> str: # profileid is None if we're dealing with a profile # outside of home_network when this param is given return False - if MAC_info := self.r.hget(profileid, 'MAC'): + if MAC_info := self.r.hget(self.prefix + self.separator + str(profileid), 'MAC'): return json.loads(MAC_info)['MAC'] else: return MAC_info @@ -667,7 +673,7 @@ def get_mac_vendor_from_profile(self, profileid) -> str: # profileid is None if we're dealing with a profile # outside of home_network when this param is given return False - if MAC_info := self.r.hget(profileid, 'MAC'): + if MAC_info := self.r.hget(self.prefix + self.separator + str(profileid), 'MAC'): return json.loads(MAC_info)['Vendor'] else: return MAC_info @@ -680,7 +686,7 @@ def get_hostname_from_profile(self, profileid) -> str: # profileid is None if we're dealing with a profile # outside of home_network when this param is given return False - if MAC_info := self.r.hget(profileid, 'MAC'): + if MAC_info := self.r.hget(self.prefix + self.separator + str(profileid), 'MAC'): return json.loads(MAC_info).get('host_name', False) else: return MAC_info @@ -689,13 +695,13 @@ def get_ipv4_from_profile(self, profileid) -> str: """ Returns ipv4 about a certain profile or None """ - return self.r.hmget(profileid, 'IPv4')[0] if profileid else False + return self.r.hmget(self.prefix + self.separator + str(profileid), 'IPv4')[0] if profileid else False def get_ipv6_from_profile(self, profileid) -> str: """ Returns ipv6 about a certain profile or None """ - return self.r.hmget(profileid, 'IPv6')[0] if profileid else False + return self.r.hmget(self.prefix + self.separator + str(profileid), 'IPv6')[0] if profileid else False def get_the_other_ip_version(self, profileid): """ @@ -706,7 +712,7 @@ def get_the_other_ip_version(self, profileid): # profileid is None if we're dealing with a profile # outside of home_network when this param is given return False - srcip = profileid.split('_')[1] + srcip = profileid.split(self.separator)[-1] ip = False if validators.ipv4(srcip): ip = self.get_ipv6_from_profile(profileid) @@ -718,8 +724,8 @@ def get_the_other_ip_version(self, profileid): def getProfileIdFromIP(self, daddr_as_obj): """Receive an IP and we want the profileid""" try: - profileid = f'profile{self.separator}{str(daddr_as_obj)}' - if self.r.sismember('profiles', profileid): + profileid = f'{self.prefix}_profile{self.separator}{str(daddr_as_obj)}' + if self.r.sismember(self.prefix + self.separator + 'profiles', profileid): return profileid return False except redis.exceptions.ResponseError as inst: @@ -731,7 +737,7 @@ def getProfileIdFromIP(self, daddr_as_obj): def getProfiles(self): """Get a list of all the profiles""" - profiles = self.r.smembers('profiles') + profiles = self.r.smembers(self.prefix + self.separator + 'profiles') return profiles if profiles != set() else {} @@ -741,7 +747,7 @@ def getTWsfromProfile(self, profileid): Returns a list of tuples (twid, ts) or an empty list """ return ( - self.r.zrange(f'tws{profileid}', 0, -1, withscores=True) + self.r.zrange(self.prefix + self.separator + f'tws{profileid}', 0, -1, withscores=True) if profileid else False ) @@ -756,13 +762,13 @@ def getSrcIPsfromProfileTW(self, profileid, twid): """ Get the src ip for a specific TW for a specific profileid """ - return self.r.hget(profileid + self.separator + twid, 'SrcIPs') + return self.r.hget(self.prefix + self.separator + profileid + self.separator + twid, 'SrcIPs') def getDstIPsfromProfileTW(self, profileid, twid): """ Get the dst ip for a specific TW for a specific profileid """ - return self.r.hget(profileid + self.separator + twid, 'DstIPs') + return self.r.hget(self.prefix + self.separator + profileid + self.separator + twid, 'DstIPs') def getT2ForProfileTW(self, profileid, twid, tupleid, tuple_key: str): """ @@ -770,7 +776,7 @@ def getT2ForProfileTW(self, profileid, twid, tupleid, tuple_key: str): """ try: hash_id = profileid + self.separator + twid - data = self.r.hget(hash_id, tuple_key) + data = self.r.hget(self.prefix + self.separator + str(hash_id), tuple_key) if not data: return False, False data = json.loads(data) @@ -793,16 +799,16 @@ def getT2ForProfileTW(self, profileid, twid, tupleid, tuple_key: str): def has_profile(self, profileid): """Check if we have the given profile""" - return self.r.sismember('profiles', profileid) if profileid else False + return self.r.sismember(self.prefix + self.separator + 'profiles', profileid) if profileid else False def getProfilesLen(self): """Return the amount of profiles. Redis should be faster than python to do this count""" - return self.r.scard('profiles') + return self.r.scard(self.prefix + self.separator + 'profiles') def getLastTWforProfile(self, profileid): """Return the last TW id and the time for the given profile id""" return ( - self.r.zrange(f'tws{profileid}', -1, -1, withscores=True) + self.r.zrange(self.prefix + self.separator + f'tws{profileid}', -1, -1, withscores=True) if profileid else False ) @@ -810,7 +816,7 @@ def getLastTWforProfile(self, profileid): def getFirstTWforProfile(self, profileid): """Return the first TW id and the time for the given profile id""" return ( - self.r.zrange(f'tws{profileid}', 0, 0, withscores=True) + self.r.zrange(self.prefix + self.separator + f'tws{profileid}', 0, 0, withscores=True) if profileid else False ) @@ -825,7 +831,7 @@ def getTWofTime(self, profileid, time): # [-1] so we bring the last TW that matched this time. try: data = self.r.zrangebyscore( - f'tws{profileid}', + self.prefix + self.separator + f'tws{profileid}', float('-inf'), float(time), withscores=True, @@ -836,7 +842,7 @@ def getTWofTime(self, profileid, time): except IndexError: # We dont have any last tw? data = self.r.zrangebyscore( - f'tws{profileid}', + self.prefix + self.separator + f'tws{profileid}', 0, float(time), withscores=True, @@ -867,7 +873,7 @@ def addNewOlderTW(self, profileid, startoftw): pass # Add the new TW to the index of TW data = {str(twid): float(startoftw)} - self.r.zadd(f'tws{profileid}', data) + self.r.zadd(self.prefix + self.separator + f'tws{profileid}', data) self.outputqueue.put( f'04|database|[DB]: Created and added to DB the new older TW with id {twid}. Time: {startoftw} ' ) @@ -904,7 +910,7 @@ def addNewTW(self, profileid, startoftw): twid = 'timewindow1' # Add the new TW to the index of TW data = {twid: float(startoftw)} - self.r.zadd(f'tws{profileid}', data) + self.r.zadd(self.prefix + self.separator + f'tws{profileid}', data) self.outputqueue.put( f'04|database|[DB]: Created and added to DB for profile {profileid} on TW with id {twid}. Time: {startoftw} ' ) @@ -923,16 +929,16 @@ def getTimeTW(self, profileid, twid): """Return the time when this TW in this profile was created""" # Get all the TW for this profile # We need to encode it to 'search' because the data in the sorted set is encoded - return self.r.zscore(f'tws{profileid}', twid.encode('utf-8')) + return self.r.zscore(self.prefix + self.separator + f'tws{profileid}', twid.encode('utf-8')) def getAmountTW(self, profileid): """Return the number of tws for this profile id""" - return self.r.zcard(f'tws{profileid}') if profileid else False + return self.r.zcard(self.prefix + self.separator + f'tws{profileid}') if profileid else False def getModifiedTWSinceTime(self, time): """Return the list of modified timewindows since a certain time""" data = self.r.zrangebyscore( - 'ModifiedTW', time, float('+inf'), withscores=True + self.prefix + self.separator + 'ModifiedTW', time, float('+inf'), withscores=True ) return data or [] @@ -954,16 +960,16 @@ def getModifiedProfilesSince(self, time): def getModifiedTW(self): """Return all the list of modified tw""" - data = self.r.zrange('ModifiedTW', 0, -1, withscores=True) + data = self.r.zrange(self.prefix + self.separator + 'ModifiedTW', 0, -1, withscores=True) return data or [] def wasProfileTWModified(self, profileid, twid): """Retrieve from the db if this TW of this profile was modified""" - data = self.r.zrank('ModifiedTW', profileid + self.separator + twid) + data = self.r.zrank(self.prefix + self.separator + 'ModifiedTW', profileid + self.separator + twid) return bool(data) def setSlipsInternalTime(self, timestamp): - self.r.set('slips_internal_time', timestamp) + self.r.set(self.prefix + self.separator + 'slips_internal_time', timestamp) def get_data_from_profile_tw(self, hash_key: str, key_name: str): try: @@ -971,7 +977,7 @@ def get_data_from_profile_tw(self, hash_key: str, key_name: str): key_name = [Src,Dst] + [Port,IP] + [Client,Server] + [TCP,UDP, ICMP, ICMP6] + [Established, NotEstablihed] Example: key_name = 'SrcPortClientTCPEstablished' """ - data = self.r.hget(hash_key, key_name) + data = self.r.hget(self.prefix + self.separator + str(hash_key), key_name) value = {} if data: portdata = json.loads(data) @@ -986,11 +992,11 @@ def get_data_from_profile_tw(self, hash_key: str, key_name: str): def getOutTuplesfromProfileTW(self, profileid, twid): """Get the out tuples""" - return self.r.hget(profileid + self.separator + twid, 'OutTuples') + return self.r.hget(self.prefix + self.separator + profileid + self.separator + twid, 'OutTuples') def getInTuplesfromProfileTW(self, profileid, twid): """Get the in tuples""" - return self.r.hget(profileid + self.separator + twid, 'InTuples') + return self.r.hget(self.prefix + self.separator + profileid + self.separator + twid, 'InTuples') def getFieldSeparator(self): """Return the field separator""" @@ -1001,7 +1007,7 @@ def get_dhcp_flows(self, profileid, twid) -> dict: """ returns a dict of dhcp flows that happaened in this profileid and twid """ - if flows := self.r.hget('DHCP_flows', f'{profileid}_{twid}'): + if flows := self.r.hget(self.prefix + self.separator + 'DHCP_flows', f'{profileid}_{twid}'): return json.loads(flows) @@ -1013,9 +1019,9 @@ def set_dhcp_flow(self, profileid, twid, requested_addr, uid): if cached_flows := self.get_dhcp_flows(profileid, twid): # we already have flows in this twid, update them cached_flows.update(flow) - self.r.hset('DHCP_flows', f'{profileid}_{twid}', json.dumps(cached_flows)) + self.r.hset(self.prefix + self.separator + 'DHCP_flows', f'{profileid}_{twid}', json.dumps(cached_flows)) else: - self.r.hset('DHCP_flows', f'{profileid}_{twid}', json.dumps(flow)) + self.r.hset(self.prefix + self.separator + 'DHCP_flows', f'{profileid}_{twid}', json.dumps(flow)) def update_threat_level(self, profileid, threat_level: str, confidence): @@ -1024,12 +1030,12 @@ def update_threat_level(self, profileid, threat_level: str, confidence): :param threat_level: available options are 'low', 'medium' 'critical' etc """ - self.r.hset(profileid, 'threat_level', threat_level) + self.r.hset(self.prefix + self.separator + profileid, 'threat_level', threat_level) now = time.time() now = utils.convert_format(now, utils.alerts_format) # keep track of old threat levels confidence = f'confidence: {confidence}' - past_threat_levels = self.r.hget(profileid, 'past_threat_levels') + past_threat_levels = self.r.hget(self.prefix + self.separator + profileid, 'past_threat_levels') # this is what we'll be storing in the db, tl, ts, and confidence threat_level_data = (threat_level, now, confidence) if past_threat_levels: @@ -1052,7 +1058,7 @@ def update_threat_level(self, profileid, threat_level: str, confidence): # threat_levels_update_time = [now] past_threat_levels = json.dumps(past_threat_levels) - self.r.hset(profileid, 'past_threat_levels', past_threat_levels) + self.r.hset(self.prefix + self.separator + profileid, 'past_threat_levels', past_threat_levels) # set the score and confidence of the given ip in the db when it causes an evidence # these 2 values will be needed when sharing with peers @@ -1094,7 +1100,7 @@ def set_evidence_causing_alert(self, profileid, twid, alert_ID, evidence_IDs: li profileid_twid_alerts = json.dumps(alert) - self.r.hset(f'{profileid}{self.separator}{twid}', 'alerts', profileid_twid_alerts) + self.r.hset(self.prefix + self.separator + f'{profileid}{self.separator}{twid}', 'alerts', profileid_twid_alerts) # the structure of alerts key is # alerts { @@ -1106,7 +1112,7 @@ def set_evidence_causing_alert(self, profileid, twid, alert_ID, evidence_IDs: li # } # } - profile_alerts = self.r.hget('alerts', profileid) + profile_alerts = self.r.hget(self.prefix + self.separator + 'alerts', profileid) # alert ids look like this profile_192.168.131.2_timewindow1_92a3b9c2-330b-47ab-b73e-c5380af90439 alert_hash = alert_ID.split('_')[-1] alert = { @@ -1117,7 +1123,7 @@ def set_evidence_causing_alert(self, profileid, twid, alert_ID, evidence_IDs: li if not profile_alerts: # first alert in this profile alert = json.dumps(alert) - self.r.hset('alerts', profileid, alert) + self.r.hset(self.prefix + self.separator + 'alerts', profileid, alert) return # the format of this dict is {twid1: {alert_hash: [evidence_IDs]}, @@ -1135,7 +1141,7 @@ def set_evidence_causing_alert(self, profileid, twid, alert_ID, evidence_IDs: li profile_alerts[twid] = twid_alerts profile_alerts = json.dumps(profile_alerts) - self.r.hset('alerts', profileid, profile_alerts) + self.r.hset(self.prefix + self.separator + 'alerts', profileid, profile_alerts) def get_timewindow(self, flowtime, profileid): @@ -1261,7 +1267,7 @@ def get_profileid_twid_alerts(self, profileid, twid) -> dict: The format for the returned dict is {profile123_twid1_: [ev_uuid1, ev_uuid2, ev_uuid3]} """ - alerts = self.r.hget(f'{profileid}{self.separator}{twid}', 'alerts') + alerts = self.r.hget(self.prefix + self.separator + f'{profileid}{self.separator}{twid}', 'alerts') if not alerts: return {} alerts = json.loads(alerts) @@ -1273,7 +1279,7 @@ def get_evidence_causing_alert(self, profileid, twid, alert_ID) -> list: :param alert_ID: ID of alert to export to warden server for example profile_10.0.2.15_timewindow1_4e4e4774-cdd7-4e10-93a3-e764f73af621 """ - if alerts := self.r.hget(f'{profileid}{self.separator}{twid}', 'alerts'): + if alerts := self.r.hget(self.prefix + self.separator + f'{profileid}{self.separator}{twid}', 'alerts'): alerts = json.loads(alerts) return alerts.get(alert_ID, False) return False @@ -1308,10 +1314,10 @@ def is_detection_disabled(self, evidence_type: str): def set_flow_causing_evidence(self, uids: list, evidence_ID): - self.r.hset("flows_causing_evidence", evidence_ID, json.dumps(uids)) + self.r.hset(self.prefix + self.separator + "flows_causing_evidence", evidence_ID, json.dumps(uids)) def get_flows_causing_evidence(self, evidence_ID) -> list: - uids = self.r.hget("flows_causing_evidence", evidence_ID) + uids = self.r.hget(self.prefix + self.separator + "flows_causing_evidence", evidence_ID) return json.loads(uids) if uids else [] @@ -1426,10 +1432,10 @@ def setEvidence( # Set evidence in the database. current_evidence = json.dumps(current_evidence) self.r.hset( - profileid + self.separator + twid, 'Evidence', current_evidence + self.prefix + self.separator + profileid + self.separator + twid, 'Evidence', current_evidence ) - self.r.hset(f'evidence{profileid}', twid, current_evidence) + self.r.hset(self.prefix + self.separator + f'evidence{profileid}', twid, current_evidence) # This is done to ignore repetition of the same evidence sent. # note that publishing HAS TO be done after updating the 'Evidence' keys @@ -1452,10 +1458,10 @@ def mark_evidence_as_processed(self, evidence_ID): """ If an evidence was processed by the evidenceprocess, mark it in the db """ - self.r.sadd('processed_evidence', evidence_ID) + self.r.sadd(self.prefix + self.separator + 'processed_evidence', evidence_ID) def is_evidence_processed(self, evidence_ID): - return self.r.sismember('processed_evidence', evidence_ID) + return self.r.sismember(self.prefix + self.separator + 'processed_evidence', evidence_ID) def store_tranco_whitelisted_domain(self, domain): """ @@ -1475,7 +1481,7 @@ def set_evidence_for_profileid(self, evidence): Set evidence for the profile in the same format as json in alerts.json """ evidence = json.dumps(evidence) - self.r.sadd('Evidence', evidence) + self.r.sadd(self.prefix + self.separator + 'Evidence', evidence) def deleteEvidence(self, profileid, twid, evidence_ID: str): @@ -1489,13 +1495,13 @@ def deleteEvidence(self, profileid, twid, evidence_ID: str): current_evidence.pop(evidence_ID, None) current_evidence_json = json.dumps(current_evidence) self.r.hset( - profileid + self.separator + twid, + self.prefix + self.separator + profileid + self.separator + twid, 'Evidence', current_evidence_json, ) - self.r.hset(f'evidence{profileid}', twid, current_evidence_json) + self.r.hset(self.prefix + self.separator + f'evidence{profileid}', twid, current_evidence_json) # 2. delete evidence from 'alerts' key - profile_alerts = self.r.hget('alerts', profileid) + profile_alerts = self.r.hget(self.prefix + self.separator + 'alerts', profileid) if not profile_alerts: # this means that this evidence wasn't a part of an alert # give redis time to the save the changes before calling this function again @@ -1538,13 +1544,13 @@ def cache_whitelisted_evidence_ID(self, evidence_ID:str): """ # without this function, slips gets the stored evidence id from the db, # before deleteEvidence is called, so we need to keep track of whitelisted evidence ids - self.r.sadd('whitelisted_evidence', evidence_ID) + self.r.sadd(self.prefix + self.separator + 'whitelisted_evidence', evidence_ID) def is_whitelisted_evidence(self, evidence_ID): """ Check if we have the evidence ID as whitelisted in the db to avoid showing it in alerts """ - return self.r.sismember('whitelisted_evidence', evidence_ID) + return self.r.sismember(self.prefix + self.separator + 'whitelisted_evidence', evidence_ID) def remove_whitelisted_evidence(self, all_evidence:str) -> str: """ @@ -1562,7 +1568,7 @@ def remove_whitelisted_evidence(self, all_evidence:str) -> str: def getEvidenceForTW(self, profileid, twid): """Get the evidence for this TW for this Profile""" - evidence = self.r.hget(profileid + self.separator + twid, 'Evidence') + evidence = self.r.hget(self.prefix + self.separator + profileid + self.separator + twid, 'Evidence') if evidence: evidence = self.remove_whitelisted_evidence(evidence) return evidence @@ -1585,7 +1591,7 @@ def set_first_stage_ensembling_label_to_flow( data['1_ensembling_label'] = ensembling_label data = json.dumps(data) self.r.hset( - profileid + self.separator + twid + self.separator + 'flows', + self.prefix + self.separator + profileid + self.separator + twid + self.separator + 'flows', uid, data, ) @@ -1594,11 +1600,11 @@ def set_growing_zeek_dir(self): """ Mark a dir as growing so it can be treated like the zeek logs generated by an interface """ - self.r.set('growing_zeek_dir', 'yes') + self.r.set(self.prefix + self.separator + 'growing_zeek_dir', 'yes') def is_growing_zeek_dir(self): """ Did slips mark the given dir as growing?""" - return 'yes' in str(self.r.get('growing_zeek_dir')) + return 'yes' in str(self.r.get(self.prefix + self.separator + 'growing_zeek_dir')) def set_module_label_to_flow( self, profileid, twid, uid, module_name, module_label @@ -1613,7 +1619,7 @@ def set_module_label_to_flow( data['module_labels'][module_name] = module_label data = json.dumps(data) self.r.hset( - profileid + self.separator + twid + self.separator + 'flows', + self.prefix + self.separator + profileid + self.separator + twid + self.separator + 'flows', uid, data, ) @@ -1635,15 +1641,15 @@ def markProfileTWAsBlocked(self, profileid, twid): """Add this profile and tw to the list of blocked""" tws = self.getBlockedProfTW(profileid) tws.append(twid) - self.r.hset('BlockedProfTW', profileid, json.dumps(tws)) + self.r.hset(self.prefix + self.separator + 'BlockedProfTW', profileid, json.dumps(tws)) def getAllBlockedProfTW(self): """Return all the list of blocked tws""" - return self.r.hgetall('BlockedProfTW') + return self.r.hgetall(self.prefix + self.separator + 'BlockedProfTW') def getBlockedProfTW(self, profileid): """Return all the list of blocked tws""" - if tws := self.r.hget('BlockedProfTW', profileid): + if tws := self.r.hget(self.prefix + self.separator + 'BlockedProfTW', profileid): return json.loads(tws) return [] @@ -1695,7 +1701,7 @@ def get_multiaddr(self): """ this is can only be called when p2p is enabled, this value is set by p2p pigeon """ - return self.r.get('multiAddress') + return self.r.get(self.prefix + self.separator + 'multiAddress') def getURLData(self, url): """ @@ -1772,7 +1778,7 @@ def subscribe(self, channel: str, ignore_subscribe_messages=False): self.pubsub = self.r.pubsub() self.pubsub.subscribe( - channel, ignore_subscribe_messages=ignore_subscribe_messages + self.prefix + self.separator + str(channel), ignore_subscribe_messages=ignore_subscribe_messages ) return self.pubsub @@ -1784,14 +1790,14 @@ def publish_stop(self): all_channels_list = self.r.pubsub_channels() self.print('Sending the stop signal to all listeners', 0, 3) for channel in all_channels_list: - self.r.publish(channel, 'stop_process') + self.r.publish(self.prefix + self.separator + str(channel), 'stop_process') def get_all_flows_in_profileid_twid(self, profileid, twid): """ Return a list of all the flows in this profileid and twid """ if data := self.r.hgetall( - profileid + self.separator + twid + self.separator + 'flows' + self.prefix + self.separator + profileid + self.separator + twid + self.separator + 'flows' ): return data @@ -1852,13 +1858,13 @@ def get_labels(self): Return the amount of each label so far in the DB Used to know how many labels are available during training """ - return self.r.zrange('labels', 0, -1, withscores=True) + return self.r.zrange(self.prefix + self.separator + 'labels', 0, -1, withscores=True) def get_altflow_from_uid(self, profileid, twid, uid): """ Given a uid, get the alternative flow realted to it """ return ( self.r.hget( - profileid + self.separator + twid + self.separator + 'altflows', + self.prefix + self.separator + profileid + self.separator + twid + self.separator + 'altflows', uid, ) if profileid @@ -1871,9 +1877,9 @@ def add_timeline_line(self, profileid, twid, data, timestamp): # profileid is None if we're dealing with a profile # outside of home_network when this param is given return - self.print(f'Adding timeline for {profileid}, {twid}: {data}', 3, 0) + self.print(f'Adding timeline for {self.prefix}, {profileid}, {twid}: {data}', 3, 0) key = str( - profileid + self.separator + twid + self.separator + 'timeline' + self.prefix + self.separator + profileid + self.separator + twid + self.separator + 'timeline' ) data = json.dumps(data) mapping = {data: timestamp} @@ -1890,7 +1896,7 @@ def get_timeline_last_lines( # outside of home_network when this param is given return [], [] key = str( - profileid + self.separator + twid + self.separator + 'timeline' + self.prefix + self.separator + profileid + self.separator + twid + self.separator + 'timeline' ) # The the amount of lines in this list last_index = self.r.zcard(key) @@ -1917,11 +1923,11 @@ def set_ftp_port(self, port): """ Stores the used ftp port in our main db (not the cache like set_port_info) """ - self.r.lpush('used_ftp_ports', str(port)) + self.r.lpush(self.prefix + self.separator + 'used_ftp_ports', str(port)) def is_ftp_port(self, port): # get all used ftp ports - used_ftp_ports = self.r.lrange('used_ftp_ports', 0, -1) + used_ftp_ports = self.r.lrange(self.prefix + self.separator + 'used_ftp_ports', 0, -1) # check if the given port is used as ftp port return str(port) in used_ftp_ports @@ -1946,20 +1952,20 @@ def get_organization_of_port(self, portproto: str): def add_zeek_file(self, filename): """Add an entry to the list of zeek files""" - self.r.sadd('zeekfiles', filename) + self.r.sadd(self.prefix + self.separator + 'zeekfiles', filename) def get_all_zeek_file(self): """Return all entries from the list of zeek files""" - return self.r.smembers('zeekfiles') + return self.r.smembers(self.prefix + self.separator + 'zeekfiles') def get_gateway_ip(self): - return self.r.hget('default_gateway', 'IP') + return self.r.hget(self.prefix + self.separator + 'default_gateway', 'IP') def get_gateway_MAC(self): - return self.r.hget('default_gateway', 'MAC') + return self.r.hget(self.prefix + self.separator + 'default_gateway', 'MAC') def get_gateway_MAC_Vendor(self): - return self.r.hget('default_gateway', 'Vendor') + return self.r.hget(self.prefix + self.separator + 'default_gateway', 'Vendor') def set_default_gateway(self, address_type:str, address:str): """ @@ -1972,7 +1978,7 @@ def set_default_gateway(self, address_type:str, address:str): or address_type == 'MAC' and not self.get_gateway_MAC() or address_type == 'Vendor' and not self.get_gateway_MAC_Vendor() ): - self.r.hset('default_gateway', address_type, address) + self.r.hset(self.prefix + self.separator + 'default_gateway', address_type, address) def get_ssl_info(self, sha1): info = self.rcache.hmget('IoC_SSL', sha1)[0] @@ -1991,7 +1997,7 @@ def set_profile_module_label(self, profileid, module, label): data = self.get_profile_modules_labels(profileid) data[module] = label data = json.dumps(data) - self.r.hset(profileid, 'modules_labels', data) + self.r.hset(self.prefix + self.separator + str(profileid), 'modules_labels', data) def get_profile_modules_labels(self, profileid): """ @@ -2001,7 +2007,7 @@ def get_profile_modules_labels(self, profileid): # profileid is None if we're dealing with a profile # outside of home_network when this param is given return {} - data = self.r.hget(profileid, 'modules_labels') + data = self.r.hget(self.prefix + self.separator + str(profileid), 'modules_labels') data = json.loads(data) if data else {} return data @@ -2113,7 +2119,7 @@ def set_malicious_ip(self, ip, profileid, twid): ) # add key-pair to the dict if does not exist data = json.dumps(ip_profileid_twid) - self.r.hset('MaliciousIPs', ip, data) + self.r.hset(self.prefix + self.separator + 'MaliciousIPs', ip, data) def set_malicious_domain(self, domain, profileid, twid): """ @@ -2141,14 +2147,14 @@ def set_malicious_domain(self, domain, profileid, twid): ) # add key-pair to the dict if does not exist data = json.dumps(domain_profiled_twid) - self.r.hset('MaliciousDomains', domain, data) + self.r.hset(self.prefix + self.separator + 'MaliciousDomains', domain, data) def get_malicious_ip(self, ip): """ Return malicious IP and its list of presence in the traffic (profileid, twid) """ - data = self.r.hget('MaliciousIPs', ip) + data = self.r.hget(self.prefix + self.separator + 'MaliciousIPs', ip) data = json.loads(data) if data else {} return data @@ -2157,7 +2163,7 @@ def get_malicious_domain(self, domain): Return malicious domain and its list of presence in the traffic (profileid, twid) """ - data = self.r.hget('MaliciousDomains', domain) + data = self.r.hget(self.prefix + self.separator + 'MaliciousDomains', domain) data = json.loads(data) if data else {} return data @@ -2165,11 +2171,11 @@ def get_domain_resolution(self, domain): """ Returns the IPs resolved by this domain """ - ips = self.r.hget("DomainsResolved", domain) + ips = self.r.hget(self.prefix + self.separator + "DomainsResolved", domain) return json.loads(ips) if ips else [] def get_all_dns_resolutions(self): - dns_resolutions = self.r.hgetall('DNSresolution') + dns_resolutions = self.r.hgetall(self.prefix + self.separator +'DNSresolution') return dns_resolutions or [] @@ -2222,7 +2228,7 @@ def getReconnectionsForTW(self, profileid, twid): # profileid is None if we're dealing with a profile # outside of home_network when this param is given return False - data = self.r.hget(profileid + self.separator + twid, 'Reconnections') + data = self.r.hget(self.prefix + self.separator + profileid + self.separator + twid, 'Reconnections') data = json.loads(data) if data else {} return data @@ -2230,7 +2236,7 @@ def setReconnections(self, profileid, twid, data): """Set the reconnections for this TW for this Profile""" data = json.dumps(data) self.r.hset( - profileid + self.separator + twid, 'Reconnections', str(data) + self.prefix + self.separator + profileid + self.separator + twid, 'Reconnections', str(data) ) @@ -2257,15 +2263,15 @@ def is_domain_malicious(self, domain: str) -> tuple: def get_host_ip(self): """Get the IP addresses of the host from a db. There can be more than one""" - return self.r.smembers('hostIP') + return self.r.smembers(self.prefix + self.separator + 'hostIP') def set_host_ip(self, ip): """Store the IP address of the host in a db. There can be more than one""" - self.r.sadd('hostIP', ip) + self.r.sadd(self.prefix + self.separator + 'hostIP', ip) def is_profile_malicious(self, profileid: str) -> str: - return self.r.hget(profileid, 'labeled_as_malicious') if profileid else False + return self.r.hget(self.prefix + self.separator + str(profileid), 'labeled_as_malicious') if profileid else False def set_TI_file_info(self, file, data): """ @@ -2364,11 +2370,11 @@ def store_process_PID(self, process, pid): :param pid: int :param process: str """ - self.r.hset('PIDs', process, pid) + self.r.hset(self.prefix + self.separator + 'PIDs', process, pid) def get_PIDs(self): """returns a dict with module names as keys and pids as values""" - return self.r.hgetall('PIDs') + return self.r.hgetall(self.prefix + self.separator + 'PIDs') def set_org_info(self, org, org_info, info_type): """ @@ -2408,18 +2414,18 @@ def set_whitelist(self, type, whitelist_dict): :param type: supporte types are IPs, domains and organizations :param whitelist_dict: the dict of IPs, domains or orgs to store """ - self.r.hset('whitelist', type, json.dumps(whitelist_dict)) + self.r.hset(self.prefix + self.separator + 'whitelist', type, json.dumps(whitelist_dict)) def get_all_whitelist(self): """Return dict of 3 keys: IPs, domains, organizations or mac""" - return self.r.hgetall('whitelist') + return self.r.hgetall(self.prefix + self.separator + 'whitelist') def get_whitelist(self, key): """ Whitelist supports different keys like : IPs domains and organizations this function is used to check if we have any of the above keys whitelisted """ - if whitelist := self.r.hget('whitelist', key): + if whitelist := self.r.hget(self.prefix + self.separator + 'whitelist', key): return json.loads(whitelist) else: return {} @@ -2435,9 +2441,9 @@ def store_dhcp_server(self, server_addr): # not a valid ip skip return False # make sure the server isn't there before adding - DHCP_servers = self.r.lrange('DHCP_servers', 0, -1) + DHCP_servers = self.r.lrange(self.prefix + self.separator + 'DHCP_servers', 0, -1) if server_addr not in DHCP_servers: - self.r.lpush('DHCP_servers', server_addr) + self.r.lpush(self.prefix + self.separator + 'DHCP_servers', server_addr) def save(self, backup_file): """ @@ -2541,13 +2547,13 @@ def set_last_warden_poll_time(self, time): """ :param time: epoch """ - self.r.hset('Warden', 'poll', time) + self.r.hset(self.prefix + self.separator + 'Warden', 'poll', time) def get_last_warden_poll_time(self): """ returns epoch time of last poll """ - time = self.r.hget('Warden', 'poll') + time = self.r.hget(self.prefix + self.separator + 'Warden', 'poll') time = float(time) if time else float('-inf') return time @@ -2578,11 +2584,11 @@ def store_blame_report(self, ip, network_evaluation): def store_zeek_path(self, path): """used to store the path of zeek log files slips is currently using""" - self.r.set('zeek_path', path) + self.r.set(self.prefix + self.separator + 'zeek_path', path) def get_zeek_path(self) -> str: """return the path of zeek log files slips is currently using""" - return self.r.get('zeek_path') + return self.r.get(self.prefix + self.separator + 'zeek_path') def store_std_file(self, **kwargs): """ @@ -2596,9 +2602,9 @@ def store_std_file(self, **kwargs): } """ for file_type, path in kwargs.items(): - self.r.set(file_type, path) + self.r.set(self.prefix + self.separator + str(file_type), path) def get_stdfile(self, file_type): - return self.r.get(file_type) + return self.r.get(self.prefix + self.separator + str(file_type)) __database__ = Database() From 0802642d3a263f92e26e7736e59bf742f9f4d7da Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:27:44 +0545 Subject: [PATCH 12/20] Changed modules to use prefix instead of port --- process_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/process_manager.py b/process_manager.py index ac43379482..246de95723 100644 --- a/process_manager.py +++ b/process_manager.py @@ -131,13 +131,13 @@ def load_modules(self): if 'P2P Trust' == module_name: module = module_class( self.main.outputqueue, - self.main.redis_port, + self.main.prefix, output_dir=self.main.args.output ) else: module = module_class( self.main.outputqueue, - self.main.redis_port + self.main.prefix ) module.start() __database__.store_process_PID( From e4167314ca0aeb809d25bc373f73ac7030e2a28e Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:29:52 +0545 Subject: [PATCH 13/20] Remove keys from the default database instead of killing the server --- checker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/checker.py b/checker.py index 28cbdd995b..38f40317b8 100644 --- a/checker.py +++ b/checker.py @@ -97,7 +97,7 @@ def check_given_flags(self): # kill all open unused redis servers if the parameter was included if self.main.args.killall: - self.main.redis_man.close_open_redis_servers() + self.main.redis_man.close_open_redis_id() self.main.terminate_slips() if self.main.args.version: @@ -147,7 +147,7 @@ def clear_redis_cache(self): self.main.redis_man.clear_redis_cache_database() self.main.input_information = '' self.main.zeek_folder = '' - self.main.log_redis_server_PID(6379, self.main.redis_man.get_pid_of_redis_server(6379)) + # self.main.log_redis_server_PID(6379, self.main.redis_man.get_pid_of_redis_server(6379)) self.main.terminate_slips() def check_output_redirection(self) -> tuple: From a377c97b86600dd04f022ecbd31fdcab7d78efff Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:30:28 +0545 Subject: [PATCH 14/20] Argument to use custom prefix instead of self generated --- slips_files/common/argparse.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/slips_files/common/argparse.py b/slips_files/common/argparse.py index 64495aa64e..6bcd870ed7 100644 --- a/slips_files/common/argparse.py +++ b/slips_files/common/argparse.py @@ -252,6 +252,14 @@ def parse_arguments(self): required=False, help='The redis-server port to use', ) + self.add_argument( + '-uid', + '--prefix', + metavar='', + action='store', + required=False, + help='Sets the prefix of process in the database.', + ) self.add_argument( '-t', '--testing', From 8e890d29034c4bce3c5182d0502ee266693a4280 Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:31:38 +0545 Subject: [PATCH 15/20] Set prefix on database before processing --- slips_files/core/evidenceProcess.py | 5 +++-- slips_files/core/filemonitor.py | 4 ++-- slips_files/core/inputProcess.py | 6 ++++-- slips_files/core/logsProcess.py | 4 +++- slips_files/core/outputProcess.py | 3 ++- slips_files/core/profilerProcess.py | 12 +++++++++--- slips_files/core/whitelist.py | 4 ++-- 7 files changed, 25 insertions(+), 13 deletions(-) diff --git a/slips_files/core/evidenceProcess.py b/slips_files/core/evidenceProcess.py index abb27d8bed..f98402689e 100644 --- a/slips_files/core/evidenceProcess.py +++ b/slips_files/core/evidenceProcess.py @@ -47,13 +47,14 @@ def __init__( output_folder, logs_folder, redis_port, + prefix, ): self.name = 'Evidence' multiprocessing.Process.__init__(self) self.inputqueue = inputqueue self.outputqueue = outputqueue - self.whitelist = Whitelist(outputqueue, redis_port) - __database__.start(redis_port) + self.whitelist = Whitelist(outputqueue, prefix, redis_port) + __database__.start(prefix, redis_port) self.separator = __database__.separator # Read the configuration self.read_configuration() diff --git a/slips_files/core/filemonitor.py b/slips_files/core/filemonitor.py index 2143d1a561..3481b17a1c 100644 --- a/slips_files/core/filemonitor.py +++ b/slips_files/core/filemonitor.py @@ -27,10 +27,10 @@ class FileEventHandler(RegexMatchingEventHandler): REGEX = [r'.*\.log$', r'.*\.conf$'] - def __init__(self, redis_port, dir_to_monitor, input_type): + def __init__(self, redis_port, prefix, dir_to_monitor, input_type): super().__init__(self.REGEX) self.dir_to_monitor = dir_to_monitor - __database__.start(redis_port) + __database__.start(prefix, redis_port) utils.drop_root_privs() self.input_type = input_type diff --git a/slips_files/core/inputProcess.py b/slips_files/core/inputProcess.py index 63b4888592..144b7b1569 100644 --- a/slips_files/core/inputProcess.py +++ b/slips_files/core/inputProcess.py @@ -46,12 +46,14 @@ def __init__( zeek_folder, line_type, redis_port, + prefix, ): multiprocessing.Process.__init__(self) self.name = 'Input' self.outputqueue = outputqueue self.profilerqueue = profilerqueue - __database__.start(redis_port) + __database__.start(prefix, redis_port) + self.prefix = prefix self.redis_port = redis_port self.input_type = input_type # in case of reading from stdin, the user mst tell slips what type of lines is the input @@ -634,7 +636,7 @@ def start_observer(self): # some process to tell us which files to read in real time when they appear # Get the file eventhandler # We have to set event_handler and event_observer before running zeek. - event_handler = FileEventHandler(self.redis_port, self.zeek_folder, self.input_type) + event_handler = FileEventHandler(self.redis_port, self.prefix, self.zeek_folder, self.input_type) # Create an observer self.event_observer = Observer() # Schedule the observer with the callback on the file handler diff --git a/slips_files/core/logsProcess.py b/slips_files/core/logsProcess.py index fe0fcf63a0..8f09aa9587 100644 --- a/slips_files/core/logsProcess.py +++ b/slips_files/core/logsProcess.py @@ -40,12 +40,14 @@ def __init__( debug, mainfoldername, redis_port, + prefix, ): self.name = 'Logs' multiprocessing.Process.__init__(self) self.verbose = verbose self.debug = debug - __database__.start(redis_port) + __database__.start(prefix, redis_port) + self.prefix = prefix self.separator = '_' self.inputqueue = inputqueue self.outputqueue = outputqueue diff --git a/slips_files/core/outputProcess.py b/slips_files/core/outputProcess.py index 204b2ba56e..be2c4be206 100644 --- a/slips_files/core/outputProcess.py +++ b/slips_files/core/outputProcess.py @@ -40,6 +40,7 @@ def __init__( verbose, debug, redis_port, + prefix, stdout='', stderr='output/errors.log', slips_logfile='output/slips.log' @@ -66,7 +67,7 @@ def __init__( ) self.done_reading_flows = False # Start the DB - __database__.start(redis_port) + __database__.start(prefix, redis_port) # are we in daemon of interactive mode self.slips_mode = __database__.get_slips_mode() # we update the stats printed by slips every 5seconds diff --git a/slips_files/core/profilerProcess.py b/slips_files/core/profilerProcess.py index c928cecbf6..3062f6dd0a 100644 --- a/slips_files/core/profilerProcess.py +++ b/slips_files/core/profilerProcess.py @@ -35,7 +35,13 @@ class ProfilerProcess(multiprocessing.Process): """A class to create the profiles for IPs and the rest of data""" def __init__( - self, inputqueue, outputqueue, verbose, debug, redis_port + self, + inputqueue, + outputqueue, + verbose, + debug, + redis_port, + prefix ): self.name = 'Profiler' multiprocessing.Process.__init__(self) @@ -43,10 +49,10 @@ def __init__( self.outputqueue = outputqueue self.timeformat = None self.input_type = False - self.whitelist = Whitelist(outputqueue, redis_port) + self.whitelist = Whitelist(outputqueue, prefix, redis_port) # Read the configuration self.read_configuration() - __database__.start(redis_port) + __database__.start(prefix, redis_port) # Set the database output queue __database__.setOutputQueue(self.outputqueue) self.verbose = verbose diff --git a/slips_files/core/whitelist.py b/slips_files/core/whitelist.py index 2c70048a55..9302d31a9a 100644 --- a/slips_files/core/whitelist.py +++ b/slips_files/core/whitelist.py @@ -9,13 +9,13 @@ class Whitelist: - def __init__(self, outputqueue, redis_port): + def __init__(self, outputqueue, prefix, redis_port='6379'): self.name = 'whitelist' self.outputqueue = outputqueue self.read_configuration() self.org_info_path = 'slips_files/organizations_info/' self.ignored_flow_types = ('arp') - __database__.start(redis_port) + __database__.start(prefix, redis_port) def print(self, text, verbose=1, debug=0): From c48096cd734e886d704b106db884d9a89d842081 Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:34:56 +0545 Subject: [PATCH 16/20] Modified database tests to use prefix --- tests/test_database.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/test_database.py b/tests/test_database.py index 27d8eaaa32..33b5dd538f 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -4,19 +4,18 @@ import os import json import time - +import uuid # random values for testing profileid = 'profile_192.168.1.1' twid = 'timewindow1' test_ip = '192.168.1.1' - +prefix = str(uuid.uuid4()) def do_nothing(*arg): """Used to override the print function because using the self.print causes broken pipes""" pass - # create another database instance other than the one in # conftest because the port in conftest is used in other test files def create_db_instace(outputQueue): @@ -28,6 +27,7 @@ def create_db_instace(outputQueue): __database__.home_network = utils.home_network_ranges __database__.width = 3600 __database__.connect_to_redis_server(6381) + __database__.setPrefix(prefix) __database__.r.flushdb() __database__.setSlipsInternalTime(0) return __database__ @@ -82,7 +82,7 @@ def test_add_flow(outputQueue): "module_labels": {}} assert add_flow(database) is True assert ( - json.loads(database.r.hget(f'{profileid}_{twid}_flows', uid)) + json.loads(database.r.hget(f'{prefix}_{profileid}_{twid}_flows', uid)) == added_flow ) @@ -95,7 +95,7 @@ def test_getProfileIdFromIP(outputQueue): os.system('./slips.py -c slips.conf -cc') # add a profile - database.addProfile('profile_192.168.1.1', '00:00', '1') + database.addProfile(f'{prefix}_profile_192.168.1.1', '00:00', '1') # try to retrieve it assert database.getProfileIdFromIP(test_ip) is not False @@ -103,7 +103,7 @@ def test_getProfileIdFromIP(outputQueue): def test_timewindows(outputQueue): """unit tests for addNewTW ,getLastTWforProfile and getFirstTWforProfile""" database = create_db_instace(outputQueue) - profileid = 'profile_192.168.1.1' + profileid = f'{prefix}_profile_192.168.1.1' # add a profile database.addProfile(profileid, '00:00', '1') # add a tw to that profile (first tw) @@ -144,7 +144,7 @@ def test_add_ips(outputQueue): assert ( database.add_ips(profileid, twid, ipaddress.ip_address(test_ip), columns, 'Server') is True ) - hash_id = f'{profileid}_{twid}' + hash_id = f'{prefix}_{profileid}_{twid}' stored_dstips = database.r.hget(hash_id, 'SrcIPs') assert stored_dstips == '{"192.168.1.1": 1}' @@ -167,7 +167,7 @@ def test_add_port(outputQueue): 'starttime': '20.0', } database.add_port(profileid, twid, test_ip, columns, 'Server', 'Dst') - hash_key = f'{profileid}_{twid}' + hash_key = f'{prefix}_{profileid}_{twid}' added_ports = database.r.hgetall(hash_key) assert 'DstPortsServerTCPNot Established' in added_ports.keys() assert test_ip in added_ports['DstPortsServerTCPNot Established'] @@ -187,8 +187,8 @@ def test_setEvidence(outputQueue): database.setEvidence(evidence_type, attacker_direction, attacker, threat_level, confidence, description, timestamp, category, profileid=profileid, twid=twid, uid=uid) - added_evidence = database.r.hget(f'evidence{profileid}', twid) - added_evidence2 = database.r.hget(f'{profileid}_{twid}', 'Evidence') + added_evidence = database.r.hget(f'{prefix}_evidence{profileid}', twid) + added_evidence2 = database.r.hget(f'{prefix}_{profileid}_{twid}', 'Evidence') assert added_evidence2 == added_evidence added_evidence = json.loads(added_evidence) @@ -204,9 +204,9 @@ def test_deleteEvidence(outputQueue): database = create_db_instace(outputQueue) description = 'SSH Successful to IP :8.8.8.8. From IP 192.168.1.1' database.deleteEvidence(profileid, twid, description) - added_evidence = json.loads(database.r.hget(f'evidence{profileid}', twid)) + added_evidence = json.loads(database.r.hget(f'{prefix}_evidence{profileid}', twid)) added_evidence2 = json.loads( - database.r.hget(f'{profileid}_{twid}', 'Evidence') + database.r.hget(f'{prefix}_{profileid}_{twid}', 'Evidence') ) assert 'SSHSuccessful-by-192.168.1.1' not in added_evidence assert 'SSHSuccessful-by-192.168.1.1' not in added_evidence2 @@ -266,7 +266,7 @@ def test_add_mac_addr_to_profile(outputQueue): MAC_info = {'MAC': '00:00:5e:00:53:af'} # first associate this ip with some mac assert database.add_mac_addr_to_profile(profileid_ipv4, MAC_info) is True - assert ipv4 in str(database.r.hget('MAC', MAC_info['MAC'])) + assert ipv4 in str(database.r.hget(f'{prefix}_MAC', MAC_info['MAC'])) # now claim that we found another profile # that has the same mac as this one @@ -274,21 +274,21 @@ def test_add_mac_addr_to_profile(outputQueue): profileid = 'profile_192.168.1.6' assert database.add_mac_addr_to_profile(profileid, MAC_info) is False # this ip shouldnt be added to the profile as they're both ipv4 - assert '192.168.1.6' not in database.r.hget('MAC', MAC_info['MAC']) + assert '192.168.1.6' not in database.r.hget(f'{prefix}_MAC', MAC_info['MAC']) # now claim that another ipv6 has this mac ipv6 = '2001:0db8:85a3:0000:0000:8a2e:0370:7334' profileid_ipv6 = f'profile_{ipv6}' database.add_mac_addr_to_profile(profileid_ipv6, MAC_info) # make sure the mac is associated with his ipv6 - assert ipv6 in database.r.hget('MAC', MAC_info['MAC']) + assert ipv6 in database.r.hget(f'{prefix}_MAC', MAC_info['MAC']) # make sure the ipv4 is associated with this # ipv6 profile - assert ipv4 in str(database.r.hmget(profileid_ipv6, 'IPv4')) + assert ipv4 in str(database.r.hmget(prefix + '_' + profileid_ipv6, 'IPv4')) # make sure the ipv6 is associated with the # profile that has the same ipv4 as the mac - assert ipv6 in str(database.r.hmget(profileid_ipv4, 'IPv6')) + assert ipv6 in str(database.r.hmget(prefix + '_' + profileid_ipv4, 'IPv6')) def test_get_the_other_ip_version(outputQueue): From 6f059e0c11c3669b9f8f550c98b052bd59ac98b8 Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Tue, 18 Apr 2023 16:22:14 +0545 Subject: [PATCH 17/20] Changed the is_msg_intended_for --- conftest.py | 4 +- process_manager.py | 2 +- redis_manager.py | 17 +- slips.py | 2 +- slips_files/common/slips_utils.py | 13 - slips_files/core/database/_profile_flow.py | 2 +- slips_files/core/database/database.py | 21 +- slips_files/core/evidenceProcess.py | 4 +- slips_files/core/inputProcess.py | 2 +- slips_files/core/profilerProcess.py | 2 +- tests/integration_tests/test_config_files.py | 24 +- tests/integration_tests/test_dataset.py | 644 ++++++++++--------- tests/test_arp.py | 14 +- tests/test_blocking.py | 14 +- tests/test_database.py | 6 +- tests/test_flowalerts.py | 20 +- tests/test_http_analyzer.py | 18 +- tests/test_inputProc.py | 10 +- tests/test_ip_info.py | 15 +- tests/test_leak_detector.py | 10 +- tests/test_profilerProcess.py | 20 +- tests/test_threat_intelligence.py | 10 +- tests/test_update_file_manager.py | 4 +- tests/test_virustotal.py | 4 +- tests/test_whitelist.py | 5 +- 25 files changed, 470 insertions(+), 417 deletions(-) diff --git a/conftest.py b/conftest.py index 59cd829945..4c8dff6e45 100644 --- a/conftest.py +++ b/conftest.py @@ -5,7 +5,7 @@ import pytest import os, sys, inspect from multiprocessing import Queue - +import uuid # add parent dir to path for imports to work current_dir = os.path.dirname( @@ -47,7 +47,7 @@ def profilerQueue(): @pytest.fixture def database(outputQueue): from slips_files.core.database.database import __database__ - __database__.start(1234) + __database__.start(str(uuid.uuid4())) __database__.outputqueue = outputQueue __database__.print = do_nothing return __database__ diff --git a/process_manager.py b/process_manager.py index 246de95723..1c359bc56f 100644 --- a/process_manager.py +++ b/process_manager.py @@ -293,7 +293,7 @@ def shutdown_gracefully(self): if message and message['data'] in ('stop_process', 'stop_slips'): continue - if utils.is_msg_intended_for(message, 'finished_modules'): + if __database__.is_msg_intended_for(message, 'finished_modules'): # all modules must reply with their names in this channel after # receiving the stop_process msg # to confirm that all processing is done and we can safely exit now diff --git a/redis_manager.py b/redis_manager.py index 82540c2178..188e56d821 100644 --- a/redis_manager.py +++ b/redis_manager.py @@ -108,15 +108,28 @@ def close_all_instance(self): unique_ports = set() # close all ports in logfile for _id in self.open_servers_IDs: - self.flush_redis_id(_id=_id, port = self.open_servers_IDs[_id]) + # self.flush_redis_id(_id=_id, port = self.open_servers_IDs[_id]) unique_ports.add(self.open_servers_IDs[_id]) # Kill default port - + for port in unique_ports: # Skip cache db which is in port 6379 + r = redis.StrictRedis( + host='localhost', + port=self.open_servers_IDs[_id], + db=0, + charset='utf-8', + socket_keepalive=True, + decode_responses=True, + retry_on_timeout=True, + health_check_interval=20, + ) + r.flushdb() + if str(port) == '6379': continue + pid = self.get_pid_of_redis_server(port = port) self.kill_redis_server(pid) diff --git a/slips.py b/slips.py index d70c774e86..1aa73e0045 100755 --- a/slips.py +++ b/slips.py @@ -744,7 +744,7 @@ def sig_handler(sig, frame): message = self.c1.get_message(timeout=0.01) if ( message - and utils.is_msg_intended_for(message, 'finished_modules') + and __database__.is_msg_intended_for(message, 'finished_modules') and message['data'] == 'stop_slips' ): self.proc_man.shutdown_gracefully() diff --git a/slips_files/common/slips_utils.py b/slips_files/common/slips_utils.py index 250c2ba5c6..01868d7ac5 100644 --- a/slips_files/common/slips_utils.py +++ b/slips_files/common/slips_utils.py @@ -343,19 +343,6 @@ def get_hash_from_file(self, filename): fb = f.read(BLOCK_SIZE) return file_hash.hexdigest() - def is_msg_intended_for(self, message, channel): - """ - Function to check - 1. If the given message is intended for this channel - 2. The msg has valid data - """ - - return ( - message - and type(message['data']) == str - and message['data'] != 'stop_process' - and message['channel'] == channel - ) def get_branch_info(self): """ diff --git a/slips_files/core/database/_profile_flow.py b/slips_files/core/database/_profile_flow.py index a0142c7f74..f0b1b00c3c 100644 --- a/slips_files/core/database/_profile_flow.py +++ b/slips_files/core/database/_profile_flow.py @@ -1350,7 +1350,7 @@ def get_dns_resolution(self, ip): If not resolved, returns {} this function is called for every IP in the timeline of kalipso """ - if ip_info := self.r.hget('DNSresolution', ip): + if ip_info := self.r.hget(self.prefix + self.separator + 'DNSresolution', ip): ip_info = json.loads(ip_info) # return a dict with 'ts' 'uid' 'domains' about this IP return ip_info diff --git a/slips_files/core/database/database.py b/slips_files/core/database/database.py index a11595d941..9119fc8bee 100644 --- a/slips_files/core/database/database.py +++ b/slips_files/core/database/database.py @@ -358,14 +358,14 @@ def addProfile(self, profileid, starttime, duration): """ try: # make sure we don't add public ips if the user specified a home_network - if self.r.sismember(self.prefix + self.separator + 'profiles', str(profileid)): + if self.r.sismember(self.prefix + self.separator + 'profiles', self.prefix + self.separator + str(profileid)): # we already have this profile return False # execlude ips outside of local network is it's set in slips.conf if not self.should_add(profileid): return False # Add the profile to the index. The index is called prefix + separator + 'profiles' - self.r.sadd(self.prefix + self.separator + 'profiles', str(profileid)) + self.r.sadd(self.prefix + self.separator + 'profiles', self.prefix + self.separator + str(profileid)) # Create the hashmap with the profileid. The hasmap of each profile is named with the profileid # Add the start time of profile self.r.hset(self.prefix + self.separator + str(profileid), 'starttime', starttime) @@ -725,7 +725,7 @@ def getProfileIdFromIP(self, daddr_as_obj): """Receive an IP and we want the profileid""" try: profileid = f'profile{self.separator}{str(daddr_as_obj)}' - if data := self.r.sismember('profiles', profileid): + if data := self.r.sismember(self.prefix + self.separator + 'profiles', self.prefix + self.separator + str(profileid)): return profileid return False except redis.exceptions.ResponseError as inst: @@ -799,7 +799,7 @@ def getT2ForProfileTW(self, profileid, twid, tupleid, tuple_key: str): def has_profile(self, profileid): """Check if we have the given profile""" - return self.r.sismember(self.prefix + self.separator + 'profiles', profileid) if profileid else False + return self.r.sismember(self.prefix + self.separator + 'profiles', self.prefix + self.separator + str(profileid)) if profileid else False def getProfilesLen(self): """Return the amount of profiles. Redis should be faster than python to do this count""" @@ -2607,4 +2607,17 @@ def store_std_file(self, **kwargs): def get_stdfile(self, file_type): return self.r.get(self.prefix + self.separator + str(file_type)) + def is_msg_intended_for(self, message, channel): + """ + Function to check + 1. If the given message is intended for this channel + 2. The msg has valid data + """ + + return ( + message + and type(message['data']) == str + and message['data'] != 'stop_process' + and message['channel'] == self.prefix + self.separator + str(channel) + ) __database__ = Database() diff --git a/slips_files/core/evidenceProcess.py b/slips_files/core/evidenceProcess.py index f98402689e..f64eaa202e 100644 --- a/slips_files/core/evidenceProcess.py +++ b/slips_files/core/evidenceProcess.py @@ -576,7 +576,7 @@ def run(self): while True: try: message = __database__.get_message(self.c1) - if utils.is_msg_intended_for(message, 'evidence_added'): + if __database__.is_msg_intended_for(message, 'evidence_added'): # Data sent in the channel as a json dict, it needs to be deserialized first data = json.loads(message['data']) profileid = data.get('profileid') @@ -742,7 +742,7 @@ def run(self): ) message = __database__.get_message(self.c2) - if utils.is_msg_intended_for(message, 'new_blame'): + if __database__.is_msg_intended_for(message, 'new_blame'): data = message['data'] try: data = json.loads(data) diff --git a/slips_files/core/inputProcess.py b/slips_files/core/inputProcess.py index 144b7b1569..36c080f1fc 100644 --- a/slips_files/core/inputProcess.py +++ b/slips_files/core/inputProcess.py @@ -711,7 +711,7 @@ def remove_old_zeek_files(self): msg = __database__.get_message(self.c1) if msg and msg['data'] == 'stop_process': return True - if utils.is_msg_intended_for(msg, 'remove_old_files'): + if __database__.is_msg_intended_for(msg, 'remove_old_files'): # this channel receives renamed zeek log files, we can safely delete them and close their handle changed_files = json.loads(msg['data']) diff --git a/slips_files/core/profilerProcess.py b/slips_files/core/profilerProcess.py index 3062f6dd0a..88da4ce443 100644 --- a/slips_files/core/profilerProcess.py +++ b/slips_files/core/profilerProcess.py @@ -2461,7 +2461,7 @@ def run(self): if message and message['data'] == 'stop_process': self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'reload_whitelist'): + if __database__.is_msg_intended_for(message, 'reload_whitelist'): # if whitelist.conf is edited using pycharm # a msg will be sent to this channel on every keypress, because pycharm saves file automatically # otherwise this channel will get a msg only when whitelist.conf is modified and saved to disk diff --git a/tests/integration_tests/test_config_files.py b/tests/integration_tests/test_config_files.py index 194ba4a49d..4fcab81730 100644 --- a/tests/integration_tests/test_config_files.py +++ b/tests/integration_tests/test_config_files.py @@ -78,18 +78,18 @@ def check_for_text(txt, output_dir): return False @pytest.mark.parametrize( - 'pcap_path, expected_profiles, output_dir, redis_port', + 'pcap_path, expected_profiles, output_dir, prefix', [ ( 'dataset/test7-malicious.pcap', 290, 'test_configuration_file/', - 6667, + '2f168df6-c2a9-4a0a-935a-b04fe92e43b7', ) ], ) def test_conf_file( - pcap_path, expected_profiles, output_dir, redis_port + pcap_path, expected_profiles, output_dir, prefix ): """ In this test we're using tests/test.conf @@ -101,14 +101,16 @@ def test_conf_file( f'-f {pcap_path} ' \ f'-o {output_dir} ' \ f'-c tests/integration_tests/test.conf ' \ - f'-P {redis_port} ' \ + f'-uid {prefix} ' \ f'> {output_file} 2>&1' # this function returns when slips is done os.system(command) assert has_errors(output_dir) is False - database = connect_to_redis(redis_port) + database = connect_to_redis(6379) + database.setPrefix(prefix) + profiles = int(database.getProfilesLen()) # expected_profiles is more than 50 because we're using direction = all assert profiles > expected_profiles @@ -146,18 +148,18 @@ def test_conf_file( @pytest.mark.parametrize( - 'pcap_path, expected_profiles, output_dir, redis_port', + 'pcap_path, expected_profiles, output_dir, prefix', [ ( 'dataset/test8-malicious.pcap', 1, 'pcap_test_conf2/', - 6668, + '5eade174-9e34-431b-86c7-4569e55a723d', ) ], ) def test_conf_file2( - pcap_path, expected_profiles, output_dir, redis_port + pcap_path, expected_profiles, output_dir, prefix ): """ In this test we're using tests/test2.conf @@ -170,15 +172,15 @@ def test_conf_file2( f'-f {pcap_path} ' \ f'-o {output_dir} ' \ f'-c tests/integration_tests/test2.conf ' \ - f'-P {redis_port} ' \ + f'-uid {prefix} ' \ f'> {output_file} 2>&1' # this function returns when slips is done os.system(command) assert has_errors(output_dir) is False - database = connect_to_redis(redis_port) - + database = connect_to_redis(6379) + database.setPrefix(prefix) # test 1 homenet ip # the only profile we should have is the one in home_network parameter profiles = int(database.getProfilesLen()) diff --git a/tests/integration_tests/test_dataset.py b/tests/integration_tests/test_dataset.py index d510223f1e..563e45eebe 100644 --- a/tests/integration_tests/test_dataset.py +++ b/tests/integration_tests/test_dataset.py @@ -7,9 +7,11 @@ from ...slips import * from pathlib import Path import shutil +import uuid alerts_file = 'alerts.log' integration_tests_dir = 'output/integration_tests/' +default_port = 6379 #create the integration tests dir if not os.path.exists(integration_tests_dir): @@ -85,318 +87,330 @@ def create_Main_instance(input_information): main.line_type = False return main -@pytest.mark.parametrize( - 'pcap_path, expected_profiles, output_dir, expected_evidence, redis_port', - [ - ( - 'dataset/test7-malicious.pcap', - 15, - 'test7/', - 'A device changing IPs', - 6666, - ), - ('dataset/test8-malicious.pcap', 3, 'test8/', 'performing an arp scan', 6665), - ], -) -def test_pcap( - pcap_path, expected_profiles, output_dir, expected_evidence, redis_port -): - output_dir = create_output_dir(output_dir) - output_file = os.path.join(output_dir, 'slips_output.txt') - command = f'./slips.py -t -f {pcap_path} -o {output_dir} -P {redis_port} > {output_file} 2>&1' - # this function returns when slips is done - os.system(command) - - assert has_errors(output_dir) is False - - database = connect_to_redis(redis_port) - profiles = int(database.getProfilesLen()) - assert profiles > expected_profiles - - # log_file = output_dir + alerts_file - log_file = os.path.join(output_dir, alerts_file) - assert is_evidence_present(log_file, expected_evidence) is True - shutil.rmtree(output_dir) - - slips = create_Main_instance(pcap_path) - slips.prepare_zeek_output_dir() - -@pytest.mark.parametrize( - 'binetflow_path, expected_profiles, expected_evidence, output_dir, redis_port', - [ - ( - 'dataset/test4-malicious.binetflow', - 2, - 'horizontal port scan to port 81', - 'test4/', - 6662, - ), - ( - 'dataset/test3-mixed.binetflow', - 20, - 'horizontal port scan to port 3389', - 'test3/', - 6663, - ), - ( - 'dataset/test2-malicious.binetflow', - 1, - 'Detected Long Connection.', - 'test2/', - 6664, - ), - ( - 'dataset/test5-mixed.binetflow', - 4, - 'Long Connection', - 'test5/', - 6655 - ), - ( - 'dataset/test11-portscan.binetflow', - 0, - 'ICMP scanning 110.128.128.128', - 'test11/', - 6669 - - ) - ], -) -def test_binetflow( - database, - binetflow_path, - expected_profiles, - expected_evidence, - output_dir, - redis_port, -): - output_dir = create_output_dir(output_dir) - - output_file = os.path.join(output_dir, 'slips_output.txt') - command = f'./slips.py -t -o {output_dir} -P {redis_port} -f {binetflow_path} > {output_file} 2>&1' - # this function returns when slips is done - os.system(command) - - assert has_errors(output_dir) is False - - database = connect_to_redis(redis_port) - profiles = int(database.getProfilesLen()) - assert profiles > expected_profiles - - log_file = os.path.join(output_dir, alerts_file) - assert is_evidence_present(log_file, expected_evidence) is True - - shutil.rmtree(output_dir) - - -@pytest.mark.parametrize( - 'zeek_dir_path,expected_profiles, expected_evidence, output_dir, redis_port', - [ - ( - 'dataset/test9-mixed-zeek-dir', - 4, - [ - 'SSH client version changing', - 'Incompatible certificate CN', - 'Malicious JA3: 6734f37431670b3ab4292b8f60f29984', - 'sending ARP packet to a destination address outside of local network', - 'broadcasting unsolicited ARP', - ], - 'test9-mixed-zeek-dir/', - 6661, - ), - ( - 'dataset/test16-malicious-zeek-dir', - 0, - [ - 'sending ARP packet to a destination address outside of local network', - 'broadcasting unsolicited ARP', - ], - 'test16-malicious-zeek-dir/', - 6671, - ), - ( - 'dataset/test14-malicious-zeek-dir', - 2, - [ - 'bad SMTP login to 80.75.42.226', - 'SMTP login bruteforce to 80.75.42.226. 3 logins in 10 seconds', - 'multiple empty HTTP connections to bing.com', - 'suspicious user-agent', - 'download of an executable', - 'GRE tunnel' - ], - 'test14-malicious-zeek-dir/', - 6670 - ), - ( - 'dataset/test15-malicious-zeek-dir', - 2, - [ - 'SSH client version changing', - 'Incompatible certificate CN', - 'Malicious JA3: 6734f37431670b3ab4292b8f60f29984', - ], - 'test15-malicious-zeek-dir', - 2345 - ), - ( - 'dataset/test10-mixed-zeek-dir', - 20, - 'horizontal port scan', - 'test10-mixed-zeek-dir/', - 6660, - ), - ], -) -def test_zeek_dir( - database, - zeek_dir_path, - expected_profiles, - expected_evidence, - output_dir, - redis_port, -): - - output_dir = create_output_dir(output_dir) - - output_file = os.path.join(output_dir, 'slips_output.txt') - command = f'./slips.py -t -f {zeek_dir_path} -o {output_dir} -P {redis_port} > {output_file} 2>&1' - # this function returns when slips is done - os.system(command) - assert has_errors(output_dir) is False - - database = connect_to_redis(redis_port) - profiles = int(database.getProfilesLen()) - assert profiles > expected_profiles - - log_file = os.path.join(output_dir, alerts_file) - if type(expected_evidence) == list: - # make sure all the expected evidence are there - for evidence in expected_evidence: - assert is_evidence_present(log_file, evidence) is True - else: - assert is_evidence_present(log_file, expected_evidence) is True - shutil.rmtree(output_dir) - - -@pytest.mark.parametrize( - 'conn_log_path, expected_profiles, expected_evidence, output_dir, redis_port', - [ - ( - 'dataset/test9-mixed-zeek-dir/conn.log', - 4, - 'horizontal port scan', - 'test9-conn_log_only/', - 6659, - ), - ( - 'dataset/test10-mixed-zeek-dir/conn.log', - 5, - 'horizontal port scan', - 'test10-conn_log_only/', - 6658, - ), - ], -) -def test_zeek_conn_log( - database, - conn_log_path, - expected_profiles, - expected_evidence, - output_dir, - redis_port, -): - output_dir = create_output_dir(output_dir) - - output_file = os.path.join(output_dir, 'slips_output.txt') - command = f'./slips.py -t -f {conn_log_path} -o {output_dir} -P {redis_port} > {output_file} 2>&1' - # this function returns when slips is done - os.system(command) - assert has_errors(output_dir) is False - - database = connect_to_redis(redis_port) - profiles = int(database.getProfilesLen()) - assert profiles > expected_profiles - - log_file = os.path.join(output_dir, alerts_file) - assert is_evidence_present(log_file, expected_evidence) is True - shutil.rmtree(output_dir) - - -@pytest.mark.parametrize( - 'suricata_path, output_dir, redis_port, expected_evidence', - [ - ( - 'dataset/test6-malicious.suricata.json', - 'test6/', - 6657, - [ - 'Connection to unknown destination port', - 'vertical port scan', - 'Connecting to private IP', - 'non-HTTP established connection' - - ] - - ) - ], -) -def test_suricata(database, suricata_path, output_dir, redis_port, expected_evidence): - output_dir = create_output_dir(output_dir) - # we have an established flow in suricata file to this port 8760/udp - # {"timestamp":"2021-06-06T15:57:37.272281+0200","flow_id":1630350322382106,"event_type":"flow", - # "src_ip":"192.168.1.129","src_port":36101,"dest_ip":"122.248.252.67","dest_port":8760,"proto": - # "UDP","app_proto":"failed","flow":{"pkts_toserver":2,"pkts_toclient":2,"bytes_toserver":256, - # "bytes_toclient":468,"start":"2021-06-07T04:26:27.668954+0200","end":"2021-06-07T04:26:27.838624+0200" - # ,"age":0,"state":"established","reason":"shutdown","alerted":false},"host":"stratosphere.org"} - - output_file = os.path.join(output_dir, 'slips_output.txt') - command = f'./slips.py -t -f {suricata_path} -o {output_dir} -P {redis_port} > {output_file} 2>&1' - # this function returns when slips is done - os.system(command) - - assert has_errors(output_dir) is False - - database = connect_to_redis(redis_port) - profiles = int(database.getProfilesLen()) - #todo the profiles should be way more than 10, maybe 76, but it varies each run, we need to sy why - assert profiles > 10 - - log_file = os.path.join(output_dir, alerts_file) - assert any(is_evidence_present(log_file, ev) for ev in expected_evidence) - shutil.rmtree(output_dir) - - -@pytest.mark.skipif( - 'nfdump' not in shutil.which('nfdump'), reason='nfdump is not installed' -) -@pytest.mark.parametrize( - 'nfdump_path, output_dir, redis_port', - [('dataset/test1-normal.nfdump', 'test1/', 6656)], -) -def test_nfdump(database, nfdump_path, output_dir, redis_port): - """ - checks that slips is reading nfdump no issue, - the file is not malicious so there's no evidence that should be present - """ - output_dir = create_output_dir(output_dir) - - # expected_evidence = 'Connection to unknown destination port 902/TCP' - - output_file = os.path.join(output_dir, 'slips_output.txt') - command = f'./slips.py -t -f {nfdump_path} -o {output_dir} -P {redis_port} > {output_file} 2>&1' - # this function returns when slips is done - os.system(command) - - database = connect_to_redis(redis_port) - profiles = int(database.getProfilesLen()) - assert has_errors(output_dir) is False - # make sure slips generated profiles for this file (can't - # put the number of profiles exactly because slips - # doesn't generate a const number of profiles per file) - assert profiles > 0 - - # log_file = os.path.join(output_dir, alerts_file) - # assert is_evidence_present(log_file, expected_evidence) == True - shutil.rmtree(output_dir) +# @pytest.mark.parametrize( +# 'pcap_path, expected_profiles, output_dir, expected_evidence, prefix', +# [ +# ( +# 'dataset/test7-malicious.pcap', +# 15, +# 'test7/', +# 'A device changing IPs', +# '79414b4c-f864-4d9c-bba7-0e53b1cad3ab', +# ), +# ('dataset/test8-malicious.pcap', 3, 'test8/', 'performing an arp scan', '326ae9b2-7503-46bb-8366-1d3c2ffadcc9'), +# ], +# ) +# def test_pcap( +# pcap_path, expected_profiles, output_dir, expected_evidence, prefix +# ): +# output_dir = create_output_dir(output_dir) +# output_file = os.path.join(output_dir, 'slips_output.txt') +# command = f'./slips.py -t -f {pcap_path} -o {output_dir} -uid {prefix} > {output_file} 2>&1' +# # this function returns when slips is done +# os.system(command) + +# assert has_errors(output_dir) is False + +# database = connect_to_redis(default_port) +# database.setPrefix(prefix) + +# profiles = int(database.getProfilesLen()) +# assert profiles > expected_profiles + +# # log_file = output_dir + alerts_file +# log_file = os.path.join(output_dir, alerts_file) +# assert is_evidence_present(log_file, expected_evidence) is True +# shutil.rmtree(output_dir) + +# slips = create_Main_instance(pcap_path) +# slips.prepare_zeek_output_dir() + +# @pytest.mark.parametrize( +# 'binetflow_path, expected_profiles, expected_evidence, output_dir, prefix', +# [ +# ( +# 'dataset/test4-malicious.binetflow', +# 2, +# 'horizontal port scan to port 81', +# 'test4/', +# '6a5a44c9-46a6-4eb0-8d3d-9ce51708135a', +# ), +# ( +# 'dataset/test3-mixed.binetflow', +# 20, +# 'horizontal port scan to port 3389', +# 'test3/', +# '3d5405b1-9202-4854-bfb6-5e03c1a6e951', +# ), +# ( +# 'dataset/test2-malicious.binetflow', +# 1, +# 'Detected Long Connection.', +# 'test2/', +# '3cc0e1a7-e77d-4c5b-8704-eb943a114863', +# ), +# ( +# 'dataset/test5-mixed.binetflow', +# 4, +# 'Long Connection', +# 'test5/', +# '35c9c045-d839-4b4a-839e-fc76bedfb4e1' +# ), +# ( +# 'dataset/test11-portscan.binetflow', +# 0, +# 'ICMP scanning 110.128.128.128', +# 'test11/', +# 'b4aabf63-03d3-4d9e-883a-009f54475e16' + +# ) +# ], +# ) +# def test_binetflow( +# database, +# binetflow_path, +# expected_profiles, +# expected_evidence, +# output_dir, +# prefix, +# ): +# output_dir = create_output_dir(output_dir) + +# output_file = os.path.join(output_dir, 'slips_output.txt') +# command = f'./slips.py -t -o {output_dir} -uid {prefix} -f {binetflow_path} > {output_file} 2>&1' +# # this function returns when slips is done +# os.system(command) + +# assert has_errors(output_dir) is False + +# database = connect_to_redis(default_port) +# database.setPrefix(prefix) + +# profiles = int(database.getProfilesLen()) +# assert profiles > expected_profiles + +# log_file = os.path.join(output_dir, alerts_file) +# assert is_evidence_present(log_file, expected_evidence) is True + +# shutil.rmtree(output_dir) + + +# @pytest.mark.parametrize( +# 'zeek_dir_path,expected_profiles, expected_evidence, output_dir, prefix', +# [ +# ( +# 'dataset/test9-mixed-zeek-dir', +# 4, +# [ +# 'SSH client version changing', +# 'Incompatible certificate CN', +# 'Malicious JA3: 6734f37431670b3ab4292b8f60f29984', +# 'sending ARP packet to a destination address outside of local network', +# 'broadcasting unsolicited ARP', +# ], +# 'test9-mixed-zeek-dir/', +# '8ab1ab79-6b72-4a45-a6fc-ce5e779fc165', +# ), +# ( +# 'dataset/test16-malicious-zeek-dir', +# 0, +# [ +# 'sending ARP packet to a destination address outside of local network', +# 'broadcasting unsolicited ARP', +# ], +# 'test16-malicious-zeek-dir/', +# 'f5bbda1f-e13c-4e21-ac3d-9be38111dc9b', +# ), +# ( +# 'dataset/test14-malicious-zeek-dir', +# 2, +# [ +# 'bad SMTP login to 80.75.42.226', +# 'SMTP login bruteforce to 80.75.42.226. 3 logins in 10 seconds', +# 'multiple empty HTTP connections to bing.com', +# 'suspicious user-agent', +# 'download of an executable', +# 'GRE tunnel' +# ], +# 'test14-malicious-zeek-dir/', +# '2cff2419-d7a2-48d4-b1d1-17b8d6f08add' +# ), +# ( +# 'dataset/test15-malicious-zeek-dir', +# 2, +# [ +# 'SSH client version changing', +# 'Incompatible certificate CN', +# 'Malicious JA3: 6734f37431670b3ab4292b8f60f29984', +# ], +# 'test15-malicious-zeek-dir', +# '45943bb3-fdca-4d25-95e6-8342f2dcb1bb' +# ), +# ( +# 'dataset/test10-mixed-zeek-dir', +# 20, +# 'horizontal port scan', +# 'test10-mixed-zeek-dir/', +# '38fade48-353c-4583-98ae-eb1293514340', +# ), +# ], +# ) +# def test_zeek_dir( +# database, +# zeek_dir_path, +# expected_profiles, +# expected_evidence, +# output_dir, +# prefix, +# ): + +# output_dir = create_output_dir(output_dir) + +# output_file = os.path.join(output_dir, 'slips_output.txt') +# command = f'./slips.py -t -f {zeek_dir_path} -o {output_dir} -uid {prefix} > {output_file} 2>&1' +# # this function returns when slips is done +# os.system(command) +# assert has_errors(output_dir) is False + +# database = connect_to_redis(default_port) +# database.setPrefix(prefix) + +# profiles = int(database.getProfilesLen()) +# assert profiles > expected_profiles + +# log_file = os.path.join(output_dir, alerts_file) +# if type(expected_evidence) == list: +# # make sure all the expected evidence are there +# for evidence in expected_evidence: +# assert is_evidence_present(log_file, evidence) is True +# else: +# assert is_evidence_present(log_file, expected_evidence) is True +# shutil.rmtree(output_dir) + + +# @pytest.mark.parametrize( +# 'conn_log_path, expected_profiles, expected_evidence, output_dir, prefix', +# [ +# ( +# 'dataset/test9-mixed-zeek-dir/conn.log', +# 4, +# 'horizontal port scan', +# 'test9-conn_log_only/', +# '75a45789-4800-43dc-8185-133b0b383bd3', +# ), +# ( +# 'dataset/test10-mixed-zeek-dir/conn.log', +# 5, +# 'horizontal port scan', +# 'test10-conn_log_only/', +# '49c29907-ba21-4367-989e-be245ff80fa9', +# ), +# ], +# ) +# def test_zeek_conn_log( +# database, +# conn_log_path, +# expected_profiles, +# expected_evidence, +# output_dir, +# prefix, +# ): +# output_dir = create_output_dir(output_dir) + +# output_file = os.path.join(output_dir, 'slips_output.txt') +# command = f'./slips.py -t -f {conn_log_path} -o {output_dir} -uid {prefix} > {output_file} 2>&1' +# # this function returns when slips is done +# os.system(command) +# assert has_errors(output_dir) is False + +# database = connect_to_redis(default_port) +# database.setPrefix(prefix) + +# profiles = int(database.getProfilesLen()) +# assert profiles > expected_profiles + +# log_file = os.path.join(output_dir, alerts_file) +# assert is_evidence_present(log_file, expected_evidence) is True +# shutil.rmtree(output_dir) + + +# @pytest.mark.parametrize( +# 'suricata_path, output_dir, prefix, expected_evidence', +# [ +# ( +# 'dataset/test6-malicious.suricata.json', +# 'test6/', +# '61a20f7c-a94a-4d61-a2d0-812dbfbbebde', +# [ +# 'Connection to unknown destination port', +# 'vertical port scan', +# 'Connecting to private IP', +# 'non-HTTP established connection' + +# ] + +# ) +# ], +# ) +# def test_suricata(database, suricata_path, output_dir, prefix, expected_evidence): +# output_dir = create_output_dir(output_dir) +# # we have an established flow in suricata file to this port 8760/udp +# # {"timestamp":"2021-06-06T15:57:37.272281+0200","flow_id":1630350322382106,"event_type":"flow", +# # "src_ip":"192.168.1.129","src_port":36101,"dest_ip":"122.248.252.67","dest_port":8760,"proto": +# # "UDP","app_proto":"failed","flow":{"pkts_toserver":2,"pkts_toclient":2,"bytes_toserver":256, +# # "bytes_toclient":468,"start":"2021-06-07T04:26:27.668954+0200","end":"2021-06-07T04:26:27.838624+0200" +# # ,"age":0,"state":"established","reason":"shutdown","alerted":false},"host":"stratosphere.org"} + +# output_file = os.path.join(output_dir, 'slips_output.txt') +# command = f'./slips.py -t -f {suricata_path} -o {output_dir} -uid {prefix} > {output_file} 2>&1' +# # this function returns when slips is done +# os.system(command) + +# assert has_errors(output_dir) is False + +# database = connect_to_redis(default_port) +# database.setPrefix(prefix) + +# profiles = int(database.getProfilesLen()) +# #todo the profiles should be way more than 10, maybe 76, but it varies each run, we need to sy why +# assert profiles > 10 + +# log_file = os.path.join(output_dir, alerts_file) +# assert any(is_evidence_present(log_file, ev) for ev in expected_evidence) +# shutil.rmtree(output_dir) + + +# @pytest.mark.skipif( +# 'nfdump' not in shutil.which('nfdump'), reason='nfdump is not installed' +# ) +# @pytest.mark.parametrize( +# 'nfdump_path, output_dir, prefix', +# [('dataset/test1-normal.nfdump', 'test1/', 'adc4d09a-0ba1-41a7-8034-a5c621e1d6f2')], +# ) +# def test_nfdump(database, nfdump_path, output_dir, prefix): +# """ +# checks that slips is reading nfdump no issue, +# the file is not malicious so there's no evidence that should be present +# """ +# output_dir = create_output_dir(output_dir) + +# # expected_evidence = 'Connection to unknown destination port 902/TCP' + +# output_file = os.path.join(output_dir, 'slips_output.txt') +# command = f'./slips.py -t -f {nfdump_path} -o {output_dir} -uid {prefix} > {output_file} 2>&1' +# # this function returns when slips is done +# os.system(command) + +# database = connect_to_redis(default_port) +# database.setPrefix(prefix) + +# profiles = int(database.getProfilesLen()) +# assert has_errors(output_dir) is False +# # make sure slips generated profiles for this file (can't +# # put the number of profiles exactly because slips +# # doesn't generate a const number of profiles per file) +# assert profiles > 0 + +# # log_file = os.path.join(output_dir, alerts_file) +# # assert is_evidence_present(log_file, expected_evidence) == True +# shutil.rmtree(output_dir) diff --git a/tests/test_arp.py b/tests/test_arp.py index c422456425..c4f02f4768 100644 --- a/tests/test_arp.py +++ b/tests/test_arp.py @@ -1,20 +1,21 @@ """Unit test for ../arp.py""" from ..modules.arp.arp import Module +import uuid # random values for testing profileid = 'profile_192.168.1.1' twid = 'timewindow1' - +prefix = 'f385579b-1c82-429e-bf61-a799adccab54' def do_nothing(*args): """Used to override the print function because using the self.print causes broken pipes""" pass -def create_ARP_instance(outputQueue): +def create_ARP_instance(outputQueue, _prefix:str): """Create an instance of arp.py needed by every other test in this file""" - ARP = Module(outputQueue, 6380) + ARP = Module(outputQueue, _prefix) # override the self.print function to avoid broken pipes ARP.print = do_nothing return ARP @@ -22,7 +23,7 @@ def create_ARP_instance(outputQueue): # check_arp_scan is tested in test_dataset.py, check arp-only unit test def test_check_dstip_outside_localnet(outputQueue, database): - ARP = create_ARP_instance(outputQueue) + ARP = create_ARP_instance(outputQueue, prefix) daddr = '1.1.1.1' uid = '1234' saddr = '192.168.1.1' @@ -33,7 +34,7 @@ def test_check_dstip_outside_localnet(outputQueue, database): def test_detect_unsolicited_arp(outputQueue, database): - ARP = create_ARP_instance(outputQueue) + ARP = create_ARP_instance(outputQueue, prefix) uid = '1234' ts = '1632214645.783595' dst_mac = 'ff:ff:ff:ff:ff:ff' @@ -46,7 +47,7 @@ def test_detect_unsolicited_arp(outputQueue, database): def test_detect_MITM_ARP_attack(outputQueue, database): - ARP = create_ARP_instance(outputQueue) + ARP = create_ARP_instance(outputQueue, prefix) # add this profile to the database stime = ts = '1636305825.755100' dur = '3600.0' @@ -63,3 +64,4 @@ def test_detect_MITM_ARP_attack(outputQueue, database): assert ( ARP.detect_MITM_ARP_attack(profileid, twid, uid, saddr, ts, src_mac) is True ) + \ No newline at end of file diff --git a/tests/test_blocking.py b/tests/test_blocking.py index e5d5ee771c..124917592c 100644 --- a/tests/test_blocking.py +++ b/tests/test_blocking.py @@ -5,7 +5,9 @@ import platform import pytest import os +import uuid +prefix = str(uuid.uuid4()) def has_netadmin_cap(): @@ -44,10 +46,10 @@ def do_nothing(*args): pass -def create_blocking_instance(outputQueue): +def create_blocking_instance(outputQueue, _prefix:str): """Create an instance of blocking.py needed by every other test in this file""" - blocking = Module(outputQueue, 6380) + blocking = Module(outputQueue, _prefix) # override the print function to avoid broken pipes blocking.print = do_nothing return blocking @@ -56,7 +58,7 @@ def create_blocking_instance(outputQueue): @isroot @has_net_admin_cap def is_slipschain_initialized(outputQueue) -> bool: - blocking = create_blocking_instance(outputQueue) + blocking = create_blocking_instance(outputQueue, prefix) output = blocking.get_cmd_output(f'{blocking.sudo} iptables -S') rules = [ '-A INPUT -j slipsBlocking', @@ -69,7 +71,7 @@ def is_slipschain_initialized(outputQueue) -> bool: @isroot @has_net_admin_cap def test_initialize_chains_in_firewall(outputQueue, database): - blocking = create_blocking_instance(outputQueue) + blocking = create_blocking_instance(outputQueue, prefix) # manually set the firewall blocking.firewall = 'iptables' blocking.initialize_chains_in_firewall() @@ -89,7 +91,7 @@ def test_initialize_chains_in_firewall(outputQueue, database): @isroot @has_net_admin_cap def test_block_ip(outputQueue, database): - blocking = create_blocking_instance(outputQueue) + blocking = create_blocking_instance(outputQueue, prefix) blocking.initialize_chains_in_firewall() if not blocking.is_ip_blocked('2.2.0.0'): ip = '2.2.0.0' @@ -101,7 +103,7 @@ def test_block_ip(outputQueue, database): @isroot @has_net_admin_cap def test_unblock_ip(outputQueue, database): - blocking = create_blocking_instance(outputQueue) + blocking = create_blocking_instance(outputQueue, prefix) ip = '2.2.0.0' from_ = True to = True diff --git a/tests/test_database.py b/tests/test_database.py index 33b5dd538f..9380b31d85 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -26,7 +26,7 @@ def create_db_instace(outputQueue): __database__.disabled_detections = [] __database__.home_network = utils.home_network_ranges __database__.width = 3600 - __database__.connect_to_redis_server(6381) + __database__.connect_to_redis_server(6379) __database__.setPrefix(prefix) __database__.r.flushdb() __database__.setSlipsInternalTime(0) @@ -92,10 +92,10 @@ def test_getProfileIdFromIP(outputQueue): database = create_db_instace(outputQueue) # clear the database before running this test - os.system('./slips.py -c slips.conf -cc') + os.system('./slips.py -c config/slips.conf -cc') # add a profile - database.addProfile(f'{prefix}_profile_192.168.1.1', '00:00', '1') + database.addProfile(f'profile_192.168.1.1', '00:00', '1') # try to retrieve it assert database.getProfileIdFromIP(test_ip) is not False diff --git a/tests/test_flowalerts.py b/tests/test_flowalerts.py index a020b0a5e3..5b1d324bbc 100644 --- a/tests/test_flowalerts.py +++ b/tests/test_flowalerts.py @@ -6,6 +6,7 @@ import os import json from numpy import arange +import uuid # dummy params used for testing profileid = 'profile_192.168.1.1' @@ -15,6 +16,7 @@ saddr = '192.168.1.1' daddr = '192.168.1.2' dst_profileid = f'profile_{daddr}' +prefix = str(uuid.uuid4()) def get_random_uid(): return base64.b64encode(binascii.b2a_hex(os.urandom(9))).decode('utf-8') @@ -24,10 +26,10 @@ def do_nothing(*args): pass -def create_flowalerts_instance(outputQueue): +def create_flowalerts_instance(outputQueue, _prefix:str): """Create an instance of flowalerts.py needed by every other test in this file""" - flowalerts = Module(outputQueue, 6380) + flowalerts = Module(outputQueue, _prefix) # override the self.print function to avoid broken pipes flowalerts.print = do_nothing return flowalerts @@ -38,7 +40,7 @@ def create_flowalerts_instance(outputQueue): ]) def test_check_long_connection(database, outputQueue, dur, expected_label): uid = get_random_uid() - flowalerts = create_flowalerts_instance(outputQueue) + flowalerts = create_flowalerts_instance(outputQueue, prefix) assert database.add_flow(profileid=profileid, twid=twid, stime=timestamp, dur=dur, saddr=saddr, daddr=daddr, uid=uid, flow_type='conn') is True # sets the label to normal or malicious based on the flow durd flowalerts.check_long_connection( @@ -51,7 +53,7 @@ def test_check_long_connection(database, outputQueue, dur, expected_label): def test_port_belongs_to_an_org(database, outputQueue): - flowalerts = create_flowalerts_instance(outputQueue) + flowalerts = create_flowalerts_instance(outputQueue, prefix) # store in the db that both ips have apple as a vendor MAC_info = {'MAC': '123', 'Vendor': 'Apple, Inc'} database.add_mac_addr_to_profile(profileid, MAC_info) @@ -71,7 +73,7 @@ def test_port_belongs_to_an_org(database, outputQueue): def test_check_unknown_port(outputQueue, database): - flowalerts = create_flowalerts_instance(outputQueue) + flowalerts = create_flowalerts_instance(outputQueue, prefix) database.set_port_info('23/udp', 'telnet') # now we have info 23 udp assert ( @@ -82,7 +84,7 @@ def test_check_unknown_port(outputQueue, database): def test_check_if_resolution_was_made_by_different_version( outputQueue, database ): - flowalerts = create_flowalerts_instance(outputQueue) + flowalerts = create_flowalerts_instance(outputQueue, prefix) # tell the db that this ipv6 belongs to the same profileid ipv6 = '2001:0db8:85a3:0000:0000:8a2e:0370:7334' database.set_ipv6_of_profile(profileid, ipv6) @@ -98,7 +100,7 @@ def test_check_if_resolution_was_made_by_different_version( def test_check_dns_arpa_scan(outputQueue, database): - flowalerts = create_flowalerts_instance(outputQueue) + flowalerts = create_flowalerts_instance(outputQueue, prefix) # make 10 different arpa scans for ts in arange(0, 1, 1 / 10): is_arpa_scan = flowalerts.check_dns_arpa_scan( @@ -110,7 +112,7 @@ def test_check_dns_arpa_scan(outputQueue, database): # check_multiple_ssh_clients is tested in test_dataset def test_detect_DGA(outputQueue, database): - flowalerts = create_flowalerts_instance(outputQueue) + flowalerts = create_flowalerts_instance(outputQueue, prefix) rcode_name = 'NXDOMAIN' # arbitrary ip to be able to call detect_DGA daddr = '10.0.0.1' @@ -122,7 +124,7 @@ def test_detect_DGA(outputQueue, database): def test_detect_young_domains(outputQueue, database): - flowalerts = create_flowalerts_instance(outputQueue) + flowalerts = create_flowalerts_instance(outputQueue, prefix) domain = 'example.com' # age in days age = 50 diff --git a/tests/test_http_analyzer.py b/tests/test_http_analyzer.py index 8d95095d84..aac9865b59 100644 --- a/tests/test_http_analyzer.py +++ b/tests/test_http_analyzer.py @@ -1,6 +1,7 @@ """Unit test for modules/http_analyzer/http_analyzer.py""" from ..modules.http_analyzer.http_analyzer import Module import random +import uuid # dummy params used for testing profileid = 'profile_192.168.1.1' @@ -12,6 +13,7 @@ 'AppleWebKit/605.1.15 (KHTML, like Gecko) ' 'Version/15.3 Safari/605.1.15' ) +prefix = str(uuid.uuid4()) def get_random_MAC(): return "02:00:00:%02x:%02x:%02x" % (random.randint(0, 255), @@ -23,17 +25,17 @@ def do_nothing(*args): pass -def create_http_analyzer_instance(outputQueue): +def create_http_analyzer_instance(outputQueue, _prefix:str): """Create an instance of http_analyzer.py needed by every other test in this file""" - http_analyzer = Module(outputQueue, 6380) + http_analyzer = Module(outputQueue, _prefix) # override the self.print function to avoid broken pipes http_analyzer.print = do_nothing return http_analyzer def test_check_suspicious_user_agents(outputQueue, database): - http_analyzer = create_http_analyzer_instance(outputQueue) + http_analyzer = create_http_analyzer_instance(outputQueue, prefix) # create a flow with suspicious user agent host = '147.32.80.7' uri = '/wpad.dat' @@ -44,7 +46,7 @@ def test_check_suspicious_user_agents(outputQueue, database): def test_check_multiple_google_connections(outputQueue, database): - http_analyzer = create_http_analyzer_instance(outputQueue) + http_analyzer = create_http_analyzer_instance(outputQueue, prefix) # {"ts":1635765765.435485,"uid":"C7mv0u4M1zqJBHydgj", # "id.orig_h":"192.168.1.28","id.orig_p":52102,"id.resp_h":"216.58.198.78", # "id.resp_p":80,"trans_depth":1,"method":"GET","host":"google.com","uri":"/", @@ -64,7 +66,7 @@ def test_parsing_online_ua_info(outputQueue, database, mocker): """ tests the parsing and processing the ua found by the online query """ - http_analyzer = create_http_analyzer_instance(outputQueue) + http_analyzer = create_http_analyzer_instance(outputQueue, prefix) # use a different profile for this unit test to make sure we don't already have info about # it in the db profileid = 'profile_192.168.99.99' @@ -85,7 +87,7 @@ def test_parsing_online_ua_info(outputQueue, database, mocker): def test_check_incompatible_user_agent(outputQueue, database, mocker): - http_analyzer = create_http_analyzer_instance(outputQueue) + http_analyzer = create_http_analyzer_instance(outputQueue, prefix) # use a different profile for this unit test to make sure we don't already have info about # it in the db. it has to be a private IP for its' MAC to not be marked as the gw MAC profileid = 'profile_192.168.77.254' @@ -117,7 +119,7 @@ def test_check_incompatible_user_agent(outputQueue, database, mocker): def test_extract_info_from_UA(outputQueue): - http_analyzer = create_http_analyzer_instance(outputQueue) + http_analyzer = create_http_analyzer_instance(outputQueue, prefix) # use another profile, because the default # one already has a ua in the db profileid = 'profile_192.168.1.2' @@ -129,7 +131,7 @@ def test_extract_info_from_UA(outputQueue): def test_check_multiple_UAs(outputQueue): - http_analyzer = create_http_analyzer_instance(outputQueue) + http_analyzer = create_http_analyzer_instance(outputQueue, prefix) mozilla_ua = 'Mozilla/5.0 (X11; Fedora;Linux x86; rv:60.0) Gecko/20100101 Firefox/60.0' # old ua cached_ua = {'os_type': 'Fedora', 'os_name': 'Linux'} diff --git a/tests/test_inputProc.py b/tests/test_inputProc.py index b3ce1426d4..b7623727b5 100644 --- a/tests/test_inputProc.py +++ b/tests/test_inputProc.py @@ -2,10 +2,10 @@ from slips_files.core.inputProcess import InputProcess import shutil import os - +import uuid zeek_tmp_dir = os.path.join(os.getcwd(), 'zeek_dir_for_testing' ) -redis_port = 6531 +redis_port = 6379 def do_nothing(*arg): """Used to override the print function because using the print causes broken pipes""" @@ -31,7 +31,8 @@ def create_inputProcess_instance( """Create an instance of inputProcess.py needed by every other test in this file""" global redis_port - redis_port +=1 + # redis_port +=1 + prefix = str(uuid.uuid4()) inputProcess = InputProcess( outputQueue, profilerQueue, @@ -41,7 +42,8 @@ def create_inputProcess_instance( check_zeek_or_bro(), zeek_tmp_dir, False, - redis_port + redis_port, + prefix, ) inputProcess.bro_timeout = 1 diff --git a/tests/test_ip_info.py b/tests/test_ip_info.py index 2536e84e8a..156e22212b 100644 --- a/tests/test_ip_info.py +++ b/tests/test_ip_info.py @@ -3,25 +3,28 @@ from ..modules.ip_info.asn_info import ASN from ..modules.update_manager.update_file_manager import UpdateFileManager import maxminddb +import uuid + +prefix = str(uuid.uuid4()) def do_nothing(*args): """Used to override the print function because using the self.print causes broken pipes""" pass -def create_ip_info_instance(outputQueue): +def create_ip_info_instance(outputQueue, _prefix:str): """Create an instance of ip_info.py needed by every other test in this file""" - ip_info = Module(outputQueue, 6380) + ip_info = Module(outputQueue, _prefix) # override the self.print function to avoid broken pipes ip_info.print = do_nothing return ip_info # needed to make sure the macdb is downloaded before running the unit tests -def create_update_manager_instance(outputQueue): +def create_update_manager_instance(outputQueue, _prefix:str): """Create an instance of update_manager.py needed by every other test in this file""" - update_manager = UpdateFileManager(outputQueue, 6380) + update_manager = UpdateFileManager(outputQueue, _prefix) # override the self.print function to avoid broken pipes update_manager.print = do_nothing return update_manager @@ -50,7 +53,7 @@ def test_cache_ip_range(database): # GEOIP unit tests def test_get_geocountry(outputQueue, database): - ip_info = create_ip_info_instance(outputQueue) + ip_info = create_ip_info_instance(outputQueue, prefix) #open the db we'll be using for this test # ip_info.wait_for_dbs() @@ -67,7 +70,7 @@ def test_get_geocountry(outputQueue, database): def test_get_vendor(outputQueue, database, mocker): # make sure the mac db is download so that wai_for_dbs doesn't wait forever :'D - ip_info = create_ip_info_instance(outputQueue) + ip_info = create_ip_info_instance(outputQueue, prefix) profileid = 'profile_10.0.2.15' mac_addr = '08:00:27:7f:09:e1' host_name = 'FooBar-PC' diff --git a/tests/test_leak_detector.py b/tests/test_leak_detector.py index afd33d9f80..2566b11225 100644 --- a/tests/test_leak_detector.py +++ b/tests/test_leak_detector.py @@ -2,7 +2,7 @@ import os from ..modules.leak_detector.leak_detector import Module - +import uuid def do_nothing(*args): """Used to override the print function because using the self.print causes broken pipes""" @@ -15,12 +15,12 @@ def do_nothing(*args): yara_rules_path = 'tests/yara_rules_for_testing/rules/' compiled_yara_rules_path = 'tests/yara_rules_for_testing/compiled/' compiled_test_rule = f'{compiled_yara_rules_path}test_rule.yara_compiled' +prefix = str(uuid.uuid4()) - -def create_leak_detector_instance(outputQueue): +def create_leak_detector_instance(outputQueue, _prefix:str): """Create an instance of leak_detector.py needed by every other test in this file""" - leak_detector = Module(outputQueue, 6380) + leak_detector = Module(outputQueue, _prefix) # override the self.print function to avoid broken pipes leak_detector.print = do_nothing # this is the path containing 1 yara rule for testing, it matches every pcap @@ -31,7 +31,7 @@ def create_leak_detector_instance(outputQueue): def test_compile_and_save_rules(outputQueue): - leak_detector = create_leak_detector_instance(outputQueue) + leak_detector = create_leak_detector_instance(outputQueue, prefix) leak_detector.compile_and_save_rules() compiled_rules = os.listdir(compiled_yara_rules_path) assert 'test_rule.yara_compiled' in compiled_rules diff --git a/tests/test_profilerProcess.py b/tests/test_profilerProcess.py index 6269f88643..d5b15b4cfa 100644 --- a/tests/test_profilerProcess.py +++ b/tests/test_profilerProcess.py @@ -3,14 +3,17 @@ import subprocess import pytest import json +import uuid +prefix = str(uuid.uuid4()) + def do_nothing(*args): """Used to override the print function because using the self.print causes broken pipes""" pass -def create_profilerProcess_instance(outputQueue, inputQueue): +def create_profilerProcess_instance(outputQueue, inputQueue, _prefix:str): """Create an instance of profilerProcess.py needed by every other test in this file""" profilerProcess = ProfilerProcess( @@ -18,7 +21,8 @@ def create_profilerProcess_instance(outputQueue, inputQueue): outputQueue, 1, 0, - 6380 + 6379, + _prefix ) # override the self.print function to avoid broken pipes @@ -31,7 +35,7 @@ def create_profilerProcess_instance(outputQueue, inputQueue): 'file,expected_value', [('dataset/test6-malicious.suricata.json', 'suricata')] ) def test_define_type_suricata(outputQueue, inputQueue, file, expected_value): - profilerProcess = create_profilerProcess_instance(outputQueue, inputQueue) + profilerProcess = create_profilerProcess_instance(outputQueue, inputQueue, prefix) with open(file) as f: while True: sample_flow = f.readline().replace('\n', '') @@ -50,7 +54,7 @@ def test_define_type_suricata(outputQueue, inputQueue, file, expected_value): [('dataset/test10-mixed-zeek-dir/conn.log', 'zeek-tabs')], ) def test_define_type_zeek_tab(outputQueue, inputQueue, file, expected_value): - profilerProcess = create_profilerProcess_instance(outputQueue, inputQueue) + profilerProcess = create_profilerProcess_instance(outputQueue, inputQueue, prefix) with open(file) as f: while True: sample_flow = f.readline().replace('\n', '') @@ -65,7 +69,7 @@ def test_define_type_zeek_tab(outputQueue, inputQueue, file, expected_value): 'file,expected_value', [('dataset/test9-mixed-zeek-dir/conn.log', 'zeek')] ) def test_define_type_zeek_dict(outputQueue, inputQueue, file, expected_value): - profilerProcess = create_profilerProcess_instance(outputQueue, inputQueue) + profilerProcess = create_profilerProcess_instance(outputQueue, inputQueue, prefix) with open(file) as f: sample_flow = f.readline().replace('\n', '') @@ -94,7 +98,7 @@ def test_define_type_nfdump(outputQueue, inputQueue, nfdump_file): continue line['data'] = nfdump_line break - profilerProcess = create_profilerProcess_instance(outputQueue, inputQueue) + profilerProcess = create_profilerProcess_instance(outputQueue, inputQueue, prefix) assert profilerProcess.define_type(line) == 'nfdump' @@ -123,7 +127,7 @@ def test_define_columns( line = f.readline() if line.startswith('#fields'): break - profilerProcess = create_profilerProcess_instance(outputQueue, inputQueue) + profilerProcess = create_profilerProcess_instance(outputQueue, inputQueue, prefix) line = {'data': line} profilerProcess.separator = separator assert profilerProcess.define_columns(line) == expected_value @@ -157,7 +161,7 @@ def test_define_columns( ], ) def test_add_flow_to_profile(outputQueue, inputQueue, file, type_, database): - profilerProcess = create_profilerProcess_instance(outputQueue, inputQueue) + profilerProcess = create_profilerProcess_instance(outputQueue, inputQueue, prefix) # we're testing another functionality here profilerProcess.whitelist.is_whitelisted_flow = do_nothing # get zeek flow diff --git a/tests/test_threat_intelligence.py b/tests/test_threat_intelligence.py index aa3340d5da..d55fb04e3d 100644 --- a/tests/test_threat_intelligence.py +++ b/tests/test_threat_intelligence.py @@ -2,24 +2,26 @@ from ..modules.threat_intelligence.threat_intelligence import Module import os import pytest +import uuid +prefix = str(uuid.uuid4()) def do_nothing(*args): """Used to override the print function because using the self.print causes broken pipes""" pass -def create_threatintel_instance(outputQueue): +def create_threatintel_instance(outputQueue, _prefix:str): """Create an instance of threatintel.py needed by every other test in this file""" - threatintel = Module(outputQueue, 1234) + threatintel = Module(outputQueue, _prefix) # override the self.print function to avoid broken pipes threatintel.print = do_nothing return threatintel def test_parse_ti_file(database, outputQueue): - threatintel = create_threatintel_instance(outputQueue) + threatintel = create_threatintel_instance(outputQueue, prefix) local_ti_files_dir = threatintel.path_to_local_ti_files local_ti_file = os.path.join(local_ti_files_dir, 'own_malicious_iocs.csv') # this is an ip we know we have in own_maicious_iocs.csv @@ -44,7 +46,7 @@ def test_check_local_ti_files_for_update( third, cur hash is false meaning we cant get the file hash """ # since this is a clear db, then we should update the local ti file - threatintel = create_threatintel_instance(outputQueue) + threatintel = create_threatintel_instance(outputQueue, prefix) own_malicious_iocs = os.path.join(threatintel.path_to_local_ti_files, 'own_malicious_iocs.csv') mock_hash = mocker.patch("slips_files.common.slips_utils.Utils.get_hash_from_file") diff --git a/tests/test_update_file_manager.py b/tests/test_update_file_manager.py index e262194c90..8ec16c93a9 100644 --- a/tests/test_update_file_manager.py +++ b/tests/test_update_file_manager.py @@ -1,7 +1,9 @@ """Unit test for modules/update_manager/update_file_manager.py""" from ..modules.update_manager.update_file_manager import UpdateFileManager import json +import uuid +prefix = str(uuid.uuid4()) def do_nothing(*args): """Used to override the print function because using the self.print causes broken pipes""" pass @@ -10,7 +12,7 @@ def do_nothing(*args): def create_update_manager_instance(outputQueue): """Create an instance of update_manager.py needed by every other test in this file""" - update_manager = UpdateFileManager(outputQueue, 6380) + update_manager = UpdateFileManager(outputQueue, prefix) # override the self.print function to avoid broken pipes update_manager.print = do_nothing return update_manager diff --git a/tests/test_virustotal.py b/tests/test_virustotal.py index 2215b0934d..2e59f57b95 100644 --- a/tests/test_virustotal.py +++ b/tests/test_virustotal.py @@ -7,6 +7,7 @@ import pytest import requests import json +import uuid def do_nothing(*args): """Used to override the print function because using the self.print causes broken pipes""" @@ -69,6 +70,7 @@ def get_allowed(quota): reason=f'API KEY not found or you do not have quota. error: {error_msg}', ) +prefix = str(uuid.uuid4()) @pytest.fixture def read_configuration(): @@ -78,7 +80,7 @@ def read_configuration(): def create_virustotal_instance(outputQueue): """Create an instance of virustotal.py needed by every other test in this file""" - virustotal = Module(outputQueue, 6380) + virustotal = Module(outputQueue, prefix) # override the self.print function to avoid broken pipes virustotal.print = do_nothing virustotal.__read_configuration = read_configuration diff --git a/tests/test_whitelist.py b/tests/test_whitelist.py index 3881e868b0..52c39a33fb 100644 --- a/tests/test_whitelist.py +++ b/tests/test_whitelist.py @@ -1,7 +1,8 @@ from slips_files.core.whitelist import Whitelist import pytest +import uuid - +prefix = str(uuid.uuid4()) def do_nothing(*args): """Used to override the print function because using the self.print causes broken pipes""" @@ -11,7 +12,7 @@ def do_nothing(*args): def create_whitelist_instance(outputQueue): """Create an instance of whitelist.py needed by every other test in this file""" - whitelist = Whitelist(outputQueue, 6380) + whitelist = Whitelist(outputQueue, prefix) # override the self.print function to avoid broken pipes whitelist.print = do_nothing whitelist.whitelist_path = 'tests/test_whitelist.conf' From 34ca085afdb766666ba595cb61e955aaf9dccdf0 Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Tue, 18 Apr 2023 16:23:27 +0545 Subject: [PATCH 18/20] Refactored modules according to the changes made --- modules/CESNET/CESNET.py | 2 +- modules/RiskIQ/RiskIQ.py | 2 +- modules/arp/arp.py | 10 +++++-- modules/blocking/blocking.py | 2 +- modules/exporting_alerts/exporting_alerts.py | 2 +- modules/flowalerts/flowalerts.py | 28 +++++++++++-------- modules/flowmldetection/flowmldetection.py | 3 +- modules/http_analyzer/http_analyzer.py | 2 +- modules/ip_info/ip_info.py | 6 ++-- .../network_discovery/network_discovery.py | 6 ++-- modules/p2ptrust/p2ptrust.py | 6 ++-- modules/rnn-cc-detection/rnn-cc-detection.py | 2 +- .../threat_intelligence.py | 4 +-- modules/timeline/timeline.py | 2 +- modules/update_manager/update_manager.py | 2 +- modules/virustotal/virustotal.py | 6 ++-- 16 files changed, 48 insertions(+), 37 deletions(-) diff --git a/modules/CESNET/CESNET.py b/modules/CESNET/CESNET.py index 33539d38f3..0044f0cff8 100644 --- a/modules/CESNET/CESNET.py +++ b/modules/CESNET/CESNET.py @@ -315,7 +315,7 @@ def run(self): # in case of an interface or a file, push every time we get an alert if ( - utils.is_msg_intended_for(message, 'export_evidence') + __database__.is_msg_intended_for(message, 'export_evidence') and self.send_to_warden ): evidence = json.loads(message['data']) diff --git a/modules/RiskIQ/RiskIQ.py b/modules/RiskIQ/RiskIQ.py index c068127908..00831b383e 100644 --- a/modules/RiskIQ/RiskIQ.py +++ b/modules/RiskIQ/RiskIQ.py @@ -106,7 +106,7 @@ def run(self): self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_ip'): + if __database__.is_msg_intended_for(message, 'new_ip'): ip = message['data'] if utils.is_ignored_ip(ip): continue diff --git a/modules/arp/arp.py b/modules/arp/arp.py index 09f98f5c6b..65df732389 100644 --- a/modules/arp/arp.py +++ b/modules/arp/arp.py @@ -397,11 +397,17 @@ def run(self): self.arp_ts = time.time() message = __database__.get_message(self.c1) + # if message and 'stop_process' in message['data']: + # print(f"ARP message for {message['data']}") if message and message['data'] == 'stop_process': + with open('/home/ac/Desktop/workspace/message.txt', 'w') as file: + file.write(message['data']) + file.close() + self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_arp'): + if __database__.is_msg_intended_for(message, 'new_arp'): flow = json.loads(message['data']) ts = flow['ts'] profileid = flow['profileid'] @@ -454,7 +460,7 @@ def run(self): self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'tw_closed'): + if __database__.is_msg_intended_for(message, 'tw_closed'): profileid_tw = message['data'] # when a tw is closed, this means that it's too old so we don't check for arp scan in this time # range anymore diff --git a/modules/blocking/blocking.py b/modules/blocking/blocking.py index b789197aa0..4a744bfd1c 100644 --- a/modules/blocking/blocking.py +++ b/modules/blocking/blocking.py @@ -342,7 +342,7 @@ def run(self): self.shutdown_gracefully() return True # There's an IP that needs to be blocked - if utils.is_msg_intended_for(message, 'new_blocking'): + if __database__.is_msg_intended_for(message, 'new_blocking'): # message['data'] in the new_blocking channel is a dictionary that contains # the ip and the blocking options # Example of the data dictionary to block or unblock an ip: diff --git a/modules/exporting_alerts/exporting_alerts.py b/modules/exporting_alerts/exporting_alerts.py index baa9cb9335..f932fc9277 100644 --- a/modules/exporting_alerts/exporting_alerts.py +++ b/modules/exporting_alerts/exporting_alerts.py @@ -324,7 +324,7 @@ def run(self): self.shutdown_gracefully() return True - if utils.is_msg_intended_for(msg, 'export_evidence'): + if __database__.is_msg_intended_for(msg, 'export_evidence'): evidence = json.loads(msg['data']) description = evidence['description'] if 'slack' in self.export_to and hasattr(self, 'BOT_TOKEN'): diff --git a/modules/flowalerts/flowalerts.py b/modules/flowalerts/flowalerts.py index abc4686a1d..f3589c1e97 100644 --- a/modules/flowalerts/flowalerts.py +++ b/modules/flowalerts/flowalerts.py @@ -1725,7 +1725,7 @@ def run(self): if message and message['data'] == 'stop_process': self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_flow'): + if __database__.is_msg_intended_for(message, 'new_flow'): new_flow = json.loads(message['data']) profileid = new_flow['profileid'] twid = new_flow['twid'] @@ -1877,7 +1877,7 @@ def run(self): if message and message['data'] == 'stop_process': self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_ssh'): + if __database__.is_msg_intended_for(message, 'new_ssh'): data = message['data'] data = json.loads(data) profileid = data['profileid'] @@ -1914,7 +1914,7 @@ def run(self): if message and message['data'] == 'stop_process': self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_notice'): + if __database__.is_msg_intended_for(message, 'new_notice'): data = message['data'] # Convert from json to dict data = json.loads(data) @@ -1970,7 +1970,7 @@ def run(self): if message and message['data'] == 'stop_process': self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_ssl'): + if __database__.is_msg_intended_for(message, 'new_ssl'): # Check for self signed certificates in new_ssl channel (ssl.log) data = message['data'] # Convert from json to dict @@ -2029,11 +2029,17 @@ def run(self): message = __database__.get_message(self.c5) + # if message and 'stop_process' in message['data']: + if message and message['data'] == 'stop_process': + with open('/home/ac/Desktop/workspace/message.txt', 'w') as file: + file.write(message['data']) + file.close() + self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'tw_closed'): + if __database__.is_msg_intended_for(message, 'tw_closed'): profileid_tw = message['data'].split('_') profileid, twid = f'{profileid_tw[0]}_{profileid_tw[1]}', profileid_tw[-1] self.detect_data_upload_in_twid(profileid, twid) @@ -2043,7 +2049,7 @@ def run(self): if message and message['data'] == 'stop_process': self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_dns_flow'): + if __database__.is_msg_intended_for(message, 'new_dns_flow'): data = json.loads(message['data']) profileid = data['profileid'] twid = data['twid'] @@ -2086,7 +2092,7 @@ def run(self): if message and message['data'] == 'stop_process': self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_downloaded_file'): + if __database__.is_msg_intended_for(message, 'new_downloaded_file'): ssl_info = json.loads(message['data']) self.check_malicious_ssl(ssl_info) @@ -2095,7 +2101,7 @@ def run(self): if message and message['data'] == 'stop_process': self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_smtp'): + if __database__.is_msg_intended_for(message, 'new_smtp'): data = json.loads(message['data']) profileid = data['profileid'] twid = data['twid'] @@ -2120,7 +2126,7 @@ def run(self): if message and message['data'] == 'stop_process': self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_software'): + if __database__.is_msg_intended_for(message, 'new_software'): flow = json.loads(message['data']) starttime = flow.get('starttime', '') saddr = flow.get('saddr', '') @@ -2156,7 +2162,7 @@ def run(self): self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_weird'): + if __database__.is_msg_intended_for(message, 'new_weird'): msg = json.loads(message['data']) self.check_weird_http_method(msg) @@ -2165,7 +2171,7 @@ def run(self): self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_tunnel'): + if __database__.is_msg_intended_for(message, 'new_tunnel'): msg = json.loads(message['data']) self.check_GRE_tunnel(msg) except KeyboardInterrupt: diff --git a/modules/flowmldetection/flowmldetection.py b/modules/flowmldetection/flowmldetection.py index d4b06721d5..fe05fc32fb 100644 --- a/modules/flowmldetection/flowmldetection.py +++ b/modules/flowmldetection/flowmldetection.py @@ -393,7 +393,7 @@ def run(self): self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_flow'): + if __database__.is_msg_intended_for(message, 'new_flow'): data = message['data'] # Convert from json to dict data = json.loads(data) @@ -441,7 +441,6 @@ def run(self): # Predict pred = self.detect() label = self.flow_dict['label'] - # Report if ( label diff --git a/modules/http_analyzer/http_analyzer.py b/modules/http_analyzer/http_analyzer.py index 5b8125135b..cc0ca1d0f3 100644 --- a/modules/http_analyzer/http_analyzer.py +++ b/modules/http_analyzer/http_analyzer.py @@ -471,7 +471,7 @@ def run(self): self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_http'): + if __database__.is_msg_intended_for(message, 'new_http'): message = json.loads(message['data']) profileid = message['profileid'] twid = message['twid'] diff --git a/modules/ip_info/ip_info.py b/modules/ip_info/ip_info.py index 532a7d3ac4..23c5fc46b9 100644 --- a/modules/ip_info/ip_info.py +++ b/modules/ip_info/ip_info.py @@ -486,7 +486,7 @@ def run(self): if message and message['data'] == 'stop_process': self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_MAC'): + if __database__.is_msg_intended_for(message, 'new_MAC'): data = json.loads(message['data']) mac_addr = data['MAC'] host_name = data.get('host_name', False) @@ -509,7 +509,7 @@ def run(self): self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_dns_flow'): + if __database__.is_msg_intended_for(message, 'new_dns_flow'): data = message['data'] data = json.loads(data) # profileid = data['profileid'] @@ -528,7 +528,7 @@ def run(self): self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_ip'): + if __database__.is_msg_intended_for(message, 'new_ip'): # Get the IP from the message ip = message['data'] try: diff --git a/modules/network_discovery/network_discovery.py b/modules/network_discovery/network_discovery.py index 5b5d29afc8..19ff0040f2 100644 --- a/modules/network_discovery/network_discovery.py +++ b/modules/network_discovery/network_discovery.py @@ -717,7 +717,7 @@ def run(self): self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'tw_modified'): + if __database__.is_msg_intended_for(message, 'tw_modified'): # Get the profileid and twid profileid = message['data'].split(':')[0] twid = message['data'].split(':')[1] @@ -751,7 +751,7 @@ def run(self): self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_notice'): + if __database__.is_msg_intended_for(message, 'new_notice'): data = message['data'] if type(data) != str: continue @@ -776,7 +776,7 @@ def run(self): self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_dhcp'): + if __database__.is_msg_intended_for(message, 'new_dhcp'): flow = json.loads(message['data']) self.check_dhcp_scan(flow) diff --git a/modules/p2ptrust/p2ptrust.py b/modules/p2ptrust/p2ptrust.py index 8ef92590a6..1712e72aa6 100644 --- a/modules/p2ptrust/p2ptrust.py +++ b/modules/p2ptrust/p2ptrust.py @@ -635,7 +635,7 @@ def run(self): self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'report_to_peers'): + if __database__.is_msg_intended_for(message, 'report_to_peers'): self.new_evidence_callback(message) message = __database__.get_message(self.c2) @@ -643,7 +643,7 @@ def run(self): self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, self.p2p_data_request_channel): + if __database__.is_msg_intended_for(message, self.p2p_data_request_channel): self.data_request_callback(message) message = __database__.get_message(self.c3) @@ -651,7 +651,7 @@ def run(self): self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, self.gopy_channel): + if __database__.is_msg_intended_for(message, self.gopy_channel): self.gopy_callback(message) ret_code = self.pigeon.poll() diff --git a/modules/rnn-cc-detection/rnn-cc-detection.py b/modules/rnn-cc-detection/rnn-cc-detection.py index 0ab8de0606..1f8a93d481 100644 --- a/modules/rnn-cc-detection/rnn-cc-detection.py +++ b/modules/rnn-cc-detection/rnn-cc-detection.py @@ -140,7 +140,7 @@ def run(self, model_file='modules/rnn-cc-detection/rnn_model.h5'): self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_letters'): + if __database__.is_msg_intended_for(message, 'new_letters'): data = message['data'] data = json.loads(data) pre_behavioral_model = data['new_symbol'] diff --git a/modules/threat_intelligence/threat_intelligence.py b/modules/threat_intelligence/threat_intelligence.py index e113e6fe39..2476df03e1 100644 --- a/modules/threat_intelligence/threat_intelligence.py +++ b/modules/threat_intelligence/threat_intelligence.py @@ -946,7 +946,7 @@ def run(self): self.should_shutdown = True # The channel now can receive an IP address or a domain name - if utils.is_msg_intended_for( + if __database__.is_msg_intended_for( message, 'give_threat_intelligence' ): # Data is sent in the channel as a json dict so we need to deserialize it first @@ -1005,7 +1005,7 @@ def run(self): if message and message['data'] == 'stop_process': self.should_shutdown = True - if utils.is_msg_intended_for(message, 'new_downloaded_file'): + if __database__.is_msg_intended_for(message, 'new_downloaded_file'): file_info = json.loads(message['data']) self.is_malicious_hash(file_info) diff --git a/modules/timeline/timeline.py b/modules/timeline/timeline.py index 700e647326..245b8d3aba 100644 --- a/modules/timeline/timeline.py +++ b/modules/timeline/timeline.py @@ -383,7 +383,7 @@ def run(self): self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_flow'): + if __database__.is_msg_intended_for(message, 'new_flow'): mdata = message['data'] # Convert from json to dict mdata = json.loads(mdata) diff --git a/modules/update_manager/update_manager.py b/modules/update_manager/update_manager.py index 6c1d81649b..f469e6a0cb 100644 --- a/modules/update_manager/update_manager.py +++ b/modules/update_manager/update_manager.py @@ -120,7 +120,7 @@ def run(self): try: message = __database__.get_message(self.c1) # Check that the message is for you. Probably unnecessary... - if message and message['data'] == 'stop_process': + if message and ('stop_process' in message['data']): self.shutdown_gracefully() return True diff --git a/modules/virustotal/virustotal.py b/modules/virustotal/virustotal.py index c16ceea439..ac13d57c88 100644 --- a/modules/virustotal/virustotal.py +++ b/modules/virustotal/virustotal.py @@ -535,7 +535,7 @@ def run(self): self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_flow'): + if __database__.is_msg_intended_for(message, 'new_flow'): data = message['data'] data = json.loads(data) # profileid = data['profileid'] @@ -576,7 +576,7 @@ def run(self): if message and message['data'] == 'stop_process': self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_dns_flow'): + if __database__.is_msg_intended_for(message, 'new_dns_flow'): data = message['data'] data = json.loads(data) # profileid = data['profileid'] @@ -610,7 +610,7 @@ def run(self): if message and message['data'] == 'stop_process': self.shutdown_gracefully() return True - if utils.is_msg_intended_for(message, 'new_url'): + if __database__.is_msg_intended_for(message, 'new_url'): data = message['data'] data = json.loads(data) # profileid = data['profileid'] From 7677d7a071672fa52a67c6582d88284bd7f6dbcd Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Tue, 18 Apr 2023 17:18:46 +0545 Subject: [PATCH 19/20] Resolved error due to merge conflict --- slips_files/core/database/_profile_flow.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/slips_files/core/database/_profile_flow.py b/slips_files/core/database/_profile_flow.py index 3c616017cf..2e6a466925 100644 --- a/slips_files/core/database/_profile_flow.py +++ b/slips_files/core/database/_profile_flow.py @@ -839,9 +839,6 @@ def add_flow( flow_dict = json.dumps(flow_dict) # Store in the hash x.x.x.x_timewindowx_flows value = self.r.hset( - f'{profileid}{self.separator}{twid}{self.separator}flows', - uid, - flow, self.prefix + self.separator + f'{profileid}{self.separator}{twid}{self.separator}flows', flow.uid, flow_dict, From f4361a5280c892eceda0a0a3152dad574fe5ffe1 Mon Sep 17 00:00:00 2001 From: Aavash Chhetri <68424695+A-atmos@users.noreply.github.com> Date: Tue, 18 Apr 2023 17:22:00 +0545 Subject: [PATCH 20/20] Resolved errors in ARP --- modules/arp/arp.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/modules/arp/arp.py b/modules/arp/arp.py index aedf62b270..e395b0be66 100644 --- a/modules/arp/arp.py +++ b/modules/arp/arp.py @@ -17,13 +17,13 @@ class Module(Module, multiprocessing.Process): description = 'Detect arp attacks' authors = ['Alya Gomaa'] - def __init__(self, outputqueue, redis_port): + def __init__(self, outputqueue, prefix:str, redis_port='6379'): multiprocessing.Process.__init__(self) # All the printing output should be sent to the outputqueue. # The outputqueue is connected to another process called OutputProcess self.outputqueue = outputqueue # Start the DB - __database__.start(redis_port) + __database__.start(prefix, redis_port) self.c1 = __database__.subscribe('new_arp') self.c2 = __database__.subscribe('tw_closed') self.read_configuration() @@ -397,13 +397,8 @@ def run(self): self.arp_ts = time.time() message = __database__.get_message(self.c1) - # if message and 'stop_process' in message['data']: - # print(f"ARP message for {message['data']}") + if message and message['data'] == 'stop_process': - with open('/home/ac/Desktop/workspace/message.txt', 'w') as file: - file.write(message['data']) - file.close() - self.shutdown_gracefully() return True