Adding new versions of the Multiprocessing implementation

Programas/Last_work/Multiprocess/bruteforce_sip-dip_threehash.py

@@ -1,28 +1,43 @@
-# Guarda lista de puertos de cada dip por cada sip
+####################################################
+# Researcher: Sara Schwarz
+# Advisor: Dr. Jose Ortiz
+#
+# Program Objective:
+#
+#In this work we seek to develop algorithms (brute-force approach, TRW, etc) to detect network and port
+#scanners on large-scale networks traffc such as those of Internet Service Providers,
+#Big Data research centers, and Science DMZ networks implemented in research institutions using network
+#flows. This specific program will follow a brute-force approach algorithm, of reading the network flows
+#and recording for each source ip the destination ips and the different destination ports it connected to.
+#Later, the number of connected ports will be compared with a threshold to classify the source ips as
+#either suspicious scanners or not.
+###############################################################
 
 from silk import *
 
 
-startDate = "2018/09/1"
-endDate = "2018/09/30"
-#Para filtrar por puertos. Pero no queremos todavia
-#minPort = 20
-#maxPort = 5000
 
+#input => list of files where each file has a list of network flows
+#output => threeway-hash table of the form {sip:{dip:{dport:total # of each dport}}}
+def verify_type(filename):
+    #print "Files", len(filename)
 
-def verify_type():
-    x = 0
-    dportHash = {} #contains amount of dport per each sip
-    for filename in FGlob(classname="all", type="all", start_date=startDate, end_date=endDate, site_config_file="/etc/silk/conf-v9/silk.conf", data_rootdir="/home/scratch/flow/rwflowpack/"):
-        for rec in silkfile_open(filename, READ):#reading the flow file
+    count_flow = 0
+    dportHash = {} #will contain amount of dport per dip per each sip
+    #iterates through each file that has a list of network flows
+    for file in filename:
+
+        #iterates through each network flow, and reads it using silkfile_open, SiLks Flow Repository function
+        for rec in silkfile_open(file, READ):
+            count_flow +=1
             sip = str(rec.sip)
             dip = str(rec.dip)
             dport = rec.dport
-            if (':' in sip): #Si en el paso anterior se vio que no
-                # print "heeloo", x
-                # x+=1
-                #                                             #tiene el length de puertos requerido, se ignora
+            #For our current research, we are limiting to only verifying IPv4 network flows
+            if (':' in sip):
                 continue
+            #using the hash table structure, keeps record of all the times each
+            #destination port for each destination ip was connected to by the same source ip
             else:
                 if sip in dportHash:
                     if dip in dportHash[sip]:
@@ -34,29 +49,40 @@ def verify_type():
                         dportHash[sip][dip] = {dport : 1}
                 else:
                     dportHash[sip] = { dip: {dport: 1} }
+
+
+    #print "flows", count_flow
     return dportHash
 
 
 #MAIN
-otherHash = {}
+#for accesing the network flows during a specific interval of time
+startDate = "2018/08/1"
+endDate = "2018/10/15"
+otherHash = {}#hash table for suspicious source ips with destination ips
 counter = 0
-files = FGlob(classname="all", type="all", start_date=startDate, end_date=endDate, site_config_file="/etc/silk/conf-v9/silk.conf", data_rootdir="/home/scratch/flow/rwflowpack/")
-files = [x for x in files]
-print "Flow", len(files)
+threshold = 100
+#object for retrieving files from Silk data store. Specified the interval of time, the silk configuration file location, and the silk data location
+files1 = FGlob(classname="all", type="all", start_date=startDate, end_date=endDate, site_config_file="/etc/silk/conf-v9/silk.conf", data_rootdir="/home/scratch/flow/rwflowpack/")
+files = [x for x in files1]
+#****************************************
+#  Verifying the source ips
+#****************************************
 flowHash = verify_type()
-print "After flow", len(flowHash)
-for sips in flowHash: #se itera por todos los dip y sus counters o puertos
+#Will iterate through the hash of sips and dports
+#Verify whether the number of different dports per desitnation ip, connected to by the same source ip
+#is greater than our threshold. If it is, the source ip will be added to the hash suspicious sips.
+
+for sips in flowHash:
     for dips, dports in flowHash[sips].items():
-        if len(dports) >= 100: #si la cantidad de puertos es mayor o igual a 100, nos interesan
-                                #y por lo tanto se guardan en un hash
-            print "DIP", dips, len(dports)
+        if len(dports) >= threshold:
             if sips in otherHash:
                 otherHash[sips][dips] = dports
             else:
                 otherHash[sips] = {dips: dports}
 
 for dips, dports in otherHash.items():
-    counter +=1 #para contar los elementos del hash
+    counter +=1
+    #prints the total number of suspicious source ips.
 
 print counter
-#print otherHash

Programas/Last_work/Multiprocess/map_bruteforce_three.py

@@ -1,5 +1,21 @@
-# Guarda lista de puertos de cada dip por cada sip
-#ftp remote edit
+####################################################
+# Researcher: Sara Schwarz
+# Advisor: Dr. Jose Ortiz
+# VERSION #1
+# Program Objective:
+#
+#In this work we seek to develop algorithms (brute-force approach, TRW, etc) to detect network and port
+#scanners on large-scale networks traffc such as those of Internet Service Providers,
+#Big Data research centers, and Science DMZ networks implemented in research institutions using network
+#flows. This specific program will follow a brute-force approach algorithm, of reading the network flows
+#and recording for each source ip the destination ips and the different destination ports it connected to.
+#Later, the number of connected ports will be compared with a threshold to classify the source ips as
+#either suspicious scanners or not.
+#To run this algorithm we will be using high-performance methods for computing such as Map and Reduce,
+#specifically Python's Pool Class Library.
+###############################################################
+
+
 from silk import *
 import multiprocessing as mp
 
@@ -63,36 +79,46 @@ def join_hash(list):
 
 
 def main():
-    startDate = "2018/09/1"
-    endDate = "2018/09/30"
-    otherHash = {}
+    #for accesing the network flows during a specific interval of time
+    startDate = "2018/08/1"
+    endDate = "2018/10/15"
+    otherHash = {} #hash table for suspicious source ips with destination ips
     counter = 0
-    process_num = 8
-    pool = mp.Pool(processes=process_num)
+    threshold = 100
+    #object for retrieving files from Silk data store. Specified the interval of time, the silk configuration file location, and the silk data location
     files = FGlob(classname="all", type="all", start_date=startDate, end_date=endDate, site_config_file="/etc/silk/conf-v9/silk.conf", data_rootdir="/home/scratch/flow/rwflowpack/")
-
-    files = [x for x in files]
-    print len(files)
-    fileHash = pool.map(verify_type, files) # FGlob(classname="all", type="all", start_date=startDate, end_date=endDate, site_config_file="/etc/silk/conf-v9/silk.conf", data_rootdir="/home/scratch/flow/rwflowpack/"))
+#****************************************
+# Using Pythons Pool Class for multiprocessing
+#****************************************
+    process_num = 2
+    pool = mp.Pool(processes=process_num)
+    #change files1 from FGlob object type to a list of silk data files
+    files1 = [x for x in files]
+    #Using map from Pool, returns a list of a hash per process, being the hash the output of verify_type
+    fileHash = pool.map(verify_type, files1)
+    #returns the hash of all hashes merged.
     flowHash = join_hash(fileHash)
-    print "FLOW", len(flowHash)
-    for sips in flowHash: #se itera por todos los dip y sus counters o puertos
-        #print sips
+    #print len(flowHash)
+    #****************************************
+#****************************************
+#  Verifying the source ips
+#****************************************
+    #Will iterate through the hash of sips and dports
+    #Verify whether the number of different dports per desitnation ip, connected to by the same source ip
+    #is greater than our threshold. If it is, the source ip will be added to the hash suspicious sips.
+    for sips in flowHash:
         for dips, dports in flowHash[sips].items():
-            #print "Dip", dips, dports
-            if len(dports) >= 100: #si la cantidad de puertos es mayor o igual a 100, nos interesan
-                                #y por lo tanto se guardan en un hash
-                print "DIP", dips, len(dports)
+            if len(dports) >= threshold:
                 if sips in otherHash:
                     otherHash[sips][dips] = dports
                 else:
                     otherHash[sips] = {dips: dports}
 
     for dips, dports in otherHash.items():
-        counter +=1 #para contar los elementos del hash
-
+        counter +=1
+    #prints the total number of suspicious source ips.
     print counter
-#print otherHash
+
 
 if __name__== "__main__":
   main()

Programas/Last_work/Multiprocess/map_bruteforce_three_2.py

@@ -0,0 +1,169 @@
+####################################################
+# Researcher: Sara Schwarz
+# Advisor: Dr. Jose Ortiz
+# VERSION #2
+# Program Objective:
+#
+#In this work we seek to develop algorithms (brute-force approach, TRW, etc) to detect network and port
+#scanners on large-scale networks traffc such as those of Internet Service Providers,
+#Big Data research centers, and Science DMZ networks implemented in research institutions using network
+#flows. This specific program will follow a brute-force approach algorithm, of reading the network flows
+#and recording for each source ip the destination ips and the different destination ports it connected to.
+#Later, the number of connected ports will be compared with a threshold to classify the source ips as
+#either suspicious scanners or not.
+#To run this algorithm we will be using high-performance methods for computing such as Map and Reduce,
+#specifically Python's Pool Class Library.
+###############################################################
+
+
+from silk import *
+import multiprocessing as mp
+
+
+#input => list of files where each file has a list of network flows
+#output => threeway-hash table of the form {sip:{dip:{dport:total # of each dport}}}
+def verify_type(filename):
+    #print "Files", len(filename)
+
+    count_flow = 0
+    dportHash = {} #will contain amount of dport per dip per each sip
+    #iterates through each file that has a list of network flows
+    for file in filename:
+
+        #iterates through each network flow, and reads it using silkfile_open, SiLks Flow Repository function
+        for rec in silkfile_open(file, READ):
+            count_flow +=1
+            sip = str(rec.sip)
+            dip = str(rec.dip)
+            dport = rec.dport
+            #For our current research, we are limiting to only verifying IPv4 network flows
+            if (':' in sip):
+                continue
+            #using the hash table structure, keeps record of all the times each
+            #destination port for each destination ip was connected to by the same source ip
+            else:
+                if sip in dportHash:
+                    if dip in dportHash[sip]:
+                        if dport in dportHash[sip][dip]:
+                            dportHash[sip][dip][dport] += 1
+                        else:
+                            dportHash[sip][dip][dport] = 1
+                    else:
+                        dportHash[sip][dip] = {dport : 1}
+                else:
+                    dportHash[sip] = { dip: {dport: 1} }
+
+
+    #print "flows", count_flow
+    return dportHash
+
+#input => list of hash tables of the form {sip:{dip:{dport:total # of each dport}}}
+#output => hash table of the form {sip:{dip:{dport:total # of each dport}}}
+#Method => iterates through each hash
+def join_hash(list):
+    complete_hash ={} #will contain the hash table of all the hash tables once formed
+
+    for i in list: #will iterate through each hash of the list
+        #will iterate through each source ip key as sip with the hash table of the form {dip:{dport:#}} as the value
+        for sip, hash in i.items():
+            #verify if that source ip is already in the merged hash.
+            if sip in complete_hash:
+                #will iterate through each destination ip key as dip with the hash table of the form {dport:#} as the value
+                for dip, dports in i[sip].items():
+                    #verify if that destination ip is already in the hash of the sip's value in the merged hash
+                    if dip in complete_hash[sip]:
+                        #will iterate through each destination port key as number with the total # of times it was connected as the value
+                        for number, value in dports.items():
+                            #verify if that destination port is already in the hash of the dip's value in the hash of the sips'value in the merged hash
+                            if number in complete_hash[sip]:
+                                #Add the value to the dport's value in the merged hash
+                                complete_hash[sip][dip][number] += value
+
+                            else:
+                                #Add the destination port with its value to the merged hash
+                                complete_hash[sip][dip][number]= value
+                    else:
+                        #Add the destination ip with its hash table of dports to the merged hash
+                        complete_hash[sip][dip]= dports
+            else:
+                #Add the source ip with its hash table of destination ip and dports to the merged hash
+                complete_hash[sip]= hash
+    return complete_hash
+
+
+def main():
+    #for accesing the network flows during a specific interval of time
+    startDate = "2018/08/1"
+    endDate = "2018/10/15"
+    otherHash = {} #hash table for suspicious source ips with destination ips
+    counter = 0
+    threshold = 100
+    #object for retrieving files from Silk data store. Specified the interval of time, the silk configuration file location, and the silk data location
+    files = FGlob(classname="all", type="all", start_date=startDate, end_date=endDate, site_config_file="/etc/silk/conf-v9/silk.conf", data_rootdir="/home/scratch/flow/rwflowpack/")
+#****************************************
+# Using Pythons Pool Class for multiprocessing
+#****************************************
+    process_num = 2
+    pool = mp.Pool(processes=process_num)
+    #change files1 from FGlob object type to a list of silk data files
+    files1 = [x for x in files]
+    files_list = []
+    #Send each process an equal amount of files
+    blocksize = len(files1) / process_num
+    for x in range(process_num):
+        files_list.append(files1[0:blocksize])
+        files1 = files1[blocksize:]
+    #in the case that the number of files is not divisible by the number of processes
+    for i in files1:
+        files_list[files1.index(i)].append(i)
+    #Using map from Pool, returns a list of a hash per process, being the hash the output of verify_type
+    fileHash = pool.map(verify_type, files_list)
+    #returns the hash of all hashes merged.
+    flowHash = join_hash(fileHash)
+    #print len(flowHash)
+#****************************************
+
+#****************************************
+#  Verifying the source ips
+#****************************************
+    #Will iterate through the hash of sips and dports
+    #Verify whether the number of different dports per desitnation ip, connected to by the same source ip
+    #is greater than our threshold. If it is, the source ip will be added to the hash suspicious sips.
+    for sips in flowHash:
+        for dips, dports in flowHash[sips].items():
+            if len(dports) >= threshold:
+                if sips in otherHash:
+                    otherHash[sips][dips] = dports
+                else:
+                    otherHash[sips] = {dips: dports}
+
+    for dips, dports in otherHash.items():
+        counter +=1
+    #prints the total number of suspicious source ips.
+    print counter
+
+
+if __name__== "__main__":
+  main()
+
+
+
+
+  # def Filter(fd, sip, dip, sport, dport, dportHash):
+  #
+  # 	if dport > 1024 and (sport <= 1024 or (sport >= 8000 and sport < 9000)):
+  # 		return
+  # 	if sip in dportHash:
+  # #       	if dip in dportHash[sip]["dips"]:
+  # #        	        dportHash[sip]["dips"][dip] += 1
+  # #		else:
+  # #			dportHash[sip]["dips"][dip] = 1
+  # 		if dport in dportHash[sip]["dports"]:
+  #                	        dportHash[sip]["dports"][dport] += 1
+  # 			#return
+  #                	else:
+  #                		dportHash[sip]["dports"][dport] = 1
+  #      	else:
+  # 		dportHash[sip] = {"dports": {}}
+  # 		dportHash[sip]["dips"] = {}
+  # 	#fd.write("%s:%s:%s:%s\n" % (sip, dip, sport, dport))

Programas/Last_work/Multiprocess/trial_map.py

@@ -0,0 +1,24 @@
+#Trial program to practice using Pool class
+
+import multiprocessing as mp
+
+
+process_num = 8
+pool = mp.Pool(processes=process_num)
+files1 = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26]
+
+files_list = []
+quenepas_blocksize = len(files1) / process_num
+for x in range(process_num):
+    files_list.append(files1[0:quenepas_blocksize])
+    files1 = files1[quenepas_blocksize:]
+    print files1
+print len(files1)
+print files_list
+for i in files1:
+    print files1.index(i)
+    print i
+    files_list[files1.index(i)].append(i)
+
+print files1
+print files_list

Programas/Last_work/Multiprocess/trw.py

@@ -1,32 +1,47 @@
-##################################
-#   TRW sin reduccion           #
-#      Para Data Set de la Uni    #
-################################
+####################################################
+# Researcher: Sara Schwarz
+# Advisor: Dr. Jose Ortiz
+# VERSION #2
+# Program Objective:
+#
+#In this work we seek to develop algorithms (brute-force approach, TRW, etc) to detect network and port
+#scanners on large-scale networks traffc such as those of Internet Service Providers,
+#Big Data research centers, and Science DMZ networks implemented in research institutions using network
+#flows. This specific program will follow the threshold random walk algorithm, of reading the network flows
+#and recording for each source ip amount of succesful connections and failed connections.
+#            Succesful connections => It means to have completed the three way handshake, to have received the
+#                                       Acknowledgment from the desitnaiton node.
+#            Failed connections => It means to not have completed the three way handshake, to have not received
+#                                   any answer, meaning it only has the Syncronization.
+#Later, the ratio between failed and succesful connections will be compared with a threshold to classify the source ips as
+#either suspicious scanners or not.
+###############################################################
 
 
 
 from silk import *
-startDate = "2018/08/14"
-endDate = "2018/08/15"
-p = 2
+
 
 def Analisis():
     counter = 0 # borrar luego
-    sampleHash={} #hash para contener los dip con el numero de conecciones y failed coneccciones
+    counter_files=0
+    sampleHash={} #hash contains each sip with their dip and failed and succesful connections
     flow_counter = 0
     for filename in FGlob(classname="all", type="all", start_date=startDate, end_date=endDate, site_config_file="/etc/silk/conf-v9/silk.conf", data_rootdir="/home/scratch/flow/rwflowpack/"):
+        counter_files +=1
         for rec in silkfile_open(filename, READ):#reading the flow file
             flow_counter += 1
             if (':' in str(rec.sip)):
 				continue
 	    else:
-            	connection = [0] * 2 #Lista para contener los valores de conecciones failed y conecciones buenas
-            	sip = str(rec.sip) #Devuelve el ip en notacion punto-decimal
+            	connection = [0] * 2 #array to contain the amount of failed connections and amount of succesful conenctions
+            	sip = str(rec.sip)
             	flags = str(rec.tcpflags)
             #print sip, flags
 	    #counter +=1
-            	if 'A' in flags: ####arreglar
-                	connection[1]=1 #good conections
+                #verify if the network flow contains the Acknowledgment flag, which will imply a succesful connection
+            	if 'A' in flags:
+                	connection[1]=1 #succesful conections
                 else:
                 	connection [0] =1 #failed conections
             	if sip in sampleHash:
@@ -35,27 +50,49 @@ def Analisis():
             	else:
                 	sampleHash[sip] = [connection[0], connection[1]]
 	    #print sampleHash
-    #print flow_counter
+    # print "flows", flow_counter
+    # print counter_files
     return sampleHash
 
-sip_connections_list = Analisis()
+def main():
+    startDate = "2018/06/1"
+    endDate = "2018/06/30"
+    p_ratio = 2 #threshold of the ratio. This ratio is chosen.
+    p_counter = 100 #if there are only failed connections, rather than the ratio,
+                    #the total number of failed connection is compared to this threshold.
+    sip_connections_list = Analisis()
+    #Receives the list hash of sip with the number of connections per dip
+    #print len(sip_connections_list)
 #print sip_connections_list
-sipList = {"sipList":[]}
-for sip in sip_connections_list:
-          if (sip_connections_list[sip][1] != 0) and ((sip_connections_list[sip][0] / sip_connections_list[sip][1])  < 1) : #si la cantidad de succesful
-                                #b                #g                    #connections es mas que failed connections
-                      #not scanner, ignore
-                      continue
-          elif (sip_connections_list[sip][1] != 0) and ((sip_connections_list[sip][0] / sip_connections_list[sip][1])  < p): #mas failed que succesful, pero no llega al threshold
-                      #not scanner, ignore
-                      continue                          #se debe tener en cuenta que es suspicious pero no taaanto
-          elif (sip_connections_list[sip][1] == 0 and sip_connections_list[sip][0] > 10): #el ratio de failed a succesful llega al threshold pautado
-                            #scanner, oh oh
-                    hash = {sip:sip_connections_list[sip]}
-                    sipList["sipList"].append(hash)
-          else:
-                            #scanner, oh oh
-                    hash = {sip:sip_connections_list[sip]}
-                    sipList["sipList"].append(hash)
-
-print sipList
+    counter = 1
+    sipList = {"sipList":[]}
+    for sip in sip_connections_list:
+        #print counter, sip, "sip0", sip_connections_list[sip][0], "sip1", sip_connections_list[sip] [1]
+        counter +=1
+
+        #compares the ratio of succesful connections to failed connections with a given threshold
+        #If the amount is larger than the threshold it is added to a list of suspicious sips.
+        if (sip_connections_list[sip][1] != 0) and ((sip_connections_list[sip][0] / sip_connections_list[sip][1])  > p_ratio):
+            hash = {sip:sip_connections_list[sip]}
+            #print hash
+            sipList["sipList"].append(hash)
+            continue
+
+        elif (sip_connections_list[sip][1] == 0 and sip_connections_list[sip][0] > p_counter):
+
+            hash = {sip:sip_connections_list[sip]}
+            #print hash
+            sipList["sipList"].append(hash)
+            continue
+        #If it does not reach the threshold, the network flow is ignored.
+        else:
+            continue
+
+    counter = 0
+    for i in sipList["sipList"]:
+        counter +=1
+    print counter
+
+
+if __name__== "__main__":
+  main()

Programas/Last_work/Multiprocess/trw_map.py

@@ -0,0 +1,137 @@
+####################################################
+# Researcher: Sara Schwarz
+# Advisor: Dr. Jose Ortiz
+# VERSION #2
+# Program Objective:
+#
+#In this work we seek to develop algorithms (brute-force approach, TRW, etc) to detect network and port
+#scanners on large-scale networks traffc such as those of Internet Service Providers,
+#Big Data research centers, and Science DMZ networks implemented in research institutions using network
+#flows. This specific program will follow the threshold random walk algorithm, of reading the network flows
+#and recording for each source ip amount of succesful connections and failed connections.
+#            Succesful connections => It means to have completed the three way handshake, to have received the
+#                                       Acknowledgment from the desitnaiton node.
+#            Failed connections => It means to not have completed the three way handshake, to have not received
+#                                   any answer, meaning it only has the Syncronization.
+#Later, the ratio between failed and succesful connections will be compared with a threshold to classify the source ips as
+#either suspicious scanners or not.
+#To run this algorithm we will be using high-performance methods for computing such as Map and Reduce,
+#specifically Python's Pool Class Library.
+###############################################################
+
+from silk import *
+import multiprocessing as mp
+
+
+def Analisis():
+    counter = 0 # borrar luego
+    counter_files=0
+    sampleHash={} #hash contains each sip with their dip and failed and succesful connections
+    flow_counter = 0
+    for filename in FGlob(classname="all", type="all", start_date=startDate, end_date=endDate, site_config_file="/etc/silk/conf-v9/silk.conf", data_rootdir="/home/scratch/flow/rwflowpack/"):
+        counter_files +=1
+        for rec in silkfile_open(filename, READ):#reading the flow file
+            flow_counter += 1
+            if (':' in str(rec.sip)):
+				continue
+	    else:
+            	connection = [0] * 2 #array to contain the amount of failed connections and amount of succesful conenctions
+            	sip = str(rec.sip)
+            	flags = str(rec.tcpflags)
+            #print sip, flags
+	    #counter +=1
+                #verify if the network flow contains the Acknowledgment flag, which will imply a succesful connection
+            	if 'A' in flags:
+                	connection[1]=1 #succesful conections
+                else:
+                	connection [0] =1 #failed conections
+            	if sip in sampleHash:
+	                sampleHash[sip][0]+= connection[0]
+	                sampleHash[sip][1]+= connection[1]
+            	else:
+                	sampleHash[sip] = [connection[0], connection[1]]
+	    #print sampleHash
+    # print "flows", flow_counter
+    # print counter_files
+    return sampleHash
+
+
+
+def merge_list(list_hash):
+    sampleHash = {}
+    for sip_hash in list_hash:
+        #print sip_hash
+        for sip, arr in sip_hash.items():
+            if sip in sampleHash:
+                sampleHash[sip][0]+= arr[0]
+                sampleHash[sip][1]+= arr[1]
+            else:
+                sampleHash[sip] = [arr[0], arr[1]]
+    return sampleHash
+
+
+
+def main():
+    startDate = "2018/06/1"
+    endDate = "2018/06/30"
+    p_ratio = 2 #threshold of the ratio. This ratio is chosen.
+    p_counter = 100 #if there are only failed connections, rather than the ratio,
+                    #the total number of failed connection is compared to this threshold.
+    process_num = 2
+    pool = mp.Pool(processes=process_num)
+    files1 = FGlob(classname="all", type="all", start_date=startDate, end_date=endDate, site_config_file="/etc/silk/conf-v9/silk.conf", data_rootdir="/home/scratch/flow/rwflowpack/")
+#****************************************
+
+#****************************************
+    files1 = [x for x in files1] #change files1 from FGlob object to a list
+        #print len(files1)
+    files_list = []
+    quenepas_blocksize = len(files1) / process_num
+    for x in range(process_num):
+        files_list.append(files1[0:quenepas_blocksize])
+        files1 = files1[quenepas_blocksize:]
+
+    for i in files1:
+        files_list[files1.index(i)].append(i)
+
+#****************************************
+
+#****************************************
+
+    fileHash = pool.map(Analisis, files_list) # FGlob(classname="all", type="all", start_date=startDate, end_date=endDate, site_config_file="/etc/silk/conf-v9/silk.conf", data_rootdir="/home/scratch/flow/rwflowpack/"))
+    #print fileHash[0]['136.145.231.48']
+    sip_connections_list = merge_list(fileHash)
+    #print sip_connections_list['136.145.231.48']
+#print sip_connections_list
+counter = 1
+sipList = {"sipList":[]}
+for sip in sip_connections_list:
+    #print counter, sip, "sip0", sip_connections_list[sip][0], "sip1", sip_connections_list[sip] [1]
+    counter +=1
+
+    #compares the ratio of succesful connections to failed connections with a given threshold
+    #If the amount is larger than the threshold it is added to a list of suspicious sips.
+    if (sip_connections_list[sip][1] != 0) and ((sip_connections_list[sip][0] / sip_connections_list[sip][1])  > p_ratio):
+        hash = {sip:sip_connections_list[sip]}
+        #print hash
+        sipList["sipList"].append(hash)
+        continue
+
+    elif (sip_connections_list[sip][1] == 0 and sip_connections_list[sip][0] > p_counter):
+
+        hash = {sip:sip_connections_list[sip]}
+        #print hash
+        sipList["sipList"].append(hash)
+        continue
+    #If it does not reach the threshold, the network flow is ignored.
+    else:
+        continue
+
+counter = 0
+for i in sipList["sipList"]:
+    counter +=1
+print counter
+
+
+if __name__== "__main__":
+main()