# -*- coding: utf-8 -*-
'''
Created on 2015-02-22

BurpSmartBuster
@author: @pathetiq
@thanks: Abhineet & @theguly
@version: 0.3
@summary: This is a Burp Suite extension which discover content with a smart touch. A bit like “DirBuster” and “Burp Discover Content”,
          but smarter and being integrated into Burp Suite this plugin looks at words in pages, the domain name, the current directories and filename
          to help you find hidden files, directories and information you usually don't with a static dictionary file that brute force its way on the web server.

@bug: URL with variable, no file, no extension or weird variable separate by ; :, etc. breaks the directories/files listing
@todo: technology detection and scanning, community files, add 404 detection in output, threads speeds and adjustments
@todo: Add results to an issue. add tested files somewhere, add found file to sitemap.

'''
import os
os.environ["NLTK_DATA"] = os.path.join(os.getcwd(), "nltk_data")

#sys imports
import sys

#Find the jython path where our prerequisites packages are installed
import site
for site in site.getsitepackages():
    sys.path.append(site)
#Examples of paths if needed
#sys.path.append("/home/USERNAME/.local/lib/python2.7/site-packages/")
#sys.path.append("/usr/local/lib/python2.7/site-packages")
##sys.path.append("/usr/lib/python2.7/dist-packages/")
#sys.path.append("/home/USERNAME/Documents/Apps/TextBlob")
#sys.path.append("/home/USERNAME/Documents/Apps/nltk")

#burp imports
from burp import IBurpExtender
from burp import IScanIssue
from burp import IScannerCheck
from burp import IScannerInsertionPoint
from burp import IHttpListener
from burp import IBurpExtenderCallbacks

#UI Import
from burp import IContextMenuFactory
from java.util import List, ArrayList
from burp import ITab
from javax.swing import JPanel, JLabel, JMenuItem, JTextField, JList, DefaultListModel, JButton, JFileChooser
from javax.swing import JScrollPane, ListSelectionModel, GroupLayout, ButtonGroup, JRadioButton
from java.awt import Dimension
from java.awt import Toolkit
from java.awt.datatransfer import StringSelection

#utils imports
from array import array
from java.io import PrintWriter
from java.net import URL
import os
import ConfigParser
import json
import logging
from tld import get_tld
import hashlib
import random

#spidering
from bs4 import BeautifulSoup
import Queue

#Parse HTML comments
from bs4 import Comment
import re
from urlparse import urlparse

#requester
import requests
import csv
from collections import deque
import threading

#text tokenization & natural language lib
locals()
#TODO: REVALIDATE the following : file /usr/local/lib/python2.7/dist-packages/nltk/internals.py line 902 has been change to remove os.getgroups() to compile in Burp...Jhython?
#http://textminingonline.com/getting-started-with-textblob
from textblob import TextBlob




'''----------------------------------------------------------------------------------------------------------------------------------------
BurpSmartBuster Logging object and config
----------------------------------------------------------------------------------------------------------------------------------------'''
class Logger():

    LOG_FILENAME = 'BSB.log'
    DEFAULT_LEVEL = logging.DEBUG

    def __init__(self,name=LOG_FILENAME,level=DEFAULT_LEVEL):

        #define configs
        self._default_level=level
        self._name = name
        print "Log file is: " + name

        logging.basicConfig(filename=self._name+".log",
                            level=self._default_level,
                            format="%(asctime)s - [%(levelname)s] [%(threadName)s] (%(funcName)s:%(lineno)d) %(message)s",
                            )

        self._logger = logging.getLogger(name)
        return

    def getLogger(self):
        return self._logger


'''----------------------------------------------------------------------------------------------------------------------------------------
BurpSmartBuster main class (BurpExtender)
----------------------------------------------------------------------------------------------------------------------------------------'''
class BurpExtender(IBurpExtender, IScanIssue, IScannerCheck, IScannerInsertionPoint,IHttpListener, IBurpExtenderCallbacks, IContextMenuFactory, ITab):

    # definitions
    EXTENSION_NAME = "BurpSmartBuster"
    AUTHOR = "@pathetiq"

    def registerExtenderCallbacks(self, callbacks):
        # keep a reference to our callbacks object
        self._callbacks = callbacks

        # obtain an extension helpers object
        self._helpers = callbacks.getHelpers()

        # define stdout writer
        self._stdout = PrintWriter(callbacks.getStdout(), True)

        print(self.EXTENSION_NAME + ' by ' + self.AUTHOR)
        print('================================')
        print('This extension will create new requests for ALL "in scope" HTTP request made through Burp. Make sure to filter scope items')
        print('For help or any information see the github page or contact the author on twitter.')
        print('Note: The Spider currently only supports English, see author github page for new language installation instructions')

        # set our extension name
        callbacks.setExtensionName(self.EXTENSION_NAME)
        callbacks.registerScannerCheck(self)
        callbacks.registerHttpListener(self)
        callbacks.registerContextMenuFactory(self)

        #Initialize tab details

        #fields of options setBounds(x,y,width,heigth)
        self.verboseLabel = JLabel("Verbose")
        self.verboseLabel.setBounds(10,10,130,30)

        self.yesVerboseButton = JRadioButton("Yes")
        self.yesVerboseButton.setSelected(True)
        self.yesVerboseButton.setBounds(10,40,50,30)
        self.noVerboseButton = JRadioButton("No")
        self.noVerboseButton.setBounds(70,40,50,30)

        self.buttonGroup = ButtonGroup()
        self.buttonGroup.add(self.yesVerboseButton)
        self.buttonGroup.add(self.noVerboseButton)

        self.spiderPagesLabel = JLabel("Spider: Nbr of pages")
        self.spiderPagesLabel.setBounds(10,70,200,30)
        self.spiderPagesTextField = JTextField(300)
        self.spiderPagesTextField.setText("5")
        self.spiderPagesTextField.setBounds(10,100,300,30)
        self.spiderPagesTextField.setPreferredSize( Dimension( 250, 20 ) )

        self.spiderRecPagesLabel = JLabel("Recursive: Nbr of pages")
        self.spiderRecPagesLabel.setBounds(10,130,250,30)
        self.spiderRecPagesTextField = JTextField(300)
        self.spiderRecPagesTextField.setText("3")
        self.spiderRecPagesTextField.setBounds(10,160,300,30)
        self.spiderRecPagesTextField.setPreferredSize( Dimension( 250, 20 ) )

        self.fileTypeLabel = JLabel("Ignore Filetypes")
        self.fileTypeLabel.setBounds(10,190,130,30)
        self.fileTypeTextField = JTextField(300)
        self.fileTypeTextField.setText("gif,jpg,png,css,js,ico,woff")
        self.fileTypeTextField.setBounds(10,220,300,30)
        self.fileTypeTextField.setPreferredSize( Dimension( 250, 20 ) )

        self.inScopeLabel = JLabel("Scan in-scope URLs only?")
        self.inScopeLabel.setBounds(10,250,200 ,30)

        self.yesInScopeButton = JRadioButton("Yes")
        self.yesInScopeButton.setBounds(10,280,50,30)
        self.yesInScopeButton.setSelected(True)
        self.noInScopeButton = JRadioButton("No")
        self.noInScopeButton.setBounds(70,280,50,30)

        self.buttonGroup1 = ButtonGroup()
        self.buttonGroup1.add(self.yesInScopeButton)
        self.buttonGroup1.add(self.noInScopeButton)

        self.refreshConfigButton = JButton("Update Configuration", actionPerformed=self.updateConfig)
        self.refreshConfigButton.setBounds(10,310,200,30)

        #Jlist to contain the results
        self.list = JList([])
        self.list.setSelectionMode(ListSelectionModel.MULTIPLE_INTERVAL_SELECTION)
        self.list.setLayoutOrientation(JList.VERTICAL)
        self.list.setVisibleRowCount(-1)
        self.listScroller = JScrollPane(self.list,JScrollPane.VERTICAL_SCROLLBAR_AS_NEEDED,JScrollPane.HORIZONTAL_SCROLLBAR_AS_NEEDED)
        self.listScroller.setBounds(510,40,500,500)
        #self.listScroller.setPreferredSize(Dimension(400, 500))

        self.urlFoundLabel = JLabel("URLs Found")
        self.urlFoundLabel.setBounds(510,10,130,30)
        self.listScroller.setPreferredSize(Dimension(500, 100))
        self.listScroller.setViewportView(self.list)

        self.clearListButton = JButton("Clear list", actionPerformed=self.clearList)
        self.clearListButton.setBounds(350,40,150,30)

        self.copyListButton = JButton("Copy Selected", actionPerformed=self.copyList)
        self.copyListButton.setBounds(350,70,150,30)

        self.deleteListButton = JButton("Delete Selected", actionPerformed=self.deleteSelected)
        self.deleteListButton.setBounds(350,100,150,30)

        self.exportListButton = JButton("Export list", actionPerformed=self.exportList)
        self.exportListButton.setBounds(350,130,150,30)


        #main panel
        self.mainpanel = JPanel()
        self.mainpanel.setLayout(None)

        self.mainpanel.add(self.verboseLabel)
        self.mainpanel.add(self.yesVerboseButton)
        self.mainpanel.add(self.noVerboseButton)
        self.mainpanel.add(self.spiderPagesLabel)
        self.mainpanel.add(self.spiderPagesTextField)
        self.mainpanel.add(self.spiderRecPagesLabel)
        self.mainpanel.add(self.spiderRecPagesTextField)
        self.mainpanel.add(self.fileTypeLabel)
        self.mainpanel.add(self.fileTypeTextField)
        self.mainpanel.add(self.inScopeLabel)
        self.mainpanel.add(self.yesInScopeButton)
        self.mainpanel.add(self.noInScopeButton)
        self.mainpanel.add(self.refreshConfigButton)
        self.mainpanel.add(self.urlFoundLabel)
        self.mainpanel.add(self.listScroller)
        self.mainpanel.add(self.clearListButton)
        self.mainpanel.add(self.copyListButton)
        self.mainpanel.add(self.deleteListButton)
        self.mainpanel.add(self.exportListButton)

        callbacks.customizeUiComponent(self.mainpanel)
        callbacks.addSuiteTab(self)

        #set default config file name and values

        #only smart is use, keeping other for future development
        self._configSmart_Local = False
        self._configSmart_Smart = True
        self._configSmart_File = False
        self._configSmart_Spider = False
        self._trailingSlash = True

        #To be fetch from the UI settings
        self._configSpider_NumberOfPages = 5
        self._verbose = False
        self._ignoreFileType = ["gif","jpg","png","css","js","ico","woff"]
        #keeping to use it
        self._configInScope_only = True
        self._configSpider_NumberOfPages = 5

        #Get a logger object for logging into file
        loggerTemp = Logger(self.EXTENSION_NAME,logging.DEBUG)
        self._logger= loggerTemp.getLogger()

        #get the config file, will overwrite default config if the ini file is different
        #self.getSmartConfiguration()

        #get config from the UI
        self.updateConfig("")

        #words gather on the page from the spidering
        self._words = {}
        self._mergedWords = {}

        #robots.txt list
        self._robots = {}
        self._robotsScanned = {}

        #sitemap.xml list
        self._sitemap = {}

        #url in comments
        self._urlsInComment = {}

        #domain names to query current url/path/files for hidden items
        self._smartDomain = {}

        #sitemap and robots scanned once
        self._siteRobotScanned = {}

        #Load our BSB json data
        self._jsonFile = "data.json"
        jsonfile = open(self._jsonFile)
        self._parsed_json = json.load(jsonfile)
        jsonfile.close()

        #define the request object to use each time we need to call a URL
        self._requestor = Requestor(self._logger,self)

        #Variable to define if unique data has already been grabbed
        self._smartRequestData = {}
        self._smartRequestPath = {}
        self._smartRequestFiles = {}
        #number of time the spider have run
        self._spiderRan = {} #Array of domain. If domain exist. Spider did ran!

        return

    '''
    Graphic Functions
    '''
    def createMenuItems(self, contextMenuInvocation):
        self._contextMenuData = contextMenuInvocation.getSelectedMessages()
        menu_list = ArrayList()
        menu_list.add(JMenuItem("Send to BurpSmartBuster",actionPerformed=self.menuItemClicked))
        return menu_list

    def menuItemClicked(self, event):
        data = self.getURLdata(self._contextMenuData[0],True)
        self._logger.info("SMARTREQUEST FOR: "+data.getUrl().toString())
        self._logger.debug("Executing: smartRequest() from menuItemClicked")
        thread = threading.Thread(
            target=self.smartRequest,
            name="Thread-smartRequest",
            args=[data],)
        thread.start()

    # Implement ITab
    def getTabCaption(self):
        return self.EXTENSION_NAME

    # Return our panel and button we setup. Components of our extension's tab
    def getUiComponent(self):
        return self.mainpanel

    '''------------------------------------------------
    Extension Unloaded
    ------------------------------------------------'''
    def extensionUnloaded(self):
        self._logger.info("Extension was unloaded")
        return

    '''------------------------------------------------
    VERBOSE FUNCTION

    Display each tested URL
    ------------------------------------------------'''
    def verbose(self,text):
        #Is verbose on or off from config file?
        if self._verbose == True:
            print "[VERBOSE]: "+text
        return

    '''------------------------------------------------
    GRAPHICAL FUNCTIONS for BUTTONS
    ------------------------------------------------'''

    def getRecursiveConfig(self):
        return int(self.spiderRecPagesTextField.getText())

    #refresh the config from the UI
    def updateConfig(self,meh):
        self._configSpider_NumberOfPages = int(self.spiderPagesTextField.getText())

        if self.yesVerboseButton.isSelected():
            self._verbose = True
        else:
            self._verbose = False

        if self.yesInScopeButton.isSelected():
            self._configInScope_only = True
        else:
            self._configInScope_only = False

        fileType = []
        fileTypeStr = self.fileTypeTextField.getText()
        self._ignoreFileType = self.fileTypeTextField.getText().split(",")

        self._logger.info("Config changed: " + "spiderNbrPages=" + str(self._configSpider_NumberOfPages) + ", Verbose is:" + str(self._verbose) + ", InScope is:" + str(self._configInScope_only) + ", fileTypeIgnored: " + str(self._ignoreFileType))
        print "Now using config: " + "spiderNbrPages=" + str(self._configSpider_NumberOfPages) + ", Verbose is:" + str(self._verbose) + ", InScope is:" + str(self._configInScope_only) + ", fileTypeIgnored: " + str(self._ignoreFileType)

        return

    #add a URL to the list
    def addURL(self,url):
        list = self.getListData()
        list.append(url)

        self.list.setListData(list)
        return

    #return the who list
    def getListData(self):
        list = []

        for i in range(0, self.list.getModel().getSize()):
            list.append(self.list.getModel().getElementAt(i))

        return list

    #Clear the list
    def clearList(self,meh):
        self.list.setListData([])
        return

    #Copy to clipboard
    def copyList(self,meh):
        clipboard = Toolkit.getDefaultToolkit().getSystemClipboard()
        list = self.getListData()
        selected = self.list.getSelectedIndices().tolist()

        copied = ""
        urls = ""
        for i in selected:
            url = str(list[i]).split(',')[0]
            urls = urls+str(url)+"\n"

        clipboard.setContents(StringSelection(urls), None)

        return

    #Delete selected item from the list
    def deleteSelected(self,meh):
        x = self.list.getSelectedIndices().tolist()
        list = self.getListData()

        for i in reversed(x):
            del list[i]

        self.list.setListData(list)
        return

    #TODO: save as the list
    def exportList(self,meh):
        fd = JFileChooser()
        dialog = fd.showDialog(self.mainpanel, "Save List As")

        dataList = self.getListData()

        urls = ""

        if dialog == JFileChooser.APPROVE_OPTION:
            file = fd.getSelectedFile()
            path = file.getCanonicalPath()

            try:
                with open(path, 'w') as exportFile:
                    for item in dataList:
                        url = str(item).split(',')[0]
                        exportFile.write(url+"\n")
            except IOError as e:
                print "Error exporting list: " + str(e)
                self._logger.debug("Error exporting list to: " + path + ", Error: " + str(e))

        return

    '''------------------------------------------------------------------------------------------------
    MAIN FUNCTION / WHERE EVERYTHING STARTS

    For every request which isn't created from the Extender(this might have to be change)
    The request is analyse and related to the config options new request are create to test if
    specific files/paths/directories exists.
    ------------------------------------------------------------------------------------------------'''
    def processHttpMessage(self, toolFlag, messageIsRequest, messageInfo): #IHttpRequestResponse message info


        #TODO: not from repeater and intruder --> set in ini file too! --> and toolFlag != self._callbacks.TOOL_EXTENDER

        #This is required to not LOOP Forever as our plugin generate requests!
        if toolFlag == self._callbacks.TOOL_PROXY and toolFlag != self._callbacks.TOOL_EXTENDER and toolFlag != self._callbacks.TOOL_SCANNER:

            #Get an Urldata object to use later
            data = self.getURLdata(messageInfo,messageIsRequest)

            #VERIFICATION: if URL is in scope we do scan
            if not self._callbacks.isInScope(data.getUrl()):
                #self._callbacks.includeInScope(url)
                self._logger.info("URL not in scope: " + data.getUrl().toString())
                return

            if messageIsRequest:
                self._logger.debug("Entering: processHttpMessage() REQUEST")
                self._logger.debug("Request from domain: "+data.getDomain())

                #REJECT specific extension on request
                if data.getFileExt() in self._ignoreFileType:
                    self._logger.info("FILETYPE IGNORED: " + data.getUrl().toString())
                    return

                ###############################################
                # Decide which mode to use based on ini config
                ###############################################

                #from browsed file only
                if self._configSmart_Smart:
                    self._logger.info("SMARTREQUEST FOR: "+data.getUrl().toString())
                    self._logger.debug("Executing: smartRequest()")
                    thread = threading.Thread(
                        target=self.smartRequest,
                        name="Thread-smartRequest",
                        args=[data],
                    )
                    thread.start()
                    thread.join()

                #wordlist adjust with the domain name
                elif self._configSmart_Local:
                    self._logger.debug("Executing: localRequest()")
                    self.localRequest(data)

                #your own wordlist, no smart here
                elif self._configSmart_File:
                    self._logger.debug("Executing: fileRequest()")
                    self.fileRequest(data)

                #spidered items only. Like smart but it browse for you.
                elif self._configSmart_Spider:
                    self._logger.debug("Executing: spiderRequest()")
                    self.spiderRequest(data)

            else: #if response
                self._logger.debug("Entering: processHttpMessage() RESPONSE")

                ###############################################
                # Decide which mode to use based on ini config
                ###############################################
                #VERIFICATION: if URL is in scope we do scan
                #if not self._callbacks.isInScope(data.getUrl()):
                #    #self._callbacks.includeInScope(url)
                #    self._logger.info("URL %s not in scope: " % data.getUrl())
                #    return

                #from browsed file only
                #TODO: sniff JS and CSS file for URLS
                #if self._configSmart_Smart:
                self._logger.debug("Executing: getUrlInComments()")
                thread = threading.Thread(
                    target=self.getUrlInComments,
                    name="Thread-getUrlInComments",
                    args=[data],
                )
                thread.start()
                thread.join()
        return

    '''----------------------------------------------------------------------------------------------------------
    BurpSmartBuster main class (BurpExtender)
    Only spidering to gather the more page and test those
    ----------------------------------------------------------------------------------------------------------'''
    def spiderRequest(self, data):
        return

    '''----------------------------------------------------------------------------------------------------------
    Use BSB files on all visited page
    ----------------------------------------------------------------------------------------------------------'''
    def localRequest(self, data):
        return

    '''----------------------------------------------------------------------------------------------------------
    Use user supply file on all visited page
    ----------------------------------------------------------------------------------------------------------'''
    def fileRequest(self, data):
        return


    '''----------------------------------------------------------------------------------------------------------
    Use the logic, based on the BSB files and data from the website
    This is where all the magic happens.

    We want to :
    - Call some file extension for the file we browsed to
        -TODO:  Get a huge list
            - Extension
            - User file, windows, linux, osx
    - Call some path when browsing a new path (even when it is a file)
        - default path list
    - Call some files when browsing a new path
        - user files windows, osx, linux
        - backup list
        - autosave list
        - svn, git list
        - CMS
        - Web server, etc.
    - Get robots.txt and sitemap data
    - Brute force up to 2 or 3 letters of files names and path on all found path which is not cms/git/etc.

    - Future version: Parse HTML comments for path


    - If they exist, we add them to XXX?
    - If new path exists, let's go recursive (new class?)
    - If file exists: add to sitemap + verbose + log

    @param data: UrlData object containing all information about the URL
    ----------------------------------------------------------------------------------------------------------'''
    def smartRequest(self,data):

        #Current request variables
        domain = data.getDomain()
        url = data.getUrl()

        ##################### FETCH DATA ###############################
        # Gather smart data once before sending requests
        ################################################################
        self._logger.debug("Has the Data been gathered for? : "+ str(url))
        if domain not in self._smartRequestData:
            try:
                self._smartRequestData[domain] = True
                self._logger.debug("no")
                self._logger.info("Fetching data for: "+ domain)

                print "getting data for:" + str(url)
                self.getSmartData(data)

            except Exception as e:
                print "exception:"+ e
                self._smartRequestData[domain] = False
                return False
        else:
            self._logger.debug("yes")

        # Execution of request with the received data:
        # - spider
        # - sitemap
        # - robots
        # - current directories
        # - commentsInUrl
        # json data:
        # - extension files
        # - common basic cms files
        # - common server files
        # - common user files
        # - common test files
        # - common repositories files
        # -
        # -
        '''
        For the current directories (path)
        - Test a path/file for a category of path/files
            - If a tested path/files exist (200/401/403/500) scan other files + - add to sitemap and LOG + add issues?
            - If not skip it
            - go 3 deep max and retest all

        TODO future version:
        Pseudo algo:
        Si le present url est un fichier:
        - Si c'Est un fichier php... tester phps extension.
        - si c'Est un fichier asmx, tester les wsdl

        Si c'Est un path:
        - si ca inclus un path dans sharepoint, tester les sharepoints
        - si ca inclus un fichier de wordpress ou drupal, tester quelques fichiers cms
            - Si on trouve un répertoire de type X, effectuer une recherche sur les fichiers de type X dans le repertoire trouvé
        '''





        #Current request data
        baseUrl = data.getBaseUrl()
        path = data.getPath()
        filename = data.getFilename()
        extension = data.getFileExt()
        print "CURRENT FILE: " + baseUrl + "," + filename + "," + extension
        #data.json sections: extensions, fileprefix, filesuffix, files, directories

        #test local file
        #if current url is a file: test extentions + intelligent details
        #AND we test current file with prefix and suffix

        #testing directories
        #if current URL have some directories test them out
        #Test them with FILES and DIRECTORIES. Including the current directory (last in path)

        #with the smart data test robots path and files
        #test N url from sitemap
        #in current paths test files and path using domainname and domain without the tld
        #with filename generated + extensions and path/filenamegenerated
        '''
        print "EXTENSIONS"
        for extension in self._parsed_json["extensions"]:
            print extension["name"]
        print "SUFFIX PREFIX"
        for prefix in self._parsed_json["fileprefix"]:
            print prefix["name"]
        for suffix in self._parsed_json["filesuffix"]:
            print suffix["name"]

        print "FILES"
        for files in self._parsed_json["files"]:
            print files["name"]
        '''

        print "DIRECTORIES"

        #Directories data information
        directories = data.getDirectories()
        directory = "/"
        slash = "" #force slash or not var

        #get options foir trailing slash. By default it's ON
        if self._trailingSlash:
            slash = "/"

        ##################### EXECUTE DATA.json REQUESTS ###################
        # Build Request to be execute based on our data.json
        # and getSmartData results
        ################################################################

        #TODO: important put tested directories and files in a dictionnary or array
        #TODO: important put tested directories and files in a dictionnary or array
        #TODO: important put tested directories and files in a dictionnary or array
        #TODO: important put tested directories and files in a dictionnary or arrayà


        ########################
        # Technology scanner
        ########################
        '''
        - do a request to root dir
        - get response (check for redirect)
        - check headers
        - check file extensions
        - depending on results scan X files.
          - Set current domain technologyVar to X
        '''

        ################
        #Scan the root directory!
        ################
        print "DIR: "+str(directories)

        if not directories:
            directories = ["/"]

        # response will be dealed in requestor
        for dir in directories:
            print "TESTING: " + dir
            if dir == "/":
                directory = "/"
            else:
                directory = directory+dir+"/" #test all directories: / /a/ /a/b/ /a/b/c/ ...

            #call our directories inside all request directires
            for dir2 in self._parsed_json["directories"]:
                self.verbose("RequestDir for: "+baseUrl+directory+dir2["name"]+slash)
                self._requestor.addRequest(baseUrl+directory+dir2["name"]+slash,data)

            # call directories based on domain information: url/a/b/c/smartDomain , url/a/b/smartDomain/, etc.
            #print "SMARTDOMAIN"+self._smartDomain
            for dir2 in self._smartDomain[domain]:
                self.verbose("RequestSmartDomain for: " + baseUrl + directory + dir2)
                self._requestor.addRequest(baseUrl + directory + dir2,data)

                #in each directory call smartDomain.extensions
                for ext in self._parsed_json["extensions"]:
                    self.verbose("RequestSmartDomain.ext for: " + baseUrl + directory + dir2 + ext["name"])
                    self._requestor.addRequest(baseUrl + directory + dir2 + ext["name"],data)

            #call our files in all directories
            #print "parsed json"+self._parsed_json["files"]
            for files in self._parsed_json["files"]:
                self.verbose("RequestFile for: "+baseUrl+directory+files["name"])
                self._requestor.addRequest(baseUrl+directory+files["name"],data)


        ################
        #If URL is a file, let's try to add some extension to the file
        ################
        if extension:

            #replace current file extension for our extension
            tempFilenameUrl = baseUrl+directory+filename
            tempFilenameUrl1 = baseUrl+directory+filename+"."+extension
            for ext in self._parsed_json["extensions"]:
                self.verbose("RequestExt for: "+ tempFilenameUrl+ext["name"])
                self.verbose("RequestFileExt for: "+ tempFilenameUrl1+ext["name"])
                self._requestor.addRequest(tempFilenameUrl+ext["name"],data)
                self._requestor.addRequest(tempFilenameUrl1+ext["name"],data)

            #add a prefix to current file
            tempFilenameUrl = baseUrl+directory
            for prefix in self._parsed_json["fileprefix"]:
                tempFilenameUrl1 = tempFilenameUrl+prefix["name"]+filename+"."+extension
                self.verbose("RequestPrefix for: "+tempFilenameUrl1)
                self._requestor.addRequest(tempFilenameUrl1,data)

            #add suffix to current file
            tempFilenameUrl = baseUrl+directory
            for suffix in self._parsed_json["filesuffix"]:
                tempFilenameUrl1 = tempFilenameUrl+filename+suffix["name"]+"."+extension
                self.verbose("RequestSuffix for: "+tempFilenameUrl1)
                self._requestor.addRequest(tempFilenameUrl1,data)



        #make sure we have some data
        #print "DATA RECEIVED"
        #print self._words[domain]
        #print self._mergedWords ##need to call the emrge function if needed
        #print self._robots[domain]
        #print str(len(self._sitemap[domain]))
        #print str(self._urlsInComment[domain])



        ##################### EXECUTE SMART REQUESTS ###################
        # Build Request to be execute based on our data.json
        # and getSmartData results
        ################################################################

        #list of smart directories
        smartDirectories = {}

        #list of smart files (add our extension to it)
        smartfiles = {}

        ################
        #Request N pages from sitemap
        ################
        if domain not in self._siteRobotScanned: #Do it once
            self._siteRobotScanned[domain] = True #done for this domain

            tmpSiteMap = []
            for i in range(0,self._configSpider_NumberOfPages): #get N number of pages from ini config
                tmpSiteMap.append(self._sitemap[domain][i])

            #Requests files and directories from robots.txt
            tmpRobots = []
            for line in self._robots[domain]:

                #in case robots.txt use ending wildcard we remove it
                if line.endswith("*"):
                    line = line[:-1]
                #TODO: Test if directory or file is not 404 ??
                tmpRobots.append(baseUrl+line)

            ################
            # requests all value for N sitemap url
            ################
            for link in tmpSiteMap:

                if link.endswith("/"): #scan directories and files

                    for dir2 in self._parsed_json["directories"]:
                        self.verbose("RequestSiteMap dir/file for: " + link + dir2["name"] + slash)
                        self._requestor.addRequest(link + dir2["name"] + slash,data)

                    for files in self._parsed_json["files"]:
                        self.verbose("RequestSiteMap dir/file for: " + link + files["name"])
                        self._requestor.addRequest(link + files["name"],data)

                else:  #scan extensions and suffix/prefix
                    # call our files in all directories
                    for ext in self._parsed_json["extensions"]:
                        self.verbose("RequestSitemap file/ext/ext for: " + link + ext["name"])
                        self._requestor.addRequest(link + ext["name"],data)

                        #Get the file extension of the current sitemap url to replace the extension
                        tmpUrl = urlparse(link)
                        if len(tmpUrl.path.split(".")[-1:]) > 1:
                            newUrl = ".".join(tmpUrl.path.split(".")[:-1])+ext["name"]
                            self.verbose("RequestSiteMap file/ext for: " + newUrl)
                            self._requestor.addRequest(newUrl,data)

            ################
            #requests all values for robots path
            ################
            for link in tmpRobots:
                tmpUrl = baseUrl + link
                if link.endswith("/"):  # scan directories and files
                    for dir2 in self._parsed_json["directories"]:
                        self.verbose("RequestRobots dir/file for: " + tmpUrl + dir2["name"] + slash)
                        self._requestor.addRequest(tmpUrl + dir2["name"] + slash,data)

                    for files in self._parsed_json["files"]:
                        self.verbose("RequestRobots dir/file for: " + tmpUrl + files["name"])
                        self._requestor.addRequest(tmpUrl + files["name"],data)
                else:
                    for ext in self._parsed_json["extensions"]:
                        self.verbose("RequestRobots file/ext/ext for: " + tmpUrl + ext["name"])
                        self._requestor.addRequest(tmpUrl + ext["name"],data)

                        #Get the file extension of the current sitemap url to replace the extension
                        tmpUrl1 = urlparse(link)
                        if len(tmpUrl1.path.split(".")[-1:]) > 1:
                            newUrl = ".".join(tmpUrl1.path.split(".")[:-1])+ext["name"]
                            self.verbose("RequestRobots file/ext for: " + newUrl)
                            self._requestor.addRequest(newUrl,data)


        #TODO :  path and words/merge words

        ################
        #Request from words
        ################
        #print self._words


        #TODO: loop over: sitemap (done), robots (done), words/mergedwords(fixed for textblob required), bruteforce(later)  Maybe comments data?
        # - add the data to our stack to request and parse by the Requestor object
        # - Get current query path and files & Filter out static object from the request (images,etc.)
        #filter out: gif,jpg,png,css,ico


        print "Done. Waiting for more URL...!"

    '''----------------------------------------------------------------------------------------------------------
    Get the data for smartRequest(), it will fills our list of words which will be our smart logic data to create
    multiple new HTTP requests. This data should be gather once.
    ----------------------------------------------------------------------------------------------------------'''
    #TODO: split some of this works in different functions
    def getSmartData(self, data):

        ################################################################
        # Get the url and its data to create the new smart requests
        ################################################################
        urlString = str(data.getUrl()) #cast to cast to stop the TYPEerror on URL()
        domain = data.getDomain()
        netloc = data.getNetloc()
        directories = data.getDirectories()
        lastDirectory = data.getLastDirectory()
        params = data.getParams()
        fileExt = data.getFileExt()
        completeUrl = data.getCompleteURL()
        baseUrl = data.getBaseUrl()

        #Java URL to be used with Burp API
        url = URL(urlString)
        self._logger.debug("Current URLString: "+urlString)
        ######################### SPIDER EXECUTION #####################
        # Get some words from the web page: do it once!
        # Note: This step could be threaded using Queue.Queue but there is
        # little advantage as we need to wait to get all the value anyway
        ################################################################

        self._logger.debug("Has the Spider ran for? : "+ domain)
        if domain not in self._spiderRan: #doing it once
            self._spiderRan[domain] = True
            self._logger.debug("No")

            #self._mergedWords[domain] = {}
            #self._words[domain] = {}

            #Start URL, number of page to spider through, request class object to use

            spider = Spider(data, self._configSpider_NumberOfPages, self._requestor,self._logger)
            spider.runSpidering()

            #Get words from the spidering
            self._words[domain] = spider.getWords()
            #Get merged words
            #spider.mergeWords()
            #self._mergedWords[domain] = spider.getMergedWords()

            self._logger.debug("Length of Words: "+ str(len(self._words[domain])))
            #self._logger.debug("Length of MergedWords: "+ str(len(self._mergedWords[domain])))
            self._logger.info("SPIDER DONE")
        else:
            self._logger.debug("Yes")

        ################################################################
        # Get robots.txt (once)
        # Retrieve unique path and files from the robots.txt
        ################################################################
        if domain not in self._robots: #do it once
            print " robot "

            #get the file
            queueRobot = Queue.Queue(1)
            self._logger.info("robot")
            thread = threading.Thread(
                target=self._requestor.runRequest,
                name="Thread-Robots",
                args=[baseUrl+"/robots.txt", queueRobot],
            )
            thread.start()
            thread.join()
            response = queueRobot.get()

            #Parse the file for disallow lines
            robotList = []
            for item in response.content.split('\n'):
                if item:
                    i = item.split(':')
                    if i[0].lower() == "disallow" and i[1] not in robotList:
                        robotList.append(i[1])

            #add to domain list
            self._robots[domain] = robotList

            self._logger.debug("ROBOT LIST for : " + domain + ":")
            for item in self._robots[domain]:
                self._logger.debug(item)

            self._logger.info("ROBOTS DONE")

        else:
            print "no robot"
            self._logger.debug("Robots.txt already checked for: " + baseUrl)

        ################################################################
        # Get sitemap.xml (once)
        # test those url for all files/extensions if not in local deque yet
        ################################################################
        if domain not in self._sitemap:
            print " sitemap "
            queueSitemap = Queue.Queue(1)
            thread = threading.Thread(
                target=self._requestor.runRequest,
                name="Thread-Sitemap",
                args=[baseUrl+"/sitemap.xml", queueSitemap],
            )
            thread.start()
            thread.join()

            response = queueSitemap.get()
            soup = BeautifulSoup(response.content, "html.parser")

            #Parse the XML TODO: for N instance related to .ini config
            sitemapList = []
            for url in soup.findAll("loc"):
                sitemapList.append(url.text)

            self._sitemap[domain] = sitemapList

            self._logger.debug("Sitemap.xml nbr of items: "+str(len(self._sitemap[domain])))

            self._logger.info("SITEMAP DONE")
        else:
            print "no sitemap"

        ################################################################
        # Get domain name relative values
        # test those names for directory, files with extension
        ################################################################
        print "smartDomain"
        tmpDomValue = []

        if domain == "localhost":
            tmpDomValue.append(domain)
        else:
            tld = get_tld(urlString, as_object=True)
            tmpDomValue.append(tld.domain)
            tmpDomValue.append(tld.tld)

            if tld.subdomain:
                tmpDomValue.append("".join(tld.subdomain+"." + tld.tld))


        self._smartDomain[domain] = tmpDomValue

        ######################## BRUTE FORCE DATA ######################
        # 1, 2 or 3 letters brute force of current directory
        # Has the current directory been test already? No: do it
        #brute force function or object?
        ################################################################
        #TODO: Later version
        #charset = "abcdefghijklmnopqrstuvwxyz0123456789_-"
        #for a in itertools.product(charset,repeat=2):
        #    sub="".join(a)


        return True


    '''----------------------------------------------------------------------------------------------------------
    Get the information inside response for smartRequest()
    It will look for URL and email domain inside HTML comments

    @todo: Optimize the IFs in the comment for loop!
    ----------------------------------------------------------------------------------------------------------'''
    def getUrlInComments(self,data):

        ################### CURRENT DIRECTORIES/FILES ##################
        # Get current directory(ies)
        # validate if tested already
        # If not deal with: test directories and files at currentPath
        # New class object?
        ################################################################
        responseData = data.getResponseData()

        #TODO: Parse HTML files for comments for Path and file

        #if you have a response
        if responseData:
            soup = BeautifulSoup(responseData, "html.parser")
            comments=soup.find_all(string=lambda text:isinstance(text,Comment))
            regUrl = r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»“”‘’]))"
            regEmail = r"[\w\.-]+@[\w\.-]+"
            urlsInComments = []
            emailsInComments= []
            urlsInComment = []
            emailsInComment = []

            for comment in comments:
                #get urls
                urlsComments = re.findall(regUrl,comment)
                #parse url, does the domain the same as our current domain?
                if urlsComments:
                    for url in urlsComments[0]:
                        if url:
                            #Get URLs
                            tempData = urlparse(url)
                            domainInUrlTemp = '{uri.netloc}'.format(uri=tempData).split('.')
                            domainInUrl = ".".join(domainInUrlTemp)

                            #TODO: url will need to be verify if in scope when we call it : keep the URL path/file for scan
                            urlsInComment = re.findall(regUrl,comment)
                            urlsInComments.append(urlsInComment)

                    #get emails
                    emailsInComment = re.findall(regEmail, comment)
                    emailsInComments.append(emailsInComment)
            self._logger.debug("url in comments and email in comments:")

            #get list only
            if urlsInComments and urlsInComments[0]:
                if type(urlsInComment[0]) is tuple:
                    self._urlsInComment[data.getDomain] = urlsInComment[0]
            #TODO: use email in another version?
            if emailsInComments and emailsInComments[0]:
                if type(emailsInComments[0]) is tuple:
                    emailsInComments = emailsInComments[0]

            self._logger.debug(urlsInComments)
            self._logger.debug(emailsInComments)

            self._logger.info("COMMENTS DONE")


            #TODO: finish these function to gather the information from the data.json
        '''
    Function which is accessing smart list of Path to look into by the smart request function
    '''
    def getSmartListPath(self):
        return

    '''
    Function which is accessing smart list of file extension to look into by the smart request function
    '''
    def getSmartListExt(self):
        return

    '''
    Function which is accessing smart list of directories to look into by the smart request function
    '''
    def getSmartDirectories(self):
        return

    '''
    Function which is accessing smart list of files to look into by the smart request function
    '''
    def getSmartFiles(self):
        return

    '''
    This functions split all informations of the URL for further use in the smartRequest function
    @param messageInfo: last request executed with all its information
    '''
    def getURLdata(self,messageInfo,messageIsRequest):

        analyzedRequest = self._helpers.analyzeRequest(messageInfo)
        url = analyzedRequest.getUrl()
        self._logger.debug(url)

        parsed = urlparse(url.toString())

        '''debug info
        print 'scheme  :', parsed.scheme
        print 'netloc  :', parsed.netloc
        print 'path    :', parsed.path
        print 'params  :', parsed.params
        print 'query   :', parsed.query
        print 'fragment:', parsed.fragment
        print 'username:', parsed.username
        print 'password:', parsed.password
        print 'hostname:', parsed.hostname, '(netloc in lower case)'
        print 'port    :', parsed.port
        '''

        #Is there any parameters?
        params = analyzedRequest.getParameters()

        for p in params:
            self._logger.debug("Query var: "+p.getName())
            self._logger.debug("Query value: "+p.getValue())

        #getURL, needs to be a string before parsing it with urlparse
        completeURL = url.toString()
        self._logger.debug("Complete URL: "+completeURL)

        #URL sans port/dir/params
        baseURL = messageInfo.getHttpService().toString()
        self._logger.debug("Base URL: "+baseURL)


        #Get path including directories and file extension
        path = urlparse(completeURL).path.encode("utf-8")
        filename = path.split('/')[-1:].pop().split('.')[:1].pop()
        fileExt = path.split('.')[1:]
        fileExt = "".join(fileExt)
        directories = path.split('/')[1:-1]
        directory = "/".join(directories)
        if len(fileExt) > 0:
            self._logger.debug("Directories: "+str(directories)[1:-1])
            self._logger.debug("Directory: "+directory)
            self._logger.debug("File Extension: "+fileExt)
            self._logger.debug("URL Path: "+path)
            self._logger.debug("Filename: "+filename)
        else:
            self._logger.debug("No file Extension, directory is: "+path)

        #Get domain and netloc
        netloc = parsed.netloc.encode("utf-8")
        domain = netloc.split(':')[0]

        self._logger.debug("Domain/: "+domain)

        '''
        print "Complete URL: "+completeURL
        print "Domain: "+domain
        print "Netloc: "+ netloc
        print "Query value: "+p.getValue()
        print "Query var: "+p.getName()
        print "Directories: "+str(directories)[1:-1]
        print "Directories2: "+str(directories)
        print "Directory: "+directory
        print "File Extension: "+fileExt
        print "URL Path: "+path
        print "Filename: "+filename
        print "Base URL: "+baseURL
        '''

        responseData = ""
        if not messageIsRequest: #when it's a response, get the response data
            content = messageInfo.getResponse()
            response = self._helpers.analyzeResponse(content)
            responseData = self._helpers.bytesToString(content[response.getBodyOffset():])

            #data = UrlData("",headers,"","","","","","",responseData,self._logger)

        data = UrlData(url,domain,netloc,directories,params,filename,fileExt,baseURL,completeURL,path,responseData,self._logger)
        return data

    # This method is called when multiple issues are reported for the same URL
    # In this case we are checking if the issue detail is different, as the
    # issues from our scans include affected parameters/values in the detail,
    # which we will want to report as unique issue instances
    def consolidateDuplicateIssues(self, existingIssue, newIssue):
        if (existingIssue.getIssueDetail() == newIssue.getIssueDetail()):
            return -1
        else:
            return 0

    #Have to be implemented
    def doPassiveScan(self, baseRequestResponse):
        pass

    #Have to be implemented
    def doActiveScan(self, baseRequestResponse, insertionPoint):
        pass
'''
Multithreaded class to execute queries out of the Queue.Queue

Also get the response and validate the 404 type
'''
class RequestorWorker(threading.Thread):

    def __init__(self, threadID, name, queue, error404, logger, requestor, UI, recursiveURLs):

        #Sahred Queue between Thread Workers
        self._id = threadID
        self._name = name
        self._queue = queue #request queue received from the Requestor
        self._threadLock = threading.Lock()
        self._alive = True
        threading.Thread.__init__(self)
        self.daemon = True
        #self._responseQueue = responseQueue
        self._error404 = error404
        self._logger = logger
        self._requestor = requestor
        self._ui = UI
        self._recursiveURLs = recursiveURLs

        self._acceptedCode = (200,400,401,403,500)

        #TODO: Set a randomizer of user-agent and add the option in .ini file
        self._headers = {
            'User-Agent': 'Mozilla/5.0'
        }

        return

    '''
    Return type of 404 for requested domain

    @param domain: domain to fetch error 404 type
    '''
    def _getError404(self,url):
        #Get domain and netloc
        parsed = urlparse(url)
        netloc = parsed.netloc.encode("utf-8")
        domain = netloc.split(':')[0]
        return self._error404[domain]

    def run(self):
        while(self._alive):
            #waiting for queue
            #print "Waiting for queue: "+self._name
            url = self._queue.get()

            #print "TASK RECEIVED: " + url + " From: " + self._name

            self._logger.debug(self._name+" requesting(URL): " + url)
            self._logger.info(self._name+" requesting(URL): " + url)
            #print "[Requesting] " + url

            #TODO: randomizedUserAgent
            #TODO: - 302 (redirect) --> parse the redirect URL (in scope ok, in sitemap stop, not in site map  add to queue : 200+window.location or JS isn't catch yet

            response = requests.get(url, headers=self._headers, allow_redirects=False)

            if response.status_code in self._acceptedCode:
                #add no false positive to site map
                code = self._getError404(url)
                print "[URL EXISTS](Response: " +str(response.status_code)+ ") | 404 type:" + str(code) +" | FOR URL: "+ str(url)

                #False positive logic.
                #TODO: can be update or upgraded for sure! :)
                fp = ""


                '''
                si 404
                    si response 200 ok
                    si response 401
                    si response 403
                    si response 300
                    si response 500
                si 403
                    si response 200
                    si response 401 fp
                    si response 403 fp
                    si response 300
                    si response 500 fp
                si 500
                    si response 200
                    si response 401
                    si response 403
                    si response 300
                    si response 500 fp
                si intext
                    si response 200 need reverification fp
                    si response 401
                    si response 403
                    si response 300
                    si response 500
                si 300
                    si response 200
                    si response 401
                    si response 403
                    si response 300 fp
                    si response 500
                '''

                #if the current request is a 403 and the 404 page isn't a 403 page, should be false positive
                if response.status_code == 403 and code != 403:
                    fp = " ,False Positive"
                #if current response is a 200 and the 404 page was inside a 200 code page, it can be a false positive
                elif response.status_code == 200 and code == "404 in page":
                    fp = " ,False Positive"
                #if 404 page is inside a 200 response code, a 300 redirect page or a 403, many possible false positive
                elif code == "404 in page" or code == 300 or code == 403:
                    fp = " ,Possible False Positive"
                #code is 200 or whatnot
                else: #TODO: define all directory in a list and add to the recursive list+validate latest directory of current url to see if it is in list, if not add it
                    print 200
                    #if it's a direct directory, let's recurse... if not recurse too much already!
                    #if urlparse(url).path[-1] == '/' and self._recursiveURLs.get(str(url), 0) <= self._ui.getRecursiveConfig():
                    #    self._recursiveURLs[str(url)] = self._recursiveURLs.get(str(url), 0) + 1 #adjust the recursed level for that directory
                    #    self._requestor.runRequest(url,Queue.Queue(1))

                #add code to the Jlist here
                print url
                self._ui.addURL(url + " , ("+str(response.status_code)+")" + fp)



                #TODO: add page to SiteMap if not there already?

                #TODO: issue = SmartBusterIssue()
                #might need to parse the url into data for the issue?
                #issue=ScanIssue(baseRequestResponse.getHttpService(), self._helpers.analyzeRequest(baseRequestResponse).getUrl(), httpmsgs, ISSUE_NAME, ISSUE_DETAIL, SEVERITY, CONFIDENCE, REMEDIATION_DETAIL, ISSUE_BACKGROUND, REMEDIATION_BACKGROUND)
                #self._callbacks.addScanIssue(issue)

'''----------------------------------------------------------------------------------------------------------------------------------------
Class to hold the Request data

- Using Requests API we use a Queue to append HTTP requests to be executed.
- If the requests return a 200/401/403/500 we add them to the sitemap and add them to our list of URL/Dir/file found
- Can save data found to csv
----------------------------------------------------------------------------------------------------------------------------------------'''
class Requestor():
    '''
    Initialize

    '''
    def __init__(self,logger,UI):

        #Queue to hold URL to request
        #Each item will be a URL string str(URL)
        self._requestQueue = Queue.Queue(0)
        self._logger = logger

        #hold type of 404 error by domain
        self._error404 = {}

        #hold url that are being recursive
        self._recursiveURLs = []

        #Queue to hold URL and their response code
        #Each item will be a list (url,code)
        #self._responseQueue = deque()

         #TODO: Set a randomizer of user-agent and add the option in .ini file
        self._headers = {
            'User-Agent': 'Mozilla/5.0'
        }

        self._logger.debug("Requestor object created")

        threads = [] #list containing threads

        #1 thread needed for infofestival. Don't know how to split the pages between workers
        for i in range(0,40):#TODO: Set a number of thread in UI
            t = RequestorWorker(i,"RequestorWorker-"+str(i),self._requestQueue,self._error404, logger, self, UI, self._recursiveURLs)
            threads.append(t)
            t.start()

        return


    '''
    Add a request to the queue to be execute by a thread worker (RequestorWorker)

    @param url: the URL to get a response from
    '''
    def addRequest(self,url,data):


        #print "ADDING: "+ url

        #get the 404 details for the current domain
        self._define404(data)
        self._requestQueue.put(url) ##see if we can put the type404 inside the queue along with the url
        return

    '''
    Define 404 type of the current domain
    '''
    def _define404(self,data):

        domain = data.getDomain()
        #only do once per domain
        if domain not in self._error404:

            code = 404
            errorQueue = Queue.Queue(0)

            #get a 404 page
            m = hashlib.md5()
            m.update(str(random.random()))

            url = data.getBaseUrl()+"/"+m.hexdigest()
            print url
            self.runRequest(url,errorQueue)
            response = errorQueue.get()

            #if website use standard 404 error, everything is good
            if response.status_code == 404:
                code = 404

            #if website used a 3xx code
            if 310 - response.status_code < 11 and 310 - response.status_code > 0:
                code = 300

            if response.status_code == 403:
                code = 403

            #if website use a 5xx code
            if 510 - response.status_code < 11 and 510 - response.status_code > 0:
                code = 500

            #if website use a 200
            if response.status_code == 200:

                soup = BeautifulSoup(response.content, "html.parser")

                ################################
                #TODO: more use case to add
                ################################
                if soup.findAll(text=re.compile("page not found")):
                    code = "404 in page"
                elif soup.findAll(text=re.compile("404")):
                    code = "404 in page"
                elif soup.findAll(text=re.compile("page does not exist")):
                    code = "404 in page"
                elif soup.findAll(text=re.compile("error 404")):
                    code = "404 in page"

            #define which code is refer to a 404
            self._error404[domain] = code

        return

    '''
    Run a NON DELAYED (no thread workers) request and save the url:response code to the response deque class variable

    @param url: the URL to request and get a response
    @param responseQueue: thread safe queue to send the response back to the spider or other objects
    '''
    def runRequest(self,url,responseQueue):

        #TODO: After thread is done, in thread read the _requestQeue object

        self._logger.debug("runRequest(URL): "+url)
        self._logger.info("EXECUTING REQUEST FOR: "+url)
        response = requests.get(url,  headers=self._headers, allow_redirects=False)
        responseQueue.put(response)

        #TODO: Get code
        #TODO: add page to SiteMap if not there already?


        self._logger.debug("runRequest done  for: "+url)

        return

    #TODO randomizedUserAgent
    def randomizedUserAgent(self):
        return




'''----------------------------------------------------------------------------------------------------------------------------------------
Class to hold the Spidering data

- Based on: http://www.netinstructions.com/how-to-make-a-web-crawler-in-under-50-lines-of-python-code/
  Uses BeautifulSoup, require to download/install it.
----------------------------------------------------------------------------------------------------------------------------------------'''
class Spider():

    '''
    Initialize

    @param startUrl: the URL to start the spidering

    '''
    def __init__(self, data, maxPages, requestObj, logger):
        self._data = data
        self._words = []
        self._mergedWords = []
        self._maxPages = int(maxPages)
        self._requestor = requestObj
        self._queue = Queue.Queue(self._maxPages)
        self._domain = data.getDomain()
        self._logger = logger
        self._logger.debug("Spider object created")

    '''
    Run the spidering

    @return: list of all words found
    @todo: use TextBlob for other language, right now mostly only english based words will be categorized correctly.
    '''
    def runSpidering(self):

        urlString = str(self._data.getUrl())
        url = URL(urlString)

        print "Spider, URL: " + urlString
        #Get the words from the URL, starting with the startUrl
        link_list = [urlString]

        #Counter
        pagesVisited = 0

        self._logger.debug("Max pages to visit: " + str(self._maxPages))

        while int(pagesVisited) < int(self._maxPages):
            self._logger.debug("Nbr Page Visited: " + str(pagesVisited) + " / " + str(self._maxPages))
            self._logger.debug("Visiting: " + link_list[pagesVisited])
            visitingUrl = link_list[pagesVisited]
            pagesVisited = pagesVisited+1
            print "Visiting URL: "+visitingUrl
            try:
                #??? Fix the url retrieve.
                #If it starts with / we add the domain to it
                if self._domain not in visitingUrl:
                    if visitingUrl.startswith("/"):
                        visitingUrl = visitingUrl[1:]
                        #TODO: startswith /#

                    link_list[pagesVisited] = self._data.getCompleteURL() + visitingUrl
                    visitingUrl = link_list[pagesVisited]

                #send an asynchronus HTTP request and wait for the response
                thread = threading.Thread(
                                target=self._requestor.runRequest,
                                name="Thread-Spider",
                                args=[visitingUrl, self._queue],
                                )
                thread.start()
                thread.join()
                response = self._queue.get()
                self._logger.debug("Response received from: "+visitingUrl)

                #Get the soup
                soup = BeautifulSoup(response.content, "html.parser")

                #Get the visible text
                [s.extract() for s in soup(['style', 'script', '[document]', 'head', 'title'])]
                visible_texts = soup.getText()#.encode('utf-8').strip()
                #Get the text blob
                blob = TextBlob(visible_texts)

                #Get the words : TODO: add the 1000 value in the bsb.ini?
                if len(blob.words) <= 1000: #merging 2 words and up to 1000  (cpu intensivity)
                    for words,tag in blob.tags:
                        #Get only noun and numbers
                        if tag.startswith("NN") or tag == "CD":
                            self._words.append(words)

                self._logger.debug("Size of WORDS: " + str(len(self._words)))

                #Get the links for next pages or stop
                aSoup = soup.findAll("a")
                if len(aSoup) > 0:
                    for i in aSoup:
                        #Do not use previous page, index or anchors
                        if not i['href'].startswith("#") and not i['href'] == "/" and not i['href'] in i and not i['href'].startswith("/#") and not i['href'].startswith("//"):
                            link_list.append(i['href'])
                else:
                    self._logger.debug("No words on: "+visitingUrl)
                    break

            except KeyError:
                self._logger.error("SpiderError: KeyError")
                pass
            except requests.exceptions.RequestException as e:
                self._logger.error("SpiderError: "+e.reason)
                pass

        return self._words

    '''
    Merge the obtained words from the spidering

    @return: List of all words mixed with each others
    '''
    def mergeWords(self):
        if len(self._words) > 1:

            #original list of words that we want to mix
            listOriginal = self._words

            #merging all words together
            for words in listOriginal:
                for wordsToMerge in listOriginal:
                    self._mergedWords.append(words+wordsToMerge)

            return True
        else:
            return False


    '''
    @return: List of all words mixed with each others
    Note: The return words needs to be convert to utf-8
    '''
    def getMergedWords(self):
         return self._mergedWords

    '''
    @return: List of all words
    Note: The return words needs to be convert to utf-8
    '''
    def getWords(self):
         return self._words


'''----------------------------------------------------------------------------------------------------------------------------------------
Class to share community data to annonimized server
----------------------------------------------------------------------------------------------------------------------------------------'''
class technologyScanner():

    def __init__(self, optIn, logger):
        self._optIn = optIn
        self._logger = logger

        self._logger.debug("CommunityData Object Created")

        return

'''----------------------------------------------------------------------------------------------------------------------------------------
Class to share community data to annonimized server
----------------------------------------------------------------------------------------------------------------------------------------'''
class communityData():

    def __init__(self, optIn, logger):
        self._optIn = optIn
        self._logger = logger

        self._logger.debug("CommunityData Object Created")

        return

    def submitData(self,fileName,isFile):
        if self._optIn:

            #prepare the request to submit to the server
            if isFile:
                print "Data is a file"
                #data to sent is a file
            else:
                print "data is a directory"
                #data to sent is a directory

            #contact the server
            print "contacting the server with data: " + fileName
        return

'''----------------------------------------------------------------------------------------------------------------------------------------
Class to hold the URL data in separated parts
----------------------------------------------------------------------------------------------------------------------------------------'''
class UrlData():

    def __init__(self,url,domain,netloc,directories,params,filename, fileExt,baseURL,completeURL,path,responseData,logger):
        self._url = url
        self._domain = domain
        self._netloc = netloc
        self._directories = directories
        self._params = params
        self._fileExt = fileExt
        self._baseURL = baseURL
        self._completeURL = completeURL
        self._responseData = responseData
        self._logger = logger
        self._path = path
        self._filename = filename

        self._logger.debug("UrlData object created")
        return

    def getPath(self):
        return self._path

    def getFilename(self):
        return self._filename

    def getResponseHeaders(self):
        if not self._url:
            return self._domain

    def getResponseData(self):
        return self._responseData

    def getBaseUrl(self):
        return self._baseURL

    def getCompleteURL(self):
        return self._completeURL

    def getUrl(self):
        return self._url

    def getDomain(self):
        return self._domain

    def getNetloc(self):
        return self._netloc

    def getDirectories(self):
        return self._directories

    def getLastDirectory(self):
        if len(self._directories) > 0:
            return self._directories[len(self._directories)-1]
        else:
            return ""

    def getParams(self):
        return self._params

    def getFileExt(self):
        return self._fileExt
'--------------------------------------------------------------------'



'''--------------------------------------------------------------------
Class to hold the Issues found
@TODO: see for Sitemap instead of issue or WITh issues
--------------------------------------------------------------------'''
class SmartBusterIssue(IScanIssue):
  '''This is our custom IScanIssue class implementation.'''
  def __init__(self, httpService, url, httpMessages, issueName, issueDetail, severity, confidence, remediationDetail, issueBackground, remediationBackground):
      self._issueName = issueName
      self._httpService = httpService
      self._url = url
      self._httpMessages = httpMessages
      self._issueDetail = issueDetail
      self._severity = severity
      self._confidence = confidence
      self._remediationDetail = remediationDetail
      self._issueBackground = issueBackground
      self._remediationBackground = remediationBackground


  def getConfidence(self):
      return self._confidence

  def getHttpMessages(self):
      return self._httpMessages
      #return None

  def getHttpService(self):
      return self._httpService

  def getIssueBackground(self):
      return self._issueBackground

  def getIssueDetail(self):
      return self._issueDetail

  def getIssueName(self):
      return self._issueName

  def getIssueType(self):
      return 0

  def getRemediationBackground(self):
      return self._remediationBackground

  def getRemediationDetail(self):
      return self._remediationDetail

  def getSeverity(self):
      return self._severity

  def getUrl(self):
      return self._url

  def getHost(self):
      return 'localhost'

  def getPort(self):
      return int(80)
