import os

# This python code iterates over the .ttl files generated by the crawler to generate a seeds.txt file which contains a line-separated list of URLs to visit. 
# All the URLs in the seeds.txt file will be added to the seeds of the crawl if this file is found in the root folder where the crawler is run.

out = open('seeds.txt','w')

def parse_file(file):
    print file
    f = open(file,'r') 
    for line in f:
        if 'oa:hasTarget' in line:
            after = line.split('oa:hasTarget',1)[1]
            url = after.split()[0]
            url = url.replace('<','')
            url = url.replace('>','')
            if not 'index.php' in url:
              out.write(url+"\n") 
    f.close()


rootdir = os.getcwd()
for subdir, dirs, files in os.walk(rootdir):
    for file in files:
        #print os.path.join(subdir, file)
        filepath = subdir + os.sep + file

        if filepath.endswith(".ttl"):
            parse_file(filepath)
			
out.close()
