#! /usr/bin/python

# Script to recursively find places on the web that accept comments.
# Written by Peter Lambert (c) 2020, all rights reserved.

import time, urllib
from os import path

# Wether to print as it goes
debug = True

# Number of addresses to check per site visited
max_per_site = 100

# Files for storing data. These can be changed.
# Put starting websites in the churn file.
# Good results will be placed in the targets file.
churn = 'churn.txt'
targets = 'targets.txt'
pulp = 'pulp.txt'
log = 'log.txt'


def append_data(logfile, line):
	''' Appends a line to a file. '''
	
	with open(logfile, 'a+') as f:
		f.write(line + '\n')
		f.close()

def load_data(logfile):
	''' Loads data from a file as an array of the lines in the file. '''
	
	with open(logfile, 'r') as f:
		my_array = f.read().split('\n')
		f.close()
		
		return my_array

def find_links(s):
	''' Returns an array of links found in a string.'''
	res = []
	
	# Split on the start of a link
	for t in s.split('<a href="')[1:]:
		# Cut off at the end of the link
		res.append(t.split('"')[0])
		
	return res

# Some variables for us to work with
num_checked = 0
num_found = 0
sites = {}

for needed_file in (churn, pulp, log, targets):
	if not path.exists(needed_file):
		f = open(needed_file, 'a+')
		f.close()

# Load the list of previous checks
checked = load_data(pulp) + load_data(targets)

# Take input of starting websites from file churn
to_check = load_data(churn)

for source in to_check:
	if debug:
		print 'checking {0}'.format(source)
	
	# Grab Page
	try:
		page = urllib.urlopen(source).read()
		num_checked += 1
		checked.append(source)
		to_check.remove(source)

		# Grab outbound URL's
		# Check if there is a comment box

		# how to tell if there is a comment box? On Wordpress it is:
		# <textarea name="comment"
		
		# If you find a different comment box type, add it as an "or" statement here.

		if '<textarea name="comment"' in page:
			num_found += 1
			
			# Append found results to list of URLs with boxes
			append_data(targets, source)
			
		else:
			append_data(pulp, source)
		
		# Add any new links into the list to check
		# - Maybe this should be in the if block above and only look for links on pages that take comments?
		for link in find_links(page):
			# Make sure it is a link and not just an anchor or mailto
			if link.startswith('http://'):
				# Find the parent site from the link 
				site = link.split('http://')[1].split('/')[0]
				if not sites.has_key(site):
					sites[site] = 0
				
				if link not in checked \
						and link not in to_check \
						and sites[site] < max_per_site:
					to_check.append(link)
					sites[site] += 1
					# Are we supposed to put these in the churn file? if so, uncomment next line
					#append_data(churn, link)

	except:
		print "Could not load {0}".format(source)

# Report number of URL's found and checked
append_data(log, '{0} Found {1} good out of {2} sites checked.'.format(time.ctime(), 
		num_found, num_checked))