DSACancellationChecker/DSACheckerClasses.py at master · bendholmes/DSACancellationChecker · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
import urllib, urllib2, cookielib
from bs4 import BeautifulSoup
from datetime import datetime
import re

class Page:
	fields = {}
	url = None
	connection = None
	html = None # BeautifulSoup object
	cookieJar = None
	opener = None
	response = None

	def __init__(self, url, cj):
		self.url = url
		self.cookieJar = cj

	def connect(self):
		print "---> Connecting to %s" % (self.url,)

		self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookieJar))
		self.opener.addheaders.append(('User-agent', 'Mozilla/4.0'))

		if self.fields:
			data = urllib.urlencode(self.fields)
			self.response = self.opener.open(self.url, data)
			print "-----> Sending data:"
			for c in self.fields.keys():
				print "-------> %s = %s" % (c, self.fields[c][:20])
		else:
			self.response = self.opener.open(self.url)

		self.html = BeautifulSoup(self.response.read())

		# save the pages for diagnostic info
		# save = open(re.sub(r'\W+', '', self.html.title.string) + '.html', 'w')
		# save.write(str(self.html))
		# save.close()