f-8classUrlManager(object):def__init__(self):self.new_urls=set()self.old_urls=set()#添加URLdefadd_new_url(self,url):ifurlisNone:returnifurlnotinself.new_urlsandurlnotinself.old_urls:self.new_urls.add(url)defadd_new_urls(self,urls):ifurlsisNoneorlen(urls)==0:returnforurlinurls:self.add_new_url(url)defhas_new_url(self):returnlen(self.new_urls)!=0defget_new_url(self):new_url=self.new_urls.pop()self.old_urls.add(new_url)returnnew_urlhtml_downloader.py#coding:utf-8importurllib2classHtmlDownloader(object):defdownload(self,url):ifurlisNone:returnNoneresponse=urllib2.urlopen(url)#如果请求码不是200,则表示请求不成功。#典型的错误包括404(页面无法找到),403(请求禁止),401(待验证的请求)#5XX回应代码以“5”开头的状态码表示服务器端发现自己出现错误,不能继续执行请求ifresponse.getcode()!=200:print"downloadhtmlfailed"returnNonereturnresponse.read()url_parse.py#coding:utf-8importurlparse
全文预览
