#?-*-?coding:?utf-8?-*-
from?baike_moon?import??html_downloader,?html_parser,?html_outputer,?url_manager



class?SpiderMain(object):
????def?__init__(self):
????????self.urls?=?url_manager.UrlManager()
????????self.downloader?=?html_downloader.HtmlDownloader()
????????self.parser?=?html_parser.HtmlParser()
????????self.outputer??=html_outputer.HtmlOutputer()
????
????
????def?craw(self,?root_url):
????????count?=?1
????????self.urls.add_new_url(root_url)
????????while?self.urls.has_new_url():
????????????#try:
????????????????new_url?=?self.urls.get_new_url()
????????????????print?'craw?%d:%s'?%(count,new_url)
????????????????html_cont?=?self.downloader.download(new_url)
????????????????new_urls,new_data?=?self.parser.parse(new_url,html_cont)
????????????????self.urls.add_new_urls(new_urls)
????????????????self.outputer.collect_data(new_data)
????????????????
????????????????if?count?==?1000:
????????????????????break
????????????????count?=?count+1
????????????#except:
???????????#?????print?'爬取失敗'
?????????
????????self.outputer.output_html()
????????

if?__name__=="__main__":
????root_url?=?"http://baike.baidu.com/view/21087.htm"
????obj_spider?=?SpiderMain()
????obj_spider.craw(root_url)

報(bào)錯(cuò)：

pydev_imports.execfile(file, globals, locals) ?# execute the script

? File "E:\eclipse python\????\baike_moon\spider_main.py", line 38, in <module>

? ? obj_spider.craw(root_url)

? File "E:\eclipse python\????\baike_moon\spider_main.py", line 22, in craw

? ? new_urls,new_data = self.parser.parse(new_url,html_cont)

TypeError: 'set' object is not callable