TypeError: 'NoneType' object is not iterable
import?url_manager,html_downloader,html_parser,html_outputer
class?SpiderMain(object):
????????"""docstring?for?SpiderMain"""
????????def?__init__(self):
????????????????self.urls?=?url_manager.UrlManager()
????????????????self.downloader?=?html_downloader.HtmlDownloader()
????????????????self.parser?=?html_parser.HtmlParser()
????????????????self.outputer?=?html_outputer.HtmlOutputer()
????????def?craw(self,root_url):
????????????????count?=?1
????????????????self.urls.add_new_url(root_url)
????????????????while?self.urls.has_new_url():
????????????????????????print(count)
????????????????????????#?try:
????????????????????????new_url?=?self.urls.get_new_url()
????????????????????????print('craw?%d?:?%s'%(count,new_url))
????????????????????????html_cont?=?self.downloader.download(new_url)
????????????????????????new_urls,new_data?=?self.parser.parse(new_url,html_cont)
????????????????????????self.urls.add_new_urls(new_urls)
????????????????????????self.outputer.collect_data(new_data)
????????????????????????#?except:
????????????????????????#???????print('craw?failed')
????????????????????????if?count?==?1000:
????????????????????????????????break
????????????????????????count?=?count?+?1
????????????????????????print(count)
????????????????self.outputer.output_html()
if?__name__?==?"__main__":
????????root_url?=?"https://baike.baidu.com/item/Python"
????????obj_spider?=?SpiderMain()
????????obj_spider.craw(root_url)運(yùn)行結(jié)果如下：
1
craw 1 : https://baike.baidu.com/item/Python
Traceback (most recent call last):
? File "C:\Users\liukai7\Desktop\New folder\新浪微博爬蟲\spider_main.py", line 44, in <module>
? ? obj_spider.craw(root_url)
? File "C:\Users\liukai7\Desktop\New folder\新浪微博爬蟲\spider_main.py", line 25, in craw
? ? new_urls,new_data = self.parser.parse(new_url,html_cont)
TypeError: 'NoneType' object is not iterable
不知道問題在哪里，請(qǐng)幫忙看下
2017-12-01
解析器第10行集合名字寫錯(cuò)了，改過來試一下~
2017-11-10
不知道你解決了沒有。是你指定的解析器寫錯(cuò)了。Line 41 少了一個(gè)r
2017-10-30
from?bs4?import?BeautifulSoup import?re import?urllib.parse class?HtmlParser(object): def?_get_new_urls(self,page_url,soup): new_url?=?set() links?=?soup.find_all('a',href=re.compile(r'/item/\w+')) for?link?in?links: new_url?=?link['href'] #new_full_url?=?urlparse.urljoin(page_url,new_url)python?2.7 new_full_url?=?urllib.parse.urljoin(page_url,?new_url) new_urls.add(new_full_url) return?new_urls def?_get_new_data(self,page_url,soup): res_data?=?{} #url res_data['url']?=?page_url #<dd?class="lemmaWgt-lemmaTitle-title"><h1>Python</h1> title_node?=?soup.find('dd',class_=?"lemmaWgt-lemmaTitle-title").find('h1') res_data['title']?=?title_node.get_text() #<div?class="lemma-summary"?label-module="lemmaSummary"> summary_node?=?soup.find('div',class_="lemma-summary") res_data['summary_node']?=?summary_node.get_text() return?res_data def?parse(self,page_url,html_cont): if?page_url?is?None?or?html_cont?is?None: return soup?=?BeautifulSoup(html_cont,'html.paser',from_encoding='utf-8') new_urls?=?self._get_new_urls(page_url,soup) new_data?=?self._get_new_data(page_url,soup) return?new_urls,new_data2017-10-27
從第25行可知解析器返回了None，所以你的解析器應(yīng)該是有問題
2017-10-27
從這里看沒什么問題，可能是你解析器哪里寫錯(cuò)了吧，可以貼出來看看