直接就爬取失敗了 ，是什么原因？
直接就爬取失敗了 ，是什么原因？ craw 1 : http://baike.baidu.com/view/21087.htm craw failed Process finished with exit code 0
11 回答
首頁(yè)上一頁(yè)12下一頁(yè)尾頁(yè)
舉報(bào)
0/150
提交
取消
直接就爬取失敗了 ，是什么原因？ craw 1 : http://baike.baidu.com/view/21087.htm craw failed Process finished with exit code 0
舉報(bào)
2016-05-15
html_parser.py 的代碼上來(lái)看看
2017-02-16
也有可能是你的get_text()方法寫成了gettext()方法，方法名錯(cuò)誤導(dǎo)致失敗。可以一步步打印試著調(diào)試
2016-09-01
from?bs4?import?BeautifulSoup 這一句我的錯(cuò)誤，
from BeautifulSoup import BeautifulSoup 改成這個(gè)就行了，有影響嗎？？
2016-05-15
?def?_get_new_urls(self,?page_url,?soup): ????????Movie_urls?=?set() ????????#?./nowplaying/ ????????#"div#screening?div.screening-hd?h2?span?a[abs:href]" ????????MovieNode?=?soup.find_all('div',class_='screening-hd').find('h2').find_all('a') ????????for?link?in?MovieNode: ????????????????new_url?=?link['href'] ????????????????new_full_url?=?urlparse.urljoin(page_url,new_url)???#匹配成與page_url相同的格式，補(bǔ)全相對(duì)url ????????????????Movie_urls.add(new_full_url) ????????print?Movie_urls ????????return??Movie_urls2016-05-15
html_downloader.py
2016-05-15
''' 輸出器 ''' class?HtmlOutputer(object): ???? ????def?__init__(self): ????????#維護(hù)收集的數(shù)據(jù) ????????self.datas?=?[] ???? ????def?collect_data(self,?data): ????????if?data?is?None: ????????????return ???????? ????????self.datas.append(data) ???? ????def?output_html(self): ????????fout?=?open('output.html',?'w') ???????? ????????fout.write('<html>') ????????fout.write('<body>') ????????fout.write('<table>') ???????? ????????for?data?in?self.datas: ????????????fout.write('<tr>') ????????????fout.write('<td>%s</td>'%data['url'].encode('utf-8')) ????????????fout.write('<td>%s</td>'%data['title'].encode('utf-8')) ????????????fout.write('<td>%s</td>'%data['summary'].encode('utf-8')) ????????????fout.write('</tr>') ????????fout.write('</table>') ????????fout.write('</body>') ????????fout.write('</html>')html_outputer.py
2016-05-15
url_manager.py
2016-05-15
spider_main.py
2016-05-15
from?bs4?import?BeautifulSoup import?re import?urlparse ''' 解析器 ''' class?HtmlParser(object): ???? ????#獲取頁(yè)面中所有相關(guān)的url ????def?_get_new_urls(self,?page_url,?soup): ???????? ????????new_urls?=?set() ???????? ????????#?查找這種格式的url?/view/123.htm ????????links?=?soup.find_all('a',?href=re.compile(r"/view/\d+\.htm")) ???????? ????????for?link?in?links: ????????????new_url?=?link['href'] ????????????#把new_url按照page_url的格式，拼接成一個(gè)完整的url ????????????new_full_url?=?urlparse.urljoin(page_url,?new_url) ????????????new_urls.add(new_full_url) ???? ????????return?new_urls ???? ????def?_get_new_data(self,?page_url,?soup): ????????res_data?=?{} ???????? ????????#url ????????res_data['url']?=?page_url ???????? ????????#得到標(biāo)題?<dd?class="lemmaWgt-lemmaTitle-title">?<h1>Python</h1> ????????title_node?=?soup.find('dd',?class_='lemmaWgt-lemmaTitle-title').find('h1') ????????res_data['title']?=?title_node.get_text() ???????? ????????#<div?class="lemma-summary"?label-module="lemmaSummary"> ????????summary_node?=?soup.find('div',?class_='lemma-summary') ????????#獲得summary正文的內(nèi)容 ????????res_data['summary']?=?summary_node.get_text() ???????? ????????return?res_data ???????? ????#?從cont中解析出兩個(gè)數(shù)據(jù)，新的url列表和數(shù)據(jù) ????def?parser(self,?page_url,?html_cont): ????????if?page_url?is?None?or?html_cont?is?None: ????????????return ???????? ????????soup?=?BeautifulSoup(html_cont,?'html.parser',?from_encoding='urf-8') ???????? ????????new_urls?=?self._get_new_urls(page_url,?soup) ????????new_data?=?self._get_new_data(page_url,?soup) ???????? ????????return?new_urls,?new_data2016-05-15
class HtmlParser(object):
? ? def parse(self,page_url,html_cont):
? ? ? ? if page_url is None or html_cont is None:
? ? ? ? ? ? return
? ? ? ? soup = BeautifulSoup(html_cont,'html.parser',from_encoding ='utf-8')
? ? ? ? new_urls = self._get_new_urls(page_url,soup)
? ? ? ? new_data = self._get_new_data(page_url,soup)
? ? ? ? return new_urls,new_data
? ? def _get_new_urls(self, page_url, soup):
? ? ? ? new_urls = set()
? ? ? ? #/view/123.htm
? ? ? ? links =soup.find_all('a',href = re.compiler(r"/view/\d+.html"))
? ? ? ? for link in links:
? ? ? ? ? ? new_url = link['href']
? ? ? ? ? ? new_full_url = urlparse.urljoin(page_url,new_url) ? #匹配成與page_url相同的格式，補(bǔ)全url
? ? ? ? ? ? new_urls.add(new_full_url)
? ? ? ? return ?new_urls
? ? def _get_new_data(self, page_url, soup):
? ? ? ? res_data ={}
? ? ? ? res_data['url']= page_url
? ? #<dd class="lemmaWgt-lemmaTitle-title"><h1>Python</h1>
? ? ? ? title_node =soup.find('dd',class_="lemmaWgt-lemmaTitle-title").find("h1")
? ? ? ? res_data['title']=title_node.get_text()
? ? #lemma-summary
? ? ? ? summary_node = soup.find('div',class_ ="lemma-summary")
? ? ? ? res_data['summary'] = summary_node.get_text()
? ? ? ? return res_data