#?-*-?coding:?utf-8?-*
#html解析器
import?re
import?urlparse

from?bs4?import?BeautifulSoup


class?HtmlParser(object):
????def?parse(self,?page_url,?html_cont):
????????if?page_url?is?None?or?html_cont?is?None:
????????????return
????????soup?=?BeautifulSoup(html_cont,?'html.parser',?from_encoding='utf-8')
????????new_urls?=?self._get_new_urls(page_url,?soup)
????????new_data?=?self._get_new_data(page_url,?soup)

????def?_get_new_urls(self,?page_url,?soup):
????????new_urls?=?set()
????????#?頁面其他url的格式，在原url后面拼接上#1，#2，#3...
????????links?=?soup.find_all('a',href=re.compile(r'#\d+'))
????????for?link?in?links:
????????????new_url?=?link['href']
????????????new_full_url?=?urlparse.urljoin(page_url,new_url)??#拼接成一個(gè)完整的url
????????????new_urls.add(new_full_url)
????????return?new_urls


????def?_get_new_data(self,?page_url,?soup):
????????res_data?=?{}?#字典類型，存放數(shù)據(jù)
????????#url
????????res_data['url']?=?page_url

????????#獲取title?<dd?class="lemmaWgt-lemmaTitle-title"><h1>Python</h1>
????????#<div?class="para-title?level-2"?label-module="para-title"><h2?class="title-text"><span?class="title-prefix">Python</span>下載Python</h2>
????????title_node?=?soup.find('dd',class_="lemmaWgt-lemmaTitle-title").find('h1')
????????res_data['title']=title_node.get_text()
????????print?res_data['title']

????????#獲取summary?<div?class="lemma-summary"?label-module="lemmaSummary">
????????summary_node?=?soup.find('div',class_="lemma-summary")
????????res_data['summary']?=?summary_node.get_text()
????????print?res_data['summary']

????????return?res_data
????????print?res_data