from scrapy import Spider, Requestfrom selenium import webdriverclass MySpider(Spider): name = "my_spider" def __init__(self): self.browser = webdriver.Chrome(executable_path='E:/chromedriver') self.browser.set_page_load_timeout(100) def closed(self,spider): print("spider closed") self.browser.close() def start_requests(self): start_urls = [] with open("target_urls.txt", 'r', encoding='utf-8') as f: for line in f: url_id, url = line.split('\t\t') start_urls.append(url) for url in start_urls: yield Request(url=url, callback=self.parse) def parse(self, response): yield { 'target_url': response.url, 'comments': response.xpath('//div[@class="comments"]//em//text()').extract() }以上是我的scrapy代碼。我scrapy crawl my_spider -o comments.json用來運行爬蟲。您可能會注意到，對于每個 my url，都有一個url_id與之相關的獨特之處。如何將每個抓取的結果與url_id. 理想情況下，我想url_id將 .yield 輸出結果存儲在comments.json.
2 回答
繁花不似錦
TA貢獻1851條經(jīng)驗 獲得超4個贊
meta例如，嘗試傳入?yún)?shù)。我對你的代碼做了一些更新：
def start_requests(self):
with open("target_urls.txt", 'r', encoding='utf-8') as f:
for line in f:
url_id, url = line.split('\t\t')
yield Request(url, self.parse, meta={'url_id': url_id, 'original_url': url})
def parse(self, response):
yield {
'target_url': response.meta['original_url'],
'url_id': response.meta['url_id'],
'comments': response.xpath('//div[@class="comments"]//em//text()').extract()
}
江戶川亂折騰
TA貢獻1851條經(jīng)驗 獲得超5個贊
回答問題和評論，嘗試這樣的事情：
from scrapy import Spider, Request
from selenium import webdriver
class MySpider(Spider):
name = "my_spider"
def __init__(self):
self.browser = webdriver.Chrome(executable_path='E:/chromedriver')
self.browser.set_page_load_timeout(100)
def closed(self,spider):
print("spider closed")
self.browser.close()
def start_requests(self):
with open("target_urls.txt", 'r', encoding='utf-8') as f:
for line in f:
url_id, url = line.split('\t\t')
yield Request(url=url, callback=self.parse, meta={'url_id':url_id,'url':url})
def parse(self, response):
yield {
'target_url': response.meta['url'],
'comments': response.xpath('//div[@class="comments"]//em//text()').extract(),
'url_id':response.meta['url_id']
}
正如上一個答案中所說，您可以使用 META ( http://scrapingauthority.com/scrapy-meta )在各種方法之間傳遞參數(shù)。
添加回答
舉報
0/150
提交
取消