當我在 cmd 中編寫此命令時爬行報價 -o item.csv -au=test_user_name -ap=test_passporw_name -a urls= http://books.toscrape.com/它顯示raise ValueError('Missing scheme in request url: %s' % self._url) ValueError: Missing scheme in request url: h# -*- coding: utf-8 -*-from scrapy.contrib.spiders.init import InitSpiderfrom scrapy.http import Request, FormRequestfrom scrapy.contrib.linkextractors.sgml import SgmlLinkExtractorfrom scrapy.contrib.spiders import Rulefrom scrapy.utils.response import open_in_browserfrom scrapy.spider import BaseSpiderfrom scrapy.selector import HtmlXPathSelectorclass QuotesSpider(InitSpider): name = 'quotes' allowed_domains = ['quotes.toscrape.com'] login_page='http://quotes.toscrape.com/login' start_urls = [''] username='' password='' def __init__(self,u,p,urls): self.username=u self.password=p self.start_urls=urls def init_request(self): #"""This function is called before crawling starts.""" return Request(url=self.login_page, callback=self.login) def login(self, response): csrf_token=response.xpath('//*[@name="csrf_token"]//@value').extract_first() return FormRequest.from_response(response, formdata={'csrf_token': csrf_token, 'username': self.username, 'password': self.password, }, callback=self.check_login_response) def check_login_response(self, response): # open_in_browser(response) #"""Check the response returned by a login request to see if we aresuccessfully logged in.""" if "Logout" in response.body: self.log("\n\n\nSuccessfully logged in. Let's start crawling!\n\n\n") # Now the crawling can begin.. return self.initialized() # ****THIS LINE FIXED THE LAST PROBLEM*****
2 回答

莫回無
TA貢獻1865條經(jīng)驗 獲得超7個贊
self.start_urls=urls
生成start_urls
一個字符串而不是一個列表。
這使得該字符串中的每個字符都被解釋為一個 url。
只需start_urls
列出一個列表,您的代碼應該可以工作:
self.start_urls = [urls]
此外,您不需要將變量初始化為虛擬值,也不需要自己解析 csrf_token(使用時會自動完成FormRequest.from_response()
)
附帶說明一下,您的代碼看起來像是為相當舊的 scrapy 版本編寫的 - 您的大部分導入已被移動、重命名或棄用。
也許您應該通過快速重讀文檔來刷新代碼。
添加回答
舉報
0/150
提交
取消