Reorganize example code
This commit is contained in:
parent
7c10d1c4b0
commit
6580c266dd
8 changed files with 143 additions and 56 deletions
36
code/exercise-solutions/exercise-4.py
Normal file
36
code/exercise-solutions/exercise-4.py
Normal file
|
@ -0,0 +1,36 @@
|
|||
import scrapy
|
||||
|
||||
|
||||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes_complete"
|
||||
allowed_domains = ["quotes.toscrape.com"]
|
||||
start_urls = ["https://quotes.toscrape.com"]
|
||||
|
||||
def parse(self, response):
|
||||
quotes = response.css(".quote")
|
||||
for quote in quotes:
|
||||
about_url = response.urljoin(quote.css("span a::attr(href)").get())
|
||||
|
||||
quote_info = {
|
||||
"quote": quote.css(".text::text").get(),
|
||||
"author": quote.css(".author::text").get(),
|
||||
"author_url": about_url,
|
||||
"tags": quote.css(".tag *::text").getall(),
|
||||
}
|
||||
|
||||
yield scrapy.Request(
|
||||
about_url,
|
||||
callback=self.parse_about_page,
|
||||
meta={"quote_info": quote_info},
|
||||
dont_filter=True,
|
||||
)
|
||||
|
||||
yield scrapy.Request(
|
||||
response.urljoin(response.css(".next a::attr(href)").get()),
|
||||
)
|
||||
|
||||
def parse_about_page(self, response):
|
||||
quote = response.meta["quote_info"]
|
||||
author_born_date = response.css(".author-born-date::text").get()
|
||||
quote["author_born_date"] = author_born_date
|
||||
yield quote
|
Loading…
Add table
Add a link
Reference in a new issue