rennerocha/tutorial-pyconus2024-gathering-data
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Reorganize example code

Browse source
This commit is contained in:
Renne Rocha 2024-05-12 10:18:16 -03:00
parent 7c10d1c4b0
commit 6580c266dd
8 changed files with 143 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

16
code/exercise-1.py
View file

@ -7,15 +7,9 @@ class QuotesSpider(scrapy.Spider):
start_urls = ["https://quotes.toscrape.com"]
def parse(self, response):
quotes = response.css(".quote")
for quote in quotes:
yield {
"quote": quote.css(".text::text").get(),
"author": quote.css(".author::text").get(),
"author_url": response.urljoin(quote.css("span a::attr(href)").get()),
"tags": quote.css(".tag *::text").getall(),
}
# 1. Get the list of quotes availabe at the page
yield scrapy.Request(
response.urljoin(response.css(".next a::attr(href)").get())
)
# 2. Parse each quote found and yield the quote item
# 3. Follow the next page link
...