Curitiba #42
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| from dateparser import parse | ||
| import datetime as dt | ||
| import re | ||
|
|
||
| from gazette.spiders.base import BaseGazetteSpider | ||
|
|
||
| import scrapy | ||
|
|
||
| from gazette.items import Gazette | ||
|
|
||
|
|
||
| class PrCuritibaSpider(BaseGazetteSpider): | ||
| TERRITORY_ID = '4106902' | ||
| name = 'pr_curitiba' | ||
| allowed_domains = ['legisladocexterno.curitiba.pr.gov.br'] | ||
| custom_settings = { | ||
| 'DEFAULT_REQUEST_HEADERS': { | ||
| 'user-agent': 'Mozilla/5.0' | ||
| } | ||
| } | ||
|
|
||
| def start_requests(self): | ||
| """ | ||
| The Curitiba website is a statefull page, so we can't just build the | ||
| request from zero, we have to resend the viewstate with every request. | ||
| @url http://legisladocexterno.curitiba.pr.gov.br/DiarioConsultaExterna_Pesquisa.aspx | ||
| @returns requests 1 | ||
| """ | ||
| todays_date = dt.date.today() | ||
| current_year = todays_date.year | ||
| for year in range(current_year, 2006, -1): | ||
| yield scrapy.FormRequest( | ||
| 'http://legisladocexterno.curitiba.pr.gov.br/DiarioConsultaExterna_Pesquisa.aspx', | ||
| formdata={ | ||
| 'ctl00$cphMasterPrincipal$ddlGrAno': str(year) | ||
| }, | ||
| callback=self.parse_year | ||
| ) | ||
|
|
||
| def parse_year(self, response): | ||
| for i in range(12): | ||
| yield self.scrape_month(response, i) | ||
|
There was a problem hiding this comment. Why create a new method that is used only here? Just yield what you are returning in L45 and avoid increasing the complexity of the spider. |
||
|
|
||
| def scrape_month(self, response, month): | ||
| return scrapy.FormRequest.from_response( | ||
| response, | ||
| formdata={ | ||
| '__EVENTTARGET': 'ctl00$cphMasterPrincipal$TabContainer1', | ||
| '__EVENTARGUMENT': 'activeTabChanged:{}'.format(month), | ||
| 'ctl00_cphMasterPrincipal_TabContalegacyDealPooliner1_ClientState': '{{"ActiveTabIndex":{},"TabState":[true,true,true,true,true,true,true,true,true,true,true,true]}}'.format(month) | ||
| }, | ||
| meta={"month": month}, | ||
| callback=self.parse_month | ||
| ) | ||
|
|
||
| def parse_month(self, response): | ||
| page_count = len(response.css(".grid_Pager:nth-child(1) table td").extract()) | ||
| month = response.meta["month"] | ||
| # The first page of pagination cannot be accessed by page number | ||
| yield scrapy.FormRequest.from_response( | ||
| response, | ||
| formdata={ | ||
| '__EVENTTARGET': 'ctl00$cphMasterPrincipal$TabContainer1', | ||
| 'ctl00_cphMasterPrincipal_TabContalegacyDealPooliner1_ClientState': '{{"ActiveTabIndex":{},"TabState":[true,true,true,true,true,true,true,true,true,true,true,true]}}'.format(month), | ||
| '__EVENTARGUMENT': 'activeTabChanged:{}'.format(month), | ||
| }, | ||
| callback=self.parse_page, | ||
| ) | ||
| for page_number in range(2, page_count + 1): | ||
| yield scrapy.FormRequest.from_response( | ||
| response, | ||
| formdata={ | ||
| '__EVENTARGUMENT': 'Page${}'.format(page_number), | ||
| '__EVENTTARGET': 'ctl00$cphMasterPrincipal$gdvGrid2' | ||
| }, | ||
| callback=self.parse_page, | ||
| ) | ||
|
|
||
| def parse_page(self, response): | ||
| for idx, row in enumerate(response.css(".grid_Row")): | ||
| pdf_date = row.css("td:nth-child(2) span ::text").extract_first() | ||
| gazette_id = row.css("td:nth-child(3) a ::attr(data-teste)").extract_first() | ||
| parsed_date = parse(f'{pdf_date}', languages=['pt']).date() | ||
| if gazette_id == '0': | ||
| starting_offset = 3 | ||
| yield scrapy.FormRequest.from_response( | ||
| response, | ||
| formdata={ | ||
| '__LASTFOCUS': '', | ||
| '__EVENTTARGET': 'ctl00$cphMasterPrincipal$gdvGrid2$ctl{num:02d}$lnkVisualizar'.format(num=(idx+starting_offset)), | ||
| '__EVENTARGUMENT': '', | ||
| '__ASYNCPOST': 'true' | ||
| }, | ||
| callback=self.scrap_not_extra_edition, | ||
| meta={"parsed_date": parsed_date} | ||
| ) | ||
| else: | ||
| yield Gazette( | ||
| date=parsed_date, | ||
| file_urls=["http://legisladocexterno.curitiba.pr.gov.br/DiarioSuplementoConsultaExterna_Download.aspx?id={}".format(gazette_id)], | ||
| is_extra_edition=True, | ||
| territory_id=self.TERRITORY_ID, | ||
| power='executive_legislature', | ||
| scraped_at=dt.datetime.utcnow() | ||
| ) | ||
|
|
||
| def scrap_not_extra_edition(self, response): | ||
|
There was a problem hiding this comment.
|
||
| parsed_date = response.meta['parsed_date'] | ||
| gazette_id = response.selector.re_first('Id=(\d+)') | ||
| return Gazette( | ||
| date=parsed_date, | ||
| file_urls=["http://legisladocexterno.curitiba.pr.gov.br/DiarioConsultaExterna_Download.aspx?id={}".format(gazette_id)], | ||
| is_extra_edition=False, | ||
| territory_id=self.TERRITORY_ID, | ||
| power='executive_legislature', | ||
| scraped_at=dt.datetime.utcnow() | ||
| ) | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
iis not a good variable name. It seems you are trying to search for months, right? Maybemonthis a better name.Anyway, this will fail in future months (for example, considering the date of this review, december/2018). Maybe include some validation to avoid trying to get future months.