Cria spider para Rio das Ostras/RJ

data_collection/gazette/spiders/rj/rj_rio_das_ostras.py

@@ -0,0 +1,49 @@
+import re
+from datetime import date, datetime as dt
+
+import scrapy
+
+from gazette.items import Gazette
+from gazette.spiders.base import BaseGazetteSpider
+
+
+class RjRioDasOstrasSpider(BaseGazetteSpider):
+    name = "rj_rio_das_ostras"
+    TERRITORY_ID = "3304524"
+    allowed_domains = ["riodasostras.rj.gov.br"]
+    start_date = date(2001, 7, 13)
+
+    def start_requests(self):
+        for year in range(self.start_date.year, self.end_date.year + 1):
+            base_url = f"https://appro.riodasostras.rj.gov.br/riodasostrasapp_server/api/jornais/search/site?&limit=600&offset=0&orderBy=data&orderDir=desc&ano={year}"
+            yield scrapy.Request(base_url)
+
+    def parse(self, response):
+        for gazette_data in response.json():
+            raw_gazette_date = gazette_data["data"][:10]
+            gazette_date = dt.strptime(raw_gazette_date, "%Y-%m-%d").date()
+
+            if gazette_date > self.end_date:
+                continue
+
+            if gazette_date < self.start_date:
+                return
+
+            match = re.search(r"Edição.*?(\d+)", gazette_data["edicao"])
+            gazette_edition_number = "" if match is None else match.group(1)
+            is_extra_edition = bool(
+                re.search(
+                    r"anex|encar|loa|ppa|ldo|conc",
+                    gazette_data["edicao"],
+                    re.IGNORECASE,
+                )
+            )
+            gazette_url = gazette_data["link"]
+
+            yield Gazette(
+                date=gazette_date,
+                edition_number=gazette_edition_number,
+                is_extra_edition=is_extra_edition,
+                file_urls=[gazette_url],
+                power="executive_legislative",
+            )