Source code for kingfisher_scrapy.spiders.uganda_releases
import orjson
import scrapy
from kingfisher_scrapy.base_spiders import SimpleSpider
from kingfisher_scrapy.util import date_range_by_year
[docs]
class UgandaReleases(SimpleSpider):
"""
Domain
Government Procurement Portal (GPP) - Public Procurement and Disposal of Public Assets Authority (PPDA)
Spider arguments
from_date
Download only data from this year onward (YYYY format).
If ``until_date`` is provided, defaults to '2015'.
The year refers to the start of the fiscal year range, e.g. if ``from_date`` = '2019' then the fiscal year is
'2019-2020'
until_date
Download only data until this year (YYYY format).
If ``from_date`` is provided, defaults to the current year.
The year refers to the start of the fiscal year range, e.g. if ``until_date`` = '2019' then the fiscal year is
'2019-2020'
Bulk download documentation
https://gpp.ppda.go.ug/public/open-data/ocds/ocds-datasets
"""
name = "uganda_releases"
custom_settings = {
# Poll one export job at a time.
"CONCURRENT_REQUESTS": 1,
# Returns HTTP 403 if too many requests. (1 is too short.)
"DOWNLOAD_DELAY": 2,
}
# BaseSpider
date_format = "year"
date_required = True
default_from_date = "2015"
# SimpleSpider
data_type = "release_package"
url_prefix = "https://cdn.ppda.go.ug/api/open-data/v2/ocds/"
async def start(self):
# The download is asynchronous: create an export job, poll its status, then download the file.
for year in date_range_by_year(self.from_date.year, self.until_date.year):
fiscal_year = f"{year}-{year + 1}"
yield scrapy.Request(
f"{self.url_prefix}exports",
method="POST",
headers={"Content-Type": "application/json"},
body=orjson.dumps({"fy": fiscal_year, "format": "json"}),
meta={"file_name": f"{fiscal_year}.json"},
callback=self.parse_job,
)
def parse_job(self, response):
data = response.json()
if data.get("success"):
yield self.build_request(
data["status_url"],
formatter=None,
meta={
"file_name": response.request.meta["file_name"],
"wait_time": 30,
},
callback=self.parse_status,
)
else:
self.log_error_from_response(response, message=data)
def parse_status(self, response):
data = response.json()
meta = response.request.meta
match data.get("status"):
case "complete":
yield self.build_request(data["download_url"], formatter=None, meta={"file_name": meta["file_name"]})
case "queued" | "processing":
polls = meta.get("polls", 0) + 1
if polls > 20:
self.log_error_from_response(response, message=f"Gave up polling (polled {polls} times)")
return
request = response.request.copy()
request.meta["polls"] = polls
request.dont_filter = True
yield request
case _:
self.log_error_from_response(response, message=data)