diff --git a/src/api/rate_limiter.py b/src/api/rate_limiter.py index 68a2ce2..79d4a07 100644 --- a/src/api/rate_limiter.py +++ b/src/api/rate_limiter.py @@ -22,12 +22,22 @@ def __init__(self, request, log=None, message="", msg_args=()): message = f"Request %s: %s + %s {message}" log(message, request.method, request.url, request.cookies) +class RLRequestFailed(RateLimiterError): + def __init__(self, request, *args, **kwargs): + super().__init__(request, *args, **kwargs) + +class RLRequestClientError(RLRequestFailed): + def __init__(self, request, *args, **kwargs): + super().__init__(request, *args, **kwargs) class RLErrorWithPause(RateLimiterError): def __init__(self, request, time_to_wait, *args, **kwargs): super().__init__(request, *args, **kwargs) self.time_to_wait = time_to_wait +class RLRequestServerError(RLErrorWithPause): + def __init__(self, request, time_to_wait, *args, **kwargs): + super().__init__(request, time_to_wait, *args, **kwargs) # pylint: disable=too-few-public-methods class RequestEntry: @@ -101,6 +111,21 @@ def handle_get_request(self, request): match resp.status_code: case 200: return resp.json() + case 400, 401, 403, 404, 405: + # HTTP failure client side + raise RLRequestClientError( + request, + self.logger.error, + f"40X error code, client side error, useless to retry." + ) + case 500, 501, 502, 503, 504: + # HTTP failure server side + raise RLRequestServerError( + request, + 300, + self.logger.error, + f"50X error code, server side error, retry in 5 min ..." + ) case 429: try: time_to_wait = int(resp.headers["Retry-After"]) @@ -163,6 +188,14 @@ async def handle_requests(self): try: self.requests[request.key]["result"] = self.handle_get_request(request) + except RLRequestFailed as exc: + # Request failed and should not be sent again + self.requests[request.key]["result"] = None + # set the exception + self.requests[request.key]["exception"] = ( + RLRequestFailed(request, self.logger.error, "Request Failed"), + exc, + ) except RateLimiterError as exc: if isinstance(exc, RLErrorWithPause): await self.wait(exc.time_to_wait) diff --git a/src/api/rootme_api.py b/src/api/rootme_api.py index c093fc2..acc553a 100644 --- a/src/api/rootme_api.py +++ b/src/api/rootme_api.py @@ -1,6 +1,11 @@ +import logging + from os import getenv -from api.rate_limiter import RateLimiter +from api.rate_limiter import RLRequestFailed, RateLimiter +class RootMeAPIError(Exception): + def __init__(self, *args: object) -> None: + super().__init__(*args) class RootMeAPIManager: """ @@ -18,6 +23,7 @@ def __init__(self, rate_limiter: RateLimiter): else: raise RuntimeError("API_URL is not set.") self.rate_limiter = rate_limiter + self.logger = logging.getLogger(__name__) def get_rate_limiter(self): return self.rate_limiter @@ -28,11 +34,17 @@ async def get_challenge_by_id(self, _id): -> returns the raw json for now """ # use the api_key in the cookies + request_url = f"{self.API_URL}/challenges/{_id}" cookies = {"api_key": self.API_KEY.strip('"')} + request_method = "GET" # ask the rate limiter for the request - data = await self.rate_limiter.make_request( - f"{self.API_URL}/challenges/{_id}", cookies, "GET" - ) + try: + data = await self.rate_limiter.make_request( + request_url, cookies, request_method + ) + except RLRequestFailed as exc: + self.logger.error("%s Request to get challenge %s failed.", request_method, request_url) + raise RootMeAPIError() # Handle the fact the request is wrong return data async def get_user_by_id(self, _id): @@ -41,7 +53,15 @@ async def get_user_by_id(self, _id): -> returns the raw json for now """ # use the api_key in the cookies + request_url = f"{self.API_URL}/auteurs/{_id}" cookies = {"api_key": self.API_KEY.strip('"')} + request_method = "GET" # ask the rate limiter for the request - data = await self.rate_limiter.make_request(f"{self.API_URL}/auteurs/{_id}", cookies, "GET") + try: + data = await self.rate_limiter.make_request( + request_url, cookies, request_method + ) + except RLRequestFailed as exc: + self.logger.error("%s Request to get user %s failed.", request_method, request_url) + raise RootMeAPIError() return data diff --git a/src/database/db_manager.py b/src/database/db_manager.py index d25d67a..398a0fd 100644 --- a/src/database/db_manager.py +++ b/src/database/db_manager.py @@ -2,6 +2,7 @@ import sqlite3 from os import getenv, path +from api.rootme_api import RootMeAPIError from database.db_structure import ( sql_create_user_table, sql_add_user, @@ -64,8 +65,11 @@ async def add_user(self, idx): if self.has_user(idx): return None - # Retreive information from RootMe API - raw_user_data = await self.api_manager.get_user_by_id(idx) + # Retrieve information from RootMe API + try: + raw_user_data = await self.api_manager.get_user_by_id(idx) + except RootMeAPIError: + return None user = User(raw_user_data) cur.execute(sql_add_user, user.to_tuple()) @@ -102,7 +106,13 @@ async def fetch_user_new_solves(self, idx): if user is None: raise InvalidUser(idx, "DatabaseManager.fetch_user_new_solves: User %s not in database") - raw_user_data = await self.api_manager.get_user_by_id(idx) + try: + raw_user_data = await self.api_manager.get_user_by_id(idx) + except RootMeAPIError: + # If for some reason we can get the user on this iteration + # we will get him next time maybe ... + self.logger.error("User %s could not be fetch from the API, yet we keep running", idx) + return user.update_new_solves(raw_user_data) if not user.has_new_solves(): self.logger.debug("'%s' hasn't any new solves", user) @@ -110,7 +120,11 @@ async def fetch_user_new_solves(self, idx): self.logger.info("'%s' has %s new solves", user, user.nb_new_solves) for challenge_id in user.yield_new_solves(raw_user_data): - challenge_data = await self.api_manager.get_challenge_by_id(challenge_id) + try: + challenge_data = await self.api_manager.get_challenge_by_id(challenge_id) + except RootMeAPIError: + # If we can't fetch the challenge, sadly there is not much we can do + continue challenge = Challenge(challenge_id, challenge_data) self.logger.debug("'%s' solved '%s'", repr(user), repr(challenge)) yield challenge