
    3 d                     8   d Z ddlmZmZ ddlmZmZ ddlmZ ddl	m
Z
mZmZmZmZmZmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ  ee          Z ddde dddededee!e"f         dee#         dee#         dede!fdZ$ G d d          Z%dS )a  
An extension to retry failed requests that are potentially caused by temporary
problems such as a connection timeout or HTTP 500 error.

You can change the behaviour of this middleware by modifying the scraping settings:
RETRY_TIMES - how many times to retry a failed page
RETRY_HTTP_CODES - which HTTP response codes to retry

Failed pages are collected on the scraping process and rescheduled at the end,
once the spider has finished crawling all regular (non failed) pages.
    )Logger	getLogger)OptionalUnion)defer)ConnectErrorConnectionDoneConnectionLostConnectionRefusedErrorDNSLookupErrorTCPTimedOutErrorTimeoutError)ResponseFailed)TunnelError)NotConfigured)Request)Spider)global_object_name)response_status_messageunspecifiedNretry)reasonmax_retry_timespriority_adjustloggerstats_base_keyrequestspiderr   r   r   r   r   c                   |j         j        }|j         j        }| j                            dd          dz   }	|1| j                            d          }||                    d          }|	|k    r|                    d| |	|dd	|i
           |                                 }
|	|
j        d<   d|
_        ||                    d          }| j	        |z   |
_	        t          |          r
 |            }t          |t                    rt          |j                  }|                    | d           |                    | d|            |
S |                    | d           |                    d| |	|dd	|i
           dS )a  
    Returns a new :class:`~scrapy.Request` object to retry the specified
    request, or ``None`` if retries of the specified request have been
    exhausted.

    For example, in a :class:`~scrapy.Spider` callback, you could use it as
    follows::

        def parse(self, response):
            if not response.text:
                new_request_or_none = get_retry_request(
                    response.request,
                    spider=self,
                    reason='empty',
                )
                return new_request_or_none

    *spider* is the :class:`~scrapy.Spider` instance which is asking for the
    retry request. It is used to access the :ref:`settings <topics-settings>`
    and :ref:`stats <topics-stats>`, and to provide extra logging context (see
    :func:`logging.debug`).

    *reason* is a string or an :class:`Exception` object that indicates the
    reason why the request needs to be retried. It is used to name retry stats.

    *max_retry_times* is a number that determines the maximum number of times
    that *request* can be retried. If not specified or ``None``, the number is
    read from the :reqmeta:`max_retry_times` meta key of the request. If the
    :reqmeta:`max_retry_times` meta key is not defined or ``None``, the number
    is read from the :setting:`RETRY_TIMES` setting.

    *priority_adjust* is a number that determines how the priority of the new
    request changes in relation to *request*. If not specified, the number is
    read from the :setting:`RETRY_PRIORITY_ADJUST` setting.

    *logger* is the logging.Logger object to be used when logging messages

    *stats_base_key* is a string to be used as the base key for the
    retry-related job stats
    retry_timesr      Nr   RETRY_TIMESz?Retrying %(request)s (failed %(retry_times)d times): %(reason)s)r   r    r   r   )extraTRETRY_PRIORITY_ADJUSTz/countz/reason_count/z/max_reachedzGGave up retrying %(request)s (failed %(retry_times)d times): %(reason)s)crawlersettingsstatsmetagetgetintdebugcopydont_filterprioritycallable
isinstance	Exceptionr   	__class__	inc_valueerror)r   r   r   r   r   r   r   r&   r'   r    new_requests              Blib/python3.11/site-packages/scrapy/downloadermiddlewares/retry.pyget_retry_requestr7   %   s   d ~&HN E,""=!44q8K =!,**+<== 	=&oom<<Oo% MvNNV$ 	 	
 	
 	

  '||~~*5'"& 	G&oo.EFFO&//AF 	VXXFfi(( 	:'(899F>111222>AAAABBB	OO~333444
LLTK6JJ     
 4    c                   `    e Zd Zej        eeeeee	e
eeefZd Zed             Zd Zd Zd ZdS )RetryMiddlewarec                    |                     d          st          |                    d          | _        t	          d |                    d          D                       | _        |                    d          | _        d S )NRETRY_ENABLEDr"   c              3   4   K   | ]}t          |          V  d S N)int).0xs     r6   	<genexpr>z+RetryMiddleware.__init__.<locals>.<genexpr>   s9       $
 $
CFF$
 $
 $
 $
 $
 $
r8   RETRY_HTTP_CODESr$   )getboolr   r*   r   setgetlistretry_http_codesr   )selfr&   s     r6   __init__zRetryMiddleware.__init__   s    00 	 '}== # $
 $
$,,-?@@$
 $
 $
 !
 !
  (/FGGr8   c                 "     | |j                   S r>   )r&   )clsr%   s     r6   from_crawlerzRetryMiddleware.from_crawler   s    s7#$$$r8   c                     |j                             dd          r|S |j        | j        v r-t	          |j                  }|                     |||          p|S |S N
dont_retryF)r(   r)   statusrG   r   _retry)rH   r   responser   r   s        r6   process_responsez RetryMiddleware.process_response   sc    <L%00 	O?d33 	D,X_==F;;w77C8Cr8   c                     t          || j                  r2|j                            dd          s|                     |||          S d S d S rN   )r0   EXCEPTIONS_TO_RETRYr(   r)   rQ   )rH   r   	exceptionr   s       r6   process_exceptionz!RetryMiddleware.process_exception   sc    i!9:: 	;7<CSCS%D
 D
 	; ;;w	6:::	; 	; 	; 	;r8   c                     |j                             d| j                  }|j                             d| j                  }t	          |||||          S )Nr   r   )r   r   r   r   )r(   r)   r   r   r7   )rH   r   r   r   r   r   s         r6   rQ   zRetryMiddleware._retry   s^    !,**+<d>RSS!,**+<d>RSS ++
 
 
 	
r8   N)__name__
__module____qualname__r   r   r   r   r	   r   r
   r   r   IOErrorr   rU   rI   classmethodrL   rS   rW   rQ    r8   r6   r:   r:   |   s        
 	H H H % % [%  ; ; ;	
 	
 	
 	
 	
r8   r:   )&__doc__loggingr   r   typingr   r   twisted.internetr   twisted.internet.errorr   r	   r
   r   r   r   r   twisted.web.clientr   &scrapy.core.downloader.handlers.http11r   scrapy.exceptionsr   scrapy.http.requestr   scrapy.spidersr   scrapy.utils.pythonr   scrapy.utils.responser   rY   retry_loggerstrr1   r?   r7   r:   r^   r8   r6   <module>rm      s   
 
 & % % % % % % % " " " " " " " " " " " " " "                  . - - - - - > > > > > > + + + + + + ' ' ' ' ' ' ! ! ! ! ! ! 2 2 2 2 2 2 9 9 9 9 9 9y"" %2%)%)!!T T TT T #y.!	T
 c]T c]T T T T T Tn6
 6
 6
 6
 6
 6
 6
 6
 6
 6
r8   