
    3 d;                     d    d Z ddlZddlmZ ddlmZ  ej        e          Z G d d          Z	dS )zV
Url Length Spider Middleware

See documentation in docs/topics/spider-middleware.rst
    N)NotConfigured)Requestc                   <    e Zd Zd Zed             Zd Zd Zd ZdS )UrlLengthMiddlewarec                     || _         d S N)	maxlength)selfr	   s     Blib/python3.11/site-packages/scrapy/spidermiddlewares/urllength.py__init__zUrlLengthMiddleware.__init__   s    "    c                 T    |                     d          }|st           | |          S )NURLLENGTH_LIMIT)getintr   )clssettingsr	   s      r   from_settingsz!UrlLengthMiddleware.from_settings   s0    OO$566	 	 s9~~r   c                 (      fd|pdD             S )Nc              3   H   K   | ]}                     |          |V  d S r   _filter).0rr
   spiders     r   	<genexpr>z<UrlLengthMiddleware.process_spider_output.<locals>.<genexpr>   s6      CCa4<<6+B+BCCCCCCCr    r   )r
   responseresultr   s   `  `r   process_spider_outputz)UrlLengthMiddleware.process_spider_output   s%    CCCCC6<RCCCCr   c                Z   K   |pd2 3 d {V }|                      ||          r|W V  #6 d S )Nr   r   )r
   r   r   r   r   s        r   process_spider_output_asyncz/UrlLengthMiddleware.process_spider_output_async   s]      | 	 	 	 	 	 	 	!||Av&&  $||s   *c                    t          |t                    rlt          |j                  | j        k    rOt
                              d| j        |j        dd|i           |j        j        	                    d|           dS dS )	Nz4Ignoring link (url length > %(maxlength)d): %(url)s )r	   urlr   )extrazurllength/request_ignored_count)r   FT)

isinstancer   lenr#   r	   loggerinfocrawlerstats	inc_value)r
   requestr   s      r   r   zUrlLengthMiddleware._filter"   s    gw'' 		C,<,<t~,M 		KKF"nW[AA(    
 N **1& +    5tr   N)	__name__
__module____qualname__r   classmethodr   r   r!   r   r   r   r   r   r      sn        # # #   [D D D  
    r   r   )
__doc__loggingscrapy.exceptionsr   scrapy.httpr   	getLoggerr-   r'   r   r   r   r   <module>r6      s      + + + + + +      		8	$	$         r   