NgmLddlmZddlZddlZddlZddlmZmZmZm Z ddl m Z m Z m Z mZddlmZddlmZmZddlmZddlmZdd lmZdd lmZdd lmZdd lmZm Z m!Z!dd l"m#Z#ddl$m%Z%ddl&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,ddl-m.Z.m/Z/ddl0m1Z1m2Z2ddl3m4Z4m5Z5m6Z6m7Z7erddl8m9Z9m:Z:ddl;mZ>ej~e@ZAedZBGddZCGddZDGddeDZEy)) annotationsN) TYPE_CHECKINGAnyTypeVarcast)Deferred DeferredListinlineCallbacks maybeDeferred) verifyClass)Spidersignals) AddonManagerExecutionEngine)ExtensionManager) ISpiderLoader) LogFormatter) BaseSettingsSettingsoverridden_settings) SignalManager)StatsCollector)LogCounterHandlerconfigure_loggingget_scrapy_root_handlerinstall_scrapy_root_handlerlog_reactor_infolog_scrapy_info)build_from_crawler load_object)install_shutdown_handlers signal_names)install_reactoris_asyncio_reactor_installed#verify_installed_asyncio_event_loopverify_installed_reactor) GeneratorIterable) SpiderLoader)RequestFingerprinter_TceZdZ d ddZddZddZeddZddZddZ eddZ e dd Z dd Z dd Zdd Zdd ZddZy)CrawlerNct|tr tdt|ts| t |}||_|j |_|j j|j|jt||_ t||_ ||_d|_d|_d|_d|_d|_d|_d|_d|_y)Nz5The spidercls argument must be a class, not an objectF) isinstancer ValueErrordictr spiderclscopysettingsupdate_settings_update_root_log_handlerraddonsrr _init_reactorcrawling_started extensionsstats logformatterrequest_fingerprinterspiderengine)selfr3r5 init_reactors .lib/python3.12/site-packages/scrapy/crawler.py__init__zCrawler.__init__7s i (TU U h %)9)H'0"*--/  &&t}}5 %%'$0$6 &3D&9 #/# # 37,0 15BF"%) .2 cDtt|jyyN)rrr5rBs rDr7z Crawler._update_root_log_handlerVs " $ 0 ' 6 1rFc\|jjry|jj|jt |jd||_t ||jjdtjjfd|_ |jj|jtjt |jd}|j||_t#t |jd||_|jd}|jd}|j&r|r t)||nd d lm}t/|r,t1|t3r |r t5|t/t7j||_|jj;t=t?|j}t@jCd d tEjF|iy) N STATS_CLASS LOG_LEVEL)levelcBtjjSrH)loggingroot removeHandler)handlersrDz)Crawler._apply_settings..fs (B(B7(KrF LOG_FORMATTERREQUEST_FINGERPRINTER_CLASSTWISTED_REACTORASYNCIO_EVENT_LOOPrreactorz!Overridden settings: %(settings)sr5)$r5frozenr8 load_settingsr!r=rgetrOrP addHandler_Crawler__remove_handlerrconnectengine_stopped from_crawlerr>r r?r9r$twisted.internetrYrr'r%r&rr<freezer2rloggerinfopprintpformat)rBlf_cls reactor_class event_looprYdrRs @rD_apply_settingszCrawler._apply_settings[s ==    !!$--0>[}!=>tD #D 0A0A+0NO (!L T22G4J4JK%01O%P"//5%7  &CD E & " "]]+<= --(<=    z:4    $] 3+-*3J?  *77=  $T]]3 4 0:v~~a?P2Q rFc/pK|jr td|jr tddx|_|_ |j|i||_|j |j |j|_t|jj}|jj|j|t|jjy#t$r1d|_|j|jjwxYww)NzCrawling already taking placez?Cannot run Crawler.crawl() more than once on the same instance.TF)r: RuntimeErrorr;_create_spiderr@rlr7_create_enginerAiterstart_requests open_spiderr start Exceptionclose)rBargskwargsrrs rDcrawlz Crawler.crawls ==>? ? ==Q )-,   -$--t>v>DK  "  ) ) +--/DK!$++"<"<">?N++))$++~F F 1 12 2 !DM{{&kk''))   s=D6B8C98D69:D33D6cB|jj|g|i|SrH)r3ra)rBrwrxs rDrozCrawler._create_spiders#*t~~**4A$A&AArFc"tfdS)Nc$jSrH)stop)_rBs rDrSz(Crawler._create_engine..s tyy{rFrrIs`rDrpzCrawler._create_enginest%:;;rFc#K|jr7d|_|jsJt|jjyyw)zoStarts a graceful stop of the crawler and returns a deferred that is fired when the crawler is stopped.FN)r:rAr r}rIs rDr}z Crawler.stops< ==!DM;; ; 0 01 1 sAAc2|D]}t||s|cSyrH)r0)component_class components components rD_get_componentzCrawler._get_components#$I)_5  $rFcN|j||jjS)zReturn the run-time instance of an :ref:`add-on ` of the specified class or a subclass, or ``None`` if none is found. .. versionadded:: 2.12 )rr8rBclss rD get_addonzCrawler.get_addons! ""3 (:(:;;rFc|js td|j||jjjj S)axReturn the run-time instance of a :ref:`downloader middleware ` of the specified class or a subclass, or ``None`` if none is found. .. versionadded:: 2.12 This method can only be called after the crawl engine has been created, e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. z_Crawler.get_downloader_middleware() can only be called after the crawl engine has been created.)rArnr downloader middleware middlewaresrs rDget_downloader_middlewarez!Crawler.get_downloader_middlewaresH{{5 ""3 (>(>(I(I(U(UVVrFc||js td|j||jjS)aoReturn the run-time instance of an :ref:`extension ` of the specified class or a subclass, or ``None`` if none is found. .. versionadded:: 2.12 This method can only be called after the extension manager has been created, e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. zXCrawler.get_extension() can only be called after the extension manager has been created.)r<rnrrrs rD get_extensionzCrawler.get_extensions<6 ""3(C(CDDrFc|js td|j||jjjj S)ahReturn the run-time instance of a :ref:`item pipeline ` of the specified class or a subclass, or ``None`` if none is found. .. versionadded:: 2.12 This method can only be called after the crawl engine has been created, e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. zWCrawler.get_item_pipeline() can only be called after the crawl engine has been created.)rArnrscraperitemprocrrs rDget_item_pipelinezCrawler.get_item_pipelineH{{1 ""3 (;(;(D(D(P(PQQrFc|js td|j||jjjj S)apReturn the run-time instance of a :ref:`spider middleware ` of the specified class or a subclass, or ``None`` if none is found. .. versionadded:: 2.12 This method can only be called after the crawl engine has been created, e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. z[Crawler.get_spider_middleware() can only be called after the crawl engine has been created.)rArnrrspidermwrrs rDget_spider_middlewarezCrawler.get_spider_middlewarerrFNF)r3z type[Spider]r5 dict[str, Any] | Settings | NonerCbool)returnNone)rwrrxrr#Generator[Deferred[Any], Any, None])rwrrxrrr )rrrr)rtype[_T]rz Iterable[Any]r _T | None)rrrr)__name__ __module__ __qualname__rEr7rlr ryrorpr} staticmethodrrrrrrrFrDr.r.6s6:" 3333 3>7 - ^.B<22!/< <W"E$R"RrFr.ceZdZdZeddZeddZdddZ ddZ dd Z dd Z dd Z dd Z edd Zy) CrawlerRunnera This is a convenient helper class that keeps track of, manages and runs crawlers inside an already setup :mod:`~twisted.internet.reactor`. The CrawlerRunner object must be instantiated with a :class:`~scrapy.settings.Settings` object. This class shouldn't be needed (since Scrapy is responsible of using it accordingly) unless writing scripts that manually handle the crawling process. See :ref:`run-from-script` for an example. c|jSrH) _crawlersrIs rDrSzCrawlerRunner.sT^^rFzeSet of :class:`crawlers ` started by :meth:`crawl` and managed by this class.)docc|jd}t|}tt|t d|j |j S)z'Get SpiderLoader instance from settingsSPIDER_LOADER_CLASSr*)r\r!r rr from_settings frozencopy)r5cls_path loader_clss rD_get_spider_loaderz CrawlerRunner._get_spider_loadersH<< 56 * M:.NJ$<$._doneTsI MM ! !' * LL  #  ! !(D)I%I I !MrF)rr,rr,)raddryraddBoth)rBrrwrxrrks`` @rDrzCrawlerRunner._crawlOsR '" GMM4 *6 *   yyrFc~t|tr tdt|tr|S|j |S)a Return a :class:`~scrapy.crawler.Crawler` object. * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is. * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler is constructed for it. * If ``crawler_or_spidercls`` is a string, this function finds a spider with this name in a Scrapy project (using spider loader), then creates a Crawler instance for it. r)r0r r1r._create_crawler)rBrs rDrzCrawlerRunner.create_crawler\sG *F 3B  *G 4' '##$899rFct|tr|jj|}t ||j SrH)r0strrloadr.r5)rBr3s rDrzCrawlerRunner._create_crawlerrs3 i %**// :Iy$--00rFctt|jDcgc]}|jc}Scc}w)z Stops simultaneously all the crawling jobs taking place. Returns a deferred that is fired when they all have ended. )r listrr})rBcs rDr}zCrawlerRunner.stopws2 tDMM/BC/B!QVVX/BCDDCs;c#nK|jr%t|j|jr$yyw)z join() Returns a deferred that is fired when all managed :attr:`crawlers` have completed their executions. N)rr rIs rDjoinzCrawlerRunner.joins'llt||, ,lls055)r5rrr*rH)r5r)rtype[Spider] | str | CrawlerrwrrxrrDeferred[None])rr.rwrrxrrr)rrrr.)r3zstr | type[Spider]rr.rz Deferred[Any]r)rrr__doc__propertyrrrrEryrrrr}r rrrFrDrrs # 3H UU&!5:!5!5 !5  !5F  :$@: :,1 E--rFrcpeZdZdZ d d fd Zd dZd dZd dZ d ddZddZ dddZ xZ S)CrawlerProcessa A class to run multiple scrapy crawlers in a process simultaneously. This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support for starting a :mod:`~twisted.internet.reactor` and handling shutdown signals, like the keyboard interrupt command Ctrl-C. It also configures top-level logging. This utility should be a better fit than :class:`~scrapy.crawler.CrawlerRunner` if you aren't running another :mod:`~twisted.internet.reactor` within your application. The CrawlerProcess object must be instantiated with a :class:`~scrapy.settings.Settings` object. :param install_root_handler: whether to install root logging handler (default: True) This class shouldn't be needed (since Scrapy is responsible of using it accordingly) unless writing scripts that manually handle the crawling process. See :ref:`run-from-script` for an example. ct||t|j|t |jd|_yr)superrErr5r_initialized_reactor)rBr5install_root_handler __class__s rDrEzCrawlerProcess.__init__s5 "$--)=> &*/!rFcddlm}t|jt|}t j dd|i|j|jy)NrrXzDReceived %(signame)s, shutting down gracefully. Send again to force signame) rbrYr" _signal_killr#rdrecallFromThread_graceful_stop_reactorrBsignumr~rYrs rD_signal_shutdownzCrawlerProcess._signal_shutdownsJ,!$"3"34v& R    t::;rFcddlm}ttjt |}t jdd|i|j|jy)NrrXz4Received %(signame)s twice, forcing unclean shutdownr) rbrYr"signalSIG_IGNr#rdrer _stop_reactorrs rDrzCrawlerProcess._signal_killsH,!&..1v& BYPWDX  t112rFct|tr|jj|}|j }d|_t ||j |S)NT)rC)r0rrrrr.r5)rBr3rCs rDrzCrawlerProcess._create_crawlersK i %**// :I444 $(!y$--lKKrFc ddlm}|r8|j}|jry|j |j t |jd}t|||}|j|j}|j|jjd|jdd|j|r"|jd d t|j |j#| y) ao This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished :param bool install_signal_handlers: whether to install the OS signal handlers from Twisted and Scrapy (default: True) rrXN DNS_RESOLVERREACTOR_THREADPOOL_MAXSIZE) maxthreadsbeforeshutdownafterstartup)installSignalHandlers)rbrYrcalledrrr!r5r install_on_reactor getThreadPooladjustPoolsizegetintaddSystemEventTriggerr}r"rrun)rBstop_after_crawlinstall_signal_handlersrYrkresolver_classresolvertps rDrtzCrawlerProcess.starts" -  Axx IId(( )$T]]>%BC&ndGL##%  " " $ T]]%9%9:V%WX%%h DIIF "  ) )$=t?T?T   *A BrFc\|j}|j|j|SrH)r}rr)rBrks rDrz%CrawlerProcess._graceful_stop_reactors$ IIK $$$%rFcPddlm} |jy#t$rYywxYw)NrrX)rbrYr}rn)rBr~rYs rDrzCrawlerProcess._stop_reactors%,  LLN   s  %%)NT)r5rrr)rintr~rrr)r3ztype[Spider] | strrr.)TT)rrrrrrrrH)r~rrr) rrrrrErrrrtrr __classcell__)rs@rDrrso26:%)020#0 <3LNR&C $&CFJ&C &CP rFr)F __future__rrOrfrtypingrrrrtwisted.internet.deferrr r r zope.interface.verifyr scrapyr r scrapy.addonsrscrapy.core.enginerscrapy.extensionrscrapy.interfacesrscrapy.logformatterrscrapy.settingsrrrscrapy.signalmanagerrscrapy.statscollectorsrscrapy.utils.logrrrrrrscrapy.utils.miscr r!scrapy.utils.ossignalr"r#scrapy.utils.reactorr$r%r&r'collections.abcr(r)scrapy.spiderloaderr*scrapy.utils.requestr+ getLoggerrrdr,r.rrrrFrDrs" 44 ."&.-+,GG.1>I309   8 $ T]ORORd@-@-Fr]rrF