+gdXmldZddlZddlZddlZddlmZmZmZmZm Z m Z ddl Z ddl ZddlmZddlmZddlmZddlmZdd lmZdd lmZdd lmZdd lmZmZdd l m!Z!ddl"m#Z#ddl$m%Z%m&Z&m'Z'ddl(m)Z)ddl*m+Z+m,Z,e)e-Z.Gdde%Z/dZ0GddZ1Gdde1Z2dS)z Metrics base class.N)AnyDictListOptionalTupleUnion)config)Dataset) ArrowReader) ArrowWriter)DownloadConfig)DownloadManager)Features) DatasetInfo MetricInfo)camelcase_to_snakecase) deprecated) BaseFileLockFileLockTimeout) get_logger)copyfunc temp_seedc.eZdZdZfdZdZdZxZS) FileFreeLockz-Thread lock until a file **cannot** be lockedcjt||_tj|g|Ri|dSN)rfilelocksuper__init__)self lock_fileargskwargs __class__s /lib/python3.11/site-packages/datasets/metric.pyr!zFileFreeLock.__init__.s? ++ 4T444V44444c |jdd|jd|_dS#t$r|jj|_YdSwxYw)Ng{Gz?g{Gz?)timeoutpoll_intervall)racquirerelease _lock_file_fdrr#r"s r'_acquirezFileFreeLock._acquire2sy & M ! !$t ! D D D M ! ! # # #!%D     9 9 9!%!8D     9sAAAcd|_dSr)r.r/s r'_releasezFileFreeLock._release=s!r()__name__ __module__ __qualname____doc__r!r0r2 __classcell__)r&s@r'rr+s\7755555 & & &"""""""r(rct|tkrt|dkr|Sd}d||ddd||dddS)Nc@dd|DS)Nz, c34K|]}t|VdSr)repr).0xs r' z?summarize_if_long_list..format_chunk..Hs(00Qa000000r()join)chunks r' format_chunkz,summarize_if_long_list..format_chunkGs#yy00%000000r([z, ..., ])typelistlen)objrBs r'summarize_if_long_listrKCsw 99  CA x111 G||CG$$ F F\\#bcc(-C-C F F FFr(ceZdZdZdefdZedZedefdZ ede efdZ edefdZ edefd Z edefd Zedefd Zede efd Zedefd Zede eefdZede eefdZedefdZede efdZdS)MetricInfoMixinuThis base class exposes some attributes of MetricInfo at the base level of the Metric for easy access. Use the new library 🤗 Evaluate instead: https://huggingface.co/docs/evaluate infoc||_dSr _metric_info)r"rNs r'r!zMetricInfoMixin.__init__Ys r(c|jS)zN:class:`datasets.MetricInfo` object containing all the metadata in the metric.rPr/s r'rNzMetricInfoMixin.info\s   r(returnc|jjSr)rQ metric_namer/s r'namezMetricInfoMixin.namea ,,r(c|jjSr)rQ experiment_idr/s r'rYzMetricInfoMixin.experiment_ide ..r(c|jjSr)rQ descriptionr/s r'r\zMetricInfoMixin.descriptionirWr(c|jjSr)rQcitationr/s r'r^zMetricInfoMixin.citationm ))r(c|jjSr)rQfeaturesr/s r'razMetricInfoMixin.featuresqr_r(c|jjSr)rQinputs_descriptionr/s r'rcz"MetricInfoMixin.inputs_descriptionus 33r(c|jjSr)rQhomepager/s r'rezMetricInfoMixin.homepageyr_r(c|jjSr)rQlicenser/s r'rgzMetricInfoMixin.license}s ((r(c|jjSr)rQ codebase_urlsr/s r'rizMetricInfoMixin.codebase_urlsrZr(c|jjSr)rQreference_urlsr/s r'rkzMetricInfoMixin.reference_urlss //r(c|jjSr)rQ streamabler/s r'rmzMetricInfoMixin.streamables ++r(c|jjSr)rQformatr/s r'rozMetricInfoMixin.formats ''r(N)r3r4r5r6rr!propertyrNstrrVrrYr\r^rrarcrergrrirkboolrmror(r'rMrMMsF  !Z!!!!!!X!-c---X-/x}///X/-S---X-*#***X**(***X*4C444X4*(3-***X*))))X)/xS 2///X/0c 3000X0,D,,,X,( (((X(((r(rMceZdZdZed d)d eed ed eed ed edeedeedede ee ffdZ dZ dZ dZd*deeeffdZdeeeeeffdZdZdZdZddddeefdZddddZdddd Zd*d!Zdefd"Z d+d#eed$eefd%Z d&Z!dddde"ee#ffd'Z$d(Z%dS),MetricuA Metric is the base class and common API for all metrics. Use the new library 🤗 Evaluate instead: https://huggingface.co/docs/evaluate Args: config_name (``str``): This is used to define a hash specific to a metrics computation script and prevents the metric's data to be overridden when the metric loading script is modified. keep_in_memory (:obj:`bool`): keep all predictions and references in memory. Not possible in distributed settings. cache_dir (``str``): Path to a directory in which temporary prediction/references data will be stored. The data directory should be located on a shared file-system in distributed setups. num_process (``int``): specify the total number of nodes in a distributed settings. This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). process_id (``int``): specify the id of the current process in a distributed setup (between 0 and num_process-1) This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). seed (:obj:`int`, optional): If specified, this will temporarily set numpy's random seed when :func:`datasets.Metric.compute` is run. experiment_id (``str``): A specific experiment id. This is used if several distributed evaluations share the same file system. This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). max_concurrent_cache_files (``int``): Max number of concurrent metrics cache files (default 10000). timeout (``Union[int, float]``): Timeout in second for distributed setting synchronization. uOUse the new library 🤗 Evaluate instead: https://huggingface.co/docs/evaluateNFr r'd config_namekeep_in_memory cache_dir num_process process_idseedrYmax_concurrent_cache_filesr*c  v|pd|_|} t|jj| _|j| _|pd| _t|| t|tr|dkrtdt|tr||krtd|r|dkrtd||_ ||_ ||_||_t j|p t&j|_||_|>t0j^} }} } | dkr|| n|d|_n||_| |_t;jt?|j ||_ t;jt?|j!||_!t;jt?|j"||_"|j j#xj$|j%j&z c_$|j!j#xj$|j%j&z c_$|j"j#xj$|j%j&z c_$d|_'d|_(d|_)d|_*d|_+d|_,d|_-d|_.d|_/dS) Ndefaultdefault_experimentrz.'process_id' should be a number greater than 0z8'num_process' should be a number greater than process_idr zPUsing 'keep_in_memory' is not possible in distributed setting (num_process > 1).ip)0rx_inforr&r3rUrYrMr! isinstanceint ValueErrorr{r|r~ryospath expanduserr HF_METRICS_CACHE_data_dir_root_build_data_dirdata_dirnprandom get_stater}r*types MethodTypercompute add_batchadd__func__r6rNrc buf_writerwriterwriter_batch_sizedatacache_file_namerrendez_vous_lock file_paths filelocks)r"rxryrzr{r|r}rYr~r*r%rN_poss r'r!zMetric.__init__s|'3)zz||1$.2IJJ+*B.B  t,,,*c** Oj1nnMNN N+s++ Y{j/H/HWXX X  qkQ..opp p&$*D', g001Uf>UVV,,.. <!y2244 AtS1*-))T#YYaDII!DI*1 '(>(>EE )(4>*B*BDII#HTX$6$6== %%)EE%% ''49+GG'' !!TY%AA!! !%  $ $r(c<|jdnt|jS)zReturn the number of examples (predictions or predictions/references pair) currently stored in the metric's cache. Nr)rrIr/s r'__len__zMetric.__len__sK'qqS-=-==r(c Xd|jd|jd|jdt|d S)NzMetric(name: "z ", features: z , usage: """z""", stored examples: ))rVrarcrIr/s r'__repr__zMetric.__repr__sP -TY - -T] - -0 - - #D  - - - r(c|j}tj||j|j}tj|d|S)aPath of this metric in cache_dir: Will be: self._data_dir_root/self.name/self.config_name/self.hash (if not none)/ If any of these element is missing or if ``with_version=False`` the corresponding subfolders are dropped. T)exist_ok)rrrr@rVrxmakedirs)r"builder_data_dirs r'rzMetric._build_data_dirsG  .7<<(8$)TEUVV $t4444r(rSc tj|j|jd|jd|jd}d}t|jD]}t|dz} | |n#t$r|jdkrtd|d|jd d||jdz krtd |jd dttj}tj|j|jd|d|jd|jd}YwxYw||fS) zTCreate a new cache file. If the default cache file is used, we generated a new hash.-.arrowN.lockr*r z^Error in _create_cache_file: another metric instance is already using the local cache file at .. Please specify an experiment_id (currently: :) to avoid collision between distributed metric instances.zCannot acquire lock, too many metric instance are operating concurrently on this file system.You should set a larger value of max_concurrent_cache_files when creating the metric (current value is z).)rrr@rrYr{r|ranger~rr,rrrquuiduuid4)r"r* file_pathri file_uuids r'_create_cache_filezMetric._create_cache_file sGLL43E0r0rHX0r0r[_[j0r0r0rss t677  A G 344H    111,+   #q(($AzCAAGKGYAAA  7!;;;$Q-1-LQQQ    -- GLLMd&8#q#q9#q#qtGW#q#qZ^Zi#q#q#q # .(""s)BB8D<;D<c jdkrjtdjg}n fdtjD}g}t |D]\}}|dkr|j&t|dz} |j ||j#t$rtd|d |d dwxYw||fS) zGet a lock on all the cache files in a distributed setup. We wait for timeout second to let all the distributed node finish their tasks (default is 100 seconds). r Nz|Metric cache file doesn't exist. Please make sure that you call `add` or `add_batch` at least once before calling `compute`.c g|]:}tjjjdjd|d;S)rrrrr@rrYr{r=r|r"s r' z/Metric._get_all_cache_files..6s[ T]t/A,i,iDDT,i,iWa,i,i,ijjr(rrrz#Cannot acquire lock on cached file z for process .) r{rrr enumerateappendrrr,r*r)r"rrr|rrs` r'_get_all_cache_fileszMetric._get_all_cache_files*sY  q #+ >./JJ"'(8"9"9J  %.z%:%: / / !J Q  ////#I$788/$$T\$::: $$X....    $ciccV`ccc  9$$s C"C)c $fdtjD}|D]k}t|} |j|B#t $rtd|djddwxYwdS)Nc g|]:}tjjjdjd|d;S)rz .arrow.lockrrs r'rz5Metric._check_all_processes_locks..Ns[$ $ $  GLL4+=(j(j@P(j(jS](j(j(j k k$ $ $ r(rExpected to find locked file  from process  but it doesn't exist.) rr{rr,r*r-rrr|)r"expected_lock_file_namesexpected_lock_file_name nofilelocks` r'_check_all_processes_locksz!Metric._check_all_processes_locksMs$ $ $ $ #D$455$ $ $  (@ % % #%&=>>J %""4<"888 ""$$$$     C4KCC[_[jCCC  % %s A&&'B ctj|j|jd|jd}t |} ||j| n*#t$rtd|d|j ddwxYwtj|j|jd|jd}t|} ||j| dS#t$rtd|d|j d dwxYw) Nrz -0.arrow.lockrrrr -rdv.lockzCouldn't acquire lock on r)rrr@rrYr{rr,r*r-rrr|r)r"rrlock_file_namers r'_check_rendez_vouszMetric._check_rendez_vous]s"$',,t}AS>u>uVZVf>u>u>u"v"v!"9::  !   t|  4 4 4       ~0G~~W[Wf~~~   dm8J5h5hTM]5h5h5hii#N33 '  $ $T\ $ : : :  $ $ & & & & & u u uiiiW[Wfiiijjpt t usA66'B%D'D=c|j|jd|_|j$|jdkr|j|jr[t |jt|j }tj |j |_dS|jdkr|\}} t dt|j }tdi|d|D|_n#t"$rt%ddwxYw||_||_dSdS) zClose all the writing process and load/gather the data from all the nodes if main node or all_process is True. Nrra)rrNcg|]}d|iS)filenamers)r=fs r'rz$Metric._finalize..s8]8]8]Q*a8]8]8]r(zError in finalize: another metric instance is already using the local cache file. Please specify an experiment_id to avoid collision between distributed metric instances.rs)rfinalizerr|r-ryr rrrar from_bufferrgetvaluerr read_filesFileNotFoundErrorrrr)r"readerrrs r' _finalizezMetric._finalizeqso ; " K " " " = $1)<)< M ! ! # # #   ' dm+t}:]:]:]^^^F+DO,D,D,F,FGGDIII _ ! !$($=$=$?$? !J  $"; 3V3V3VWWW#__f&7&78]8]R\8]8]8]&^&^__ $    o )DO&DNNN!" !s AD++E predictions referencesc 8 ||d |+|) fdjD} |n< fdjD}|r'td|dtj fdjD}fdD}t d |Dr jdi|d_d_ j d krj j j fd jD}tj5jdi||}dddn #1swxYwYjd_` d_ nt'tt)jjD]\\} } t. d | ` d_ `d_t3j| | ]|SdS)aiCompute the metrics. Usage of positional arguments is not allowed to prevent mistakes. Args: predictions (list/array/tensor, optional): Predictions. references (list/array/tensor, optional): References. **kwargs (optional): Keyword arguments that will be forwarded to the metrics :meth:`_compute` method (see details in the docstring). Return: dict or None - Dictionary with the metrics if this metric is run on the main process (``process_id == 0``). - None if the metric is not run on the main process (``process_id != 0``). Example: ```py >>> from datasets import load_metric >>> metric = load_metric("accuracy") >>> accuracy = metric.compute(predictions=model_prediction, references=labels) ``` rNci|] }|v|d Srrsr=k all_kwargss r' z"Metric.compute..s%TTT!@S@Sa@S@S@Sr(cg|]}|v| Srsrsrs r'rz"Metric.compute..s#NNNA!::M:Ma:M:M:Mr(zMetric inputs are missing: . All required inputs are c"i|] }|| Srsrs)r= input_namers r'rz"Metric.compute..s UUU*j4UUUr(c4i|]}|jv ||Srsr)r=rr%r"s r'rz"Metric.compute..s+QQQ1!4=:P:P!VAY:P:P:Pr(c3K|]}|duV dSrrs)r=vs r'r?z!Metric.compute..s&66q}666666r(r)rGc,i|]}|j|Srs)rr=rr"s r'rz"Metric.compute..s"XXXJj$)J"7XXXr(z Removing rs)raupdaterrHanyvaluesrrrrr|r set_formatrNrorr}_computerreversedziprrloggerrrremover-) r"rrr%missing_kwargsmissing_inputsinputscompute_kwargsoutputrrrs ` ` @r'rzMetric.computes2&1 UUfU  :#5TTTTt}TTTN   n - - - -NNNNNNNN  q.qq\`aean\o\oqqVUUUt}UUUQQQQQQQQ 66fmmoo666 6 6 % DN $ $V $ $ $ # ?a   I di&6 7 7 7XXXX$-XXXF49%% C C&BBB>BB C C C C C C C C C C C C C C C*"&I  ,4DT^T_9]9]4^4^+_+_'''HiKK 7I 7 7888 $DI "&DKIi((($$&&&&M4sE$$E(+E(c  fd|D}|r'td|dtj||d|fdjDjjj jdS#tj $rtfdDrgtt  fdDd }d d t d |d t|d }n{tjddgkr7djd}dfdjD}||z }n*djdt!|dt!|}t|dwxYw)aAdd a batch of predictions and references for the metric's stack. Args: predictions (list/array/tensor, optional): Predictions. references (list/array/tensor, optional): References. Example: ```py >>> from datasets import load_metric >>> metric = load_metric("accuracy") >>> metric.add_batch(predictions=model_prediction, references=labels) ``` c&g|] }|jv |Srsrrs r'rz$Metric.add_batch..%]]]ZZt}=\=\j=\=\=\r(Bad inputs for metric: rrc"i|] }|| Srsrs)r= intput_namebatchs r'rz$Metric.add_batch..s RRR[eK0RRRr(Nc 3K|]V}t|tttkVWdSr)rInextiterr)r=crs r'r?z#Metric.add_batch..sRSS3uQx==CT%,,..-A-A(B(B$C$CCSSSSSSr(clg|]0}t|tk.|1Srs)rI)r=rrcol0s r'rz$Metric.add_batch..s9QQQs58}}E$K@P@P/P/P1/P/P/Pr(rzMismatch in the number of z (z) and rrr@Metric inputs don't match the expected format. Expected format: , c3NK|]}d|dt|V dSzInput z: NrK)r=rrs r'r?z#Metric.add_batch..sS..[eVZVV+A% BS+T+TVV......r(zPPredictions and/or references don't match the expected format. Expected format: z, Input predictions: z, Input references: )rrHrarN encode_batchr _init_writer write_batchpa ArrowInvalidrrrrIsortedr@rK) r"rrr% bad_inputsbad_col error_msgerror_msg_inputsrrs ` @@r'rzMetric.add_batchsr^]]]6]]]  trzrr]abfbo]p]prrss s +:PPPRRRRDMRRR "//66 ;        2 K # #E * * * * * 2 2 2SSSSUSSSSS DKK((QQQQQeQQQRSTrrrU4[9I9IrrQXrr\_`efm`n\o\orrr  &&<*GGGvdhdqvvv #(::....imiv...$$ -- N(, NN*@*M*MNN*@ )K)KNN Y''T 1) 2s B,,DG ) prediction referencec fd|D}|r'td|dtj||d|fdjDjjj jdS#tj $rGdjd}d fd jD}||z }t|dwxYw) aAdd one prediction and reference for the metric's stack. Args: prediction (list/array/tensor, optional): Predictions. reference (list/array/tensor, optional): References. Example: ```py >>> from datasets import load_metric >>> metric = load_metric("accuracy") >>> metric.add(predictions=model_predictions, references=labels) ``` c&g|] }|jv |Srsrrs r'rzMetric.add..rr(rrrc"i|] }|| Srsrs)r=rexamples r'rzMetric.add..s VVV; 4VVVr(Nrrc3NK|]}d|dt|V dSr r )r=rrs r'r?zMetric.add..$sS**YcTTT'=gj>Q'R'RTT******r() rrHrarNencode_examplerr writerrr@)r"rrr%rrrrs` @r'rz Metric.add s`^]]]6]]]  trzrr]abfbo]p]prrss s",IPPPVVVV VVV)$33G<< ;        2 K  g & & & & & 2 2 2r`d`mrrrI$zz****gkgt***    ) )IY''T 1  2s B++ADch|jdkr|jdkrtj|j|jd|jd}t||_ |j |n*#t$rtd|d|jddwxYw|j rDtj|_t!|jj|j|j |_ned|_|j|j%|\}}||_||_t!|jj|j|j |_|jdkrP|jdkr/||jdS|dSdS) Nr rrrrzXError in _init_writer: another metric instance is already using the local cache file at rr)rastreamr)rarr)r{r|rrr@rrYrrr, TimeoutErrorrryrBufferOutputStreamrr rNrarrrrrrr-r)r"r*rrrs r'r zMetric._init_writer*s  a  !##GLL4;M8k8kPTP`8k8k8kll (0(;(;% )11'1BBBB#   $As|AAGKGYAAA       355DO%+DOW[WmDKK#DO#+t}/D,0,C,C,E,E)'6$ ( %+$2FZ^ZpDK  a  !##//111%--/////''))))) s !A=='B$ct)aConstruct the MetricInfo object. See `MetricInfo` for details. Warning: This function is only called once and the result is cached for all following .info() calls. Returns: info: (MetricInfo) The metrics information NotImplementedErrorr/s r'rz Metric._infoQs "!r(download_config dl_managerc|]|?t}tj|jd|_d|_t|j||j}| |dS)aDownloads and prepares dataset for reading. Args: download_config (:class:`DownloadConfig`, optional): Specific download configuration parameters. dl_manager (:class:`DownloadManager`, optional): Specific download manager to use. N downloadsF) dataset_namer&r) rrrr@rrzforce_downloadrrV_download_and_prepare)r"r&r's r'download_and_preparezMetric.download_and_prepare\sz  &"0"2"2,.GLL ,T,T)16.(!YRVR_J "":.....r(cdS)aYDownloads and prepares resources for the metric. This is the internal implementation to overwrite called when user calls `download_and_prepare`. It should download all required resources for the metric. Args: dl_manager (:class:`DownloadManager`): `DownloadManager` used to download and cache data. Nrs)r"r's r'r,zMetric._download_and_preparess tr(c t)zEThis method defines the common API for all the metrics in the libraryr$)r"rrr%s r'rzMetric._compute~s!!r(ct|dr |j|jt|dr |j|jt|dr|`t|dr|`dSdS)Nrrrr)hasattrrr-rrrr/s r'__del__zMetric.__del__s 4 $ $ $)B M ! ! # # # 4+ , , ,1F1R  ! ) ) + + + 4 " "  4     r() NFNr rNNrvrw)r )NN)&r3r4r5r6rrrqrrrrfloatr!rrrrrrrrrrrdictrrrr rrrrr-r,rrrr2rsr(r'rurus2Zabb&*$#'"'+*/%(BBc]BBC= B  B  BsmB }B%(BsEz"BBBcbBH>>>        ##uS(]/C####B!%eDItH~,E&F!%!%!%!%F%%% '''( ' ' 'D&*dEEERVEEEEN(,-2-2-2-2-2^!%22222B%*%*%*%*N "z " " " "5904//!.1/_-////.   '+t"""$sTWx.""""r(ru)3r6rrrtypingrrrrrrnumpyrpyarrowrrr arrow_datasetr arrow_readerr arrow_writerr download.download_configrdownload.download_managerrrarrNrrnamingrutils.deprecation_utilsrutils.filelockrrr utils.loggingrutils.py_utilsrrr3rrrKrMrursr(r'rBs.  ::::::::::::::::""""""%%%%%%%%%%%%444444666666))))))))******//////;;;;;;;;;;%%%%%%//////// H  """""<"""0GGGB(B(B(B(B(B(B(B(Jxxxxx_xxxxxr(