
    Qd%@              	          d Z ddlZddlmZ ddlmZ ddlmZ ddlm	Z	 ddl
mZmZ  ej        e          Z G d d	          Zed
k    rddlZddlZddlmZ  ej                    Ze                    ddddd           e                    ddddd           e                    ddddg d           e                    ddddg d            e                    d!d"d#d$%           e                    d&d'd(d)d*           e                    d+d,d-d.d/           e                    d0d1d2d3d4           e                    d5d6d7dd8           e                    d9d:d;d<d=d>?           e                                \  ZZej        se                                  e              ej        d@dA eej                   z  z
  B           g Z! e"ej                  5 Z#e#D ]Z$e$%                    ej&                  Z'e'd          e(e'dCdD                    e)e'dD         *                                %                    ej+                            cZ,Z-Z.ej/        ej0        k    s: e1ej/                  dk    r	e,ej/        v s e1ej0                  dk    r!e,ej0        vre!2                    e,e-e.f           	 ddd           n# 1 swxY w Y   ej3        r* ee!  e4eej                  ej3                            Z5n ee! e4eej                            Z5ej6        rn" e7  e4e5ej8                                          ej9                     dS dS )EaC	  
Implementations of inter-annotator agreement coefficients surveyed by Artstein
and Poesio (2007), Inter-Coder Agreement for Computational Linguistics.

An agreement coefficient calculates the amount that annotators agreed on label
assignments beyond what is expected by chance.

In defining the AnnotationTask class, we use naming conventions similar to the
paper's terminology.  There are three types of objects in an annotation task:

    the coders (variables "c" and "C")
    the items to be annotated (variables "i" and "I")
    the potential categories to be assigned (variables "k" and "K")

Additionally, it is often the case that we don't want to treat two different
labels as complete disagreement, and so the AnnotationTask constructor can also
take a distance metric as a final argument.  Distance metrics are simply
functions that take two arguments, and return a value between 0.0 and 1.0
indicating the distance between them.  If not supplied, the default is binary
comparison between the arguments.

The simplest way to initialize an AnnotationTask is with a list of triples,
each containing a coder's assignment for one object in the task:

    task = AnnotationTask(data=[('c1', '1', 'v1'),('c2', '1', 'v1'),...])

Note that the data list needs to contain the same number of triples for each
individual coder, containing category values for the same set of items.

Alpha (Krippendorff 1980)
Kappa (Cohen 1960)
S (Bennet, Albert and Goldstein 1954)
Pi (Scott 1955)


TODO: Describe handling of multiple coders and missing data

Expected results from the Artstein and Poesio survey paper:

    >>> from nltk.metrics.agreement import AnnotationTask
    >>> import os.path
    >>> t = AnnotationTask(data=[x.split() for x in open(os.path.join(os.path.dirname(__file__), "artstein_poesio_example.txt"))])
    >>> t.avg_Ao()
    0.88
    >>> round(t.pi(), 5)
    0.79953
    >>> round(t.S(), 2)
    0.82

    This would have returned a wrong value (0.0) in @785fb79 as coders are in
    the wrong order. Subsequently, all values for pi(), S(), and kappa() would
    have been wrong as they are computed with avg_Ao().
    >>> t2 = AnnotationTask(data=[('b','1','stat'),('a','1','stat')])
    >>> t2.avg_Ao()
    1.0

    The following, of course, also works.
    >>> t3 = AnnotationTask(data=[('a','1','othr'),('b','1','othr')])
    >>> t3.avg_Ao()
    1.0

    N)groupby)
itemgetter)
deprecated)binary_distance)ConditionalFreqDistFreqDistc                       e Zd ZdZdefdZd Zd ZddZd Z	d Z
d	 Z ed
          dd            ZddZd Zd Zd ZddZddZd Zd Zd Zd Zd Zd Zd Zd ZddZddZdS ) AnnotationTaska/  Represents an annotation task, i.e. people assign labels to items.

    Notation tries to match notation in Artstein and Poesio (2007).

    In general, coders and items can be represented as any hashable object.
    Integers, for example, are fine, though strings are more readable.
    Labels must support the distance functions applied to them, so e.g.
    a string-edit-distance makes no sense if your labels are integers,
    whereas interval distance needs numeric values.  A notable case of this
    is the MASI metric, which requires Python sets.
    Nc                     || _         t                      | _        t                      | _        t                      | _        g | _        ||                     |           dS dS )a  Initialize an annotation task.

        The data argument can be None (to create an empty annotation task) or a sequence of 3-tuples,
        each representing a coder's labeling of an item:
        ``(coder,item,label)``

        The distance argument is a function taking two arguments (labels) and producing a numerical distance.
        The distance from a label to itself should be zero:
        ``distance(l,l) = 0``
        N)distancesetIKCdata
load_array)selfr   r   s      6lib/python3.11/site-packages/nltk/metrics/agreement.py__init__zAnnotationTask.__init__`   sY     !	OOD!!!!!     c                 T    d                     t          d | j                            S )Nz
c                     | d         d| d                              dd          dd                    | d                   S )Ncoder	item_,labels)replacejoin)xs    r   <lambda>z(AnnotationTask.__str__.<locals>.<lambda>v   sF    W:::qy00d;;;;SXXak=R=R=RT r   )r    mapr   r   s    r   __str__zAnnotationTask.__str__s   s6    {{T T	 
 
 	
r   c                     |D ]r\  }}}| j                             |           | j                            |           | j                            |           | j                            |||d           sdS )zLoad an sequence of annotation results, appending to any data already loaded.

        The argument is a sequence of 3-tuples, each representing a coder's labeling of an item:
            (coder,item,label)
        )r   r   r   N)r   addr   r   r   append)r   arrayr   r   r   s        r   r   zAnnotationTask.load_array|   s     $) 	O 	OE4FJJuFJJvFJJtIuMMNNNN		O 	Or   c                    |p| j         }t          fd|D                       }|d         k    rt          fd|D                       }nt          fd|D                       }dt          |                     |d         |d                             z
  }t                              d|           t                              d|d         |d         d|z
             |S )	z,Agreement between two coders on a given itemc              3   L   K   | ]}|d          fv |d         k    |V  dS r   r   N ).0r!   cAcBis     r   	<genexpr>z%AnnotationTask.agr.<locals>.<genexpr>   s=      OOQwZB8%;%;&	Q!OOr   r   c              3   L   K   | ]}|d          k    |d         k    |V  dS r,   r-   )r.   r!   r0   r1   s     r   r2   z%AnnotationTask.agr.<locals>.<genexpr>   9      MMA7r)9)9ai1nnannnnMMr   c              3   L   K   | ]}|d          k    |d         k    |V  dS r,   r-   )r.   r!   r/   r1   s     r   r2   z%AnnotationTask.agr.<locals>.<genexpr>   r4   r         ?r   z.Observed agreement between %s and %s on %s: %fz"Distance between "%r" and "%r": %f)r   nextfloatr   logdebug)r   r/   r0   r1   r   k1k2rets    ```    r   agrzAnnotationTask.agr   s    ty OOOOOOTOOOOOg;"MMMMMMMMMMBBMMMMMMMMMMBE$--8blCCDDD		BBAsSSS		0"X,8cTWi	
 	
 	
 
r   c                 ^    t          t          fd| j        D                                 S )Nc              3   4   K   | ]}|d          k    dV  dS )r      Nr-   )r.   r!   ks     r   r2   z$AnnotationTask.Nk.<locals>.<genexpr>   s1      BBq81A1A1A1A1A1ABBr   r8   sumr   )r   rB   s    `r   NkzAnnotationTask.Nk   s0    SBBBBDIBBBBBCCCr   c                 b    t          t          fd| j        D                                 S )Nc              3   L   K   | ]}|d          k    |d         k    dV  dS )r   r   rA   Nr-   )r.   r!   r1   rB   s     r   r2   z%AnnotationTask.Nik.<locals>.<genexpr>   s>      UUq6aAhKSTDTDTDTDTDTDTUUr   rC   )r   r1   rB   s    ``r   NikzAnnotationTask.Nik   s4    SUUUUUDIUUUUUVVVr   c                 b    t          t          fd| j        D                                 S )Nc              3   L   K   | ]}|d          k    |d         k    dV  dS )r   r   rA   Nr-   )r.   r!   crB   s     r   r2   z%AnnotationTask.Nck.<locals>.<genexpr>   s>      VVq7qQx[TUEUEUEUEUEUEUVVr   rC   )r   rK   rB   s    ``r   NckzAnnotationTask.Nck   s4    SVVVVVDIVVVVVWWWr   zUse Nk, Nik or Nck insteadc           	          ||||                      |          }nS||||                     ||          }n6||||                     ||          }nt          d|d|d|d          t                              d||||           |S )z>Implements the "n-notation" used in Artstein and Poesio (2007)Nz*You must pass either i or c, not both! (k=z,i=z,c=)zCount on N[%s,%s,%s]: %d)rE   rH   rL   
ValueErrorr9   r:   )r   rB   r1   rK   r=   s        r   NzAnnotationTask.N   s     =QY19''!**CC]q}((1a..CC]q}((1a..CCSQSSQSSQSSS   			,aAs;;;
r   c                     |p| j         }t          t          |t          |                    t          |                    S )N)key)r   r   sortedr   )r   fieldr   s      r   _grouped_datazAnnotationTask._grouped_data   s;     tyvd
5(9(9:::Ju<M<MNNNr   c                                            dfd j        D                       }t           fd|D                       t           j                  z  }t
                              d|           |S )z3Observed agreement between two coders on all items.r   c              3   4   K   | ]}|d          fv |V  dS r   Nr-   r.   r!   r/   r0   s     r   r2   z$AnnotationTask.Ao.<locals>.<genexpr>   s5      DD1QwZB8-C-CQ-C-C-C-CDDr   c              3   N   K   | ]\  }}                     ||          V   d S N)r>   )r.   r   	item_datar/   r0   r   s      r   r2   z$AnnotationTask.Ao.<locals>.<genexpr>   s9      PPi$((2r433PPPPPPr   z(Observed agreement between %s and %s: %f)rU   r   rD   lenr   r9   r:   )r   r/   r0   r   r=   s   ```  r   AozAnnotationTask.Ao   s    !!DDDDD	DDD
 
 PPPPPP4PPPPPSVFT
 T
 
 			<b"cJJJ
r   c                     d}d}| j                                         }| j         D ]0}|                    |           |D ]}| |||          z  }|dz  }1||z  }|S )zP
        Calculates the average of function results for each coder pair
        r   rA   )r   copyremove)r   functiontotalnsr/   r0   r=   s           r   _pairwise_averagez AnnotationTask._pairwise_average   s     FKKMM& 	 	BHHRLLL  "b)))Q ai
r   c                 p    |                      | j                  }t                              d|           |S )z7Average observed agreement across all coders and items.zAverage observed agreement: %f)rf   r^   r9   r:   )r   r=   s     r   avg_AozAnnotationTask.avg_Ao   s1    $$TW--		2C888
r   r6   c                 Z   d}fd| j         D             }|                     d|          D ]D\  }}||                     t          |          d         t          |          d                   z  }E|t	          | j                  |z  z  }t                              d|           |S )z=The observed disagreement for the weighted kappa coefficient.        c              3   4   K   | ]}|d          fv |V  dS rX   r-   rY   s     r   r2   z0AnnotationTask.Do_Kw_pairwise.<locals>.<genexpr>   s5      ??a'
r2h(>(>(>(>(>(>??r   r   r   z+Observed disagreement between %s and %s: %f)r   rU   r   r7   r]   r   r9   r:   )	r   r/   r0   max_distancerc   r   r1   itemdatar=   s	    ``      r   Do_Kw_pairwisezAnnotationTask.Do_Kw_pairwise   s    ?????49???--fd;; 	W 	WKAxT]]4>>(#;T(^^H=UVVVEEs46{{\12		?RMMM
r   c                 r                            fd          }t                              d|           |S )zAveraged over all labelersc                 2                         | |          S r[   )rn   r/   r0   rl   r   s     r   r"   z&AnnotationTask.Do_Kw.<locals>.<lambda>   s    4..r2|DD r   zObserved disagreement: %f)rf   r9   r:   )r   rl   r=   s   `` r   Do_KwzAnnotationTask.Do_Kw   sD    $$DDDDD
 
 			-s333
r   c                 n    dt          | j                  z  }|                                 |z
  d|z
  z  }|S )z"Bennett, Albert and Goldstein 1954r6   )r]   r   rh   )r   Aer=   s      r   SzAnnotationTask.S   s4    3tv;;{{}}r!cBh/
r   c                    d}t          d | j        D                       }|                                D ]\  }}||dz  z  }|t          | j                  t          | j                  z  dz  z  }|                                 |z
  d|z
  z  S )z_Scott 1955; here, multi-pi.
        Equivalent to K from Siegel and Castellan (1988).

        rj   c              3   &   K   | ]}|d          V  dS r   Nr-   r.   r!   s     r   r2   z$AnnotationTask.pi.<locals>.<genexpr>   s&      >>qq{>>>>>>r      rA   )r   r   itemsr]   r   r   rh   )r   rc   label_freqsrB   frt   s         r   pizAnnotationTask.pi   s    
 >>DI>>>>>%%'' 	 	DAqQTMEEs46{{S[[0Q67"q2v..r   c                    d}t          t          | j                            }t          d | j        D                       }|                                D ](}|||         |         |z  ||         |         |z  z  z  })|S )Nrj   c              3   6   K   | ]}|d          |d         fV  dS )r   r   Nr-   ry   s     r   r2   z*AnnotationTask.Ae_kappa.<locals>.<genexpr>  s/      )W)W1X;'
*C)W)W)W)W)W)Wr   )r8   r]   r   r   r   
conditions)r   r/   r0   rt   nitemsr|   rB   s          r   Ae_kappazAnnotationTask.Ae_kappa   s    s46{{##))W)WTY)W)W)WWW'')) 	P 	PA;q>"%.;q>"3E3NOOBB	r   c                     |                      ||          }|                     ||          |z
  d|z
  z  }t                              d|||           |S ) r6   z(Expected agreement between %s and %s: %f)r   r^   r9   r:   )r   r/   r0   rt   r=   s        r   kappa_pairwisezAnnotationTask.kappa_pairwise  sR    ]]2r""wwr2#b1		<b"bIII
r   c                 6    |                      | j                  S )zNCohen 1960
        Averages naively over kappas for each coder pair.

        )rf   r   r$   s    r   kappazAnnotationTask.kappa  s    
 %%d&9:::r   c                 p    |                      | j                  }|                                 |z
  d|z
  z  S )zlDavies and Fleiss 1982
        Averages over observed and expected agreements for each coder pair.

        r6   )rf   r   rh   )r   rt   s     r   multi_kappazAnnotationTask.multi_kappa  s5    
 ##DM22"sRx00r   c                 $   t          |                                          }d}|                                D ]K\  }}|                                D ]1\  }}|t          ||z            |                     ||          z  z  }2Ld|z  ||dz
  z  z  S )Nrj   r6   rA   )rD   valuesr{   r8   r   )r   r|   total_labelspairsjnjlnls           r   DisagreementzAnnotationTask.Disagreement  s    ;--//00 &&(( 	> 	>EAr$**,, > >2rBw$--1*=*===>U{llQ.>?@@r   c                    t          | j                  dk    rt          d          t          | j                  dk    rt                              d           dS t          | j                  dk    r't          | j                  dk    rt          d          d}d}t          g           }d}|                     d          D ]f\  }}t          d |D                       }t          |
                                          }|d	k     rF||z  }||                     |          |z  z  }g|t          |
                                          z  }	|                     |          }
d
|	|
z  z
  }|S )zKrippendorff 1980r   z(Cannot calculate alpha, no data present!rA   z-Only one annotation value, alpha returning 1.z8Cannot calculate alpha, only one coder and item present!rj   r   c              3   &   K   | ]}|d          V  dS rx   r-   ry   s     r   r2   z'AnnotationTask.alpha.<locals>.<genexpr>6  s&      "A"A11X;"A"A"A"A"A"Ar   rz   r6   )r]   r   rO   r9   r:   r   r   r   rU   rD   r   r   )r   total_disagreementtotal_ratingsall_valid_labels_freqtotal_dor1   rm   r|   labels_countdodek_alphas               r   alphazAnnotationTask.alpha%  sv    tv;;!GHHHtv;;!IIEFFF1tv;;!DFq 0 0WXXX  (--f55 	F 	FKAx""A"A"A"A"AAAK{113344La![0!))+66EEHH188::;;;455R-r   c                    d}t          fd| j        D                       }| j        D ]C}| j        D ]9}||         |         |         |         z  |                     ||          z  z  }:D||t	          t          | j                  d          z  z  }t                              d|           | 	                              }	d|	|z  z
  }
|
S )
Cohen 1968rj   c              3   P   K   | ] }|d          fv |d          |d         fV  !dS )r   r   Nr-   rY   s     r   r2   z9AnnotationTask.weighted_kappa_pairwise.<locals>.<genexpr>H  sK       *
 *
*+AgJ2r(<R<RQwZ8%<R<R<R<R*
 *
r   rz   z+Expected disagreement between %s and %s: %fr6   )
r   r   r   r   powr]   r   r9   r:   rn   )r   r/   r0   rl   rc   r|   r   r   DeDor=   s    ``        r   weighted_kappa_pairwisez&AnnotationTask.weighted_kappa_pairwiseE  s   ) *
 *
 *
 *
 *
/3y*
 *
 *
 
 
  	W 	WAV W WR+k"oa.@@4==QRTUCVCVVVWlSTVa%8%889		?RLLL  R((R"Wo
r   c                 8                            fd          S )r   c                 2                         | |          S r[   )r   rq   s     r   r"   z/AnnotationTask.weighted_kappa.<locals>.<lambda>W  s    477BMM r   )rf   )r   rl   s   ``r   weighted_kappazAnnotationTask.weighted_kappaT  s+    %%MMMMM
 
 	
r   r[   )NNN)r6   )__name__
__module____qualname____doc__r   r   r%   r   r>   rE   rH   rL   r   rP   rU   r^   rf   rh   rn   rr   ru   r~   r   r   r   r   r   r   r   r   r-   r   r   r
   r
   S   s       
 
 !? " " " "&
 
 

O 
O 
O   &D D DW W WX X X Z,--   .-O O O O	 	 	    
 
 
 
     
/ 
/ 
/    ; ; ;1 1 1A A A  @   
 
 
 
 
 
r   r
   __main__)r   z-dz
--distancer   r   zdistance metric to use)destdefaulthelpz-az--agreement	agreementr   z"agreement coefficient to calculatez-ez	--excludeexcluder(   z8coder names to exclude (may be specified multiple times))r   actionr   r   z-iz	--includeincludez.coder names to include, same format as excludez-fz--filefilezPfile to read labelings from, each line with three columns: 'labeler item labels')r   r   z-vz	--verboseverbose0z+how much debugging to print on stderr (0-4)z-cz--columnsep	columnsepr   zIchar/string that separates the three columns in the file, defaults to tabz-lz
--labelseplabelsepr   z[char/string that separates labels (if labelers can assign more than one), defaults to commaz-pz
--presencepresencez=convert each labeling into 1 or 0, based on presence of LABELz-Tz
--thoroughthoroughF
store_truez6calculate agreement for every subset of the annotators)r   r   r   r   2   
   )levelrA   ):r   logging	itertoolsr   operatorr   nltk.internalsr   nltk.metrics.distancer   nltk.probabilityr   r   	getLoggerr   r9   r
   optparserenltk.metricsr   OptionParserparser
add_option
parse_argsoptions	remainderr   
print_helpexitbasicConfigintr   r   openinfiler   splitr   toksstr	frozensetstripr   r   object_r   r   r   r]   r(   r   getattrtaskr   printr   shutdownr-   r   r   <module>r      sL  = =~              % % % % % % 1 1 1 1 1 1 : : : : : : : :g!!E
 E
 E
 E
 E
 E
 E
 E
P zOOOIII%%%%%% #X"$$F
!%     1     G     =     _	     :     X     j     L     E     ",,..Wi< Gb2GO(<(<#<<==== D	gl		 6v 	6 	6A777,--DQD2J	$r(..**001ABBCC #E7F GO33C((1,,'/1I1IC((1,,go1M1MUGV4555	66 6 6 6 6 6 6 6 6 6 6 6 6 6 6  I~5''(G$455g6FGG
 
 ~dGGHg6F$G$GHH 2.ggdG-..00111Gm s   !CKKK