a
    EPÎj“+  ã                   @   sö  d Z ddlmZ ddlmZmZmZmZ ddlZddlm	Z	m
Z
 ddlmZ eejƒdkrhddlmZ nddlmZ dd	lmZ ddlmZ dd
lmZ ddlmZmZ g d¢ZedƒZedƒZeeef ZG dd„ de ƒZ!dd„ Z"e"ej#ƒG dd„ de!ƒƒZ$G dd„ de!ƒZ%e"ej&ƒG dd„ de!ƒƒZ'e"ej(ƒG dd„ de!ƒƒZ)e"ej*ƒG dd„ de!ƒƒZ+e"ej,ƒG dd„ de!ƒƒZ-e"ej.ƒG dd „ d e!ƒƒZ/e"ej0ƒG d!d"„ d"e!ƒƒZ1ed#d$d%gƒZ2ee2d&d'„ d(d'„ ƒ d)d*„ Z3G d+d,„ d,e!ƒZ4e!d-œd.d/„Z5dS )0zá
Optimizer classes defined here are light wrappers over the corresponding optimizers
sourced from :mod:`jax.example_libraries.optimizers` with an interface that is better
suited for working with NumPyro inference algorithms.
é    )Ú
namedtuple)ÚAnyÚCallableÚTupleÚTypeVarN)ÚlaxÚvalue_and_grad)Ú_versiontuple)r   é   é   )Ú
optimizers)Úravel_pytree)Úminimize)Úregister_pytree_nodeÚtree_map)	ÚAdamÚAdagradÚClippedAdamÚMinimizeÚMomentumÚRMSPropÚRMSPropMomentumÚSGDÚSM3Ú_ParamsÚ	_OptStatec                   @   s‚   e Zd Zeddœdd„Zeedœdd„Zeeedœd	d
„Zee	ge
f edœdd„Zee	ge
f edœdd„Zeedœdd„ZdS )Ú_NumPyroOptimN)Úoptim_fnÚreturnc                 O   s   ||i |¤Ž\| _ | _| _d S ©N)Úinit_fnÚ	update_fnÚget_params_fn)Úselfr   ÚargsÚkwargs© r&   ú]/mounts/lovelace/software/anaconda3/envs/metadmg/lib/python3.9/site-packages/numpyro/optim.pyÚ__init__.   s    z_NumPyroOptim.__init__)Úparamsr   c                 C   s   |   |¡}t d¡|fS )z¶
        Initialize the optimizer with parameters designated to be optimized.

        :param params: a collection of numpy arrays.
        :return: initial optimizer state.
        r   )r    ÚjnpÚarray)r#   r)   Ú	opt_stater&   r&   r'   Úinit1   s    
z_NumPyroOptim.init)ÚgÚstater   c                 C   s"   |\}}|   |||¡}|d |fS )zÒ
        Gradient update for the optimizer.

        :param g: gradient information for parameters.
        :param state: current optimizer state.
        :return: new optimizer state after the update.
        é   )r!   ©r#   r.   r/   Úir,   r&   r&   r'   Úupdate;   s    z_NumPyroOptim.update©Úfnr/   c                 C   s6   |   |¡}t|dd�|ƒ\\}}}||f|  ||¡fS )aÑ  
        Performs an optimization step for the objective function `fn`.
        For most optimizers, the update is performed based on the gradient
        of the objective function w.r.t. the current state. However, for
        some optimizers such as :class:`Minimize`, the update is performed
        by reevaluating the function multiple times to get optimal
        parameters.

        :param fn: an objective function returning a pair where the first item
            is a scalar loss function to be differentiated and the second item
            is an auxiliary output.
        :param state: current optimizer state.
        :return: a pair of the output of objective function and the new optimizer state.
        T©Zhas_aux)Ú
get_paramsr   r3   )r#   r5   r/   r)   ÚoutÚauxÚgradsr&   r&   r'   Úeval_and_updateG   s    
z_NumPyroOptim.eval_and_updatec                    st   ˆ  ˆ¡}t|dd�|ƒ\\‰}‰ t t ˆ¡t tˆ ƒd ¡ ¡ @ ‡ ‡‡‡fdd„‡fdd„d¡\‰‰ˆ|fˆfS )a�  
        Like :meth:`eval_and_update` but when the value of the objective function
        or the gradients are not finite, we will not update the input `state`
        and will set the objective output to `nan`.

        :param fn: objective function.
        :param state: current optimizer state.
        :return: a pair of the output of objective function and the new optimizer state.
        Tr6   r   c                    s   ˆˆ  ˆ ˆ¡fS r   )r3   ©Ú_©r:   r8   r#   r/   r&   r'   Ú<lambda>h   ó    z6_NumPyroOptim.eval_and_stable_update.<locals>.<lambda>c                    s
   t jˆ fS r   )r*   Únanr<   ©r/   r&   r'   r?   i   r@   N)r7   r   r   Zcondr*   Úisfiniter   Úall)r#   r5   r/   r)   r9   r&   r>   r'   Úeval_and_stable_updateZ   s    


üz$_NumPyroOptim.eval_and_stable_update)r/   r   c                 C   s   |\}}|   |¡S )zž
        Get current parameter values.

        :param state: current optimizer state.
        :return: collection with current value for parameters.
        )r"   )r#   r/   r=   r,   r&   r&   r'   r7   n   s    z_NumPyroOptim.get_params)Ú__name__Ú
__module__Ú__qualname__r   r(   r   Ú_IterOptStater-   r3   r   r   r;   rE   r7   r&   r&   r&   r'   r   -   s   
r   c                    s   ‡ fdd„}|S )Nc                    s   d  ˆ j¡| _| S )NzQWrapper class for the JAX optimizer: :func:`~jax.example_libraries.optimizers.{}`)ÚformatrF   Ú__doc__)Úcls©r5   r&   r'   Ú_wrappedz   s    ÿz_add_doc.<locals>._wrappedr&   )r5   rN   r&   rM   r'   Ú_add_docy   s    rO   c                       s   e Zd Z‡ fdd„Z‡  ZS )r   c                    s$   t t| ƒjtjg|¢R i |¤Ž d S r   )Úsuperr   r(   r   Úadam©r#   r$   r%   ©Ú	__class__r&   r'   r(   …   s    zAdam.__init__©rF   rG   rH   r(   Ú__classcell__r&   r&   rS   r'   r   ƒ   s   r   c                       s.   e Zd ZdZddœ‡ fdd„
Zdd„ Z‡  ZS )r   a6  
    :class:`~numpyro.optim.Adam` optimizer with gradient clipping.

    :param float clip_norm: All gradient values will be clipped between
        `[-clip_norm, clip_norm]`.

    **Reference:**

    `A Method for Stochastic Optimization`, Diederik P. Kingma, Jimmy Ba
    https://arxiv.org/abs/1412.6980
    g      $@)Ú	clip_normc                   s*   || _ tt| ƒjtjg|¢R i |¤Ž d S r   )rW   rP   r   r(   r   rQ   )r#   rW   r$   r%   rS   r&   r'   r(   –   s    zClippedAdam.__init__c                    s4   |\}}t ‡ fdd„|ƒ}ˆ  |||¡}|d |fS )Nc                    s   t j| ˆ j ˆ jd�S )N)Za_minZa_max)r*   ZcliprW   )Zg_©r#   r&   r'   r?   ž   r@   z$ClippedAdam.update.<locals>.<lambda>r0   )r   r!   r1   r&   rX   r'   r3   š   s    ÿzClippedAdam.update)rF   rG   rH   rK   r(   r3   rV   r&   r&   rS   r'   r   ‰   s   r   c                       s   e Zd Z‡ fdd„Z‡  ZS )r   c                    s$   t t| ƒjtjg|¢R i |¤Ž d S r   )rP   r   r(   r   ÚadagradrR   rS   r&   r'   r(   ¦   s    zAdagrad.__init__rU   r&   r&   rS   r'   r   ¤   s   r   c                       s   e Zd Z‡ fdd„Z‡  ZS )r   c                    s$   t t| ƒjtjg|¢R i |¤Ž d S r   )rP   r   r(   r   ÚmomentumrR   rS   r&   r'   r(   ¬   s    zMomentum.__init__rU   r&   r&   rS   r'   r   ª   s   r   c                       s   e Zd Z‡ fdd„Z‡  ZS )r   c                    s$   t t| ƒjtjg|¢R i |¤Ž d S r   )rP   r   r(   r   ÚrmsproprR   rS   r&   r'   r(   ²   s    zRMSProp.__init__rU   r&   r&   rS   r'   r   °   s   r   c                       s   e Zd Z‡ fdd„Z‡  ZS )r   c                    s$   t t| ƒjtjg|¢R i |¤Ž d S r   )rP   r   r(   r   Úrmsprop_momentumrR   rS   r&   r'   r(   ¸   s    
ÿÿÿzRMSPropMomentum.__init__rU   r&   r&   rS   r'   r   ¶   s   r   c                       s   e Zd Z‡ fdd„Z‡  ZS )r   c                    s$   t t| ƒjtjg|¢R i |¤Ž d S r   )rP   r   r(   r   ÚsgdrR   rS   r&   r'   r(   À   s    zSGD.__init__rU   r&   r&   rS   r'   r   ¾   s   r   c                       s   e Zd Z‡ fdd„Z‡  ZS )r   c                    s$   t t| ƒjtjg|¢R i |¤Ž d S r   )rP   r   r(   r   Úsm3rR   rS   r&   r'   r(   Æ   s    zSM3.__init__rU   r&   r&   rS   r'   r   Ä   s   r   ZMinimizeStateÚflat_paramsÚ
unravel_fnc                 C   s   | j f| jffS r   )r_   r`   rB   r&   r&   r'   r?   Ò   r@   r?   c                 C   s   t |d | d ƒS )Nr   )Ú_MinimizeState)ÚdataZxsr&   r&   r'   r?   Ó   r@   c                  C   s"   dd„ } dd„ }dd„ }| ||fS )Nc                 S   s   t | ƒ\}}t||ƒS r   )r   ra   )r)   r_   r`   r&   r&   r'   r    Ø   s    z"_minimize_wrapper.<locals>.init_fnc                 S   s   |S r   r&   )r2   Z	grad_treer,   r&   r&   r'   r!   Ü   s    z$_minimize_wrapper.<locals>.update_fnc                 S   s   | \}}||ƒS r   r&   )r,   r_   r`   r&   r&   r'   r7   à   s    z%_minimize_wrapper.<locals>.get_paramsr&   )r    r!   r7   r&   r&   r'   Ú_minimize_wrapper×   s    rc   c                       s<   e Zd ZdZd‡ fdd„	Zeegef edœdd„Z	‡  Z
S )	r   a”  
    Wrapper class for the JAX minimizer: :func:`~jax.scipy.optimize.minimize`.

    .. warnings: This optimizer is intended to be used with static guides such
        as empty guides (maximum likelihood estimate), delta guides (MAP estimate),
        or :class:`~numpyro.infer.autoguide.AutoLaplaceApproximation`.
        Using this in stochastic setting is either expensive or hard to converge.

    **Example:**

    .. doctest::

        >>> from numpy.testing import assert_allclose
        >>> from jax import random
        >>> import jax.numpy as jnp
        >>> import numpyro
        >>> import numpyro.distributions as dist
        >>> from numpyro.infer import SVI, Trace_ELBO
        >>> from numpyro.infer.autoguide import AutoLaplaceApproximation

        >>> def model(x, y):
        ...     a = numpyro.sample("a", dist.Normal(0, 1))
        ...     b = numpyro.sample("b", dist.Normal(0, 1))
        ...     with numpyro.plate("N", y.shape[0]):
        ...         numpyro.sample("obs", dist.Normal(a + b * x, 0.1), obs=y)

        >>> x = jnp.linspace(0, 10, 100)
        >>> y = 3 * x + 2
        >>> optimizer = numpyro.optim.Minimize()
        >>> guide = AutoLaplaceApproximation(model)
        >>> svi = SVI(model, guide, optimizer, loss=Trace_ELBO())
        >>> init_state = svi.init(random.PRNGKey(0), x, y)
        >>> optimal_state, loss = svi.update(init_state, x, y)
        >>> params = svi.get_params(optimal_state)  # get guide's parameters
        >>> quantiles = guide.quantiles(params, 0.5)  # get means of posterior samples
        >>> assert_allclose(quantiles["a"], 2., atol=1e-3)
        >>> assert_allclose(quantiles["b"], 3., atol=1e-3)
    ÚBFGSc                    s   t ƒ  t¡ || _|| _d S r   )rP   r(   rc   Ú_methodÚ_kwargs)r#   Úmethodr%   rS   r&   r'   r(     s    zMinimize.__init__r4   c                    sb   |\}\}‰‡ ‡fdd„}t ||dfd| ji| j¤Ž}|j|j }}|d t|ˆƒf}|d f|fS )Nc                    s(   ˆ| ƒ} ˆ | ƒ\}}|d ur$t dƒ‚|S )Nz5Minimize does not support models with mutable states.)Ú
ValueError)Úxr8   r9   ©r5   r`   r&   r'   Úloss_fn  s    ÿz)Minimize.eval_and_update.<locals>.loss_fnr&   rg   r0   )r   re   rf   ri   Zfunra   )r#   r5   r/   r2   r_   rk   Úresultsr8   r&   rj   r'   r;     s    	ÿÿÿzMinimize.eval_and_update)rd   )rF   rG   rH   rK   r(   r   r   r   rI   r;   rV   r&   r&   rS   r'   r   ç   s   'r   )r   c                    s<   ddl ‰ ‡fdd„}‡ ‡fdd„}dd„ }td	d
„ |||ƒS )a  
    This function produces a ``numpyro.optim._NumPyroOptim`` instance from an
    ``optax.GradientTransformation`` so that it can be used with
    ``numpyro.infer.svi.SVI``. It is a lightweight wrapper that recreates the
    ``(init_fn, update_fn, get_params_fn)`` interface defined by
    :mod:`jax.example_libraries.optimizers`.

    :param transformation: An ``optax.GradientTransformation`` instance to wrap.
    :return: An instance of ``numpyro.optim._NumPyroOptim`` wrapping the supplied
        Optax optimizer.
    r   Nc                    s   ˆ   | ¡}| |fS r   )r-   )r)   r,   )Útransformationr&   r'   r    6  s    
z!optax_to_numpyro.<locals>.init_fnc                    s.   |\}}ˆ  |||¡\}}ˆ  ||¡}||fS r   )r3   Zapply_updates)Ústepr:   r/   r)   r,   ZupdatesZupdated_params©Úoptaxrm   r&   r'   r!   :  s    z#optax_to_numpyro.<locals>.update_fnc                 S   s   | \}}|S r   r&   )r/   r)   r=   r&   r&   r'   r"   @  s    z'optax_to_numpyro.<locals>.get_params_fnc                 S   s
   | ||fS r   r&   )ri   ÚyÚzr&   r&   r'   r?   D  r@   z"optax_to_numpyro.<locals>.<lambda>)rp   r   )rm   r    r!   r"   r&   ro   r'   Úoptax_to_numpyro(  s
    rs   )6rK   Úcollectionsr   Útypingr   r   r   r   Zjaxr   r   Znumpyro.utilr	   Ú__version__Zjax.example_librariesr   Zjax.experimentalZjax.flatten_utilr   Z	jax.numpyÚnumpyr*   Zjax.scipy.optimizer   Zjax.tree_utilr   r   Ú__all__r   r   ÚintrI   Úobjectr   rO   rQ   r   r   rY   r   rZ   r   r[   r   r\   r   r]   r   r^   r   ra   rc   r   rs   r&   r&   r&   r'   Ú<module>   sT   L

ýA