a
    ;gh+                     @   s  d Z ddlmZ ddlmZmZmZmZ ddlZddlm	Z	m
Z
 ddlmZ eejdkrhddlmZ nddlmZ dd	lmZ ddlmZ dd
lmZ ddlmZmZ g dZedZedZeeef ZG dd de Z!dd Z"e"ej#G dd de!Z$G dd de!Z%e"ej&G dd de!Z'e"ej(G dd de!Z)e"ej*G dd de!Z+e"ej,G dd de!Z-e"ej.G dd  d e!Z/e"ej0G d!d" d"e!Z1ed#d$d%gZ2ee2d&d' d(d'  d)d* Z3G d+d, d,e!Z4e!d-d.d/Z5dS )0z
Optimizer classes defined here are light wrappers over the corresponding optimizers
sourced from :mod:`jax.example_libraries.optimizers` with an interface that is better
suited for working with NumPyro inference algorithms.
    )
namedtuple)AnyCallableTupleTypeVarN)laxvalue_and_grad)_versiontuple)r         )
optimizers)ravel_pytree)minimize)register_pytree_nodetree_map)	AdamAdagradClippedAdamMinimizeMomentumRMSPropRMSPropMomentumSGDSM3_Params	_OptStatec                   @   s   e Zd ZeddddZeedddZeeedd	d
Zee	ge
f edddZee	ge
f edddZeedddZdS )_NumPyroOptimN)optim_fnreturnc                 O   s   ||i |\| _ | _| _d S N)init_fn	update_fnget_params_fn)selfr   argskwargs r&   ]/mounts/lovelace/software/anaconda3/envs/metaDMG/lib/python3.9/site-packages/numpyro/optim.py__init__.   s    z_NumPyroOptim.__init__)paramsr   c                 C   s   |  |}td|fS )z
        Initialize the optimizer with parameters designated to be optimized.

        :param params: a collection of numpy arrays.
        :return: initial optimizer state.
        r   )r    jnparray)r#   r)   	opt_stater&   r&   r'   init1   s    
z_NumPyroOptim.init)gstater   c                 C   s"   |\}}|  |||}|d |fS )z
        Gradient update for the optimizer.

        :param g: gradient information for parameters.
        :param state: current optimizer state.
        :return: new optimizer state after the update.
           )r!   r#   r.   r/   ir,   r&   r&   r'   update;   s    z_NumPyroOptim.updatefnr/   c                 C   s6   |  |}t|dd|\\}}}||f| ||fS )a  
        Performs an optimization step for the objective function `fn`.
        For most optimizers, the update is performed based on the gradient
        of the objective function w.r.t. the current state. However, for
        some optimizers such as :class:`Minimize`, the update is performed
        by reevaluating the function multiple times to get optimal
        parameters.

        :param fn: an objective function returning a pair where the first item
            is a scalar loss function to be differentiated and the second item
            is an auxiliary output.
        :param state: current optimizer state.
        :return: a pair of the output of objective function and the new optimizer state.
        TZhas_aux)
get_paramsr   r3   )r#   r5   r/   r)   outauxgradsr&   r&   r'   eval_and_updateG   s    
z_NumPyroOptim.eval_and_updatec                    st    }t|dd|\\} tttt d  @  fddfddd\|ffS )a  
        Like :meth:`eval_and_update` but when the value of the objective function
        or the gradients are not finite, we will not update the input `state`
        and will set the objective output to `nan`.

        :param fn: objective function.
        :param state: current optimizer state.
        :return: a pair of the output of objective function and the new optimizer state.
        Tr6   r   c                    s     fS r   )r3   _r:   r8   r#   r/   r&   r'   <lambda>h       z6_NumPyroOptim.eval_and_stable_update.<locals>.<lambda>c                    s
   t j fS r   )r*   nanr<   r/   r&   r'   r?   i   r@   N)r7   r   r   Zcondr*   isfiniter   all)r#   r5   r/   r)   r9   r&   r>   r'   eval_and_stable_updateZ   s    


z$_NumPyroOptim.eval_and_stable_update)r/   r   c                 C   s   |\}}|  |S )z
        Get current parameter values.

        :param state: current optimizer state.
        :return: collection with current value for parameters.
        )r"   )r#   r/   r=   r,   r&   r&   r'   r7   n   s    z_NumPyroOptim.get_params)__name__
__module____qualname__r   r(   r   _IterOptStater-   r3   r   r   r;   rE   r7   r&   r&   r&   r'   r   -   s   
r   c                    s    fdd}|S )Nc                    s   d  j| _| S )NzQWrapper class for the JAX optimizer: :func:`~jax.example_libraries.optimizers.{}`)formatrF   __doc__)clsr5   r&   r'   _wrappedz   s    z_add_doc.<locals>._wrappedr&   )r5   rN   r&   rM   r'   _add_docy   s    rO   c                       s   e Zd Z fddZ  ZS )r   c                    s$   t t| jtjg|R i | d S r   )superr   r(   r   adamr#   r$   r%   	__class__r&   r'   r(      s    zAdam.__init__rF   rG   rH   r(   __classcell__r&   r&   rS   r'   r      s   r   c                       s.   e Zd ZdZdd fdd
Zdd Z  ZS )r   a6  
    :class:`~numpyro.optim.Adam` optimizer with gradient clipping.

    :param float clip_norm: All gradient values will be clipped between
        `[-clip_norm, clip_norm]`.

    **Reference:**

    `A Method for Stochastic Optimization`, Diederik P. Kingma, Jimmy Ba
    https://arxiv.org/abs/1412.6980
    g      $@)	clip_normc                   s*   || _ tt| jtjg|R i | d S r   )rW   rP   r   r(   r   rQ   )r#   rW   r$   r%   rS   r&   r'   r(      s    zClippedAdam.__init__c                    s4   |\}}t  fdd|} |||}|d |fS )Nc                    s   t j|  j  jdS )N)Za_minZa_max)r*   ZcliprW   )Zg_r#   r&   r'   r?      r@   z$ClippedAdam.update.<locals>.<lambda>r0   )r   r!   r1   r&   rX   r'   r3      s    zClippedAdam.update)rF   rG   rH   rK   r(   r3   rV   r&   r&   rS   r'   r      s   r   c                       s   e Zd Z fddZ  ZS )r   c                    s$   t t| jtjg|R i | d S r   )rP   r   r(   r   adagradrR   rS   r&   r'   r(      s    zAdagrad.__init__rU   r&   r&   rS   r'   r      s   r   c                       s   e Zd Z fddZ  ZS )r   c                    s$   t t| jtjg|R i | d S r   )rP   r   r(   r   momentumrR   rS   r&   r'   r(      s    zMomentum.__init__rU   r&   r&   rS   r'   r      s   r   c                       s   e Zd Z fddZ  ZS )r   c                    s$   t t| jtjg|R i | d S r   )rP   r   r(   r   rmsproprR   rS   r&   r'   r(      s    zRMSProp.__init__rU   r&   r&   rS   r'   r      s   r   c                       s   e Zd Z fddZ  ZS )r   c                    s$   t t| jtjg|R i | d S r   )rP   r   r(   r   rmsprop_momentumrR   rS   r&   r'   r(      s    
zRMSPropMomentum.__init__rU   r&   r&   rS   r'   r      s   r   c                       s   e Zd Z fddZ  ZS )r   c                    s$   t t| jtjg|R i | d S r   )rP   r   r(   r   sgdrR   rS   r&   r'   r(      s    zSGD.__init__rU   r&   r&   rS   r'   r      s   r   c                       s   e Zd Z fddZ  ZS )r   c                    s$   t t| jtjg|R i | d S r   )rP   r   r(   r   sm3rR   rS   r&   r'   r(      s    zSM3.__init__rU   r&   r&   rS   r'   r      s   r   ZMinimizeStateflat_params
unravel_fnc                 C   s   | j f| jffS r   )r_   r`   rB   r&   r&   r'   r?      r@   r?   c                 C   s   t |d | d S )Nr   )_MinimizeState)dataZxsr&   r&   r'   r?      r@   c                  C   s"   dd } dd }dd }| ||fS )Nc                 S   s   t | \}}t||S r   )r   ra   )r)   r_   r`   r&   r&   r'   r       s    z"_minimize_wrapper.<locals>.init_fnc                 S   s   |S r   r&   )r2   Z	grad_treer,   r&   r&   r'   r!      s    z$_minimize_wrapper.<locals>.update_fnc                 S   s   | \}}||S r   r&   )r,   r_   r`   r&   r&   r'   r7      s    z%_minimize_wrapper.<locals>.get_paramsr&   )r    r!   r7   r&   r&   r'   _minimize_wrapper   s    rc   c                       s<   e Zd ZdZd fdd	Zeegef edddZ	  Z
S )	r   a  
    Wrapper class for the JAX minimizer: :func:`~jax.scipy.optimize.minimize`.

    .. warnings: This optimizer is intended to be used with static guides such
        as empty guides (maximum likelihood estimate), delta guides (MAP estimate),
        or :class:`~numpyro.infer.autoguide.AutoLaplaceApproximation`.
        Using this in stochastic setting is either expensive or hard to converge.

    **Example:**

    .. doctest::

        >>> from numpy.testing import assert_allclose
        >>> from jax import random
        >>> import jax.numpy as jnp
        >>> import numpyro
        >>> import numpyro.distributions as dist
        >>> from numpyro.infer import SVI, Trace_ELBO
        >>> from numpyro.infer.autoguide import AutoLaplaceApproximation

        >>> def model(x, y):
        ...     a = numpyro.sample("a", dist.Normal(0, 1))
        ...     b = numpyro.sample("b", dist.Normal(0, 1))
        ...     with numpyro.plate("N", y.shape[0]):
        ...         numpyro.sample("obs", dist.Normal(a + b * x, 0.1), obs=y)

        >>> x = jnp.linspace(0, 10, 100)
        >>> y = 3 * x + 2
        >>> optimizer = numpyro.optim.Minimize()
        >>> guide = AutoLaplaceApproximation(model)
        >>> svi = SVI(model, guide, optimizer, loss=Trace_ELBO())
        >>> init_state = svi.init(random.PRNGKey(0), x, y)
        >>> optimal_state, loss = svi.update(init_state, x, y)
        >>> params = svi.get_params(optimal_state)  # get guide's parameters
        >>> quantiles = guide.quantiles(params, 0.5)  # get means of posterior samples
        >>> assert_allclose(quantiles["a"], 2., atol=1e-3)
        >>> assert_allclose(quantiles["b"], 3., atol=1e-3)
    BFGSc                    s   t  t || _|| _d S r   )rP   r(   rc   _method_kwargs)r#   methodr%   rS   r&   r'   r(     s    zMinimize.__init__r4   c                    sb   |\}\} fdd}t ||dfd| ji| j}|j|j }}|d t|f}|d f|fS )Nc                    s(   | }  | \}}|d ur$t d|S )Nz5Minimize does not support models with mutable states.)
ValueError)xr8   r9   r5   r`   r&   r'   loss_fn  s    z)Minimize.eval_and_update.<locals>.loss_fnr&   rg   r0   )r   re   rf   ri   Zfunra   )r#   r5   r/   r2   r_   rk   resultsr8   r&   rj   r'   r;     s    	zMinimize.eval_and_update)rd   )rF   rG   rH   rK   r(   r   r   r   rI   r;   rV   r&   r&   rS   r'   r      s   'r   )r   c                    s<   ddl  fdd} fdd}dd }td	d
 |||S )a  
    This function produces a ``numpyro.optim._NumPyroOptim`` instance from an
    ``optax.GradientTransformation`` so that it can be used with
    ``numpyro.infer.svi.SVI``. It is a lightweight wrapper that recreates the
    ``(init_fn, update_fn, get_params_fn)`` interface defined by
    :mod:`jax.example_libraries.optimizers`.

    :param transformation: An ``optax.GradientTransformation`` instance to wrap.
    :return: An instance of ``numpyro.optim._NumPyroOptim`` wrapping the supplied
        Optax optimizer.
    r   Nc                    s     | }| |fS r   )r-   )r)   r,   )transformationr&   r'   r    6  s    
z!optax_to_numpyro.<locals>.init_fnc                    s.   |\}} |||\}} ||}||fS r   )r3   Zapply_updates)stepr:   r/   r)   r,   ZupdatesZupdated_paramsoptaxrm   r&   r'   r!   :  s    z#optax_to_numpyro.<locals>.update_fnc                 S   s   | \}}|S r   r&   )r/   r)   r=   r&   r&   r'   r"   @  s    z'optax_to_numpyro.<locals>.get_params_fnc                 S   s
   | ||fS r   r&   )ri   yzr&   r&   r'   r?   D  r@   z"optax_to_numpyro.<locals>.<lambda>)rp   r   )rm   r    r!   r"   r&   ro   r'   optax_to_numpyro(  s
    rs   )6rK   collectionsr   typingr   r   r   r   Zjaxr   r   Znumpyro.utilr	   __version__Zjax.example_librariesr   Zjax.experimentalZjax.flatten_utilr   Z	jax.numpynumpyr*   Zjax.scipy.optimizer   Zjax.tree_utilr   r   __all__r   r   intrI   objectr   rO   rQ   r   r   rY   r   rZ   r   r[   r   r\   r   r]   r   r^   r   ra   rc   r   rs   r&   r&   r&   r'   <module>   sT   L

A