@inproceedings{49e4ed515b0d4f7b8ec2777b00c583c9,
title = "Density Estimation of Processes with Memory via Donsker Vardhan",
abstract = "Density estimation plays an important role in modeling random variables (RVs) with continuous alphabets. This work provides an algorithm that estimates the probability density function (PDF) of stationary and ergodic random processes using recurrent neural networks (RNNs). The main idea is to decompose the target PDF into a known auxiliary PDF and a likelihood ratio between the target and auxiliary PDFs. The algorithm focuses on estimating the likelihood ratio using the Donsker Vardhan (DV) variational formula of Kullback Leibler (KL) divergence. Together, the maximizer of the DV formula and the auxiliary PDF are used to construct the estimator of the target PDF in the form of a Gibbs density. The obtained estimator converges to the target PDF in total variation (TV) and in distribution. Also, we show that proposed estimator minimizes the cross entropy (CE) between the target and auxiliary distribution, and that with a proper choice of the auxiliary distribution, it defines a tight upper bound on the entropy rate. We demonstrate this approach by estimating the density of a Gaussian hidden Markov model.",
keywords = "density estimation, Donsker-Vardhan, recurrent-neural-network",
author = "Ziv Aharoni and Dor Tsur and Permuter, {Haim H.}",
note = "Publisher Copyright: {\textcopyright} 2022 IEEE.; 2022 IEEE International Symposium on Information Theory, ISIT 2022 ; Conference date: 26-06-2022 Through 01-07-2022",
year = "2022",
month = aug,
day = "3",
doi = "10.1109/ISIT50566.2022.9834775",
language = "English",
series = "IEEE International Symposium on Information Theory - Proceedings",
publisher = "Institute of Electrical and Electronics Engineers",
pages = "330--335",
booktitle = "2022 IEEE International Symposium on Information Theory, ISIT 2022",
address = "United States",
}