@inproceedings{5f0379df2e6a457b9e9675a8aae54523,

title = "Recomposing the Reinforcement Learning Building Blocks with Hypernetworks",

abstract = "The Reinforcement Learning (RL) building blocks, i.e. Q-functions and policy networks, usually take elements from the cartesian product of two domains as input. In particular, the input of the Q-function is both the state and the action, and in multi-task problems (Meta-RL) the policy can take a state and a context. Standard architectures tend to ignore these variables' underlying interpretations and simply concatenate their features into a single vector. In this work, we argue that this choice may lead to poor gradient estimation in actor-critic algorithms and high variance learning steps in Meta-RL algorithms. To consider the interaction between the input variables, we suggest using a Hypernetwork architecture where a primary network determines the weights of a conditional dynamic network. We show that this approach improves the gradient approximation and reduces the learning step variance, which both accelerates learning and improves the final performance. We demonstrate a consistent improvement across different locomotion tasks and different algorithms both in RL (TD3 and SAC) and in Meta-RL (MAML and PEARL).",

author = "Elad Sarafian and Shai Keynan and Sarit Kraus",

note = "Publisher Copyright: Copyright {\textcopyright} 2021 by the author(s); 38th International Conference on Machine Learning, ICML 2021 ; Conference date: 18-07-2021 Through 24-07-2021",

year = "2021",

language = "الإنجليزيّة",

series = "Proceedings of Machine Learning Research",

publisher = "ML Research Press",

pages = "9301--9312",

booktitle = "Proceedings of the 38th International Conference on Machine Learning, ICML 2021",

}