strs <- c("N(0, 1)", "N(N(0.1, 1), 1)", "N(U(0, 1), 1)", "N(0, T(0, 1))", "N(N(0, 1), N(0, 1))")
p <- "(?:\\G(?!^)\\s*,\\s*|^N\\()\\K(?:\\d+|\\w+(\\([^()]*(?:(?1)[^()]*)*\\)))(?=\\s*,|\\)$)"
res <- regmatches(strs, gregexpr(p, strs, perl=TRUE))
res
c3RycyA8LSBjKCJOKDAsIDEpIiwgIk4oTigwLjEsIDEpLCAxKSIsICJOKFUoMCwgMSksIDEpIiwgIk4oMCwgVCgwLCAxKSkiLCAiTihOKDAsIDEpLCBOKDAsIDEpKSIpCnAgPC0gIig/OlxcRyg/IV4pXFxzKixcXHMqfF5OXFwoKVxcSyg/OlxcZCt8XFx3KyhcXChbXigpXSooPzooPzEpW14oKV0qKSpcXCkpKSg/PVxccyosfFxcKSQpIgpyZXMgPC0gcmVnbWF0Y2hlcyhzdHJzLCBncmVnZXhwcihwLCBzdHJzLCBwZXJsPVRSVUUpKQpyZXM=