x <- "{(i,j) foo (k,l) bar (m,n,o)} should match on (h,j), (a,s), and (i,o,g)."
regmatches(x, gregexpr("(?:\\G(?!\\A)|{)[^{}]*?\\K\\([^()]*\\)(?=[^{}]*})", x, perl=TRUE))