x <- "{(i,j) foo (k,l) bar (m,n,o)} should match on (h,j), (a,s), and (i,o,g)."
regmatches(x, gregexpr("(?:\\G(?!\\A)|{)[^{}]*?\\K\\([^()]*\\)(?=[^{}]*})", x, perl=TRUE))
eCA8LSAieyhpLGopIGZvbyAoayxsKSBiYXIgKG0sbixvKX0gc2hvdWxkIG1hdGNoIG9uIChoLGopLCAoYSxzKSwgYW5kIChpLG8sZykuIgpyZWdtYXRjaGVzKHgsIGdyZWdleHByKCIoPzpcXEcoPyFcXEEpfHspW157fV0qP1xcS1xcKFteKCldKlxcKSg/PVtee31dKn0pIiwgeCwgcGVybD1UUlVFKSk=