<?php
$s = '\\o{a}\\o{a}{b}';
echo "$s\n";
preg_match('~\\\o(\{(?>[^{}\\\]++|(?1)|\\.)+\}){2}~', $s, $match);
$s = '\\o{{a}}{b}';
echo "$s\n";
preg_match('~\\\o(\{(?>[^{}\\\]++|(?1)|\\.)+\}){2}~', $s, $match);
# Or if you want the individual parameters
preg_match('~\\\o(\{(?>[^{}\\\]++|(?1)|\\.)+\})(\{(?>[^{}\\\]++|(?1)|\\.)+\})~', $s, $match);
PD9waHAKCiRzID0gJ1xcb3thfVxcb3thfXtifSc7CmVjaG8gIiRzXG4iOwpwcmVnX21hdGNoKCd+XFxcbyhceyg/Pltee31cXFxdKyt8KD8xKXxcXC4pK1x9KXsyfX4nLCAkcywgJG1hdGNoKTsKcHJpbnRfcigkbWF0Y2gpOwoKJHMgPSAnXFxve3thfX17Yn0nOwplY2hvICIkc1xuIjsKcHJlZ19tYXRjaCgnflxcXG8oXHsoPz5bXnt9XFxcXSsrfCg/MSl8XFwuKStcfSl7Mn1+JywgJHMsICRtYXRjaCk7CnByaW50X3IoJG1hdGNoKTsKCiMgT3IgaWYgeW91IHdhbnQgdGhlIGluZGl2aWR1YWwgcGFyYW1ldGVycwpwcmVnX21hdGNoKCd+XFxcbyhceyg/Pltee31cXFxdKyt8KD8xKXxcXC4pK1x9KShceyg/Pltee31cXFxdKyt8KD8xKXxcXC4pK1x9KX4nLCAkcywgJG1hdGNoKTsKcHJpbnRfcigkbWF0Y2gpOw==