#!/bin/bash
s="<</key1(value1)/key2(value2)/key3(value3)/key4(outside \(inside\) outside)>>"
sed 's/.*key4(\([^\()]*\(\\.[^\()]*\)*\)).*/\1/' <<< "$s"
sed -E 's/.*key4\(([^\()]*(\\.[^\()]*)*)\).*/\1/' <<< "$s"
IyEvYmluL2Jhc2gKCnM9Ijw8L2tleTEodmFsdWUxKS9rZXkyKHZhbHVlMikva2V5Myh2YWx1ZTMpL2tleTQob3V0c2lkZSBcKGluc2lkZVwpIG91dHNpZGUpPj4iCgpzZWQgJ3MvLiprZXk0KFwoW15cKCldKlwoXFwuW15cKCldKlwpKlwpKS4qL1wxLycgIDw8PCAiJHMiCnNlZCAtRSAncy8uKmtleTRcKChbXlwoKV0qKFxcLlteXCgpXSopKilcKS4qL1wxLycgPDw8ICIkcyI=