<?php

$sym_rx = "~(?:([^-'\"()*,.:…;?`\s]+)|([-'\"()*,.:…;?`]))(\s)*~u";
$str = "'this' is Mary! (a dog - not a human)";
echo preg_replace_callback($sym_rx, function ($m) {
	$ins = !empty($m[2]) ? $m[2] . "\\SYM" : $m[1] . "\\not SYM";
	return !empty($m[3]) ? $ins . $m[3] : $ins . " ";
}, $str);