# If you need to add 1 to the end of strings not having X or Y
test.dat <- c("abcde", "abcXe", "abcdY", "abcXY", "abYcXY", "abcYX")
sub("^([^XY]*)$", "\\11", sub("^([^XY]*)(Y)([^X]*)$|(.*)(X)", "\\1\\41\\3\\5\\2", test.dat))

library(stringr)
str_replace(str_replace(test.dat, "^([^XY]*)(Y)([^X]*)$|(.*)(X)", "\\1\\41\\3\\5\\2"), "^([^XY]*)$", "\\11")