str = [[
function test(arg1, arg2) {
dosomething(0x12f, "String");
}
]]
str = str:gsub("%s*(%w+)%s*", "%1§")
str = str:gsub("(%W)%s*", "%1§")
for token in str:gmatch("[^§]+") do
print(token)
end
c3RyID0gW1sKICAgIGZ1bmN0aW9uIHRlc3QoYXJnMSwgYXJnMikgewogICAgICAgIGRvc29tZXRoaW5nKDB4MTJmLCAiU3RyaW5nIik7CiAgICB9Cl1dCgpzdHIgPSBzdHI6Z3N1YigiJXMqKCV3KyklcyoiLCAiJTHCpyIpCnN0ciA9IHN0cjpnc3ViKCIoJVcpJXMqIiwgIiUxwqciKQpmb3IgdG9rZW4gaW4gc3RyOmdtYXRjaCgiW17Cp10rIikgZG8KICAgIHByaW50KHRva2VuKQplbmQ=