import re
import json
 
# a input document of sentences
document="These are oranges and apples and and pears, but not pinapples\nThese are oranges and apples and pears, but not pinapples"
 
 
# uncomment to test UNICODE
document="तुम मुझे दोस्त कहते कहते हो"
 
sentences=[] # sentences
seen = {} # map if a token has been see already!
 
# split into sentences
lines=document.splitlines()
 
for index,line in enumerate(lines):
 
  print("Line:%d %s" % (index,line))
 
  # split token that are words
  # LP: (for Simon ;P we do not care of punct at all!
  rgx = re.compile("([\w][\w']*\w)")
  tokens=rgx.findall(line)
 
  # uncomment to test UNICODE
  tokens=["तुम","मुझे","दोस्त","कहते","कहते","हो"]
 
  print("Tokens:",tokens)
 
  sentence={} # a sentence
  items=[] # word tokens
 
  # for each token word
  for index_word,word in enumerate(tokens):
 
    # uncomment to test UNICODE
    my_regex = r"(?<!\S){}(?!\S)".format(re.escape(word))
    #my_regex = r"\b(?=\w)" + re.escape(word) + r"\b(?!\w)"
    r = re.compile(my_regex, flags=re.I | re.X | re.UNICODE)
 
    item = {}
    # for each matched token in sentence
    for m in r.finditer(document):
 
      token=m.group()
      characterOffsetBegin=m.start()
      characterOffsetEnd=characterOffsetBegin+len(m.group()) - 1 # LP: star from 0
 
      print ("word:%s characterOffsetBegin:%d characterOffsetEnd:%d" % (token, characterOffsetBegin, characterOffsetEnd) )
 
      found=-1
      if word in seen:
        found=seen[word]
 
      if characterOffsetBegin > found:
        # store last word has been seen
        seen[word] = characterOffsetBegin
        item['index']=index_word+1 #// word index starts from 1
        item['word']=token
        item['characterOffsetBegin'] = characterOffsetBegin;
        item['characterOffsetEnd'] = characterOffsetEnd;
        items.append(item)
        break
 
  sentence['text']=line
  sentence['tokens']=items
  sentences.append(sentence)
 
print(json.dumps(sentences, indent=4, sort_keys=True))
 
print("------ testing ------")
text=''
for sentence in sentences:
  for token in sentence['tokens']:
    # LP: we get the token from a slice in original text
    text = text + document[token['characterOffsetBegin']:token['characterOffsetEnd']+1] + " "
  text = text + '\n'
print(text)

aW1wb3J0IHJlCmltcG9ydCBqc29uCgojIGEgaW5wdXQgZG9jdW1lbnQgb2Ygc2VudGVuY2VzCmRvY3VtZW50PSJUaGVzZSBhcmUgb3JhbmdlcyBhbmQgYXBwbGVzIGFuZCBhbmQgcGVhcnMsIGJ1dCBub3QgcGluYXBwbGVzXG5UaGVzZSBhcmUgb3JhbmdlcyBhbmQgYXBwbGVzIGFuZCBwZWFycywgYnV0IG5vdCBwaW5hcHBsZXMiCgoKIyB1bmNvbW1lbnQgdG8gdGVzdCBVTklDT0RFCmRvY3VtZW50PSLgpKTgpYHgpK4g4KSu4KWB4KSd4KWHIOCkpuCli+CkuOCljeCkpCDgpJXgpLngpKTgpYcg4KSV4KS54KSk4KWHIOCkueCliyIKCnNlbnRlbmNlcz1bXSAjIHNlbnRlbmNlcwpzZWVuID0ge30gIyBtYXAgaWYgYSB0b2tlbiBoYXMgYmVlbiBzZWUgYWxyZWFkeSEKCiMgc3BsaXQgaW50byBzZW50ZW5jZXMKbGluZXM9ZG9jdW1lbnQuc3BsaXRsaW5lcygpCgpmb3IgaW5kZXgsbGluZSBpbiBlbnVtZXJhdGUobGluZXMpOgoKICBwcmludCgiTGluZTolZCAlcyIgJSAoaW5kZXgsbGluZSkpCgogICMgc3BsaXQgdG9rZW4gdGhhdCBhcmUgd29yZHMKICAjIExQOiAoZm9yIFNpbW9uIDtQIHdlIGRvIG5vdCBjYXJlIG9mIHB1bmN0IGF0IGFsbCEKICByZ3ggPSByZS5jb21waWxlKCIoW1x3XVtcdyddKlx3KSIpCiAgdG9rZW5zPXJneC5maW5kYWxsKGxpbmUpCgogICMgdW5jb21tZW50IHRvIHRlc3QgVU5JQ09ERQogIHRva2Vucz1bIuCkpOClgeCkriIsIuCkruClgeCkneClhyIsIuCkpuCli+CkuOCljeCkpCIsIuCkleCkueCkpOClhyIsIuCkleCkueCkpOClhyIsIuCkueCliyJdCgogIHByaW50KCJUb2tlbnM6Iix0b2tlbnMpCgogIHNlbnRlbmNlPXt9ICMgYSBzZW50ZW5jZQogIGl0ZW1zPVtdICMgd29yZCB0b2tlbnMKCiAgIyBmb3IgZWFjaCB0b2tlbiB3b3JkCiAgZm9yIGluZGV4X3dvcmQsd29yZCBpbiBlbnVtZXJhdGUodG9rZW5zKToKCiAgICAjIHVuY29tbWVudCB0byB0ZXN0IFVOSUNPREUKICAgIG15X3JlZ2V4ID0gciIoPzwhXFMpe30oPyFcUykiLmZvcm1hdChyZS5lc2NhcGUod29yZCkpCiAgICAjbXlfcmVnZXggPSByIlxiKD89XHcpIiArIHJlLmVzY2FwZSh3b3JkKSArIHIiXGIoPyFcdykiCiAgICByID0gcmUuY29tcGlsZShteV9yZWdleCwgZmxhZ3M9cmUuSSB8IHJlLlggfCByZS5VTklDT0RFKQoKICAgIGl0ZW0gPSB7fQogICAgIyBmb3IgZWFjaCBtYXRjaGVkIHRva2VuIGluIHNlbnRlbmNlCiAgICBmb3IgbSBpbiByLmZpbmRpdGVyKGRvY3VtZW50KToKCiAgICAgIHRva2VuPW0uZ3JvdXAoKQogICAgICBjaGFyYWN0ZXJPZmZzZXRCZWdpbj1tLnN0YXJ0KCkKICAgICAgY2hhcmFjdGVyT2Zmc2V0RW5kPWNoYXJhY3Rlck9mZnNldEJlZ2luK2xlbihtLmdyb3VwKCkpIC0gMSAjIExQOiBzdGFyIGZyb20gMAoKICAgICAgcHJpbnQgKCJ3b3JkOiVzIGNoYXJhY3Rlck9mZnNldEJlZ2luOiVkIGNoYXJhY3Rlck9mZnNldEVuZDolZCIgJSAodG9rZW4sIGNoYXJhY3Rlck9mZnNldEJlZ2luLCBjaGFyYWN0ZXJPZmZzZXRFbmQpICkKCiAgICAgIGZvdW5kPS0xCiAgICAgIGlmIHdvcmQgaW4gc2VlbjoKICAgICAgICBmb3VuZD1zZWVuW3dvcmRdCgogICAgICBpZiBjaGFyYWN0ZXJPZmZzZXRCZWdpbiA+IGZvdW5kOgogICAgICAgICMgc3RvcmUgbGFzdCB3b3JkIGhhcyBiZWVuIHNlZW4KICAgICAgICBzZWVuW3dvcmRdID0gY2hhcmFjdGVyT2Zmc2V0QmVnaW4KICAgICAgICBpdGVtWydpbmRleCddPWluZGV4X3dvcmQrMSAjLy8gd29yZCBpbmRleCBzdGFydHMgZnJvbSAxCiAgICAgICAgaXRlbVsnd29yZCddPXRva2VuCiAgICAgICAgaXRlbVsnY2hhcmFjdGVyT2Zmc2V0QmVnaW4nXSA9IGNoYXJhY3Rlck9mZnNldEJlZ2luOwogICAgICAgIGl0ZW1bJ2NoYXJhY3Rlck9mZnNldEVuZCddID0gY2hhcmFjdGVyT2Zmc2V0RW5kOwogICAgICAgIGl0ZW1zLmFwcGVuZChpdGVtKQogICAgICAgIGJyZWFrCgogIHNlbnRlbmNlWyd0ZXh0J109bGluZQogIHNlbnRlbmNlWyd0b2tlbnMnXT1pdGVtcwogIHNlbnRlbmNlcy5hcHBlbmQoc2VudGVuY2UpCgpwcmludChqc29uLmR1bXBzKHNlbnRlbmNlcywgaW5kZW50PTQsIHNvcnRfa2V5cz1UcnVlKSkKCnByaW50KCItLS0tLS0gdGVzdGluZyAtLS0tLS0iKQp0ZXh0PScnCmZvciBzZW50ZW5jZSBpbiBzZW50ZW5jZXM6CiAgZm9yIHRva2VuIGluIHNlbnRlbmNlWyd0b2tlbnMnXToKICAgICMgTFA6IHdlIGdldCB0aGUgdG9rZW4gZnJvbSBhIHNsaWNlIGluIG9yaWdpbmFsIHRleHQKICAgIHRleHQgPSB0ZXh0ICsgZG9jdW1lbnRbdG9rZW5bJ2NoYXJhY3Rlck9mZnNldEJlZ2luJ106dG9rZW5bJ2NoYXJhY3Rlck9mZnNldEVuZCddKzFdICsgIiAiCiAgdGV4dCA9IHRleHQgKyAnXG4nCnByaW50KHRleHQp