Indexer for search engine in python

import sys
import os
import urllib
import re
from bs4 import BeautifulSoup
#first time
#from whoosh.fields import Schema, TEXT, KEYWORD, ID, STORED ,NUMERIC
#from whoosh.index import create_in, open_dir
#from whoosh.query import *
import whoosh.index as index
ix = index.open_dir("index")
#then onwards
from whoosh.analysis.tokenizers import RegexTokenizer
from whoosh.analysis.tokenizers import IDTokenizer
#--------------------------------------------------------------------------------------
#creating the schema
schema = Schema(lid= TEXT(stored=True),ldata=TEXT(stored=True))
#creating the index
if not os.path.exists("index"):
    os.mkdir("index")
ix = create_in("index",schema)
ix = open_dir("index")
writer = ix.writer()
#--------------------------------------------------------------------------------------
soup = BeautifulSoup(open("39/390099"))
#tag = soup.find('title')
#print(tag.text)
#print(soup.prettify())
#for link in soup.find_all('a'):
#   print(link.get('href'))
#print(soup.get_text())
text1=soup.get_text()
text2=text1.split()

def LowercaseFilter(tokens):
    for t in tokens:
        t.text = t.text.lower()
        yield t

regt = RegexTokenizer()
idt = IDTokenizer()
#-----------------------------------------------------------------------------

for token in regt(text1):
   writer.add_document(lid=unicode(LowercaseFilter(token.text.encode('utf-8'))),ldata=unicode(LowercaseFilter(token.text.encode('utf-8'))))
   break
   #print unicode(token.text.encode('utf-8'))
  
#length=len(words_list)


 
   #print unicode(token) , fdist1[token]
 
   #print words_list[i]


#------------------------------------------------------------------------------------------------------

testVar=raw_input("enter a search keyword : ")
from whoosh.qparser import QueryParser
ix.searcher().documents()
with ix.searcher() as searcher:
    #query = QueryParser("ldata", ix.schema).parse(u'methylotrophus')
    query = QueryParser("ldata", ix.schema).parse(testVar)
    results = searcher.search(query)
    for result in results:
        print result