python - skip-thought Theano model - list not declared -
i have defined list x
take contents of file , feed preprocessor of skip-though model in theano. error says x not defined
.
https://github.com/ryankiros/skip-thoughts/blob/master/skipthoughts.py
def encode(model, x, use_norm=true, verbose=true, batch_size=128, use_eos=false): """ encode sentences in list x. each entry return vector """ # first, preprocessing x = preprocess(x) # word dictionary , init d = defaultdict(lambda : 0) w in model['utable'].keys(): d[w] = 1 ufeatures = numpy.zeros((len(x), model['uoptions']['dim']), dtype='float32') bfeatures = numpy.zeros((len(x), 2 * model['boptions']['dim']), dtype='float32') # length dictionary ds = defaultdict(list) captions = [s.split() s in x] i,s in enumerate(captions): ds[len(s)].append(i) def preprocess(text): """ preprocess text encoder """ x = [] #for _chat.txt list_______________________________________ f= open("/var/www/html/_chat.txt", "r") file_list = f.readlines() first_three = file_list[1:3] x = [x.replace('\t',' ') x in first_three] x = [x.replace('\x00','') x in x] x = [x.replace(' \r\n','') x in x] #______________________________________________________________ sent_detector = nltk.data.load('tokenizers/punkt/english.pickle') t in text: sents = sent_detector.tokenize(t) result = '' s in sents: tokens = word_tokenize(s) result += ' ' + ' '.join(tokens) x.append(result) return x
Comments
Post a Comment