We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 179b9f5 commit 4364b39Copy full SHA for 4364b39
chatbot/textdata.py
@@ -366,11 +366,7 @@ def mergeSentences(sentences, fromEnd=False):
366
}
367
new_mapping = {} # Map the full words ids to the new one (TODO: Should be a list)
368
newId = 0
369
- for wordId, count in tqdm(
370
- [(i, self.idCount[i]) for i in range(len(self.idCount))],
371
- desc='Filter words:',
372
- leave=False
373
- ):
+ for wordId, count in [(i, self.idCount[i]) for i in range(len(self.idCount))]: # Iterate in order
374
if (count <= self.args.filterVocab and
375
wordId not in specialTokens): # Cadidate to filtering (Warning: don't filter special token)
376
new_mapping[wordId] = self.unknownToken
0 commit comments