在python中计算没有停用词的tfidf矩阵

Question

我正在尝试计算一个tfidf没有停用词的矩阵。这是我的代码：

def removeStopWords(documents):

    stop_words = set(stopwords.words('italian'))

    english_stop_words = set(stopwords.words('english'))

    stop_words.update(list(set(english_stop_words)))

    for d in documents:

        document = d['document']

        word_tokens = word_tokenize(document)

         filtered_sentence = ''

        for w in word_tokens:

            if not inStopwords(w, stop_words):

                 filtered_sentence = w + ' ' + filtered_sentence

        d['document'] = filtered_sentence[:-1]

    return calculateTFIDF(documents)

def calculateTFIDF(corpus):

    tfidf = TfidfVectorizer()

    x = tfidf.fit_transform(corpus)

    df_tfidf = pd.DataFrame(x.toarray(), columns=tfidf.get_feature_names())

    return {c: s[s > 0] for c, s in zip(df_tfidf, df_tfidf.T.values)}

但是当我返回矩阵（使用形式{word:value}）时，它还包含一些停用词，例如whenor il。我该如何解决？谢谢

Answer 1

有更好的方法来删除 TF-IDF 计算中的停用词。有TfidfVectorizer一个参数stop_words，您可以在其中传递要排除的单词集合。from nltk.corpus import stopwordsfrom sklearn.feature_extraction.text import TfidfVectorizerimport pandas as pddocuments = ['I went to the barbershop when my hair was long.', 'The barbershop was closed.']# create set of stopwords to removestop_words = set(stopwords.words('italian'))english_stop_words = set(stopwords.words('english'))stop_words.update(english_stop_words)# check if word in stop wordsprint('when' in stop_words)  # Trueprint('il' in stop_words)  # True# else add word to the setprint('went' in stop_words)  # Falsestop_words.add('went')# create tf-idf from original documentstfidf = TfidfVectorizer(stop_words=stop_words)x = tfidf.fit_transform(documents)df_tfidf = pd.DataFrame(x.toarray(), columns=tfidf.get_feature_names())print({c: s[s > 0] for c, s in zip(df_tfidf, df_tfidf.T.values)})# {'barbershop': array([0.44943642, 0.57973867]), 'closed': array([0.81480247]), 'hair': array([0.6316672]), 'long': array([0.6316672])}