Python文本数据分析：新闻分类任务（贝叶斯，TF-IDF词向量）

基本思路

1.文本分析
2.TF-IDF关键词提取
3.LDA建模
4.基于贝叶斯算法进行新闻分类

1.文本分析

11.查看数据

import pandas as pd
import jieba
df_news = pd.read_table('./data/val.txt',names=['category','theme','URL','content'],encoding='utf-8')
df_news = df_news.dropna()
print(df_news.head())

1.2转换为llist格式

print(df_news.shape)
content = df_news.content.values.tolist()
print (content[1000])

1.3使用jieba分词

ontent_S = []
for line in content:
    current_segment = jieba.lcut(line)
    if len(current_segment) > 1 and current_segment != '\r\n': #换行符
        content_S.append(current_segment)
print(content_S[1000]) #(list of list)

1.4转换为DataFrame格式

df_content=pd.DataFrame({'content_S':content_S})
print(df_content.head())

1.5使用停用词

stopwords=pd.read_csv("stopwords.txt",index_col=False,sep="\t",quoting=3,names=['stopword'], encoding='utf-8') #list
#print(stopwords.head(20))

def drop_stopwords(contents, stopwords):
    contents_clean = []
    all_words = []
    for line in contents:
        line_clean = []
        for word in line:
            if word in stopwords:
                continue
            line_clean.append(word)
            all_words.append(str(word))
        contents_clean.append(line_clean)
    return contents_clean, all_words
    # print (contents_clean)
contents = df_content.content_S.values.tolist()
stopwords = stopwords.stopword.values.tolist()
contents_clean, all_words = drop_stopwords(contents, stopwords) ###使用停用词
df_content=pd.DataFrame({'contents_clean':contents_clean}) ##每一列的分词
print(df_content.head())
df_all_words=pd.DataFrame({'all_words':all_words})  ###所有词语
print(df_all_words.head())

1.6查看词频

import numpy
words_count=df_all_words.groupby(by=['all_words'])['all_words'].agg({"count":numpy.size})##分组大小
print(words_count.head())
words_count=words_count.reset_index()
print(words_count.head())
words_count=words_count.sort_values(by=["count"],ascending=False)##排序
print(words_count.head())

1.7生成词云

from wordcloud import WordCloud
import matplotlib.pyplot as plt
import matplotlib
matplotlib.rcParams['figure.figsize'] = (10.0, 5.0)
wordcloud=WordCloud(font_path="./data/simhei.ttf",background_color="white",max_font_size=80)
word_frequence = {x[0]:x[1] for x in words_count.head(100).values}
wordcloud=wordcloud.fit_words(word_frequence)
plt.imshow(wordcloud)
plt.show()

2.TF-IDF关键词提取

2.1 提取关键词

import jieba.analyse
index = 2400
print (df_news['content'][index])
content_S_str = "".join(content_S[index])
print ("  ".join(jieba.analyse.extract_tags(content_S_str, topK=5, withWeight=False)))#3输出前五个关键词

3.LDA建模

更多参数详情
https://blog.csdn.net/sinat_26917383/article/details/79357700#1__27

3.1建模

from gensim import corpora, models, similarities
import gensim
#做映射，相当于词袋
dictionary = corpora.Dictionary(contents_clean) ##格式要求：list of list形式，分词好的的整个语料
corpus = [dictionary.doc2bow(sentence) for sentence in contents_clean]  #语料
lda = gensim.models.ldamodel.LdaModel(corpus=corpus, id2word=dictionary, num_topics=20) #类似Kmeans自己指定K值
print (lda.print_topic(1, topn=5)) ##第一个主题，关键词5个

3.2 输出20个主题的的关键词和权重

for topic in lda.print_topics(num_topics=20, num_words=5):
    print (topic[1])

4.基于贝叶斯算法进行新闻分类

4.1 转换数据

df_train=pd.DataFrame({'contents_clean':contents_clean,'label':df_news['category']})
print(df_train.tail())
print(df_train.label.unique())

label_mapping = {"汽车": 1, "财经": 2, "科技": 3, "健康": 4, "体育":5, "教育": 6,"文化": 7,"军事": 8,"娱乐": 9,"时尚": 0}
df_train['label'] = df_train['label'].map(label_mapping) ##变换label
print(df_train.head())

4.2 创建训练、测试集

from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(df_train['contents_clean'].values, df_train['label'].values, random_state=1)
#print(x_train)
#print(x_train[0])
#x_train = x_train.flatten()
words = []
for line_index in range(len(x_train)):  
    try:
        #x_train[line_index][word_index] = str(x_train[line_index][word_index])
        words.append(' '.join(x_train[line_index]))
    except:
        print (line_index)
print(words[0])
print (len(words))

4.3 构建贝叶斯模型

from sklearn.feature_extraction.text import CountVectorizer
vec = CountVectorizer(analyzer='word', max_features=4000,  lowercase = False)
vec.fit(words)
from sklearn.naive_bayes import MultinomialNB
classifier = MultinomialNB()
classifier.fit(vec.transform(words), y_train)
test_words = []
for line_index in range(len(x_test)):
    try:
        #x_train[line_index][word_index] = str(x_train[line_index][word_index])
        test_words.append(' '.join(x_test[line_index]))
    except:
         print (line_index)
print('test_words[0]',test_words[0])
print('test_words_sorce',classifier.score(vec.transform(test_words), y_test))

4.4 TF-IDF模型

from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(analyzer='word', max_features=4000,  lowercase = False)
vectorizer.fit(words)

from sklearn.naive_bayes import MultinomialNB
classifier = MultinomialNB()
classifier.fit(vectorizer.transform(words), y_train)
print(classifier.score(vectorizer.transform(test_words), y_test))

不同的模型预测结果也不同