87 lines
2.4 KiB
Python
87 lines
2.4 KiB
Python
|
import nltk
|
|||
|
|
from nltk.tokenize import word_tokenize
|
|||
|
|
from nltk import pos_tag
|
|||
|
|
import jieba.posseg as pseg
|
|||
|
|
|
|||
|
|
# 下载相关数据
|
|||
|
|
nltk.download('punkt')
|
|||
|
|
nltk.download('averaged_perceptron_tagger')
|
|||
|
|
|
|||
|
|
|
|||
|
|
from nltk.stem import WordNetLemmatizer
|
|||
|
|
import string
|
|||
|
|
import re
|
|||
|
|
|
|||
|
|
def preprocess_eng(text):
|
|||
|
|
'''
|
|||
|
|
英文文本预处理：小写化，去除标点（待定），去除特殊符号，只保留单词
|
|||
|
|
拼写是否正确：是，因为是从ecoinvent导入的，没有拼写错误；
|
|||
|
|
词干提取(stemming)和词形还原(lemmatization)：可以处理一下，有的提取不准确，不做此操作
|
|||
|
|
'''
|
|||
|
|
# 去除标点
|
|||
|
|
text = text.translate(str.maketrans('', '', string.punctuation))
|
|||
|
|
# 去除数字
|
|||
|
|
text = re.sub(r'\d+', ' ', text)
|
|||
|
|
# 去除多余字符
|
|||
|
|
text = re.sub(r'[^A-Za-z0-9\s]', '', text)
|
|||
|
|
# 去除多余空格
|
|||
|
|
text = re.sub(r'\s+', ' ', text)
|
|||
|
|
return text
|
|||
|
|
|
|||
|
|
def preprocess_zh(text):
|
|||
|
|
'''
|
|||
|
|
中文文本预处理:只保留中文内容，去除英文、数字和标点
|
|||
|
|
'''
|
|||
|
|
text = str(text)
|
|||
|
|
# 去除英文
|
|||
|
|
text = re.sub(r'[a-zA-Z]',' ',text)
|
|||
|
|
text = re.sub(r'\d', ' ', text)
|
|||
|
|
# 去除中文标点符号
|
|||
|
|
text = re.sub(r'[，。！？、；：“”（）《》【】-]', ' ', text)
|
|||
|
|
# 去除英文标点符号
|
|||
|
|
text = re.sub(r'[.,!?;:"\'\(\)\[\]{}]', ' ', text)
|
|||
|
|
# 去除空格
|
|||
|
|
text = re.sub(r'\s+','',text)
|
|||
|
|
|
|||
|
|
return text
|
|||
|
|
|
|||
|
|
# 英文名词处理
|
|||
|
|
def get_noun_en(text):
|
|||
|
|
# 分词
|
|||
|
|
words = word_tokenize(text)
|
|||
|
|
# 词性标注
|
|||
|
|
tagged = pos_tag(words)
|
|||
|
|
|
|||
|
|
# 提取名词
|
|||
|
|
nouns = [word for word, tag in tagged if tag.startswith('NN')]
|
|||
|
|
noun = ' '.join(nouns)
|
|||
|
|
return noun
|
|||
|
|
|
|||
|
|
# 中文名词提取
|
|||
|
|
def get_noun_zh(text):
|
|||
|
|
x = str(text)
|
|||
|
|
if x=='nan':
|
|||
|
|
return ''
|
|||
|
|
words = pseg.cut(text)
|
|||
|
|
nouns = [word for word, flag in words if flag.startswith('n')]
|
|||
|
|
noun = ' '.join(nouns)
|
|||
|
|
return noun
|
|||
|
|
|
|||
|
|
def has_no_chinese(text):
|
|||
|
|
"""
|
|||
|
|
判断一个文本是否不包含中文字符
|
|||
|
|
|
|||
|
|
参数:
|
|||
|
|
text (str): 需要检查的文本
|
|||
|
|
|
|||
|
|
返回:
|
|||
|
|
bool: 如果文本中没有中文字符返回True，否则返回False
|
|||
|
|
"""
|
|||
|
|
for char in text:
|
|||
|
|
if '\u4e00' <= char <= '\u9fff' or \
|
|||
|
|
'\u3400' <= char <= '\u4dbf' or \
|
|||
|
|
'\u2f00' <= char <= '\u2fdf' or \
|
|||
|
|
'\u3100' <= char <= '\u312f' or \
|
|||
|
|
'\u31a0' <= char <= '\u31bf':
|
|||
|
|
return False
|
|||
|
|
return True
|