3 NLTK Tricks for Advanced Text Preprocessing & Linguistic Analysis

import re
import time

# 示例语料库
raw_texts = [
    "We are studying neural networks and deep learning.",
    "The decision tree is a popular model in machine learning.",
    "A neural network can have many layers."
] * 5000

cleaned_texts = []
for text in raw_texts:
    # 手动替换领域术语
    text = re.sub(r"\bneural networks?\b", "neural_network", text, flags=re.IGNORECASE)
    text = re.sub(r"\bdecision trees?\b", "decision_tree", text, flags=re.IGNORECASE)
    text = re.sub(r"\bmachine learnings?\b", "machine_learning", text, flags=re.IGNORECASE)
    
    # 分词处理后的字符串
    tokens = text.lower().split()
    cleaned_texts.append(tokens)

print("示例分词：", cleaned_texts[0])

示例分词： ['we', 'are', 'studying', 'neural_network', 'and', 'deep', 'learning.']

import nltk
from nltk.tokenize import word_tokenize, MWETokenizer
import time

# 确保已下载 NLTK 资源
nltk.download('punkt', quiet=True)

raw_texts = [
    "We are studying neural networks and deep learning.",
    "The decision tree is a popular model in machine learning.",
    "A neural network can have many layers."
] * 5000

# 初始化分词器并注册 MWE 元组
mwe_tokenizer = MWETokenizer([
    ('neural', 'network'),
    ('neural', 'networks'),
    ('decision', 'tree'),
    ('decision', 'trees'),
    ('machine', 'learning')
], separator='_')

cleaned_texts_mwe = []
for text in raw_texts:
    # 使用 NLTK 标准分词器对单词进行分词
    tokens = word_tokenize(text.lower())
    # 合并指定的多词表达式
    merged_tokens = mwe_tokenizer.tokenize(tokens)
    cleaned_texts_mwe.append(merged_tokens)

print("示例分词：", cleaned_texts_mwe[0])

import nltk
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize

nltk.download('punkt', quiet=True)
nltk.download('wordnet', quiet=True)

sentence = "The feet of the running runners are getting better and faster."
tokens = word_tokenize(sentence.lower())

lemmatizer = WordNetLemmatizer()

# 简单的词形还原：假设所有词都是名词
naive_lemmas = [lemmatizer.lemmatize(token) for token in tokens]
print("Tokens:      ", tokens)
print("Naive Lemmas:", naive_lemmas)

Tokens:       ['the', 'feet', 'of', 'the', 'running', 'runners', 'are', 'getting', 'better', 'and', 'faster', '.']
Naive Lemmas: ['the', 'foot', 'of', 'the', 'running', 'runner', 'are', 'getting', 'better', 'and', 'faster', '.']

import nltk
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize
from nltk.corpus import wordnet

# 下载词性标注器资源
nltk.download('punkt', quiet=True)
nltk.download('wordnet', quiet=True)
nltk.download('averaged_perceptron_tagger', quiet=True)

sentence = "The feet of the running runners are getting better and faster."
tokens = word_tokenize(sentence.lower())

# 为每个词生成词性标签
pos_tags = nltk.pos_tag(tokens)

# 将 Penn Treebank 标签映射到 WordNet 标签
def get_wordnet_pos(treebank_tag):
    if treebank_tag.startswith('J'):
        return wordnet.ADJ
    elif treebank_tag.startswith('V'):
        return wordnet.VERB
    elif treebank_tag.startswith('N'):
        return wordnet.NOUN
    elif treebank_tag.startswith('R'):
        return wordnet.ADV
    else:
        # 默认使用 WordNet 的名词处理方式
        return None

lemmatizer = WordNetLemmatizer()

# 使用映射的词性标签进行词形还原
context_lemmas = []
for token, tag in pos_tags:
    wn_tag = get_wordnet_pos(tag)
    if wn_tag:
        lemma = lemmatizer.lemmatize(token, pos=wn_tag)
    else:
        lemma = lemmatizer.lemmatize(token)
    context_lemmas.append(lemma)

print("POS Tagged:    ", pos_tags)
print("Context Lemmas:", context_lemmas)

POS 标注:     [('the', 'DT'), ('feet', 'NNS'), ('of', 'IN'), ('the', 'DT'), ('running', 'NN'), ('runners', 'NNS'), ('are', 'VBP'), ('getting', 'VBG'), ('better', 'RBR'), ('and', 'CC'), ('faster', 'RBR'), ('.', '.')]
上下文词形: ['the', 'foot', 'of', 'the', 'running', 'runner', 'be', 'get', 'well', 'and', 'faster', '.']

from collections import Counter
import nltk
from nltk.tokenize import word_tokenize
from nltk.util import bigrams

# 示例语料库
corpus = """
Natural language processing is an active field of AI. Machine learning plays a key role 
in natural language processing. Deep learning architectures have revolutionized natural 
language processing. We need machine learning models to solve these natural language tasks.
"""
tokens = word_tokenize(corpus.lower())

# 提取并统计原始二元组
raw_bigrams = list(bigrams(tokens))
bigram_counts = Counter(raw_bigrams)

print("Top 5 Raw Bigrams:")
for bigram, freq in bigram_counts.most_common(5):
    print(f"{bigram}: {freq}")

Top 5 Raw Bigrams:
('natural', 'language'): 4
('language', 'processing'): 3
('machine', 'learning'): 2
('processing', '.'): 2
('processing', 'is'): 1

import nltk
from nltk.collocations import BigramCollocationFinder
from nltk.metrics.association import BigramAssocMeasures
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

nltk.download('punkt', quiet=True)
nltk.download('stopwords', quiet=True)

corpus = """
Natural language processing is an active field of AI. Machine learning plays a key role 
in natural language processing. Deep learning architectures have revolutionized natural 
language processing. We need machine learning models to solve these natural language tasks.
"""
tokens = word_tokenize(corpus.lower())

# Initialize the collocation finder
finder = BigramCollocationFinder.from_words(tokens)

# Filter out punctuation and stop words
stop_words = set(stopwords.words('english'))
filter_stops = lambda w: w in stop_words or not w.isalnum()
finder.apply_word_filter(filter_stops)

# Filter out bigrams that occur less than N times
finder.apply_freq_filter(2)

# Score bigrams using pointwise mutual information
pmi_measures = BigramAssocMeasures()
top_collocations = finder.score_ngrams(pmi_measures.pmi)

print("Top Collocations by PMI:")
for bigram, pmi_score in top_collocations[:5]:
    # Formulate a clean print representation
    phrase = " ".join(bigram)
    print(f"Phrase: {phrase:<30} | PMI Score: {pmi_score:.4f}")

Top Collocations by PMI:
Phrase: machine learning               | PMI Score: 3.8074
Phrase: language processing            | PMI Score: 3.3923
Phrase: natural language               | PMI Score: 3.3923