sudo pip install spacy

python setup.py install

python -m spacy.en.download all

import spacy
nlp = spacy.load(“en”)

document = unicode(open(filename).read().decode('utf8'))
document = nlp(document)

dir(document)
>> [ 'doc', 'ents', … 'mem']

# document 的首個(gè)單詞
document[0]
>> Nice
# document 的最后一個(gè)單詞  
document[len(document)-5]
>> boston
# 列出 document 中的句子
list(document.sents)
>> [ Nice place Better than some reviews give it credit for.,
 Overall, the rooms were a bit small but nice.,
...
Everything was clean, the view was wonderful and it is very well located (the Prudential Center makes shopping and eating easy and the T is nearby for jaunts out and about the city).]

# 獲得所有標(biāo)注
all_tags = {w.pos: w.pos_ for w in document}
>> {97:  u'SYM', 98: u'VERB', 99: u'X', 101: u'SPACE', 82: u'ADJ', 83: u'ADP', 84: u'ADV', 87: u'CCONJ', 88: u'DET', 89: u'INTJ', 90: u'NOUN', 91: u'NUM', 92: u'PART', 93: u'PRON', 94: u'PROPN', 95: u'PUNCT'}
# document 中第一個(gè)句子的詞性標(biāo)注
for word in list(document.sents)[0]:  
    print word, word.tag_
>> ( Nice, u'JJ') (place, u'NN') (Better, u'NNP') (than, u'IN') (some, u'DT') (reviews, u'NNS') (give, u'VBP') (it, u'PRP') (creit, u'NN') (for, u'IN') (., u'.')

#一些參數(shù)定義
noisy_pos_tags = [“PROP”]
min_token_length = 2
#檢查 token 是不是噪音的函數(shù)
def isNoise(token):     
    is_noise = False
    if token.pos_ in noisy_pos_tags:
        is_noise = True
    elif token.is_stop == True:
        is_noise = True
    elif len(token.string) <= min_token_length:
        is_noise = True
    return is_noise
def cleanup(token, lower = True):
    if lower:
       token = token.lower()
    return token.strip()
# 評(píng)論中最常用的單詞
from collections import Counter
cleaned_list = [cleanup(word.string) for word in document if not isNoise(word)]
Counter(cleaned_list) .most_common(5)
>> [( u'hotel', 683), (u'room', 652), (u'great', 300),  (u'sheraton', 285), (u'location', 271)]

labels = set([w.label_ for w in document.ents])
for label in labels:
    entities = [cleanup(e.string, lower=False) for e in document.ents if label==e.label_]
    entities = list(set(entities))
    print label,entities

# 取出所有句中包含“hotel”單詞的評(píng)論
hotel = [sent for sent in document.sents if 'hotel' in sent.string.lower()]
# 創(chuàng)建依存樹(shù)
sentence = hotel[2] for word in sentence:
print word, ': ', str(list(word.children))
>> A :  []  cab :  [A, from]
from :  [airport, to]
the :  []
airport :  [the]
to :  [hotel]
the :  [] hotel :  
[the] can :  []
be :  [cab, can, cheaper, .]
cheaper :  [than] than :  
[shuttles]
the :  []
shuttles :  [the, depending]
depending :  [time] what :  []
time :  [what, of] of :  [day]
the :  [] day :  
[the, go] you :  
[]
go :  [you]
. :  []

# 檢查修飾某個(gè)單詞的所有形容詞
def pos_words (sentence, token, ptag):
    sentences = [sent for sent in sentence.sents if token in sent.string]     
    pwrds = []
    for sent in sentences:
        for word in sent:
            if character in word.string:
                   pwrds.extend([child.string.strip() for child in word.children
                                                      if child.pos_ == ptag] )
    return Counter(pwrds).most_common(10)
pos_words(document, 'hotel', “ADJ”)
>> [(u'other', 20), (u'great', 10), (u'good', 7), (u'better', 6), (u'nice', 6), (u'different', 5), (u'many', 5), (u'best', 4), (u'my', 4), (u'wonderful', 3)]

# 生成名詞短語(yǔ)
doc = nlp(u'I love data science on analytics vidhya')
for np in doc.noun_chunks:
    print np.text, np.root.dep_, np.root.head.text
>> I nsubj love
   data science dobj love
   analytics pobj on

from numpy import dot
from numpy.linalg import norm
from spacy.en import English
parser = English()
# 生成“apple”的詞向量 
apple = parser.vocab[u'apple']
# 余弦相似性計(jì)算函數(shù)
cosine = lambda v1, v2: dot(v1, v2) / (norm(v1) * norm(v2))
others = list({w for w in parser.vocab if w.has_vector and w.orth_.islower() and w.lower_ != unicode("apple")})
# 根據(jù)相似性值進(jìn)行排序
others.sort(key=lambda w: cosine(w.vector, apple.vector))
others.reverse()
print "top most similar words to apple:"
for word in others[:10]:
    print word.orth_
>> apples iphone f ruit juice cherry lemon banana pie mac orange

from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS as stopwords
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import accuracy_score
from sklearn.base import TransformerMixin
from sklearn.pipeline import Pipeline
from sklearn.svm import LinearSVC
import string
punctuations = string.punctuation
from spacy.en import English
parser = English()
# 使用 spaCy 自定義 transformer
class predictors(TransformerMixin):
    def transform(self, X, **transform_params):
        return [clean_text(text) for text in X]
    def fit(self, X, y=None, **fit_params):
        return self
    def get_params(self, deep=True):
        return {}
# 進(jìn)行文本清洗的實(shí)用的基本函數(shù)
def clean_text(text):     
    return text.strip().lower()

#創(chuàng)建 spaCy tokenizer，解析句子并生成 token
#也可以用詞向量函數(shù)來(lái)代替它
def spacy_tokenizer(sentence):
    tokens = parser(sentence)
    tokens = [tok.lemma_.lower().strip() if tok.lemma_ != "-PRON-" else tok.lower_ for tok in tokens]
    tokens = [tok for tok in tokens if (tok not in stopwords and tok not in punctuations)]     return tokens
#創(chuàng)建 vectorizer 對(duì)象，生成特征向量，以此可以自定義 spaCy 的 tokenizer
vectorizer = CountVectorizer(tokenizer = spacy_tokenizer, ngram_range=(1,1)) classifier = LinearSVC()

# 創(chuàng)建管道，進(jìn)行文本清洗、tokenize、向量化、分類操作
pipe = Pipeline([("cleaner", predictors()),
                 ('vectorizer', vectorizer),
                 ('classifier', classifier)])
# Load sample data
train = [('I love this sandwich.', 'pos'),          
         ('this is an amazing place!', 'pos'),
         ('I feel very good about these beers.', 'pos'),
         ('this is my best work.', 'pos'),
         ("what an awesome view", 'pos'),
         ('I do not like this restaurant', 'neg'),
         ('I am tired of this stuff.', 'neg'),
         ("I can't deal with this", 'neg'),
         ('he is my sworn enemy!', 'neg'),          
         ('my boss is horrible.', 'neg')]
test =   [('the beer was good.', 'pos'),     
         ('I do not enjoy my job', 'neg'),
         ("I ain't feelin dandy today.", 'neg'),
         ("I feel amazing!", 'pos'),
         ('Gary is a good friend of mine.', 'pos'),
         ("I can't believe I'm doing this.", 'neg')]
# 創(chuàng)建模型并計(jì)算準(zhǔn)確率
pipe.fit([x[0] for x in train], [x[1] for x in train])
pred_data = pipe.predict([x[0] for x in test])
for (sample, pred) in zip(test, pred_data):
    print sample, pred
print "Accuracy:", accuracy_score([x[1] for x in test], pred_data)
>>    ('the beer was good.', 'pos') pos
      ('I do not enjoy my job', 'neg') neg
      ("I ain't feelin dandy today.", 'neg') neg
      ('I feel amazing!', 'pos') pos
      ('Gary is a good friend of mine.', 'pos') pos
      ("I can't believe I'm doing this.", 'neg') neg
      Accuracy: 1.0