sudo pip install spacy

python setup.py install

python -m spacy.en.download all

import spacy
nlp = spacy.load(“en”)

document = unicode(open(filename).read().decode('utf8'))
document = nlp(document)

dir(document)
>> [ 'doc', 'ents', … 'mem']

# document 的首個單詞
document[0]
>> Nice
# document 的最后一個單詞  
document[len(document)-5]
>> boston
# 列出 document 中的句子
list(document.sents)
>> [ Nice place Better than some reviews give it credit for.,
 Overall, the rooms were a bit small but nice.,
...
Everything was clean, the view was wonderful and it is very well located (the Prudential Center makes shopping and eating easy and the T is nearby for jaunts out and about the city).]

# 獲得所有標注
all_tags = {w.pos: w.pos_ for w in document}
>> {97:  u'SYM', 98: u'VERB', 99: u'X', 101: u'SPACE', 82: u'ADJ', 83: u'ADP', 84: u'ADV', 87: u'CCONJ', 88: u'DET', 89: u'INTJ', 90: u'NOUN', 91: u'NUM', 92: u'PART', 93: u'PRON', 94: u'PROPN', 95: u'PUNCT'}
# document 中第一個句子的詞性標注
for word in list(document.sents)[0]:  
    print word, word.tag_
>> ( Nice, u'JJ') (place, u'NN') (Better, u'NNP') (than, u'IN') (some, u'DT') (reviews, u'NNS') (give, u'VBP') (it, u'PRP') (creit, u'NN') (for, u'IN') (., u'.')

#一些參數定義
noisy_pos_tags = [“PROP”]
min_token_length = 2
#檢查 token 是不是噪音的函數
def isNoise(token):     
    is_noise = False
    if token.pos_ in noisy_pos_tags:
        is_noise = True
    elif token.is_stop == True:
        is_noise = True
    elif len(token.string) <= min_token_length:
        is_noise = True
    return is_noise
def cleanup(token, lower = True):
    if lower:
       token = token.lower()
    return token.strip()
# 評論中最常用的單詞
from collections import Counter
cleaned_list = [cleanup(word.string) for word in document if not isNoise(word)]
Counter(cleaned_list) .most_common(5)
>> [( u'hotel', 683), (u'room', 652), (u'great', 300),  (u'sheraton', 285), (u'location', 271)]

labels = set([w.label_ for w in document.ents])
for label in labels:
    entities = [cleanup(e.string, lower=False) for e in document.ents if label==e.label_]
    entities = list(set(entities))
    print label,entities

# 取出所有句中包含“hotel”單詞的評論
hotel = [sent for sent in document.sents if 'hotel' in sent.string.lower()]
# 創(chuàng)建依存樹
sentence = hotel[2] for word in sentence:
print word, ': ', str(list(word.children))
>> A :  []  cab :  [A, from]
from :  [airport, to]
the :  []
airport :  [the]
to :  [hotel]
the :  [] hotel :  
[the] can :  []
be :  [cab, can, cheaper, .]
cheaper :  [than] than :  
[shuttles]
the :  []
shuttles :  [the, depending]
depending :  [time] what :  []
time :  [what, of] of :  [day]
the :  [] day :  
[the, go] you :  
[]
go :  [you]
. :  []

# 檢查修飾某個單詞的所有形容詞
def pos_words (sentence, token, ptag):
    sentences = [sent for sent in sentence.sents if token in sent.string]     
    pwrds = []
    for sent in sentences:
        for word in sent:
            if character in word.string:
                   pwrds.extend([child.string.strip() for child in word.children
                                                      if child.pos_ == ptag] )
    return Counter(pwrds).most_common(10)
pos_words(document, 'hotel', “ADJ”)
>> [(u'other', 20), (u'great', 10), (u'good', 7), (u'better', 6), (u'nice', 6), (u'different', 5), (u'many', 5), (u'best', 4), (u'my', 4), (u'wonderful', 3)]

# 生成名詞短語
doc = nlp(u'I love data science on analytics vidhya')
for np in doc.noun_chunks:
    print np.text, np.root.dep_, np.root.head.text
>> I nsubj love
   data science dobj love
   analytics pobj on

from numpy import dot
from numpy.linalg import norm
from spacy.en import English
parser = English()
# 生成“apple”的詞向量 
apple = parser.vocab[u'apple']
# 余弦相似性計算函數
cosine = lambda v1, v2: dot(v1, v2) / (norm(v1) * norm(v2))
others = list({w for w in parser.vocab if w.has_vector and w.orth_.islower() and w.lower_ != unicode("apple")})
# 根據相似性值進行排序
others.sort(key=lambda w: cosine(w.vector, apple.vector))
others.reverse()
print "top most similar words to apple:"
for word in others[:10]:
    print word.orth_
>> apples iphone f ruit juice cherry lemon banana pie mac orange

from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS as stopwords
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import accuracy_score
from sklearn.base import TransformerMixin
from sklearn.pipeline import Pipeline
from sklearn.svm import LinearSVC
import string
punctuations = string.punctuation
from spacy.en import English
parser = English()
# 使用 spaCy 自定義 transformer
class predictors(TransformerMixin):
    def transform(self, X, **transform_params):
        return [clean_text(text) for text in X]
    def fit(self, X, y=None, **fit_params):
        return self
    def get_params(self, deep=True):
        return {}
# 進行文本清洗的實用的基本函數
def clean_text(text):     
    return text.strip().lower()

#創(chuàng)建 spaCy tokenizer，解析句子并生成 token
#也可以用詞向量函數來代替它
def spacy_tokenizer(sentence):
    tokens = parser(sentence)
    tokens = [tok.lemma_.lower().strip() if tok.lemma_ != "-PRON-" else tok.lower_ for tok in tokens]
    tokens = [tok for tok in tokens if (tok not in stopwords and tok not in punctuations)]     return tokens
#創(chuàng)建 vectorizer 對象，生成特征向量，以此可以自定義 spaCy 的 tokenizer
vectorizer = CountVectorizer(tokenizer = spacy_tokenizer, ngram_range=(1,1)) classifier = LinearSVC()

# 創(chuàng)建管道，進行文本清洗、tokenize、向量化、分類操作
pipe = Pipeline([("cleaner", predictors()),
                 ('vectorizer', vectorizer),
                 ('classifier', classifier)])
# Load sample data
train = [('I love this sandwich.', 'pos'),          
         ('this is an amazing place!', 'pos'),
         ('I feel very good about these beers.', 'pos'),
         ('this is my best work.', 'pos'),
         ("what an awesome view", 'pos'),
         ('I do not like this restaurant', 'neg'),
         ('I am tired of this stuff.', 'neg'),
         ("I can't deal with this", 'neg'),
         ('he is my sworn enemy!', 'neg'),          
         ('my boss is horrible.', 'neg')]
test =   [('the beer was good.', 'pos'),     
         ('I do not enjoy my job', 'neg'),
         ("I ain't feelin dandy today.", 'neg'),
         ("I feel amazing!", 'pos'),
         ('Gary is a good friend of mine.', 'pos'),
         ("I can't believe I'm doing this.", 'neg')]
# 創(chuàng)建模型并計算準確率
pipe.fit([x[0] for x in train], [x[1] for x in train])
pred_data = pipe.predict([x[0] for x in test])
for (sample, pred) in zip(test, pred_data):
    print sample, pred
print "Accuracy:", accuracy_score([x[1] for x in test], pred_data)
>>    ('the beer was good.', 'pos') pos
      ('I do not enjoy my job', 'neg') neg
      ("I ain't feelin dandy today.", 'neg') neg
      ('I feel amazing!', 'pos') pos
      ('Gary is a good friend of mine.', 'pos') pos
      ("I can't believe I'm doing this.", 'neg') neg
      Accuracy: 1.0