[GitHub]Python 3 Text-Processing with NLTK 3 Cookbook.

是 否 +2 论坛币

k人 参与回答

经管之家送您一份

应届毕业生专属福利!

求职就业群

赵安豆老师微信：zhaoandou666

经管之家联合CDA

送您一个全额奖学金名额~ !

立即领取

感谢您参与论坛问题回答

经管之家送您两个论坛币！

+2 论坛币

本帖隐藏的内容

nltk3-cookbook-master.zip (37.06 KB)

2017-7-7 00:22:35 上传

nltk3-cookbook

本帖隐藏的内容

https://github.com/karanmilan/Automatic-Answer-Evaluation/blob/master/Python%203%20Text%20Processing%20with%20NLTK%203%20Cookbook.pdf

Python 3 code and corpus examples for the Python 3 Text-Processing with NLTK 3 Cookbook.

扫码加我 拉你入群

请注明：姓名-公司-职位

以便审核进群资格，未注明则拒绝

分享0 收藏0 回帖

关键词：Processing Cookbook processI Process python

1. ==========================
   
2. Setting up a Custom Corpus
   
3. ==========================
   
4. 
   
5. >>> import os, os.path
   
6. >>> path = os.path.expanduser('~/nltk_data')
   
7. >>> if not os.path.exists(path):
   
8. ...                os.mkdir(path)
   
9. >>> os.path.exists(path)
   
10. True
    
11. >>> import nltk.data
    
12. >>> path in nltk.data.path
    
13. True
    
14. 
    
15. >>> nltk.data.load('corpora/cookbook/mywords.txt', format='raw')
    
16. b'nltk\\n'
    
17. 
    
18. >>> nltk.data.load('corpora/cookbook/synonyms.yaml')
    
19. {'bday': 'birthday'}

复制代码

1. ===========================
   
2. Creating a Word List Corpus
   
3. ===========================
   
4. 
   
5. >>> from nltk.corpus.reader import WordListCorpusReader
   
6. >>> reader = WordListCorpusReader('.', ['wordlist'])
   
7. >>> reader.words()
   
8. ['nltk', 'corpus', 'corpora', 'wordnet']
   
9. >>> reader.fileids()
   
10. ['wordlist']
    
11. 
    
12. >>> reader.raw()
    
13. 'nltk\\ncorpus\\ncorpora\\nwordnet\\n'
    
14. >>> from nltk.tokenize import line_tokenize
    
15. >>> line_tokenize(reader.raw())
    
16. ['nltk', 'corpus', 'corpora', 'wordnet']
    
17. 
    
18. >>> from nltk.corpus import names
    
19. >>> names.fileids()
    
20. ['female.txt', 'male.txt']
    
21. >>> len(names.words('female.txt'))
    
22. 5001
    
23. >>> len(names.words('male.txt'))
    
24. 2943
    
25. 
    
26. >>> from nltk.corpus import words
    
27. >>> words.fileids()
    
28. ['en', 'en-basic']
    
29. >>> len(words.words('en-basic'))
    
30. 850
    
31. >>> len(words.words('en'))
    
32. 234936

复制代码

1. Creating a Part-of-Speech Tagged Word Corpus
   
2. ============================================
   
3. 
   
4. >>> from nltk.corpus.reader import TaggedCorpusReader
   
5. >>> reader = TaggedCorpusReader('.', r'.*\.pos')
   
6. >>> reader.words()
   
7. ['The', 'expense', 'and', 'time', 'involved', 'are', ...]
   
8. >>> reader.tagged_words()
   
9. [('The', 'AT-TL'), ('expense', 'NN'), ('and', 'CC'), ...]
   
10. >>> reader.sents()
    
11. [['The', 'expense', 'and', 'time', 'involved', 'are', 'astronomical', '.']]
    
12. >>> reader.tagged_sents()
    
13. [[('The', 'AT-TL'), ('expense', 'NN'), ('and', 'CC'), ('time', 'NN'), ('involved', 'VBN'), ('are', 'BER'), ('astronomical', 'JJ'), ('.', '.')]]
    
14. >>> reader.paras()
    
15. [[['The', 'expense', 'and', 'time', 'involved', 'are', 'astronomical', '.']]]
    
16. >>> reader.tagged_paras()
    
17. [[[('The', 'AT-TL'), ('expense', 'NN'), ('and', 'CC'), ('time', 'NN'), ('involved', 'VBN'), ('are', 'BER'), ('astronomical', 'JJ'), ('.', '.')]]]
    
18. 
    
19. >>> from nltk.tokenize import SpaceTokenizer
    
20. >>> reader = TaggedCorpusReader('.', r'.*\.pos', word_tokenizer=SpaceTokenizer())
    
21. >>> reader.words()
    
22. ['The', 'expense', 'and', 'time', 'involved', 'are', ...]
    
23. 
    
24. >>> from nltk.tokenize import LineTokenizer
    
25. >>> reader = TaggedCorpusReader('.', r'.*\.pos', sent_tokenizer=LineTokenizer())
    
26. >>> reader.sents()
    
27. [['The', 'expense', 'and', 'time', 'involved', 'are', 'astronomical', '.']]
    
28. 
    
29. >>> reader = TaggedCorpusReader('.', r'.*\.pos', tagset='en-brown')
    
30. >>> reader.tagged_words(tagset='universal')
    
31. [('The', 'DET'), ('expense', 'NOUN'), ('and', 'CONJ'), ...]
    
32. 
    
33. >>> from nltk.corpus import treebank
    
34. >>> treebank.tagged_words()
    
35. [('Pierre', 'NNP'), ('Vinken', 'NNP'), (',', ','), ...]
    
36. >>> treebank.tagged_words(tagset='universal')
    
37. [('Pierre', 'NOUN'), ('Vinken', 'NOUN'), (',', '.'), ...]
    
38. >>> treebank.tagged_words(tagset='brown')
    
39. [('Pierre', 'UNK'), ('Vinken', 'UNK'), (',', 'UNK'), ...]

复制代码

1. ================================
   
2. Creating a Chunked Phrase Corpus
   
3. ================================
   
4. 
   
5. >>> from nltk.corpus.reader import ChunkedCorpusReader
   
6. >>> reader = ChunkedCorpusReader('.', r'.*\.chunk')
   
7. >>> reader.chunked_words()
   
8. [Tree('NP', [('Earlier', 'JJR'), ('staff-reduction', 'NN'), ('moves', 'NNS')]), ('have', 'VBP'), ...]
   
9. >>> reader.chunked_sents()
   
10. [Tree('S', [Tree('NP', [('Earlier', 'JJR'), ('staff-reduction', 'NN'), ('moves', 'NNS')]), ('have', 'VBP'), ('trimmed', 'VBN'), ('about', 'IN'), Tree('NP', [('300', 'CD'), ('jobs', 'NNS')]), (',', ','), Tree('NP', [('the', 'DT'), ('spokesman', 'NN')]), ('said', 'VBD'), ('.', '.')])]
    
11. >>> reader.chunked_paras()
    
12. [[Tree('S', [Tree('NP', [('Earlier', 'JJR'), ('staff-reduction', 'NN'), ('moves', 'NNS')]), ('have', 'VBP'), ('trimmed', 'VBN'), ('about', 'IN'), Tree('NP', [('300', 'CD'), ('jobs', 'NNS')]), (',', ','), Tree('NP', [('the', 'DT'), ('spokesman', 'NN')]), ('said', 'VBD'), ('.', '.')])]]
    
13. 
    
14. >>> from nltk.corpus.reader import ConllChunkCorpusReader
    
15. >>> conllreader = ConllChunkCorpusReader('.', r'.*\.iob', ('NP', 'VP', 'PP'))
    
16. >>> conllreader.chunked_words()
    
17. [Tree('NP', [('Mr.', 'NNP'), ('Meador', 'NNP')]), Tree('VP', [('had', 'VBD'), ('been', 'VBN')]), ...]
    
18. >>> conllreader.chunked_sents()
    
19. [Tree('S', [Tree('NP', [('Mr.', 'NNP'), ('Meador', 'NNP')]), Tree('VP', [('had', 'VBD'), ('been', 'VBN')]), Tree('NP', [('executive', 'JJ'), ('vice', 'NN'), ('president', 'NN')]), Tree('PP', [('of', 'IN')]), Tree('NP', [('Balcor', 'NNP')]), ('.', '.')])]
    
20. >>> conllreader.iob_words()
    
21. [('Mr.', 'NNP', 'B-NP'), ('Meador', 'NNP', 'I-NP'), ...]
    
22. >>> conllreader.iob_sents()
    
23. [[('Mr.', 'NNP', 'B-NP'), ('Meador', 'NNP', 'I-NP'), ('had', 'VBD', 'B-VP'), ('been', 'VBN', 'I-VP'), ('executive', 'JJ', 'B-NP'), ('vice', 'NN', 'I-NP'), ('president', 'NN', 'I-NP'), ('of', 'IN', 'B-PP'), ('Balcor', 'NNP', 'B-NP'), ('.', '.', 'O')]]
    
24. 
    
25. >>> reader.chunked_words()[0].leaves()
    
26. [('Earlier', 'JJR'), ('staff-reduction', 'NN'), ('moves', 'NNS')]
    
27. >>> reader.chunked_sents()[0].leaves()
    
28. [('Earlier', 'JJR'), ('staff-reduction', 'NN'), ('moves', 'NNS'), ('have', 'VBP'), ('trimmed', 'VBN'), ('about', 'IN'), ('300', 'CD'), ('jobs', 'NNS'), (',', ','), ('the', 'DT'), ('spokesman', 'NN'), ('said', 'VBD'), ('.', '.')]
    
29. >>> reader.chunked_paras()[0][0].leaves()
    
30. [('Earlier', 'JJR'), ('staff-reduction', 'NN'), ('moves', 'NNS'), ('have', 'VBP'), ('trimmed', 'VBN'), ('about', 'IN'), ('300', 'CD'), ('jobs', 'NNS'), (',', ','), ('the', 'DT'), ('spokesman', 'NN'), ('said', 'VBD'), ('.', '.')]

复制代码

1. ==================================
   
2. Creating a Categorized Text Corpus
   
3. ==================================
   
4. 
   
5. >>> from nltk.corpus import brown
   
6. >>> brown.categories()
   
7. ['adventure', 'belles_lettres', 'editorial', 'fiction', 'government', 'hobbies', 'humor', 'learned', 'lore', 'mystery', 'news', 'religion', 'reviews', 'romance', 'science_fiction']
   
8. 
   
9. >>> from nltk.corpus.reader import CategorizedPlaintextCorpusReader
   
10. >>> reader = CategorizedPlaintextCorpusReader('.', r'movie_.*\.txt', cat_pattern=r'movie_(\w+)\.txt')
    
11. >>> reader.categories()
    
12. ['neg', 'pos']
    
13. >>> reader.fileids(categories=['neg'])
    
14. ['movie_neg.txt']
    
15. >>> reader.fileids(categories=['pos'])
    
16. ['movie_pos.txt']
    
17. 
    
18. >>> reader = CategorizedPlaintextCorpusReader('.', r'movie_.*\.txt', cat_map={'movie_pos.txt': ['pos'], 'movie_neg.txt': ['neg']})
    
19. >>> reader.categories()
    
20. ['neg', 'pos']

复制代码

1. ===================================
   
2. Creating a Categorized Chunk Corpus
   
3. ===================================
   
4. 
   
5. >>> import nltk.data
   
6. >>> from catchunked import CategorizedChunkedCorpusReader
   
7. >>> path = nltk.data.find('corpora/treebank/tagged')
   
8. >>> reader = CategorizedChunkedCorpusReader(path, r'wsj_.*\.pos', cat_pattern=r'wsj_(.*)\.pos')
   
9. >>> len(reader.categories()) == len(reader.fileids())
   
10. True
    
11. >>> len(reader.chunked_sents(categories=['0001']))
    
12. 16
    
13. 
    
14. >>> import nltk.data
    
15. >>> from catchunked import CategorizedConllChunkCorpusReader
    
16. >>> path = nltk.data.find('corpora/conll2000')
    
17. >>> reader = CategorizedConllChunkCorpusReader(path, r'.*\.txt', ('NP','VP','PP'), cat_pattern=r'(.*)\.txt')
    
18. >>> reader.categories()
    
19. ['test', 'train']
    
20. >>> reader.fileids()
    
21. ['test.txt', 'train.txt']
    
22. >>> len(reader.chunked_sents(categories=['test']))
    
23. 2012

复制代码

谢谢分享

谢谢 分享！