Python Social Media Analytics

1. Stream API
   
2. In [ ]:
   
3. from pymongo import MongoClient
   
4. from requests_oauthlib import OAuth1
   
5. 
   
6. client = MongoClient('mongodb://localhost:27017/')
   
7. db = client['test']
   
8. collection = db['test']
   
9. 
   
10. url = 'https://stream.Twitter.com/1.1/statuses/filter.json'
    
11. auth = OAuth1(consumer_key, consumer_secret, access_token, access_token_secret)
    
12. 
    
13. pms = {'track' : 'premier league -filter:retweets AND -filter:replies', 'lang': 'en'}
    
14. 
    
15. res = requests.post(url, auth=auth, params = pms, stream = True)
    
16. 
    
17. for line in res.iter_lines():   
    
18.     if line:
    
19.         tweet = json.loads(line)
    
20.         try:
    
21.            collection.insert(tweet)
    
22.         except:
    
23.             pass

复制代码

1. Customized sentiment analysis
   
2. In [75]:
   
3. #You need to tag the dataset before
   
4. dataset  = pd.read_pickle('tagged.pickle')
   
5. 
   
6. classes = ['pos', 'neu', 'neg']
   
7. train_data = dataset['final'][0:80]
   
8. train_labels = dataset['label'][0:80]
   
9. test_data = dataset['final'][80:96]
   
10. test_labels = dataset['label'][80:96]
    
11. train_data = list(train_data.apply(' '.join))
    
12. test_data = list(test_data.apply(' '.join))
    
13. In [77]:
    
14. from sklearn.feature_extraction.text import TfidfVectorizer
    
15. from sklearn.naive_bayes import MultinomialNB
    
16. 
    
17. vectorizer = TfidfVectorizer(min_df=5, max_df = 0.8, sublinear_tf=True, use_idf=True)
    
18. 
    
19. train_vectors = vectorizer.fit_transform(train_data)
    
20. test_vectors = vectorizer.transform(test_data)
    
21. 
    
22. ### Perform a logistic regression model, and fit with X and y
    
23. 
    
24. nb = MultinomialNB()
    
25. 
    
26. nb.fit(train_vectors, train_labels).score(test_vectors, test_labels)

复制代码

Many thanks

1. NER Recognition
   
2. In [ ]:
   
3. from nltk.tag import StanfordNERTagger
   
4. from collections import Counter
   
5. import numpy as np
   
6. 
   
7. st = StanfordNERTagger('path_to_your_folder/english.all.3class.distsim.crf.ser.gz')
   
8. st.tag(sentence.split())
   
9. 
   
10. for r in tweets:
    
11.     lst_tags = st.tag(r.split())
    
12. 
    
13. for tup in lst_tags:
    
14.     if(tup[1] != 'O'):
    
15.         entities.append(tup)
    
16. In [ ]:
    
17. organizations = df_entities[df_entities['ner'].str.contains("ORGANIZATION")]
    
18. 
    
19. cnt = Counter(organizations['word'])
    
20. cnt.most_common(10)

复制代码

1. Getting the data
   
2. In [1]:
   
3. def get_statistics(video_id):
   
4. 
   
5.     url = "https://www.googleapis.com/youtube/v3/videos"
   
6.     pms = {'key': api_key, 'id': video_id, 'part':'contentDetails,statistics'}
   
7.     res = requests.get(url, params = pms)
   
8.     data = res.json()
   
9. 
   
10.     return(data)
    
11. In [3]:
    
12. def get_channel_videos(channel_id='UC-2Y8dQb0S6DtpxNgAKoJKA'):
    
13. 
    
14.         url = "https://www.googleapis.com/youtube/v3/search"
    
15. 
    
16.         pms = {'type': 'video', 'id' : id, 'key': api_key, 'channelId': channel_id, 'part':'snippet', 'order':'viewCount','maxResults':50}
    
17. 
    
18.         res = requests.get(url, params = pms)
    
19. 
    
20.         print("Connection status: %s" % res)
    
21. 
    
22.         data = res.json()
    
23. 
    
24.         #print(data)
    
25. 
    
26.         lst = []
    
27. 
    
28.         for video in data['items']:
    
29.                 video_stats = get_statistics(video['id']['videoId'])
    
30.                 #print(video['snippet']['title'])
    
31.                 results_json = {
    
32.                         'channelTitle' : video['snippet']['channelTitle'],
    
33.                         'title' : video['snippet']['title'],
    
34.                         'publishedAt' : video['snippet']['publishedAt'],
    
35.                         'videoId' : video['id']['videoId'],
    
36.                         'viewCount' : video_stats['items'][0]['statistics']['viewCount'],
    
37.                         'commentCount' : video_stats['items'][0]['statistics']['commentCount'],
    
38.                         'likeCount' : video_stats['items'][0]['statistics']['likeCount'],
    
39.                         'dislikeCount' : video_stats['items'][0]['statistics']['dislikeCount'],
    
40.                 }
    
41. 
    
42.                 lst.append(results_json)
    
43. 
    
44.         df = pd.read_json(json.dumps(lst))
    
45. 
    
46.         return(df)
    
47. In [5]:
    
48. import requests
    
49. 
    
50. url = 'https://www.googleapis.com/youtube/v3/videos'
    
51. 
    
52. pms = {'part': 'snippet, statistics', 'id' : 'YQUpg795iBo', 'key': 'xxx'}
    
53. 
    
54. res = requests.get(url, params = pms)
    
55. 
    
56. data = res.json()
    
57. In [6]:
    
58. url = 'https://www.googleapis.com/youtube/v3/commentThreads'
    
59. 
    
60. full_data = []                        # return list
    
61. page = ''                                # init paging
    
62. 
    
63. while True:
    
64. 
    
65.         pms = {'part': 'snippet', 'videoId' : id, 'maxResults' : 100, 'key': api_key, 'pageToken': page}
    
66.         res = requests.get(url, params = pms)
    
67. 
    
68.         print("Connection status: %s" % res)
    
69. 
    
70.         data = res.json()
    
71.         full_data.extend(data['items'])
    
72. 
    
73.         print("Just downloaded: %s, Total: %s" % (len(data['items']), len(full_data)))
    
74. 
    
75.         try:
    
76.                 page = data['nextPageToken']
    
77.         except:
    
78.                 break

复制代码

好书，学习了

谢谢分享

看看看