[GitHub]Practical Data Science Cookbook - Second Edition

是 否 +2 论坛币

k人 参与回答

经管之家送您一份

应届毕业生专属福利!

求职就业群

赵安豆老师微信：zhaoandou666

经管之家联合CDA

送您一个全额奖学金名额~ !

立即领取

感谢您参与论坛问题回答

经管之家送您两个论坛币！

+2 论坛币

本帖隐藏的内容

Practical-Data-Science-Cookbook-Second-Edition-master.zip (55.02 MB, 需要: 1 个论坛币)

2017-7-7 04:16:04 上传

需要: 1 个论坛币  [购买]

扫码加我 拉你入群

请注明：姓名-公司-职位

以便审核进群资格，未注明则拒绝

分享0 收藏0 回帖

关键词：Data Science Practical Cookbook Edition Science

1. '''
   
2. Understanding the Twitter API v1.1
   
3. '''
   
4. from twython import Twython
   
5. 
   
6. API_KEY = 'INSERT HERE'
   
7. API_SECRET = 'INSERT HERE'
   
8. 
   
9. ACCESS_TOKEN = 'INSERT HERE'
   
10. ACCESS_TOKEN_SECRET = 'INSERT HERE'
    
11. 
    
12. 
    
13. twitter = Twython(API_KEY, API_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
    
14. 
    
15. #testing our new twitter interface
    
16. print twitter.get_user_timeline()
    
17. 
    
18. print twitter.get_last_function_header('x-rate-limit-remaining')
    
19. 
    
20. 
    
21. #from the How it Works section:
    
22. def  twitter_oauth_login():
    
23.   API_KEY = 'INSERT HERE'
    
24.   API_SECRET = 'INSERT HERE'
    
25. 
    
26.   ACCESS_TOKEN = 'INSERT HERE'
    
27.   ACCESS_TOKEN_SECRET = 'INSERT HERE'
    
28. 
    
29.   twitter = Twython(API_KEY, API_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
    
30.   return(twitter)
    
31. 
    
32. 
    
33. '''
    
34. Determining your Twitter followers and friends
    
35. '''
    
36. 
    
37. twitter = twitter_oauth_login()
    
38. 
    
39. friends_ids = twitter.get_friends_ids(count=5000)
    
40. friends_ids = friends_ids['ids']
    
41. 
    
42. followers_ids = twitter.get_followers_ids(count=5000)
    
43. followers_ids = followers_ids['ids']
    
44. 
    
45. print len(friends_ids), len(followers_ids)
    
46. 
    
47. 
    
48. friends_set = set(friends_ids)
    
49. followers_set = set(followers_ids)
    
50. 
    
51. print('Number of Twitter users who either are our friend or follow you (union):')
    
52. print(len(friends_set.union(followers_set)))
    
53. 
    
54. len(friends_set | followers_set)
    
55. len(set(friends_ids+followers_ids))
    
56. 
    
57. print('Number of Twitter users who follow you and are your friend (intersection):')
    
58. print(len(friends_set & followers_set))
    
59. 
    
60. print("Number of Twitter users you follow that don't follow you (set difference):")
    
61. print(len(friends_set - followers_set))
    
62. 
    
63. print("Number of Twitter users who follow you that you don't follow (set difference):")
    
64. print(len(followers_set - friends_set))
    
65. 
    
66. 
    
67. '''
    
68. Pulling Twitter User Profiles
    
69. '''
    
70. 
    
71. 
    
72. def pull_users_profiles(ids):
    
73.     users = []
    
74.     for i in range(0, len(ids), 100):
    
75.         batch = ids[i:i + 100]
    
76.         users += twitter.lookup_user(user_id=batch)
    
77.         print(twitter.get_lastfunction_header('x-rate-limit-remaining'))
    
78.     return (users)
    
79. 
    
80. friends_profiles = pull_users_profiles(friends_ids)
    
81. followers_profiles = pull_users_profiles(followers_ids)
    
82. 
    
83. friends_screen_names = [p['screen_name'] for p in friends_profiles]
    
84. 
    
85. print friends_screen_names
    
86. 
    
87. #from There's more section
    
88. 
    
89. friends_screen_names = [p['screen_name'] for p in friends_profiles if 'screen_name' in p]
    
90. 
    
91. friends_screen_names = [p.get('screen_name',{}) for p in friends_profiles]
    
92. 
    
93. 
    
94. 
    
95. 
    
96. '''
    
97. Making requests without running afoul of Twitter's rate limit
    
98. '''
    
99. 
    
100. import time
     
101. import math
     
102. 
     
103. rate_limit_window = 15 * 60 #900 seconds
     
104. 
     
105. def pull_users_profiles_limit_aware(ids):
     
106.     users = []
     
107.     start_time = time.time()
     
108.     # Must look up users in
     
109.     for i in range(0, len(ids), 10):
     
110.         batch = ids[i:i + 10]
     
111.         users += twitter.lookup_user(user_id=batch)
     
112.         calls_left = float(twitter.get_lastfunction_header('x-rate-limit-remaining'))
     
113.         time_remaining_in_window = rate_limit_window - (time.time()-start_time)
     
114.         sleep_duration = math.ceil(time_remaining_in_window/calls_left)
     
115.         print('Sleeping for: ' + str(sleep_duration) + ' seconds; ' + str(calls_left) + ' API calls remaining')
     
116.         time.sleep(sleep_duration)
     
117. 
     
118.     return (users)
     
119. 
     
120. friends_profiles = pull_users_profiles_limit_aware(friends_ids)
     
121. followers_profiles = pull_users_profiles_limit_aware(followers_ids)
     
122. 
     
123. 
     
124. 
     
125. 
     
126. '''
     
127. Storing JSON data to disk
     
128. '''
     
129. 
     
130. import json
     
131. def save_json(filename, data):
     
132.     with open(filename, 'wb') as outfile:
     
133.         json.dump(data, outfile)
     
134. 
     
135. def load_json(filename):
     
136.     with open(filename) as infile:
     
137.         data = json.load(infile)
     
138.     return data
     
139. 
     
140. fname  = 'test_friends_profiles.json'
     
141. save_json(fname, friends_profiles)
     
142. 
     
143. test_reload = load_json(fname)
     
144. print(test_reload[0])
     
145. 
     
146. '''
     
147. Storing user profiles in MongoDB using PyMongo
     
148. '''
     
149. 
     
150. import pymongo
     
151. 
     
152. host_string = "mongodb://localhost"
     
153. port = 27017
     
154. mongo_client = pymongo.MongoClient(host_string, port)
     
155. 
     
156. # get a reference to the mongodb database 'test'
     
157. mongo_db = mongo_client['test']
     
158. 
     
159. # get a reference to the 'user profiles' collection in the 'test' database
     
160. user_profiles_collection = mongo_db['user_profiles']
     
161. 
     
162. user_profiles_collection.insert(friends_profiles)
     
163. user_profiles_collection.insert(followers_profiles)
     
164. 
     
165. 
     
166. #from How it works section
     
167. def save_json_data_to_mongo(data, mongo_db,
     
168.                             mongo_db_collection,
     
169.                             host_string = "localhost",
     
170.                             port = 27017):
     
171.     mongo_client = pymongo.MongoClient(host_string, port)
     
172.     mongo_db = mongo_client[mongo_db]
     
173.     collection = mongo_db[mongo_db_collection]
     
174.     inserted_object_ids = collection.insert(data)
     
175.     return(inserted_object_ids)
     
176. 
     
177. 
     
178. '''
     
179. Exploring geographic information available in profiles
     
180. '''
     
181. 
     
182. 
     
183. fname = 'test_friends_profiles.json'
     
184. load_json(fname)
     
185. 
     
186. 
     
187. geo_enabled = [p['geo_enabled'] for p in friends_profiles]
     
188. print geo_enabled.count(1)
     
189. 
     
190. 
     
191. location = [p['location'] for p in friends_profiles]
     
192. print location.count('')
     
193. 
     
194. 
     
195. print(set(location))
     
196. 
     
197. time_zone = [p['time_zone'] for p in friends_profiles]
     
198. print time_zone.count(None)
     
199. print(set(time_zone))
     
200. 
     
201. status_geo = [p['status']['geo'] for p in friends_profiles if ('status' in p and p['status']['geo'] is not None)]
     
202. if status_geo: print status_geo[0]
     
203. print len(status_geo)
     
204. 
     
205. 
     
206. 
     
207. '''
     
208. Plotting geo spatial data in Python
     
209. '''
     
210. 
     
211. status_geo = []
     
212. status_geo_screen_names = []
     
213. for fp in friends_profiles:
     
214.     if ('status' in fp and fp['status']['geo'] is not None and 'screen_name' in fp):
     
215.         status_geo.append(fp['status']['geo'])
     
216.         status_geo_screen_names.append(fp['screen_name'])
     
217. 
     
218. 
     
219. 
     
220. import folium
     
221. from itertools import izip
     
222. 
     
223. #Let Folium determine the scale
     
224. map = folium.Map(location=[48, -102], zoom_start=3)
     
225. 
     
226. for sg, sn in izip(status_geo, status_geo_screen_names):
     
227.     map.simple_marker(sg['coordinates'], popup=str(sn))
     
228. 
     
229. map.create_map(path='us_states.html')

复制代码

ReneeBK 发表于 2017-7-7 04:17
**** 本内容被作者隐藏 ****

必须学会！

kankan

看看      

thanks  for sharing

thanks for your sharing