Programming Collective Intelligence

1. import time
   
2. import urllib2
   
3. import xml.dom.minidom
   
4. 
   
5. kayakkey='YOUR KEY HERE'
   
6. 
   
7. def getkayaksession():
   
8.   # Construct the URL to start a session
   
9.   url='http://www.kayak.com/k/ident/apisession?token=%s&version=1' % kayakkey
   
10.   
    
11.   # Parse the resulting XML
    
12.   doc=xml.dom.minidom.parseString(urllib2.urlopen(url).read())
    
13.   
    
14.   # Find <sid>xxxxxxxx</sid>
    
15.   sid=doc.getElementsByTagName('sid')[0].firstChild.data
    
16.   return sid
    
17. 
    
18. def flightsearch(sid,origin,destination,depart_date):
    
19.   
    
20.   # Construct search URL
    
21.   url='http://www.kayak.com/s/apisearch?basicmode=true&oneway=y&origin=%s' % origin
    
22.   url+='&destination=%s&depart_date=%s' % (destination,depart_date)
    
23.   url+='&return_date=none&depart_time=a&return_time=a'
    
24.   url+='&travelers=1&cabin=e&action=doFlights&apimode=1'
    
25.   url+='&_sid_=%s&version=1' % (sid)
    
26. 
    
27.   # Get the XML
    
28.   doc=xml.dom.minidom.parseString(urllib2.urlopen(url).read())
    
29. 
    
30.   # Extract the search ID
    
31.   searchid=doc.getElementsByTagName('searchid')[0].firstChild.data
    
32. 
    
33.   return searchid
    
34. 
    
35. def flightsearchresults(sid,searchid):
    
36.   def parseprice(p):
    
37.     return float(p[1:].replace(',',''))
    
38. 
    
39.   # Polling loop
    
40.   while 1:
    
41.     time.sleep(2)
    
42. 
    
43.     # Construct URL for polling
    
44.     url='http://www.kayak.com/s/basic/flight?'
    
45.     url+='searchid=%s&c=5&apimode=1&_sid_=%s&version=1' % (searchid,sid)
    
46.     doc=xml.dom.minidom.parseString(urllib2.urlopen(url).read())
    
47. 
    
48.     # Look for morepending tag, and wait until it is no longer true
    
49.     morepending=doc.getElementsByTagName('morepending')[0].firstChild
    
50.     if morepending==None or morepending.data=='false': break
    
51. 
    
52.   # Now download the complete list
    
53.   url='http://www.kayak.com/s/basic/flight?'
    
54.   url+='searchid=%s&c=999&apimode=1&_sid_=%s&version=1' % (searchid,sid)
    
55.   doc=xml.dom.minidom.parseString(urllib2.urlopen(url).read())
    
56. 
    
57.   # Get the various elements as lists
    
58.   prices=doc.getElementsByTagName('price')
    
59.   departures=doc.getElementsByTagName('depart')
    
60.   arrivals=doc.getElementsByTagName('arrive')  
    
61. 
    
62.   # Zip them together
    
63.   return zip([p.firstChild.data.split(' ')[1] for p in departures],
    
64.              [p.firstChild.data.split(' ')[1] for p in arrivals],
    
65.              [parseprice(p.firstChild.data) for p in prices])
    
66. 
    
67. 
    
68. def createschedule(people,dest,dep,ret):
    
69.   # Get a session id for these searches
    
70.   sid=getkayaksession()
    
71.   flights={}
    
72.   
    
73.   for p in people:
    
74.     name,origin=p
    
75.     # Outbound flight
    
76.     searchid=flightsearch(sid,origin,dest,dep)
    
77.     flights[(origin,dest)]=flightsearchresults(sid,searchid)
    
78.    
    
79.     # Return flight
    
80.     searchid=flightsearch(sid,dest,origin,ret)
    
81.     flights[(dest,origin)]=flightsearchresults(sid,searchid)
    
82.    
    
83.   return flights

复制代码

1. import random
   
2. import math
   
3. 
   
4. # The dorms, each of which has two available spaces
   
5. dorms=['Zeus','Athena','Hercules','Bacchus','Pluto']
   
6. 
   
7. # People, along with their first and second choices
   
8. prefs=[('Toby', ('Bacchus', 'Hercules')),
   
9.        ('Steve', ('Zeus', 'Pluto')),
   
10.        ('Karen', ('Athena', 'Zeus')),
    
11.        ('Sarah', ('Zeus', 'Pluto')),
    
12.        ('Dave', ('Athena', 'Bacchus')),
    
13.        ('Jeff', ('Hercules', 'Pluto')),
    
14.        ('Fred', ('Pluto', 'Athena')),
    
15.        ('Suzie', ('Bacchus', 'Hercules')),
    
16.        ('Laura', ('Bacchus', 'Hercules')),
    
17.        ('James', ('Hercules', 'Athena'))]
    
18. 
    
19. # [(0,9),(0,8),(0,7),(0,6),...,(0,0)]
    
20. domain=[(0,(len(dorms)*2)-i-1) for i in range(0,len(dorms)*2)]
    
21. 
    
22. def printsolution(vec):
    
23.   slots=[]
    
24.   # Create two slots for each dorm
    
25.   for i in range(len(dorms)): slots+=[i,i]
    
26. 
    
27.   # Loop over each students assignment
    
28.   for i in range(len(vec)):
    
29.     x=int(vec[i])
    
30. 
    
31.     # Choose the slot from the remaining ones
    
32.     dorm=dorms[slots[x]]
    
33.     # Show the student and assigned dorm
    
34.     print prefs[i][0],dorm
    
35.     # Remove this slot
    
36.     del slots[x]
    
37. 
    
38. def dormcost(vec):
    
39.   cost=0
    
40.   # Create list a of slots
    
41.   slots=[0,0,1,1,2,2,3,3,4,4]
    
42. 
    
43.   # Loop over each student
    
44.   for i in range(len(vec)):
    
45.     x=int(vec[i])
    
46.     dorm=dorms[slots[x]]
    
47.     pref=prefs[i][1]
    
48.     # First choice costs 0, second choice costs 1
    
49.     if pref[0]==dorm: cost+=0
    
50.     elif pref[1]==dorm: cost+=1
    
51.     else: cost+=3
    
52.     # Not on the list costs 3
    
53. 
    
54.     # Remove selected slot
    
55.     del slots[x]
    
56.    
    
57.   return cost

复制代码

1. import feedparser
   
2. import re
   
3. 
   
4. # Takes a filename of URL of a blog feed and classifies the entries
   
5. def read(feed,classifier):
   
6.   # Get feed entries and loop over them
   
7.   f=feedparser.parse(feed)
   
8.   for entry in f['entries']:
   
9.     print
   
10.     print '-----'
    
11.     # Print the contents of the entry
    
12.     print 'Title:     '+entry['title'].encode('utf-8')
    
13.     print 'Publisher: '+entry['publisher'].encode('utf-8')
    
14.     print
    
15.     print entry['summary'].encode('utf-8')
    
16.    
    
17. 
    
18.     # Combine all the text to create one item for the classifier
    
19.     fulltext='%s\n%s\n%s' % (entry['title'],entry['publisher'],entry['summary'])
    
20. 
    
21.     # Print the best guess at the current category
    
22.     print 'Guess: '+str(classifier.classify(entry))
    
23. 
    
24.     # Ask the user to specify the correct category and train on that
    
25.     cl=raw_input('Enter category: ')
    
26.     classifier.train(entry,cl)
    
27. 
    
28. 
    
29. def entryfeatures(entry):
    
30.   splitter=re.compile('\\W*')
    
31.   f={}
    
32.   
    
33.   # Extract the title words and annotate
    
34.   titlewords=[s.lower() for s in splitter.split(entry['title'])
    
35.           if len(s)>2 and len(s)<20]
    
36.   for w in titlewords: f['Title:'+w]=1
    
37.   
    
38.   # Extract the summary words
    
39.   summarywords=[s.lower() for s in splitter.split(entry['summary'])
    
40.           if len(s)>2 and len(s)<20]
    
41. 
    
42.   # Count uppercase words
    
43.   uc=0
    
44.   for i in range(len(summarywords)):
    
45.     w=summarywords[i]
    
46.     f[w]=1
    
47.     if w.isupper(): uc+=1
    
48.    
    
49.     # Get word pairs in summary as features
    
50.     if i<len(summarywords)-1:
    
51.       twowords=' '.join(summarywords[i:i+1])
    
52.       f[twowords]=1
    
53.    
    
54.   # Keep creator and publisher whole
    
55.   f['Publisher:'+entry['publisher']]=1
    
56. 
    
57.   # UPPERCASE is a virtual word flagging too much shouting  
    
58.   if float(uc)/len(summarywords)>0.3: f['UPPERCASE']=1
    
59.   
    
60.   return f

复制代码

1. from pysqlite2 import dbapi2 as sqlite
   
2. import re
   
3. import math
   
4. 
   
5. def getwords(doc):
   
6.   splitter=re.compile('\\W*')
   
7.   print doc
   
8.   # Split the words by non-alpha characters
   
9.   words=[s.lower() for s in splitter.split(doc)
   
10.           if len(s)>2 and len(s)<20]
    
11.   
    
12.   # Return the unique set of words only
    
13.   return dict([(w,1) for w in words])
    
14. 
    
15. class classifier:
    
16.   def __init__(self,getfeatures,filename=None):
    
17.     # Counts of feature/category combinations
    
18.     self.fc={}
    
19.     # Counts of documents in each category
    
20.     self.cc={}
    
21.     self.getfeatures=getfeatures
    
22.    
    
23.   def setdb(self,dbfile):
    
24.     self.con=sqlite.connect(dbfile)   
    
25.     self.con.execute('create table if not exists fc(feature,category,count)')
    
26.     self.con.execute('create table if not exists cc(category,count)')
    
27. 
    
28. 
    
29.   def incf(self,f,cat):
    
30.     count=self.fcount(f,cat)
    
31.     if count==0:
    
32.       self.con.execute("insert into fc values ('%s','%s',1)"
    
33.                        % (f,cat))
    
34.     else:
    
35.       self.con.execute(
    
36.         "update fc set count=%d where feature='%s' and category='%s'"
    
37.         % (count+1,f,cat))
    
38.   
    
39.   def fcount(self,f,cat):
    
40.     res=self.con.execute(
    
41.       'select count from fc where feature="%s" and category="%s"'
    
42.       %(f,cat)).fetchone()
    
43.     if res==None: return 0
    
44.     else: return float(res[0])
    
45. 
    
46.   def incc(self,cat):
    
47.     count=self.catcount(cat)
    
48.     if count==0:
    
49.       self.con.execute("insert into cc values ('%s',1)" % (cat))
    
50.     else:
    
51.       self.con.execute("update cc set count=%d where category='%s'"
    
52.                        % (count+1,cat))   
    
53. 
    
54.   def catcount(self,cat):
    
55.     res=self.con.execute('select count from cc where category="%s"'
    
56.                          %(cat)).fetchone()
    
57.     if res==None: return 0
    
58.     else: return float(res[0])
    
59. 
    
60.   def categories(self):
    
61.     cur=self.con.execute('select category from cc');
    
62.     return [d[0] for d in cur]
    
63. 
    
64.   def totalcount(self):
    
65.     res=self.con.execute('select sum(count) from cc').fetchone();
    
66.     if res==None: return 0
    
67.     return res[0]
    
68. 
    
69. 
    
70.   def train(self,item,cat):
    
71.     features=self.getfeatures(item)
    
72.     # Increment the count for every feature with this category
    
73.     for f in features:
    
74.       self.incf(f,cat)
    
75. 
    
76.     # Increment the count for this category
    
77.     self.incc(cat)
    
78.     self.con.commit()
    
79. 
    
80.   def fprob(self,f,cat):
    
81.     if self.catcount(cat)==0: return 0
    
82. 
    
83.     # The total number of times this feature appeared in this
    
84.     # category divided by the total number of items in this category
    
85.     return self.fcount(f,cat)/self.catcount(cat)
    
86. 
    
87.   def weightedprob(self,f,cat,prf,weight=1.0,ap=0.5):
    
88.     # Calculate current probability
    
89.     basicprob=prf(f,cat)
    
90. 
    
91.     # Count the number of times this feature has appeared in
    
92.     # all categories
    
93.     totals=sum([self.fcount(f,c) for c in self.categories()])
    
94. 
    
95.     # Calculate the weighted average
    
96.     bp=((weight*ap)+(totals*basicprob))/(weight+totals)
    
97.     return bp
    
98. 
    
99. 
    
100. 
     
101. 
     
102. class naivebayes(classifier):
     
103.   
     
104.   def __init__(self,getfeatures):
     
105.     classifier.__init__(self,getfeatures)
     
106.     self.thresholds={}
     
107.   
     
108.   def docprob(self,item,cat):
     
109.     features=self.getfeatures(item)   
     
110. 
     
111.     # Multiply the probabilities of all the features together
     
112.     p=1
     
113.     for f in features: p*=self.weightedprob(f,cat,self.fprob)
     
114.     return p
     
115. 
     
116.   def prob(self,item,cat):
     
117.     catprob=self.catcount(cat)/self.totalcount()
     
118.     docprob=self.docprob(item,cat)
     
119.     return docprob*catprob
     
120.   
     
121.   def setthreshold(self,cat,t):
     
122.     self.thresholds[cat]=t
     
123.    
     
124.   def getthreshold(self,cat):
     
125.     if cat not in self.thresholds: return 1.0
     
126.     return self.thresholds[cat]
     
127.   
     
128.   def classify(self,item,default=None):
     
129.     probs={}
     
130.     # Find the category with the highest probability
     
131.     max=0.0
     
132.     for cat in self.categories():
     
133.       probs[cat]=self.prob(item,cat)
     
134.       if probs[cat]>max:
     
135.         max=probs[cat]
     
136.         best=cat
     
137. 
     
138.     # Make sure the probability exceeds threshold*next best
     
139.     for cat in probs:
     
140.       if cat==best: continue
     
141.       if probs[cat]*self.getthreshold(best)>probs[best]: return default
     
142.     return best
     
143. 
     
144. class fisherclassifier(classifier):
     
145.   def cprob(self,f,cat):
     
146.     # The frequency of this feature in this category   
     
147.     clf=self.fprob(f,cat)
     
148.     if clf==0: return 0
     
149. 
     
150.     # The frequency of this feature in all the categories
     
151.     freqsum=sum([self.fprob(f,c) for c in self.categories()])
     
152. 
     
153.     # The probability is the frequency in this category divided by
     
154.     # the overall frequency
     
155.     p=clf/(freqsum)
     
156.    
     
157.     return p
     
158.   def fisherprob(self,item,cat):
     
159.     # Multiply all the probabilities together
     
160.     p=1
     
161.     features=self.getfeatures(item)
     
162.     for f in features:
     
163.       p*=(self.weightedprob(f,cat,self.cprob))
     
164. 
     
165.     # Take the natural log and multiply by -2
     
166.     fscore=-2*math.log(p)
     
167. 
     
168.     # Use the inverse chi2 function to get a probability
     
169.     return self.invchi2(fscore,len(features)*2)
     
170.   def invchi2(self,chi, df):
     
171.     m = chi / 2.0
     
172.     sum = term = math.exp(-m)
     
173.     for i in range(1, df//2):
     
174.         term *= m / i
     
175.         sum += term
     
176.     return min(sum, 1.0)
     
177.   def __init__(self,getfeatures):
     
178.     classifier.__init__(self,getfeatures)
     
179.     self.minimums={}
     
180. 
     
181.   def setminimum(self,cat,min):
     
182.     self.minimums[cat]=min
     
183.   
     
184.   def getminimum(self,cat):
     
185.     if cat not in self.minimums: return 0
     
186.     return self.minimums[cat]
     
187.   def classify(self,item,default=None):
     
188.     # Loop through looking for the best result
     
189.     best=default
     
190.     max=0.0
     
191.     for c in self.categories():
     
192.       p=self.fisherprob(item,c)
     
193.       # Make sure it exceeds its minimum
     
194.       if p>self.getminimum(c) and p>max:
     
195.         best=c
     
196.         max=p
     
197.     return best
     
198. 
     
199. 
     
200. def sampletrain(cl):
     
201.   cl.train('Nobody owns the water.','good')
     
202.   cl.train('the quick rabbit jumps fences','good')
     
203.   cl.train('buy pharmaceuticals now','bad')
     
204.   cl.train('make quick money at the online casino','bad')
     
205.   cl.train('the quick brown fox jumps','good')

复制代码

have a look

不明觉厉

一看就好高大上。。。。。

好书，感谢分享

xie xie
xie xie
xie xie