楼主: Lisrelchen
2954 19

Top 20 Python Machine Learning Open Source Projects [推广有奖]

11楼
Lisrelchen 发表于 2016-4-7 09:31:55
  1. import os, sys; sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))

  3. from pattern.web import Newsfeed, plaintext, URL
  4. from pattern.db  import date

  6. # This example reads a given RSS or Atom newsfeed channel.
  7. # Some example feeds to try out:
  8. NATURE  = "http://feeds.nature.com/nature/rss/current"
  9. SCIENCE = "http://www.sciencemag.org/rss/podcast.xml"
  10. NYT     = "http://rss.nytimes.com/services/xml/rss/nyt/GlobalHome.xml"
  11. TIME    = "http://feeds.feedburner.com/time/topstories"
  12. CNN     = "http://rss.cnn.com/rss/edition.rss"

  14. engine = Newsfeed()

  16. for result in engine.search(CNN, cached=True):
  17.     print result.title.upper()
  18.     print plaintext(result.text) # Remove HTML formatting.
  19.     print result.url
  20.     print result.date
  21.     print

  23. # News item URL's lead to the page with the full article.
  24. # This page can have any kind of formatting.
  25. # There is no default way to read it.
  26. # But we could just download the source HTML and convert it to plain text:

  28. #html = URL(result.url).download()
  29. #print plaintext(html)

  31. # The resulting text may contain a lot of garbage.
  32. # A better way is to use a DOM parser to select the HTML elements we want.
  33. # This is demonstrated in one of the next examples.
复制代码
12楼
Lisrelchen 发表于 2016-4-7 09:37:41
  1. import os, sys; sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))

  3. from pattern.web import Wikipedia

  5. # This example retrieves an article from Wikipedia (http://en.wikipedia.org).
  6. # Wikipedia queries request the article HTML source from the server. This can be slow.
  7. # It is a good idea to cache results from Wikipedia locally,
  8. # and to set a high timeout when calling Wikipedia.search().

  10. engine = Wikipedia(language="en")

  12. # Contrary to the other search engines in the pattern.web module,
  13. # Wikipedia simply returns one WikipediaArticle object (or None),
  14. # instead of a list of results.
  15. article = engine.search("alice in wonderland", cached=True, timeout=30)

  17. print article.title               # Article title (may differ from the search query).
  18. print
  19. print article.languages["fr"]     # Article in French, can be retrieved with Wikipedia(language="fr").
  20. print article.links[:10], "..."   # List of linked Wikipedia articles.
  21. print article.external[:5], "..." # List of external URL's.
  22. print

  24. #print article.source # The full article content as HTML.
  25. #print article.string # The full article content, plain text with HTML tags stripped.

  27. # An article is made up of different sections with a title.
  28. # WikipediaArticle.sections is a list of WikipediaSection objects.
  29. # Each section has a title + content and can have a linked parent section or child sections.
  30. for s in article.sections:
  31.     print s.title.upper()
  32.     print
  33.     print s.content # = ArticleSection.string, minus the title.
  34.     print
复制代码
13楼
Lisrelchen 发表于 2016-4-7 09:38:46
  1. import os, sys; sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))

  3. from pattern.web import Wiktionary, DOM
  4. from pattern.db import csv, pd

  6. # This example retrieves male and female given names from Wiktionary (http://en.wiktionary.org).
  7. # It then trains a classifier that can predict the gender of unknown names (about 78% correct).
  8. # The classifier is small (80KB) and fast.

  10. w = Wiktionary(language="en")
  11. f = csv() # csv() is a short alias for Datasheet().

  13. # Collect male and female given names from Wiktionary.
  14. # Store the data as (name, gender)-rows in a CSV-file.
  15. # The pd() function returns the parent directory of the current script,
  16. # so pd("given-names.csv") = pattern/examples/01-web/given-names.csv.

  18. for gender in ("male", "female"):
  19.     for ch in ("abcdefghijklmnopqrstuvwxyz"):
  20.         p = w.search("Appendix:%s_given_names/%s" % (gender.capitalize(), ch.capitalize()), cached=True)
  21.         for name in p.links:
  22.             if not name.startswith("Appendix:"):
  23.                 f.append((name, gender[0]))
  24.         f.save(pd("given-names.csv"))
  25.         print ch, gender

  27. # Create a classifier that predicts gender based on name.

  29. from pattern.vector import SVM, chngrams, count, kfoldcv

  31. class GenderByName(SVM):

  33.     def train(self, name, gender=None):
  34.         SVM.train(self, self.vector(name), gender)

  36.     def classify(self, name):
  37.         return SVM.classify(self, self.vector(name))

  39.     def vector(self, name):
  40.         """ Returns a dictionary with character bigrams and suffix.
  41.             For example, "Felix" => {"Fe":1, "el":1, "li":1, "ix":1, "ix$":1, 5:1}
  42.         """
  43.         v = chngrams(name, n=2)
  44.         v = count(v)
  45.         v[name[-2:]+"$"] = 1
  46.         v[len(name)] = 1
  47.         return v

  49. data = csv(pd("given-names.csv"))

  51. # Test average (accuracy, precision, recall, F-score, standard deviation).

  53. print kfoldcv(GenderByName, data, folds=3) # (0.81, 0.79, 0.77, 0.78, 0.00)

  55. # Train and save the classifier in the current folder.
  56. # With final=True, discards the original training data (= smaller file).

  58. g = GenderByName(train=data)
  59. g.save(pd("gender-by-name.svm"), final=True)

  61. # Next time, we can simply load the trained classifier.
  62. # Keep in mind that the script that loads the classifier
  63. # must include the code for the GenderByName class description,
  64. # otherwise Python won't know how to load the data.

  66. g = GenderByName.load(pd("gender-by-name.svm"))

  68. for name in (
  69.   "Felix",
  70.   "Felicia",
  71.   "Rover",
  72.   "Kitty",
  73.   "Legolas",
  74.   "Arwen",
  75.   "Jabba",
  76.   "Leia",
  77.   "Flash",
  78.   "Barbarella"):
  79.     print name, g.classify(name)

  81. # In the example above, Arwen and Jabba are misclassified.
  82. # We can of course improve the classifier by hand:

  84. #g.train("Arwen", gender="f")
  85. #g.train("Jabba", gender="m")
  86. #g.save(pd("gender-by-name.svm"), final=True)
  87. #print g.classify("Arwen")
  88. #print g.classify("Jabba")
复制代码
14楼
hanszhu 发表于 2016-4-7 10:12:15
  1. # -*- coding: utf-8 *-*
  2. import os, sys; sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))

  4. from pattern.web import Wikia

  6. # This example retrieves articled from Wikia (http://www.wikia.com).
  7. # Wikia is a collection of thousands of wikis based on MediaWiki.
  8. # Wikipedia is based on MediaWiki too.
  9. # Wikia queries request the article HTML source from the server. This can be slow.

  11. domain = "monkeyisland" # "Look behind you, a three-headed monkey!"

  13. # Alternatively, you can call this script from the commandline
  14. # and specify another domain: python 09-wikia.py "Bieberpedia".
  15. if len(sys.argv) > 1:
  16.     domain = sys.argv[1]

  18. w = Wikia(domain, language="en")

  20. # Like Wikipedia, we can search for articles by title with Wikia.search():
  21. print w.search("Three Headed Monkey")

  23. # However, we may not know exactly what kind of articles exist,
  24. # three-headed monkey" for example does not redirect to the above article.

  26. # We can iterate through all articles with the Wikia.articles() method
  27. # (note that Wikipedia also has a Wikipedia.articles() method).
  28. # The "count" parameter sets the number of article titles to retrieve per query.
  29. # Retrieving the full article for each article takes another query. This can be slow.
  30. i = 0
  31. for article in w.articles(count=2, cached=True):
  32.     print
  33.     print article.title
  34.     #print article.plaintext()
  35.     i += 1
  36.     if i >= 3:
  37.         break

  39. # Alternatively, we can retrieve just the titles,
  40. # and only retrieve the full articles for the titles we need:
  41. i = 0
  42. for title in w.index(count=2):
  43.     print
  44.     print title
  45.     #article = w.search(title)
  46.     #print article.plaintext()
  47.     i += 1
  48.     if i >= 3:
  49.         break
复制代码
15楼
hanszhu 发表于 2016-4-7 10:13:13
  1. # -*- coding: utf-8 *-*
  2. import os, sys; sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))

  4. from pattern.web import DBPedia

  6. dbp = DBPedia()

  8. # DBPedia is a database of structured information mined from Wikipedia.
  9. # DBPedia data is stored as RDF triples: (subject, predicate, object),
  10. # e.g., X is-a Actor, Y is-a Country, Z has-birthplace Country, ...
  11. # If you know about pattern.graph (or graphs in general),
  12. # this triple format should look familiar.

  14. # DBPedia can be queried using SPARQL:
  15. # http://dbpedia.org/sparql
  16. # http://www.w3.org/TR/rdf-sparql-query/
  17. # A SPARQL query yields rows that match all triples in the WHERE clause.
  18. # A SPARQL query uses ?wildcards in triple subject/object to select fields.

  20. # 1) Search DBPedia for actors.

  22. # Variables are indicated with a "?" prefix.
  23. # Variables will be bound to the corresponding part of each matched triple.
  24. # The "a" is short for "is of the class".
  25. # The "prefix" statement creates a shorthand for a given namespace.
  26. # To see what semantic constraints are available in "dbo" (for example):
  27. # http://dbpedia.org/ontology/
  28. q = """
  29. prefix dbo: <http://dbpedia.org/ontology/>
  30. select ?actor where {
  31.     ?actor a dbo:Actor.
  32. }
  33. """
  34. for result in dbp.search(q, start=1, count=10):
  35.     print result.actor
  36. print
  37.    
  38. # You may notice that each Result.actor is of the form:
  39. # "http://dbpedia.org/resource/[NAME]"
  40. # This kind of string is a subclass of unicode: DBPediaResource.
  41. # DBPediaResource has a DBPediaResource.name property (see below).

  43. # 2) Search DBPedia for actors and their place of birth.

  45. q = """
  46. prefix dbo: <http://dbpedia.org/ontology/>
  47. select ?actor ?place where {
  48.     ?actor a dbo:Actor.
  49.     ?actor dbo:birthPlace ?place.
  50. }
  51. order by ?actor
  52. """
  53. for r in dbp.search(q, start=1, count=10):
  54.     print "%s (%s)" % (r.actor.name, r.place.name)
  55. print

  57. # You will notice that the results now include duplicates,
  58. # the same actor with a city name, and with a country name.
  59. # We could refine ?place by including the following triple:
  60. # "?place a dbo:Country."

  62. # 3) Search DBPedia for actors born in 1970.

  64. # Each result must match both triples, i.e.,
  65. # X is an actor + X is born on Y.
  66. # We don't want to filter by month and day (e.g., "1970-12-31"),
  67. # so we use a regular expression instead with filter():
  68. q = """
  69. prefix dbo: <http://dbpedia.org/ontology/>
  70. select ?actor ?date where {
  71.     ?actor a dbo:Actor.
  72.     ?actor dbo:birthDate ?date.
  73.     filter(regex(str(?date), "1970-..-.."))
  74. }
  75. order by ?date
  76. """
  77. for r in dbp.search(q, start=1, count=10):
  78.     print "%s (%s)" % (r.actor.name, r.date)
  79. print

  81. # We could also make this query shorter,
  82. # by combining the two ?actor triples into one:
  83. # "?actor a dbo:Actor; dbo:birthDate ?date."

  85. # 4) A more advanced example, in German:

  87. q = """
  88. prefix dbo: <http://dbpedia.org/ontology/>
  89. prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>
  90. select ?actor ?place where {
  91.     ?_actor a dbo:Actor.
  92.     ?_actor dbo:birthPlace ?_place.
  93.     ?_actor rdfs:label ?actor.
  94.     ?_place rdfs:label ?place.
  95.     filter(lang(?actor) = "de" && lang(?place) = "de")
  96. }
  97. order by ?actor
  98. """
  99. for r in dbp.search(q, start=1, count=10):
  100.     print "%s (%s)" % (r.actor, r.place)
  101. print

  103. # This extracts a German label for each matched DBPedia resource.
  104. # - X is an actor,
  105. # - X is born in Y,
  106. # - X has a German label A,
  107. # - Y has a German label B,
  108. # - Retrieve A and B.

  110. # For example, say one of the matched resources was:
  111. # "<http://dbpedia.org/page/Erwin_Schrödinger>"
  112. # If you open this URL in a browser,
  113. # you will see all the available semantic properties and their values.
  114. # One of the properties is "rdfs:label": a human-readable & multilingual label.

  116. # 5) Find triples involving cats.

  118. # <http://purl.org/dc/terms/subject>
  119. # means: "is in the category of".
  120. q = """
  121. prefix dbo: <http://dbpedia.org/ontology/>
  122. prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>
  123. select ?cat ?relation ?concept where {
  124.     ?cat <http://purl.org/dc/terms/subject> <http://dbpedia.org/resource/Category:Felis>.
  125.     ?cat ?_relation ?_concept.
  126.     ?_relation rdfs:label ?relation.
  127.     ?_concept rdfs:label ?concept.
  128.     filter(lang(?relation) = "en" && lang(?concept) = "en")
  129. } order by ?cat
  130. """
  131. for r in dbp.search(q, start=1, count=10):
  132.     print "%s ---%s--> %s" % (r.cat.name, r.relation.ljust(10, "-"), r.concept)
  133. print

  135. # 6) People whose first name includes "édouard"

  137. q = u"""
  138. prefix dbo: <http://dbpedia.org/ontology/>
  139. prefix foaf: <http://xmlns.com/foaf/0.1/>
  140. select ?person ?name where {
  141.     ?person a dbo:Person.
  142.     ?person foaf:givenName ?name.
  143.     filter(regex(?name, "édouard"))
  144. }
  145. """
  146. for result in dbp.search(q, start=1, count=10, cached=False):
  147.     print "%s (%s)" % (result.person.name, result.name)
  148. print
复制代码
16楼
hanszhu 发表于 2016-4-7 10:14:13
  1. import os, sys; sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))

  3. from pattern.web import Facebook, NEWS, COMMENTS, LIKES
  4. from pattern.db  import Datasheet, pprint, pd

  6. # The Facebook API can be used to search public status updates (no license needed).

  8. # It can also be used to get status updates, comments and persons that liked it,
  9. # from a given profile or product page.
  10. # This requires a personal license key.
  11. # If you are logged in to Facebook, you can get a license key here:
  12. # http://www.clips.ua.ac.be/pattern-facebook
  13. # (We don't / can't store your information).

  15. # 1) Searching for public status updates.
  16. #    Search for all status updates that contain the word "horrible".

  18. try:
  19.     # We'll store the status updates in a Datasheet.
  20.     # A Datasheet is a table of rows and columns that can be exported as a CSV-file.
  21.     # In the first column, we'll store a unique id for each status update.
  22.     # We only want to add new status updates, i.e., those we haven't seen yet.
  23.     # With an index on the first column we can quickly check if an id already exists.
  24.     table = Datasheet.load(pd("opinions.csv"))
  25.     index = set(table.columns[0])
  26. except:
  27.     table = Datasheet()
  28.     index = set()

  30. fb = Facebook()

  32. # With Facebook.search(cached=False), a "live" request is sent to Facebook:
  33. # we get the most recent results instead of those in the local cache.
  34. # Keeping a local cache can also be useful (e.g., while testing)
  35. # because a query is instant when it is executed the second time.
  36. for status in fb.search("horrible", count=25, cached=False):
  37.     print "=" * 100
  38.     print status.id
  39.     print status.text
  40.     print status.author # Yields an (id, name)-tuple.
  41.     print status.date
  42.     print status.likes
  43.     print status.comments
  44.     print
  45.     # Only add the tweet to the table if it doesn't already exists.
  46.     if len(table) == 0 or status.id not in index:
  47.         table.append([status.id, status.text])
  48.         index.add(status.id)

  50. # Create a .csv in pattern/examples/01-web/
  51. table.save(pd("opinions.csv"))

  53. # 2) Status updates from specific profiles.
  54. #    For this you need a personal license key:
  55. #    http://www.clips.ua.ac.be/pattern-facebook

  57. license = ""

  59. if license != "":
  60.     fb = Facebook(license)
  61.     # Facebook.profile() returns a dictionary with author info.
  62.     # By default, this is your own profile.
  63.     # You can also supply the id of another profile,
  64.     # or the name of a product page.
  65.     me = fb.profile()["id"]
  66.     for status in fb.search(me, type=NEWS, count=30, cached=False):
  67.         print "-" * 100
  68.         print status.id    # Status update unique id.
  69.         print status.title # Status title (i.e., the id of the page or event given as URL).
  70.         print status.text  # Status update text.
  71.         print status.url   # Status update image, external link, ...
  72.         if status.comments > 0:
  73.             # Retrieve comments on the status update.
  74.             print "%s comments:" % status.comments
  75.             print [(x.author, x.text, x.likes) for x in fb.search(status.id, type=COMMENTS)]
  76.         if status.likes > 0:
  77.             # Retrieve likes on the status update.
  78.             print "%s likes:" % status.likes
  79.             print [x.author for x in fb.search(status.id, type=LIKES)]
  80.         print
复制代码
17楼
hanszhu 发表于 2016-4-7 10:15:57
  1. import os, sys; sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))

  3. from pattern.web import URL, DOM, plaintext
  4. from pattern.web import NODE, TEXT, COMMENT, ELEMENT, DOCUMENT

  6. # The pattern.web module has a number of convenient search engines, as demonstrated.
  7. # But often you will need to handle the HTML in web pages of your interest manually.
  8. # The DOM object can be used for this, similar to the Javascript DOM.
  9. # The DOM (Document Object Model) parses a string of HTML
  10. # and returns a tree of nested Element objects.
  11. # The DOM elements can then be searched by tag name, CSS id, CSS class, ...

  13. # For example, top news entries on Reddit are coded as:
  14. # <div class="entry">
  15. #     <p class="title">
  16. #         <a class="title " href="http://i.imgur.com/yDyPu8P.jpg">Bagel the bengal, destroyer of boxes</a>
  17. #     ...
  18. # </div>
  19. #
  20. # ... which - naturally - is a picture of a cat.
  21. url = URL("http://www.reddit.com/top/")
  22. dom = DOM(url.download(cached=True))
  23. #print dom.body.content
  24. for e in dom.by_tag("div.entry")[:5]: # Top 5 reddit entries.
  25.     for a in e.by_tag("a.title")[:1]: # First <a class="title"> in entry.
  26.         print plaintext(a.content)
  27.         print a.attrs["href"]
  28.         print
  29.         
  30. # The links in the HTML source code may be relative,
  31. # e.g., "../img.jpg" instead of "www.domain.com/img.jpg".
  32. # We can get the absolute URL by prepending the base URL.
  33. # However, this can get messy with anchors, trailing slashes and redirected URL's.
  34. # A good way to get absolute URL's is to use the module's abs() function:
  35. from pattern.web import abs
  36. url = URL("http://nodebox.net")
  37. for link in DOM(url.download()).by_tag("a"):
  38.     link = link.attrs.get("href","")
  39.     link = abs(link, base=url.redirect or url.string)
  40.     #print link

  42. # The DOM object is a tree of nested Element and Text objects.
  43. # All objects inherit from Node (check the source code).

  45. # Node.type       : NODE, TEXT, COMMENT, ELEMENT or DOM
  46. # Node.parent     : Parent Node object.
  47. # Node.children   : List of child Node objects.
  48. # Node.next       : Next Node in Node.parent.children.
  49. # Node.previous   : Previous Node in Node.parent.children.

  51. # DOM.head        : Element with tag name "head".
  52. # DOM.body        : Element with tag name "body".

  54. # Element.tag     : Element tag name, e.g. "body".
  55. # Element.attrs   : Dictionary of tag attributes, e.g. {"class": "header"}
  56. # Element.content : Element HTML content as a string.
  57. # Element.source  : Element tag + content

  59. # Element.get_element_by_id(value)
  60. # Element.get_elements_by_tagname(value)
  61. # Element.get_elements_by_classname(value)
  62. # Element.get_elements_by_attribute(name=value)

  64. # You can also use shorter aliases (we prefer them):
  65. # Element.by_id(), by_tag(), by_class(), by_attr().

  67. # The tag name passed to Element.by_tag() can include
  68. # a class (e.g., "div.message") or an id (e.g., "div#header").

  70. # For example:
  71. # In the <head> tag, retrieve the <meta name="keywords"> element.
  72. # Get the string value of its "content" attribute and split into a list:
  73. dom = DOM(URL("http://www.clips.ua.ac.be").download())
  74. kw = dom.head.by_attr(name="keywords")[0]
  75. kw = kw.attrs["content"]
  76. kw = [x.strip() for x in kw.split(",")]
  77. print kw
  78. print

  80. # If you know CSS, you can also use short and handy CSS selectors:
  81. # http://www.w3.org/TR/CSS2/selector.html
  82. # Element(selector) will return a list of nested elements that match the given string.
  83. dom = DOM(URL("http://www.clips.ua.ac.be").download())
  84. for e in dom("div#sidebar-left li div:first-child span"):
  85.     print plaintext(e.content)
  86.     print
复制代码
18楼
hanszhu 发表于 2016-4-7 10:17:50

19楼
Nicolle 学生认证  发表于 2016-4-7 10:37:25
提示: 作者被禁止或删除 内容自动屏蔽
20楼
Nicolle 学生认证  发表于 2016-4-7 10:39:15
提示: 作者被禁止或删除 内容自动屏蔽
返回列表
12
高级模式
B Color Image Link Quote Code Smilies
您需要登录后才可以回帖 登录 | 我要注册

本版微信群
加好友,备注jltj
拉您入交流群

京ICP备16021002号-2 京B2-20170662号 京公网安备 11010802022788号 论坛法律顾问：王进律师 知识产权保护声明   免责及隐私声明

GMT+8, 2026-1-3 01:14