Chalk: Scala Library for Natural Language Processing Library

是 否 +2 论坛币

k人 参与回答

经管之家送您一份

应届毕业生专属福利!

求职就业群

赵安豆老师微信：zhaoandou666

经管之家联合CDA

送您一个全额奖学金名额~ !

立即领取

感谢您参与论坛问题回答

经管之家送您两个论坛币！

+2 论坛币

本帖隐藏的内容

https://github.com/scalanlp/chalk

扫码加我 拉你入群

请注明：姓名-公司-职位

以便审核进群资格，未注明则拒绝

分享0 收藏0 回帖

关键词：Processing Language processI Library Process Natural

1. package chalk.corpora
   
2. 
   
3. import io.Source
   
4. import collection.mutable.ArrayBuffer
   
5. import nak.data.{Observation, Example}
   
6. import java.io.{File, FileInputStream, InputStream}
   
7. 
   
8. /**
   
9. * Reads tag sequences in the conll shared task format. See http://mlcomp.org/faq/domains "Sequence Tagging" for the spec.
   
10. * @author dlwh
    
11. */
    
12. object CONLLSequenceReader {
    
13.   def readTrain(f: InputStream, name: String = "sequence", splitToken:String =" "):Iterator[Example[IndexedSeq[String],IndexedSeq[IndexedSeq[String]]]] = {
    
14.     val source = Source.fromInputStream(f).getLines()
    
15.     new Iterator[Example[IndexedSeq[String],IndexedSeq[IndexedSeq[String]]]] {
    
16.       def hasNext = source.hasNext
    
17.       var index = 0
    
18.       def next():Example[IndexedSeq[String],IndexedSeq[IndexedSeq[String]]] = {
    
19.         val inputs = new ArrayBuffer[IndexedSeq[String]]()
    
20.         val outputs = new ArrayBuffer[String]
    
21.         import scala.util.control.Breaks._
    
22.         breakable {
    
23.           while(source.hasNext) {
    
24.             val line = source.next()
    
25.             if(line.trim().isEmpty) break
    
26. 
    
27.             val split = line.split(splitToken)
    
28.             inputs += split.take(split.length -1).toIndexedSeq
    
29.             outputs += split.last
    
30.           }
    
31.         }
    
32.         val id = name + "-" + index
    
33.         index += 1
    
34.         Example(outputs, inputs, id)
    
35.       }
    
36.     }
    
37.   }
    
38. 
    
39.   /**
    
40.    * This format reads a CONLL file with the last column (i.e. the label) missing. If you have the label,
    
41.    * use readTrain, even if you plan on testing with it. Silly, I know.
    
42.    * @param f
    
43.    * @param name
    
44.    * @return
    
45.    */
    
46.   def readTest(f: InputStream, name: String = "test-sequence", splitToken: String = " "):Iterator[Observation[IndexedSeq[IndexedSeq[String]]]] = {
    
47.     val source = Source.fromInputStream(f).getLines()
    
48.     new Iterator[Observation[IndexedSeq[IndexedSeq[String]]]] {
    
49.       def hasNext = source.hasNext
    
50.       var index = 0
    
51.       def next() = {
    
52.         val inputs = new ArrayBuffer[IndexedSeq[String]]()
    
53.         import scala.util.control.Breaks._
    
54.         breakable {
    
55.           while(source.hasNext) {
    
56.             val line = source.next()
    
57.             if(line.trim().isEmpty) break
    
58. 
    
59.             val split = line.split(splitToken)
    
60.             inputs += split
    
61.           }
    
62.         }
    
63.         val id = name + "-" + index
    
64.         index += 1
    
65.         Observation(inputs,id)
    
66.       }
    
67.     }
    
68.   }
    
69. 
    
70.   def main(args: Array[String]) {
    
71.     println(readTrain(new FileInputStream(new File(args(0)))).length)
    
72.     println(readTest(new FileInputStream(new File(args(1)))).length)
    
73.   }
    
74. 
    
75. }

复制代码

1. package chalk.corpora
   
2. 
   
3. /*
   
4. Copyright 2009 David Hall, Daniel Ramage
   
5. 
   
6. Licensed under the Apache License, Version 2.0 (the "License")
   
7. you may not use this file except in compliance with the License.
   
8. You may obtain a copy of the License at
   
9. 
   
10. http://www.apache.org/licenses/LICENSE-2.0
    
11. 
    
12. Unless required by applicable law or agreed to in writing, software
    
13. distributed under the License is distributed on an "AS IS" BASIS,
    
14. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    
15. See the License for the specific language governing permissions and
    
16. limitations under the License.
    
17. */
    
18. 
    
19. 
    
20. //import breeze.util.Implicits._
    
21. import java.net._
    
22. import java.io.File
    
23. import scala.xml.XML
    
24. 
    
25. /**
    
26. * Class to help load corpora. Each Corpus is represented by a class
    
27. * of type "Corpus" which knows how to load the data.
    
28. *
    
29. * Corpora are located in one of two ways:
    
30. * <ul>
    
31. *  <li> Provide a class name for the corpus for a class already in the class path.</li>
    
32. *  <li>2) Located through one or more public repositories, usually your
    
33. *     MAVEN_HOME and the breeze maven repository.
    
34. *     
    
35. *     Files located in this way must be installed in:
    
36. *      /path/to/repository/org/breeze/corpora/(corpus name in lower case)/(version)/(corpus-name)-(version).jar
    
37. *
    
38. *     The jar file itself must have a file named breeze-corpus.xml which has the format:
    
39. *   <code>
    
40. *       <corpus>
    
41. *         <classname>absolute classname</classname>
    
42. *       </corpus>
    
43. *   </code>
    
44. *    This class is then created and returned.
    
45. *   </li>
    
46. * </ul>
    
47. *
    
48. * @author dlwh
    
49. */
    
50. class Corpora(val repositories: Seq[URL]) {
    
51.   import Corpora._
    
52.   def this(remoteURL: String) = this( Array(CorpusUtils.defaultCorpusRepo, new URL(remoteURL)))
    
53. 
    
54.   /**
    
55.   * Create a new instance of the corpus.
    
56.   */
    
57.   def load[T](clss: Class[U] forSome {type U<:Corpus[T]}) = {
    
58.     clss.newInstance
    
59.   }
    
60. 
    
61.   /**
    
62.   * Either loads the Corpus with the given class name, or searches the repositories for the corpus with the (lowercase) name.
    
63.   */
    
64.   def load[T](name: String):Corpus[T] = {
    
65.     val clss = cForName[T](name) orElse cForName[T]("breeze.corpora."+name)
    
66.     clss match {
    
67.       case None => locateCorpus(name)
    
68.       case Some(c) => load(c.asSubclass(classOf[Corpus[T]]))
    
69.     }
    
70.   }
    
71. 
    
72. 
    
73.   /**
    
74.   * Searches for a corpus in the available repositories.
    
75.   */
    
76.   def locateCorpus[T](name: String) : Corpus[T] = {
    
77.     val jarLocation = findJar(name)
    
78.     val cl = new URLClassLoader(Array(jarLocation))
    
79.     val className = {
    
80.       val strm = cl.getResourceAsStream("breeze-corpus.xml")
    
81.       val xml = XML.load(strm)
    
82.       println(xml)
    
83.       strm.close()
    
84.       (xml \ "classname").text.trim()
    
85.     }
    
86.     println(className + " " + jarLocation)
    
87.     load(cl.loadClass(className).asSubclass(classOf[Corpus[T]]))
    
88.   }
    
89. 
    
90.   private def findJar(name: String) = {
    
91.     val lowerName = name.toLowerCase
    
92.     val urls = repositories.iterator
    
93. 
    
94.     var jar: URL = null
    
95.     while(urls.hasNext && (jar eq null)) {
    
96.       val url = urls.next
    
97.       try { // is it a directory?
    
98.         val f = new File(url.toURI)
    
99.         val location = new File(new File(f, "org/breeze/corpora/"), lowerName)
    
100.         println(location)
     
101.         if(location.exists && location.isDirectory) {
     
102.           val latestVersion = location.listFiles.filter(_.isDirectory).reduceLeft( (x,y) =>
     
103.             new File(lexicographicOrder(x.getName,y.getName))
     
104.           )
     
105.           println(latestVersion)
     
106.           jar = latestVersion.listFiles.filter(_.getName.endsWith(".jar"))(0).toURI.toURL
     
107.         }
     
108.       } catch { // not a file, so we'll hope it's a maven repo
     
109.         case _:Exception =>
     
110.         val location = url.toString +"/org/breeze/corpora/"+lowerName
     
111.         val metadata = new URL(location + "/maven-metadata.xml")
     
112.         try {
     
113.           val stream = metadata.openStream()
     
114.           try {
     
115.             val xml = XML.load(stream);
     
116.             val latestVersion = (xml \ "version").text
     
117.             jar = new URL(location + "/" + latestVersion + "/"+lowerName + "-"+latestVersion + ".jar")
     
118.           } finally {
     
119.             stream.close()
     
120.           }
     
121.         } catch {
     
122.           case _:Exception => ()
     
123.         }
     
124.       }
     
125.     }
     
126.     if(jar == null) throw new RuntimeException("Couldn't find the corpus " + name)
     
127.     jar
     
128.   }
     
129. 
     
130. 
     
131.   private def lexicographicOrder(f1: String, f2: String) = {
     
132.     if(f1.split(".").zip(f2.split(".")).forall { case (s1,s2) =>
     
133.       try {
     
134.         s1.toInt > s2.toInt
     
135.       } catch {
     
136.         case _:Exception => s1 > s2
     
137.       }
     
138.     }) f1 else f2
     
139.   }
     
140. 
     
141.   private def cForName[T](name: String) = try {
     
142.     Some(Class.forName(name).asInstanceOf[Class[Corpus[T]]])
     
143.   } catch {
     
144.     case _: Exception => None
     
145.   }
     
146. 
     
147. }
     
148. 
     
149. object Corpora extends Corpora("http://repo.breeze.org/repo/") {
     
150. }
     
151. 
     
152. object CorpusUtils {
     
153.   protected[corpora] def defaultCorpusRepo = {
     
154.     val repoFile = {
     
155.       var a = System.getenv("SCALANLP_CORPORA")
     
156.       if(a == null) a = System.getenv("MAVEN_HOME")
     
157.       if(a == null) a = System.getenv("HOME") + "/.m2/"
     
158.     a } + "repository/"
     
159.     new File(repoFile).toURI.toURL
     
160.   }
     
161. 
     
162. }

复制代码

1. package chalk.corpora;
   
2. 
   
3. /*
   
4. Copyright 2009 David Hall, Daniel Ramage
   
5. 
   
6. Licensed under the Apache License, Version 2.0 (the "License");
   
7. you may not use this file except in compliance with the License.
   
8. You may obtain a copy of the License at
   
9. 
   
10. http://www.apache.org/licenses/LICENSE-2.0
    
11. 
    
12. Unless required by applicable law or agreed to in writing, software
    
13. distributed under the License is distributed on an "AS IS" BASIS,
    
14. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    
15. See the License for the specific language governing permissions and
    
16. limitations under the License.
    
17. */
    
18. 
    
19. 
    
20. import scala.io._;
    
21. import java.io.Reader;
    
22. import java.net._;
    
23. import java.io._;
    
24. import scala.collection.mutable.ArrayBuffer;
    
25. 
    
26. /**
    
27. * Represents a corpus of something.
    
28. *
    
29. * @author dlwh
    
30. */
    
31. trait Corpus[+T] {
    
32.   def name : String;
    
33.   /** Splits could be "train" "test", or "small", "large", or whatever. */
    
34.   def splits : Map[String,Seq[T]];
    
35.   def license = "Unknown";
    
36.   def author = "Unknown";
    
37. }
    
38. 
    
39. 
    
40. /**
    
41. * Pretty useful for creating a corpus from a jar file where the
    
42. * resources are packaged in the jar file. Simply specify a mapFun
    
43. * to transform the data and and put a "data/categories" file in the
    
44. * jar that has "splitName resourceName" pairs separated by a space,
    
45. * where the resourceName points to another resource with a list of
    
46. * files to be used in that split. It's easier than it sounds.
    
47. *
    
48. * @author dlwh
    
49. */
    
50. trait JarCorpus[+T] extends Corpus[T] {
    
51.   protected def mapFun(category:String, path:String, s: InputStream):T
    
52. 
    
53.   // Utility to read "k v" pairs from a resource.
    
54.   protected def stringsFromFile(cl: ClassLoader, s : String) = {
    
55.     val strm = cl.getResourceAsStream(s)
    
56.     val src = Source.fromInputStream(strm);
    
57.     val result = (for{ line <- src.getLines();
    
58.                       trimmed = line.trim}
    
59.                       yield trimmed).toIndexedSeq;
    
60.     strm.close();
    
61.     result;
    
62.   }
    
63. 
    
64.   // place for top level categories files
    
65.   protected def categoryFile: String = "data/categories";
    
66. 
    
67.   /**
    
68.   * Provides a list of (categoryName,resourceName), where the
    
69.   * categoryName is the name of the split (like "train") and the
    
70.   * resourceName is a resource path that has a list of the files
    
71.   * to load.
    
72.   */
    
73.   protected def categories: Seq[(String,String)] = stringsFromFile(classLoader, categoryFile) map { x=>
    
74.     val res = x.split(" ");
    
75.     (res(0),res(1));
    
76.   };
    
77. 
    
78.   lazy val splits = {
    
79.     Map[String,Seq[T]]() ++ categories.map { case(cat,res) =>
    
80.       val paths = stringsFromFile(classLoader,res).toStream;
    
81.       val data = for (path <- paths) yield {
    
82.         val strm = classLoader.getResourceAsStream(path)
    
83.         val result = mapFun(cat,path,new BufferedInputStream(strm));
    
84.         strm.close();
    
85.         result;
    
86.       }
    
87.       (cat,data)
    
88.     }
    
89.   }
    
90. 
    
91.   protected lazy val classLocation = {
    
92.     this.getClass.getProtectionDomain.getCodeSource.getLocation;
    
93.   }
    
94. 
    
95.   protected lazy val classLoader = {
    
96.     this.getClass.getClassLoader;
    
97.   }
    
98. }

复制代码

1. package chalk.data;
   
2. 
   
3. /*
   
4. Copyright 2009 David Hall, Daniel Ramage
   
5. 
   
6. Licensed under the Apache License, Version 2.0 (the "License");
   
7. you may not use this file except in compliance with the License.
   
8. You may obtain a copy of the License at
   
9. 
   
10. http://www.apache.org/licenses/LICENSE-2.0
    
11. 
    
12. Unless required by applicable law or agreed to in writing, software
    
13. distributed under the License is distributed on an "AS IS" BASIS,
    
14. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    
15. See the License for the specific language governing permissions and
    
16. limitations under the License.
    
17. */
    
18. 
    
19. import scala.collection.mutable.StringBuilder;
    
20. import scala.io.Source;
    
21. import java.io.File;
    
22. 
    
23. import nak.data.{Observation,Example}
    
24. 
    
25. /**
    
26. * Represents a sequence of text. This is just a string with an id, and a nice map method.
    
27. *
    
28. * @author dlwh
    
29. */
    
30. case class Text(id:String, contents: String) extends Observation[String] {
    
31.   def features = contents;
    
32.    
    
33.   def withLabel[L](l:L) = new LabeledText[L](id,l,contents);
    
34. }
    
35. 
    
36. object Text {
    
37. def fromFile(f :File) = {
    
38.    new Text(f.getName,Source.fromFile(f).getLines().toSeq.foldLeft(new StringBuilder)( _ append _).toString);
    
39. }
    
40. }
    
41. 
    
42. /**
    
43. * A text with a label.
    
44. * @author dlwh
    
45. */
    
46. class LabeledText[L](id:String,val label:L, contents: String) extends Text(id,contents) with Example[L,String];
    
47. 
    
48. object LabeledText {
    
49.   /**
    
50.    * Creates a new text with the file as the id, and the directory name as the label.
    
51.    */
    
52.   def fromFile(f:File) = {
    
53.     new Text(f.getName,Source.fromFile(f).getLines().toSeq.foldLeft(new StringBuffer)(_ append _).toString).withLabel(f.getParentFile.getName);
    
54.   }
    
55. }

复制代码

1. package chalk.data;
   
2. 
   
3. /*
   
4. Copyright 2009 David Hall, Daniel Ramage
   
5. 
   
6. Licensed under the Apache License, Version 2.0 (the "License");
   
7. you may not use this file except in compliance with the License.
   
8. You may obtain a copy of the License at
   
9. 
   
10. http://www.apache.org/licenses/LICENSE-2.0
    
11. 
    
12. Unless required by applicable law or agreed to in writing, software
    
13. distributed under the License is distributed on an "AS IS" BASIS,
    
14. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    
15. See the License for the specific language governing permissions and
    
16. limitations under the License.
    
17. */
    
18. 
    
19. import scala.collection.mutable.StringBuilder;
    
20. import scala.io.Source;
    
21. import java.io.File;
    
22. 
    
23. import nak.data.{Observation,Example}
    
24. 
    
25. /**
    
26. * Represents a sequence of text. This is just a string with an id, and a nice map method.
    
27. *
    
28. * @author dlwh
    
29. */
    
30. case class Text(id:String, contents: String) extends Observation[String] {
    
31.   def features = contents;
    
32.    
    
33.   def withLabel[L](l:L) = new LabeledText[L](id,l,contents);
    
34. }
    
35. 
    
36. object Text {
    
37. def fromFile(f :File) = {
    
38.    new Text(f.getName,Source.fromFile(f).getLines().toSeq.foldLeft(new StringBuilder)( _ append _).toString);
    
39. }
    
40. }
    
41. 
    
42. /**
    
43. * A text with a label.
    
44. * @author dlwh
    
45. */
    
46. class LabeledText[L](id:String,val label:L, contents: String) extends Text(id,contents) with Example[L,String];
    
47. 
    
48. object LabeledText {
    
49.   /**
    
50.    * Creates a new text with the file as the id, and the directory name as the label.
    
51.    */
    
52.   def fromFile(f:File) = {
    
53.     new Text(f.getName,Source.fromFile(f).getLines().toSeq.foldLeft(new StringBuffer)(_ append _).toString).withLabel(f.getParentFile.getName);
    
54.   }
    
55. }

复制代码

1. package chalk.slab
   
2. 
   
3. import akka.actor.{Actor,ActorLogging,ActorSystem,Props}
   
4. import akka.pattern.{ask,pipe}
   
5. import akka.util.Timeout
   
6. import scala.collection.mutable.ListBuffer
   
7. import scala.concurrent.duration._
   
8. import scala.concurrent.Future
   
9. 
   
10. /**
    
11.   * An actor that mixes-in an AnalysisFunction and hands Slabs contained in Process messages over
    
12.   * to the function.
    
13.   */
    
14. trait AnalysisComponent[C,B,I<:B,O<:B] extends Actor with ActorLogging with AnalysisFunction[C,B,I,O] {
    
15.   import AnalysisComponent._
    
16.   def receive = {
    
17.     case Process(slab) => sender ! apply(slab.asInstanceOf[Slab[C,B,I]])
    
18.   }
    
19. }
    
20. 
    
21. /**
    
22.   * Companion object, e.g. to hold messages that can be processed by an AnalysisComponent actor.
    
23.   */
    
24. object AnalysisComponent {
    
25.   case class Process[C,B,I<:B](slab: Slab[C,B,I])
    
26. }
    
27. 
    
28. trait StringAnalysisComponent[I<:Span,O<:Span]
    
29.     extends AnalysisComponent[String,Span,I,O]
    
30. 
    
31. /**
    
32.   * An actor that uses SentenceSegmenter.
    
33.   */
    
34. class SentenceSegmenterActor extends SentenceSegmenter[Span]
    
35.     with StringAnalysisComponent[Span,Sentence]
    
36. 
    
37. /**
    
38.   * An actor that uses Tokenizer.
    
39.   */
    
40. class TokenizerActor extends Tokenizer[Sentence] with StringAnalysisComponent[Sentence, Token]
    
41. 
    
42. 
    
43. /**
    
44.   * An analysis engine that runs Slabs through a pipeline of AnalysisComponents. It currently
    
45.   * requires explicit declaration of the analyzers, but this would ideally be done through
    
46.   * configuration. No compile-time consistency check for the types in the pipeline is performed.
    
47.   * Anyhoo, this gives the basic idea for how an actor based AnalysisEngine might work, so
    
48.   * it should be a good starting point.
    
49.   */
    
50. class AnalysisEngine extends Actor with ActorLogging {
    
51. 
    
52.   import AnalysisComponent._
    
53.   import AnalysisEngine._
    
54.   import Span._
    
55.   import Slab.StringSlab
    
56.   implicit val ec = context.dispatcher
    
57.   implicit val timeout = Timeout(10 seconds)
    
58.   
    
59.   val sentenceSegmenter = context.system.actorOf(Props[SentenceSegmenterActor])
    
60.   val tokenizer = context.system.actorOf(Props[TokenizerActor])
    
61. 
    
62.   def receive = {
    
63.     case Process(slab) =>
    
64.       log.info("Processing slab:\n " + slab.content)
    
65.       (for {
    
66.         slab1 <- (sentenceSegmenter ? Process(slab)).mapTo[StringSlab[Sentence]]
    
67.         slab2 <- (tokenizer ? Process(slab1)).mapTo[StringSlab[Sentence with Token]]
    
68.       } yield {
    
69.         slab2
    
70.       }) pipeTo sender
    
71. 
    
72.     case ProcessCorpus(texts) =>
    
73.       Future.traverse(texts)(text => self ? Process(Slab(text))) pipeTo sender
    
74.   }
    
75. }
    
76. 
    
77. /**
    
78.   * Example application doing actor based Slab processing.
    
79.   */
    
80. object AnalysisEngine {
    
81. 
    
82.   case class ProcessCorpus(corpus: Iterator[String])
    
83.   
    
84.   import AnalysisComponent._
    
85.   import Span._
    
86.   import Slab.StringSlab
    
87. 
    
88.   val text1 = "Here is an example text. It has four sentences and it mentions Jimi Hendrix and Austin, Texas! In this third sentence, it also brings up Led Zeppelin and Radiohead, but does it ask a question? It also has a straggler sentence that doesn't end with punctuation"
    
89. 
    
90.   val text2 = "How much wood can a woodchuck chuck? Can a woodchuck chuck wood?"
    
91. 
    
92.   val text3 = "The Association for Computational Linguistics is proud to present its first Lifetime Achievement Award to Prof. Aravind Joshi of the University of Pennsylvania. Aravind Joshi was born in 1929 in Pune, India, where he completed his secondary education as well as his first degree in Mechanical and Electrical Engineering, the latter in 1950. He worked as a research assistant in Linguistics at Penn from 1958-60, while completing his Ph.D. in Electrical Engineering, in 1960. Joshi's work and the work of his Penn colleagues at the frontiers of Cognitive Science was rewarded in 1991 by the establishment of a National Science Foundation Science and Technology Center for Research in Cognitive Science, which Aravind Joshi co-directed until 2001. Dr. Joshi has supervised thirty-six Ph.D. theses to-date, on topics including information and coding theory, and also pure linguistics."
    
93.   
    
94.   def main(args: Array[String]) {
    
95. 
    
96.     val system = ActorSystem("ChalkSystem")
    
97.     implicit val ec = system.dispatcher
    
98.     implicit val timeout = Timeout(10 seconds)
    
99. 
    
100.     val engine = system.actorOf(Props[AnalysisEngine])
     
101.     val corpus = Iterator(text1,text2,text3)
     
102.    
     
103.     for {
     
104.       slabs <- (engine ? ProcessCorpus(corpus)).mapTo[Iterator[StringSlab[Sentence with Token]]]
     
105.       slab <- slabs
     
106.     } {
     
107.       // Notice that the last sentence (lacking EOS char) is missing.
     
108.       val sentences = slab.iterator[Sentence].toList
     
109.       println("\nSENTENCES\n\n" + sentences.map(_.in(slab).content).mkString("\n"))
     
110.       
     
111.       val tokens = slab.iterator[Token].toList
     
112.       println("\nTOKENS\n\n" + tokens.map(_.in(slab).content).mkString("\n"))
     
113.     }
     
114.     Thread.sleep(3000)
     
115.     system.shutdown
     
116.   }
     
117. 
     
118. }

复制代码

thanks for sharing.

感谢分享