finished gutenberg

data/stopwords.txt

@@ -1 +1 @@
-a,able,about,across,after,all,almost,also,am,among,an,and,any,are,as,at,be,because,been,but,by,can,cannot,could,dear,did,do,does,either,else,ever,every,for,from,get,got,had,has,have,he,her,hers,him,his,how,however,i,if,in,into,is,it,its,just,least,let,like,likely,may,me,might,most,must,my,neither,no,nor,not,of,off,often,on,only,or,other,our,own,rather,said,say,says,she,should,since,so,some,than,that,the,their,them,then,there,these,they,this,tis,to,too,twas,us,wants,was,we,were,what,when,where,which,while,who,whom,why,will,with,would,yet,you,your,one,out,more,now,first,two,very,such,same,shall,upon,before,therefore,great,made,even,same,work,make,being,through,here,way,true,see,time,those,place,much,without,body,whole,another,thus,set,new,given,both,above,well,part,between,end,order,each,form,gutenberg
+a,able,about,across,after,all,almost,also,am,among,an,and,any,are,as,at,be,because,been,but,by,can,cannot,could,dear,did,do,does,either,else,ever,every,for,from,get,got,had,has,have,he,her,hers,him,his,how,however,i,if,in,into,is,it,its,just,least,let,like,likely,may,me,might,most,must,my,neither,no,nor,not,of,off,often,on,only,or,other,our,own,rather,said,say,says,she,should,since,so,some,than,that,the,their,them,then,there,these,they,this,tis,to,too,twas,us,wants,was,we,were,what,when,where,which,while,who,whom,why,will,with,would,yet,you,your,one,out,more,now,first,two,very,such,same,shall,upon,before,therefore,great,made,even,same,work,make,being,through,here,way,true,see,time,those,place,much,without,body,whole,another,thus,set,new,given,both,above,well,part,between,end,order,each,form,gutenberg,project,electronic,refund,source,around,example,origin,copyright,trademark,itself,paragraph,further,including,spiral,united,limited,together,license,things,others,toward,present,showed,second,against,observed,sometimes,during,within,little,inside,quality,especially,methods,making,necessity,single,although,probably,regarded,public,applied,information,degree,higher,become,unless,larger,actual,recent,showing,bottom,section,agreement,typical,double,access,easily,manufacture,readily,formed,already,except,certain,spaces,online,general,region,former,question,opinion,probable,though,thought,greater,enough,nothing,perhaps,common,matter,themselves,arguments,answer,matter,whether,developed,illustration,foundation,donations,produced,works,terms,found,large,archive,used,name,type,forms,many,mold,near,line,kind,full,flat,base,over,copy,http,ebook,once,under,plain,until,small,fuel,later,away,came,seen,long,laws,soft,find,view,exit,free,latter,known,space,less,open,upper,agree,sand,round,means,study,color,rain,fact,easy,black,built,wood,outside,forth,ebooks,give,side,copies,paid,several,extent,provided,point,reason,charge,comply,word,examples,attached,shape,abundant,provide,indented,central,quite,left,nearly,seems,permission,materials,designs,loops,come,white,conical,method,distributed,upward,beginning,coated,status,again,tight,additional,important,distribution,different,domain,received,affected,ends,regard,idea,take,effect,plan,named,result,available,smooth,called,convex,lines,states,instead,variety,shaped,placed,simply,show,using,distribute,whence,liable,hand,occur,handle,support,indentation,relationship,points,removed,contact,refer,four,kinds,modern,life,people,associated,rectangular,surface,alone,took,answered,none,always,entered,right,third,call,hour,knew,sight,hear,bring,reply,gave,real,

gutenberg.rb

@@ -1,4 +1,4 @@
-require_relative 'lib/simple_predictor'
+# require_relative 'lib/simple_predictor'
 require_relative 'lib/complex_predictor'
 
 def run!(predictor_klass, opts={})
@@ -23,6 +23,8 @@ def run!(predictor_klass, opts={})
   puts "Accuracy: #{accuracy}"
 end
 
-run!(SimplePredictor)
+# run!(SimplePredictor)
 run!(ComplexPredictor, debug: true)
 
+
+

lib/complex_predictor.rb

@@ -6,7 +6,23 @@ class ComplexPredictor < Predictor
   #
   # Returns nothing.
   def train!
-    @data = {}
+    @top_frequency = {}
+    @all_books.each do |key, value|
+      frequency = Hash.new(0)
+      value[0][1].each do |val|
+        if good_token?(val)
+          frequency[val] +=1
+        end
+      end
+      frequency = frequency.sort_by {|key, val| val}.reverse!
+      frequency = frequency[0..129]
+      frequency.flatten!
+      frequency.delete_if {|x| x.is_a? Integer}
+      @top_frequency[key] = frequency
+    end
+    mkfile = File.open("/home/daniel/code/algorithms/lib/txt/train.txt","a")
+    mkfile.write(@top_frequency)
+    mkfile.close
   end
 
   # Public: Predicts category.
@@ -15,8 +31,29 @@ def train!
   #
   # Returns a category.
   def predict(tokens)
-    # Always predict astronomy, for now.
-    :astronomy
+    matches = {}
+    token_freq = Hash.new(0)
+    tokens.each do |val|
+      if good_token?(val)
+        token_freq[val] +=1
+      end
+    end
+    token_freq = token_freq.sort_by {|key, val| val}.reverse!
+    token_freq = token_freq[0..129]
+    token_freq.flatten!
+    token_freq.delete_if {|x| x.is_a? Integer}
+
+    @top_frequency.each do |key, val|
+      compare = val & token_freq
+      matches[key] = compare.length
+    end
+    mkfile = File.open("/home/daniel/code/algorithms/lib/txt/predict.txt","a")
+    mkfile.write(matches)
+    mkfile.close
+    return matches.sort_by {|k,v| v}.reverse!.first[0]
+
+
+
   end
 end
 

lib/predictor.rb

@@ -2,7 +2,7 @@
 require 'find'
 
 class Predictor
-  CATEGORIES = [:astronomy, :philosophy, :religion, :archeology]
+  CATEGORIES = [:philosophy,:astronomy, :archeology,:religion]
   STOP_WORDS = Set.new(File.read('data/stopwords.txt').split(','))
 
   # Public: Initialize object.
@@ -42,7 +42,7 @@ def predict(tokens)
   # Returns true if you should use this token. In our project, "token" is
   # synonymous with "word".
   def good_token?(token)
-    !STOP_WORDS.include?(token) && token.size > 2
+    !STOP_WORDS.include?(token) && token.size > 3
   end
 
   #############################################################################