From 8b8ebf56357d5c8a6ced1b8fc5b1732d7539b35c Mon Sep 17 00:00:00 2001 From: Richard Degenne Date: Thu, 14 Mar 2019 16:12:20 +0100 Subject: [PATCH 01/10] Added source_id to entities --- lib/textrazor/entity.rb | 2 +- spec/lib/textrazor/entity_spec.rb | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/textrazor/entity.rb b/lib/textrazor/entity.rb index fb8e7d2..06454b7 100644 --- a/lib/textrazor/entity.rb +++ b/lib/textrazor/entity.rb @@ -6,7 +6,7 @@ class Entity attr_reader :id, :type, :matching_tokens, :entity_id, :freebase_types, :confidence_score, :wiki_link, :matched_text, :freebase_id, :relevance_score, :entity_english_id, - :starting_pos, :ending_pos, :data, :wikidata_id + :starting_pos, :ending_pos, :data, :wikidata_id, :source_id def initialize(params = {}) @type = [] diff --git a/spec/lib/textrazor/entity_spec.rb b/spec/lib/textrazor/entity_spec.rb index 8354a9a..8c12b63 100644 --- a/spec/lib/textrazor/entity_spec.rb +++ b/spec/lib/textrazor/entity_spec.rb @@ -29,6 +29,7 @@ module TextRazor "data" => { "type" => ['person', 'company'] }, + "sourceId" => "some-dictionary", "wikidataId" => 'Q7330070' } end @@ -48,6 +49,7 @@ module TextRazor expect(entity.starting_pos).to eq(3) expect(entity.ending_pos).to eq(20) expect(entity.data['type']).to match_array(['person', 'company']) + expect(entity.source_id).to eq("some-dictionary") expect(entity.wikidata_id).to eq('Q7330070') end end From 43e736ad55380d8963336ba4cdef5edf06604208 Mon Sep 17 00:00:00 2001 From: Richard Degenne Date: Thu, 14 Mar 2019 16:17:14 +0100 Subject: [PATCH 02/10] Added Dictionary model --- lib/textrazor.rb | 1 + lib/textrazor/dictionary.rb | 27 ++++++++++ spec/lib/textrazor/dictionary_spec.rb | 77 +++++++++++++++++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 lib/textrazor/dictionary.rb create mode 100644 spec/lib/textrazor/dictionary_spec.rb diff --git a/lib/textrazor.rb b/lib/textrazor.rb index b7202c8..caa5c58 100644 --- a/lib/textrazor.rb +++ b/lib/textrazor.rb @@ -1,6 +1,7 @@ require "textrazor/version" require "textrazor/configuration" require "textrazor/util" +require "textrazor/dictionary" require "textrazor/client" require "textrazor/request" require "textrazor/response" diff --git a/lib/textrazor/dictionary.rb b/lib/textrazor/dictionary.rb new file mode 100644 index 0000000..9a2161a --- /dev/null +++ b/lib/textrazor/dictionary.rb @@ -0,0 +1,27 @@ +module TextRazor + + class Dictionary + + include Util + + attr_reader :id, :match_type, :case_insensitive, :language + + def initialize(params = {}) + initialize_params params + end + + def to_h + { + "matchType" => match_type, + "caseInsensitive" => case_insensitive, + "language" => language + }.reject { |_, v| v.nil? } + end + + def valid? + !id.nil? && !id.empty? + end + + end + +end diff --git a/spec/lib/textrazor/dictionary_spec.rb b/spec/lib/textrazor/dictionary_spec.rb new file mode 100644 index 0000000..2265566 --- /dev/null +++ b/spec/lib/textrazor/dictionary_spec.rb @@ -0,0 +1,77 @@ +require "spec_helper" + +module TextRazor + + describe Dictionary do + + let(:dictionary) do + Dictionary.create_from_hash(dictionary_hash) + end + + let(:dictionary_hash) do + { + "id" => "some-dictionary", + "matchType" => "TOKEN", + "caseInsensitive" => true, + "language" => "eng" + } + end + + context "#create_from_hash" do + + it "should create a new instance" do + expect(dictionary.id).to eq("some-dictionary") + expect(dictionary.match_type).to eq("TOKEN") + expect(dictionary.case_insensitive).to be(true) + expect(dictionary.language).to eq("eng") + end + end + + + context "#to_h" do + + it "should have the right values" do + expect(dictionary.to_h).to eq( + "matchType" => "TOKEN", + "caseInsensitive" => true, + "language" => "eng" + ) + end + + context "with default values" do + + let(:dictionary_hash) do + { + "id" => "some-dictionary" + } + end + + it "should not add them" do + expect(dictionary.to_h).to eq({}) + end + end + end + + context "#valid?" do + + context "when it is valid" do + + it "should return true" do + expect(dictionary.valid?).to be true + end + + end + + context "when it is not valid" do + let(:dictionary_hash) do + {} + end + + it "should return false" do + expect(dictionary.valid?).to be false + end + end + end + + end +end From cdc5c9621e4f1a6a00def951291129056cac8d01 Mon Sep 17 00:00:00 2001 From: Richard Degenne Date: Thu, 14 Mar 2019 16:24:07 +0100 Subject: [PATCH 03/10] Revamped API calls --- lib/textrazor.rb | 1 + lib/textrazor/api_response.rb | 54 ++++++++++++ lib/textrazor/client.rb | 6 +- lib/textrazor/request.rb | 14 ++- lib/textrazor/response.rb | 46 +--------- spec/lib/textrazor/api_response_spec.rb | 110 ++++++++++++++++++++++++ spec/lib/textrazor/client_spec.rb | 4 +- spec/lib/textrazor/request_spec.rb | 60 ++++++++++--- spec/lib/textrazor/response_spec.rb | 94 -------------------- 9 files changed, 229 insertions(+), 160 deletions(-) create mode 100644 lib/textrazor/api_response.rb create mode 100644 spec/lib/textrazor/api_response_spec.rb diff --git a/lib/textrazor.rb b/lib/textrazor.rb index caa5c58..465a5bd 100644 --- a/lib/textrazor.rb +++ b/lib/textrazor.rb @@ -4,6 +4,7 @@ require "textrazor/dictionary" require "textrazor/client" require "textrazor/request" +require "textrazor/api_response" require "textrazor/response" require "textrazor/category" require "textrazor/topic" diff --git a/lib/textrazor/api_response.rb b/lib/textrazor/api_response.rb new file mode 100644 index 0000000..5000af6 --- /dev/null +++ b/lib/textrazor/api_response.rb @@ -0,0 +1,54 @@ +require 'json' + +module TextRazor + + class ApiResponse + + BadRequest = Class.new(StandardError) + Unauthorised = Class.new(StandardError) + RequestEntityTooLong = Class.new(StandardError) + + attr_reader :raw_response, :time + + def initialize(http_response) + code = http_response.code + body = http_response.body + + raise BadRequest.new(body) if bad_request?(code) + raise Unauthorised.new(body) if unauthorised?(code) + raise RequestEntityTooLong.new(body) if request_entity_too_long?(code) + + json_body = ::JSON::parse(body, symbolize_names: true) + + @time = json_body[:time].to_f + @ok = json_body[:ok] + @raw_response = json_body[:response] + end + + def ok? + @ok + end + + #TODO: Not in a successful response + #def error + #end + + #def message + #end + + private + + def bad_request?(code) + code == 400 + end + + def unauthorised?(code) + code == 401 + end + + def request_entity_too_long?(code) + code == 413 + end + end + +end diff --git a/lib/textrazor/client.rb b/lib/textrazor/client.rb index c35878f..07a0dcf 100644 --- a/lib/textrazor/client.rb +++ b/lib/textrazor/client.rb @@ -30,11 +30,7 @@ def initialize(api_key, options = {}) def analyse(text) assert_text(text) - options = { - api_key: api_key - }.merge(request_options) - - Response.new(Request.post(text, options)) + Response.new(Request.post(api_key, text, **request_options)) end def self.topics(api_key, text, options = {}) diff --git a/lib/textrazor/request.rb b/lib/textrazor/request.rb index f069e19..fadd999 100644 --- a/lib/textrazor/request.rb +++ b/lib/textrazor/request.rb @@ -18,18 +18,22 @@ class Request classifiers: 'classifiers' } - def self.post(text, options) + def self.post(api_key, text, **options) ::RestClient.post( TextRazor.configuration.url, build_query(text, options), - accept_encoding: 'gzip' + build_headers(api_key) ) end + def self.url(path = '/') + File.join(TextRazor.configuration.url, path) + end + private def self.build_query(text, options) - query = {"text" => text, "apiKey" => options.delete(:api_key)} + query = { 'text' => text } options.each do |key, value| value = value.join(",") if value.is_a?(Array) @@ -39,6 +43,10 @@ def self.build_query(text, options) query end + def self.build_headers(api_key) + { x_textrazor_key: api_key, accept_encoding: 'gzip' } + end + end end diff --git a/lib/textrazor/response.rb b/lib/textrazor/response.rb index f7d8fd7..21d504e 100644 --- a/lib/textrazor/response.rb +++ b/lib/textrazor/response.rb @@ -2,39 +2,7 @@ module TextRazor - class Response - - BadRequest = Class.new(StandardError) - Unauthorised = Class.new(StandardError) - RequestEntityTooLong = Class.new(StandardError) - - attr_reader :raw_response, :time - - def initialize(http_response) - code = http_response.code - body = http_response.body - - raise BadRequest.new(body) if bad_request?(code) - raise Unauthorised.new(body) if unauthorised?(code) - raise RequestEntityTooLong.new(body) if request_entity_too_long?(code) - - json_body = ::JSON::parse(body, symbolize_names: true) - - @time = json_body[:time].to_f - @ok = json_body[:ok] - @raw_response = json_body[:response] - end - - def ok? - @ok - end - - #TODO: Not in a successful response - #def error - #end - - #def message - #end + class Response < ApiResponse def custom_annotation_output @custom_annotation_output ||= raw_response[:customAnnotationOutput] @@ -98,18 +66,6 @@ def language_is_reliable? private - def bad_request?(code) - code == 400 - end - - def unauthorised?(code) - code == 401 - end - - def request_entity_too_long?(code) - code == 413 - end - def parse_entailments parse(:entailment, raw_response[:entailments]) end diff --git a/spec/lib/textrazor/api_response_spec.rb b/spec/lib/textrazor/api_response_spec.rb new file mode 100644 index 0000000..5774f47 --- /dev/null +++ b/spec/lib/textrazor/api_response_spec.rb @@ -0,0 +1,110 @@ +require 'spec_helper' + +module TextRazor + + describe ApiResponse do + + let(:http_response) do + ::OpenStruct.new(code: 200, body: body) + end + + let(:response) do + Response.new(http_response) + end + + describe "#initialize" do + + context "when HTTP response code is 200" do + + it "should create an instance of Response" do + body = "{\"response\":\"{}\"}" + http_response = ::OpenStruct.new code: 200, body: body + + expect(JSON).to receive(:parse). + with(body, {symbolize_names: true}). + and_return({"response" => "{}"}) + + Response.new(http_response) + end + + end + + context "when HTTP response code is 400" do + + it "should raise an exception" do + http_response = ::OpenStruct.new code: 400 + + expect{ Response.new(http_response) }. + to raise_error(Response::BadRequest) + end + + end + + context "when HTTP response code is 401" do + + it "should raise an exception" do + http_response = ::OpenStruct.new code: 401 + + expect{ Response.new(http_response) }. + to raise_error(Response::Unauthorised) + end + + end + + context "when HTTP response code is 413" do + + it "should raise an exception" do + http_response = ::OpenStruct.new code: 413 + + expect{ Response.new(http_response) }. + to raise_error(Response::RequestEntityTooLong) + end + + end + + end + + describe '#time' do + + it 'returns time taken to process request' do + body = {time: "0.013219"}.to_json + http_response = ::OpenStruct.new code: 200, body: body + response = Response.new(http_response) + + time = response.time + + expect(time).to eq(0.013219) + end + + end + + describe '#ok?' do + + context 'when successfully analysed' do + + it 'returns true' do + body = {ok: true}.to_json + http_response = ::OpenStruct.new code: 200, body: body + response = Response.new(http_response) + + expect(response).to be_ok + end + + end + + context 'when unsuccessfully analysed' do + + it 'returns false' do + body = {ok: false}.to_json + http_response = ::OpenStruct.new code: 200, body: body + response = Response.new(http_response) + + expect(response).to_not be_ok + end + + end + + end + end + +end diff --git a/spec/lib/textrazor/client_spec.rb b/spec/lib/textrazor/client_spec.rb index 51e14d6..f2cc4a0 100644 --- a/spec/lib/textrazor/client_spec.rb +++ b/spec/lib/textrazor/client_spec.rb @@ -124,10 +124,10 @@ module TextRazor request = BasicObject.new expect(Request).to receive(:post). - with('text', {api_key: 'api_key', extractors: %w(entities topics words), cleanup_mode: 'raw', + with(api_key, 'text', extractors: %w(entities topics words), cleanup_mode: 'raw', cleanup_return_cleaned: true, cleanup_return_raw: true, language: 'fre', filter_dbpedia_types: %w(type1), filter_freebase_types: %w(type2), - allow_overlap: false, dictionaries: %w(test), classifiers: 'textrazor_newscodes'}). + allow_overlap: false, dictionaries: %w(test), classifiers: 'textrazor_newscodes'). and_return(request) expect(Response).to receive(:new).with(request) diff --git a/spec/lib/textrazor/request_spec.rb b/spec/lib/textrazor/request_spec.rb index 9b2f826..2c01bfd 100644 --- a/spec/lib/textrazor/request_spec.rb +++ b/spec/lib/textrazor/request_spec.rb @@ -4,18 +4,56 @@ module TextRazor describe Request do + let(:api_key) do + "api_key" + end + + describe ".url" do + after :each do + TextRazor.reset + end + + let(:url) do + TextRazor::Request.url + end + + context "with the config 'secure' set to false" do + before :each do + TextRazor.configure do |config| + config.secure = false + end + end + + it "returns the unsecured URL" do + expect(url).to eq 'http://api.textrazor.com/' + end + end + + context "with the config 'secure' set to true" do + before :each do + TextRazor.configure do |config| + config.secure = true + end + end + + it "returns the unsecured URL" do + expect(url).to eq 'https://api.textrazor.com/' + end + end + end + context ".post" do context "default options" do it "should make correct calls" do - options = {api_key: 'api_key', extractors: %w(entities topics words dependency-trees relations entailments)} + options = { extractors: %w(entities topics words dependency-trees relations entailments) } expect(::RestClient).to receive(:post). - with("https://api.textrazor.com", { "text" => 'text', "apiKey" => 'api_key', - "extractors" => "entities,topics,words,dependency-trees,relations,entailments" }, accept_encoding: 'gzip') + with("https://api.textrazor.com", { "text" => 'text', + "extractors" => "entities,topics,words,dependency-trees,relations,entailments" }, accept_encoding: 'gzip', x_textrazor_key: 'api_key') - Request.post('text', options) + Request.post(api_key, 'text', **options) end end @@ -23,19 +61,19 @@ module TextRazor context "custom options" do it "should make correct calls" do - options = {api_key: 'api_key', extractors: %w(entities topics words), cleanup_mode: 'raw', + options = { extractors: %w(entities topics words), cleanup_mode: 'raw', cleanup_return_cleaned: true, cleanup_return_raw: true, language: 'fre', filter_dbpedia_types: %w(type1), filter_freebase_types: %w(type2), allow_overlap: false, enrichment_queries: 'queries', classifiers: 'textrazor_iab'} expect(::RestClient).to receive(:post). - with("https://api.textrazor.com", { "text" => 'text', "apiKey" => 'api_key', "extractors" => "entities,topics,words", - "cleanup.mode" => "raw", "cleanup.returnCleaned" => true, "cleanup.returnRaw" => true, "languageOverride" => 'fre', - "entities.filterDbpediaTypes" => "type1", "entities.filterFreebaseTypes" => "type2" , "entities.allowOverlap" => false, - "entities.enrichmentQueries" => "queries", "classifiers" => 'textrazor_iab'}, - accept_encoding: 'gzip') + with("https://api.textrazor.com", { "text" => 'text', "extractors" => "entities,topics,words", + "cleanup.mode" => "raw", "cleanup.returnCleaned" => true, "cleanup.returnRaw" => true, "languageOverride" => 'fre', + "entities.filterDbpediaTypes" => "type1", "entities.filterFreebaseTypes" => "type2" , "entities.allowOverlap" => false, + "entities.enrichmentQueries" => "queries", "classifiers" => 'textrazor_iab'}, + accept_encoding: 'gzip', x_textrazor_key: 'api_key') - Request.post('text', options) + Request.post(api_key, 'text', options) end end diff --git a/spec/lib/textrazor/response_spec.rb b/spec/lib/textrazor/response_spec.rb index 2c286e3..7a85008 100644 --- a/spec/lib/textrazor/response_spec.rb +++ b/spec/lib/textrazor/response_spec.rb @@ -12,100 +12,6 @@ module TextRazor Response.new(http_response) end - describe "#initialize" do - - context "when HTTP response code is 200" do - - it "should create an instance of Response" do - body = "{\"response\":\"{}\"}" - http_response = ::OpenStruct.new code: 200, body: body - - expect(JSON).to receive(:parse). - with(body, {symbolize_names: true}). - and_return({"response" => "{}"}) - - Response.new(http_response) - end - - end - - context "when HTTP response code is 400" do - - it "should raise an exception" do - http_response = ::OpenStruct.new code: 400 - - expect{ Response.new(http_response) }. - to raise_error(Response::BadRequest) - end - - end - - context "when HTTP response code is 401" do - - it "should raise an exception" do - http_response = ::OpenStruct.new code: 401 - - expect{ Response.new(http_response) }. - to raise_error(Response::Unauthorised) - end - - end - - context "when HTTP response code is 413" do - - it "should raise an exception" do - http_response = ::OpenStruct.new code: 413 - - expect{ Response.new(http_response) }. - to raise_error(Response::RequestEntityTooLong) - end - - end - - end - - describe '#time' do - - it 'returns time taken to process request' do - body = {time: "0.013219"}.to_json - http_response = ::OpenStruct.new code: 200, body: body - response = Response.new(http_response) - - time = response.time - - expect(time).to eq(0.013219) - end - - end - - describe '#ok?' do - - context 'when successfully analysed' do - - it 'returns true' do - body = {ok: true}.to_json - http_response = ::OpenStruct.new code: 200, body: body - response = Response.new(http_response) - - expect(response).to be_ok - end - - end - - context 'when unsuccessfully analysed' do - - it 'returns false' do - body = {ok: false}.to_json - http_response = ::OpenStruct.new code: 200, body: body - response = Response.new(http_response) - - expect(response).to_not be_ok - end - - end - - end - describe '#custom_annotation_output' do it 'returns raw text' do From cd134182bcbf378254823bca62bb10a96f401aef Mon Sep 17 00:00:00 2001 From: Richard Degenne Date: Thu, 14 Mar 2019 16:33:08 +0100 Subject: [PATCH 04/10] Added dictionary creation --- README.md | 12 ++++++++++++ lib/textrazor/client.rb | 12 ++++++++++++ lib/textrazor/request.rb | 9 +++++++++ spec/lib/textrazor/client_spec.rb | 25 +++++++++++++++++++++++++ spec/lib/textrazor/request_spec.rb | 16 ++++++++++++++++ 5 files changed, 74 insertions(+) diff --git a/README.md b/README.md index 5db0c21..3d0fe24 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,18 @@ TextRazor.phrases('api_key', 'text') ``` +### Dictionaries + +You can manage dictionaries and their entries through the dictionary API. + +#### Creating a dictionary + +``` +client = TextRazor::Client.new('api_key') + +client.create_dictionary('my-dictionary', case_insensitive: true) +``` + ## Next steps Only implemented this for topics, entities, words and phrases. Also, implement diff --git a/lib/textrazor/client.rb b/lib/textrazor/client.rb index 07a0dcf..d4ff415 100644 --- a/lib/textrazor/client.rb +++ b/lib/textrazor/client.rb @@ -8,6 +8,8 @@ class Client UnsupportedExtractor = Class.new(StandardError) UnsupportedCleanupMode = Class.new(StandardError) + InvalidDictionary = Class.new(StandardError) + DEFAULT_EXTRACTORS = ['entities', 'topics', 'words', 'phrases', 'dependency-trees', 'relations', 'entailments', 'senses'] @@ -33,6 +35,12 @@ def analyse(text) Response.new(Request.post(api_key, text, **request_options)) end + def create_dictionary(id, **options) + dictionary = Dictionary.new(id: id, **options) + assert_dictionary(dictionary) + Request.create_dictionary(api_key, dictionary) + end + def self.topics(api_key, text, options = {}) new(api_key, options.merge(extractors: ['topics'])). analyse(text). @@ -122,6 +130,10 @@ def is_text_bigger_than_200_kb?(text) text.bytesize/1024.0 > 200 end + def assert_dictionary(dictionary) + raise InvalidDictionary, "Dictionary is invalid" unless dictionary.valid? + end + end end diff --git a/lib/textrazor/request.rb b/lib/textrazor/request.rb index fadd999..9aa3d2b 100644 --- a/lib/textrazor/request.rb +++ b/lib/textrazor/request.rb @@ -26,6 +26,15 @@ def self.post(api_key, text, **options) ) end + def self.create_dictionary(api_key, dictionary, **options) + ::RestClient.put( + url("/entities/#{dictionary.id}"), + dictionary.to_h.to_json, + build_headers(api_key) + ) + dictionary + end + def self.url(path = '/') File.join(TextRazor.configuration.url, path) end diff --git a/spec/lib/textrazor/client_spec.rb b/spec/lib/textrazor/client_spec.rb index f2cc4a0..3ac99ec 100644 --- a/spec/lib/textrazor/client_spec.rb +++ b/spec/lib/textrazor/client_spec.rb @@ -170,6 +170,31 @@ module TextRazor end + context "#create_dictionary" do + + it "make correct calls" do + expect(Request).to receive(:create_dictionary). + with(api_key, a_kind_of(Dictionary)) + client.create_dictionary("id") + end + + context "with an invalid dictionary" do + + it "raises an exception" do + expect { client.create_dictionary('') }. + to raise_error(Client::InvalidDictionary) + end + end + end + + context "#delete_dictionary" do + it "make correct calls" do + expect(Request).to receive(:delete_dictionary). + with(api_key, "id") + client.delete_dictionary("id") + end + end + context ".topics" do it "makes correct calls" do diff --git a/spec/lib/textrazor/request_spec.rb b/spec/lib/textrazor/request_spec.rb index 2c01bfd..eed6667 100644 --- a/spec/lib/textrazor/request_spec.rb +++ b/spec/lib/textrazor/request_spec.rb @@ -80,6 +80,22 @@ module TextRazor end + context ".create_dictionary" do + + let(:dictionary) do + Dictionary.new(id: "id", match_type: "token", case_insensitive: true, language: "eng") + end + + it "should make correct calls" do + expect(::RestClient).to receive(:put). + with("https://api.textrazor.com/entities/id", + { "matchType" => "token", "caseInsensitive" => true, "language" => "eng" }.to_json, + accept_encoding: 'gzip', x_textrazor_key: 'api_key') + + Request.create_dictionary(api_key, dictionary) + end + end + end end From 9da60acae6af39306449dc46e023673c8a7542ef Mon Sep 17 00:00:00 2001 From: Richard Degenne Date: Thu, 14 Mar 2019 16:35:33 +0100 Subject: [PATCH 05/10] Added Dictionary deletion --- README.md | 6 ++++++ lib/textrazor/client.rb | 4 ++++ lib/textrazor/request.rb | 8 ++++++++ spec/lib/textrazor/request_spec.rb | 12 ++++++++++++ 4 files changed, 30 insertions(+) diff --git a/README.md b/README.md index 3d0fe24..4ed5d4e 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,12 @@ client = TextRazor::Client.new('api_key') client.create_dictionary('my-dictionary', case_insensitive: true) ``` +#### Deleting a dctionary + +``` +client.delete_dictionary('my-dictionary') +``` + ## Next steps Only implemented this for topics, entities, words and phrases. Also, implement diff --git a/lib/textrazor/client.rb b/lib/textrazor/client.rb index d4ff415..afca591 100644 --- a/lib/textrazor/client.rb +++ b/lib/textrazor/client.rb @@ -41,6 +41,10 @@ def create_dictionary(id, **options) Request.create_dictionary(api_key, dictionary) end + def delete_dictionary(dictionary_id) + Request.delete_dictionary(api_key, dictionary_id) + end + def self.topics(api_key, text, options = {}) new(api_key, options.merge(extractors: ['topics'])). analyse(text). diff --git a/lib/textrazor/request.rb b/lib/textrazor/request.rb index 9aa3d2b..51c0c47 100644 --- a/lib/textrazor/request.rb +++ b/lib/textrazor/request.rb @@ -35,6 +35,14 @@ def self.create_dictionary(api_key, dictionary, **options) dictionary end + def self.delete_dictionary(api_key, dictionary_id) + ::RestClient.delete( + url("entities/#{dictionary_id}"), + build_headers(api_key) + ) + true + end + def self.url(path = '/') File.join(TextRazor.configuration.url, path) end diff --git a/spec/lib/textrazor/request_spec.rb b/spec/lib/textrazor/request_spec.rb index eed6667..469364c 100644 --- a/spec/lib/textrazor/request_spec.rb +++ b/spec/lib/textrazor/request_spec.rb @@ -96,6 +96,18 @@ module TextRazor end end + context ".delete_dictionary" do + + it "should make correct calls" do + expect(::RestClient).to receive(:delete). + with("https://api.textrazor.com/entities/id", + accept_encoding: 'gzip', x_textrazor_key: 'api_key' + ) + + Request.delete_dictionary(api_key, "id") + end + end + end end From fe018b0ca728e0c0b654897db3ebacca7ef249f5 Mon Sep 17 00:00:00 2001 From: Richard Degenne Date: Thu, 14 Mar 2019 16:36:13 +0100 Subject: [PATCH 06/10] Added DictionaryEntry --- lib/textrazor.rb | 1 + lib/textrazor/dictionary_entry.rb | 27 ++++++++ spec/lib/textrazor/dictionary_entry_spec.rb | 71 +++++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 lib/textrazor/dictionary_entry.rb create mode 100644 spec/lib/textrazor/dictionary_entry_spec.rb diff --git a/lib/textrazor.rb b/lib/textrazor.rb index 465a5bd..3ce6246 100644 --- a/lib/textrazor.rb +++ b/lib/textrazor.rb @@ -2,6 +2,7 @@ require "textrazor/configuration" require "textrazor/util" require "textrazor/dictionary" +require "textrazor/dictionary_entry" require "textrazor/client" require "textrazor/request" require "textrazor/api_response" diff --git a/lib/textrazor/dictionary_entry.rb b/lib/textrazor/dictionary_entry.rb new file mode 100644 index 0000000..5a25bfb --- /dev/null +++ b/lib/textrazor/dictionary_entry.rb @@ -0,0 +1,27 @@ +module TextRazor + + class DictionaryEntry + + include Util + + attr_reader :id, :text, :data + + def initialize(params = {}) + initialize_params params + end + + def to_h + { + "id" => id, + "text" => text, + "data" => data + }.reject { |_, v| v.nil? || v.empty? } + end + + def valid? + !text.nil? && !text.empty? + end + + end +end + diff --git a/spec/lib/textrazor/dictionary_entry_spec.rb b/spec/lib/textrazor/dictionary_entry_spec.rb new file mode 100644 index 0000000..6caba1a --- /dev/null +++ b/spec/lib/textrazor/dictionary_entry_spec.rb @@ -0,0 +1,71 @@ +require "spec_helper" + +module TextRazor + + describe DictionaryEntry do + + let(:dictionary_entry) do + DictionaryEntry.create_from_hash(dictionary_entry_hash) + end + + let(:dictionary_entry_hash) do + { + "id" => "some-dictionary-entry", + "text" => "some text", + "data" => { "key" => ["value1", "value2"] } + } + end + + context "#create_from_hash" do + + it "should create a new instance" do + expect(dictionary_entry.id).to eq("some-dictionary-entry") + expect(dictionary_entry.text).to eq("some text") + expect(dictionary_entry.data).to eq("key" => ["value1", "value2"]) + end + end + + + context "#to_h" do + + it "should have the right values" do + expect(dictionary_entry.to_h).to eq(dictionary_entry_hash) + end + + context "with default values" do + + let(:dictionary_entry_hash) do + { + "text" => "some text" + } + end + + it "should not add them" do + expect(dictionary_entry.to_h).to eq("text" => "some text") + end + end + end + + context "#valid?" do + + context "when it is valid" do + + it "should return true" do + expect(dictionary_entry.valid?).to be true + end + + end + + context "when it is not valid" do + let(:dictionary_entry_hash) do + {} + end + + it "should return false" do + expect(dictionary_entry.valid?).to be false + end + end + end + + end +end From ca0f5c834dabdbbe95a255d87641af8f9f9d1d77 Mon Sep 17 00:00:00 2001 From: Richard Degenne Date: Thu, 14 Mar 2019 16:38:33 +0100 Subject: [PATCH 07/10] Added dictionary entry creation --- README.md | 6 ++++++ lib/textrazor/client.rb | 12 ++++++++++++ lib/textrazor/request.rb | 9 +++++++++ spec/lib/textrazor/client_spec.rb | 25 +++++++++++++++++++++++++ spec/lib/textrazor/request_spec.rb | 16 ++++++++++++++++ 5 files changed, 68 insertions(+) diff --git a/README.md b/README.md index 4ed5d4e..aab5710 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,12 @@ client = TextRazor::Client.new('api_key') client.create_dictionary('my-dictionary', case_insensitive: true) ``` +#### Adding entries to a dictionary + +``` +client.create_dictionary_entries('my-dictionary', [{id: 'my-entry', text: 'Text to be matched'}]) +``` + #### Deleting a dctionary ``` diff --git a/lib/textrazor/client.rb b/lib/textrazor/client.rb index afca591..a250dff 100644 --- a/lib/textrazor/client.rb +++ b/lib/textrazor/client.rb @@ -9,6 +9,7 @@ class Client UnsupportedCleanupMode = Class.new(StandardError) InvalidDictionary = Class.new(StandardError) + InvalidDictionaryEntry = Class.new(StandardError) DEFAULT_EXTRACTORS = ['entities', 'topics', 'words', 'phrases', 'dependency-trees', 'relations', 'entailments', 'senses'] @@ -45,6 +46,13 @@ def delete_dictionary(dictionary_id) Request.delete_dictionary(api_key, dictionary_id) end + def create_dictionary_entries(dictionary_id, dictionary_entry_hashes) + dictionary_entries = dictionary_entry_hashes.map do |entry_hash| + DictionaryEntry.new(entry_hash).tap { |e| assert_dictionary_entry(e) } + end + Request.create_dictionary_entries(api_key, dictionary_id, dictionary_entries) + end + def self.topics(api_key, text, options = {}) new(api_key, options.merge(extractors: ['topics'])). analyse(text). @@ -138,6 +146,10 @@ def assert_dictionary(dictionary) raise InvalidDictionary, "Dictionary is invalid" unless dictionary.valid? end + def assert_dictionary_entry(entry) + raise InvalidDictionaryEntry, "Entry is invalid" unless entry.valid? + end + end end diff --git a/lib/textrazor/request.rb b/lib/textrazor/request.rb index 51c0c47..982f50c 100644 --- a/lib/textrazor/request.rb +++ b/lib/textrazor/request.rb @@ -43,6 +43,15 @@ def self.delete_dictionary(api_key, dictionary_id) true end + def self.create_dictionary_entries(api_key, dictionary_id, dictionary_entries) + ::RestClient.post( + url("entities/#{dictionary_id}/"), + dictionary_entries.map(&:to_h).to_json, + build_headers(api_key) + ) + dictionary_entries + end + def self.url(path = '/') File.join(TextRazor.configuration.url, path) end diff --git a/spec/lib/textrazor/client_spec.rb b/spec/lib/textrazor/client_spec.rb index 3ac99ec..b53a85d 100644 --- a/spec/lib/textrazor/client_spec.rb +++ b/spec/lib/textrazor/client_spec.rb @@ -195,6 +195,31 @@ module TextRazor end end + context "#create_dictionary_entries" do + + let(:dictionary_entries_hash) do + [{text: "text"}] + end + + it "make correct calls" do + expect(Request).to receive(:create_dictionary_entries). + with(api_key, "dictionary_id", all(a_kind_of(DictionaryEntry))) + client.create_dictionary_entries("dictionary_id", dictionary_entries_hash) + end + + context "with an invalid dictionary" do + + let(:dictionary_entries_hash) do + [{ text: '' }] + end + + it "raises an exception" do + expect { client.create_dictionary_entries("dictionary_id", dictionary_entries_hash) }. + to raise_error(Client::InvalidDictionaryEntry) + end + end + end + context ".topics" do it "makes correct calls" do diff --git a/spec/lib/textrazor/request_spec.rb b/spec/lib/textrazor/request_spec.rb index 469364c..5530cb1 100644 --- a/spec/lib/textrazor/request_spec.rb +++ b/spec/lib/textrazor/request_spec.rb @@ -108,6 +108,22 @@ module TextRazor end end + context ".create_dictionary_entries" do + + let(:dictionary_entries) do + [DictionaryEntry.new(id: "id", text: "text", data: {key: ["value1", "value2"]})] + end + + it "should make correct calls" do + expect(::RestClient).to receive(:post). + with("https://api.textrazor.com/entities/dictionary_id/", + [{ "id" => "id", "text" => "text", "data" => { "key" => ["value1", "value2"] }}].to_json, + accept_encoding: 'gzip', x_textrazor_key: 'api_key') + + Request.create_dictionary_entries(api_key, "dictionary_id", dictionary_entries) + end + end + end end From 958e262e75835251e4663041822dab678ad160c4 Mon Sep 17 00:00:00 2001 From: Richard Degenne Date: Thu, 14 Mar 2019 16:39:40 +0100 Subject: [PATCH 08/10] Added dictionary entry deletion --- README.md | 7 +++++++ lib/textrazor/client.rb | 4 ++++ lib/textrazor/request.rb | 8 ++++++++ spec/lib/textrazor/client_spec.rb | 8 ++++++++ spec/lib/textrazor/request_spec.rb | 11 +++++++++++ 5 files changed, 38 insertions(+) diff --git a/README.md b/README.md index aab5710..04954e7 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,13 @@ client.create_dictionary('my-dictionary', case_insensitive: true) client.create_dictionary_entries('my-dictionary', [{id: 'my-entry', text: 'Text to be matched'}]) ``` + +#### Deleting a dictionary entry + +``` +client.delete_dictionary_entry('my-dictionary', 'my-entry') +``` + #### Deleting a dctionary ``` diff --git a/lib/textrazor/client.rb b/lib/textrazor/client.rb index a250dff..5c2b310 100644 --- a/lib/textrazor/client.rb +++ b/lib/textrazor/client.rb @@ -53,6 +53,10 @@ def create_dictionary_entries(dictionary_id, dictionary_entry_hashes) Request.create_dictionary_entries(api_key, dictionary_id, dictionary_entries) end + def delete_dictionary_entry(dictionary_id, dictionary_entry_id) + Request.delete_dictionary_entry(api_key, dictionary_id, dictionary_entry_id) + end + def self.topics(api_key, text, options = {}) new(api_key, options.merge(extractors: ['topics'])). analyse(text). diff --git a/lib/textrazor/request.rb b/lib/textrazor/request.rb index 982f50c..35e2037 100644 --- a/lib/textrazor/request.rb +++ b/lib/textrazor/request.rb @@ -52,6 +52,14 @@ def self.create_dictionary_entries(api_key, dictionary_id, dictionary_entries) dictionary_entries end + def self.delete_dictionary_entry(api_key, dictionary_id, dictionary_entry_id) + ::RestClient.delete( + url("entities/#{dictionary_id}/#{dictionary_entry_id}"), + build_headers(api_key) + ) + true + end + def self.url(path = '/') File.join(TextRazor.configuration.url, path) end diff --git a/spec/lib/textrazor/client_spec.rb b/spec/lib/textrazor/client_spec.rb index b53a85d..519e63d 100644 --- a/spec/lib/textrazor/client_spec.rb +++ b/spec/lib/textrazor/client_spec.rb @@ -220,6 +220,14 @@ module TextRazor end end + context "#delete_dictionary_entry" do + it "make correct calls" do + expect(Request).to receive(:delete_dictionary_entry). + with(api_key, "dictionary_id", "dictionary_entry_id") + client.delete_dictionary_entry("dictionary_id", "dictionary_entry_id") + end + end + context ".topics" do it "makes correct calls" do diff --git a/spec/lib/textrazor/request_spec.rb b/spec/lib/textrazor/request_spec.rb index 5530cb1..b77f84c 100644 --- a/spec/lib/textrazor/request_spec.rb +++ b/spec/lib/textrazor/request_spec.rb @@ -124,6 +124,17 @@ module TextRazor end end + context ".delete_dictionary_entry" do + + it "should make correct calls" do + expect(::RestClient).to receive(:delete). + with("https://api.textrazor.com/entities/dictionary_id/dictionary_entry_id", + accept_encoding: 'gzip', x_textrazor_key: 'api_key' + ) + + Request.delete_dictionary_entry(api_key, "dictionary_id", "dictionary_entry_id") + end + end end end From 07dcd06be9ebac3278890703ebdcfb42e2f69d7a Mon Sep 17 00:00:00 2001 From: Richard Degenne Date: Thu, 14 Mar 2019 18:26:57 +0100 Subject: [PATCH 09/10] Added dictionary entry retrieval --- README.md | 8 ++++++++ lib/textrazor/client.rb | 8 ++++++++ lib/textrazor/request.rb | 7 +++++++ spec/lib/textrazor/request_spec.rb | 11 +++++++++++ 4 files changed, 34 insertions(+) diff --git a/README.md b/README.md index 04954e7..717a400 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,14 @@ client.create_dictionary('my-dictionary', case_insensitive: true) client.create_dictionary_entries('my-dictionary', [{id: 'my-entry', text: 'Text to be matched'}]) ``` +#### Getting entries from a dictionary + +``` +client.get_dictionary_entries('my-dictionary') + +# Using pagination +client.get_dictionary_entries('my-dictionary', limit: 20, offset: 0) +``` #### Deleting a dictionary entry diff --git a/lib/textrazor/client.rb b/lib/textrazor/client.rb index 5c2b310..9f45e73 100644 --- a/lib/textrazor/client.rb +++ b/lib/textrazor/client.rb @@ -42,6 +42,14 @@ def create_dictionary(id, **options) Request.create_dictionary(api_key, dictionary) end + def get_dictionary_entries(dictionary_id, limit: 0, offset: 0) + response = ApiResponse.new( + Request.get_dictionary_entries(api_key, dictionary_id, limit: limit, offset: offset) + ) + return [] unless response.raw_response.key?(:entries) + response.raw_response[:entries].map { |hash| DictionaryEntry.create_from_hash(hash) } + end + def delete_dictionary(dictionary_id) Request.delete_dictionary(api_key, dictionary_id) end diff --git a/lib/textrazor/request.rb b/lib/textrazor/request.rb index 35e2037..e8e27e7 100644 --- a/lib/textrazor/request.rb +++ b/lib/textrazor/request.rb @@ -35,6 +35,13 @@ def self.create_dictionary(api_key, dictionary, **options) dictionary end + def self.get_dictionary_entries(api_key, dictionary_id, limit:, offset:) + ::RestClient.get( + url("entities/#{dictionary_id}/_all?limit=#{limit}&offset=#{offset}"), + build_headers(api_key) + ) + end + def self.delete_dictionary(api_key, dictionary_id) ::RestClient.delete( url("entities/#{dictionary_id}"), diff --git a/spec/lib/textrazor/request_spec.rb b/spec/lib/textrazor/request_spec.rb index b77f84c..607d285 100644 --- a/spec/lib/textrazor/request_spec.rb +++ b/spec/lib/textrazor/request_spec.rb @@ -96,6 +96,17 @@ module TextRazor end end + context ".get_dictionary_entries" do + + it "should make correct calls" do + expect(::RestClient).to receive(:get). + with("https://api.textrazor.com/entities/id/_all?limit=10&offset=20", + accept_encoding: 'gzip', x_textrazor_key: 'api_key') + + Request.get_dictionary_entries(api_key, 'id', limit: 10, offset: 20) + end + end + context ".delete_dictionary" do it "should make correct calls" do From 1a501735915cdd3a45bb5756abb1cd40977920af Mon Sep 17 00:00:00 2001 From: Richard Degenne Date: Wed, 7 Apr 2021 12:33:55 +0200 Subject: [PATCH 10/10] Removed custom accept-encoding headers --- lib/textrazor/request.rb | 2 +- spec/lib/textrazor/request_spec.rb | 14 +++++++------- textrazor.gemspec | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/textrazor/request.rb b/lib/textrazor/request.rb index e8e27e7..f0aeeff 100644 --- a/lib/textrazor/request.rb +++ b/lib/textrazor/request.rb @@ -85,7 +85,7 @@ def self.build_query(text, options) end def self.build_headers(api_key) - { x_textrazor_key: api_key, accept_encoding: 'gzip' } + { x_textrazor_key: api_key } end end diff --git a/spec/lib/textrazor/request_spec.rb b/spec/lib/textrazor/request_spec.rb index 607d285..3573490 100644 --- a/spec/lib/textrazor/request_spec.rb +++ b/spec/lib/textrazor/request_spec.rb @@ -51,7 +51,7 @@ module TextRazor expect(::RestClient).to receive(:post). with("https://api.textrazor.com", { "text" => 'text', - "extractors" => "entities,topics,words,dependency-trees,relations,entailments" }, accept_encoding: 'gzip', x_textrazor_key: 'api_key') + "extractors" => "entities,topics,words,dependency-trees,relations,entailments" }, x_textrazor_key: 'api_key') Request.post(api_key, 'text', **options) end @@ -71,7 +71,7 @@ module TextRazor "cleanup.mode" => "raw", "cleanup.returnCleaned" => true, "cleanup.returnRaw" => true, "languageOverride" => 'fre', "entities.filterDbpediaTypes" => "type1", "entities.filterFreebaseTypes" => "type2" , "entities.allowOverlap" => false, "entities.enrichmentQueries" => "queries", "classifiers" => 'textrazor_iab'}, - accept_encoding: 'gzip', x_textrazor_key: 'api_key') + x_textrazor_key: 'api_key') Request.post(api_key, 'text', options) end @@ -90,7 +90,7 @@ module TextRazor expect(::RestClient).to receive(:put). with("https://api.textrazor.com/entities/id", { "matchType" => "token", "caseInsensitive" => true, "language" => "eng" }.to_json, - accept_encoding: 'gzip', x_textrazor_key: 'api_key') + x_textrazor_key: 'api_key') Request.create_dictionary(api_key, dictionary) end @@ -101,7 +101,7 @@ module TextRazor it "should make correct calls" do expect(::RestClient).to receive(:get). with("https://api.textrazor.com/entities/id/_all?limit=10&offset=20", - accept_encoding: 'gzip', x_textrazor_key: 'api_key') + x_textrazor_key: 'api_key') Request.get_dictionary_entries(api_key, 'id', limit: 10, offset: 20) end @@ -112,7 +112,7 @@ module TextRazor it "should make correct calls" do expect(::RestClient).to receive(:delete). with("https://api.textrazor.com/entities/id", - accept_encoding: 'gzip', x_textrazor_key: 'api_key' + x_textrazor_key: 'api_key' ) Request.delete_dictionary(api_key, "id") @@ -129,7 +129,7 @@ module TextRazor expect(::RestClient).to receive(:post). with("https://api.textrazor.com/entities/dictionary_id/", [{ "id" => "id", "text" => "text", "data" => { "key" => ["value1", "value2"] }}].to_json, - accept_encoding: 'gzip', x_textrazor_key: 'api_key') + x_textrazor_key: 'api_key') Request.create_dictionary_entries(api_key, "dictionary_id", dictionary_entries) end @@ -140,7 +140,7 @@ module TextRazor it "should make correct calls" do expect(::RestClient).to receive(:delete). with("https://api.textrazor.com/entities/dictionary_id/dictionary_entry_id", - accept_encoding: 'gzip', x_textrazor_key: 'api_key' + x_textrazor_key: 'api_key' ) Request.delete_dictionary_entry(api_key, "dictionary_id", "dictionary_entry_id") diff --git a/textrazor.gemspec b/textrazor.gemspec index e184cb4..d9e9cd4 100644 --- a/textrazor.gemspec +++ b/textrazor.gemspec @@ -18,7 +18,7 @@ Gem::Specification.new do |spec| spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) spec.require_paths = ["lib"] - spec.add_dependency "rest-client" + spec.add_dependency "rest-client", "~> 2.1" spec.add_dependency "fast_open_struct" spec.add_development_dependency "bundler", "~> 1.3"