From 87ffbb334d51adf872db2faaf087c98d25b3025d Mon Sep 17 00:00:00 2001 From: benjessop12 Date: Wed, 8 Apr 2020 17:11:17 +0100 Subject: [PATCH 1/5] OpencBot configuration from Analysis App This commit allows a fetcher bot to retrieve information about the bot from Analysis App's bots.json output. We will need the jurisdiction_code to be able to pull the data from the JSON. I considered using a Proc select method of getting this data, as it reads nicer, bu this is slightly more performative. --- lib/openc_bot.rb | 18 +++++++++++------- lib/openc_bot/config.rb | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) create mode 100644 lib/openc_bot/config.rb diff --git a/lib/openc_bot.rb b/lib/openc_bot.rb index e24fe5c5..6efab0f2 100644 --- a/lib/openc_bot.rb +++ b/lib/openc_bot.rb @@ -84,17 +84,21 @@ def statsd_namespace StatsD.server = "sys1:8125" StatsD.logger = Logger.new("/dev/null") if bot_env != :production - if respond_to?(:inferred_jurisdiction_code) && inferred_jurisdiction_code - "fetcher_bot.#{bot_env}.#{inferred_jurisdiction_code}" - elsif is_a?(Module) - "fetcher_bot.#{bot_env}.#{name.downcase}" - else - "fetcher_bot.#{bot_env}.#{self.class.name.downcase}" - end + "fetcher_bot.#{bot_env}.#{jurisdiction_code}" .sub("companiesfetcher", "") end end + def jurisdiction_code + if respond_to?(:inferred_jurisdiction_code) && inferred_jurisdiction_code + inferred_jurisdiction_code + elsif is_a?(Module) + name.downcase + else + self.class.name.downcase + end + end + def db_name if is_a?(Module) "#{name.downcase}.db" diff --git a/lib/openc_bot/config.rb b/lib/openc_bot/config.rb new file mode 100644 index 00000000..5529536d --- /dev/null +++ b/lib/openc_bot/config.rb @@ -0,0 +1,14 @@ +require "json" +require "net/http" +require "ostruct" + +module OpencBot + # This is called from company_fetcher_bot + module Config + BOTS_JSON_URL = ENV.fetch("ANALYSIS_BOTS_JSON_URL").freeze + + def db_config + @db_config ||= OpenStruct.new(JSON.parse(Net::HTTP.get(URI(BOTS_JSON_URL))).select { |bot| bot["bot_id"] == "#{OpencBot::CompanyFetcherBot.jurisdiction_code}_companies" }.first) + end + end +end From b2bf0bf350c46a7dcb77f969d58aea552a8079bc Mon Sep 17 00:00:00 2001 From: benjessop12 Date: Mon, 13 Apr 2020 15:01:01 +0100 Subject: [PATCH 2/5] Accessor method to match new Analysis Field PR #53 in Analysis App will return a hash value based on what is stored in the database, to which we will allow the fetcher to access as an OpenStruct. --- lib/openc_bot/company_fetcher_bot.rb | 2 ++ lib/openc_bot/config.rb | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/openc_bot/company_fetcher_bot.rb b/lib/openc_bot/company_fetcher_bot.rb index 8a170131..37c64cae 100644 --- a/lib/openc_bot/company_fetcher_bot.rb +++ b/lib/openc_bot/company_fetcher_bot.rb @@ -2,6 +2,7 @@ require "openc_bot/helpers/incremental_search" require "openc_bot/helpers/alpha_search" require "openc_bot/helpers/reporting" +require "openc_bot/config" module OpencBot module CompanyFetcherBot @@ -9,6 +10,7 @@ module CompanyFetcherBot include OpencBot::Helpers::IncrementalSearch include OpencBot::Helpers::AlphaSearch include OpencBot::Helpers::Reporting + include OpencBot::Config STDOUT.sync = true STDERR.sync = true diff --git a/lib/openc_bot/config.rb b/lib/openc_bot/config.rb index 5529536d..83ffbeda 100644 --- a/lib/openc_bot/config.rb +++ b/lib/openc_bot/config.rb @@ -5,10 +5,22 @@ module OpencBot # This is called from company_fetcher_bot module Config - BOTS_JSON_URL = ENV.fetch("ANALYSIS_BOTS_JSON_URL").freeze + BOTS_JSON_URL = ENV["ANALYSIS_BOTS_JSON_URL"] + + def set_variables + config?.marshal_dump.each { |k, v| instance_variable_set("@#{k}", v) } unless config?.nil? + end def db_config - @db_config ||= OpenStruct.new(JSON.parse(Net::HTTP.get(URI(BOTS_JSON_URL))).select { |bot| bot["bot_id"] == "#{OpencBot::CompanyFetcherBot.jurisdiction_code}_companies" }.first) + return if BOTS_JSON_URL.nil? + + @db_config ||= JSON.parse(Net::HTTP.get(URI(BOTS_JSON_URL))).select { |bot| bot["bot_id"] == "#{jurisdiction_code}_companies" }.first + end + + def config? + OpenStruct.new(db_config["bot_config"]) + rescue NoMethodError + nil end end end From 980ede7f61d5ccd2f9ca777993b4899004652635 Mon Sep 17 00:00:00 2001 From: benjessop12 Date: Mon, 13 Apr 2020 15:05:43 +0100 Subject: [PATCH 3/5] Gemfile bump Travis was complaining about this, so this is to make the integration tests pass. --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index a1b8f245..27255a25 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -34,7 +34,7 @@ GEM addressable (2.7.0) public_suffix (>= 2.0.2, < 5.0) ast (2.4.0) - backports (3.17.0) + backports (3.17.1) byebug (10.0.2) coderay (1.1.2) concurrent-ruby (1.1.6) @@ -58,9 +58,9 @@ GEM nokogiri (1.9.1) mini_portile2 (~> 2.4.0) parallel (1.19.1) - parser (2.7.1.0) + parser (2.7.1.1) ast (~> 2.4.0) - pry (0.13.0) + pry (0.13.1) coderay (~> 1.1) method_source (~> 1.0) public_suffix (3.1.1) From 50c1e237c9130d02aad0fe124c3e2dbca938b94e Mon Sep 17 00:00:00 2001 From: benjessop12 Date: Tue, 14 Apr 2020 13:24:28 +0100 Subject: [PATCH 4/5] Spec tests for config.rb Spec tests to indicate the behaviour of the new module. --- spec/lib/openc_bot/config_spec.rb | 62 +++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 spec/lib/openc_bot/config_spec.rb diff --git a/spec/lib/openc_bot/config_spec.rb b/spec/lib/openc_bot/config_spec.rb new file mode 100644 index 00000000..71d64c2a --- /dev/null +++ b/spec/lib/openc_bot/config_spec.rb @@ -0,0 +1,62 @@ +require "spec_helper" +require "openc_bot" +require "openc_bot/company_fetcher_bot" +require "openc_bot/config" + +module ModuleThatIncludesConfig + extend OpencBot + extend OpencBot::CompanyFetcherBot + extend OpencBot::Config +end + +describe OpencBot::Config do + let(:expected_output) { OpenStruct.new(bot_config: { foo: "bar" }) } + + describe "#set_variables" do + before do + allow(ModuleThatIncludesConfig).to receive(:config?).and_return(expected_output) + end + + it "sends a request to config def" do + ModuleThatIncludesConfig.set_variables + expect(ModuleThatIncludesConfig).to have_received(:config?).twice + end + end + + describe "#db_config" do + context "when BOTS_JSON_URL is not set" do + it "returns nil" do + expect(ModuleThatIncludesConfig.db_config).to eq(nil) + end + end + + context "when BOTS_JSON_URL is set" do + before do + stub_const("OpencBot::Config::BOTS_JSON_URL", "someendpoint") + ModuleThatIncludesConfig.instance_variable_set(:@db_config, expected_output) + end + + it "returns an object" do + expect(ModuleThatIncludesConfig.db_config).to eq(expected_output) + end + end + end + + describe "#config?" do + context "when db_config returns an object" do + before do + allow(ModuleThatIncludesConfig).to receive(:db_config).and_return(expected_output) + end + + it "returns the bot_config struct from the db_config object that is queryable" do + expect(ModuleThatIncludesConfig.config?.foo).to eq("bar") + end + end + + context "when db_config does not return an object" do + it "returns nil" do + expect(ModuleThatIncludesConfig.config?).to eq(nil) + end + end + end +end From 8c39c96afb6f535e0e6409192f521add5dcf3045 Mon Sep 17 00:00:00 2001 From: benjessop12 Date: Wed, 15 Apr 2020 15:26:16 +0100 Subject: [PATCH 5/5] Return if called before There are many entry points for running an openc_bot, so we will need to return this if already called. --- lib/openc_bot/config.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/openc_bot/config.rb b/lib/openc_bot/config.rb index 83ffbeda..71b2a965 100644 --- a/lib/openc_bot/config.rb +++ b/lib/openc_bot/config.rb @@ -8,6 +8,8 @@ module Config BOTS_JSON_URL = ENV["ANALYSIS_BOTS_JSON_URL"] def set_variables + return if @db_config + config?.marshal_dump.each { |k, v| instance_variable_set("@#{k}", v) } unless config?.nil? end