From ed675d9583397b7e57db93fa5b47fffd39fec774 Mon Sep 17 00:00:00 2001 From: klea Date: Fri, 21 Aug 2026 03:39:04 +0000 Subject: [PATCH 1/3] bot: Handle UAs by loading the JSON Trying to remove CouchDB --- bot/brain.rb | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/bot/brain.rb b/bot/brain.rb index 5f1e6b9c..0d61c6da 100644 --- a/bot/brain.rb +++ b/bot/brain.rb @@ -1,4 +1,5 @@ require 'addressable/uri' +require 'json' require File.expand_path('../../lib/job', __FILE__) require File.expand_path('../../lib/pipeline_info', __FILE__) @@ -125,7 +126,20 @@ def request_archive(m, target, params, depth=:inf, url_file=false) if h[:user_agent_alias] ua_alias = h[:user_agent_alias] - user_agent = couchdb.user_agent_for_alias(ua_alias) + + user_agents = {} + + Dir.glob(File.join(File.expand_path('../../db/user_agents', __FILE__), "*.json")).each do |file| + data = JSON.parse(File.read(file)) + + data["agents"].each do |agent| + agent["aliases"].each do |alias_name| + user_agents[alias_name] = agent["name"] + end + end + end + + user_agent = user_agents[ua_alias] if !user_agent reply m, %Q{Sorry, I don't know what the user agent "#{ua_alias}" is.} From e29a425ea500e2da0e6e6e1239e7b693ca0437d5 Mon Sep 17 00:00:00 2001 From: klea Date: Fri, 21 Aug 2026 05:09:18 +0000 Subject: [PATCH 2/3] bot: Remove CouchDB usage of ignore handling --- bot/brain.rb | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/bot/brain.rb b/bot/brain.rb index 0d61c6da..7d4f559a 100644 --- a/bot/brain.rb +++ b/bot/brain.rb @@ -274,7 +274,23 @@ def add_ignore_sets(m, job, names, need_authorization=true) return unless names && !names.empty? - ignore_pairs = couchdb.resolve_ignore_sets(names) + unknown = [] + ignore_pairs = [] + + names.each do |name| + file = File.join(File.expand_path('../../db/ignore_patterns', __FILE__), "#{name}.json") + + if File.file?(file) + data = JSON.parse(File.read(file)) + + data.fetch('patterns', []).map do |pattern| + ignore_pairs << [name, pattern] + end + else + unknown << name + [] + end + end resolved = ignore_pairs.map(&:first).uniq patterns = ignore_pairs.map(&:last) @@ -287,8 +303,6 @@ def add_ignore_sets(m, job, names, need_authorization=true) reply m, %Q{Added #{patterns.length} patterns from ignore set #{resolved.first} to job #{job.ident}.} end - unknown = names - resolved - if !unknown.empty? reply m, "The following sets are unknown: #{unknown.join(', ')}" end From 552499daff24ccb0dd01b572cdceb3a15bbe9407 Mon Sep 17 00:00:00 2001 From: klea Date: Fri, 21 Aug 2026 05:12:28 +0000 Subject: [PATCH 3/3] Remove couchdb uses I think there might be a few missing --- bot/bot.rb | 6 +----- bot/brain.rb | 4 +--- cogs/ignore_pattern_updater.rb | 8 -------- cogs/start.rb | 14 -------------- cogs/user_agent_updater.rb | 12 ------------ 5 files changed, 2 insertions(+), 42 deletions(-) delete mode 100644 cogs/ignore_pattern_updater.rb delete mode 100644 cogs/user_agent_updater.rb diff --git a/bot/bot.rb b/bot/bot.rb index 467caf4c..47b08f4d 100644 --- a/bot/bot.rb +++ b/bot/bot.rb @@ -7,7 +7,6 @@ require File.expand_path('../command_patterns', __FILE__) require File.expand_path('../finish_notifier', __FILE__) require File.expand_path('../pipeline_notifier', __FILE__) -require File.expand_path('../../lib/couchdb', __FILE__) opts = Trollop.options do opt :server, 'IRC server, expressed as a URI (irc://SERVER:PORT or ircs://SERVER:PORT for SSL)', :type => String @@ -17,8 +16,6 @@ opt :schemes, 'Comma-separated list of acceptable URI schemes', :default => 'http,https,ftp' opt :redis, 'URL of Redis server', :default => ENV['REDIS_URL'] || 'redis://localhost:6379/0' opt :password, 'IRC server password', :default => nil, :type => String - opt :db, 'URL of CouchDB database', :default => ENV['COUCHDB_URL'] || 'http://localhost:5984/archivebot' - opt :db_credentials, 'Credentials for CouchDB database (USERNAME:PASSWORD)', :type => String, :default => nil end redis = Redis.new(:url => opts[:redis]) @@ -54,8 +51,7 @@ c.messages_per_second = 1.0 end - couchdb = Couchdb.new(URI(opts[:db]), opts[:db_credentials]) - brain = Brain.new(schemes, redis, couchdb) + brain = Brain.new(schemes, redis) on :message, CommandPatterns::ARCHIVE do |m, target, params| brain.request_archive(m, target, params) diff --git a/bot/brain.rb b/bot/brain.rb index 7d4f559a..d346935e 100644 --- a/bot/brain.rb +++ b/bot/brain.rb @@ -18,13 +18,11 @@ class Brain include AddIgnoreSets include PipelineOptions - attr_reader :couchdb attr_reader :redis attr_reader :schemes attr_reader :url_pattern - def initialize(schemes, redis, couchdb) - @couchdb = couchdb + def initialize(schemes, redis) @redis = redis @schemes = schemes @url_pattern ||= %r{(?:#{schemes.join('|')})://.+} diff --git a/cogs/ignore_pattern_updater.rb b/cogs/ignore_pattern_updater.rb deleted file mode 100644 index c1edfa05..00000000 --- a/cogs/ignore_pattern_updater.rb +++ /dev/null @@ -1,8 +0,0 @@ -require File.expand_path('../couchdb_doc_updater', __FILE__) -require File.expand_path('../ignore_pattern_id_generation', __FILE__) - -# Watches the ignore patterns directory and installs new ignore pattern sets as -# they're updated. -class IgnorePatternUpdater < CouchdbDocUpdater - include IgnorePatternIdGeneration -end diff --git a/cogs/start.rb b/cogs/start.rb index 198ef26b..619f5708 100644 --- a/cogs/start.rb +++ b/cogs/start.rb @@ -2,8 +2,6 @@ require 'trollop' require 'uri' -require File.expand_path('../ignore_pattern_updater', __FILE__) -require File.expand_path('../user_agent_updater', __FILE__) require File.expand_path('../../lib/job', __FILE__) require File.expand_path('../../lib/redis_subscriber', __FILE__) require File.expand_path('../../lib/shared_config', __FILE__) @@ -35,16 +33,6 @@ def on_receive(ident) Reaper.supervise_as :reaper, opts[:redis] TwitterTweeter.supervise_as :twitter_tweeter, opts[:redis], opts[:twitter_config] -ignore_patterns_path = File.expand_path('../../db/ignore_patterns', __FILE__) - -IgnorePatternUpdater.supervise_as :ignore_pattern_updater, - ignore_patterns_path, db_uri, opts[:db_credentials] - -user_agents_path = File.expand_path('../../db/user_agents', __FILE__) - -UserAgentUpdater.supervise_as :user_agent_updater, - user_agents_path, db_uri, opts[:db_credentials] - # This should be started after all actors it references have started, which is # why it's the last actor to start up. if opts[:twitter_config] @@ -55,8 +43,6 @@ def on_receive(ident) if opts[:twitter_config] Celluloid::Actor[:broadcaster].stop end - Celluloid::Actor[:ignore_pattern_updater].stop - Celluloid::Actor[:user_agent_updater].stop end trap('INT') do diff --git a/cogs/user_agent_updater.rb b/cogs/user_agent_updater.rb deleted file mode 100644 index 56bcdf55..00000000 --- a/cogs/user_agent_updater.rb +++ /dev/null @@ -1,12 +0,0 @@ -require File.expand_path('../couchdb_doc_updater', __FILE__) - -# Watches the user agents directory and installs new user agent aliases as -# they're updated. -class UserAgentUpdater < CouchdbDocUpdater - def id_from_path(path) - basename = File.basename(path) - doc_id = basename.sub(File.extname(path), '') - - "user_agents:#{doc_id}" - end -end