diff --git a/Appraisals b/Appraisals
index 2b84303..834d201 100644
--- a/Appraisals
+++ b/Appraisals
@@ -17,24 +17,20 @@ appraise "rack_1_6" do
   gem "rack-test", ">= 0.6"
 end
 
-appraise 'rails_6_0' do
-  gem 'actionpack', '~> 6.0.0'
-  gem 'activesupport', '~> 6.0.0'
+appraise 'rails_6-0' do
+  gem 'railties', '~> 6.0.0'
 end
 
 appraise 'rails_5-2' do
-  gem 'actionpack', '~> 5.2.0'
-  gem 'activesupport', '~> 5.2.0'
+  gem 'railties', '~> 5.2.0'
 end
 
 appraise 'rails_5-1' do
-  gem 'actionpack', '~> 5.1.0'
-  gem 'activesupport', '~> 5.1.0'
+  gem 'railties', '~> 5.1.0'
 end
 
 appraise 'rails_4-2' do
-  gem 'actionpack', '~> 4.2.0'
-  gem 'activesupport', '~> 4.2.0'
+  gem 'railties', '~> 4.2.0'
 
   # Override rack-test version constraint by making it more loose
   # so it's compatible with actionpack 4.2.x
diff --git a/gemfiles/rails_4_2.gemfile b/gemfiles/rails_4_2.gemfile
index eb09083..055cf9f 100644
--- a/gemfiles/rails_4_2.gemfile
+++ b/gemfiles/rails_4_2.gemfile
@@ -2,8 +2,7 @@
 
 source "https://rubygems.org"
 
-gem "actionpack", "~> 4.2.0"
-gem "activesupport", "~> 4.2.0"
+gem "railties", "~> 4.2.0"
 gem "rack-test", ">= 0.6"
 
 gemspec path: "../"
diff --git a/gemfiles/rails_5_1.gemfile b/gemfiles/rails_5_1.gemfile
index d05d8e4..66a5a0b 100644
--- a/gemfiles/rails_5_1.gemfile
+++ b/gemfiles/rails_5_1.gemfile
@@ -2,7 +2,6 @@
 
 source "https://rubygems.org"
 
-gem "actionpack", "~> 5.1.0"
-gem "activesupport", "~> 5.1.0"
+gem "railties", "~> 5.1.0"
 
 gemspec path: "../"
diff --git a/gemfiles/rails_5_2.gemfile b/gemfiles/rails_5_2.gemfile
index d12b8c6..8b2627f 100644
--- a/gemfiles/rails_5_2.gemfile
+++ b/gemfiles/rails_5_2.gemfile
@@ -2,7 +2,6 @@
 
 source "https://rubygems.org"
 
-gem "actionpack", "~> 5.2.0"
-gem "activesupport", "~> 5.2.0"
+gem "railties", "~> 5.2.0"
 
 gemspec path: "../"
diff --git a/gemfiles/rails_6_0.gemfile b/gemfiles/rails_6_0.gemfile
index 8e0b558..4cd55a8 100644
--- a/gemfiles/rails_6_0.gemfile
+++ b/gemfiles/rails_6_0.gemfile
@@ -2,7 +2,6 @@
 
 source "https://rubygems.org"
 
-gem "actionpack", "~> 6.0.0"
-gem "activesupport", "~> 6.0.0"
+gem "railties", "~> 6.0.0"
 
 gemspec path: "../"
diff --git a/lib/rack/attack.rb b/lib/rack/attack.rb
index e38b4ba..c8d4ad1 100644
--- a/lib/rack/attack.rb
+++ b/lib/rack/attack.rb
@@ -6,7 +6,7 @@ require 'rack/attack/path_normalizer'
 require 'rack/attack/request'
 require "ipaddr"
 
-require 'rack/attack/railtie' if defined?(Rails)
+require 'rack/attack/railtie' if defined?(::Rails)
 
 module Rack
   class Attack
diff --git a/lib/rack/attack/railtie.rb b/lib/rack/attack/railtie.rb
index 5136560..973bebd 100644
--- a/lib/rack/attack/railtie.rb
+++ b/lib/rack/attack/railtie.rb
@@ -2,10 +2,23 @@
 
 module Rack
   class Attack
-    class Railtie < Rails::Railtie
-      config.after_initialize do |app|
-        include_middleware = app.middleware.none? { |m| m == Rack::Attack }
-        app.middleware.use(Rack::Attack) if include_middleware
+    class Railtie < ::Rails::Railtie
+      initializer 'rack.attack.middleware', after: :load_config_initializers, before: :build_middleware_stack do |app|
+        middlewares = app.config.middleware
+
+        operations =
+          if Gem::Version.new(Rails::VERSION::STRING) >= Gem::Version.new("5")
+            middlewares.send(:operations) + middlewares.send(:delete_operations)
+          else
+            middlewares.instance_variable_get(:@operations)
+          end
+
+        use_middleware = operations.none? do |operation|
+          middleware = operation[1]
+          middleware.include?(Rack::Attack)
+        end
+
+        middlewares.use(Rack::Attack) if use_middleware
       end
     end
   end
diff --git a/rack-attack.gemspec b/rack-attack.gemspec
index 94d627e..dfdd09d 100644
--- a/rack-attack.gemspec
+++ b/rack-attack.gemspec
@@ -46,8 +46,5 @@ Gem::Specification.new do |s|
     s.add_development_dependency 'byebug', '~> 11.0'
   end
 
-  # The following are potential runtime dependencies users may have,
-  # which rack-attack uses only for testing compatibility in test suite.
-  s.add_development_dependency 'actionpack', '~> 5.2'
-  s.add_development_dependency 'activesupport', '~> 5.2'
+  s.add_development_dependency 'railties', '>= 4.2'
 end
diff --git a/spec/acceptance/rails_middleware_spec.rb b/spec/acceptance/rails_middleware_spec.rb
new file mode 100644
index 0000000..c9053be
--- /dev/null
+++ b/spec/acceptance/rails_middleware_spec.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative "../spec_helper"
+
+if defined?(Rails) && Gem::Version.new(Rails::VERSION::STRING) >= Gem::Version.new("5")
+  describe "Middleware for Rails" do
+    before do
+      @app = Class.new(Rails::Application) do
+        config.eager_load = false
+        config.logger = Logger.new(nil) # avoid creating the log/ directory automatically
+        config.cache_store = :null_store # avoid creating tmp/ directory for cache
+      end
+    end
+
+    it "is enabled by default" do
+      @app.initialize!
+      assert_equal 1, @app.middleware.count(Rack::Attack)
+    end
+
+    it "is not added when it was added explicitly" do
+      @app.config.middleware.use(Rack::Attack)
+      @app.initialize!
+      assert_equal 1, @app.middleware.count(Rack::Attack)
+    end
+
+    it "is not added when it was explicitly deleted" do
+      @app.config.middleware.delete(Rack::Attack)
+      @app.initialize!
+      refute @app.middleware.include?(Rack::Attack)
+    end
+  end
+end
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 49d8873..a1728cb 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -5,8 +5,7 @@ require "bundler/setup"
 require "minitest/autorun"
 require "minitest/pride"
 require "rack/test"
-require 'active_support'
-require 'action_dispatch'
+require "rails"
 
 require "rack/attack"
 
@@ -30,6 +29,7 @@ class MiniTest::Spec
   include Rack::Test::Methods
 
   before do
+    Rails.cache = nil
     @_original_throttled_response = Rack::Attack.throttled_response
     @_original_blocklisted_response = Rack::Attack.blocklisted_response
   end