diff --git a/spec/acceptance/stores/active_support_dalli_store_spec.rb b/spec/acceptance/stores/active_support_dalli_store_spec.rb
new file mode 100644
index 0000000..b3be95f
--- /dev/null
+++ b/spec/acceptance/stores/active_support_dalli_store_spec.rb
@@ -0,0 +1,39 @@
+require_relative "../../spec_helper"
+require_relative "../../support/cache_store_helper"
+
+require "active_support/cache/dalli_store"
+require "timecop"
+
+describe "ActiveSupport::Cache::DalliStore as a cache backend" do
+  before do
+    Rack::Attack.cache.store = ActiveSupport::Cache::DalliStore.new
+  end
+
+  after do
+    Rack::Attack.cache.store.clear
+  end
+
+  it_works_for_cache_backed_features
+
+  it "doesn't leak keys" do
+    Rack::Attack.throttle("by ip", limit: 1, period: 1) do |request|
+      request.ip
+    end
+
+    key = nil
+
+    # Freeze time during these statement to be sure that the key used by rack attack is the same
+    # we pre-calculate in local variable `key`
+    Timecop.freeze do
+      key = "rack::attack:#{Time.now.to_i}:by ip:1.2.3.4"
+
+      get "/", {}, "REMOTE_ADDR" => "1.2.3.4"
+    end
+
+    assert Rack::Attack.cache.store.fetch(key)
+
+    sleep 2.1
+
+    assert_nil Rack::Attack.cache.store.fetch(key)
+  end
+end
diff --git a/spec/acceptance/stores/mem_cache_store_spec.rb b/spec/acceptance/stores/active_support_mem_cache_store_spec.rb
similarity index 92%
rename from spec/acceptance/stores/mem_cache_store_spec.rb
rename to spec/acceptance/stores/active_support_mem_cache_store_spec.rb
index 6c593f1..1a4e7f7 100644
--- a/spec/acceptance/stores/mem_cache_store_spec.rb
+++ b/spec/acceptance/stores/active_support_mem_cache_store_spec.rb
@@ -3,7 +3,7 @@ require_relative "../../support/cache_store_helper"
 
 require "timecop"
 
-describe "MemCacheStore as a cache backend" do
+describe "ActiveSupport::Cache::MemCacheStore as a cache backend" do
   before do
     Rack::Attack.cache.store = ActiveSupport::Cache::MemCacheStore.new
   end
diff --git a/spec/acceptance/stores/active_support_memory_store_spec.rb b/spec/acceptance/stores/active_support_memory_store_spec.rb
new file mode 100644
index 0000000..94de429
--- /dev/null
+++ b/spec/acceptance/stores/active_support_memory_store_spec.rb
@@ -0,0 +1,38 @@
+require_relative "../../spec_helper"
+require_relative "../../support/cache_store_helper"
+
+require "timecop"
+
+describe "ActiveSupport::Cache::MemoryStore as a cache backend" do
+  before do
+    Rack::Attack.cache.store = ActiveSupport::Cache::MemoryStore.new
+  end
+
+  after do
+    Rack::Attack.cache.store.clear
+  end
+
+  it_works_for_cache_backed_features
+
+  it "doesn't leak keys" do
+    Rack::Attack.throttle("by ip", limit: 1, period: 1) do |request|
+      request.ip
+    end
+
+    key = nil
+
+    # Freeze time during these statement to be sure that the key used by rack attack is the same
+    # we pre-calculate in local variable `key`
+    Timecop.freeze do
+      key = "rack::attack:#{Time.now.to_i}:by ip:1.2.3.4"
+
+      get "/", {}, "REMOTE_ADDR" => "1.2.3.4"
+    end
+
+    assert Rack::Attack.cache.store.fetch(key)
+
+    sleep 2.1
+
+    assert_nil Rack::Attack.cache.store.fetch(key)
+  end
+end
diff --git a/spec/acceptance/stores/redis_cache_store_pooled_spec.rb b/spec/acceptance/stores/active_support_redis_cache_store_pooled_spec.rb
similarity index 92%
rename from spec/acceptance/stores/redis_cache_store_pooled_spec.rb
rename to spec/acceptance/stores/active_support_redis_cache_store_pooled_spec.rb
index cd54b30..8c006bb 100644
--- a/spec/acceptance/stores/redis_cache_store_pooled_spec.rb
+++ b/spec/acceptance/stores/active_support_redis_cache_store_pooled_spec.rb
@@ -4,7 +4,7 @@ require_relative "../../support/cache_store_helper"
 require "timecop"
 
 if ActiveSupport.version >= Gem::Version.new("5.2.0")
-  describe "RedisCacheStore (pooled) as a cache backend" do
+  describe "ActiveSupport::Cache::RedisCacheStore (pooled) as a cache backend" do
     before do
       Rack::Attack.cache.store = ActiveSupport::Cache::RedisCacheStore.new(pool_size: 2)
     end
diff --git a/spec/acceptance/stores/redis_cache_store_spec.rb b/spec/acceptance/stores/active_support_redis_cache_store_spec.rb
similarity index 92%
rename from spec/acceptance/stores/redis_cache_store_spec.rb
rename to spec/acceptance/stores/active_support_redis_cache_store_spec.rb
index b819749..495a6a2 100644
--- a/spec/acceptance/stores/redis_cache_store_spec.rb
+++ b/spec/acceptance/stores/active_support_redis_cache_store_spec.rb
@@ -4,7 +4,7 @@ require_relative "../../support/cache_store_helper"
 require "timecop"
 
 if ActiveSupport.version >= Gem::Version.new("5.2.0")
-  describe "RedisCacheStore as a cache backend" do
+  describe "ActiveSupport::Cache::RedisCacheStore as a cache backend" do
     before do
       Rack::Attack.cache.store = ActiveSupport::Cache::RedisCacheStore.new
     end
diff --git a/spec/acceptance/stores/active_support_redis_store_spec.rb b/spec/acceptance/stores/active_support_redis_store_spec.rb
new file mode 100644
index 0000000..0b446e4
--- /dev/null
+++ b/spec/acceptance/stores/active_support_redis_store_spec.rb
@@ -0,0 +1,39 @@
+require_relative "../../spec_helper"
+require_relative "../../support/cache_store_helper"
+
+require "redis-activesupport"
+require "timecop"
+
+describe "ActiveSupport::Cache::RedisStore as a cache backend" do
+  before do
+    Rack::Attack.cache.store = ActiveSupport::Cache::RedisStore.new
+  end
+
+  after do
+    Rack::Attack.cache.store.flushdb
+  end
+
+  it_works_for_cache_backed_features
+
+  it "doesn't leak keys" do
+    Rack::Attack.throttle("by ip", limit: 1, period: 1) do |request|
+      request.ip
+    end
+
+    key = nil
+
+    # Freeze time during these statement to be sure that the key used by rack attack is the same
+    # we pre-calculate in local variable `key`
+    Timecop.freeze do
+      key = "rack::attack:#{Time.now.to_i}:by ip:1.2.3.4"
+
+      get "/", {}, "REMOTE_ADDR" => "1.2.3.4"
+    end
+
+    assert Rack::Attack.cache.store.read(key)
+
+    sleep 2.1
+
+    assert_nil Rack::Attack.cache.store.read(key)
+  end
+end
diff --git a/spec/acceptance/stores/connection_pool_dalli_client_spec.rb b/spec/acceptance/stores/connection_pool_dalli_client_spec.rb
new file mode 100644
index 0000000..a852d03
--- /dev/null
+++ b/spec/acceptance/stores/connection_pool_dalli_client_spec.rb
@@ -0,0 +1,40 @@
+require_relative "../../spec_helper"
+require_relative "../../support/cache_store_helper"
+
+require "connection_pool"
+require "dalli"
+require "timecop"
+
+describe "ConnectionPool with Dalli::Client as a cache backend" do
+  before do
+    Rack::Attack.cache.store = ConnectionPool.new { Dalli::Client.new }
+  end
+
+  after do
+    Rack::Attack.cache.store.with { |client| client.flush_all }
+  end
+
+  it_works_for_cache_backed_features
+
+  it "doesn't leak keys" do
+    Rack::Attack.throttle("by ip", limit: 1, period: 1) do |request|
+      request.ip
+    end
+
+    key = nil
+
+    # Freeze time during these statement to be sure that the key used by rack attack is the same
+    # we pre-calculate in local variable `key`
+    Timecop.freeze do
+      key = "rack::attack:#{Time.now.to_i}:by ip:1.2.3.4"
+
+      get "/", {}, "REMOTE_ADDR" => "1.2.3.4"
+    end
+
+    assert(Rack::Attack.cache.store.with { |client| client.fetch(key) })
+
+    sleep 2.1
+
+    assert_nil(Rack::Attack.cache.store.with { |client| client.fetch(key) })
+  end
+end
diff --git a/spec/acceptance/stores/dalli_client_spec.rb b/spec/acceptance/stores/dalli_client_spec.rb
new file mode 100644
index 0000000..e01ab3f
--- /dev/null
+++ b/spec/acceptance/stores/dalli_client_spec.rb
@@ -0,0 +1,39 @@
+require_relative "../../spec_helper"
+require_relative "../../support/cache_store_helper"
+
+require "dalli"
+require "timecop"
+
+describe "Dalli::Client as a cache backend" do
+  before do
+    Rack::Attack.cache.store = Dalli::Client.new
+  end
+
+  after do
+    Rack::Attack.cache.store.flush_all
+  end
+
+  it_works_for_cache_backed_features
+
+  it "doesn't leak keys" do
+    Rack::Attack.throttle("by ip", limit: 1, period: 1) do |request|
+      request.ip
+    end
+
+    key = nil
+
+    # Freeze time during these statement to be sure that the key used by rack attack is the same
+    # we pre-calculate in local variable `key`
+    Timecop.freeze do
+      key = "rack::attack:#{Time.now.to_i}:by ip:1.2.3.4"
+
+      get "/", {}, "REMOTE_ADDR" => "1.2.3.4"
+    end
+
+    assert Rack::Attack.cache.store.fetch(key)
+
+    sleep 2.1
+
+    assert_nil Rack::Attack.cache.store.fetch(key)
+  end
+end
diff --git a/spec/acceptance/stores/redis_store_spec.rb b/spec/acceptance/stores/redis_store_spec.rb
new file mode 100644
index 0000000..17e5a0a
--- /dev/null
+++ b/spec/acceptance/stores/redis_store_spec.rb
@@ -0,0 +1,39 @@
+require_relative "../../spec_helper"
+require_relative "../../support/cache_store_helper"
+
+require "redis-store"
+require "timecop"
+
+describe "ActiveSupport::Cache::RedisStore as a cache backend" do
+  before do
+    Rack::Attack.cache.store = ::Redis::Store.new
+  end
+
+  after do
+    Rack::Attack.cache.store.flushdb
+  end
+
+  it_works_for_cache_backed_features
+
+  it "doesn't leak keys" do
+    Rack::Attack.throttle("by ip", limit: 1, period: 1) do |request|
+      request.ip
+    end
+
+    key = nil
+
+    # Freeze time during these statement to be sure that the key used by rack attack is the same
+    # we pre-calculate in local variable `key`
+    Timecop.freeze do
+      key = "rack::attack:#{Time.now.to_i}:by ip:1.2.3.4"
+
+      get "/", {}, "REMOTE_ADDR" => "1.2.3.4"
+    end
+
+    assert Rack::Attack.cache.store.read(key)
+
+    sleep 2.1
+
+    assert_nil Rack::Attack.cache.store.read(key)
+  end
+end
diff --git a/spec/integration/rack_attack_cache_spec.rb b/spec/integration/rack_attack_cache_spec.rb
deleted file mode 100644
index 98f5568..0000000
--- a/spec/integration/rack_attack_cache_spec.rb
+++ /dev/null
@@ -1,124 +0,0 @@
-require_relative '../spec_helper'
-
-describe Rack::Attack::Cache do
-  # A convenience method for deleting a key from cache.
-  # Slightly different than @cache.delete, which adds a prefix.
-  def delete(key)
-    if @cache.store.respond_to?(:delete)
-      @cache.store.delete(key)
-    else
-      @cache.store.del(key)
-    end
-  end
-
-  def sleep_until_expired
-    sleep(@expires_in * 1.1) # Add 10% to reduce errors
-  end
-
-  require 'active_support/cache/dalli_store'
-  require 'active_support/cache/mem_cache_store'
-  require 'active_support/cache/redis_store'
-  require 'active_support/cache/redis_cache_store' if ActiveSupport.version.to_s.to_f >= 5.2
-  require 'connection_pool'
-
-  cache_stores = [
-    ActiveSupport::Cache::MemoryStore.new,
-    ActiveSupport::Cache::DalliStore.new("127.0.0.1"),
-    ActiveSupport::Cache::RedisStore.new("127.0.0.1"),
-    ActiveSupport::Cache::MemCacheStore.new("127.0.0.1"),
-    Dalli::Client.new,
-    ConnectionPool.new { Dalli::Client.new },
-    Redis::Store.new
-  ]
-
-  cache_stores << ActiveSupport::Cache::RedisCacheStore.new if defined?(ActiveSupport::Cache::RedisCacheStore)
-
-  cache_stores.each do |store|
-    store = Rack::Attack::StoreProxy.build(store)
-
-    describe "with #{store.class}" do
-      before do
-        @cache = Rack::Attack::Cache.new
-        @key = "rack::attack:cache-test-key"
-        @expires_in = 1
-        @cache.store = store
-        delete(@key)
-      end
-
-      after { delete(@key) }
-
-      describe "do_count once" do
-        it "should be 1" do
-          @cache.send(:do_count, @key, @expires_in).must_equal 1
-        end
-      end
-
-      describe "do_count twice" do
-        it "must be 2" do
-          @cache.send(:do_count, @key, @expires_in)
-          @cache.send(:do_count, @key, @expires_in).must_equal 2
-        end
-      end
-
-      describe "do_count after expires_in" do
-        it "must be 1" do
-          @cache.send(:do_count, @key, @expires_in)
-          sleep_until_expired
-          @cache.send(:do_count, @key, @expires_in).must_equal 1
-        end
-      end
-
-      describe "write" do
-        it "should write a value to the store with prefix" do
-          @cache.write("cache-test-key", "foobar", 1)
-          store.read(@key).must_equal "foobar"
-        end
-      end
-
-      describe "write after expiry" do
-        it "must not have a value" do
-          @cache.write("cache-test-key", "foobar", @expires_in)
-          sleep_until_expired
-          store.read(@key).must_be :nil?
-        end
-      end
-
-      describe "read" do
-        it "must read the value with a prefix" do
-          store.write(@key, "foobar", :expires_in => @expires_in)
-          @cache.read("cache-test-key").must_equal "foobar"
-        end
-      end
-
-      describe "delete" do
-        it "must delete the value" do
-          store.write(@key, "foobar", :expires_in => @expires_in)
-          @cache.read('cache-test-key').must_equal "foobar"
-          store.delete(@key)
-          assert_nil @cache.read('cache-test-key')
-        end
-      end
-
-      describe "cache#delete" do
-        it "must delete the value" do
-          @cache.write("cache-test-key", "foobar", 1)
-          store.read(@key).must_equal "foobar"
-          @cache.delete('cache-test-key')
-          store.read(@key).must_be :nil?
-        end
-      end
-
-      describe "reset_count" do
-        it "must delete the value" do
-          period = 1.minute
-          unprefixed_key = 'cache-test-key'
-          @cache.count(unprefixed_key, period)
-          period_key, _ = @cache.send(:key_and_expiry, 'cache-test-key', period)
-          store.read(period_key).to_i.must_equal 1
-          @cache.reset_count(unprefixed_key, period)
-          assert_nil store.read(period_key)
-        end
-      end
-    end
-  end
-end