diff --git a/spec/acceptance/stores/connection_pool_dalli_client_spec.rb b/spec/acceptance/stores/connection_pool_dalli_client_spec.rb
new file mode 100644
index 0000000..a852d03
--- /dev/null
+++ b/spec/acceptance/stores/connection_pool_dalli_client_spec.rb
@@ -0,0 +1,40 @@
+require_relative "../../spec_helper"
+require_relative "../../support/cache_store_helper"
+
+require "connection_pool"
+require "dalli"
+require "timecop"
+
+describe "ConnectionPool with Dalli::Client as a cache backend" do
+  before do
+    Rack::Attack.cache.store = ConnectionPool.new { Dalli::Client.new }
+  end
+
+  after do
+    Rack::Attack.cache.store.with { |client| client.flush_all }
+  end
+
+  it_works_for_cache_backed_features
+
+  it "doesn't leak keys" do
+    Rack::Attack.throttle("by ip", limit: 1, period: 1) do |request|
+      request.ip
+    end
+
+    key = nil
+
+    # Freeze time during these statement to be sure that the key used by rack attack is the same
+    # we pre-calculate in local variable `key`
+    Timecop.freeze do
+      key = "rack::attack:#{Time.now.to_i}:by ip:1.2.3.4"
+
+      get "/", {}, "REMOTE_ADDR" => "1.2.3.4"
+    end
+
+    assert(Rack::Attack.cache.store.with { |client| client.fetch(key) })
+
+    sleep 2.1
+
+    assert_nil(Rack::Attack.cache.store.with { |client| client.fetch(key) })
+  end
+end
diff --git a/spec/integration/rack_attack_cache_spec.rb b/spec/integration/rack_attack_cache_spec.rb
deleted file mode 100644
index 25148d8..0000000
--- a/spec/integration/rack_attack_cache_spec.rb
+++ /dev/null
@@ -1,106 +0,0 @@
-require_relative '../spec_helper'
-
-describe Rack::Attack::Cache do
-  # A convenience method for deleting a key from cache.
-  # Slightly different than @cache.delete, which adds a prefix.
-  def delete(key)
-    if @cache.store.respond_to?(:delete)
-      @cache.store.delete(key)
-    else
-      @cache.store.del(key)
-    end
-  end
-
-  def sleep_until_expired
-    sleep(@expires_in * 1.1) # Add 10% to reduce errors
-  end
-
-  require 'connection_pool'
-
-  store = Rack::Attack::StoreProxy.build(ConnectionPool.new { Dalli::Client.new })
-
-  describe "with #{store.class}" do
-    before do
-      @cache = Rack::Attack::Cache.new
-      @key = "rack::attack:cache-test-key"
-      @expires_in = 1
-      @cache.store = store
-      delete(@key)
-    end
-
-    after { delete(@key) }
-
-    describe "do_count once" do
-      it "should be 1" do
-        @cache.send(:do_count, @key, @expires_in).must_equal 1
-      end
-    end
-
-    describe "do_count twice" do
-      it "must be 2" do
-        @cache.send(:do_count, @key, @expires_in)
-        @cache.send(:do_count, @key, @expires_in).must_equal 2
-      end
-    end
-
-    describe "do_count after expires_in" do
-      it "must be 1" do
-        @cache.send(:do_count, @key, @expires_in)
-        sleep_until_expired
-        @cache.send(:do_count, @key, @expires_in).must_equal 1
-      end
-    end
-
-    describe "write" do
-      it "should write a value to the store with prefix" do
-        @cache.write("cache-test-key", "foobar", 1)
-        store.read(@key).must_equal "foobar"
-      end
-    end
-
-    describe "write after expiry" do
-      it "must not have a value" do
-        @cache.write("cache-test-key", "foobar", @expires_in)
-        sleep_until_expired
-        store.read(@key).must_be :nil?
-      end
-    end
-
-    describe "read" do
-      it "must read the value with a prefix" do
-        store.write(@key, "foobar", :expires_in => @expires_in)
-        @cache.read("cache-test-key").must_equal "foobar"
-      end
-    end
-
-    describe "delete" do
-      it "must delete the value" do
-        store.write(@key, "foobar", :expires_in => @expires_in)
-        @cache.read('cache-test-key').must_equal "foobar"
-        store.delete(@key)
-        assert_nil @cache.read('cache-test-key')
-      end
-    end
-
-    describe "cache#delete" do
-      it "must delete the value" do
-        @cache.write("cache-test-key", "foobar", 1)
-        store.read(@key).must_equal "foobar"
-        @cache.delete('cache-test-key')
-        store.read(@key).must_be :nil?
-      end
-    end
-
-    describe "reset_count" do
-      it "must delete the value" do
-        period = 1.minute
-        unprefixed_key = 'cache-test-key'
-        @cache.count(unprefixed_key, period)
-        period_key, _ = @cache.send(:key_and_expiry, 'cache-test-key', period)
-        store.read(period_key).to_i.must_equal 1
-        @cache.reset_count(unprefixed_key, period)
-        assert_nil store.read(period_key)
-      end
-    end
-  end
-end