diff --git a/config.go b/config.go
index 19cfdaf..db3138d 100644
--- a/config.go
+++ b/config.go
@@ -153,7 +153,7 @@ func (r rule) Matches(m *irc.Message, event *string) bool {
 	}
 
 	// Check on permit
-	if r.DisableOnPermit && timerStore.HasPermit(m.User) {
+	if r.DisableOnPermit && timerStore.HasPermit(m.Params[0], m.User) {
 		logger.Trace("Non-Match: Permit")
 		return false
 	}
diff --git a/irc.go b/irc.go
index 2b3ab5d..81586ca 100644
--- a/irc.go
+++ b/irc.go
@@ -125,7 +125,7 @@ func (i ircHandler) handlePermit(m *irc.Message) {
 	username := msgParts[1]
 
 	log.WithField("user", username).Debug("Added permit")
-	timerStore.Add(timerStore.NormalizeUsername(username))
+	timerStore.AddPermit(m.Params[0], username)
 
 	go handleMessage(i.c, m, eventTypePermit)
 }
diff --git a/timers.go b/timers.go
index 6ce47f3..85d228c 100644
--- a/timers.go
+++ b/timers.go
@@ -1,6 +1,8 @@
 package main
 
 import (
+	"crypto/sha256"
+	"fmt"
 	"strings"
 	"sync"
 	"time"
@@ -27,6 +29,10 @@ func (t *timer) Add(id string) {
 	t.timers[id] = time.Now()
 }
 
+func (t *timer) AddPermit(channel, username string) {
+	t.Add(t.getPermitTimerKey(channel, username))
+}
+
 func (t *timer) Has(id string, validity time.Duration) bool {
 	t.lock.RLock()
 	defer t.lock.RUnlock()
@@ -34,10 +40,16 @@ func (t *timer) Has(id string, validity time.Duration) bool {
 	return time.Since(t.timers[id]) < validity
 }
 
-func (t *timer) HasPermit(username string) bool {
-	return t.Has(t.NormalizeUsername(username), config.PermitTimeout)
+func (t *timer) HasPermit(channel, username string) bool {
+	return t.Has(t.getPermitTimerKey(channel, username), config.PermitTimeout)
 }
 
 func (t timer) NormalizeUsername(username string) string {
 	return strings.ToLower(strings.TrimLeft(username, "@"))
 }
+
+func (t timer) getPermitTimerKey(channel, username string) string {
+	h := sha256.New()
+	fmt.Fprintf(h, "%s:%s", channel, t.NormalizeUsername(username))
+	return fmt.Sprintf("sha256:%x", h.Sum(nil))
+}