diff --git a/watch/inotify.go b/watch/inotify.go
index 4478f1e..408a68d 100644
--- a/watch/inotify.go
+++ b/watch/inotify.go
@@ -75,7 +75,6 @@ func (fw *InotifyFileWatcher) ChangeEvents(t *tomb.Tomb, pos int64) (*FileChange
 	fw.Size = pos
 
 	go func() {
-		defer RemoveWatch(fw.Filename)
 
 		events := Events(fw.Filename)
 
@@ -88,9 +87,11 @@ func (fw *InotifyFileWatcher) ChangeEvents(t *tomb.Tomb, pos int64) (*FileChange
 			select {
 			case evt, ok = <-events:
 				if !ok {
+					RemoveWatch(fw.Filename)
 					return
 				}
 			case <-t.Dying():
+				RemoveWatch(fw.Filename)
 				return
 			}
 
@@ -99,6 +100,7 @@ func (fw *InotifyFileWatcher) ChangeEvents(t *tomb.Tomb, pos int64) (*FileChange
 				fallthrough
 
 			case evt.Op&fsnotify.Rename == fsnotify.Rename:
+				RemoveWatch(fw.Filename)
 				changes.NotifyDeleted()
 				return
 
@@ -106,6 +108,7 @@ func (fw *InotifyFileWatcher) ChangeEvents(t *tomb.Tomb, pos int64) (*FileChange
 				fi, err := os.Stat(fw.Filename)
 				if err != nil {
 					if os.IsNotExist(err) {
+						RemoveWatch(fw.Filename)
 						changes.NotifyDeleted()
 						return
 					}