MaxLineSize now splits longer lines without discarding the rest

implements http://bugs.activestate.com/show_bug.cgi?id=95745

tail.go

@@ -21,7 +21,7 @@ type Config struct {
 	ReOpen      bool // -F
 	MustExist   bool // if false, wait for the file to exist before beginning to tail.
 	Poll        bool // if true, do not use inotify but use polling
-	MaxLineSize int  // if > 0, limit the line size (discarding the rest)
+	MaxLineSize int  // if > 0, limit the line size (rest of the line would be returned as next lines)
 }
 
 type Tail struct {
@@ -114,18 +114,7 @@ func (tail *Tail) reopen() error {
 }
 
 func (tail *Tail) readLine() ([]byte, error) {
-	line, isPrefix, err := tail.reader.ReadLine()
-
-	if isPrefix && err == nil {
-		// line is longer than what we can accept. 
-		// ignore the rest of this line.
-		for {
-			_, isPrefix, err := tail.reader.ReadLine()
-			if !isPrefix || err != nil {
-				return line, err
-			}
-		}
-	}
+	line, _, err := tail.reader.ReadLine()
 	return line, err
 }
 
@@ -154,14 +143,21 @@ func (tail *Tail) tailFileSync() {
 		}
 	}
 
-	tail.reader = bufio.NewReaderSize(tail.file, tail.MaxLineSize)
+	tail.reader = bufio.NewReader(tail.file)
 
 	for {
 		line, err := tail.readLine()
 
 		if err == nil {
 			if line != nil {
-				tail.Lines <- &Line{string(line), getCurrentTime()}
+				now := getCurrentTime()
+				if tail.MaxLineSize > 0 && len(line) > tail.MaxLineSize {
+					for _, line := range partitionString(string(line), tail.MaxLineSize) {
+						tail.Lines <- &Line{line, now}
+					}
+				}else{
+					tail.Lines <- &Line{string(line), now}
+				}
 			}
 		} else {
 			if err != io.EOF {
@@ -192,7 +188,7 @@ func (tail *Tail) tailFileSync() {
 								return
 							}
 							log.Printf("Successfully reopened %s", tail.Filename)
-							tail.reader = bufio.NewReaderSize(tail.file, tail.MaxLineSize)
+							tail.reader = bufio.NewReader(tail.file)
 							changes = nil // XXX: how to kill changes' goroutine?
 							continue
 						} else {
@@ -221,3 +217,27 @@ func (tail *Tail) tailFileSync() {
 func getCurrentTime() int64 {
 	return time.Now().UTC().Unix()
 }
+
+// partitionString partitions the string into chunks of given size,
+// with the last chunk of variable size.
+func partitionString(s string, chunkSize int) []string {
+	if chunkSize <= 0 {
+		panic("invalid chunkSize")
+	}
+	length := len(s)
+	chunks := 1 + length/chunkSize 
+	start := 0
+	end := chunkSize
+	parts := make([]string, 0, chunks)
+	for {
+		if end > length {
+			end = length
+		}
+		parts = append(parts, s[start:end])
+		if end == length {
+			break
+		}
+		start, end = end, end+chunkSize
+	}
+	return parts
+}

tail_test.go

@@ -1,3 +1,6 @@
+// TODO:
+//  * repeat all the tests with Poll:true
+
 package tail
 
 import (
@@ -33,6 +36,19 @@ func TestMustExist(t *testing.T) {
 	tail.Stop()
 }
 
+func TestMaxLineSize(t *testing.T) {
+	fix := NewFixture("maxlinesize", t)
+	fix.CreateFile("test.txt", "hello\nworld\n")
+	tail := fix.StartTail("test.txt", Config{Follow: true, Location: -1, MaxLineSize: 3})
+	go fix.VerifyTail(tail, []string{"hel", "lo", "wor", "ld"})
+
+	// Delete after a reasonable delay, to give tail sufficient time
+	// to read all lines.
+	<-time.After(100 * time.Millisecond)
+	fix.RemoveFile("test.txt")
+	tail.Stop()
+}
+
 func TestLocationFull(t *testing.T) {
 	fix := NewFixture("location-full", t)
 	fix.CreateFile("test.txt", "hello\nworld\n")
@@ -83,7 +99,7 @@ func TestReOpen(t *testing.T) {
 	// Delete after a reasonable delay, to give tail sufficient time
 	// to read all lines.
 	<-time.After(100 * time.Millisecond)
-	fix.RemoveFile("test.txt") // TODO
+	fix.RemoveFile("test.txt")
 	
 	tail.Stop()
 }
@@ -153,7 +169,7 @@ func (fix Fixture) VerifyTail(tail *Tail, lines []string) {
 	for idx, line := range lines {
 		tailedLine, ok := <-tail.Lines
 		if !ok {
-			fix.t.Fatalf("insufficient lines from tail; expecting %v", lines[idx+1:])
+			fix.t.Fatalf("tail ended early; expecting more: %v", lines[idx:])
 		}
 		if tailedLine == nil {
 			fix.t.Fatalf("tail.Lines returned nil; not possible")