Fine tune diff highlighting (#36592)

modules/git/catfile_batch_reader.go

@@ -12,11 +12,15 @@ import (
 	"math"
 	"strconv"
 	"strings"
+	"sync/atomic"
+	"time"
 
 	"code.gitea.io/gitea/modules/git/gitcmd"
 	"code.gitea.io/gitea/modules/log"
 )
 
+var catFileBatchDebugWaitClose atomic.Int64
+
 type catFileBatchCommunicator struct {
 	cancel      context.CancelFunc
 	reqWriter   io.Writer
@@ -36,7 +40,14 @@ func newCatFileBatch(ctx context.Context, repoPath string, cmdCatFile *gitcmd.Co
 	ctx, ctxCancel := context.WithCancelCause(ctx)
 
 	// We often want to feed the commits in order into cat-file --batch, followed by their trees and subtrees as necessary.
-	stdinWriter, stdoutReader, pipeClose := cmdCatFile.MakeStdinStdoutPipe()
+	stdinWriter, stdoutReader, stdPipeClose := cmdCatFile.MakeStdinStdoutPipe()
+	pipeClose := func() {
+		if delay := catFileBatchDebugWaitClose.Load(); delay > 0 {
+			time.Sleep(time.Duration(delay)) // for testing purpose only
+		}
+		stdPipeClose()
+	}
+
 	ret = &catFileBatchCommunicator{
 		debugGitCmd: cmdCatFile,
 		cancel:      func() { ctxCancel(nil) },

modules/git/catfile_batch_test.go

@@ -5,9 +5,11 @@ package git
 
 import (
 	"io"
+	"os"
 	"path/filepath"
 	"sync"
 	"testing"
+	"time"
 
 	"code.gitea.io/gitea/modules/test"
 
@@ -37,6 +39,45 @@ func testCatFileBatch(t *testing.T) {
 		require.Error(t, err)
 	})
 
+	simulateQueryTerminated := func(pipeCloseDelay, pipeReadDelay time.Duration) (errRead error) {
+		catFileBatchDebugWaitClose.Store(int64(pipeCloseDelay))
+		defer catFileBatchDebugWaitClose.Store(0)
+		batch, err := NewBatch(t.Context(), filepath.Join(testReposDir, "repo1_bare"))
+		require.NoError(t, err)
+		defer batch.Close()
+		_, _ = batch.QueryInfo("e2129701f1a4d54dc44f03c93bca0a2aec7c5449")
+		var c *catFileBatchCommunicator
+		switch b := batch.(type) {
+		case *catFileBatchLegacy:
+			c = b.batchCheck
+			_, _ = c.reqWriter.Write([]byte("in-complete-line-"))
+		case *catFileBatchCommand:
+			c = b.batch
+			_, _ = c.reqWriter.Write([]byte("info"))
+		default:
+			t.FailNow()
+		}
+
+		wg := sync.WaitGroup{}
+		wg.Go(func() {
+			time.Sleep(pipeReadDelay)
+			var n int
+			n, errRead = c.respReader.Read(make([]byte, 100))
+			assert.Zero(t, n)
+		})
+		time.Sleep(10 * time.Millisecond)
+		c.debugGitCmd.DebugKill()
+		wg.Wait()
+		return errRead
+	}
+
+	t.Run("QueryTerminated", func(t *testing.T) {
+		err := simulateQueryTerminated(0, 20*time.Millisecond)
+		assert.ErrorIs(t, err, os.ErrClosed) // pipes are closed faster
+		err = simulateQueryTerminated(40*time.Millisecond, 20*time.Millisecond)
+		assert.ErrorIs(t, err, io.EOF) // reader is faster
+	})
+
 	batch, err := NewBatch(t.Context(), filepath.Join(testReposDir, "repo1_bare"))
 	require.NoError(t, err)
 	defer batch.Close()
@@ -60,30 +101,4 @@ func testCatFileBatch(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, "file1\n", string(content))
 	})
-
-	t.Run("QueryTerminated", func(t *testing.T) {
-		var c *catFileBatchCommunicator
-		switch b := batch.(type) {
-		case *catFileBatchLegacy:
-			c = b.batchCheck
-			_, _ = c.reqWriter.Write([]byte("in-complete-line-"))
-		case *catFileBatchCommand:
-			c = b.batch
-			_, _ = c.reqWriter.Write([]byte("info"))
-		default:
-			t.FailNow()
-			return
-		}
-
-		wg := sync.WaitGroup{}
-		wg.Go(func() {
-			buf := make([]byte, 100)
-			_, _ = c.respReader.Read(buf)
-			n, errRead := c.respReader.Read(buf)
-			assert.Zero(t, n)
-			assert.ErrorIs(t, errRead, io.EOF) // the pipe is closed due to command being killed
-		})
-		c.debugGitCmd.DebugKill()
-		wg.Wait()
-	})
 }

modules/git/gitcmd/pipe.go

@@ -9,6 +9,14 @@ import (
 )
 
 type PipeBufferReader interface {
+	// Read should be used in the same goroutine as command's Wait
+	// When Reader in one goroutine, command's Wait in another goroutine, then the command exits, the pipe will be closed:
+	// * If the Reader goroutine reads faster, it will read all remaining data and then get io.EOF
+	//   * But this io.EOF doesn't mean the Reader has gotten complete data, the data might still be corrupted
+	// * If the Reader goroutine reads slower, it will get os.ErrClosed because the os.Pipe is closed ahead when the command exits
+	//
+	// When using 2 goroutines, no clear solution to distinguish these two cases or make Reader knows whether the data is complete
+	// It should avoid using Reader in a different goroutine than the command if the Read error needs to be handled.
 	Read(p []byte) (n int, err error)
 	Bytes() []byte
 }