wp

txlog/reader.go

@@ -5,16 +5,20 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"hash/crc32"
 	"io"
 	"os"
 
-	"gordenko.dev/dima/qb/bin"
+	bin "gordenko.dev/dima/bin/little"
+	"gordenko.dev/dima/qb/util"
 )
 
+const readBufferSize = 8 * 1024 * 1024
+
 type Reader struct {
-	file   *os.File
-	reader *bufio.Reader
+	file    *os.File
+	reader  *bufio.Reader
+	current []any
+	next    []any
 }
 
 type ReaderOptions struct {
@@ -37,7 +41,7 @@ func NewReader(opt ReaderOptions) (*Reader, error) {
 
 	return &Reader{
 		file:   file,
-		reader: bufio.NewReaderSize(file, 1024*1024),
+		reader: bufio.NewReaderSize(file, readBufferSize),
 	}, nil
 }
 
@@ -45,42 +49,62 @@ func (s *Reader) Close() {
 	s.file.Close()
 }
 
-func (s *Reader) ReadPacket() (uint32, []any, bool, error) {
-	prefix := make([]byte, packetPrefixSize)
-	n, err := s.reader.Read(prefix)
-	if err != nil {
-		if err == io.EOF && n == 0 {
-			return 0, nil, true, nil
-		} else {
-			return 0, nil, false, fmt.Errorf("read packet prefix: %s", err)
+func (s *Reader) NextPacket() ([]any, bool, error) {
+	var err error
+	if s.current == nil {
+		s.current, err = s.readPacket()
+		if err != nil {
+			return nil, false, err
+		}
+		if len(s.current) > 0 {
+			s.next, err = s.readPacket()
+			if err != nil {
+				return nil, false, err
+			}
 		}
 	}
 
-	length := bin.GetUint32(prefix[lengthIdx:])
-	storedCRC := bin.GetUint32(prefix[checksumIdx:])
-	lsn := bin.GetUint32(prefix[lsnIdx:])
+	current := s.current
+	done := s.next == nil
 
-	body, err := bin.ReadN(s.reader, int(length))
+	s.current = s.next
+	s.next, err = s.readPacket()
 	if err != nil {
-		return 0, nil, false, fmt.Errorf("read packet body: %s", err)
+		return nil, false, err
+	}
+	return current, done, nil
+}
+
+func (s *Reader) readPacket() (_ []any, err error) {
+	payloadSize, err := bin.ReadVarSize(s.reader)
+	if err != nil {
+		if err == io.EOF {
+			return nil, nil
+		} else {
+			return
+		}
 	}
 
-	hasher := crc32.NewIEEE()
-	hasher.Write(prefix[lsnIdx:])
+	storedCRC, err := bin.ReadUint32(s.reader)
+	if err != nil {
+		return
+	}
+
+	body, err := bin.ReadN(s.reader, int(payloadSize))
+	if err != nil {
+		return
+	}
+
+	hasher := util.NewHasher()
 	hasher.Write(body)
 
 	calculatedCRC := hasher.Sum32()
 
-	if calculatedCRC != storedCRC {
-		return 0, nil, false, fmt.Errorf("stored CRC %d != calculated CRC %d",
-			storedCRC, calculatedCRC)
+	if calculatedCRC == storedCRC {
+		return s.parseRecords(body)
 	}
-
-	records, err := s.parseRecords(body)
-	if err != nil {
-		return 0, nil, false, err
-	}
-	return lsn, records, false, nil
+	//s.logger.Printf("stored CRC %d != calculated CRC %d", storedCRC, calculatedCRC)
+	return nil, nil
 }
 
 func (s *Reader) parseRecords(body []byte) ([]any, error) {