wp

storage/wal_reader.go

@@ -3,10 +3,8 @@ package storage
 import (
 	"bufio"
 	"bytes"
-	"errors"
 	"fmt"
 	"io"
-	"os"
 
 	bin "gordenko.dev/dima/bin/little"
 	"gordenko.dev/dima/qb/util"
@@ -15,155 +13,133 @@ import (
 const readBufferSize = 8 * 1024 * 1024
 
 type WALReader struct {
-	file    *os.File
-	reader  *bufio.Reader
-	current []any
-	next    []any
+	//file   *os.File
+	reader *bufio.Reader
+	next   []any
+	done   bool
 }
 
-type WALReaderOptions struct {
-	FileName   string
-	BufferSize int
-}
+// type WALReaderOptions struct {
+// 	File       *os.File
+// 	BufferSize int
+// }
 
-func NewWALReader(opt WALReaderOptions) (*WALReader, error) {
-	if opt.FileName == "" {
-		return nil, errors.New("FileName option is required")
-	}
-	if opt.BufferSize <= 0 {
-		return nil, errors.New("BufferSize option is required")
-	}
-
-	file, err := os.Open(opt.FileName)
-	if err != nil {
-		return nil, err
-	}
+func NewWALReader(src io.Reader) (*WALReader, error) {
+	// if opt.FileName == "" {
+	// 	return nil, errors.New("FileName option is required")
+	// }
+	// if opt.BufferSize <= 0 {
+	// 	return nil, errors.New("BufferSize option is required")
+	// }
 
 	return &WALReader{
-		file:   file,
-		reader: bufio.NewReaderSize(file, readBufferSize),
+		//file:   file,
+		reader: bufio.NewReaderSize(src, readBufferSize),
 	}, nil
 }
 
-func (s *WALReader) Close() {
-	s.file.Close()
-}
+// func (s *WALReader) Close() {
+// 	s.file.Close()
+// }
 
-func (s *WALReader) NextPacket() ([]any, bool, error) {
-	var err error
-	if s.current == nil {
-		s.current, err = s.readPacket()
+// (packet records, isLastPacket, error)
+func (s *WALReader) NextPacket() (_ []any, _ bool, err error) {
+	if s.done {
+		return
+	}
+	var current []any
+	if s.next != nil {
+		current = s.next
+	} else {
+		current, err = s.readPacket()
 		if err != nil {
-			return nil, false, err
+			return
 		}
-		if len(s.current) > 0 {
-			s.next, err = s.readPacket()
-			if err != nil {
-				return nil, false, err
-			}
-		}
-	}
-
-	current := s.current
-	done := s.next == nil
-
-	s.current = s.next
-	s.next, err = s.readPacket()
-	if err != nil {
-		return nil, false, err
-	}
-	return current, done, nil
-}
-
-func (s *WALReader) readPacket() (_ []any, err error) {
-	payloadSize, err := bin.ReadVarSize(s.reader)
-	if err != nil {
-		if err == io.EOF {
-			return nil, nil
-		} else {
+		if current == nil {
 			return
 		}
 	}
-
-	storedCRC, err := bin.ReadUint32(s.reader)
+	s.next, err = s.readPacket()
 	if err != nil {
 		return
 	}
-
-	body, err := bin.ReadN(s.reader, int(payloadSize))
-	if err != nil {
-		return
-	}
-
-	hasher := util.NewHasher()
-	hasher.Write(body)
-
-	calculatedCRC := hasher.Sum32()
-
-	if calculatedCRC == storedCRC {
-		return s.parseRecords(body)
-	}
-	//s.logger.Printf("stored CRC %d != calculated CRC %d", storedCRC, calculatedCRC)
-	return nil, nil
+	return current, s.next == nil, nil
 }
 
-func (s *WALReader) parseRecords(body []byte) ([]any, error) {
+func (s *WALReader) readPacket() (_ []any, err error) {
+	bodySize, err := bin.ReadVarSize(s.reader) // fix add n
+	if err != nil {
+		if err == io.EOF {
+			s.done = true
+			return nil, nil
+		}
+		return
+	}
+	body, err := bin.ReadN(s.reader, int(bodySize))
+	if err != nil {
+		return
+	}
+	writtenCRC, err := bin.ReadUint32(s.reader)
+	if err != nil {
+		return
+	}
+	calculatedCRC := util.CalculateCRC32(body)
+	if calculatedCRC != writtenCRC {
+		return nil, fmt.Errorf("written CRC %d != calculated CRC %d",
+			writtenCRC, calculatedCRC)
+	}
+	return s.parseRecords(body)
+}
+
+func (s *WALReader) parseRecords(body []byte) (_ []any, err error) {
 	var (
 		src     = bytes.NewBuffer(body)
 		records []any
 	)
-
 	for {
-		recordType, err := src.ReadByte()
+		var recordType byte
+		recordType, err = src.ReadByte()
 		if err != nil {
 			if err == io.EOF {
 				return records, nil
 			}
-			return nil, err
+			return
 		}
-
 		switch recordType {
 		case CodeMetricAdd:
 			rec := new(MetricAddRecord)
 			if err = rec.Parse(src); err != nil {
-				return nil, err
+				return
 			}
-			records = append(records, rec)
+			records = append(records, *rec)
 
 		case CodeMetricDelete:
 			rec := new(MetricDeleteRecord)
 			if err = rec.Parse(src); err != nil {
-				return nil, err
+				return
 			}
-			records = append(records, rec)
+			records = append(records, *rec)
 
-		// case CodeAppendedMeasure:
-		// 	rec := new(AppendedMeasure)
-		// 	if err = rec.Parse(src); err != nil {
-		// 		return nil, err
-		// 	}
-		// 	records = append(records, rec)
+		case CodeMeasuresAppend:
+			rec := new(MeasuresAppendRecord)
+			if err = rec.Parse(src); err != nil {
+				return
+			}
+			records = append(records, *rec)
 
-		// case CodeAppendedMeasures:
-		// 	rec := new(AppendedMeasures)
-		// 	if err = rec.Parse(src); err != nil {
-		// 		return nil, err
-		// 	}
-		// 	records = append(records, rec)
-
-		// case CodeAppendedPages:
-		// 	rec := new(AppendedPages)
-		// 	if err = rec.Parse(src); err != nil {
-		// 		return nil, err
-		// 	}
-		// 	records = append(records, rec)
+		case CodeMeasuresAppendWithGrow:
+			rec := new(MeasuresAppendWithGrowRecord)
+			if err = rec.Parse(src); err != nil {
+				return
+			}
+			records = append(records, *rec)
 
 		case CodeMeasuresDelete:
 			rec := new(MeasuresDeleteRecord)
 			if err = rec.Parse(src); err != nil {
-				return nil, err
+				return
 			}
-			records = append(records, rec)
+			records = append(records, *rec)
 
 		default:
 			return nil, fmt.Errorf("unknown record type code: %d", recordType)