package storage import ( "bufio" "bytes" "fmt" "io" bin "gordenko.dev/dima/bin/little" "gordenko.dev/dima/qb/util" ) const readBufferSize = 8 * 1024 * 1024 type WALReader struct { //file *os.File reader *bufio.Reader next []any done bool } // type WALReaderOptions struct { // File *os.File // BufferSize int // } func NewWALReader(src io.Reader) (*WALReader, error) { // if opt.FileName == "" { // return nil, errors.New("FileName option is required") // } // if opt.BufferSize <= 0 { // return nil, errors.New("BufferSize option is required") // } return &WALReader{ //file: file, reader: bufio.NewReaderSize(src, readBufferSize), }, nil } // func (s *WALReader) Close() { // s.file.Close() // } // (packet records, isLastPacket, error) func (s *WALReader) NextPacket() (_ []any, _ bool, err error) { if s.done { return } var current []any if s.next != nil { current = s.next } else { current, err = s.readPacket() if err != nil { err = fmt.Errorf("readPacket: %s", err) return } if current == nil { return } } s.next, err = s.readPacket() if err != nil { err = fmt.Errorf("readPacket: %s", err) return } return current, s.next == nil, nil } func (s *WALReader) readPacket() (_ []any, err error) { bodySize, err := bin.ReadVarSize(s.reader) // fix add n if err != nil { if err == io.EOF { s.done = true return nil, nil } return } body, err := bin.ReadN(s.reader, int(bodySize)) if err != nil { return } writtenCRC, err := bin.ReadUint32(s.reader) if err != nil { return } calculatedCRC := util.CalculateCRC32(body) if calculatedCRC != writtenCRC { //fmt.Printf("bodySize: %d\n", bodySize) //fmt.Printf("body: % x\n", body) return nil, fmt.Errorf("written CRC %d != calculated CRC %d", writtenCRC, calculatedCRC) } //fmt.Println("SAME CRC") return s.parseRecords(body) } func (s *WALReader) parseRecords(body []byte) (_ []any, err error) { var ( src = bytes.NewBuffer(body) records []any ) for { var recordType byte recordType, err = src.ReadByte() if err != nil { if err == io.EOF { return records, nil } return } switch recordType { case CodeMetricAdd: rec := new(MetricAddRecord) if err = rec.Parse(src); err != nil { return } records = append(records, *rec) case CodeMetricDelete: rec := new(MetricDeleteRecord) if err = rec.Parse(src); err != nil { return } records = append(records, *rec) case CodeMeasuresAppend: rec := new(MeasuresAppendRecord) if err = rec.Parse(src); err != nil { return } records = append(records, *rec) case CodeMeasuresAppendWithGrow: rec := new(MeasuresAppendWithGrowRecord) if err = rec.Parse(src); err != nil { return } records = append(records, *rec) case CodeMeasuresDelete: rec := new(MeasuresDeleteRecord) if err = rec.Parse(src); err != nil { return } records = append(records, *rec) default: return nil, fmt.Errorf("unknown record type code: %d", recordType) } } } // HELPERS // func (s *WALReader) Seek(offset int64) error { // ret, err := s.file.Seek(offset, 0) // if err != nil { // return err // } // if ret != offset { // return fmt.Errorf("ret %d != offset %d", ret, offset) // } // return nil // }