package storage import ( "bufio" "bytes" "errors" "fmt" "io" "os" bin "gordenko.dev/dima/bin/little" "gordenko.dev/dima/qb/util" ) const readBufferSize = 8 * 1024 * 1024 type WALReader struct { file *os.File reader *bufio.Reader current []any next []any } type WALReaderOptions struct { FileName string BufferSize int } func NewWALReader(opt WALReaderOptions) (*WALReader, error) { if opt.FileName == "" { return nil, errors.New("FileName option is required") } if opt.BufferSize <= 0 { return nil, errors.New("BufferSize option is required") } file, err := os.Open(opt.FileName) if err != nil { return nil, err } return &WALReader{ file: file, reader: bufio.NewReaderSize(file, readBufferSize), }, nil } func (s *WALReader) Close() { s.file.Close() } func (s *WALReader) NextPacket() ([]any, bool, error) { var err error if s.current == nil { s.current, err = s.readPacket() if err != nil { return nil, false, err } if len(s.current) > 0 { s.next, err = s.readPacket() if err != nil { return nil, false, err } } } current := s.current done := s.next == nil s.current = s.next s.next, err = s.readPacket() if err != nil { return nil, false, err } return current, done, nil } func (s *WALReader) readPacket() (_ []any, err error) { payloadSize, err := bin.ReadVarSize(s.reader) if err != nil { if err == io.EOF { return nil, nil } else { return } } storedCRC, err := bin.ReadUint32(s.reader) if err != nil { return } body, err := bin.ReadN(s.reader, int(payloadSize)) if err != nil { return } hasher := util.NewHasher() hasher.Write(body) calculatedCRC := hasher.Sum32() if calculatedCRC == storedCRC { return s.parseRecords(body) } //s.logger.Printf("stored CRC %d != calculated CRC %d", storedCRC, calculatedCRC) return nil, nil } func (s *WALReader) parseRecords(body []byte) ([]any, error) { var ( src = bytes.NewBuffer(body) records []any ) for { recordType, err := src.ReadByte() if err != nil { if err == io.EOF { return records, nil } return nil, err } switch recordType { case CodeMetricAdd: rec := new(MetricAddRecord) if err = rec.Parse(src); err != nil { return nil, err } records = append(records, rec) case CodeMetricDelete: rec := new(MetricDeleteRecord) if err = rec.Parse(src); err != nil { return nil, err } records = append(records, rec) // case CodeAppendedMeasure: // rec := new(AppendedMeasure) // if err = rec.Parse(src); err != nil { // return nil, err // } // records = append(records, rec) // case CodeAppendedMeasures: // rec := new(AppendedMeasures) // if err = rec.Parse(src); err != nil { // return nil, err // } // records = append(records, rec) // case CodeAppendedPages: // rec := new(AppendedPages) // if err = rec.Parse(src); err != nil { // return nil, err // } // records = append(records, rec) case CodeMeasuresDelete: rec := new(MeasuresDeleteRecord) if err = rec.Parse(src); err != nil { return nil, err } records = append(records, rec) default: return nil, fmt.Errorf("unknown record type code: %d", recordType) } } } // HELPERS // func (s *WALReader) Seek(offset int64) error { // ret, err := s.file.Seek(offset, 0) // if err != nil { // return err // } // if ret != offset { // return fmt.Errorf("ret %d != offset %d", ret, offset) // } // return nil // }