package worker import ( "bufio" "fmt" "io" "os" bin "gordenko.dev/dima/bin/little" "gordenko.dev/dima/qb" "gordenko.dev/dima/qb/storage" "gordenko.dev/dima/qb/util" ) /* Формат: metricsQty - varsize [metric]* де metric - це: index free list frozen pages - varsize indexFreeList size - varsize indexFreeList - Nb data free list frozen pages - varsize dataFreeList size - varsize dataFreeList - Nb CRC32 - 4b */ const readBufferSize = 4 * 1024 * 1024 type WriteSnapshotIn struct { SnapshotNumber int WriteBufferSize int Metrics map[uint32]*Metric FrozenIndexPagesCount int IndexPageNumbers []uint32 FrozenDataPagesCount int DataPageNumbers []uint32 } func writeSnapshot(filePath string, in WriteSnapshotIn) (err error) { file, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) if err != nil { qb.Abort(qb.WriteSnapshotFailed, err) } var ( bufferedWriter = bufio.NewWriterSize(file, in.WriteBufferSize) hasher = util.NewHasher() ) dst := io.MultiWriter(bufferedWriter, hasher) // _, err = bin.WriteVarSize(dst, len(in.Metrics)) if err != nil { return } for metricID, metric := range in.Metrics { err = bin.WriteUint32(dst, metricID) if err != nil { return } err = metric.WriteTo(dst) if err != nil { return } } // free index pages err = writeFreePages(dst, in.FrozenIndexPagesCount, in.IndexPageNumbers) if err != nil { return } // free data pages err = writeFreePages(dst, in.FrozenDataPagesCount, in.DataPageNumbers) if err != nil { return } err = bufferedWriter.Flush() if err != nil { return } // CRC32 err = bin.WriteUint32(file, hasher.Sum32()) if err != nil { return } return file.Close() // prevLogNumber := logNumber - 1 // prevChanges := filepath.Join(s.dir, fmt.Sprintf("%d.changes", prevLogNumber)) // prevSnapshot := filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", prevLogNumber)) // isExist, err := isFileExist(prevChanges) // if err != nil { // return // } // if isExist { // err = os.Remove(prevChanges) // if err != nil { // qb.Abort(qb.DeletePrevChangesFileFailed, err) // } // } // isExist, err = isFileExist(prevSnapshot) // if err != nil { // return // } // if isExist { // err = os.Remove(prevSnapshot) // if err != nil { // qb.Abort(qb.DeletePrevSnapshotFileFailed, err) // } // } } type ReadSnapshotOut struct { Metrics map[uint32]*storage.ReplayMetric FrozenIndexPagesCount int IndexPageNumbers []uint32 FrozenDataPagesCount int DataPageNumbers []uint32 } func ReadSnapshot(filePath string) (out ReadSnapshotOut, err error) { var ( metrics = make(map[uint32]*storage.ReplayMetric) frozenIndexPagesCount int indexPageNumbers []uint32 frozenDataPagesCount int dataPageNumbers []uint32 hasher = util.NewHasher() ) file, err := os.Open(filePath) if err != nil { return } defer file.Close() stat, err := file.Stat() if err != nil { return } fileSize := stat.Size() if fileSize < 4 { err = fmt.Errorf("snapshot %s is corrupted", filePath) return } payloadSize := fileSize - 4 // 2. Створюємо великий буфер для NVMe (4 MB) bufferedReader := bufio.NewReaderSize(file, readBufferSize) // 3. Обмежуємо читання лише розміром Payload limitReader := io.LimitReader(bufferedReader, payloadSize) // 4. Створюємо хеш та обгортку TeeReader, яка рахує CRC32 на льоту src := io.TeeReader(limitReader, hasher) // читаю payload metricsQty, err := bin.ReadVarSize(src) if err != nil { return } for range metricsQty { var ( metricID uint32 buf = make([]byte, storage.DataPageSize) m = &storage.ReplayMetric{ Buf: buf, } metricType byte levelsCount int ) metricID, err = bin.ReadUint32(src) if err != nil { return } fmt.Println("metricID", metricID) metricType, err = bin.ReadByte(src) if err != nil { return } m.MetricType = qb.MetricType(metricType) m.FracDigits, err = bin.ReadByte(src) if err != nil { return } fmt.Println("m.MetricType", m.MetricType) fmt.Println("m.FracDigits", m.FracDigits) m.LastPageNo, err = bin.ReadUint32(src) if err != nil { return } fmt.Println("m.LastPageNo", m.LastPageNo) m.TimestampsSize, err = bin.ReadUint16AsInt(src) if err != nil { return } m.ValuesSize, err = bin.ReadUint16AsInt(src) if err != nil { return } fmt.Println("m.TimestampsSize", m.TimestampsSize, "m.ValuesSize", m.ValuesSize) err = bin.ReadNInto(src, buf[storage.DataPagePayloadSize-m.TimestampsSize:storage.DataPagePayloadSize]) if err != nil { return } err = bin.ReadNInto(src, buf[:m.ValuesSize]) if err != nil { return } fmt.Printf("% x\n", buf) // levelsCount, err = bin.ReadVarSize(src) if err != nil { return } fmt.Println("levels count:", levelsCount) for range levelsCount { var ( buf = make([]byte, storage.IndexPageSize) count int ) count, err = bin.ReadVarSize(src) if err != nil { return } fmt.Println("records count:", count) err = bin.ReadNInto(src, buf[:count*storage.IndexRecordSize]) if err != nil { return } fmt.Printf("% x\n", buf) m.IndexLevelTails = append(m.IndexLevelTails, storage.IndexLevelTail{ Buffer: buf, RecordsCount: count, }) } metrics[metricID] = m } // index pages frozenIndexPagesCount, indexPageNumbers, err = readFreePages(src) if err != nil { return } // data pages frozenDataPagesCount, dataPageNumbers, err = readFreePages(src) if err != nil { return } // verify CRC32 expectedCRC, err := bin.ReadUint32(bufferedReader) if err != nil { return } calculatedCRC := hasher.Sum32() if expectedCRC != calculatedCRC { err = fmt.Errorf("%s is corrupted. Calculated CRC %d not equal expected CRC %d", filePath, calculatedCRC, expectedCRC) return } return ReadSnapshotOut{ Metrics: metrics, FrozenIndexPagesCount: frozenIndexPagesCount, IndexPageNumbers: indexPageNumbers, FrozenDataPagesCount: frozenDataPagesCount, DataPageNumbers: dataPageNumbers, }, nil } // HELPERS func writeFreePages(dst io.Writer, frozenPagesCount int, pageNumbers []uint32) (err error) { _, err = bin.WriteVarSize(dst, frozenPagesCount) if err != nil { return } fmt.Println("write frozenPagesCount:", frozenPagesCount) _, err = bin.WriteVarSize(dst, len(pageNumbers)) if err != nil { return } for _, pageNo := range pageNumbers { err = bin.WriteUint32(dst, pageNo) if err != nil { return } } return } func readFreePages(src io.Reader) (_ int, _ []uint32, err error) { var ( frozenPagesCount int pageNumbers []uint32 ) frozenPagesCount, err = bin.ReadVarSize(src) if err != nil { return } fmt.Println("read frozenPagesCount:", frozenPagesCount) pageNumbersCount, err := bin.ReadVarSize(src) if err != nil { return } for range pageNumbersCount { var pageNo uint32 pageNo, err = bin.ReadUint32(src) if err != nil { return } pageNumbers = append(pageNumbers, pageNo) } return frozenPagesCount, pageNumbers, nil }