package database import ( "fmt" "hash/crc32" "io" "os" "path/filepath" bin "gordenko.dev/dima/bin/little" "gordenko.dev/dima/qb" "gordenko.dev/dima/qb/atree" "gordenko.dev/dima/qb/chunkenc" "gordenko.dev/dima/qb/conbuf" "gordenko.dev/dima/qb/freelist" ) /* Формат: metricsQty - varuint [metric]* де metric - це: metricID - 4b metricType - 1b fracDigits - 1b lastPageNo - 4b since - 4b sinceValue - 8b until - 4b untilValue - 8b timestamps size - 2b values size - 2b timestams payload - Nb values payload - Nb dataFreeList size - varsize dataFreeList - Nb indexFreeList size - varsize indexFreeList - Nb CRC32 - 4b */ const metricHeaderSize = 38 func (s *Database) dumpSnapshot(logNumber int) (err error) { var ( fileName = filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", logNumber)) hasher = crc32.NewIEEE() prefix = make([]byte, metricHeaderSize) ) file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY, 0770) if err != nil { return } dst := io.MultiWriter(file, hasher) _, err = bin.WriteVarUint64(dst, uint64(len(s.metrics))) if err != nil { return } for metricID, metric := range s.metrics { tSize := metric.Timestamps.Size() vSize := metric.Values.Size() bin.PutUint32(prefix[0:], metricID) prefix[4] = byte(metric.MetricType) prefix[5] = metric.FracDigits bin.PutUint32(prefix[6:], metric.LastPageNo) bin.PutUint32(prefix[10:], metric.Since) bin.PutFloat64(prefix[14:], metric.SinceValue) bin.PutUint32(prefix[22:], metric.Until) bin.PutFloat64(prefix[26:], metric.UntilValue) bin.PutUint16(prefix[34:], uint16(tSize)) bin.PutUint16(prefix[36:], uint16(vSize)) _, err = dst.Write(prefix) if err != nil { return } // copy timestamps writeChunks(dst, metric.Timestamps.Chunks(), tSize) // copy values writeChunks(dst, metric.Values.Chunks(), vSize) } // free data pages err = freeListWriteTo(s.freeList, dst) if err != nil { return } // free index pages err = freeListWriteTo(s.freeList, dst) if err != nil { return } bin.WriteUint32(file, hasher.Sum32()) err = file.Sync() if err != nil { return } err = file.Close() if err != nil { return } // prevLogNumber := logNumber - 1 // prevChanges := filepath.Join(s.dir, fmt.Sprintf("%d.changes", prevLogNumber)) // prevSnapshot := filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", prevLogNumber)) // isExist, err := isFileExist(prevChanges) // if err != nil { // return // } // if isExist { // err = os.Remove(prevChanges) // if err != nil { // qb.Abort(qb.DeletePrevChangesFileFailed, err) // } // } // isExist, err = isFileExist(prevSnapshot) // if err != nil { // return // } // if isExist { // err = os.Remove(prevSnapshot) // if err != nil { // qb.Abort(qb.DeletePrevSnapshotFileFailed, err) // } // } return } func (s *Database) loadSnapshot(fileName string) (err error) { var ( hasher = crc32.NewIEEE() header = make([]byte, metricHeaderSize) body = make([]byte, atree.PageSize) ) file, err := os.Open(fileName) if err != nil { return } src := io.TeeReader(file, hasher) metricsQty, err := bin.ReadVarSize(src) if err != nil { return } for range metricsQty { var metric _metric err = bin.ReadNInto(src, header) if err != nil { return } metricID, _ := bin.GetUint32(header[0:]) metric.MetricType = qb.MetricType(header[4]) metric.FracDigits = header[5] metric.LastPageNo, _ = bin.GetUint32(header[6:]) metric.Since, _ = bin.GetUint32(header[10:]) metric.SinceValue, _ = bin.GetFloat64(header[14:]) metric.Until, _ = bin.GetUint32(header[22:]) metric.UntilValue, _ = bin.GetFloat64(header[26:]) tSize, _ := bin.GetUint16(header[34:]) vSize, _ := bin.GetUint16(header[36:]) buf := body[:tSize] err = bin.ReadNInto(src, buf) if err != nil { return } metric.Timestamps = chunkenc.NewReverseTimeDeltaCompressor( conbuf.NewFromBuffer(buf), int(tSize), ) buf = body[:vSize] err = bin.ReadNInto(src, buf) if err != nil { return } if metric.MetricType == qb.Cumulative { metric.Values = chunkenc.NewReverseCumulativeDeltaCompressor( conbuf.NewFromBuffer(buf), int(vSize), metric.FracDigits, ) } else { metric.Values = chunkenc.NewReverseInstantDeltaCompressor( conbuf.NewFromBuffer(buf), int(vSize), metric.FracDigits, ) } s.metrics[metricID] = &metric } err = restoreFreeList(s.freeList, src) if err != nil { return fmt.Errorf("restore dataFreeList: %s", err) } err = restoreFreeList(s.freeList, src) if err != nil { return fmt.Errorf("restore indexFreeList: %s", err) } calculatedChecksum := hasher.Sum32() writtenChecksum, err := bin.ReadUint32(file) if err != nil { return } if calculatedChecksum != writtenChecksum { return fmt.Errorf("calculated checksum %d not equal written checksum %d", calculatedChecksum, writtenChecksum) } return } // HELPERS func freeListWriteTo(freeList *freelist.FreeList, dst io.Writer) error { serialized, err := freeList.Serialize() if err != nil { qb.Abort(qb.FailedFreeListSerialize, err) } _, err = bin.WriteVarSize(dst, len(serialized)) if err != nil { return err } _, err = dst.Write(serialized) if err != nil { return err } return nil } func restoreFreeList(freeList *freelist.FreeList, src io.Reader) error { size, err := bin.ReadVarSize(src) if err != nil { return err } serialized, err := bin.ReadN(src, size) if err != nil { return err } freeList.Restore(serialized) return nil } func writeChunks(dst io.Writer, chunks [][]byte, size int) (err error) { remaining := size for _, buf := range chunks { if remaining < len(buf) { buf = buf[:remaining] } _, err = dst.Write(buf) if err != nil { return } remaining -= len(buf) if remaining == 0 { break } } return }