qb/database/snapshot.go

package database

import (
	"fmt"
	"hash/crc32"
	"io"
	"os"
	"path/filepath"

	bin "gordenko.dev/dima/bin/little"
	"gordenko.dev/dima/qb"
	"gordenko.dev/dima/qb/atree"
	"gordenko.dev/dima/qb/chunkenc"
	"gordenko.dev/dima/qb/conbuf"
	"gordenko.dev/dima/qb/freelist"
)

/*
Формат:
metricsQty - varuint
[metric]*
де metric - це:
metricID - 4b
metricType - 1b
fracDigits - 1b
lastPageNo - 4b
since - 4b
sinceValue - 8b
until - 4b
untilValue - 8b
timestamps size - 2b
values size - 2b
timestams payload - Nb
values payload - Nb
dataFreeList size - varsize
dataFreeList - Nb
indexFreeList size - varsize
indexFreeList - Nb
CRC32 - 4b
*/

const metricHeaderSize = 38

func (s *Database) dumpSnapshot(logNumber int) (err error) {
	var (
		fileName = filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", logNumber))
		hasher   = crc32.NewIEEE()
		prefix   = make([]byte, metricHeaderSize)
	)

	file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY, 0770)
	if err != nil {
		return
	}

	dst := io.MultiWriter(file, hasher)

	_, err = bin.WriteVarUint64(dst, uint64(len(s.metrics)))
	if err != nil {
		return
	}

	for metricID, metric := range s.metrics {
		tSize := metric.Timestamps.Size()
		vSize := metric.Values.Size()

		bin.PutUint32(prefix[0:], metricID)
		prefix[4] = byte(metric.MetricType)
		prefix[5] = metric.FracDigits
		bin.PutUint32(prefix[6:], metric.LastPageNo)
		bin.PutUint32(prefix[10:], metric.Since)
		bin.PutFloat64(prefix[14:], metric.SinceValue)
		bin.PutUint32(prefix[22:], metric.Until)
		bin.PutFloat64(prefix[26:], metric.UntilValue)
		bin.PutUint16(prefix[34:], uint16(tSize))
		bin.PutUint16(prefix[36:], uint16(vSize))

		_, err = dst.Write(prefix)
		if err != nil {
			return
		}
		// copy timestamps
		writeChunks(dst, metric.Timestamps.Chunks(), tSize)
		// copy values
		writeChunks(dst, metric.Values.Chunks(), vSize)

	}
	// free data pages
	err = freeListWriteTo(s.freeList, dst)
	if err != nil {
		return
	}
	// free index pages
	err = freeListWriteTo(s.freeList, dst)
	if err != nil {
		return
	}

	bin.WriteUint32(file, hasher.Sum32())

	err = file.Sync()
	if err != nil {
		return
	}

	err = file.Close()
	if err != nil {
		return
	}

	// prevLogNumber := logNumber - 1
	// prevChanges := filepath.Join(s.dir, fmt.Sprintf("%d.changes", prevLogNumber))
	// prevSnapshot := filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", prevLogNumber))

	// isExist, err := isFileExist(prevChanges)
	// if err != nil {
	// 	return
	// }

	// if isExist {
	// 	err = os.Remove(prevChanges)
	// 	if err != nil {
	// 		qb.Abort(qb.DeletePrevChangesFileFailed, err)
	// 	}
	// }

	// isExist, err = isFileExist(prevSnapshot)
	// if err != nil {
	// 	return
	// }

	// if isExist {
	// 	err = os.Remove(prevSnapshot)
	// 	if err != nil {
	// 		qb.Abort(qb.DeletePrevSnapshotFileFailed, err)
	// 	}
	// }
	return
}

func (s *Database) loadSnapshot(fileName string) (err error) {
	var (
		hasher = crc32.NewIEEE()
		header = make([]byte, metricHeaderSize)
		body   = make([]byte, atree.PageSize)
	)

	file, err := os.Open(fileName)
	if err != nil {
		return
	}

	src := io.TeeReader(file, hasher)
	metricsQty, err := bin.ReadVarSize(src)
	if err != nil {
		return
	}

	for range metricsQty {
		var metric _metric
		err = bin.ReadNInto(src, header)
		if err != nil {
			return
		}

		metricID, _ := bin.GetUint32(header[0:])
		metric.MetricType = qb.MetricType(header[4])
		metric.FracDigits = header[5]
		metric.LastPageNo, _ = bin.GetUint32(header[6:])
		metric.Since, _ = bin.GetUint32(header[10:])
		metric.SinceValue, _ = bin.GetFloat64(header[14:])
		metric.Until, _ = bin.GetUint32(header[22:])
		metric.UntilValue, _ = bin.GetFloat64(header[26:])
		tSize, _ := bin.GetUint16(header[34:])
		vSize, _ := bin.GetUint16(header[36:])

		buf := body[:tSize]
		err = bin.ReadNInto(src, buf)
		if err != nil {
			return
		}
		metric.Timestamps = chunkenc.NewReverseTimeDeltaCompressor(
			conbuf.NewFromBuffer(buf),
			int(tSize),
		)

		buf = body[:vSize]
		err = bin.ReadNInto(src, buf)
		if err != nil {
			return
		}

		if metric.MetricType == qb.Cumulative {
			metric.Values = chunkenc.NewReverseCumulativeDeltaCompressor(
				conbuf.NewFromBuffer(buf),
				int(vSize),
				metric.FracDigits,
			)
		} else {
			metric.Values = chunkenc.NewReverseInstantDeltaCompressor(
				conbuf.NewFromBuffer(buf),
				int(vSize),
				metric.FracDigits,
			)
		}
		s.metrics[metricID] = &metric
	}

	err = restoreFreeList(s.freeList, src)
	if err != nil {
		return fmt.Errorf("restore dataFreeList: %s", err)
	}

	err = restoreFreeList(s.freeList, src)
	if err != nil {
		return fmt.Errorf("restore indexFreeList: %s", err)
	}

	calculatedChecksum := hasher.Sum32()

	writtenChecksum, err := bin.ReadUint32(file)
	if err != nil {
		return
	}

	if calculatedChecksum != writtenChecksum {
		return fmt.Errorf("calculated checksum %d not equal written checksum %d", calculatedChecksum, writtenChecksum)
	}
	return
}

// HELPERS

func freeListWriteTo(freeList *freelist.FreeList, dst io.Writer) error {
	serialized, err := freeList.Serialize()
	if err != nil {
		qb.Abort(qb.FailedFreeListSerialize, err)
	}
	_, err = bin.WriteVarSize(dst, len(serialized))
	if err != nil {
		return err
	}
	_, err = dst.Write(serialized)
	if err != nil {
		return err
	}
	return nil
}

func restoreFreeList(freeList *freelist.FreeList, src io.Reader) error {
	size, err := bin.ReadVarSize(src)
	if err != nil {
		return err
	}
	serialized, err := bin.ReadN(src, size)
	if err != nil {
		return err
	}
	freeList.Restore(serialized)
	return nil
}

func writeChunks(dst io.Writer, chunks [][]byte, size int) (err error) {
	remaining := size
	for _, buf := range chunks {
		if remaining < len(buf) {
			buf = buf[:remaining]
		}
		_, err = dst.Write(buf)
		if err != nil {
			return
		}
		remaining -= len(buf)
		if remaining == 0 {
			break
		}
	}
	return
}