dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-05-21 23:38:52 +03:00
parent 793df29e11
commit e7ec6083cb
21 changed files with 1234 additions and 1017 deletions
Download Patch File Download Diff File Expand all files Collapse all files

284
database/snapshot.go
View File

@@ -7,21 +7,22 @@ import (
"os"
"path/filepath"
octopus "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/bin"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/chunkenc"
"gordenko.dev/dima/qb/conbuf"
"gordenko.dev/dima/qb/freelist"
)
/*
Формат:
//lsn - varuint (останній LSN, що змінив дані у RAM)
metricsQty - varuint
[metric]*
где metric - це:
де metric - це:
metricID - 4b
metricType - 1b
fracDigits - 1b
rootPageNo - 4b
lastPageNo - 4b
since - 4b
sinceValue - 8b
@@ -31,14 +32,14 @@ timestamps size - 2b
values size - 2b
timestams payload - Nb
values payload - Nb
dataFreeList size - varuint
dataFreeList size - varsize
dataFreeList - Nb
indexFreeList size - varuint
indexFreeList size - varsize
indexFreeList - Nb
CRC32 - 4b
*/
const metricHeaderSize = 42
const metricHeaderSize = 38
func (s *Database) dumpSnapshot(logNumber int) (err error) {
var (
@@ -66,49 +67,23 @@ func (s *Database) dumpSnapshot(logNumber int) (err error) {
bin.PutUint32(prefix[0:], metricID)
prefix[4] = byte(metric.MetricType)
prefix[5] = metric.FracDigits
bin.PutUint32(prefix[6:], metric.RootPageNo)
bin.PutUint32(prefix[10:], metric.LastPageNo)
bin.PutUint32(prefix[14:], metric.Since)
bin.PutFloat64(prefix[18:], metric.SinceValue)
bin.PutUint32(prefix[26:], metric.Until)
bin.PutFloat64(prefix[30:], metric.UntilValue)
bin.PutUint16(prefix[38:], uint16(tSize))
bin.PutUint16(prefix[40:], uint16(vSize))
bin.PutUint32(prefix[6:], metric.LastPageNo)
bin.PutUint32(prefix[10:], metric.Since)
bin.PutFloat64(prefix[14:], metric.SinceValue)
bin.PutUint32(prefix[22:], metric.Until)
bin.PutFloat64(prefix[26:], metric.UntilValue)
bin.PutUint16(prefix[34:], uint16(tSize))
bin.PutUint16(prefix[36:], uint16(vSize))
_, err = dst.Write(prefix)
if err != nil {
return
}
// copy timestamps
remaining := tSize
for _, buf := range metric.TimestampsBuf.Chunks() {
if remaining < len(buf) {
buf = buf[:remaining]
}
_, err = dst.Write(buf)
if err != nil {
return
}
remaining -= len(buf)
if remaining == 0 {
break
}
}
writeChunks(dst, metric.Timestamps.Chunks(), tSize)
// copy values
remaining = vSize
for _, buf := range metric.ValuesBuf.Chunks() {
if remaining < len(buf) {
buf = buf[:remaining]
}
_, err = dst.Write(buf)
if err != nil {
return
}
remaining -= len(buf)
if remaining == 0 {
break
}
}
writeChunks(dst, metric.Values.Chunks(), vSize)
}
// free data pages
err = freeListWriteTo(s.freeList, dst)
@@ -133,123 +108,124 @@ func (s *Database) dumpSnapshot(logNumber int) (err error) {
return
}
prevLogNumber := logNumber - 1
prevChanges := filepath.Join(s.dir, fmt.Sprintf("%d.changes", prevLogNumber))
prevSnapshot := filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", prevLogNumber))
// prevLogNumber := logNumber - 1
// prevChanges := filepath.Join(s.dir, fmt.Sprintf("%d.changes", prevLogNumber))
// prevSnapshot := filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", prevLogNumber))
isExist, err := isFileExist(prevChanges)
if err != nil {
return
}
// isExist, err := isFileExist(prevChanges)
// if err != nil {
// return
// }
if isExist {
err = os.Remove(prevChanges)
if err != nil {
octopus.Abort(octopus.DeletePrevChangesFileFailed, err)
}
}
// if isExist {
// err = os.Remove(prevChanges)
// if err != nil {
// qb.Abort(qb.DeletePrevChangesFileFailed, err)
// }
// }
isExist, err = isFileExist(prevSnapshot)
if err != nil {
return
}
// isExist, err = isFileExist(prevSnapshot)
// if err != nil {
// return
// }
if isExist {
err = os.Remove(prevSnapshot)
if err != nil {
octopus.Abort(octopus.DeletePrevSnapshotFileFailed, err)
}
}
// if isExist {
// err = os.Remove(prevSnapshot)
// if err != nil {
// qb.Abort(qb.DeletePrevSnapshotFileFailed, err)
// }
// }
return
}
func (s *Database) loadSnapshot(fileName string) (err error) {
// FIX
// var (
// hasher = crc32.NewIEEE()
// metricsQty int
// header = make([]byte, metricHeaderSize)
// body = make([]byte, atree.PageSize)
// )
var (
hasher = crc32.NewIEEE()
header = make([]byte, metricHeaderSize)
body = make([]byte, atree.PageSize)
)
// file, err := os.Open(fileName)
// if err != nil {
// return
// }
file, err := os.Open(fileName)
if err != nil {
return
}
// src := io.TeeReader(file, hasher)
// u64, _, err := bin.ReadVarUint64(src)
// if err != nil {
// return
// }
// metricsQty = int(u64)
src := io.TeeReader(file, hasher)
metricsQty, err := bin.ReadVarSize(src)
if err != nil {
return
}
// for range metricsQty {
// var metric _metric
// err = bin.ReadNInto(src, header)
// if err != nil {
// return
// }
for range metricsQty {
var metric _metric
err = bin.ReadNInto(src, header)
if err != nil {
return
}
// metricID := bin.GetUint32(header[0:])
// metric.MetricType = octopus.MetricType(header[4])
// metric.FracDigits = header[5]
// metric.RootPageNo = bin.GetUint32(header[6:])
// metric.LastPageNo = bin.GetUint32(header[10:])
// metric.Since = bin.GetUint32(header[14:])
// metric.SinceValue = bin.GetFloat64(header[18:])
// metric.Until = bin.GetUint32(header[26:])
// metric.UntilValue = bin.GetFloat64(header[30:])
// tSize := bin.GetUint16(header[38:])
// vSize := bin.GetUint16(header[40:])
metricID, _ := bin.GetUint32(header[0:])
metric.MetricType = qb.MetricType(header[4])
metric.FracDigits = header[5]
metric.LastPageNo, _ = bin.GetUint32(header[6:])
metric.Since, _ = bin.GetUint32(header[10:])
metric.SinceValue, _ = bin.GetFloat64(header[14:])
metric.Until, _ = bin.GetUint32(header[22:])
metric.UntilValue, _ = bin.GetFloat64(header[26:])
tSize, _ := bin.GetUint16(header[34:])
vSize, _ := bin.GetUint16(header[36:])
// buf := body[:tSize]
// err = bin.ReadNInto(src, buf)
// if err != nil {
// return
// }
// metric.TimestampsBuf = conbuf.NewFromBuffer(buf)
buf := body[:tSize]
err = bin.ReadNInto(src, buf)
if err != nil {
return
}
metric.Timestamps = chunkenc.NewReverseTimeDeltaCompressor(
conbuf.NewFromBuffer(buf),
int(tSize),
)
// buf = body[:vSize]
// err = bin.ReadNInto(src, buf)
// if err != nil {
// return
// }
// metric.ValuesBuf = conbuf.NewFromBuffer(buf)
buf = body[:vSize]
err = bin.ReadNInto(src, buf)
if err != nil {
return
}
// metric.Timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor(
// metric.TimestampsBuf, int(tSize))
if metric.MetricType == qb.Cumulative {
metric.Values = chunkenc.NewReverseCumulativeDeltaCompressor(
conbuf.NewFromBuffer(buf),
int(vSize),
metric.FracDigits,
)
} else {
metric.Values = chunkenc.NewReverseInstantDeltaCompressor(
conbuf.NewFromBuffer(buf),
int(vSize),
metric.FracDigits,
)
}
s.metrics[metricID] = &metric
}
// if metric.MetricType == octopus.Cumulative {
// metric.Values = chunkenc.NewReverseCumulativeDeltaCompressor(
// metric.ValuesBuf, int(vSize), metric.FracDigits)
// } else {
// metric.Values = chunkenc.NewReverseInstantDeltaCompressor(
// metric.ValuesBuf, int(vSize), metric.FracDigits)
// }
// s.metrics[metricID] = &metric
// }
err = restoreFreeList(s.freeList, src)
if err != nil {
return fmt.Errorf("restore dataFreeList: %s", err)
}
// err = restoreFreeList(s.freeList, src)
// if err != nil {
// return fmt.Errorf("restore dataFreeList: %s", err)
// }
err = restoreFreeList(s.freeList, src)
if err != nil {
return fmt.Errorf("restore indexFreeList: %s", err)
}
// err = restoreFreeList(s.freeList, src)
// if err != nil {
// return fmt.Errorf("restore indexFreeList: %s", err)
// }
calculatedChecksum := hasher.Sum32()
// calculatedChecksum := hasher.Sum32()
writtenChecksum, err := bin.ReadUint32(file)
if err != nil {
return
}
// writtenChecksum, err := bin.ReadUint32(file)
// if err != nil {
// return
// }
// if calculatedChecksum != writtenChecksum {
// return fmt.Errorf("calculated checksum %d not equal written checksum %d", calculatedChecksum, writtenChecksum)
// }
if calculatedChecksum != writtenChecksum {
return fmt.Errorf("calculated checksum %d not equal written checksum %d", calculatedChecksum, writtenChecksum)
}
return
}
@@ -258,9 +234,9 @@ func (s *Database) loadSnapshot(fileName string) (err error) {
func freeListWriteTo(freeList *freelist.FreeList, dst io.Writer) error {
serialized, err := freeList.Serialize()
if err != nil {
octopus.Abort(octopus.FailedFreeListSerialize, err)
qb.Abort(qb.FailedFreeListSerialize, err)
}
_, err = bin.WriteVarUint64(dst, uint64(len(serialized)))
_, err = bin.WriteVarSize(dst, len(serialized))
if err != nil {
return err
}
@@ -272,14 +248,32 @@ func freeListWriteTo(freeList *freelist.FreeList, dst io.Writer) error {
}
func restoreFreeList(freeList *freelist.FreeList, src io.Reader) error {
size, _, err := bin.ReadVarUint64(src)
size, err := bin.ReadVarSize(src)
if err != nil {
return err
}
serialized, err := bin.ReadN(src, int(size))
serialized, err := bin.ReadN(src, size)
if err != nil {
return err
}
freeList.Restore(serialized)
return nil
}
func writeChunks(dst io.Writer, chunks [][]byte, size int) (err error) {
remaining := size
for _, buf := range chunks {
if remaining < len(buf) {
buf = buf[:remaining]
}
_, err = dst.Write(buf)
if err != nil {
return
}
remaining -= len(buf)
if remaining == 0 {
break
}
}
return
}