dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-06-14 07:57:01 +03:00
parent 685c48f94f
commit 181031a753
14 changed files with 626 additions and 591 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
worker/metric.go
View File

@@ -6,6 +6,7 @@ import (
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/inbox"
"gordenko.dev/dima/qb/storage"
)
@@ -36,6 +37,14 @@ type Metric struct {
capturedState *CapturedState
}
func (s *Metric) MetricType() qb.MetricType {
return s.metricType
}
func (s *Metric) FracDigits() byte {
return s.fracDigits
}
func (s *Metric) LastValue() float64 {
if s.capturedState != nil {
return s.capturedState.LastValue
@@ -67,7 +76,7 @@ func (s *Metric) OnMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) {
}
// func (s *_metric) startAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(storage.AppendedMeasures)) {
func (s *Metric) StartAppendMeasures(req AppendMeasuresReq, tmp []byte, saveToStorage func(any)) {
func (s *Metric) StartAppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox *inbox.Inbox) {
if s.capturedState != nil {
s.WaitQueue = append(s.WaitQueue, req)
}
@@ -171,7 +180,7 @@ func (s *Metric) StartAppendMeasures(req AppendMeasuresReq, tmp []byte, saveToSt
if len(pages) > 0 {
// пишу в storage довгим шляхом через redo файл і запис в data файл
saveToStorage(storage.MeasuresAppendWithGrow{
storageInbox.Push(storage.MeasuresAppendWithGrow{
MetricID: req.MetricID,
LastPageNo: s.lastPageNo,
TimestampsRewindOffset: timestampsRewindOffset,
@@ -188,7 +197,7 @@ func (s *Metric) StartAppendMeasures(req AppendMeasuresReq, tmp []byte, saveToSt
})
} else {
// короткий шлях - запис лише в storage
saveToStorage(storage.MeasuresAppend{
storageInbox.Push(storage.MeasuresAppend{
MetricID: req.MetricID,
TimestampsRewindOffset: timestampsRewindOffset,
ValuesRewindOffset: valuesRewindOffset,

321
worker/snapshot.go Normal file
View File

@@ -0,0 +1,321 @@
package worker
import (
"bufio"
"fmt"
"io"
"os"
"path/filepath"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/storage"
"gordenko.dev/dima/qb/util"
)
/*
Формат:
metricsQty - varsize
[metric]*
де metric - це:
index free list frozen pages - varsize
indexFreeList size - varsize
indexFreeList - Nb
data free list frozen pages - varsize
dataFreeList size - varsize
dataFreeList - Nb
CRC32 - 4b
*/
const readBufferSize = 4 * 1024 * 1024 // 8mb
type WriteSnapshotIn struct {
SnapshotNumber int
Dir string
WriteBufferSize int
Metrics map[uint32]*Metric
FrozenIndexPagesCount int
IndexPageNumbers []uint32
FrozenDataPagesCount int
DataPageNumbers []uint32
}
func WriteSnapshot(in WriteSnapshotIn) (err error) {
var (
fileName = filepath.Join(in.Dir, fmt.Sprintf("%d.snapshot", in.SnapshotNumber))
hasher = util.NewHasher()
)
file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY, 0666) //0770)
if err != nil {
return
}
dst := io.MultiWriter(bufio.NewWriterSize(file, in.WriteBufferSize), hasher)
_, err = bin.WriteVarSize(dst, len(in.Metrics))
if err != nil {
return
}
for metricID, metric := range in.Metrics {
err = bin.WriteUint32(dst, metricID)
if err != nil {
return
}
err = metric.WriteTo(dst)
if err != nil {
return
}
}
// free index pages
err = writeFreePages(dst, in.FrozenIndexPagesCount, in.IndexPageNumbers)
if err != nil {
return
}
// free data pages
err = writeFreePages(dst, in.FrozenDataPagesCount, in.DataPageNumbers)
if err != nil {
return
}
// CRC32
bin.WriteUint32(file, hasher.Sum32())
err = file.Close()
if err != nil {
return
}
// prevLogNumber := logNumber - 1
// prevChanges := filepath.Join(s.dir, fmt.Sprintf("%d.changes", prevLogNumber))
// prevSnapshot := filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", prevLogNumber))
// isExist, err := isFileExist(prevChanges)
// if err != nil {
// return
// }
// if isExist {
// err = os.Remove(prevChanges)
// if err != nil {
// qb.Abort(qb.DeletePrevChangesFileFailed, err)
// }
// }
// isExist, err = isFileExist(prevSnapshot)
// if err != nil {
// return
// }
// if isExist {
// err = os.Remove(prevSnapshot)
// if err != nil {
// qb.Abort(qb.DeletePrevSnapshotFileFailed, err)
// }
// }
return
}
type ReadSnapshotOut struct {
Metrics map[uint32]*storage.ReplayMetric
FrozenIndexPagesCount int
IndexPageNumbers []uint32
FrozenDataPagesCount int
DataPageNumbers []uint32
}
func ReadSnapshot(fileName string) (out ReadSnapshotOut, err error) {
var (
metrics = make(map[uint32]*storage.ReplayMetric)
frozenIndexPagesCount int
indexPageNumbers []uint32
frozenDataPagesCount int
dataPageNumbers []uint32
hasher = util.NewHasher()
)
file, err := os.Open(fileName)
if err != nil {
return
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return
}
fileSize := stat.Size()
if fileSize == 0 {
return ReadSnapshotOut{
Metrics: metrics,
}, nil
}
if fileSize < 4 {
err = fmt.Errorf("%s is corrupted", fileName)
return
}
payloadSize := fileSize - 4
// 2. Створюємо великий буфер для NVMe (4 MB)
bufferedReader := bufio.NewReaderSize(file, readBufferSize)
// 3. Обмежуємо читання лише розміром Payload
limitReader := io.LimitReader(bufferedReader, payloadSize)
// 4. Створюємо хеш та обгортку TeeReader, яка рахує CRC32 на льоту
src := io.TeeReader(limitReader, hasher)
// читаю payload
metricsQty, err := bin.ReadVarSize(src)
if err != nil {
return
}
for range metricsQty {
var (
metricID uint32
buf = make([]byte, storage.DataPageSize)
m = &storage.ReplayMetric{
Buf: buf,
}
metricType byte
levelsCount int
)
metricID, err = bin.ReadUint32(src)
if err != nil {
return
}
metricType, err = bin.ReadByte(src)
if err != nil {
return
}
m.MetricType = qb.MetricType(metricType)
m.FracDigits, err = bin.ReadByte(src)
if err != nil {
return
}
m.LastPageNo, err = bin.ReadUint32(src)
if err != nil {
return
}
m.TimestampsSize, err = bin.ReadUint16AsInt(src)
if err != nil {
return
}
m.ValuesSize, err = bin.ReadUint16AsInt(src)
if err != nil {
return
}
err = bin.ReadNInto(src, buf[len(buf)-m.TimestampsSize:])
if err != nil {
return
}
err = bin.ReadNInto(src, buf[:m.ValuesSize])
if err != nil {
return
}
//
levelsCount, err = bin.ReadVarSize(src)
if err != nil {
return
}
for range levelsCount {
var (
buf = make([]byte, storage.IndexPageSize)
count int
)
count, err = bin.ReadVarSize(src)
if err != nil {
return
}
err = bin.ReadNInto(src, buf[:count*storage.IndexRecordSize])
if err != nil {
return
}
m.IndexLevelTails = append(m.IndexLevelTails, storage.IndexLevelTail{
Buffer: buf,
RecordsCount: count,
})
}
metrics[metricID] = m
}
// index pages
frozenIndexPagesCount, indexPageNumbers, err = readFreePages(src)
if err != nil {
return
}
// data pages
frozenDataPagesCount, dataPageNumbers, err = readFreePages(src)
if err != nil {
return
}
// verify CRC32
expectedCRC, err := bin.ReadUint32(bufferedReader)
if err != nil {
return
}
calculatedCRC := hasher.Sum32()
if expectedCRC != calculatedCRC {
err = fmt.Errorf("%s is corrupted. Calculated CRC %d not equal expected CRC %d",
fileName, calculatedCRC, expectedCRC)
return
}
return ReadSnapshotOut{
Metrics: metrics,
FrozenIndexPagesCount: frozenIndexPagesCount,
IndexPageNumbers: indexPageNumbers,
FrozenDataPagesCount: frozenDataPagesCount,
DataPageNumbers: dataPageNumbers,
}, nil
}
// HELPERS
func writeFreePages(dst io.Writer, frozenPagesCount int, pageNumbers []uint32) (err error) {
_, err = bin.WriteVarSize(dst, frozenPagesCount)
if err != nil {
return
}
_, err = bin.WriteVarSize(dst, len(pageNumbers))
if err != nil {
return
}
for _, pageNo := range pageNumbers {
err = bin.WriteUint32(dst, pageNo)
if err != nil {
return
}
}
return
}
func readFreePages(src io.Reader) (_ int, _ []uint32, err error) {
var (
frozenPagesCount int
pageNumbers []uint32
)
frozenPagesCount, err = bin.ReadVarSize(src)
if err != nil {
return
}
pageNumbersCount, err := bin.ReadVarSize(src)
if err != nil {
return
}
for range pageNumbersCount {
var pageNo uint32
pageNo, err = bin.ReadUint32(src)
if err != nil {
return
}
pageNumbers = append(pageNumbers, pageNo)
}
return frozenPagesCount, pageNumbers, nil
}

90
worker/snapshot_test.go Normal file
View File

@@ -0,0 +1,90 @@
package worker
import (
"fmt"
"slices"
"testing"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/storage"
)
func TestWriteReadSnapshot(t *testing.T) {
metrics := make(map[uint32]*Metric)
metrics[1] = &Metric{
metricType: qb.Cumulative,
fracDigits: 3,
lastPageNo: 1000,
buffer: []byte{},
}
in := WriteSnapshotIn{
SnapshotNumber: 1,
Dir: ".",
WriteBufferSize: 8 * 1024 * 1024,
Metrics: metrics,
FrozenIndexPagesCount: 7,
IndexPageNumbers: []uint32{
1, 2, 3, 4, 5,
},
FrozenDataPagesCount: 8,
DataPageNumbers: []uint32{
6, 7, 8,
},
}
err := WriteSnapshot(in)
if err != nil {
t.Fatalf("writeSnapshot: %s", err)
}
out, err := ReadSnapshot("1.snapshot")
if err != nil {
return
}
if out.FrozenIndexPagesCount != in.FrozenIndexPagesCount {
t.Fatalf("FrozenIndexPagesCount: got %d are not equal expected %d",
out.FrozenIndexPagesCount, in.FrozenIndexPagesCount)
}
if out.FrozenDataPagesCount != in.FrozenDataPagesCount {
t.Fatalf("FrozenDataPagesCount: got %d are not equal expected %d",
out.FrozenDataPagesCount, in.FrozenDataPagesCount)
}
if !slices.Equal(out.IndexPageNumbers, in.IndexPageNumbers) {
t.Fatalf("IndexPageNumbers: got %v are not equal expected %v",
out.IndexPageNumbers, in.IndexPageNumbers)
}
if !slices.Equal(out.DataPageNumbers, in.DataPageNumbers) {
t.Fatalf("DataPageNumbers: got %v are not equal expected %v",
out.DataPageNumbers, in.DataPageNumbers)
}
for metricID, replay := range out.Metrics {
origin, ok := metrics[metricID]
if !ok {
t.Fatalf("decoded metricID %d not found in metrics", metricID)
}
err = cmpMetric(replay, origin)
if err != nil {
t.Fatalf("metric %d: %s", metricID, err)
}
}
}
func cmpMetric(replay *storage.ReplayMetric, origin *Metric) error {
if replay.MetricType != origin.metricType {
return fmt.Errorf("metricType: got %d are not equal expected %d",
replay.MetricType, origin.metricType)
}
if replay.FracDigits != origin.fracDigits {
return fmt.Errorf("fracDigits: got %d are not equal expected %d",
replay.FracDigits, origin.fracDigits)
}
if replay.LastPageNo != origin.lastPageNo {
return fmt.Errorf("lastPageNo: got %d are not equal expected %d",
replay.LastPageNo, origin.lastPageNo)
}
// if !reflect.DeepEqual() {
// return fmt.Errorf("lastPageNo: got %d are not equal expected %d",
// replay.LastPageNo, origin.lastPageNo)
// }
return nil
}

52
worker/worker.go
View File

@@ -38,37 +38,51 @@ const (
)
type Worker struct {
//mutex sync.Mutex
inbox *inbox.Inbox
storageInbox *inbox.Inbox
dir string
snapshotNumber int
tmp []byte
metrics map[uint32]*Metric
saveToStorage func(any)
exitCh chan struct{}
waitGroup *sync.WaitGroup
}
type WorkerOptions struct {
Inbox *inbox.Inbox
StorageInbox *inbox.Inbox
Metrics map[uint32]*Metric
Dir string
ExitCh chan struct{}
WaitGroup *sync.WaitGroup
type Options struct {
Inbox *inbox.Inbox
StorageInbox *inbox.Inbox
ReplayMetrics map[uint32]*storage.ReplayMetric
Dir string
ExitCh chan struct{}
WaitGroup *sync.WaitGroup
}
func NewWorker(opt WorkerOptions) *Worker {
return &Worker{
func New(opt Options) *Worker {
s := &Worker{
inbox: opt.Inbox,
storageInbox: opt.StorageInbox,
dir: opt.Dir,
tmp: make([]byte, 26),
metrics: opt.Metrics,
metrics: make(map[uint32]*Metric),
exitCh: opt.ExitCh,
waitGroup: opt.WaitGroup,
}
for metricID, x := range opt.ReplayMetrics {
databuf := x.Buf[:storage.DataPagePayloadSize]
values := enc.NewValueDeltaCompressor(x.MetricType, x.FracDigits, databuf, x.ValuesSize)
s.metrics[metricID] = &Metric{
metricType: x.MetricType,
fracDigits: x.FracDigits,
lastPageNo: x.LastPageNo,
lastValue: values.LastValue(),
buffer: x.Buf,
timestamps: enc.NewTimeDeltaCompressor(databuf, x.TimestampsSize),
values: values,
indexLevelTails: x.IndexLevelTails,
}
}
return s
}
func (s *Worker) ReleaseRLock(metricID uint32) {
@@ -182,7 +196,7 @@ func (s *Worker) processMetricQueue(metricID uint32, metric *Metric, tmp []byte)
s.GetMetric(req)
case AppendMeasuresReq:
metric.StartAppendMeasures(req, tmp, s.saveToStorage)
metric.StartAppendMeasures(req, tmp, s.storageInbox)
case DeleteMetricReq:
s.startDeleteMetric(metric, req)
@@ -259,8 +273,8 @@ func (s *Worker) GetMetric(req GetMetricReq) {
if ok {
req.ResultCh <- GetMetricResult{
ResultCode: Succeed,
MetricType: metric.metricType,
FracDigits: metric.fracDigits,
MetricType: metric.MetricType(),
FracDigits: metric.FracDigits(),
}
} else {
req.ResultCh <- GetMetricResult{
@@ -404,7 +418,7 @@ func (s *Worker) AppendMeasures(req AppendMeasuresReq) {
metric.WaitQueue = append(metric.WaitQueue, req)
return
}
metric.StartAppendMeasures(req, s.tmp, s.saveToStorage)
metric.StartAppendMeasures(req, s.tmp, s.storageInbox)
}
type RangeScanResult struct {
@@ -431,7 +445,7 @@ func (s *Worker) RangeScan(req RangeScanReq) {
}
return
}
if metric.metricType != req.MetricType {
if metric.MetricType() != req.MetricType {
req.ResultCh <- RangeScanResult{
ResultCode: WrongMetricType,
}
@@ -467,7 +481,7 @@ func (s *Worker) tryFullScan(req FullScanReq) {
}
return
}
if metric.metricType != req.MetricType {
if metric.MetricType() != req.MetricType {
req.ResultCh <- FullScanResult{
ResultCode: WrongMetricType,
}
@@ -525,7 +539,7 @@ func (s *Worker) applyCommits(req storage.Changes) {
if req.SnapshotNumberCh != nil {
s.snapshotNumber++
err := storage.WriteSnapshot(storage.WriteSnapshotIn{
err := WriteSnapshot(WriteSnapshotIn{
SnapshotNumber: s.snapshotNumber,
Dir: s.dir,
WriteBufferSize: 4 * 1024 * 1024, // 1mb