dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-06-14 07:57:01 +03:00
parent 685c48f94f
commit 181031a753
14 changed files with 626 additions and 591 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
database/database.go
View File

@@ -103,14 +103,11 @@ func New(opt Options) (_ *Database, err error) {
} }
s.worker = worker.NewWorker(worker.WorkerOptions{ s.worker = worker.New(worker.Options{
Inbox: s.workerInbox, Inbox: s.workerInbox,
StorageInbox: storageInbox, StorageInbox: storageInbox,
//Metrics: make(map[uint32]*_metric), FIX //Metrics: make(map[uint32]*_metric), FIX
Dir: opt.Dir, Dir: opt.Dir,
// SaveToStorage: func(in any) {
// fmt.Println("save to storage")
// },
ExitCh: opt.ExitCh, ExitCh: opt.ExitCh,
WaitGroup: opt.WaitGroup, WaitGroup: opt.WaitGroup,
}) })

14
freelist/freelist.go
View File

@@ -55,22 +55,10 @@ func New(opt Options) (*FreeList, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
s.deltaFile, err = os.OpenFile(opt.DeltaFilePath, os.O_RDWR|os.O_CREATE, 0666) s.deltaFile, err = os.OpenFile(opt.DeltaFilePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
if err != nil { if err != nil {
return nil, err return nil, err
} }
// info, err := s.baseFile.Stat()
// if err != nil {
// return nil, err
// }
// fileSize := info.Size()
// if fileSize > 0 {
// if (fileSize % int64(s.pageSize)) > 0 {
// return nil, fmt.Errorf("file size %d is not a multiple of page size %d",
// fileSize, s.pageSize)
// }
// s.basePagesCount = int(fileSize) / s.pageSize
// }
return s, nil return s, nil
} }

8
qb.go
View File

@@ -173,6 +173,14 @@ func GetIndexFreeListFilePath(dir string, databaseName string) string {
return filepath.Join(dir, fmt.Sprintf("%s.index_free", databaseName)) return filepath.Join(dir, fmt.Sprintf("%s.index_free", databaseName))
} }
func GetIndexFreeListDeltaFilePath(dir string, databaseName string) string {
return filepath.Join(dir, fmt.Sprintf("%s.index_free_delta", databaseName))
}
func GetDataFreeListFilePath(dir string, databaseName string) string { func GetDataFreeListFilePath(dir string, databaseName string) string {
return filepath.Join(dir, fmt.Sprintf("%s.data_free", databaseName)) return filepath.Join(dir, fmt.Sprintf("%s.data_free", databaseName))
} }
func GetDataFreeListDeltaFilePath(dir string, databaseName string) string {
return filepath.Join(dir, fmt.Sprintf("%s.data_free_delta", databaseName))
}

56
recovery/recovery.go
View File

@@ -10,6 +10,7 @@ import (
"gordenko.dev/dima/qb" "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/freelist" "gordenko.dev/dima/qb/freelist"
"gordenko.dev/dima/qb/storage" "gordenko.dev/dima/qb/storage"
"gordenko.dev/dima/qb/worker"
) )
// Result зберігає результати для обох типів файлів // Result зберігає результати для обох типів файлів
@@ -110,7 +111,6 @@ func resolveRecoveryFiles(dir string) (_ RecoveryFiles, err error) {
} }
type RecoveryReport struct { type RecoveryReport struct {
//Metrics []
SnapshotNumber int SnapshotNumber int
WAL string WAL string
IndexFreeList *freelist.FreeList IndexFreeList *freelist.FreeList
@@ -118,32 +118,38 @@ type RecoveryReport struct {
} }
func Recovery(dir string, databaseName string) (_ RecoveryReport, err error) { func Recovery(dir string, databaseName string) (_ RecoveryReport, err error) {
files, err := resolveRecoveryFiles(dir) recoveryFiles, err := resolveRecoveryFiles(dir)
if err != nil { if err != nil {
return return
} }
var ( var (
frozenIndexPageCount int
frozenDataPageCount int
freeIndexPages []uint32 freeIndexPages []uint32
freeDataPages []uint32 freeDataPages []uint32
metrics map[uint32]*storage.ReplayMetric
) )
if files.Snapshot != "" { if recoveryFiles.Snapshot != "" {
var out storage.ReadSnapshotOut var snapshot worker.ReadSnapshotOut
out, err = storage.ReadSnapshot(files.Snapshot) snapshot, err = worker.ReadSnapshot(recoveryFiles.Snapshot)
if err != nil { if err != nil {
return return
} }
// //
freeIndexPages = out.IndexPageNumbers frozenIndexPageCount = snapshot.FrozenIndexPagesCount
freeDataPages = out.DataPageNumbers frozenDataPageCount = snapshot.FrozenDataPagesCount
freeIndexPages = snapshot.IndexPageNumbers
freeDataPages = snapshot.DataPageNumbers
metrics = snapshot.Metrics
} }
if files.WAL != "" { if recoveryFiles.WAL != "" {
var ( var (
walReader *storage.WALReader walReader *storage.WALReader
walReplayer *storage.WALReplayer walReplayer *storage.WALReplayer
) )
walReader, err = storage.NewWALReader(storage.WALReaderOptions{ walReader, err = storage.NewWALReader(storage.WALReaderOptions{
//FileName: JoinWALFileName(s.dir, files.WAL), FileName: recoveryFiles.WAL,
BufferSize: 8 * 1024 * 1024, BufferSize: 8 * 1024 * 1024,
}) })
if err != nil { if err != nil {
@@ -152,7 +158,7 @@ func Recovery(dir string, databaseName string) (_ RecoveryReport, err error) {
walReplayer, err = storage.NewWALReplayer(storage.WALReplayerOptions{ walReplayer, err = storage.NewWALReplayer(storage.WALReplayerOptions{
WALReader: walReader, WALReader: walReader,
Metrics: snapshot.Metrics, Metrics: metrics,
FreeIndexPages: freeIndexPages, FreeIndexPages: freeIndexPages,
FreeDataPages: freeDataPages, FreeDataPages: freeDataPages,
}) })
@@ -170,7 +176,7 @@ func Recovery(dir string, databaseName string) (_ RecoveryReport, err error) {
if err != nil { if err != nil {
return return
} }
err = storage.WriteDataPages(dataFile, walReplayer.DataPages()) err = storage.WriteDataPages(dataFile, walReplayer.DataPagesToRewrite())
if err != nil { if err != nil {
return return
} }
@@ -183,7 +189,7 @@ func Recovery(dir string, databaseName string) (_ RecoveryReport, err error) {
if err != nil { if err != nil {
return return
} }
err = storage.WriteIndexPages(indexFile, walReplayer.IndexPages()) err = storage.WriteIndexPages(indexFile, walReplayer.IndexPagesToRewrite())
if err != nil { if err != nil {
return return
} }
@@ -191,31 +197,31 @@ func Recovery(dir string, databaseName string) (_ RecoveryReport, err error) {
if err != nil { if err != nil {
return return
} }
freeIndexPages = walReplayer.FreeIndexPages()
freeDataPages = walReplayer.FreeDataPages()
} }
indexFreeList, err := freelist.New(freelist.Options{ indexFreeList, err := freelist.New(freelist.Options{
PageSize: 2048, PageSize: 2048,
BaseFilePath: filepath.Join(dir, databaseName+".free_index"), BaseFilePath: qb.GetIndexFreeListFilePath(dir, databaseName),
DeltaFilePath: filepath.Join(dir, databaseName+".free_index_delta"), DeltaFilePath: qb.GetIndexFreeListDeltaFilePath(dir, databaseName),
FrozenPageCount: frozenIndexPageCount,
}) })
indexFreeList.AddPageNumbers(freeIndexPages)
// FIX free pages sync
dataFreeList, err := freelist.New(freelist.Options{ dataFreeList, err := freelist.New(freelist.Options{
PageSize: 2048, PageSize: 2048,
BaseFilePath: filepath.Join(dir, databaseName+".free_data"), BaseFilePath: qb.GetDataFreeListFilePath(dir, databaseName),
DeltaFilePath: filepath.Join(dir, databaseName+".free_data_delta"), DeltaFilePath: qb.GetDataFreeListDeltaFilePath(dir, databaseName),
FrozenPageCount: frozenDataPageCount,
}) })
dataFreeList.AddPageNumbers(freeDataPages)
return RecoveryReport{ return RecoveryReport{
SnapshotNumber: files.SnapshotNumber, SnapshotNumber: recoveryFiles.SnapshotNumber,
WAL: files.WAL, WAL: recoveryFiles.WAL,
IndexFreeList: indexFreeList, IndexFreeList: indexFreeList,
DataFreeList: dataFreeList, DataFreeList: dataFreeList,
}, nil }, nil
} }
//func (s *Database) relayMetricsToMetrics(replayMetrics map[uint32]*ReplayMetric) {
//for metricID, x := range replayMetrics {
//s.metrics[metricID] = x.ToMetric() FIX
//}
//}

183
storage/replay_metric.go
View File

@@ -2,101 +2,50 @@ package storage
import ( import (
"fmt" "fmt"
"io"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb" "gordenko.dev/dima/qb"
) )
type ReplayMetric struct { type ReplayMetric struct {
metricType qb.MetricType MetricType qb.MetricType
fracDigits byte FracDigits byte
lastPageNo uint32 LastPageNo uint32
buf []byte Buf []byte
databuf []byte // якщо розмір buf == DataPageSize, то databuf буде менше (мінус футер) TimestampsSize int
tSize int ValuesSize int
vSize int IndexLevelTails []IndexLevelTail // root - last element
indexLevelTails []IndexLevelTail // root - last element
} }
func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) { // return occupied after write
metricType, err := bin.ReadByte(r) func writeTimestampsWithRewind(buf []byte, occupied int, payload []byte, rewindOffset int) int {
if err != nil { // timestamps writes right-to-left
return pos := DataPagePayloadSize - occupied
} // timestamps rewind offset moves pos left-to-right
s.metricType = qb.MetricType(metricType) pos += rewindOffset
s.fracDigits, err = bin.ReadByte(r) pos -= len(payload)
if err != nil { copy(buf[pos:], payload)
return return DataPagePayloadSize - pos
}
s.lastPageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
s.tSize, err = bin.ReadUint16AsInt(r)
if err != nil {
return
}
s.vSize, err = bin.ReadUint16AsInt(r)
if err != nil {
return
}
err = bin.ReadNInto(r, s.databuf[len(s.databuf)-s.tSize:])
if err != nil {
return
}
err = bin.ReadNInto(r, s.databuf[:s.vSize])
if err != nil {
return
}
//
levelsCount, err := bin.ReadVarSize(r)
if err != nil {
return
}
for range levelsCount {
var (
buf = make([]byte, IndexPageSize)
count int
)
count, err = bin.ReadVarSize(r)
if err != nil {
return
}
err = bin.ReadNInto(r, buf[:count*IndexRecordSize])
if err != nil {
return
}
s.indexLevelTails = append(s.indexLevelTails, IndexLevelTail{
Buffer: buf,
RecordsCount: count,
})
}
return
} }
// func appendNewRecordsToIndexTail(level IndexLevelTail, records []byte) { func writeValuesWithRewind(buf []byte, occupied int, payload []byte, rewindOffset int) int {
// values writes left-to-right
// } pos := occupied
// values rewind offset moves pos right-to-left
// func countReusedIndexPages(changedIndexLevels []ChangedIndexLevel) (count int) { pos -= rewindOffset
copy(buf[pos:], payload)
// return return pos + len(payload)
// } }
func (s *ReplayMetric) MeasuresAppend(rec MeasuresAppendRecord) { func (s *ReplayMetric) MeasuresAppend(rec MeasuresAppendRecord) {
timestampsSize := rec.TimestampsRewindOffset - len(rec.Timestamps) s.TimestampsSize = writeTimestampsWithRewind(s.Buf, s.TimestampsSize,
// копіюю нові дані із WAL з урахуванням offset-ів rec.Timestamps, rec.TimestampsRewindOffset)
copy(s.databuf[rec.ValuesRewindOffset:], rec.Values) s.ValuesSize = writeValuesWithRewind(s.Buf, s.ValuesSize,
copy(s.databuf[len(s.databuf)-timestampsSize:], rec.Timestamps) rec.Values, rec.ValuesRewindOffset)
//
s.tSize = timestampsSize
s.vSize = rec.ValuesRewindOffset - len(rec.Values)
} }
type AppendMeasuresResult struct { type AppendMeasuresResult struct {
IndexPages []PageToWrite IndexPagesToRewrite []PageToWrite
DataPages []PageToWrite DataPagesToRewrite []PageToWrite
ReusedIndexPagesCount int ReusedIndexPagesCount int
ReusedDataPagesCount int ReusedDataPagesCount int
} }
@@ -117,8 +66,7 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord,
// 1. набиваю сторінки для перезапису в index файлі // 1. набиваю сторінки для перезапису в index файлі
if isLastPacket { if isLastPacket {
if head != nil { if head != nil {
indexPages = append(indexPages, indexPages = append(indexPages, s.composeHeadIndexPage(levelIdx, head))
composeHeadIndexPage(levelIdx, s.indexLevelTails[levelIdx], head))
} }
for _, x := range change.IndexPages { for _, x := range change.IndexPages {
indexPages = append(indexPages, PageToWrite{ indexPages = append(indexPages, PageToWrite{
@@ -129,8 +77,8 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord,
} }
// 2. Вношу зміни в поточні індексні хвости // 2. Вношу зміни в поточні індексні хвости
if levelIdx < len(s.indexLevelTails) { if levelIdx < len(s.IndexLevelTails) {
level := s.indexLevelTails[levelIdx] level := s.IndexLevelTails[levelIdx]
if head != nil { if head != nil {
// попередній хвіст індексного рівня перетворився на head сторінку, // попередній хвіст індексного рівня перетворився на head сторінку,
// отже TailRecords - це новий хвіст // отже TailRecords - це новий хвіст
@@ -142,13 +90,13 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord,
copy(level.Buffer[pos:], change.TailRecords) copy(level.Buffer[pos:], change.TailRecords)
level.RecordsCount += newRecordsCount level.RecordsCount += newRecordsCount
} }
s.indexLevelTails[levelIdx] = level s.IndexLevelTails[levelIdx] = level
} else { } else {
// додаю новий індексний рівень // додаю новий індексний рівень
buf := make([]byte, IndexPageSize) buf := make([]byte, IndexPageSize)
copy(buf, change.TailRecords) copy(buf, change.TailRecords)
// //
s.indexLevelTails = append(s.indexLevelTails, IndexLevelTail{ s.IndexLevelTails = append(s.IndexLevelTails, IndexLevelTail{
Buffer: buf, Buffer: buf,
RecordsCount: newRecordsCount, RecordsCount: newRecordsCount,
}) })
@@ -169,7 +117,7 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord,
// 4. набиваю сторінки для перезапису в data файлі // 4. набиваю сторінки для перезапису в data файлі
if isLastPacket { if isLastPacket {
dataPages = append(dataPages, composeHeadDataPage(s.databuf, rec.DataPageTail)) dataPages = append(dataPages, s.composeHeadDataPage(rec.DataPageTail))
for _, x := range rec.DataPages { for _, x := range rec.DataPages {
dataPages = append(dataPages, PageToWrite{ dataPages = append(dataPages, PageToWrite{
@@ -190,22 +138,22 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord,
} }
// HeadDataPage є полюбому, оскільки це транзакція із мінімум однією заповненою // HeadDataPage є полюбому, оскільки це транзакція із мінімум однією заповненою
// data сторінкою. Отже TailTimestamps і TailValues - це нові хвости data рівня. // data сторінкою. Отже TailTimestamps і TailValues - це нові хвости data рівня.
copy(s.databuf, rec.TailValues) copy(s.Buf, rec.TailValues)
pos := len(s.databuf) - len(rec.TailTimestamps) s.ValuesSize = len(rec.TailValues)
copy(s.databuf[pos:], rec.TailTimestamps)
s.tSize = len(rec.TailTimestamps) pos := DataPagePayloadSize - len(rec.TailTimestamps)
s.vSize = len(rec.TailValues) copy(s.Buf[pos:], rec.TailTimestamps)
s.TimestampsSize = len(rec.TailTimestamps)
if len(rec.DataPages) == 0 { if len(rec.DataPages) == 0 {
s.lastPageNo = rec.DataPageTail.PageNo s.LastPageNo = rec.DataPageTail.PageNo
} else { } else {
s.lastPageNo = rec.DataPages[len(rec.DataPages)-1].PageNo s.LastPageNo = rec.DataPages[len(rec.DataPages)-1].PageNo
} }
return AppendMeasuresResult{ return AppendMeasuresResult{
IndexPages: indexPages, IndexPagesToRewrite: indexPages,
DataPages: dataPages, DataPagesToRewrite: dataPages,
ReusedIndexPagesCount: reusedIndexPages, ReusedIndexPagesCount: reusedIndexPages,
ReusedDataPagesCount: reusedDataPages, ReusedDataPagesCount: reusedDataPages,
} }
@@ -213,7 +161,8 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord,
// HELPERS // HELPERS
func composeHeadIndexPage(levelIdx int, level IndexLevelTail, head *IndexPageTail) PageToWrite { func (s *ReplayMetric) composeHeadIndexPage(levelIdx int, head *IndexPageTail) PageToWrite {
level := s.IndexLevelTails[levelIdx]
// розраховую pos, з якого буду дописувати хвіст // розраховую pos, з якого буду дописувати хвіст
pos := level.RecordsCount * IndexRecordSize pos := level.RecordsCount * IndexRecordSize
// створюю копію сторінки // створюю копію сторінки
@@ -238,23 +187,21 @@ func composeHeadIndexPage(levelIdx int, level IndexLevelTail, head *IndexPageTai
} }
} }
func composeHeadDataPage(databuf []byte, head DataPageTail) PageToWrite { func (s *ReplayMetric) composeHeadDataPage(head DataPageTail) PageToWrite {
// створюю копію сторінки // copy page
var ( page := make([]byte, DataPageSize)
page = make([]byte, DataPageSize) copy(page, s.Buf)
timestampsSize = head.TimestampsRewindOffset - len(head.Timestamps) // append data
) timestampsSize := writeTimestampsWithRewind(s.Buf, s.TimestampsSize,
// копіюю поточні дані head.Timestamps, head.TimestampsRewindOffset)
copy(page, databuf) valuesSize := writeValuesWithRewind(s.Buf, s.ValuesSize,
// копіюю нові дані із WAL з урахуванням offset-ів head.Values, head.ValuesRewindOffset)
copy(page[head.ValuesRewindOffset:], head.Values)
copy(page[len(page)-timestampsSize:], head.Timestamps)
// запечатати сторінку // запечатати сторінку
calculatedCRC := SealDataPage(SealDataPageIn{ calculatedCRC := SealDataPage(SealDataPageIn{
Content: page, Content: page,
PrevPageNo: head.PrevPageNo, PrevPageNo: head.PrevPageNo,
TimestampsSize: timestampsSize, TimestampsSize: timestampsSize,
ValuesSize: head.ValuesRewindOffset + len(head.Values), ValuesSize: valuesSize,
}) })
// перевірка CRC // перевірка CRC
if calculatedCRC != head.CRC32 { if calculatedCRC != head.CRC32 {
@@ -267,19 +214,3 @@ func composeHeadDataPage(databuf []byte, head DataPageTail) PageToWrite {
Content: page, Content: page,
} }
} }
// func (s *ReplayMetric) ToMetric() *worker.metric {
// databuf := s.buf[:storage.DataPagePayloadSize]
// values := enc.NewValueDeltaCompressor(s.metricType, s.fracDigits, databuf, s.vSize)
// return &_metric{
// metricType: s.metricType,
// fracDigits: s.fracDigits,
// lastPageNo: s.lastPageNo,
// lastValue: values.LastValue(),
// buffer: s.buf,
// timestamps: enc.NewTimeDeltaCompressor(databuf, s.tSize),
// values: values,
// indexLevelTails: s.indexLevelTails,
// }
// }

8
storage/storage.go
View File

@@ -72,7 +72,7 @@ type MeasuresAppendWithGrow struct {
IndexLevelTails []IndexLevelTail IndexLevelTails []IndexLevelTail
ResultCode byte ResultCode byte
WrittenCount int WrittenCount int
ResultCh chan struct{} ResultCh chan AppendMeasuresResult
} }
type MeasuresAppend struct { type MeasuresAppend struct {
@@ -83,7 +83,7 @@ type MeasuresAppend struct {
Values []byte Values []byte
ResultCode byte ResultCode byte
WrittenCount int WrittenCount int
ResultCh chan struct{} ResultCh chan AppendMeasuresResult
} }
type MeasuresDelete struct { type MeasuresDelete struct {
@@ -100,7 +100,7 @@ type MeasuresAppendCommited struct {
MetricID uint32 MetricID uint32
ResultCode byte ResultCode byte
WrittenCount int WrittenCount int
ResultCh chan struct{} ResultCh chan AppendMeasuresResult
} }
type MeasuresAppendWithGrowCommited struct { type MeasuresAppendWithGrowCommited struct {
@@ -109,7 +109,7 @@ type MeasuresAppendWithGrowCommited struct {
Index []IndexLevelTail Index []IndexLevelTail
ResultCode byte ResultCode byte
WrittenCount int WrittenCount int
ResultCh chan struct{} ResultCh chan AppendMeasuresResult
} }
type MeasuresDeleteCommited struct { type MeasuresDeleteCommited struct {

103
storage/storage_test.go
View File

@@ -868,82 +868,49 @@ func TestWritePreparer(t *testing.T) {
} }
} }
func TestWriteReadSnapshot(t *testing.T) { func TestWriteTimestampsWithRewind(t *testing.T) {
metrics := make(map[uint32]*_metric) DataPagePayloadSize = 8
metrics[1] = &_metric{ var (
metricType: qb.Cumulative, payload = []byte{
fracDigits: 3, 0xaa, 0xbb, 0xcc,
lastPageNo: 1000,
buffer: []byte{},
} }
in := WriteSnapshotIn{ before = []byte{
SnapshotNumber: 1, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, 0x02, 0x01,
Dir: ".",
WriteBufferSize: 8 * 1024 * 1024,
Metrics: metrics,
FrozenIndexPagesCount: 7,
IndexPageNumbers: []uint32{
1, 2, 3, 4, 5,
},
FrozenDataPagesCount: 8,
DataPageNumbers: []uint32{
6, 7, 8,
},
} }
err := WriteSnapshot(in) occupied = 4
if err != nil { after = []byte{
t.Fatalf("writeSnapshot: %s", err) 0x00, 0x00, 0x00, 0xaa, 0xbb, 0xcc, 0x02, 0x01,
} }
occupiedAfter = 5
out, err := ReadSnapshot("1.snapshot") )
if err != nil { occupied = writeTimestampsWithRewind(before, occupied, payload, 2)
return if occupied != occupiedAfter {
} t.Fatalf("got occupied %d are not equal expected %d", occupied, occupiedAfter)
if out.FrozenIndexPagesCount != in.FrozenIndexPagesCount {
t.Fatalf("FrozenIndexPagesCount: got %d are not equal expected %d",
out.FrozenIndexPagesCount, in.FrozenIndexPagesCount)
}
if out.FrozenDataPagesCount != in.FrozenDataPagesCount {
t.Fatalf("FrozenDataPagesCount: got %d are not equal expected %d",
out.FrozenDataPagesCount, in.FrozenDataPagesCount)
}
if !slices.Equal(out.IndexPageNumbers, in.IndexPageNumbers) {
t.Fatalf("IndexPageNumbers: got %v are not equal expected %v",
out.IndexPageNumbers, in.IndexPageNumbers)
}
if !slices.Equal(out.DataPageNumbers, in.DataPageNumbers) {
t.Fatalf("DataPageNumbers: got %v are not equal expected %v",
out.DataPageNumbers, in.DataPageNumbers)
}
for metricID, replay := range out.Metrics {
origin, ok := metrics[metricID]
if !ok {
t.Fatalf("decoded metricID %d not found in metrics", metricID)
}
err = cmpMetric(replay, origin)
if err != nil {
t.Fatalf("metric %d: %s", metricID, err)
} }
if !bytes.Equal(before, after) {
t.Fatalf("got buf % x are not equal expected % x", before, after)
} }
} }
func cmpMetric(replay *ReplayMetric, origin *_metric) error { func TestWriteValuesWithRewind(t *testing.T) {
if replay.metricType != origin.metricType { var (
return fmt.Errorf("metricType: got %d are not equal expected %d", payload = []byte{
replay.metricType, origin.metricType) 0xaa, 0xbb, 0xcc,
} }
if replay.fracDigits != origin.fracDigits { before = []byte{
return fmt.Errorf("fracDigits: got %d are not equal expected %d", 0x01, 0x02, 0x03, 0x04, 0x00, 0x00, 0x00, 0x00,
replay.fracDigits, origin.fracDigits)
} }
if replay.lastPageNo != origin.lastPageNo { occupied = 4
return fmt.Errorf("lastPageNo: got %d are not equal expected %d", after = []byte{
replay.lastPageNo, origin.lastPageNo) 0x01, 0x02, 0xaa, 0xbb, 0xcc, 0x00, 0x00, 0x00,
}
occupiedAfter = 5
)
occupied = writeValuesWithRewind(before, occupied, payload, 2)
if occupied != occupiedAfter {
t.Fatalf("got occupied %d are not equal expected %d", occupied, occupiedAfter)
}
if !bytes.Equal(before, after) {
t.Fatalf("got buf % x are not equal expected % x", before, after)
} }
// if !refect.DeepEq{
// return fmt.Errorf("lastPageNo: got %d are not equal expected %d",
// replay.lastPageNo, origin.lastPageNo)
// }
return nil
} }

20
storage/wal_reader.go
View File

@@ -173,14 +173,14 @@ func (s *WALReader) parseRecords(body []byte) ([]any, error) {
// HELPERS // HELPERS
func (s *WALReader) Seek(offset int64) error { // func (s *WALReader) Seek(offset int64) error {
ret, err := s.file.Seek(offset, 0) // ret, err := s.file.Seek(offset, 0)
if err != nil { // if err != nil {
return err // return err
} // }
if ret != offset { // if ret != offset {
return fmt.Errorf("ret %d != offset %d", ret, offset) // return fmt.Errorf("ret %d != offset %d", ret, offset)
} // }
return nil // return nil
} // }

51
storage/wal_replayer.go
View File

@@ -36,14 +36,22 @@ func NewWALReplayer(opt WALReplayerOptions) (*WALReplayer, error) {
}, nil }, nil
} }
func (s *WALReplayer) IndexPages() []PageToWrite { func (s *WALReplayer) IndexPagesToRewrite() []PageToWrite {
return s.indexPages return s.indexPages
} }
func (s *WALReplayer) DataPages() []PageToWrite { func (s *WALReplayer) DataPagesToRewrite() []PageToWrite {
return s.dataPages return s.dataPages
} }
func (s *WALReplayer) FreeIndexPages() []uint32 {
return s.freeIndexPages
}
func (s *WALReplayer) FreeDataPages() []uint32 {
return s.freeDataPages
}
func (s *WALReplayer) Replay() error { func (s *WALReplayer) Replay() error {
for { for {
records, isLastPacket, err := s.walReader.NextPacket() records, isLastPacket, err := s.walReader.NextPacket()
@@ -101,10 +109,9 @@ func (s *WALReplayer) onMetricAdd(rec MetricAddRecord) (err error) {
buf = make([]byte, DataPageSize) buf = make([]byte, DataPageSize)
) )
s.metrics[rec.MetricID] = &ReplayMetric{ s.metrics[rec.MetricID] = &ReplayMetric{
metricType: rec.MetricType, MetricType: rec.MetricType,
fracDigits: byte(rec.FracDigits), FracDigits: rec.FracDigits,
buf: buf, Buf: buf,
databuf: buf[:DataPagePayloadSize],
} }
return return
} }
@@ -137,39 +144,11 @@ func (s *WALReplayer) onMeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord,
result := metric.MeasuresAppendWithGrow(rec, isLastPacket) result := metric.MeasuresAppendWithGrow(rec, isLastPacket)
// //
if isLastPacket { if isLastPacket {
s.indexPages = append(s.indexPages, result.IndexPages...) s.indexPages = append(s.indexPages, result.IndexPagesToRewrite...)
s.dataPages = append(s.dataPages, result.DataPages...) s.dataPages = append(s.dataPages, result.DataPagesToRewrite...)
} }
// //
s.freeIndexPages = s.freeIndexPages[:len(s.freeIndexPages)-result.ReusedIndexPagesCount] s.freeIndexPages = s.freeIndexPages[:len(s.freeIndexPages)-result.ReusedIndexPagesCount]
s.freeDataPages = s.freeDataPages[:len(s.freeDataPages)-result.ReusedDataPagesCount] s.freeDataPages = s.freeDataPages[:len(s.freeDataPages)-result.ReusedDataPagesCount]
return return
} }
//func (s *WALReplayer) collectPagesToWrite(metric *ReplayMetric, rec WALRecordAppendMeasures) {
// metric.ApplyHeadDataPage(rec.CompletedDataPage)
// s.dataPages = append(s.dataPages, PageToWrite{
// PageNo: p.PageNo,
// Content: metric.buf,
// })
// for _, x := range rec.DataPages {
// s.dataPages = append(s.dataPages, PageToWrite{
// PageNo: x.PageNo,
// Content: x.Content,
// })
// }
//metric.ApplyHeadIndexPages(rec.ChangedIndexLevels)
//}
// func (s *WALReplayer) replayPacket(records []any, isLastPacket bool) {
// for _, untyped := range records {
// switch rec := untyped.(type) {
// case WALRecordAppendMeasures:
// s.onWALRecordAppendMeasures(rec, isLastPacket)
// }
// }
// return
// }

15
worker/metric.go
View File

@@ -6,6 +6,7 @@ import (
bin "gordenko.dev/dima/bin/little" bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb" "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/inbox"
"gordenko.dev/dima/qb/storage" "gordenko.dev/dima/qb/storage"
) )
@@ -36,6 +37,14 @@ type Metric struct {
capturedState *CapturedState capturedState *CapturedState
} }
func (s *Metric) MetricType() qb.MetricType {
return s.metricType
}
func (s *Metric) FracDigits() byte {
return s.fracDigits
}
func (s *Metric) LastValue() float64 { func (s *Metric) LastValue() float64 {
if s.capturedState != nil { if s.capturedState != nil {
return s.capturedState.LastValue return s.capturedState.LastValue
@@ -67,7 +76,7 @@ func (s *Metric) OnMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) {
} }
// func (s *_metric) startAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(storage.AppendedMeasures)) { // func (s *_metric) startAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(storage.AppendedMeasures)) {
func (s *Metric) StartAppendMeasures(req AppendMeasuresReq, tmp []byte, saveToStorage func(any)) { func (s *Metric) StartAppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox *inbox.Inbox) {
if s.capturedState != nil { if s.capturedState != nil {
s.WaitQueue = append(s.WaitQueue, req) s.WaitQueue = append(s.WaitQueue, req)
} }
@@ -171,7 +180,7 @@ func (s *Metric) StartAppendMeasures(req AppendMeasuresReq, tmp []byte, saveToSt
if len(pages) > 0 { if len(pages) > 0 {
// пишу в storage довгим шляхом через redo файл і запис в data файл // пишу в storage довгим шляхом через redo файл і запис в data файл
saveToStorage(storage.MeasuresAppendWithGrow{ storageInbox.Push(storage.MeasuresAppendWithGrow{
MetricID: req.MetricID, MetricID: req.MetricID,
LastPageNo: s.lastPageNo, LastPageNo: s.lastPageNo,
TimestampsRewindOffset: timestampsRewindOffset, TimestampsRewindOffset: timestampsRewindOffset,
@@ -188,7 +197,7 @@ func (s *Metric) StartAppendMeasures(req AppendMeasuresReq, tmp []byte, saveToSt
}) })
} else { } else {
// короткий шлях - запис лише в storage // короткий шлях - запис лише в storage
saveToStorage(storage.MeasuresAppend{ storageInbox.Push(storage.MeasuresAppend{
MetricID: req.MetricID, MetricID: req.MetricID,
TimestampsRewindOffset: timestampsRewindOffset, TimestampsRewindOffset: timestampsRewindOffset,
ValuesRewindOffset: valuesRewindOffset, ValuesRewindOffset: valuesRewindOffset,

78
storage/snapshot.go → worker/snapshot.go
View File

@@ -1,4 +1,4 @@
package storage package worker
import ( import (
"bufio" "bufio"
@@ -8,6 +8,8 @@ import (
"path/filepath" "path/filepath"
bin "gordenko.dev/dima/bin/little" bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/storage"
"gordenko.dev/dima/qb/util" "gordenko.dev/dima/qb/util"
) )
@@ -28,17 +30,13 @@ dataFreeList - Nb
CRC32 - 4b CRC32 - 4b
*/ */
const ReadBufferSize = 8 * 1024 * 1024 // 8mb const readBufferSize = 4 * 1024 * 1024 // 8mb
type WritableMetric interface {
WriteTo(io.Writer) error
}
type WriteSnapshotIn struct { type WriteSnapshotIn struct {
SnapshotNumber int SnapshotNumber int
Dir string Dir string
WriteBufferSize int WriteBufferSize int
Metrics map[uint32]WritableMetric Metrics map[uint32]*Metric
FrozenIndexPagesCount int FrozenIndexPagesCount int
IndexPageNumbers []uint32 IndexPageNumbers []uint32
FrozenDataPagesCount int FrozenDataPagesCount int
@@ -122,7 +120,7 @@ func WriteSnapshot(in WriteSnapshotIn) (err error) {
} }
type ReadSnapshotOut struct { type ReadSnapshotOut struct {
Metrics map[uint32]*ReplayMetric Metrics map[uint32]*storage.ReplayMetric
FrozenIndexPagesCount int FrozenIndexPagesCount int
IndexPageNumbers []uint32 IndexPageNumbers []uint32
FrozenDataPagesCount int FrozenDataPagesCount int
@@ -131,7 +129,7 @@ type ReadSnapshotOut struct {
func ReadSnapshot(fileName string) (out ReadSnapshotOut, err error) { func ReadSnapshot(fileName string) (out ReadSnapshotOut, err error) {
var ( var (
metrics = make(map[uint32]*ReplayMetric) metrics = make(map[uint32]*storage.ReplayMetric)
frozenIndexPagesCount int frozenIndexPagesCount int
indexPageNumbers []uint32 indexPageNumbers []uint32
frozenDataPagesCount int frozenDataPagesCount int
@@ -180,22 +178,70 @@ func ReadSnapshot(fileName string) (out ReadSnapshotOut, err error) {
for range metricsQty { for range metricsQty {
var ( var (
metricID uint32 metricID uint32
buf = make([]byte, DataPageSize) buf = make([]byte, storage.DataPageSize)
// для простоти створюю буфер максимального розміру m = &storage.ReplayMetric{
metric = &ReplayMetric{ Buf: buf,
buf: buf,
databuf: buf[:DataPagePayloadSize],
} }
metricType byte
levelsCount int
) )
metricID, err = bin.ReadUint32(src) metricID, err = bin.ReadUint32(src)
if err != nil { if err != nil {
return return
} }
err = metric.ReadFrom(src) metricType, err = bin.ReadByte(src)
if err != nil { if err != nil {
return return
} }
metrics[metricID] = metric m.MetricType = qb.MetricType(metricType)
m.FracDigits, err = bin.ReadByte(src)
if err != nil {
return
}
m.LastPageNo, err = bin.ReadUint32(src)
if err != nil {
return
}
m.TimestampsSize, err = bin.ReadUint16AsInt(src)
if err != nil {
return
}
m.ValuesSize, err = bin.ReadUint16AsInt(src)
if err != nil {
return
}
err = bin.ReadNInto(src, buf[len(buf)-m.TimestampsSize:])
if err != nil {
return
}
err = bin.ReadNInto(src, buf[:m.ValuesSize])
if err != nil {
return
}
//
levelsCount, err = bin.ReadVarSize(src)
if err != nil {
return
}
for range levelsCount {
var (
buf = make([]byte, storage.IndexPageSize)
count int
)
count, err = bin.ReadVarSize(src)
if err != nil {
return
}
err = bin.ReadNInto(src, buf[:count*storage.IndexRecordSize])
if err != nil {
return
}
m.IndexLevelTails = append(m.IndexLevelTails, storage.IndexLevelTail{
Buffer: buf,
RecordsCount: count,
})
}
metrics[metricID] = m
} }
// index pages // index pages
frozenIndexPagesCount, indexPageNumbers, err = readFreePages(src) frozenIndexPagesCount, indexPageNumbers, err = readFreePages(src)

90
worker/snapshot_test.go Normal file
View File

@@ -0,0 +1,90 @@
package worker
import (
"fmt"
"slices"
"testing"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/storage"
)
func TestWriteReadSnapshot(t *testing.T) {
metrics := make(map[uint32]*Metric)
metrics[1] = &Metric{
metricType: qb.Cumulative,
fracDigits: 3,
lastPageNo: 1000,
buffer: []byte{},
}
in := WriteSnapshotIn{
SnapshotNumber: 1,
Dir: ".",
WriteBufferSize: 8 * 1024 * 1024,
Metrics: metrics,
FrozenIndexPagesCount: 7,
IndexPageNumbers: []uint32{
1, 2, 3, 4, 5,
},
FrozenDataPagesCount: 8,
DataPageNumbers: []uint32{
6, 7, 8,
},
}
err := WriteSnapshot(in)
if err != nil {
t.Fatalf("writeSnapshot: %s", err)
}
out, err := ReadSnapshot("1.snapshot")
if err != nil {
return
}
if out.FrozenIndexPagesCount != in.FrozenIndexPagesCount {
t.Fatalf("FrozenIndexPagesCount: got %d are not equal expected %d",
out.FrozenIndexPagesCount, in.FrozenIndexPagesCount)
}
if out.FrozenDataPagesCount != in.FrozenDataPagesCount {
t.Fatalf("FrozenDataPagesCount: got %d are not equal expected %d",
out.FrozenDataPagesCount, in.FrozenDataPagesCount)
}
if !slices.Equal(out.IndexPageNumbers, in.IndexPageNumbers) {
t.Fatalf("IndexPageNumbers: got %v are not equal expected %v",
out.IndexPageNumbers, in.IndexPageNumbers)
}
if !slices.Equal(out.DataPageNumbers, in.DataPageNumbers) {
t.Fatalf("DataPageNumbers: got %v are not equal expected %v",
out.DataPageNumbers, in.DataPageNumbers)
}
for metricID, replay := range out.Metrics {
origin, ok := metrics[metricID]
if !ok {
t.Fatalf("decoded metricID %d not found in metrics", metricID)
}
err = cmpMetric(replay, origin)
if err != nil {
t.Fatalf("metric %d: %s", metricID, err)
}
}
}
func cmpMetric(replay *storage.ReplayMetric, origin *Metric) error {
if replay.MetricType != origin.metricType {
return fmt.Errorf("metricType: got %d are not equal expected %d",
replay.MetricType, origin.metricType)
}
if replay.FracDigits != origin.fracDigits {
return fmt.Errorf("fracDigits: got %d are not equal expected %d",
replay.FracDigits, origin.fracDigits)
}
if replay.LastPageNo != origin.lastPageNo {
return fmt.Errorf("lastPageNo: got %d are not equal expected %d",
replay.LastPageNo, origin.lastPageNo)
}
// if !reflect.DeepEqual() {
// return fmt.Errorf("lastPageNo: got %d are not equal expected %d",
// replay.LastPageNo, origin.lastPageNo)
// }
return nil
}

42
worker/worker.go
View File

@@ -38,37 +38,51 @@ const (
) )
type Worker struct { type Worker struct {
//mutex sync.Mutex
inbox *inbox.Inbox inbox *inbox.Inbox
storageInbox *inbox.Inbox storageInbox *inbox.Inbox
dir string dir string
snapshotNumber int snapshotNumber int
tmp []byte tmp []byte
metrics map[uint32]*Metric metrics map[uint32]*Metric
saveToStorage func(any)
exitCh chan struct{} exitCh chan struct{}
waitGroup *sync.WaitGroup waitGroup *sync.WaitGroup
} }
type WorkerOptions struct { type Options struct {
Inbox *inbox.Inbox Inbox *inbox.Inbox
StorageInbox *inbox.Inbox StorageInbox *inbox.Inbox
Metrics map[uint32]*Metric ReplayMetrics map[uint32]*storage.ReplayMetric
Dir string Dir string
ExitCh chan struct{} ExitCh chan struct{}
WaitGroup *sync.WaitGroup WaitGroup *sync.WaitGroup
} }
func NewWorker(opt WorkerOptions) *Worker { func New(opt Options) *Worker {
return &Worker{ s := &Worker{
inbox: opt.Inbox, inbox: opt.Inbox,
storageInbox: opt.StorageInbox, storageInbox: opt.StorageInbox,
dir: opt.Dir, dir: opt.Dir,
tmp: make([]byte, 26), tmp: make([]byte, 26),
metrics: opt.Metrics, metrics: make(map[uint32]*Metric),
exitCh: opt.ExitCh, exitCh: opt.ExitCh,
waitGroup: opt.WaitGroup, waitGroup: opt.WaitGroup,
} }
for metricID, x := range opt.ReplayMetrics {
databuf := x.Buf[:storage.DataPagePayloadSize]
values := enc.NewValueDeltaCompressor(x.MetricType, x.FracDigits, databuf, x.ValuesSize)
s.metrics[metricID] = &Metric{
metricType: x.MetricType,
fracDigits: x.FracDigits,
lastPageNo: x.LastPageNo,
lastValue: values.LastValue(),
buffer: x.Buf,
timestamps: enc.NewTimeDeltaCompressor(databuf, x.TimestampsSize),
values: values,
indexLevelTails: x.IndexLevelTails,
}
}
return s
} }
func (s *Worker) ReleaseRLock(metricID uint32) { func (s *Worker) ReleaseRLock(metricID uint32) {
@@ -182,7 +196,7 @@ func (s *Worker) processMetricQueue(metricID uint32, metric *Metric, tmp []byte)
s.GetMetric(req) s.GetMetric(req)
case AppendMeasuresReq: case AppendMeasuresReq:
metric.StartAppendMeasures(req, tmp, s.saveToStorage) metric.StartAppendMeasures(req, tmp, s.storageInbox)
case DeleteMetricReq: case DeleteMetricReq:
s.startDeleteMetric(metric, req) s.startDeleteMetric(metric, req)
@@ -259,8 +273,8 @@ func (s *Worker) GetMetric(req GetMetricReq) {
if ok { if ok {
req.ResultCh <- GetMetricResult{ req.ResultCh <- GetMetricResult{
ResultCode: Succeed, ResultCode: Succeed,
MetricType: metric.metricType, MetricType: metric.MetricType(),
FracDigits: metric.fracDigits, FracDigits: metric.FracDigits(),
} }
} else { } else {
req.ResultCh <- GetMetricResult{ req.ResultCh <- GetMetricResult{
@@ -404,7 +418,7 @@ func (s *Worker) AppendMeasures(req AppendMeasuresReq) {
metric.WaitQueue = append(metric.WaitQueue, req) metric.WaitQueue = append(metric.WaitQueue, req)
return return
} }
metric.StartAppendMeasures(req, s.tmp, s.saveToStorage) metric.StartAppendMeasures(req, s.tmp, s.storageInbox)
} }
type RangeScanResult struct { type RangeScanResult struct {
@@ -431,7 +445,7 @@ func (s *Worker) RangeScan(req RangeScanReq) {
} }
return return
} }
if metric.metricType != req.MetricType { if metric.MetricType() != req.MetricType {
req.ResultCh <- RangeScanResult{ req.ResultCh <- RangeScanResult{
ResultCode: WrongMetricType, ResultCode: WrongMetricType,
} }
@@ -467,7 +481,7 @@ func (s *Worker) tryFullScan(req FullScanReq) {
} }
return return
} }
if metric.metricType != req.MetricType { if metric.MetricType() != req.MetricType {
req.ResultCh <- FullScanResult{ req.ResultCh <- FullScanResult{
ResultCode: WrongMetricType, ResultCode: WrongMetricType,
} }
@@ -525,7 +539,7 @@ func (s *Worker) applyCommits(req storage.Changes) {
if req.SnapshotNumberCh != nil { if req.SnapshotNumberCh != nil {
s.snapshotNumber++ s.snapshotNumber++
err := storage.WriteSnapshot(storage.WriteSnapshotIn{ err := WriteSnapshot(WriteSnapshotIn{
SnapshotNumber: s.snapshotNumber, SnapshotNumber: s.snapshotNumber,
Dir: s.dir, Dir: s.dir,
WriteBufferSize: 4 * 1024 * 1024, // 1mb WriteBufferSize: 4 * 1024 * 1024, // 1mb