dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
dima
2026-06-13 22:43:17 +00:00
parent 1d1fdb1f0b
commit 685c48f94f
28 changed files with 2492 additions and 2657 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
storage/page_ingester.go
View File

@@ -236,7 +236,7 @@ func appendIndexRecord(in appendIndexRecordIn) []byte {
buf = in.Records
)
if in.RecordsCount > 0 {
pos = in.RecordsCount * indexRecordSize
pos = in.RecordsCount * IndexRecordSize
//fmt.Printf("pos: %d\n", pos)
} else {
buf = make([]byte, IndexPageSize) // IndexPageIncSize

55
storage/pageno_provider.go Normal file
View File

@@ -0,0 +1,55 @@
package storage
import (
"errors"
"math"
"os"
"gordenko.dev/dima/qb/freelist"
)
type PageNoProvider struct {
freeList *freelist.FreeList
pagesCount uint32
}
type PageNoProviderOptions struct {
FilePath string
PageSize int
FreeList *freelist.FreeList
}
func NewPageNoProvider(opt PageNoProviderOptions) (_ *PageNoProvider, err error) {
if opt.FreeList == nil {
return nil, errors.New("FreeList option is required")
}
file, err := os.Open(opt.FilePath)
if err != nil {
return
}
defer file.Close()
info, err := file.Stat()
if err != nil {
return
}
pagesCount := info.Size() / int64(opt.PageSize)
return &PageNoProvider{
freeList: opt.FreeList,
pagesCount: uint32(pagesCount),
}, nil
}
func (s *PageNoProvider) GetPageNumber() (uint32, bool, error) {
pageNo, err := s.freeList.GetPageNumber()
if err != nil {
return 0, false, err
}
if pageNo > 0 {
return pageNo, true, nil
}
if s.pagesCount < math.MaxUint32 {
s.pagesCount++
return s.pagesCount, false, nil
}
return 0, false, errors.New("no space")
}

285
storage/replay_metric.go Normal file
View File

@@ -0,0 +1,285 @@
package storage
import (
"fmt"
"io"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
)
type ReplayMetric struct {
metricType qb.MetricType
fracDigits byte
lastPageNo uint32
buf []byte
databuf []byte // якщо розмір buf == DataPageSize, то databuf буде менше (мінус футер)
tSize int
vSize int
indexLevelTails []IndexLevelTail // root - last element
}
func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) {
metricType, err := bin.ReadByte(r)
if err != nil {
return
}
s.metricType = qb.MetricType(metricType)
s.fracDigits, err = bin.ReadByte(r)
if err != nil {
return
}
s.lastPageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
s.tSize, err = bin.ReadUint16AsInt(r)
if err != nil {
return
}
s.vSize, err = bin.ReadUint16AsInt(r)
if err != nil {
return
}
err = bin.ReadNInto(r, s.databuf[len(s.databuf)-s.tSize:])
if err != nil {
return
}
err = bin.ReadNInto(r, s.databuf[:s.vSize])
if err != nil {
return
}
//
levelsCount, err := bin.ReadVarSize(r)
if err != nil {
return
}
for range levelsCount {
var (
buf = make([]byte, IndexPageSize)
count int
)
count, err = bin.ReadVarSize(r)
if err != nil {
return
}
err = bin.ReadNInto(r, buf[:count*IndexRecordSize])
if err != nil {
return
}
s.indexLevelTails = append(s.indexLevelTails, IndexLevelTail{
Buffer: buf,
RecordsCount: count,
})
}
return
}
// func appendNewRecordsToIndexTail(level IndexLevelTail, records []byte) {
// }
// func countReusedIndexPages(changedIndexLevels []ChangedIndexLevel) (count int) {
// return
// }
func (s *ReplayMetric) MeasuresAppend(rec MeasuresAppendRecord) {
timestampsSize := rec.TimestampsRewindOffset - len(rec.Timestamps)
// копіюю нові дані із WAL з урахуванням offset-ів
copy(s.databuf[rec.ValuesRewindOffset:], rec.Values)
copy(s.databuf[len(s.databuf)-timestampsSize:], rec.Timestamps)
//
s.tSize = timestampsSize
s.vSize = rec.ValuesRewindOffset - len(rec.Values)
}
type AppendMeasuresResult struct {
IndexPages []PageToWrite
DataPages []PageToWrite
ReusedIndexPagesCount int
ReusedDataPagesCount int
}
func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord, isLastPacket bool) (_ AppendMeasuresResult) {
var (
reusedIndexPages int
reusedDataPages int
indexPages []PageToWrite
dataPages []PageToWrite
)
// Додати в index level tails недостаючі дані, або замінити
for levelIdx, change := range rec.ChangedIndexLevels {
var (
head = change.IndexPageTail
newRecordsCount = len(change.TailRecords) / IndexRecordSize
)
// 1. набиваю сторінки для перезапису в index файлі
if isLastPacket {
if head != nil {
indexPages = append(indexPages,
composeHeadIndexPage(levelIdx, s.indexLevelTails[levelIdx], head))
}
for _, x := range change.IndexPages {
indexPages = append(indexPages, PageToWrite{
PageNo: x.PageNo,
Content: x.Content,
})
}
}
// 2. Вношу зміни в поточні індексні хвости
if levelIdx < len(s.indexLevelTails) {
level := s.indexLevelTails[levelIdx]
if head != nil {
// попередній хвіст індексного рівня перетворився на head сторінку,
// отже TailRecords - це новий хвіст
copy(level.Buffer, change.TailRecords)
level.RecordsCount = newRecordsCount
} else {
// TailRecords - це нові дані, які треба додати
pos := level.RecordsCount * IndexRecordSize
copy(level.Buffer[pos:], change.TailRecords)
level.RecordsCount += newRecordsCount
}
s.indexLevelTails[levelIdx] = level
} else {
// додаю новий індексний рівень
buf := make([]byte, IndexPageSize)
copy(buf, change.TailRecords)
//
s.indexLevelTails = append(s.indexLevelTails, IndexLevelTail{
Buffer: buf,
RecordsCount: newRecordsCount,
})
}
// 3. рахую кількість reused індексних сторінок
if head != nil {
if head.Reused {
reusedIndexPages++
}
}
for _, x := range change.IndexPages {
if x.Reused {
reusedIndexPages++
}
}
}
// 4. набиваю сторінки для перезапису в data файлі
if isLastPacket {
dataPages = append(dataPages, composeHeadDataPage(s.databuf, rec.DataPageTail))
for _, x := range rec.DataPages {
dataPages = append(dataPages, PageToWrite{
PageNo: x.PageNo,
Content: x.Content,
})
}
}
// рахую кількість reused дата сторінок
if rec.DataPageTail.Reused {
reusedDataPages++
}
for _, x := range rec.DataPages {
if x.Reused {
reusedDataPages++
}
}
// HeadDataPage є полюбому, оскільки це транзакція із мінімум однією заповненою
// data сторінкою. Отже TailTimestamps і TailValues - це нові хвости data рівня.
copy(s.databuf, rec.TailValues)
pos := len(s.databuf) - len(rec.TailTimestamps)
copy(s.databuf[pos:], rec.TailTimestamps)
s.tSize = len(rec.TailTimestamps)
s.vSize = len(rec.TailValues)
if len(rec.DataPages) == 0 {
s.lastPageNo = rec.DataPageTail.PageNo
} else {
s.lastPageNo = rec.DataPages[len(rec.DataPages)-1].PageNo
}
return AppendMeasuresResult{
IndexPages: indexPages,
DataPages: dataPages,
ReusedIndexPagesCount: reusedIndexPages,
ReusedDataPagesCount: reusedDataPages,
}
}
// HELPERS
func composeHeadIndexPage(levelIdx int, level IndexLevelTail, head *IndexPageTail) PageToWrite {
// розраховую pos, з якого буду дописувати хвіст
pos := level.RecordsCount * IndexRecordSize
// створюю копію сторінки
page := make([]byte, IndexPageSize)
copy(page, level.Buffer[:pos]) // поточні дані
copy(page[pos:], head.Records)
// запечатати сторінку
calculatedCRC := SealIndexPage(SealIndexPageIn{
Content: page,
RecordsCount: level.RecordsCount + len(head.Records)/IndexRecordSize,
ZeroLevel: levelIdx == 0,
})
// перевірка CRC
if calculatedCRC != head.CRC32 {
qb.Abort(qb.WALReplayFailed,
fmt.Errorf("calculated CRC %d not equal expected %d of head page %d on index level %d",
calculatedCRC, head.CRC32, head.PageNo, levelIdx))
}
return PageToWrite{
PageNo: head.PageNo,
Content: page,
}
}
func composeHeadDataPage(databuf []byte, head DataPageTail) PageToWrite {
// створюю копію сторінки
var (
page = make([]byte, DataPageSize)
timestampsSize = head.TimestampsRewindOffset - len(head.Timestamps)
)
// копіюю поточні дані
copy(page, databuf)
// копіюю нові дані із WAL з урахуванням offset-ів
copy(page[head.ValuesRewindOffset:], head.Values)
copy(page[len(page)-timestampsSize:], head.Timestamps)
// запечатати сторінку
calculatedCRC := SealDataPage(SealDataPageIn{
Content: page,
PrevPageNo: head.PrevPageNo,
TimestampsSize: timestampsSize,
ValuesSize: head.ValuesRewindOffset + len(head.Values),
})
// перевірка CRC
if calculatedCRC != head.CRC32 {
qb.Abort(qb.WALReplayFailed,
fmt.Errorf("calculated CRC %d not equal expected %d of head data page %d",
calculatedCRC, head.CRC32, head.PageNo))
}
return PageToWrite{
PageNo: head.PageNo,
Content: page,
}
}
// func (s *ReplayMetric) ToMetric() *worker.metric {
// databuf := s.buf[:storage.DataPagePayloadSize]
// values := enc.NewValueDeltaCompressor(s.metricType, s.fracDigits, databuf, s.vSize)
// return &_metric{
// metricType: s.metricType,
// fracDigits: s.fracDigits,
// lastPageNo: s.lastPageNo,
// lastValue: values.LastValue(),
// buffer: s.buf,
// timestamps: enc.NewTimeDeltaCompressor(databuf, s.tSize),
// values: values,
// indexLevelTails: s.indexLevelTails,
// }
// }

275
storage/snapshot.go Normal file
View File

@@ -0,0 +1,275 @@
package storage
import (
"bufio"
"fmt"
"io"
"os"
"path/filepath"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb/util"
)
/*
Формат:
metricsQty - varsize
[metric]*
де metric - це:
index free list frozen pages - varsize
indexFreeList size - varsize
indexFreeList - Nb
data free list frozen pages - varsize
dataFreeList size - varsize
dataFreeList - Nb
CRC32 - 4b
*/
const ReadBufferSize = 8 * 1024 * 1024 // 8mb
type WritableMetric interface {
WriteTo(io.Writer) error
}
type WriteSnapshotIn struct {
SnapshotNumber int
Dir string
WriteBufferSize int
Metrics map[uint32]WritableMetric
FrozenIndexPagesCount int
IndexPageNumbers []uint32
FrozenDataPagesCount int
DataPageNumbers []uint32
}
func WriteSnapshot(in WriteSnapshotIn) (err error) {
var (
fileName = filepath.Join(in.Dir, fmt.Sprintf("%d.snapshot", in.SnapshotNumber))
hasher = util.NewHasher()
)
file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY, 0666) //0770)
if err != nil {
return
}
dst := io.MultiWriter(bufio.NewWriterSize(file, in.WriteBufferSize), hasher)
_, err = bin.WriteVarSize(dst, len(in.Metrics))
if err != nil {
return
}
for metricID, metric := range in.Metrics {
err = bin.WriteUint32(dst, metricID)
if err != nil {
return
}
err = metric.WriteTo(dst)
if err != nil {
return
}
}
// free index pages
err = writeFreePages(dst, in.FrozenIndexPagesCount, in.IndexPageNumbers)
if err != nil {
return
}
// free data pages
err = writeFreePages(dst, in.FrozenDataPagesCount, in.DataPageNumbers)
if err != nil {
return
}
// CRC32
bin.WriteUint32(file, hasher.Sum32())
err = file.Close()
if err != nil {
return
}
// prevLogNumber := logNumber - 1
// prevChanges := filepath.Join(s.dir, fmt.Sprintf("%d.changes", prevLogNumber))
// prevSnapshot := filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", prevLogNumber))
// isExist, err := isFileExist(prevChanges)
// if err != nil {
// return
// }
// if isExist {
// err = os.Remove(prevChanges)
// if err != nil {
// qb.Abort(qb.DeletePrevChangesFileFailed, err)
// }
// }
// isExist, err = isFileExist(prevSnapshot)
// if err != nil {
// return
// }
// if isExist {
// err = os.Remove(prevSnapshot)
// if err != nil {
// qb.Abort(qb.DeletePrevSnapshotFileFailed, err)
// }
// }
return
}
type ReadSnapshotOut struct {
Metrics map[uint32]*ReplayMetric
FrozenIndexPagesCount int
IndexPageNumbers []uint32
FrozenDataPagesCount int
DataPageNumbers []uint32
}
func ReadSnapshot(fileName string) (out ReadSnapshotOut, err error) {
var (
metrics = make(map[uint32]*ReplayMetric)
frozenIndexPagesCount int
indexPageNumbers []uint32
frozenDataPagesCount int
dataPageNumbers []uint32
hasher = util.NewHasher()
)
file, err := os.Open(fileName)
if err != nil {
return
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return
}
fileSize := stat.Size()
if fileSize == 0 {
return ReadSnapshotOut{
Metrics: metrics,
}, nil
}
if fileSize < 4 {
err = fmt.Errorf("%s is corrupted", fileName)
return
}
payloadSize := fileSize - 4
// 2. Створюємо великий буфер для NVMe (4 MB)
bufferedReader := bufio.NewReaderSize(file, readBufferSize)
// 3. Обмежуємо читання лише розміром Payload
limitReader := io.LimitReader(bufferedReader, payloadSize)
// 4. Створюємо хеш та обгортку TeeReader, яка рахує CRC32 на льоту
src := io.TeeReader(limitReader, hasher)
// читаю payload
metricsQty, err := bin.ReadVarSize(src)
if err != nil {
return
}
for range metricsQty {
var (
metricID uint32
buf = make([]byte, DataPageSize)
// для простоти створюю буфер максимального розміру
metric = &ReplayMetric{
buf: buf,
databuf: buf[:DataPagePayloadSize],
}
)
metricID, err = bin.ReadUint32(src)
if err != nil {
return
}
err = metric.ReadFrom(src)
if err != nil {
return
}
metrics[metricID] = metric
}
// index pages
frozenIndexPagesCount, indexPageNumbers, err = readFreePages(src)
if err != nil {
return
}
// data pages
frozenDataPagesCount, dataPageNumbers, err = readFreePages(src)
if err != nil {
return
}
// verify CRC32
expectedCRC, err := bin.ReadUint32(bufferedReader)
if err != nil {
return
}
calculatedCRC := hasher.Sum32()
if expectedCRC != calculatedCRC {
err = fmt.Errorf("%s is corrupted. Calculated CRC %d not equal expected CRC %d",
fileName, calculatedCRC, expectedCRC)
return
}
return ReadSnapshotOut{
Metrics: metrics,
FrozenIndexPagesCount: frozenIndexPagesCount,
IndexPageNumbers: indexPageNumbers,
FrozenDataPagesCount: frozenDataPagesCount,
DataPageNumbers: dataPageNumbers,
}, nil
}
// HELPERS
func writeFreePages(dst io.Writer, frozenPagesCount int, pageNumbers []uint32) (err error) {
_, err = bin.WriteVarSize(dst, frozenPagesCount)
if err != nil {
return
}
_, err = bin.WriteVarSize(dst, len(pageNumbers))
if err != nil {
return
}
for _, pageNo := range pageNumbers {
err = bin.WriteUint32(dst, pageNo)
if err != nil {
return
}
}
return
}
func readFreePages(src io.Reader) (_ int, _ []uint32, err error) {
var (
frozenPagesCount int
pageNumbers []uint32
)
frozenPagesCount, err = bin.ReadVarSize(src)
if err != nil {
return
}
pageNumbersCount, err := bin.ReadVarSize(src)
if err != nil {
return
}
for range pageNumbersCount {
var pageNo uint32
pageNo, err = bin.ReadUint32(src)
if err != nil {
return
}
pageNumbers = append(pageNumbers, pageNo)
}
return frozenPagesCount, pageNumbers, nil
}

4
storage/storage.go
View File

@@ -24,7 +24,7 @@ var (
indexRecordsCountIdx = IndexPageSize - 6
isZeroLevelIdx = IndexPageSize - 7
maxRecordsOnIndexPage = (IndexPageSize - IndexPageFooterSize) / indexRecordSize
maxRecordsOnIndexPage = (IndexPageSize - IndexPageFooterSize) / IndexRecordSize
// timestampSize = 4
// pairSize = timestampSize + PageNoSize
@@ -35,7 +35,7 @@ var (
const (
DataPageFooterSize = 12
IndexPageFooterSize = 7
indexRecordSize = 8
IndexRecordSize = 8
)
type IndexRecord struct {

80
storage/storage_test.go
View File

@@ -867,3 +867,83 @@ func TestWritePreparer(t *testing.T) {
fmt.Printf("%#v\n", x)
}
}
func TestWriteReadSnapshot(t *testing.T) {
metrics := make(map[uint32]*_metric)
metrics[1] = &_metric{
metricType: qb.Cumulative,
fracDigits: 3,
lastPageNo: 1000,
buffer: []byte{},
}
in := WriteSnapshotIn{
SnapshotNumber: 1,
Dir: ".",
WriteBufferSize: 8 * 1024 * 1024,
Metrics: metrics,
FrozenIndexPagesCount: 7,
IndexPageNumbers: []uint32{
1, 2, 3, 4, 5,
},
FrozenDataPagesCount: 8,
DataPageNumbers: []uint32{
6, 7, 8,
},
}
err := WriteSnapshot(in)
if err != nil {
t.Fatalf("writeSnapshot: %s", err)
}
out, err := ReadSnapshot("1.snapshot")
if err != nil {
return
}
if out.FrozenIndexPagesCount != in.FrozenIndexPagesCount {
t.Fatalf("FrozenIndexPagesCount: got %d are not equal expected %d",
out.FrozenIndexPagesCount, in.FrozenIndexPagesCount)
}
if out.FrozenDataPagesCount != in.FrozenDataPagesCount {
t.Fatalf("FrozenDataPagesCount: got %d are not equal expected %d",
out.FrozenDataPagesCount, in.FrozenDataPagesCount)
}
if !slices.Equal(out.IndexPageNumbers, in.IndexPageNumbers) {
t.Fatalf("IndexPageNumbers: got %v are not equal expected %v",
out.IndexPageNumbers, in.IndexPageNumbers)
}
if !slices.Equal(out.DataPageNumbers, in.DataPageNumbers) {
t.Fatalf("DataPageNumbers: got %v are not equal expected %v",
out.DataPageNumbers, in.DataPageNumbers)
}
for metricID, replay := range out.Metrics {
origin, ok := metrics[metricID]
if !ok {
t.Fatalf("decoded metricID %d not found in metrics", metricID)
}
err = cmpMetric(replay, origin)
if err != nil {
t.Fatalf("metric %d: %s", metricID, err)
}
}
}
func cmpMetric(replay *ReplayMetric, origin *_metric) error {
if replay.metricType != origin.metricType {
return fmt.Errorf("metricType: got %d are not equal expected %d",
replay.metricType, origin.metricType)
}
if replay.fracDigits != origin.fracDigits {
return fmt.Errorf("fracDigits: got %d are not equal expected %d",
replay.fracDigits, origin.fracDigits)
}
if replay.lastPageNo != origin.lastPageNo {
return fmt.Errorf("lastPageNo: got %d are not equal expected %d",
replay.lastPageNo, origin.lastPageNo)
}
// if !refect.DeepEq{
// return fmt.Errorf("lastPageNo: got %d are not equal expected %d",
// replay.lastPageNo, origin.lastPageNo)
// }
return nil
}

8
storage/wal_records.go
View File

@@ -194,7 +194,7 @@ func (s MeasuresAppendWithGrowRecord) Pack(w io.Writer) {
bin.WriteUint32(w, tail.PageNo)
bin.WriteBool(w, tail.Reused)
bin.WriteUint32(w, tail.CRC32)
bin.WriteVarSize(w, len(tail.Records)/indexRecordSize)
bin.WriteVarSize(w, len(tail.Records)/IndexRecordSize)
w.Write(tail.Records)
} else {
bin.WriteBool(w, false)
@@ -207,7 +207,7 @@ func (s MeasuresAppendWithGrowRecord) Pack(w io.Writer) {
w.Write(p.Content)
}
// level tail
bin.WriteVarSize(w, len(level.TailRecords)/indexRecordSize)
bin.WriteVarSize(w, len(level.TailRecords)/IndexRecordSize)
w.Write(level.TailRecords)
}
}
@@ -313,7 +313,7 @@ func (s *MeasuresAppendWithGrowRecord) Parse(r io.Reader) (err error) {
if err != nil {
return
}
tail.Records, err = bin.ReadN(r, recordsCount*indexRecordSize)
tail.Records, err = bin.ReadN(r, recordsCount*IndexRecordSize)
if err != nil {
return
}
@@ -344,7 +344,7 @@ func (s *MeasuresAppendWithGrowRecord) Parse(r io.Reader) (err error) {
if err != nil {
return
}
level.TailRecords, err = bin.ReadN(r, recordsCount*indexRecordSize)
level.TailRecords, err = bin.ReadN(r, recordsCount*IndexRecordSize)
if err != nil {
return
}

175
storage/wal_replayer.go Normal file
View File

@@ -0,0 +1,175 @@
package storage
import (
"errors"
"fmt"
)
type WALReplayer struct {
walReader *WALReader
metrics map[uint32]*ReplayMetric
freeIndexPages []uint32
freeDataPages []uint32
indexPages []PageToWrite
dataPages []PageToWrite
}
type WALReplayerOptions struct {
WALReader *WALReader
Metrics map[uint32]*ReplayMetric // from snapshot
FreeIndexPages []uint32 // from snapshot
FreeDataPages []uint32 // from snapshot
}
func NewWALReplayer(opt WALReplayerOptions) (*WALReplayer, error) {
if opt.WALReader == nil {
return nil, errors.New("required option: WALReader")
}
if opt.Metrics == nil {
return nil, errors.New("required option: Metrics")
}
return &WALReplayer{
walReader: opt.WALReader,
metrics: opt.Metrics,
freeIndexPages: opt.FreeIndexPages,
freeDataPages: opt.FreeDataPages,
}, nil
}
func (s *WALReplayer) IndexPages() []PageToWrite {
return s.indexPages
}
func (s *WALReplayer) DataPages() []PageToWrite {
return s.dataPages
}
func (s *WALReplayer) Replay() error {
for {
records, isLastPacket, err := s.walReader.NextPacket()
if err != nil {
return err
}
for _, rec := range records {
if err = s.replayRecord(rec, isLastPacket); err != nil {
return err
}
}
}
}
func (s *WALReplayer) replayRecord(untyped any, isLastPacket bool) (err error) {
switch rec := untyped.(type) {
case MeasuresAppendRecord:
if err = s.onMeasuresAppend(rec); err != nil {
return
}
case MeasuresAppendWithGrowRecord:
if err = s.onMeasuresAppendWithGrow(rec, isLastPacket); err != nil {
return
}
case MetricAddRecord:
if err = s.onMetricAdd(rec); err != nil {
return
}
case MetricDeleteRecord:
if err = s.onMetricDelete(rec); err != nil {
return
}
// case DeletedMeasures:
// metric, ok := s.metrics[rec.MetricID]
// if ok {
// metric.DeleteMeasures()
// if len(rec.FreePageNumbers) > 0 {
// s.freeList.AddPages(rec.FreePageNumbers)
// }
// }
// default:
// qb.Abort(qb.UnknownstorageRecordTypeBug,
// fmt.Errorf("bug: unknown record type %T in TransactionLog", rec))
}
return nil
}
func (s *WALReplayer) onMetricAdd(rec MetricAddRecord) (err error) {
_, ok := s.metrics[rec.MetricID]
if ok {
return fmt.Errorf("metric %d add failed: already added", rec.MetricID)
}
var (
buf = make([]byte, DataPageSize)
)
s.metrics[rec.MetricID] = &ReplayMetric{
metricType: rec.MetricType,
fracDigits: byte(rec.FracDigits),
buf: buf,
databuf: buf[:DataPagePayloadSize],
}
return
}
func (s *WALReplayer) onMetricDelete(rec MetricDeleteRecord) error {
_, ok := s.metrics[rec.MetricID]
if !ok {
return fmt.Errorf("metric %d deletion failed: not found", rec.MetricID)
}
delete(s.metrics, rec.MetricID)
s.freeIndexPages = append(s.freeIndexPages, rec.FreeIndexPages...)
s.freeDataPages = append(s.freeDataPages, rec.FreeDataPages...)
return nil
}
func (s *WALReplayer) onMeasuresAppend(rec MeasuresAppendRecord) (err error) {
metric, ok := s.metrics[rec.MetricID]
if !ok {
return fmt.Errorf("append measures failed: metric %d not found", rec.MetricID)
}
metric.MeasuresAppend(rec)
return
}
func (s *WALReplayer) onMeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord, isLastPacket bool) (err error) {
metric, ok := s.metrics[rec.MetricID]
if !ok {
return fmt.Errorf("append measures failed: metric %d not found", rec.MetricID)
}
result := metric.MeasuresAppendWithGrow(rec, isLastPacket)
//
if isLastPacket {
s.indexPages = append(s.indexPages, result.IndexPages...)
s.dataPages = append(s.dataPages, result.DataPages...)
}
//
s.freeIndexPages = s.freeIndexPages[:len(s.freeIndexPages)-result.ReusedIndexPagesCount]
s.freeDataPages = s.freeDataPages[:len(s.freeDataPages)-result.ReusedDataPagesCount]
return
}
//func (s *WALReplayer) collectPagesToWrite(metric *ReplayMetric, rec WALRecordAppendMeasures) {
// metric.ApplyHeadDataPage(rec.CompletedDataPage)
// s.dataPages = append(s.dataPages, PageToWrite{
// PageNo: p.PageNo,
// Content: metric.buf,
// })
// for _, x := range rec.DataPages {
// s.dataPages = append(s.dataPages, PageToWrite{
// PageNo: x.PageNo,
// Content: x.Content,
// })
// }
//metric.ApplyHeadIndexPages(rec.ChangedIndexLevels)
//}
// func (s *WALReplayer) replayPacket(records []any, isLastPacket bool) {
// for _, untyped := range records {
// switch rec := untyped.(type) {
// case WALRecordAppendMeasures:
// s.onWALRecordAppendMeasures(rec, isLastPacket)
// }
// }
// return
// }

173
storage/wal_writer.go
View File

@@ -4,7 +4,6 @@ import (
"bytes"
"errors"
"fmt"
"log"
"math"
"os"
"path/filepath"
@@ -12,6 +11,7 @@ import (
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/freelist"
"gordenko.dev/dima/qb/inbox"
)
const (
@@ -42,10 +42,6 @@ type FreeList interface {
GetPageNumber() (uint32, error)
}
func JoinChangesFileName(dir string, logNumber int) string {
return filepath.Join(dir, fmt.Sprintf("%d.changes", logNumber))
}
type MetricsState struct {
//Metrics []Metric
WaitCh chan struct{}
@@ -63,32 +59,37 @@ type Changes struct {
type Writer struct {
mutex sync.Mutex
inbox *inbox.Inbox
workerInbox *inbox.Inbox
dataPagesCount uint32
indexPagesCount uint32
dataFreeList *freelist.FreeList
indexFreeList *freelist.FreeList
//atree *atree.Atree
dir string
w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
wal *os.File
dataFile *os.File
indexFile *os.File
input []any
writePreparer *WritePreparer
appendToWorkerQueue func(any)
written int64
isExited bool
exitCh chan struct{}
waitGroup *sync.WaitGroup
signalCh chan struct{}
dir string
databaseName string
w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
wal *os.File
dataFile *os.File
indexFile *os.File
input []any
writePreparer *WritePreparer
written int64
isExited bool
exitCh chan struct{}
waitGroup *sync.WaitGroup
//signalCh chan struct{}
}
type WriterOptions struct {
Dir string
Inbox *inbox.Inbox
WorkerInbox *inbox.Inbox
Dir string
DatabaseName string
//SnapshotNumber int // номер журнала
AppendToWorkerQueue func(any)
DataFreeList *freelist.FreeList
IndexFreeList *freelist.FreeList
WAL string
DataFreeList *freelist.FreeList
IndexFreeList *freelist.FreeList
//Atree *atree.Atree
ExitCh chan struct{}
WaitGroup *sync.WaitGroup
@@ -101,12 +102,21 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
// if (opt.DataPageSize % 2) != 0 {
// return nil, errors.New("DataPageSize must be multiple of 2")
// }
if opt.Inbox == nil {
return nil, errors.New("Inbox option is required")
}
if opt.WorkerInbox == nil {
return nil, errors.New("WorkerInbox option is required")
}
if opt.Dir == "" {
return nil, errors.New("Dir option is required")
}
if opt.AppendToWorkerQueue == nil {
return nil, errors.New("AppendToWorkerQueue option is required")
if opt.DatabaseName == "" {
return nil, errors.New("DatabaseName option is required")
}
// if opt.AppendToWorkerQueue == nil {
// return nil, errors.New("AppendToWorkerQueue option is required")
// }
if opt.DataFreeList == nil {
return nil, errors.New("DataFreeList option is required")
}
@@ -123,9 +133,21 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
return nil, errors.New("WaitGroup option is required")
}
indexPageNoProvider, err := NewPageNoProvider(PageNoProviderOptions{
FilePath: qb.GetIndexFilePath(opt.Dir, opt.DatabaseName),
PageSize: IndexPageSize,
FreeList: opt.IndexFreeList,
})
dataPageNoProvider, err := NewPageNoProvider(PageNoProviderOptions{
FilePath: qb.GetDataFilePath(opt.Dir, opt.DatabaseName),
PageSize: DataPageSize,
FreeList: opt.DataFreeList,
})
pageIngester, err := NewPageIngester(PageIngesterOptions{
GetIndexPageNumber: s.getIndexPageNumber,
GetDataPageNumber: s.getDataPageNumber,
GetIndexPageNumber: indexPageNoProvider.GetPageNumber,
GetDataPageNumber: dataPageNoProvider.GetPageNumber,
})
if err != nil {
return nil, err
@@ -137,27 +159,31 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
})
s := &Writer{
dir: opt.Dir,
appendToWorkerQueue: opt.AppendToWorkerQueue,
dataFreeList: opt.DataFreeList,
indexFreeList: opt.IndexFreeList,
writePreparer: writePreparer,
//atree: opt.Atree,
exitCh: opt.ExitCh,
waitGroup: opt.WaitGroup,
signalCh: make(chan struct{}, 1),
inbox: opt.Inbox,
workerInbox: opt.WorkerInbox,
dir: opt.Dir,
databaseName: opt.DatabaseName,
dataFreeList: opt.DataFreeList,
indexFreeList: opt.IndexFreeList,
writePreparer: writePreparer,
exitCh: opt.ExitCh,
waitGroup: opt.WaitGroup,
}
if opt.SnapshotNumber <= 0 {
log.Fatalln("FIX FUCK NUMBER")
}
s.wal, err = os.OpenFile(
JoinChangesFileName(opt.Dir, opt.SnapshotNumber),
os.O_APPEND|os.O_WRONLY,
filePerm,
)
if err != nil {
return nil, err
if opt.WAL != "" {
s.wal, err = os.OpenFile(filepath.Join(opt.Dir, opt.WAL), os.O_APPEND|os.O_WRONLY, filePerm)
if err != nil {
return nil, err
}
} else {
s.wal, err = os.OpenFile(
qb.GetWALFilePath(opt.Dir, opt.DatabaseName, 0), // .wal_0
os.O_CREATE|os.O_APPEND|os.O_WRONLY,
filePerm,
)
if err != nil {
return nil, err
}
}
return s, nil
}
@@ -165,7 +191,7 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
func (s *Writer) Run() {
for {
select {
case <-s.signalCh:
case <-s.inbox.Ready():
if err := s.packAndWrite(); err != nil {
qb.Abort(qb.FailedWriteToTxLog, err)
}
@@ -208,16 +234,16 @@ func (s *Writer) getIndexPageNumber() (uint32, bool, error) {
}
func (s *Writer) packAndWrite() (err error) {
s.mutex.Lock()
//s.mutex.Lock()
//isExited := s.isExited
input := s.input
s.input = nil
// var exitWaitGroup *sync.WaitGroup
// if s.isExited {
// exitWaitGroup = s.waitGroup
// }
s.mutex.Unlock()
//s.mutex.Unlock()
input := s.inbox.Drain()
prepared := s.writePreparer.Prepare(input)
@@ -234,7 +260,11 @@ func (s *Writer) packAndWrite() (err error) {
}
// 4. Пишу в atree сторінки
err = s.WritePagesToAtree(prepared.WriteToIndex, prepared.WriteToData)
err = WriteDataPages(s.dataFile, prepared.WriteToData)
if err != nil {
return
}
err = WriteIndexPages(s.indexFile, prepared.WriteToIndex)
if err != nil {
return
}
@@ -262,7 +292,7 @@ func (s *Writer) packAndWrite() (err error) {
snapshotNumberCh := make(chan int, 1)
s.appendToWorkerQueue(Changes{
s.workerInbox.Push(Changes{
Commits: prepared.Commits,
SnapshotNumberCh: snapshotNumberCh,
FrozenIndexPagesCount: s.indexFreeList.Pages(),
@@ -285,7 +315,7 @@ func (s *Writer) packAndWrite() (err error) {
var err error
s.wal, err = os.OpenFile(
JoinChangesFileName(s.dir, snapshotNumber),
qb.GetWALFilePath(s.dir, s.databaseName, snapshotNumber),
os.O_CREATE|os.O_WRONLY,
filePerm,
)
@@ -293,24 +323,13 @@ func (s *Writer) packAndWrite() (err error) {
return fmt.Errorf("create new changes file: %s", err)
}
} else {
s.appendToWorkerQueue(Changes{
s.workerInbox.Push(Changes{
Commits: prepared.Commits,
})
}
return nil
}
func (s *Writer) Append(req any) {
s.mutex.Lock()
s.input = append(s.input, req)
s.mutex.Unlock()
select {
case s.signalCh <- struct{}{}:
default:
}
}
// func (s *Writer) reset() {
// s.buf.Reset()
// s.buf.Write([]byte{
@@ -350,9 +369,8 @@ type PageToWrite struct {
Content []byte
}
// writePagesToAtree - записує сторінки в .data та .index файли
func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToWrite) (err error) {
for _, p := range dataPages {
func WriteDataPages(file *os.File, pages []PageToWrite) (err error) {
for _, p := range pages {
if len(p.Content) != DataPageSize {
return fmt.Errorf("wrong data page size: %d", len(p.Content))
}
@@ -360,7 +378,7 @@ func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToW
off = int(p.PageNo-1) * DataPageSize
n int
)
n, err = s.dataFile.WriteAt(p.Content, int64(off))
n, err = file.WriteAt(p.Content, int64(off))
if err != nil {
return
}
@@ -368,7 +386,11 @@ func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToW
return fmt.Errorf("write %d instead of %d", n, DataPageSize)
}
}
for _, p := range indexPages {
return
}
func WriteIndexPages(file *os.File, pages []PageToWrite) (err error) {
for _, p := range pages {
if len(p.Content) != IndexPageSize {
return fmt.Errorf("wrong index page size: %d", len(p.Content))
}
@@ -376,7 +398,7 @@ func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToW
off = int(p.PageNo-1) * IndexPageSize
n int
)
n, err = s.indexFile.WriteAt(p.Content, int64(off))
n, err = file.WriteAt(p.Content, int64(off))
if err != nil {
return
}
@@ -384,7 +406,7 @@ func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToW
return fmt.Errorf("write %d instead of %d", n, IndexPageSize)
}
}
return nil
return
}
// API
@@ -401,13 +423,6 @@ func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToW
// ValuesBuf []byte
// }
func (s *Writer) sendSignal() {
select {
case s.signalCh <- struct{}{}:
default:
}
}
// Якщо (Pages) > 0 - незаповнену сторінку кодую в стисненому вигляді.
// Якщо (Pages) = 0 - кодую просто пари timestamp / value
// func (s *Writer) packWriteAppended(w *bytes.Buffer, req AppendedPagesReq) {

8
storage/write_preparer.go
View File

@@ -140,13 +140,13 @@ func (s *WritePreparer) Prepare(input []any) PreparedData {
if len(level.IndexPages) > 0 {
first := level.IndexPages[0]
firstCRC32, _ := bin.GetUint32(first.Content[indexCRC32Idx:])
skipSize := level.SkipRecords * indexRecordSize
skipSize := level.SkipRecords * IndexRecordSize
indexPageTail = &IndexPageTail{
PageNo: first.PageNo,
Reused: first.Reused,
CRC32: firstCRC32,
// в WAL файл попадає лише payload
Records: first.Content[skipSize : maxRecordsOnIndexPage*indexRecordSize],
Records: first.Content[skipSize : maxRecordsOnIndexPage*IndexRecordSize],
}
indexPages = level.IndexPages[1:]
} else {
@@ -156,7 +156,7 @@ func (s *WritePreparer) Prepare(input []any) PreparedData {
IndexPageTail: indexPageTail,
IndexPages: indexPages,
// в WAL файл попадає лише payload
TailRecords: level.TailRecords[:level.TailRecordsCount*indexRecordSize],
TailRecords: level.TailRecords[:level.TailRecordsCount*IndexRecordSize],
})
}
// Worker-у відправляю повний index
@@ -256,7 +256,7 @@ func getIndexRecords(buf []byte, count int) (list []IndexRecord) {
rec.Timestamp, _ = bin.GetUint32(buf[i:])
rec.PageNo, _ = bin.GetUint32(buf[i+4:])
list = append(list, rec)
i += indexRecordSize
i += IndexRecordSize
}
return
}