dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-06-14 23:12:03 +03:00
parent 181031a753
commit b32279deaf
35 changed files with 1395 additions and 1062 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
storage/replay_metric.go
View File

@@ -43,14 +43,14 @@ func (s *ReplayMetric) MeasuresAppend(rec MeasuresAppendRecord) {
rec.Values, rec.ValuesRewindOffset)
}
type AppendMeasuresResult struct {
type ReplayMeasuresAppendWithGrowResult struct {
IndexPagesToRewrite []PageToWrite
DataPagesToRewrite []PageToWrite
ReusedIndexPagesCount int
ReusedDataPagesCount int
}
func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord, isLastPacket bool) (_ AppendMeasuresResult) {
func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord, isLastPacket bool) (_ ReplayMeasuresAppendWithGrowResult) {
var (
reusedIndexPages int
reusedDataPages int
@@ -82,6 +82,7 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord,
if head != nil {
// попередній хвіст індексного рівня перетворився на head сторінку,
// отже TailRecords - це новий хвіст
clear(level.Buffer)
copy(level.Buffer, change.TailRecords)
level.RecordsCount = newRecordsCount
} else {
@@ -138,6 +139,7 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord,
}
// HeadDataPage є полюбому, оскільки це транзакція із мінімум однією заповненою
// data сторінкою. Отже TailTimestamps і TailValues - це нові хвости data рівня.
clear(s.Buf)
copy(s.Buf, rec.TailValues)
s.ValuesSize = len(rec.TailValues)
@@ -151,7 +153,7 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord,
s.LastPageNo = rec.DataPages[len(rec.DataPages)-1].PageNo
}
return AppendMeasuresResult{
return ReplayMeasuresAppendWithGrowResult{
IndexPagesToRewrite: indexPages,
DataPagesToRewrite: dataPages,
ReusedIndexPagesCount: reusedIndexPages,
@@ -192,9 +194,9 @@ func (s *ReplayMetric) composeHeadDataPage(head DataPageTail) PageToWrite {
page := make([]byte, DataPageSize)
copy(page, s.Buf)
// append data
timestampsSize := writeTimestampsWithRewind(s.Buf, s.TimestampsSize,
timestampsSize := writeTimestampsWithRewind(page, s.TimestampsSize,
head.Timestamps, head.TimestampsRewindOffset)
valuesSize := writeValuesWithRewind(s.Buf, s.ValuesSize,
valuesSize := writeValuesWithRewind(page, s.ValuesSize,
head.Values, head.ValuesRewindOffset)
// запечатати сторінку
calculatedCRC := SealDataPage(SealDataPageIn{

25
storage/storage.go
View File

@@ -49,14 +49,19 @@ type MetricAdd struct {
MetricID uint32
MetricType qb.MetricType
FracDigits byte
ResultCh chan struct{}
ResultCh chan byte
}
type MetricDelete struct {
MetricID uint32
FreeIndexPages []uint32
FreeDataPages []uint32
ResultCh chan struct{}
ResultCh chan byte
}
type MeasuresAppendResult struct {
ResultCode byte
WrittenCount int
}
type MeasuresAppendWithGrow struct {
@@ -72,7 +77,7 @@ type MeasuresAppendWithGrow struct {
IndexLevelTails []IndexLevelTail
ResultCode byte
WrittenCount int
ResultCh chan AppendMeasuresResult
ResultCh chan MeasuresAppendResult
}
type MeasuresAppend struct {
@@ -83,14 +88,14 @@ type MeasuresAppend struct {
Values []byte
ResultCode byte
WrittenCount int
ResultCh chan AppendMeasuresResult
ResultCh chan MeasuresAppendResult
}
type MeasuresDelete struct {
MetricID uint32
FreeIndexPages []uint32
FreeDataPages []uint32
ResultCh chan struct{}
ResultCh chan byte
}
// WRITE RESULTS
@@ -100,7 +105,7 @@ type MeasuresAppendCommited struct {
MetricID uint32
ResultCode byte
WrittenCount int
ResultCh chan AppendMeasuresResult
ResultCh chan MeasuresAppendResult
}
type MeasuresAppendWithGrowCommited struct {
@@ -109,20 +114,20 @@ type MeasuresAppendWithGrowCommited struct {
Index []IndexLevelTail
ResultCode byte
WrittenCount int
ResultCh chan AppendMeasuresResult
ResultCh chan MeasuresAppendResult
}
type MeasuresDeleteCommited struct {
MetricID uint32
ResultCh chan struct{}
ResultCh chan byte
}
type MetricAddCommited struct {
MetricID uint32
ResultCh chan struct{}
ResultCh chan byte
}
type MetricDeleteCommited struct {
MetricID uint32
ResultCh chan struct{}
ResultCh chan byte
}

167
storage/storage_test.go
View File

@@ -3,6 +3,7 @@ package storage
import (
"bytes"
"fmt"
"io"
"math"
"reflect"
"slices"
@@ -914,3 +915,169 @@ func TestWriteValuesWithRewind(t *testing.T) {
t.Fatalf("got buf % x are not equal expected % x", before, after)
}
}
func TestWALReader(t *testing.T) {
rec1 := MetricAddRecord{
MetricID: 100,
MetricType: qb.Cumulative,
FracDigits: 4,
}
rec2 := MetricAddRecord{
MetricID: 200,
MetricType: qb.Instant,
FracDigits: 1,
}
rec3 := MetricDeleteRecord{
MetricID: 100,
FreeIndexPages: []uint32{
1, 2, 3,
},
FreeDataPages: []uint32{
10, 20,
},
}
rec4 := MetricDeleteRecord{
MetricID: 200,
}
packet1 := createPacket([]Packable{rec1, rec2})
packet2 := createPacket([]Packable{rec3, rec4})
src := bytes.NewBuffer(nil)
src.Write(packet1)
src.Write(packet2)
walReader, err := NewWALReader(src)
if err != nil {
t.Fatal(err)
}
i := 0
for {
records, isLastPacket, err := walReader.NextPacket()
if err != nil {
t.Fatal(err)
}
if records == nil {
break
}
switch i {
case 0:
if isLastPacket {
t.Fatalf("1st packet can't be the last packet in WAL")
}
if !reflect.DeepEqual(records, []any{rec1, rec2}) {
t.Fatalf("1st packet decoded incorrectly")
}
case 1:
if !isLastPacket {
t.Fatalf("2nd packet must be the last packet in WAL")
}
if !reflect.DeepEqual(records, []any{rec3, rec4}) {
t.Fatalf("2nd packet decoded incorrectly")
}
}
i++
}
}
func createPacket(records []Packable) []byte {
w := bytes.NewBuffer(nil)
w.Write([]byte{
0, 0, 0, 0, 0, 0, 0, 0, 0, // size (max 9 byte)
})
hasher := util.NewHasher()
multi := io.MultiWriter(w, hasher)
for _, rec := range records {
rec.Pack(multi)
}
return finalizePacket(w, hasher.Sum32())
}
func TestReplayMetric(t *testing.T) {
// data
DataPageSize = 24
DataPagePayloadSize = 12
dataCRC32Idx = DataPageSize - 4
timestampsSizeIdx = DataPageSize - 6
valuesSizeIdx = DataPageSize - 8
prevPageIdx = DataPageSize - 12
// index
IndexPageSize = 24
indexCRC32Idx = IndexPageSize - 4
indexRecordsCountIdx = IndexPageSize - 6
isZeroLevelIdx = IndexPageSize - 7
maxRecordsOnIndexPage = 2
//
metric := &ReplayMetric{
Buf: []byte{
0xa1, 0xa2, 0xa3, 0xa4,
0x00, 0x00, 0x00, 0x00,
0xb4, 0xb3, 0xb2, 0xb1,
0x01, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
},
TimestampsSize: 4,
ValuesSize: 4,
IndexLevelTails: []IndexLevelTail{
{
Buffer: []byte{
0x01, 0x01, 0x01, 0x01,
0x02, 0x02, 0x02, 0x02,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
},
RecordsCount: 1,
},
},
}
result := metric.MeasuresAppendWithGrow(MeasuresAppendWithGrowRecord{
DataPageTail: DataPageTail{
PrevPageNo: 1,
CRC32: 3235260875,
TimestampsRewindOffset: 1,
Timestamps: []byte{0x55, 0x44},
ValuesRewindOffset: 2,
Values: []byte{0x33, 0x44},
},
TailTimestamps: []byte{
0xf3, 0xf2, 0xf1,
},
TailValues: []byte{
0xd1, 0xd2,
},
ChangedIndexLevels: []ChangedIndexLevel{
{
IndexPageTail: &IndexPageTail{
CRC32: 2138044007,
Records: []byte{
0x03, 0x03, 0x03, 0x03,
0x04, 0x04, 0x04, 0x04,
},
},
TailRecords: []byte{
0x08, 0x08, 0x08, 0x08,
0x09, 0x09, 0x09, 0x09,
},
},
{
TailRecords: []byte{
0x01, 0x01, 0x01, 0x01,
0x02, 0x02, 0x02, 0x02,
},
},
},
}, true) // last packet
fmt.Printf("% x\n", metric.Buf)
fmt.Printf("% x\n", result.DataPagesToRewrite[0].Content)
fmt.Printf("% x\n", result.IndexPagesToRewrite[0].Content)
fmt.Printf("%d: % x\n", metric.IndexLevelTails[0].RecordsCount, metric.IndexLevelTails[0].Buffer)
fmt.Printf("%d: % x\n", metric.IndexLevelTails[1].RecordsCount, metric.IndexLevelTails[1].Buffer)
}

186
storage/wal_reader.go
View File

@@ -3,10 +3,8 @@ package storage
import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
"os"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb/util"
@@ -15,155 +13,133 @@ import (
const readBufferSize = 8 * 1024 * 1024
type WALReader struct {
file *os.File
reader *bufio.Reader
current []any
next []any
//file *os.File
reader *bufio.Reader
next []any
done bool
}
type WALReaderOptions struct {
FileName string
BufferSize int
}
// type WALReaderOptions struct {
// File *os.File
// BufferSize int
// }
func NewWALReader(opt WALReaderOptions) (*WALReader, error) {
if opt.FileName == "" {
return nil, errors.New("FileName option is required")
}
if opt.BufferSize <= 0 {
return nil, errors.New("BufferSize option is required")
}
file, err := os.Open(opt.FileName)
if err != nil {
return nil, err
}
func NewWALReader(src io.Reader) (*WALReader, error) {
// if opt.FileName == "" {
// return nil, errors.New("FileName option is required")
// }
// if opt.BufferSize <= 0 {
// return nil, errors.New("BufferSize option is required")
// }
return &WALReader{
file: file,
reader: bufio.NewReaderSize(file, readBufferSize),
//file: file,
reader: bufio.NewReaderSize(src, readBufferSize),
}, nil
}
func (s *WALReader) Close() {
s.file.Close()
}
// func (s *WALReader) Close() {
// s.file.Close()
// }
func (s *WALReader) NextPacket() ([]any, bool, error) {
var err error
if s.current == nil {
s.current, err = s.readPacket()
// (packet records, isLastPacket, error)
func (s *WALReader) NextPacket() (_ []any, _ bool, err error) {
if s.done {
return
}
var current []any
if s.next != nil {
current = s.next
} else {
current, err = s.readPacket()
if err != nil {
return nil, false, err
return
}
if len(s.current) > 0 {
s.next, err = s.readPacket()
if err != nil {
return nil, false, err
}
}
}
current := s.current
done := s.next == nil
s.current = s.next
s.next, err = s.readPacket()
if err != nil {
return nil, false, err
}
return current, done, nil
}
func (s *WALReader) readPacket() (_ []any, err error) {
payloadSize, err := bin.ReadVarSize(s.reader)
if err != nil {
if err == io.EOF {
return nil, nil
} else {
if current == nil {
return
}
}
storedCRC, err := bin.ReadUint32(s.reader)
s.next, err = s.readPacket()
if err != nil {
return
}
body, err := bin.ReadN(s.reader, int(payloadSize))
if err != nil {
return
}
hasher := util.NewHasher()
hasher.Write(body)
calculatedCRC := hasher.Sum32()
if calculatedCRC == storedCRC {
return s.parseRecords(body)
}
//s.logger.Printf("stored CRC %d != calculated CRC %d", storedCRC, calculatedCRC)
return nil, nil
return current, s.next == nil, nil
}
func (s *WALReader) parseRecords(body []byte) ([]any, error) {
func (s *WALReader) readPacket() (_ []any, err error) {
bodySize, err := bin.ReadVarSize(s.reader) // fix add n
if err != nil {
if err == io.EOF {
s.done = true
return nil, nil
}
return
}
body, err := bin.ReadN(s.reader, int(bodySize))
if err != nil {
return
}
writtenCRC, err := bin.ReadUint32(s.reader)
if err != nil {
return
}
calculatedCRC := util.CalculateCRC32(body)
if calculatedCRC != writtenCRC {
return nil, fmt.Errorf("written CRC %d != calculated CRC %d",
writtenCRC, calculatedCRC)
}
return s.parseRecords(body)
}
func (s *WALReader) parseRecords(body []byte) (_ []any, err error) {
var (
src = bytes.NewBuffer(body)
records []any
)
for {
recordType, err := src.ReadByte()
var recordType byte
recordType, err = src.ReadByte()
if err != nil {
if err == io.EOF {
return records, nil
}
return nil, err
return
}
switch recordType {
case CodeMetricAdd:
rec := new(MetricAddRecord)
if err = rec.Parse(src); err != nil {
return nil, err
return
}
records = append(records, rec)
records = append(records, *rec)
case CodeMetricDelete:
rec := new(MetricDeleteRecord)
if err = rec.Parse(src); err != nil {
return nil, err
return
}
records = append(records, rec)
records = append(records, *rec)
// case CodeAppendedMeasure:
// rec := new(AppendedMeasure)
// if err = rec.Parse(src); err != nil {
// return nil, err
// }
// records = append(records, rec)
case CodeMeasuresAppend:
rec := new(MeasuresAppendRecord)
if err = rec.Parse(src); err != nil {
return
}
records = append(records, *rec)
// case CodeAppendedMeasures:
// rec := new(AppendedMeasures)
// if err = rec.Parse(src); err != nil {
// return nil, err
// }
// records = append(records, rec)
// case CodeAppendedPages:
// rec := new(AppendedPages)
// if err = rec.Parse(src); err != nil {
// return nil, err
// }
// records = append(records, rec)
case CodeMeasuresAppendWithGrow:
rec := new(MeasuresAppendWithGrowRecord)
if err = rec.Parse(src); err != nil {
return
}
records = append(records, *rec)
case CodeMeasuresDelete:
rec := new(MeasuresDeleteRecord)
if err = rec.Parse(src); err != nil {
return nil, err
return
}
records = append(records, rec)
records = append(records, *rec)
default:
return nil, fmt.Errorf("unknown record type code: %d", recordType)

3
storage/wal_replayer.go
View File

@@ -58,6 +58,9 @@ func (s *WALReplayer) Replay() error {
if err != nil {
return err
}
if len(records) == 0 {
return nil
}
for _, rec := range records {
if err = s.replayRecord(rec, isLastPacket); err != nil {
return err

46
storage/write_preparer.go
View File

@@ -9,6 +9,10 @@ import (
"gordenko.dev/dima/qb/util"
)
type Packable interface {
Pack(io.Writer)
}
type WritePreparer struct {
pagePreparer *PageIngester
//dataPagePayloadBound int
@@ -39,15 +43,19 @@ type PreparedData struct {
Commits []any
}
// const (
// prefixSize = 9
// crc32Size = 4
// )
func (s *WritePreparer) Prepare(input []any) PreparedData {
s.w.Write([]byte{
0, 0, 0, 0, 0, 0, 0, 0, 0, // size (max 9 byte)
0, 0, 0, 0, // crc32
})
hasher := util.NewHasher()
w := io.MultiWriter(s.w, hasher)
multi := io.MultiWriter(s.w, hasher)
// 1. Пакую всі дані в WAL буфер запису
for _, untyped := range input {
@@ -58,7 +66,7 @@ func (s *WritePreparer) Prepare(input []any) PreparedData {
MetricType: x.MetricType,
FracDigits: x.FracDigits,
}
rec.Pack(w)
rec.Pack(multi)
s.commits = append(s.commits, MetricAddCommited{
MetricID: x.MetricID,
ResultCh: x.ResultCh,
@@ -76,7 +84,7 @@ func (s *WritePreparer) Prepare(input []any) PreparedData {
FreeIndexPages: x.FreeIndexPages,
FreeDataPages: x.FreeDataPages,
}
rec.Pack(w)
rec.Pack(multi)
s.commits = append(s.commits, MetricDeleteCommited{
MetricID: x.MetricID,
ResultCh: x.ResultCh,
@@ -90,7 +98,7 @@ func (s *WritePreparer) Prepare(input []any) PreparedData {
ValuesRewindOffset: x.ValuesRewindOffset,
Values: x.Values,
}
rec.Pack(w)
rec.Pack(multi)
// Дані для Worker
s.commits = append(s.commits, MeasuresAppendCommited{
MetricID: x.MetricID,
@@ -188,9 +196,9 @@ func (s *WritePreparer) Prepare(input []any) PreparedData {
ChangedIndexLevels: changedIndexPages,
}
fmt.Printf("%#v\ns", rec.ChangedIndexLevels[0].IndexPageTail)
//fmt.Printf("%#v\ns", rec.ChangedIndexLevels[0].IndexPageTail)
rec.Pack(w)
rec.Pack(multi)
// Дані для Worker
s.commits = append(s.commits, MeasuresAppendWithGrowCommited{
MetricID: x.MetricID,
@@ -213,7 +221,7 @@ func (s *WritePreparer) Prepare(input []any) PreparedData {
FreeIndexPages: x.FreeIndexPages,
FreeDataPages: x.FreeDataPages,
}
rec.Pack(w)
rec.Pack(multi)
s.commits = append(s.commits, MeasuresDeleteCommited{
MetricID: x.MetricID,
ResultCh: x.ResultCh,
@@ -222,19 +230,11 @@ func (s *WritePreparer) Prepare(input []any) PreparedData {
}
}
// 2. Додаю розмір пакету і чексуму
packet := finalizePacket(s.w, hasher.Sum32())
//s.written += int64(len(packet)) + 12
packet := s.w.Bytes()
// write size
payloadSize := len(packet) - 13
start := 9 - bin.CountVarSize(payloadSize)
bin.PutVarSize(packet[start:], payloadSize)
// CRC32
bin.PutUint32(packet[9:], hasher.Sum32())
//
return PreparedData{
Packet: packet[start:],
Packet: packet,
WriteToIndex: s.indexPages,
WriteToData: s.dataPages,
Commits: s.commits,
@@ -260,3 +260,13 @@ func getIndexRecords(buf []byte, count int) (list []IndexRecord) {
}
return
}
func finalizePacket(w *bytes.Buffer, checksum uint32) []byte {
bin.WriteUint32(w, checksum)
packet := w.Bytes()
bodySize := len(packet) - 13 // 9 bytes prefix + 4 bytes crc32
// cut empty bytes at start
start := 9 - bin.CountVarSize(bodySize)
bin.PutVarSize(packet[start:], bodySize)
return packet[start:]
}

38
storage/wal_writer.go → storage/writer.go
View File

@@ -4,7 +4,6 @@ import (
"bytes"
"errors"
"fmt"
"math"
"os"
"path/filepath"
"sync"
@@ -203,37 +202,8 @@ func (s *Writer) Run() {
}
}
func (s *Writer) getDataPageNumber() (uint32, bool, error) {
pageNo, err := s.dataFreeList.GetPageNumber()
if err != nil {
return 0, false, err
}
if pageNo > 0 {
return pageNo, true, nil
}
if s.dataPagesCount < math.MaxUint32 {
s.dataPagesCount++
return s.dataPagesCount, false, nil
}
return 0, false, errors.New("no space")
}
func (s *Writer) getIndexPageNumber() (uint32, bool, error) {
pageNo, err := s.indexFreeList.GetPageNumber()
if err != nil {
return 0, false, err
}
if pageNo > 0 {
return pageNo, true, nil
}
if s.indexPagesCount < math.MaxUint32 {
s.indexPagesCount++
return s.indexPagesCount, false, nil
}
return 0, false, errors.New("no space")
}
func (s *Writer) packAndWrite() (err error) {
//fmt.Println("packAndWrite")
//s.mutex.Lock()
//isExited := s.isExited
@@ -245,15 +215,19 @@ func (s *Writer) packAndWrite() (err error) {
input := s.inbox.Drain()
//fmt.Printf("Drain: %#v\n", input)
prepared := s.writePreparer.Prepare(input)
//fmt.Printf("prepared: %#v\n", prepared)
// 3. Пишу на диск WAL (append)
n, err := s.wal.Write(prepared.Packet)
if err != nil {
return
}
if n != len(prepared.Packet) {
return fmt.Errorf("written %d != total size %d", n, len(prepared.Packet))
return fmt.Errorf("written %d != packet size %d", n, len(prepared.Packet))
}
if err = s.wal.Sync(); err != nil {
return