wp
This commit is contained in:
@@ -1,228 +0,0 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
|
||||
bin "gordenko.dev/dima/bin/little"
|
||||
"gordenko.dev/dima/qb/util"
|
||||
)
|
||||
|
||||
type DataPreparer struct {
|
||||
pagePreparer *PagePreparer
|
||||
//dataPagePayloadBound int
|
||||
w *bytes.Buffer
|
||||
indexPages []PageToWrite
|
||||
dataPages []PageToWrite
|
||||
writeResults []any
|
||||
}
|
||||
|
||||
type DataPreparerOptions struct {
|
||||
MinWALBufferSize int
|
||||
PagePreparer *PagePreparer
|
||||
}
|
||||
|
||||
func NewDataPreparer(opt DataPreparerOptions) *DataPreparer {
|
||||
buf := make([]byte, opt.MinWALBufferSize)
|
||||
s := &DataPreparer{
|
||||
pagePreparer: opt.PagePreparer,
|
||||
w: bytes.NewBuffer(buf),
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
type PreparedData struct {
|
||||
Packet []byte // пакет для запису в WAL
|
||||
WriteToIndex []PageToWrite
|
||||
WriteToData []PageToWrite
|
||||
WriteResults []any
|
||||
}
|
||||
|
||||
func (s *DataPreparer) Prepare(input []any) PreparedData {
|
||||
s.w.Write([]byte{
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, // size (max 9 byte)
|
||||
0, 0, 0, 0, // crc32
|
||||
})
|
||||
|
||||
hasher := util.NewHasher()
|
||||
|
||||
w := io.MultiWriter(s.w, hasher)
|
||||
|
||||
// 1. Пакую всі дані в WAL буфер запису
|
||||
for _, untyped := range input {
|
||||
switch x := untyped.(type) {
|
||||
case MetricAddRecord:
|
||||
x.Pack(w)
|
||||
// case DeletedMetric:
|
||||
// if len(x.FreePageNumbers) > 0 {
|
||||
// s.freeList.AddPageNumbers(x.FreePageNumbers)
|
||||
// }
|
||||
case MeasuresAppendRecord:
|
||||
rec := MeasuresAppendRecord{
|
||||
MetricID: x.MetricID,
|
||||
TimestampsRewindOffset: x.TimestampsRewindOffset,
|
||||
Timestamps: x.Timestamps,
|
||||
ValuesRewindOffset: x.ValuesRewindOffset,
|
||||
Values: x.Values,
|
||||
}
|
||||
|
||||
rec.WriteTo(s.w)
|
||||
|
||||
// Дані для Worker
|
||||
s.writeResults = append(s.writeResults, AppendMeasuresSummary{
|
||||
// MetricID: x.MetricID,
|
||||
// LastPageNo: sealResult.DataPages[len(sealResult.DataPages)-1].PageNo,
|
||||
// Index: indexLevelTails,
|
||||
// ResultCode: x.ResultCode,
|
||||
// WrittenCount: x.WrittenCount,
|
||||
// ResultCh: x.ResultCh,
|
||||
})
|
||||
|
||||
case AppendedMeasuresWithGrow:
|
||||
sealResult := s.pagePreparer.SealPages(SealPagesIn{
|
||||
LastPageNo: x.LastPageNo,
|
||||
IndexLevelTails: x.IndexLevelTails,
|
||||
DataPages: x.DataPages,
|
||||
})
|
||||
|
||||
// сторінки для запису в index та data файли
|
||||
for _, p := range sealResult.DataPages {
|
||||
s.dataPages = append(s.dataPages, PageToWrite{
|
||||
PageNo: p.PageNo,
|
||||
Content: p.Content,
|
||||
})
|
||||
}
|
||||
for _, level := range sealResult.IndexLevels {
|
||||
for _, p := range level.IndexPages {
|
||||
s.indexPages = append(s.indexPages, PageToWrite{
|
||||
PageNo: p.PageNo,
|
||||
Content: p.Content,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
// дані для запису в WAL
|
||||
changedIndexPages []ChangedIndexLevel
|
||||
// дані для Воркера
|
||||
indexLevelTails []IndexLevelTail
|
||||
)
|
||||
|
||||
for _, level := range sealResult.IndexLevels {
|
||||
if len(level.IndexPages) == 0 && level.SkipRecords == level.TailRecordsCount {
|
||||
break
|
||||
}
|
||||
var completedIndexPage *HeadIndexPage
|
||||
if len(level.IndexPages) > 0 {
|
||||
first := level.IndexPages[0]
|
||||
skipSize := level.SkipRecords * indexRecordSize
|
||||
completedIndexPage = &HeadIndexPage{
|
||||
PageNo: first.PageNo,
|
||||
Reused: first.Reused,
|
||||
Checksum: first.Checksum,
|
||||
Records: first.Content[skipSize:],
|
||||
}
|
||||
}
|
||||
changedIndexPages = append(changedIndexPages, ChangedIndexLevel{
|
||||
HeadIndexPage: completedIndexPage,
|
||||
IndexPages: level.IndexPages[1:],
|
||||
TailRecords: level.TailRecords[:level.TailRecordsCount*indexRecordSize],
|
||||
})
|
||||
|
||||
indexLevelTails = append(indexLevelTails, IndexLevelTail{
|
||||
Buffer: level.TailRecords,
|
||||
RecordsCount: level.TailRecordsCount,
|
||||
})
|
||||
}
|
||||
|
||||
first := sealResult.DataPages[0]
|
||||
|
||||
rec := MeasuresAppendWithGrowRecord{
|
||||
MetricID: x.MetricID,
|
||||
HeadDataPage: HeadDataPage{
|
||||
PageNo: first.PageNo,
|
||||
Reused: first.Reused,
|
||||
PrevPageNo: first.PrevPageNo,
|
||||
Checksum: first.Checksum,
|
||||
TimestampsOffset: x.TimestampsOffset,
|
||||
Timestamps: x.Timestamps,
|
||||
ValuesOffset: x.ValuesOffset,
|
||||
Values: x.Values,
|
||||
},
|
||||
DataPages: sealResult.DataPages[1:],
|
||||
ChangedIndexLevels: changedIndexPages,
|
||||
TailTimestamps: x.Timestamps,
|
||||
TailValues: x.Values,
|
||||
}
|
||||
|
||||
rec.WriteTo(s.w)
|
||||
|
||||
// Дані для Worker
|
||||
s.writeResults = append(s.writeResults, AppendMeasuresSummary{
|
||||
MetricID: x.MetricID,
|
||||
LastPageNo: sealResult.DataPages[len(sealResult.DataPages)-1].PageNo,
|
||||
Index: indexLevelTails,
|
||||
ResultCode: x.ResultCode,
|
||||
WrittenCount: x.WrittenCount,
|
||||
ResultCh: x.ResultCh,
|
||||
})
|
||||
|
||||
//case DeletedMeasures:
|
||||
//case DeletedMeasuresSince:
|
||||
}
|
||||
}
|
||||
// 2. Додаю розмір пакету і чексуму
|
||||
|
||||
//s.written += int64(len(packet)) + 12
|
||||
|
||||
packet := s.w.Bytes()
|
||||
// write size
|
||||
payloadSize := len(packet) - 13
|
||||
start := 9 - bin.CountVarSize(payloadSize)
|
||||
bin.PutVarSize(packet[start:], payloadSize)
|
||||
// checksum
|
||||
bin.PutUint32(packet[9:], hasher.Sum32())
|
||||
//
|
||||
return PreparedData{
|
||||
Packet: packet[start:],
|
||||
WriteToIndex: s.indexPages,
|
||||
WriteToData: s.dataPages,
|
||||
WriteResults: s.writeResults,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DataPreparer) Reset() {
|
||||
s.w.Reset()
|
||||
s.indexPages = nil
|
||||
s.dataPages = nil
|
||||
s.writeResults = nil
|
||||
}
|
||||
|
||||
// MeasuresToWrite
|
||||
type AppendedMeasuresWithGrow struct {
|
||||
MetricID uint32
|
||||
LastPageNo uint32
|
||||
TimestampsOffset int // offset on 1st page
|
||||
Timestamps []byte
|
||||
ValuesOffset int // offset on 1st page
|
||||
Values []byte
|
||||
IndexLevelTails []IndexLevelTail
|
||||
DataPages []DataPayload
|
||||
TailTimestamps []byte // data level tail
|
||||
TailValues []byte // data level tail
|
||||
ResultCode byte
|
||||
WrittenCount int
|
||||
ResultCh chan struct{}
|
||||
}
|
||||
|
||||
// WRITE RESULTS
|
||||
|
||||
// результат
|
||||
type AppendMeasuresSummary struct {
|
||||
MetricID uint32
|
||||
LastPageNo uint32
|
||||
Index []IndexLevelTail
|
||||
ResultCode byte
|
||||
WrittenCount int
|
||||
ResultCh chan struct{}
|
||||
}
|
||||
@@ -14,56 +14,30 @@ import (
|
||||
// levels []*IndexLevel, dataPages []*DataPage
|
||||
// fix - reduce leves
|
||||
|
||||
type PagePreparer struct {
|
||||
type PageIngester struct {
|
||||
getDataPageNumber func() (uint32, bool, error)
|
||||
getIndexPageNumber func() (uint32, bool, error)
|
||||
}
|
||||
|
||||
type PagePreparerOptions struct {
|
||||
type PageIngesterOptions struct {
|
||||
GetDataPageNumber func() (uint32, bool, error)
|
||||
GetIndexPageNumber func() (uint32, bool, error)
|
||||
}
|
||||
|
||||
func NewPagePreparer(opt PagePreparerOptions) (*PagePreparer, error) {
|
||||
func NewPageIngester(opt PageIngesterOptions) (*PageIngester, error) {
|
||||
if opt.GetIndexPageNumber == nil {
|
||||
return nil, errors.New("missing required option: GetIndexPageNumber")
|
||||
}
|
||||
if opt.GetDataPageNumber == nil {
|
||||
return nil, errors.New("missing required option: GetDataPageNumber")
|
||||
}
|
||||
s := &PagePreparer{
|
||||
s := &PageIngester{
|
||||
getIndexPageNumber: opt.GetIndexPageNumber,
|
||||
getDataPageNumber: opt.GetDataPageNumber,
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
type appendIndexRecordIn struct {
|
||||
Records []byte
|
||||
RecordsCount int
|
||||
Timestamp uint32
|
||||
PageNo uint32
|
||||
}
|
||||
|
||||
func (s *PagePreparer) appendIndexRecord(in appendIndexRecordIn) []byte {
|
||||
if in.RecordsCount < maxRecordsOnIndexPage {
|
||||
var (
|
||||
pos int
|
||||
buf = in.Records
|
||||
)
|
||||
if in.RecordsCount > 0 {
|
||||
pos = in.RecordsCount * indexRecordSize
|
||||
} else {
|
||||
buf = make([]byte, IndexPageSize) // IndexPageIncSize
|
||||
}
|
||||
bin.PutUint32(buf[pos:], in.Timestamp)
|
||||
bin.PutUint32(buf[pos+4:], in.PageNo)
|
||||
return buf
|
||||
}
|
||||
qb.Abort(qb.NoSpaceOnIndexPage, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
type IndexLevelTail struct {
|
||||
Buffer []byte // розмір більший за кількість
|
||||
RecordsCount int
|
||||
@@ -76,25 +50,22 @@ type DataPayload struct {
|
||||
ValuesSize int
|
||||
}
|
||||
|
||||
type SealPagesIn struct {
|
||||
type IngestIn struct {
|
||||
LastPageNo uint32
|
||||
IndexLevelTails []IndexLevelTail
|
||||
DataPages []DataPayload
|
||||
}
|
||||
|
||||
type SealedDataPage struct {
|
||||
PageNo uint32
|
||||
Reused bool
|
||||
PrevPageNo uint32
|
||||
Checksum uint32
|
||||
Content []byte
|
||||
PageNo uint32
|
||||
Reused bool
|
||||
Content []byte
|
||||
}
|
||||
|
||||
type SealedIndexPage struct {
|
||||
Content []byte
|
||||
PageNo uint32
|
||||
Reused bool
|
||||
Checksum uint32
|
||||
PageNo uint32
|
||||
Reused bool
|
||||
Content []byte
|
||||
}
|
||||
|
||||
// SealedIndexLevel - як зрозуміти чи потрібно щось писати в WAL чи index файл,
|
||||
@@ -115,12 +86,7 @@ type SealedIndexLevel struct {
|
||||
TailRecordsCount int
|
||||
}
|
||||
|
||||
type SealResult struct {
|
||||
DataPages []SealedDataPage
|
||||
IndexLevels []*SealedIndexLevel
|
||||
}
|
||||
|
||||
func (s *PagePreparer) SealPages(in SealPagesIn) (result SealResult) {
|
||||
func (s *PageIngester) Ingest(in IngestIn) ([]SealedDataPage, []*SealedIndexLevel) {
|
||||
var (
|
||||
sealedDataPages []SealedDataPage
|
||||
sealedIndexLevels []*SealedIndexLevel
|
||||
@@ -142,31 +108,31 @@ func (s *PagePreparer) SealPages(in SealPagesIn) (result SealResult) {
|
||||
if dataPageIdx == 0 {
|
||||
prevPageNo = in.LastPageNo
|
||||
} else {
|
||||
prevPageNo = result.DataPages[dataPageIdx-1].PageNo
|
||||
prevPageNo = sealedDataPages[dataPageIdx-1].PageNo
|
||||
}
|
||||
pageNo, reused, err := s.getDataPageNumber()
|
||||
if err != nil {
|
||||
qb.Abort(qb.FailedGetPageNumber, err)
|
||||
}
|
||||
|
||||
SealDataPage(SealDataPageIn{
|
||||
Content: d.Content,
|
||||
PrevPageNo: prevPageNo,
|
||||
TimestampsSize: d.TimestampsSize,
|
||||
ValuesSize: d.ValuesSize,
|
||||
})
|
||||
|
||||
sealedDataPages = append(sealedDataPages, SealedDataPage{
|
||||
Content: d.Content,
|
||||
PrevPageNo: prevPageNo,
|
||||
PageNo: pageNo,
|
||||
Reused: reused,
|
||||
Checksum: SealDataPage(SealDataPageIn{
|
||||
Content: d.Content,
|
||||
PrevPageNo: prevPageNo,
|
||||
TimestampsSize: d.TimestampsSize,
|
||||
ValuesSize: d.ValuesSize,
|
||||
}),
|
||||
Content: d.Content,
|
||||
PageNo: pageNo,
|
||||
Reused: reused,
|
||||
})
|
||||
upPageNo = pageNo
|
||||
for {
|
||||
if levelIdx == len(sealedIndexLevels) {
|
||||
// новий root
|
||||
sealedIndexLevels = append(sealedIndexLevels, &SealedIndexLevel{
|
||||
TailRecords: s.appendIndexRecord(appendIndexRecordIn{
|
||||
Records: nil,
|
||||
TailRecords: appendIndexRecord(appendIndexRecordIn{
|
||||
Timestamp: upTimestamp,
|
||||
PageNo: upPageNo,
|
||||
}),
|
||||
@@ -176,10 +142,11 @@ func (s *PagePreparer) SealPages(in SealPagesIn) (result SealResult) {
|
||||
}
|
||||
level := sealedIndexLevels[levelIdx]
|
||||
if level.TailRecordsCount < maxRecordsOnIndexPage {
|
||||
level.TailRecords = s.appendIndexRecord(appendIndexRecordIn{
|
||||
Records: level.TailRecords,
|
||||
Timestamp: upTimestamp,
|
||||
PageNo: upPageNo,
|
||||
level.TailRecords = appendIndexRecord(appendIndexRecordIn{
|
||||
Records: level.TailRecords,
|
||||
RecordsCount: level.TailRecordsCount,
|
||||
Timestamp: upTimestamp,
|
||||
PageNo: upPageNo,
|
||||
})
|
||||
level.TailRecordsCount++
|
||||
break
|
||||
@@ -191,20 +158,20 @@ func (s *PagePreparer) SealPages(in SealPagesIn) (result SealResult) {
|
||||
qb.Abort(qb.FailedGetPageNumber, err)
|
||||
}
|
||||
|
||||
SealIndexPage(SealIndexPageIn{
|
||||
Content: level.TailRecords,
|
||||
RecordsCount: level.TailRecordsCount,
|
||||
ZeroLevel: levelIdx == 0,
|
||||
})
|
||||
|
||||
filled := SealedIndexPage{
|
||||
Content: level.TailRecords,
|
||||
PageNo: pageNo,
|
||||
Reused: reused,
|
||||
Checksum: SealIndexPage(SealIndexPageIn{
|
||||
Content: level.TailRecords,
|
||||
RecordsCount: level.TailRecordsCount,
|
||||
ZeroLevel: levelIdx == 0,
|
||||
}),
|
||||
}
|
||||
level.IndexPages = append(level.IndexPages, filled)
|
||||
// новий tail на індексному рівні
|
||||
level.TailRecords = s.appendIndexRecord(appendIndexRecordIn{
|
||||
Records: nil,
|
||||
level.TailRecords = appendIndexRecord(appendIndexRecordIn{
|
||||
Timestamp: upTimestamp,
|
||||
PageNo: upPageNo,
|
||||
})
|
||||
@@ -215,10 +182,7 @@ func (s *PagePreparer) SealPages(in SealPagesIn) (result SealResult) {
|
||||
levelIdx++
|
||||
}
|
||||
}
|
||||
return SealResult{
|
||||
DataPages: sealedDataPages,
|
||||
IndexLevels: sealedIndexLevels,
|
||||
}
|
||||
return sealedDataPages, sealedIndexLevels
|
||||
}
|
||||
|
||||
// HELPERS
|
||||
@@ -235,7 +199,7 @@ func SealDataPage(in SealDataPageIn) (checksum uint32) {
|
||||
bin.PutUint16(in.Content[valuesSizeIdx:], uint16(in.ValuesSize))
|
||||
bin.PutUint32(in.Content[prevPageIdx:], in.PrevPageNo)
|
||||
|
||||
checksum = util.CalcChecksum(in.Content[:dataCRC32Idx])
|
||||
checksum = util.CalculateCRC32(in.Content[:dataCRC32Idx])
|
||||
bin.PutUint32(in.Content[dataCRC32Idx:], checksum)
|
||||
return
|
||||
}
|
||||
@@ -251,7 +215,33 @@ func SealIndexPage(in SealIndexPageIn) (checksum uint32) {
|
||||
if in.ZeroLevel {
|
||||
in.Content[isZeroLevelIdx] = 1
|
||||
}
|
||||
checksum = util.CalcChecksum(in.Content[:indexCRC32Idx])
|
||||
checksum = util.CalculateCRC32(in.Content[:indexCRC32Idx])
|
||||
bin.PutUint32(in.Content[indexCRC32Idx:], checksum)
|
||||
return
|
||||
}
|
||||
|
||||
type appendIndexRecordIn struct {
|
||||
Records []byte
|
||||
RecordsCount int
|
||||
Timestamp uint32
|
||||
PageNo uint32
|
||||
}
|
||||
|
||||
func appendIndexRecord(in appendIndexRecordIn) []byte {
|
||||
if in.RecordsCount < maxRecordsOnIndexPage {
|
||||
var (
|
||||
pos int
|
||||
buf = in.Records
|
||||
)
|
||||
if in.RecordsCount > 0 {
|
||||
pos = in.RecordsCount * indexRecordSize
|
||||
} else {
|
||||
buf = make([]byte, IndexPageSize) // IndexPageIncSize
|
||||
}
|
||||
bin.PutUint32(buf[pos:], in.Timestamp)
|
||||
bin.PutUint32(buf[pos+4:], in.PageNo)
|
||||
return buf
|
||||
}
|
||||
qb.Abort(qb.NoSpaceOnIndexPage, nil)
|
||||
return nil
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
package storage
|
||||
|
||||
const (
|
||||
import "gordenko.dev/dima/qb"
|
||||
|
||||
var (
|
||||
|
||||
//PageNoSize = 4
|
||||
|
||||
@@ -15,18 +17,106 @@ const (
|
||||
prevPageIdx = DataPageSize - 12
|
||||
|
||||
// index page
|
||||
IndexPageSize = 1024
|
||||
IndexPageFooterSize = 7
|
||||
IndexPageSize = 1024
|
||||
IndexPageFooterSize = 7
|
||||
//indexPageIncSize = IndexPageIncSize
|
||||
indexCRC32Idx = IndexPageSize - 4
|
||||
indexRecordsCountIdx = IndexPageSize - 6
|
||||
isZeroLevelIdx = IndexPageSize - 7
|
||||
indexRecordSize = 8
|
||||
maxRecordsOnIndexPage = (IndexPageSize - IndexPageFooterSize) / indexRecordSize
|
||||
|
||||
//indexPageIncSize = IndexPageIncSize
|
||||
indexCRC32Idx = IndexPageSize - 4
|
||||
indexRecordsCountIdx = IndexPageSize - 6
|
||||
isZeroLevelIdx = IndexPageSize - 7
|
||||
|
||||
// timestampSize = 4
|
||||
// pairSize = timestampSize + PageNoSize
|
||||
// indexFooterIdx = indexRecordsQtyIdx
|
||||
// dataFooterIdx = timestampsSizeIdx
|
||||
)
|
||||
|
||||
type IndexRecord struct {
|
||||
Timestamp uint32
|
||||
PageNo uint32
|
||||
}
|
||||
|
||||
// TASKS FROM WORKER
|
||||
|
||||
type MetricAdd struct {
|
||||
MetricID uint32
|
||||
MetricType qb.MetricType
|
||||
FracDigits byte
|
||||
ResultCh chan struct{}
|
||||
}
|
||||
|
||||
type MetricDelete struct {
|
||||
MetricID uint32
|
||||
FreeIndexPages []uint32
|
||||
FreeDataPages []uint32
|
||||
ResultCh chan struct{}
|
||||
}
|
||||
|
||||
type MeasuresAppendWithGrow struct {
|
||||
MetricID uint32
|
||||
LastPageNo uint32
|
||||
TimestampsRewindOffset int // offset on 1st page
|
||||
Timestamps []byte
|
||||
ValuesRewindOffset int // offset on 1st page
|
||||
Values []byte
|
||||
DataPages []DataPayload
|
||||
TailTimestamps []byte // data level tail
|
||||
TailValues []byte // data level tail
|
||||
IndexLevelTails []IndexLevelTail
|
||||
ResultCode byte
|
||||
WrittenCount int
|
||||
ResultCh chan struct{}
|
||||
}
|
||||
|
||||
type MeasuresAppend struct {
|
||||
MetricID uint32
|
||||
TimestampsRewindOffset int // offset on 1st page
|
||||
Timestamps []byte
|
||||
ValuesRewindOffset int // offset on 1st page
|
||||
Values []byte
|
||||
ResultCode byte
|
||||
WrittenCount int
|
||||
ResultCh chan struct{}
|
||||
}
|
||||
|
||||
type MeasuresDelete struct {
|
||||
MetricID uint32
|
||||
FreeIndexPages []uint32
|
||||
FreeDataPages []uint32
|
||||
ResultCh chan struct{}
|
||||
}
|
||||
|
||||
// WRITE RESULTS
|
||||
|
||||
// результат
|
||||
type MeasuresAppendCommited struct {
|
||||
MetricID uint32
|
||||
ResultCode byte
|
||||
WrittenCount int
|
||||
ResultCh chan struct{}
|
||||
}
|
||||
|
||||
type MeasuresAppendWithGrowCommited struct {
|
||||
MetricID uint32
|
||||
LastPageNo uint32
|
||||
Index []IndexLevelTail
|
||||
ResultCode byte
|
||||
WrittenCount int
|
||||
ResultCh chan struct{}
|
||||
}
|
||||
|
||||
type MeasuresDeleteCommited struct {
|
||||
MetricID uint32
|
||||
ResultCh chan struct{}
|
||||
}
|
||||
|
||||
type MetricAddCommited struct {
|
||||
MetricID uint32
|
||||
ResultCh chan struct{}
|
||||
}
|
||||
|
||||
type MetricDeleteCommited struct {
|
||||
MetricID uint32
|
||||
ResultCh chan struct{}
|
||||
}
|
||||
|
||||
871
storage/storage_test.go
Normal file
871
storage/storage_test.go
Normal file
@@ -0,0 +1,871 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"math"
|
||||
"reflect"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
bin "gordenko.dev/dima/bin/little"
|
||||
"gordenko.dev/dima/qb"
|
||||
"gordenko.dev/dima/qb/util"
|
||||
)
|
||||
|
||||
func TestAppendIndexRecord(t *testing.T) {
|
||||
IndexPageSize = 24
|
||||
var (
|
||||
after = []byte{
|
||||
0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
|
||||
0x02, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
}
|
||||
)
|
||||
// func creates index page buffer
|
||||
buf := appendIndexRecord(appendIndexRecordIn{
|
||||
Timestamp: 1,
|
||||
PageNo: 8,
|
||||
})
|
||||
buf = appendIndexRecord(appendIndexRecordIn{
|
||||
Records: buf,
|
||||
RecordsCount: 1,
|
||||
Timestamp: 2,
|
||||
PageNo: 9,
|
||||
})
|
||||
|
||||
if !bytes.Equal(buf, after) {
|
||||
t.Fatalf("before not equal after:\nbefore: % x\n after: % x",
|
||||
buf, after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSealIndexPage(t *testing.T) {
|
||||
IndexPageSize = 16
|
||||
indexCRC32Idx = IndexPageSize - 4
|
||||
indexRecordsCountIdx = IndexPageSize - 6
|
||||
isZeroLevelIdx = IndexPageSize - 7
|
||||
var (
|
||||
before = []byte{
|
||||
0x07, 0x07, 0x07, 0x07, 0x08, 0x08, 0x08, 0x08, 0x00, // payload
|
||||
0x00, // zero level
|
||||
0x00, 0x00, // records count
|
||||
0x00, 0x00, 0x00, 0x00, // crc32
|
||||
}
|
||||
after = []byte{
|
||||
0x07, 0x07, 0x07, 0x07, 0x08, 0x08, 0x08, 0x08, 0x00, // payload
|
||||
0x01, // zero level
|
||||
0x02, 0x00, // records count
|
||||
0x8d, 0xcc, 0x3c, 0xc0, // crc32
|
||||
}
|
||||
)
|
||||
|
||||
calculatedCRC := util.CalculateCRC32(after[:indexCRC32Idx])
|
||||
|
||||
checksum := SealIndexPage(SealIndexPageIn{
|
||||
Content: before,
|
||||
RecordsCount: 2,
|
||||
ZeroLevel: true,
|
||||
})
|
||||
|
||||
if calculatedCRC != checksum {
|
||||
t.Fatalf("calculated CRC %d not equal returned %d",
|
||||
calculatedCRC, checksum)
|
||||
}
|
||||
|
||||
if !bytes.Equal(before, after) {
|
||||
t.Fatalf("before not equal after:\nbefore: % x\n after: % x",
|
||||
before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSealDataPage(t *testing.T) {
|
||||
DataPageSize = 16
|
||||
dataCRC32Idx = DataPageSize - 4
|
||||
timestampsSizeIdx = DataPageSize - 6
|
||||
valuesSizeIdx = DataPageSize - 8
|
||||
prevPageIdx = DataPageSize - 12
|
||||
var (
|
||||
before = []byte{
|
||||
0x07, 0x07, 0x07, 0x07, // payload
|
||||
0x00, 0x00, 0x00, 0x00, // prevPageNo
|
||||
0x00, 0x00, // values size
|
||||
0x00, 0x00, // timestamps size
|
||||
0x00, 0x00, 0x00, 0x00, // crc32
|
||||
}
|
||||
after = []byte{
|
||||
0x07, 0x07, 0x07, 0x07, // payload
|
||||
0x09, 0x00, 0x00, 0x00, // prevPageNo
|
||||
0x03, 0x00, // values size
|
||||
0x01, 0x00, // timestamps size
|
||||
0xa9, 0xec, 0xc8, 0x91, // crc32
|
||||
}
|
||||
)
|
||||
|
||||
calculatedCRC := util.CalculateCRC32(after[:dataCRC32Idx])
|
||||
|
||||
checksum := SealDataPage(SealDataPageIn{
|
||||
Content: before,
|
||||
PrevPageNo: 9,
|
||||
TimestampsSize: 1,
|
||||
ValuesSize: 3,
|
||||
})
|
||||
|
||||
if calculatedCRC != checksum {
|
||||
t.Fatalf("calculated CRC %d not equal returned %d",
|
||||
calculatedCRC, checksum)
|
||||
}
|
||||
|
||||
if !bytes.Equal(before, after) {
|
||||
t.Fatalf("before not equal after:\nbefore: % x\n after: % x",
|
||||
before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPageIngester(t *testing.T) {
|
||||
// index
|
||||
IndexPageSize = 24
|
||||
indexCRC32Idx = IndexPageSize - 4
|
||||
indexRecordsCountIdx = IndexPageSize - 6
|
||||
isZeroLevelIdx = IndexPageSize - 7
|
||||
maxRecordsOnIndexPage = 2
|
||||
// data
|
||||
DataPageSize = 16
|
||||
DataPagePayloadSize = DataPageSize - DataPageFooterSize
|
||||
dataCRC32Idx = DataPageSize - 4
|
||||
timestampsSizeIdx = DataPageSize - 6
|
||||
valuesSizeIdx = DataPageSize - 8
|
||||
prevPageIdx = DataPageSize - 12
|
||||
|
||||
var testCases = []struct {
|
||||
Name string
|
||||
DataPages []DataPayload
|
||||
ExpectedIndexLevels []ExpectedIndexLevel
|
||||
ExpectedDataPages []ExpectedDataPage
|
||||
}{
|
||||
{
|
||||
Name: "create root index tail",
|
||||
DataPages: []DataPayload{
|
||||
{
|
||||
Since: 100,
|
||||
Content: makeDataPage([]byte{0x01, 0x00, 0x00, 0x00}),
|
||||
},
|
||||
},
|
||||
ExpectedDataPages: []ExpectedDataPage{
|
||||
{
|
||||
PageNo: 1,
|
||||
PrevPageNo: 0,
|
||||
Payload: []byte{0x01, 0x00, 0x00, 0x00},
|
||||
},
|
||||
},
|
||||
ExpectedIndexLevels: []ExpectedIndexLevel{
|
||||
{
|
||||
TailRecords: []IndexRecord{
|
||||
{Timestamp: 100, PageNo: 1},
|
||||
},
|
||||
TailRecordsCount: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "just append record to index tail",
|
||||
DataPages: []DataPayload{
|
||||
{
|
||||
Since: 100,
|
||||
Content: makeDataPage([]byte{0x01, 0x00, 0x00, 0x00}),
|
||||
},
|
||||
{
|
||||
Since: 200,
|
||||
Content: makeDataPage([]byte{0x02, 0x00, 0x00, 0x00}),
|
||||
},
|
||||
},
|
||||
ExpectedDataPages: []ExpectedDataPage{
|
||||
{
|
||||
PageNo: 1,
|
||||
PrevPageNo: 0,
|
||||
Payload: []byte{0x01, 0x00, 0x00, 0x00},
|
||||
},
|
||||
{
|
||||
PageNo: 2,
|
||||
PrevPageNo: 1,
|
||||
Payload: []byte{0x02, 0x00, 0x00, 0x00},
|
||||
},
|
||||
},
|
||||
ExpectedIndexLevels: []ExpectedIndexLevel{
|
||||
{
|
||||
TailRecords: []IndexRecord{
|
||||
{Timestamp: 100, PageNo: 1},
|
||||
{Timestamp: 200, PageNo: 2},
|
||||
},
|
||||
TailRecordsCount: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "create index page and upper level (root)",
|
||||
DataPages: []DataPayload{
|
||||
{
|
||||
Since: 100,
|
||||
Content: makeDataPage([]byte{0x01, 0x00, 0x00, 0x00}),
|
||||
},
|
||||
{
|
||||
Since: 200,
|
||||
Content: makeDataPage([]byte{0x02, 0x00, 0x00, 0x00}),
|
||||
},
|
||||
{
|
||||
Since: 300,
|
||||
Content: makeDataPage([]byte{0x03, 0x00, 0x00, 0x00}),
|
||||
},
|
||||
},
|
||||
ExpectedDataPages: []ExpectedDataPage{
|
||||
{
|
||||
PageNo: 1,
|
||||
PrevPageNo: 0,
|
||||
Payload: []byte{0x01, 0x00, 0x00, 0x00},
|
||||
},
|
||||
{
|
||||
PageNo: 2,
|
||||
PrevPageNo: 1,
|
||||
Payload: []byte{0x02, 0x00, 0x00, 0x00},
|
||||
},
|
||||
{
|
||||
PageNo: 3,
|
||||
PrevPageNo: 2,
|
||||
Payload: []byte{0x03, 0x00, 0x00, 0x00},
|
||||
},
|
||||
},
|
||||
ExpectedIndexLevels: []ExpectedIndexLevel{
|
||||
{
|
||||
IndexPages: []ExpectedIndexPage{
|
||||
{
|
||||
PageNo: 1,
|
||||
ZeroLevel: true,
|
||||
Records: []IndexRecord{
|
||||
{Timestamp: 100, PageNo: 1},
|
||||
{Timestamp: 200, PageNo: 2},
|
||||
},
|
||||
},
|
||||
},
|
||||
TailRecords: []IndexRecord{
|
||||
{Timestamp: 300, PageNo: 3},
|
||||
},
|
||||
TailRecordsCount: 1,
|
||||
},
|
||||
{
|
||||
TailRecords: []IndexRecord{
|
||||
{Timestamp: 100, PageNo: 1},
|
||||
},
|
||||
TailRecordsCount: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "just append record to tail on zero level",
|
||||
DataPages: []DataPayload{
|
||||
{
|
||||
Since: 100,
|
||||
Content: makeDataPage([]byte{0x01, 0x00, 0x00, 0x00}),
|
||||
},
|
||||
{
|
||||
Since: 200,
|
||||
Content: makeDataPage([]byte{0x02, 0x00, 0x00, 0x00}),
|
||||
},
|
||||
{
|
||||
Since: 300,
|
||||
Content: makeDataPage([]byte{0x03, 0x00, 0x00, 0x00}),
|
||||
},
|
||||
{
|
||||
Since: 400,
|
||||
Content: makeDataPage([]byte{0x04, 0x00, 0x00, 0x00}),
|
||||
},
|
||||
},
|
||||
ExpectedDataPages: []ExpectedDataPage{
|
||||
{
|
||||
PageNo: 1,
|
||||
PrevPageNo: 0,
|
||||
Payload: []byte{0x01, 0x00, 0x00, 0x00},
|
||||
},
|
||||
{
|
||||
PageNo: 2,
|
||||
PrevPageNo: 1,
|
||||
Payload: []byte{0x02, 0x00, 0x00, 0x00},
|
||||
},
|
||||
{
|
||||
PageNo: 3,
|
||||
PrevPageNo: 2,
|
||||
Payload: []byte{0x03, 0x00, 0x00, 0x00},
|
||||
},
|
||||
{
|
||||
PageNo: 4,
|
||||
PrevPageNo: 3,
|
||||
Payload: []byte{0x04, 0x00, 0x00, 0x00},
|
||||
},
|
||||
},
|
||||
ExpectedIndexLevels: []ExpectedIndexLevel{
|
||||
{
|
||||
IndexPages: []ExpectedIndexPage{
|
||||
{
|
||||
PageNo: 1,
|
||||
ZeroLevel: true,
|
||||
Records: []IndexRecord{
|
||||
{Timestamp: 100, PageNo: 1},
|
||||
{Timestamp: 200, PageNo: 2},
|
||||
},
|
||||
},
|
||||
},
|
||||
TailRecords: []IndexRecord{
|
||||
{Timestamp: 300, PageNo: 3},
|
||||
{Timestamp: 400, PageNo: 4},
|
||||
},
|
||||
TailRecordsCount: 2,
|
||||
},
|
||||
{
|
||||
TailRecords: []IndexRecord{
|
||||
{Timestamp: 100, PageNo: 1},
|
||||
},
|
||||
TailRecordsCount: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
pageManager := new(PageNumbersMock)
|
||||
|
||||
ingester, err := NewPageIngester(PageIngesterOptions{
|
||||
GetIndexPageNumber: pageManager.GetIndexPageNumber,
|
||||
GetDataPageNumber: pageManager.GetDataPageNumber,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sealedDataPages, sealedIndexLevels := ingester.Ingest(IngestIn{
|
||||
LastPageNo: 0,
|
||||
IndexLevelTails: nil,
|
||||
DataPages: testCase.DataPages,
|
||||
})
|
||||
|
||||
if len(testCase.ExpectedDataPages) != len(sealedDataPages) {
|
||||
t.Fatalf("%s: got %d data pages, but not %d",
|
||||
testCase.Name, len(sealedDataPages), len(testCase.ExpectedDataPages))
|
||||
}
|
||||
|
||||
for idx, page := range testCase.ExpectedDataPages {
|
||||
err := cmpDataPage(page, sealedDataPages[idx])
|
||||
if err != nil {
|
||||
t.Fatalf("%s: data page #%d: %s", testCase.Name, idx, err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(testCase.ExpectedIndexLevels) != len(sealedIndexLevels) {
|
||||
t.Fatalf("%s: got %d index levels, but not %d",
|
||||
testCase.Name, len(sealedIndexLevels), len(testCase.ExpectedIndexLevels))
|
||||
}
|
||||
|
||||
for idx, level := range testCase.ExpectedIndexLevels {
|
||||
err := cmpIndexLevel(level, *sealedIndexLevels[idx])
|
||||
if err != nil {
|
||||
t.Fatalf("%s: index level #%d: %s", testCase.Name, idx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type PageNumbersMock struct {
|
||||
indexPageNo uint32
|
||||
dataPageNo uint32
|
||||
}
|
||||
|
||||
func (s *PageNumbersMock) GetIndexPageNumber() (uint32, bool, error) {
|
||||
s.indexPageNo++
|
||||
return s.indexPageNo, false, nil
|
||||
}
|
||||
|
||||
func (s *PageNumbersMock) GetDataPageNumber() (uint32, bool, error) {
|
||||
s.dataPageNo++
|
||||
return s.dataPageNo, false, nil
|
||||
}
|
||||
|
||||
// func getIndexRecords(buf []byte, count int) (list []IndexRecord) {
|
||||
// i := 0
|
||||
// for range count {
|
||||
// var rec IndexRecord
|
||||
// rec.Timestamp, _ = bin.GetUint32(buf[i:])
|
||||
// rec.PageNo, _ = bin.GetUint32(buf[i+4:])
|
||||
// list = append(list, rec)
|
||||
// i += indexRecordSize
|
||||
// }
|
||||
// return
|
||||
// }
|
||||
|
||||
type ExpectedIndexPage struct {
|
||||
PageNo uint32
|
||||
Records []IndexRecord
|
||||
ZeroLevel bool
|
||||
}
|
||||
|
||||
func cmpIndexPage(expected ExpectedIndexPage, page SealedIndexPage) error {
|
||||
if expected.PageNo != page.PageNo {
|
||||
return fmt.Errorf("got pageNo %d not equal expected %d",
|
||||
page.PageNo, expected.PageNo)
|
||||
}
|
||||
buf := page.Content
|
||||
calculatedCRC := util.CalculateCRC32(buf[:indexCRC32Idx])
|
||||
checksum, _ := bin.GetUint32(buf[indexCRC32Idx:])
|
||||
|
||||
if calculatedCRC != checksum {
|
||||
return fmt.Errorf("calculated CRC %d not equal written %d",
|
||||
calculatedCRC, checksum)
|
||||
}
|
||||
|
||||
zeroLevel, _ := bin.GetBool(buf[isZeroLevelIdx:])
|
||||
if expected.ZeroLevel != zeroLevel {
|
||||
return fmt.Errorf("expected zero level %t not equal written %t",
|
||||
expected.ZeroLevel, zeroLevel)
|
||||
}
|
||||
|
||||
count, _ := bin.GetUint16(buf[indexRecordsCountIdx:])
|
||||
|
||||
records := getIndexRecords(buf, int(count))
|
||||
if !slices.Equal(expected.Records, records) {
|
||||
return fmt.Errorf("expected records %v not equal written %v",
|
||||
expected.Records, records)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type ExpectedIndexLevel struct {
|
||||
SkipRecords int
|
||||
IndexPages []ExpectedIndexPage
|
||||
TailRecords []IndexRecord
|
||||
TailRecordsCount int
|
||||
}
|
||||
|
||||
func cmpIndexLevel(expected ExpectedIndexLevel, level SealedIndexLevel) error {
|
||||
if expected.SkipRecords != level.SkipRecords {
|
||||
return fmt.Errorf("got skipRecords %d not equal expected %d",
|
||||
level.SkipRecords, expected.SkipRecords)
|
||||
}
|
||||
|
||||
if expected.TailRecordsCount != level.TailRecordsCount {
|
||||
return fmt.Errorf("got tailRecordsCount %d not equal expected %d",
|
||||
level.TailRecordsCount, expected.TailRecordsCount)
|
||||
}
|
||||
|
||||
records := getIndexRecords(level.TailRecords, level.TailRecordsCount)
|
||||
if !slices.Equal(expected.TailRecords, records) {
|
||||
return fmt.Errorf("expected tailRecords %v not equal written %v",
|
||||
expected.TailRecords, records)
|
||||
}
|
||||
|
||||
if len(expected.IndexPages) != len(level.IndexPages) {
|
||||
return fmt.Errorf("got %d index pages, but not %d",
|
||||
len(level.IndexPages), len(expected.IndexPages))
|
||||
}
|
||||
|
||||
for idx, page := range expected.IndexPages {
|
||||
err := cmpIndexPage(page, level.IndexPages[idx])
|
||||
if err != nil {
|
||||
return fmt.Errorf("index page #%d: %s", idx, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type ExpectedDataPage struct {
|
||||
PageNo uint32
|
||||
Payload []byte
|
||||
PrevPageNo uint32
|
||||
}
|
||||
|
||||
func cmpDataPage(expected ExpectedDataPage, page SealedDataPage) error {
|
||||
buf := page.Content
|
||||
|
||||
if expected.PageNo != page.PageNo {
|
||||
return fmt.Errorf("got pageNo %d not equal expected %d",
|
||||
page.PageNo, expected.PageNo)
|
||||
}
|
||||
prevPageNo, _ := bin.GetUint32(buf[prevPageIdx:])
|
||||
if expected.PrevPageNo != prevPageNo {
|
||||
return fmt.Errorf("written prevPageNo %d not equal expected %d",
|
||||
prevPageNo, expected.PrevPageNo)
|
||||
}
|
||||
|
||||
calculatedCRC := util.CalculateCRC32(buf[:dataCRC32Idx])
|
||||
checksum, _ := bin.GetUint32(buf[dataCRC32Idx:])
|
||||
|
||||
if calculatedCRC != checksum {
|
||||
return fmt.Errorf("calculated CRC %d not equal written %d",
|
||||
calculatedCRC, checksum)
|
||||
}
|
||||
|
||||
writtenPayload := page.Content[:DataPagePayloadSize]
|
||||
if !slices.Equal(expected.Payload, writtenPayload) {
|
||||
return fmt.Errorf("expected payload % x not equal written % x",
|
||||
expected.Payload, writtenPayload)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func makeDataPage(payload []byte) []byte {
|
||||
buf := make([]byte, DataPageSize)
|
||||
copy(buf, payload)
|
||||
return buf
|
||||
}
|
||||
|
||||
func TestMetricAddRecord(t *testing.T) {
|
||||
rec := MetricAddRecord{
|
||||
MetricID: 12345,
|
||||
MetricType: qb.Cumulative,
|
||||
FracDigits: 5,
|
||||
}
|
||||
buf := bytes.NewBuffer(nil)
|
||||
rec.Pack(buf)
|
||||
|
||||
recordType, _ := buf.ReadByte()
|
||||
|
||||
if recordType != CodeMetricAdd {
|
||||
t.Fatalf("wrong record type: %d", recordType)
|
||||
}
|
||||
|
||||
decoded := new(MetricAddRecord)
|
||||
err := decoded.Parse(buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(rec, *decoded) {
|
||||
t.Fatalf("decoded are not equal origin: %v", decoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricDeleteRecord(t *testing.T) {
|
||||
rec := MetricDeleteRecord{
|
||||
MetricID: 12345,
|
||||
FreeIndexPages: []uint32{1, math.MaxUint32},
|
||||
FreeDataPages: []uint32{1, math.MaxUint32},
|
||||
}
|
||||
buf := bytes.NewBuffer(nil)
|
||||
rec.Pack(buf)
|
||||
|
||||
recordType, _ := buf.ReadByte()
|
||||
|
||||
if recordType != CodeMetricDelete {
|
||||
t.Fatalf("wrong record type: %d", recordType)
|
||||
}
|
||||
|
||||
decoded := new(MetricDeleteRecord)
|
||||
err := decoded.Parse(buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(rec, *decoded) {
|
||||
t.Fatalf("decoded are not equal origin: %v", decoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeasuresDeleteRecord(t *testing.T) {
|
||||
rec := MeasuresDeleteRecord{
|
||||
MetricID: 12345,
|
||||
FreeIndexPages: []uint32{1, math.MaxUint32},
|
||||
FreeDataPages: []uint32{1, math.MaxUint32},
|
||||
}
|
||||
buf := bytes.NewBuffer(nil)
|
||||
rec.Pack(buf)
|
||||
|
||||
recordType, _ := buf.ReadByte()
|
||||
|
||||
if recordType != CodeMeasuresDelete {
|
||||
t.Fatalf("wrong record type: %d", recordType)
|
||||
}
|
||||
|
||||
decoded := new(MeasuresDeleteRecord)
|
||||
err := decoded.Parse(buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(rec, *decoded) {
|
||||
t.Fatalf("decoded are not equal origin: %v", decoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeasuresAppendRecord(t *testing.T) {
|
||||
rec := MeasuresAppendRecord{
|
||||
MetricID: 12345,
|
||||
TimestampsRewindOffset: 7,
|
||||
Timestamps: []byte{
|
||||
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
|
||||
},
|
||||
ValuesRewindOffset: 11,
|
||||
Values: []byte{
|
||||
0x08, 0x09, 0x0a,
|
||||
},
|
||||
}
|
||||
buf := bytes.NewBuffer(nil)
|
||||
rec.Pack(buf)
|
||||
|
||||
recordType, _ := buf.ReadByte()
|
||||
|
||||
if recordType != CodeMeasuresAppend {
|
||||
t.Fatalf("wrong record type: %d", recordType)
|
||||
}
|
||||
|
||||
decoded := new(MeasuresAppendRecord)
|
||||
err := decoded.Parse(buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(rec, *decoded) {
|
||||
t.Fatalf("decoded are not equal origin: %v", decoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeasuresAppendWithGrowRecord(t *testing.T) {
|
||||
DataPageSize = 8
|
||||
IndexPageSize = 4
|
||||
indexRecordSize = 2
|
||||
//
|
||||
rec := MeasuresAppendWithGrowRecord{
|
||||
MetricID: 12345,
|
||||
DataPageTail: DataPageTail{
|
||||
PageNo: math.MaxUint32,
|
||||
Reused: true,
|
||||
PrevPageNo: 4127831,
|
||||
CRC32: 316278321,
|
||||
TimestampsRewindOffset: 7,
|
||||
Timestamps: []byte{
|
||||
0x07, 0x07, 0x07, 0x07, 0x07,
|
||||
},
|
||||
Values: []byte{
|
||||
0x08, 0x08, 0x08, 0x08,
|
||||
},
|
||||
ValuesRewindOffset: 11,
|
||||
},
|
||||
DataPages: []SealedDataPage{
|
||||
{
|
||||
PageNo: 1000,
|
||||
Reused: true,
|
||||
//PrevPageNo: math.MaxUint32,
|
||||
//CRC32: 312,
|
||||
Content: []byte{
|
||||
0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05,
|
||||
},
|
||||
},
|
||||
{
|
||||
PageNo: 1001,
|
||||
Reused: false,
|
||||
//PrevPageNo: 1000,
|
||||
//CRC32: 432432,
|
||||
Content: []byte{
|
||||
0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
|
||||
},
|
||||
},
|
||||
},
|
||||
TailTimestamps: []byte{
|
||||
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
|
||||
},
|
||||
TailValues: []byte{
|
||||
0x08, 0x09, 0x0a,
|
||||
},
|
||||
ChangedIndexLevels: []ChangedIndexLevel{
|
||||
{
|
||||
IndexPageTail: &IndexPageTail{
|
||||
PageNo: 20000,
|
||||
Reused: true,
|
||||
CRC32: 32897,
|
||||
Records: []byte{
|
||||
0x01, 0x00, 0x00, 0x00,
|
||||
},
|
||||
},
|
||||
IndexPages: []SealedIndexPage{
|
||||
{
|
||||
PageNo: 20001,
|
||||
Reused: true,
|
||||
//CRC32: 32897,
|
||||
Content: []byte{
|
||||
0x01, 0x00, 0x00, 0x01,
|
||||
},
|
||||
},
|
||||
{
|
||||
PageNo: 20002,
|
||||
Reused: false,
|
||||
//CRC32: 3284397,
|
||||
Content: []byte{
|
||||
0x01, 0x02, 0x03, 0x04,
|
||||
},
|
||||
},
|
||||
},
|
||||
TailRecords: []byte{
|
||||
0xaa, 0xbb,
|
||||
},
|
||||
},
|
||||
{
|
||||
IndexPageTail: nil,
|
||||
TailRecords: []byte{
|
||||
0xdd, 0xee, 0xff, 0x77,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
buf := bytes.NewBuffer(nil)
|
||||
rec.Pack(buf)
|
||||
|
||||
recordType, _ := buf.ReadByte()
|
||||
|
||||
if recordType != CodeMeasuresAppendWithGrow {
|
||||
t.Fatalf("wrong record type: %d", recordType)
|
||||
}
|
||||
|
||||
decoded := new(MeasuresAppendWithGrowRecord)
|
||||
err := decoded.Parse(buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(rec, *decoded) {
|
||||
t.Fatalf("decoded are not equal origin: %v", decoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWritePreparer(t *testing.T) {
|
||||
// data page
|
||||
DataPageSize = 24
|
||||
dataCRC32Idx = DataPageSize - 4
|
||||
timestampsSizeIdx = DataPageSize - 6
|
||||
valuesSizeIdx = DataPageSize - 8
|
||||
prevPageIdx = DataPageSize - 12
|
||||
|
||||
// index page
|
||||
IndexPageSize = 16
|
||||
indexRecordSize = 4 // 2 records per index page
|
||||
maxRecordsOnIndexPage = 2
|
||||
indexCRC32Idx = IndexPageSize - 4
|
||||
indexRecordsCountIdx = IndexPageSize - 6
|
||||
isZeroLevelIdx = IndexPageSize - 7
|
||||
|
||||
pageManager := new(PageNumbersMock)
|
||||
|
||||
pageIngester, err := NewPageIngester(PageIngesterOptions{
|
||||
GetIndexPageNumber: pageManager.GetIndexPageNumber,
|
||||
GetDataPageNumber: pageManager.GetDataPageNumber,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
writePreparer := NewWritePreparer(WritePreparerOptions{
|
||||
MinWALBufferSize: 128,
|
||||
PageIngester: pageIngester,
|
||||
})
|
||||
|
||||
rec := MeasuresAppendWithGrow{
|
||||
MetricID: 12345,
|
||||
LastPageNo: 1,
|
||||
TimestampsRewindOffset: 2,
|
||||
Timestamps: []byte{
|
||||
0x04, 0x05, 0x06,
|
||||
},
|
||||
ValuesRewindOffset: 1,
|
||||
Values: []byte{
|
||||
0x04, 0x03,
|
||||
},
|
||||
DataPages: []DataPayload{
|
||||
// 1. fills existent index tail
|
||||
{
|
||||
Since: 513, // 0x01 0x02
|
||||
// full page
|
||||
Content: []byte{
|
||||
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x00, 0x00, 0x04, 0x03, 0x02, 0x01,
|
||||
0x00, 0x00, 0x00, 0x00, // prevPageNo
|
||||
0x00, 0x00, // values size
|
||||
0x00, 0x00, // timestamps size
|
||||
0x00, 0x00, 0x00, 0x00, // crc32
|
||||
},
|
||||
TimestampsSize: 6,
|
||||
ValuesSize: 4,
|
||||
},
|
||||
// 2. creates index page tail (zero level) and level 1
|
||||
{
|
||||
Since: 769, // 0x01 0x03
|
||||
// full page
|
||||
Content: []byte{
|
||||
0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0x00, 0x00, 0xa5, 0xa4, 0xa3, 0xa2, 0xa1,
|
||||
0x00, 0x00, 0x00, 0x00, // prevPageNo
|
||||
0x00, 0x00, // values size
|
||||
0x00, 0x00, // timestamps size
|
||||
0x00, 0x00, 0x00, 0x00, // crc32
|
||||
},
|
||||
TimestampsSize: 5,
|
||||
ValuesSize: 5,
|
||||
},
|
||||
// 2. fills index tail records (zero level)
|
||||
{
|
||||
Since: 1025, // 0x01 0x04
|
||||
// full page
|
||||
Content: []byte{
|
||||
0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb5, 0xb4, 0xb3, 0xb2, 0xb1,
|
||||
0x00, 0x00, 0x00, 0x00, // prevPageNo
|
||||
0x00, 0x00, // values size
|
||||
0x00, 0x00, // timestamps size
|
||||
0x00, 0x00, 0x00, 0x00, // crc32
|
||||
},
|
||||
TimestampsSize: 7,
|
||||
ValuesSize: 5,
|
||||
},
|
||||
// 3. creates index page
|
||||
{
|
||||
Since: 1025, // 0x01 0x04
|
||||
// full page
|
||||
Content: []byte{
|
||||
0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc6, 0xc5, 0xc4, 0xc3, 0xc2, 0xc1,
|
||||
0x00, 0x00, 0x00, 0x00, // prevPageNo
|
||||
0x00, 0x00, // values size
|
||||
0x00, 0x00, // timestamps size
|
||||
0x00, 0x00, 0x00, 0x00, // crc32
|
||||
},
|
||||
TimestampsSize: 6,
|
||||
ValuesSize: 6,
|
||||
},
|
||||
},
|
||||
TailTimestamps: []byte{
|
||||
0xc1, 0xc2,
|
||||
},
|
||||
TailValues: []byte{
|
||||
0xd1, 0xd2,
|
||||
},
|
||||
IndexLevelTails: []IndexLevelTail{
|
||||
{
|
||||
Buffer: []byte{
|
||||
0x01, 0x01, // since
|
||||
0x0a, 0x00, // pageNo 10
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, // empty space
|
||||
0x00, // zero level
|
||||
0x00, 0x00, // records count
|
||||
0x00, 0x00, 0x00, 0x00, // crc32
|
||||
|
||||
},
|
||||
RecordsCount: 1,
|
||||
},
|
||||
},
|
||||
WrittenCount: 10,
|
||||
}
|
||||
|
||||
prepared := writePreparer.Prepare([]any{rec})
|
||||
// fmt.Printf("%v\n", prepared)
|
||||
// pretty.PPrintln("prepared", prepared)
|
||||
|
||||
fmt.Printf("wal: % x\n", prepared.Packet)
|
||||
|
||||
fmt.Println("index to write")
|
||||
for _, p := range prepared.WriteToIndex {
|
||||
fmt.Printf("%d: % x\n", p.PageNo, p.Content)
|
||||
}
|
||||
fmt.Println("data to write")
|
||||
for _, p := range prepared.WriteToData {
|
||||
fmt.Printf("%d: % x\n", p.PageNo, p.Content)
|
||||
}
|
||||
|
||||
fmt.Println("commits")
|
||||
for _, x := range prepared.Commits {
|
||||
fmt.Printf("%#v\n", x)
|
||||
}
|
||||
}
|
||||
@@ -159,7 +159,7 @@ func (s *WALReader) parseRecords(body []byte) ([]any, error) {
|
||||
// records = append(records, rec)
|
||||
|
||||
case CodeMeasuresDelete:
|
||||
rec := new(DeletedMeasures)
|
||||
rec := new(MeasuresDeleteRecord)
|
||||
if err = rec.Parse(src); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -171,184 +171,8 @@ func (s *WALReader) parseRecords(body []byte) ([]any, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// func (s *Reader) readAddedMetric(src *bytes.Buffer) (_ AddedMetric, err error) {
|
||||
// arr, err := bin.ReadN(src, 6)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// return AddedMetric{
|
||||
// MetricID: bin.GetUint32(arr),
|
||||
// MetricType: diploma.MetricType(arr[4]),
|
||||
// FracDigits: int(arr[5]),
|
||||
// }, nil
|
||||
// }
|
||||
|
||||
// func (s *Reader) readDeletedMetric(src *bytes.Buffer) (_ DeletedMetric, err error) {
|
||||
// var rec DeletedMetric
|
||||
// rec.MetricID, err = bin.ReadUint32(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// // free data pages
|
||||
// dataQty, _, err := bin.ReadVarUint64(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// for range dataQty {
|
||||
// var pageNo uint32
|
||||
// pageNo, err = bin.ReadUint32(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// rec.FreeDataPages = append(rec.FreeDataPages, pageNo)
|
||||
// }
|
||||
// // free index pages
|
||||
// indexQty, _, err := bin.ReadVarUint64(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// for range indexQty {
|
||||
// var pageNo uint32
|
||||
// pageNo, err = bin.ReadUint32(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// rec.FreeIndexPages = append(rec.FreeIndexPages, pageNo)
|
||||
// }
|
||||
// return rec, nil
|
||||
// }
|
||||
|
||||
// func (s *Reader) readAppendedMeasure(src *bytes.Buffer) (_ AppendedMeasure, err error) {
|
||||
// arr, err := bin.ReadN(src, 16)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// return AppendedMeasure{
|
||||
// MetricID: bin.GetUint32(arr[0:]),
|
||||
// Timestamp: bin.GetUint32(arr[4:]),
|
||||
// Value: bin.GetFloat64(arr[8:]),
|
||||
// }, nil
|
||||
// }
|
||||
|
||||
// func (s *Reader) readAppendedMeasures(src *bytes.Buffer) (_ AppendedMeasures, err error) {
|
||||
// var rec AppendedMeasures
|
||||
// rec.MetricID, err = bin.ReadUint32(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// qty, err := bin.ReadUint16(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// for range qty {
|
||||
// var measure proto.Measure
|
||||
// measure.Timestamp, err = bin.ReadUint32(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// measure.Value, err = bin.ReadFloat64(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// rec.Measures = append(rec.Measures, measure)
|
||||
// }
|
||||
// return rec, nil
|
||||
// }
|
||||
|
||||
// func (s *Reader) readAppendedMeasureWithOverflow(src *bytes.Buffer) (_ AppendedMeasureWithOverflow, err error) {
|
||||
// var (
|
||||
// b byte
|
||||
// rec AppendedMeasureWithOverflow
|
||||
// )
|
||||
// rec.MetricID, err = bin.ReadUint32(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// rec.Timestamp, err = bin.ReadUint32(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// rec.Value, err = bin.ReadFloat64(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// b, err = src.ReadByte()
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// rec.IsDataPageReused = b == 1
|
||||
|
||||
// rec.DataPageNo, err = bin.ReadUint32(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// b, err = src.ReadByte()
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// if b == 1 {
|
||||
// rec.IsRootChanged = true
|
||||
// rec.RootPageNo, err = bin.ReadUint32(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// }
|
||||
// // index pages
|
||||
// indexQty, err := src.ReadByte()
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// for range indexQty {
|
||||
// var pageNo uint32
|
||||
// pageNo, err = bin.ReadUint32(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// rec.ReusedIndexPages = append(rec.ReusedIndexPages, pageNo)
|
||||
// }
|
||||
// return rec, nil
|
||||
// }
|
||||
|
||||
// func (s *Reader) readDeletedMeasures(src *bytes.Buffer) (_ DeletedMeasures, err error) {
|
||||
// var (
|
||||
// rec DeletedMeasures
|
||||
// )
|
||||
// rec.MetricID, err = bin.ReadUint32(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// // free data pages
|
||||
// rec.FreeDataPages, err = s.readFreePageNumbers(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// // free index pages
|
||||
// rec.FreeIndexPages, err = s.readFreePageNumbers(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// return rec, nil
|
||||
// }
|
||||
|
||||
// HELPERS
|
||||
|
||||
// func (s *Reader) readFreePageNumbers(src *bytes.Buffer) ([]uint32, error) {
|
||||
// var freePages []uint32
|
||||
// qty, _, err := bin.ReadVarUint64(src)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// for range qty {
|
||||
// var pageNo uint32
|
||||
// pageNo, err = bin.ReadUint32(src)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// freePages = append(freePages, pageNo)
|
||||
// }
|
||||
// return freePages, nil
|
||||
// }
|
||||
|
||||
func (s *WALReader) Seek(offset int64) error {
|
||||
ret, err := s.file.Seek(offset, 0)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,362 +1,12 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
|
||||
bin "gordenko.dev/dima/bin/little"
|
||||
"gordenko.dev/dima/qb"
|
||||
)
|
||||
|
||||
type HeadDataPage struct {
|
||||
PageNo uint32
|
||||
Reused bool
|
||||
PrevPageNo uint32
|
||||
Checksum uint32
|
||||
TimestampsOffset int // offset on 1st page
|
||||
Timestamps []byte
|
||||
ValuesOffset int // offset on 1st page
|
||||
Values []byte
|
||||
}
|
||||
|
||||
type HeadIndexPage struct {
|
||||
PageNo uint32
|
||||
Reused bool
|
||||
Checksum uint32
|
||||
Records []byte // offset не потрібен, оскільки Records додаються в кінець
|
||||
}
|
||||
|
||||
type ChangedIndexLevel struct {
|
||||
HeadIndexPage *HeadIndexPage
|
||||
IndexPages []SealedIndexPage
|
||||
TailRecords []byte // payload only
|
||||
}
|
||||
|
||||
type MeasuresAppendWithGrowRecord struct {
|
||||
MetricID uint32
|
||||
HeadDataPage HeadDataPage
|
||||
DataPages []SealedDataPage
|
||||
ChangedIndexLevels []ChangedIndexLevel
|
||||
TailTimestamps []byte // data level tail
|
||||
TailValues []byte // data level tail
|
||||
}
|
||||
|
||||
/*
|
||||
Format appended measures:
|
||||
1b - tx type
|
||||
4b - metricID
|
||||
4b - pageNo
|
||||
1b - reused
|
||||
4b - prevPageNo
|
||||
4b - page crc32
|
||||
Nb - (varsize) timestamps offset
|
||||
Nb - (varsize) timestamps tail size on 1st filled data page
|
||||
Nb - timestamps tail on 1st filled data page
|
||||
Nb - (varsize) values offset
|
||||
Nb - (varsize) values tail size on 1st filled data page
|
||||
Nb - values tail on 1st filled data page
|
||||
|
||||
Nb - (varsize) filled data pages qty
|
||||
[
|
||||
|
||||
4b - pageNo
|
||||
1b - reused
|
||||
Nb - (data page size) page content
|
||||
|
||||
]
|
||||
Nb - (varsize) timestamps payload size on tail
|
||||
Nb - timestamps payload
|
||||
Nb - (varsize) values payload size on tail
|
||||
Nb - values payload
|
||||
Nb - (varsize) qty of index levels
|
||||
[
|
||||
|
||||
// NOTE!
|
||||
// - idx = 0 - zeroLevel
|
||||
// - skipped = maxRecordsOnIndexPage - records count
|
||||
4b - pageNo of 1st index page
|
||||
1b - reused
|
||||
4b - page crc32
|
||||
Nb - (varsize) records tail count on 1st filled index page
|
||||
Nb - records tail on 1st filled index page
|
||||
|
||||
|
||||
Nb - (varsize) qty of level filled pages
|
||||
[
|
||||
4b - pageNo
|
||||
1b - reused
|
||||
Nb - (index page size) page content
|
||||
]
|
||||
Nb - (varsize) size of records level tail
|
||||
Nb - level tail records
|
||||
|
||||
]
|
||||
*/
|
||||
func (s MeasuresAppendWithGrowRecord) WriteTo(w *bytes.Buffer) {
|
||||
w.WriteByte(CodeMeasuresAppendWithGrow)
|
||||
bin.WriteUint32(w, s.MetricID)
|
||||
// completed data page
|
||||
completed := s.HeadDataPage
|
||||
bin.WriteUint32(w, completed.PageNo)
|
||||
bin.WriteBool(w, completed.Reused)
|
||||
bin.WriteUint32(w, completed.PrevPageNo)
|
||||
bin.WriteUint32(w, completed.Checksum)
|
||||
bin.WriteVarSize(w, completed.TimestampsOffset)
|
||||
bin.WriteVarSize(w, len(completed.Timestamps))
|
||||
w.Write(completed.Timestamps)
|
||||
bin.WriteVarSize(w, completed.ValuesOffset)
|
||||
bin.WriteVarSize(w, len(completed.Values))
|
||||
w.Write(completed.Values)
|
||||
// data pages
|
||||
bin.WriteVarSize(w, len(s.DataPages))
|
||||
for _, p := range s.DataPages {
|
||||
bin.WriteUint32(w, p.PageNo)
|
||||
bin.WriteBool(w, p.Reused)
|
||||
bin.WriteVarSize(w, len(p.Content))
|
||||
w.Write(p.Content)
|
||||
}
|
||||
// data tail
|
||||
bin.WriteVarSize(w, len(s.TailTimestamps))
|
||||
w.Write(s.TailTimestamps)
|
||||
bin.WriteVarSize(w, len(s.TailValues))
|
||||
w.Write(s.TailValues)
|
||||
// changed levels
|
||||
bin.WriteVarSize(w, len(s.ChangedIndexLevels))
|
||||
for _, level := range s.ChangedIndexLevels {
|
||||
// completed index page
|
||||
completed := level.HeadIndexPage
|
||||
bin.WriteUint32(w, completed.PageNo)
|
||||
bin.WriteBool(w, completed.Reused)
|
||||
bin.WriteUint32(w, completed.Checksum)
|
||||
bin.WriteVarSize(w, len(completed.Records)/indexRecordSize)
|
||||
w.Write(completed.Records)
|
||||
// full index pages
|
||||
for _, p := range level.IndexPages {
|
||||
bin.WriteUint32(w, p.PageNo)
|
||||
bin.WriteBool(w, p.Reused)
|
||||
bin.WriteVarSize(w, len(p.Content))
|
||||
w.Write(p.Content)
|
||||
}
|
||||
// level tail
|
||||
bin.WriteVarSize(w, len(level.TailRecords)/indexRecordSize)
|
||||
w.Write(level.TailRecords)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MeasuresAppendWithGrowRecord) Parse(r *bytes.Buffer) (err error) {
|
||||
var (
|
||||
size int
|
||||
recordsCount int
|
||||
)
|
||||
s.MetricID, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.HeadDataPage.PageNo, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.HeadDataPage.Reused, err = bin.ReadBool(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.HeadDataPage.PrevPageNo, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.HeadDataPage.Checksum, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.HeadDataPage.TimestampsOffset, err = bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
size, err = bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.HeadDataPage.Timestamps, err = bin.ReadN(r, size)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
size, err = bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.HeadDataPage.Values, err = bin.ReadN(r, size)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// data pages
|
||||
dataPagesQty, err := bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for range dataPagesQty {
|
||||
var (
|
||||
p SealedDataPage
|
||||
)
|
||||
p.PageNo, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
p.Reused, err = bin.ReadBool(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
size, err = bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
p.Content, err = bin.ReadN(r, size)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.DataPages = append(s.DataPages, p)
|
||||
}
|
||||
// data tail
|
||||
size, err = bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.TailTimestamps, err = bin.ReadN(r, size)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
size, err = bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.TailValues, err = bin.ReadN(r, size)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// changed levels
|
||||
changedIndexLevelsCount, err := bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for range changedIndexLevelsCount {
|
||||
var (
|
||||
indexPagesCount int
|
||||
level ChangedIndexLevel
|
||||
)
|
||||
// completed index page
|
||||
level.HeadIndexPage.PageNo, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
level.HeadIndexPage.Reused, err = bin.ReadBool(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
level.HeadIndexPage.Checksum, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
recordsCount, err = bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
level.HeadIndexPage.Records, err = bin.ReadN(r, recordsCount*indexRecordSize)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
//
|
||||
indexPagesCount, err = bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for range indexPagesCount {
|
||||
var p SealedIndexPage
|
||||
p.PageNo, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
p.Reused, err = bin.ReadBool(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
p.Checksum, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
size, err = bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
p.Content, err = bin.ReadN(r, size)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
level.IndexPages = append(level.IndexPages, p)
|
||||
}
|
||||
recordsCount, err = bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
level.TailRecords, err = bin.ReadN(r, recordsCount*indexRecordSize)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.ChangedIndexLevels = append(s.ChangedIndexLevels, level)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type MeasuresAppendRecord struct {
|
||||
MetricID uint32
|
||||
TimestampsRewindOffset int
|
||||
Timestamps []byte
|
||||
ValuesRewindOffset int
|
||||
Values []byte
|
||||
}
|
||||
|
||||
func (s MeasuresAppendRecord) WriteTo(w *bytes.Buffer) {
|
||||
w.WriteByte(CodeMeasuresAppendWithGrow)
|
||||
bin.WriteUint32(w, s.MetricID)
|
||||
bin.WriteVarSize(w, s.TimestampsRewindOffset)
|
||||
bin.WriteVarSize(w, len(s.Timestamps))
|
||||
w.Write(s.Timestamps)
|
||||
bin.WriteVarSize(w, s.ValuesRewindOffset)
|
||||
bin.WriteVarSize(w, len(s.Values))
|
||||
w.Write(s.Values)
|
||||
}
|
||||
|
||||
func (s *MeasuresAppendRecord) Parse(r *bytes.Buffer) (err error) {
|
||||
var size int
|
||||
s.MetricID, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.TimestampsRewindOffset, err = bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
size, err = bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.Timestamps, err = bin.ReadN(r, size)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.ValuesRewindOffset, err = bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
size, err = bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.Values, err = bin.ReadN(r, size)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type MetricAddRecord struct {
|
||||
MetricID uint32
|
||||
MetricType qb.MetricType
|
||||
@@ -398,11 +48,8 @@ func (s MetricDeleteRecord) Pack(w io.Writer) {
|
||||
}
|
||||
bin.PutUint32(arr[1:], s.MetricID)
|
||||
w.Write(arr)
|
||||
// fix
|
||||
// bin.WriteVarSize(w, len(s.FreePageNumbers))
|
||||
// for _, pageNo := range s.FreePageNumbers {
|
||||
// bin.WriteUint32(w, pageNo)
|
||||
// }
|
||||
packPageNumbers(w, s.FreeIndexPages)
|
||||
packPageNumbers(w, s.FreeDataPages)
|
||||
}
|
||||
|
||||
func (s *MetricDeleteRecord) Parse(src io.Reader) (err error) {
|
||||
@@ -410,61 +57,378 @@ func (s *MetricDeleteRecord) Parse(src io.Reader) (err error) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// free pages
|
||||
// fix
|
||||
// qty, err := bin.ReadVarSize(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// for range qty {
|
||||
// var pageNo uint32
|
||||
// pageNo, err = bin.ReadUint32(src)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
// //s.FreePageNumbers = append(s.FreePageNumbers, pageNo)
|
||||
// }
|
||||
return nil
|
||||
s.FreeIndexPages, err = parsePageNumbers(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.FreeDataPages, err = parsePageNumbers(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type DataPageTail struct {
|
||||
PageNo uint32
|
||||
Reused bool
|
||||
PrevPageNo uint32
|
||||
CRC32 uint32
|
||||
TimestampsRewindOffset int // offset on 1st page
|
||||
Timestamps []byte
|
||||
ValuesRewindOffset int // offset on 1st page
|
||||
Values []byte
|
||||
}
|
||||
|
||||
type IndexPageTail struct {
|
||||
PageNo uint32
|
||||
Reused bool
|
||||
CRC32 uint32
|
||||
Records []byte // offset не потрібен, оскільки Records додаються в кінець
|
||||
}
|
||||
|
||||
type ChangedIndexLevel struct {
|
||||
IndexPageTail *IndexPageTail
|
||||
IndexPages []SealedIndexPage
|
||||
TailRecords []byte // payload only
|
||||
}
|
||||
|
||||
type MeasuresAppendWithGrowRecord struct {
|
||||
MetricID uint32
|
||||
DataPageTail DataPageTail
|
||||
DataPages []SealedDataPage
|
||||
TailTimestamps []byte // data level tail
|
||||
TailValues []byte // data level tail
|
||||
ChangedIndexLevels []ChangedIndexLevel
|
||||
}
|
||||
|
||||
/*
|
||||
Format appended measures:
|
||||
1b - tx type
|
||||
4b - metricID
|
||||
4b - pageNo
|
||||
1b - reused
|
||||
4b - prevPageNo
|
||||
4b - page crc32
|
||||
1b - timestamps & values offsets
|
||||
Nb - (varsize) timestamps tail size on 1st filled data page
|
||||
Nb - timestamps tail on 1st filled data page
|
||||
Nb - (varsize) values tail size on 1st filled data page
|
||||
Nb - values tail on 1st filled data page
|
||||
|
||||
Nb - (varsize) full data pages qty
|
||||
[
|
||||
|
||||
4b - pageNo
|
||||
1b - reused
|
||||
Nb - (data page size) page content
|
||||
|
||||
]
|
||||
Nb - (varsize) timestamps payload size on tail
|
||||
Nb - timestamps payload
|
||||
Nb - (varsize) values payload size on tail
|
||||
Nb - values payload
|
||||
Nb - (varsize) qty of index levels
|
||||
[
|
||||
|
||||
// NOTE!
|
||||
// - idx = 0 - zeroLevel
|
||||
// - skipped = maxRecordsOnIndexPage - records count
|
||||
4b - pageNo of 1st index page
|
||||
1b - reused
|
||||
4b - page crc32
|
||||
Nb - (varsize) records tail count on 1st filled index page
|
||||
Nb - records tail on 1st filled index page
|
||||
|
||||
|
||||
Nb - (varsize) qty of level filled pages
|
||||
[
|
||||
4b - pageNo
|
||||
1b - reused
|
||||
Nb - (index page size) page content
|
||||
]
|
||||
Nb - (varsize) size of records level tail
|
||||
Nb - level tail records
|
||||
|
||||
]
|
||||
*/
|
||||
func (s MeasuresAppendWithGrowRecord) Pack(w io.Writer) {
|
||||
arr := []byte{
|
||||
CodeMeasuresAppendWithGrow,
|
||||
0, 0, 0, 0, //
|
||||
0, 0, 0, 0, // tail pageNo
|
||||
0, // tail reused
|
||||
0, 0, 0, 0, // tail prevPageNo
|
||||
0, 0, 0, 0, // tail CRC32
|
||||
0, // tail offsets
|
||||
}
|
||||
bin.PutUint32(arr[1:], s.MetricID)
|
||||
// data page tail
|
||||
tail := s.DataPageTail
|
||||
bin.PutUint32(arr[5:], tail.PageNo)
|
||||
if tail.Reused {
|
||||
arr[9] = 1
|
||||
}
|
||||
bin.PutUint32(arr[10:], tail.PrevPageNo)
|
||||
bin.PutUint32(arr[14:], tail.CRC32)
|
||||
arr[18] = byte(tail.TimestampsRewindOffset) | (byte(tail.ValuesRewindOffset) << 4)
|
||||
w.Write(arr)
|
||||
bin.WriteVarSized(w, tail.Timestamps)
|
||||
bin.WriteVarSized(w, tail.Values)
|
||||
// data pages
|
||||
bin.WriteVarSize(w, len(s.DataPages))
|
||||
for _, p := range s.DataPages {
|
||||
bin.WriteUint32(w, p.PageNo)
|
||||
bin.WriteBool(w, p.Reused)
|
||||
w.Write(p.Content)
|
||||
}
|
||||
// data tail
|
||||
bin.WriteVarSized(w, s.TailTimestamps)
|
||||
bin.WriteVarSized(w, s.TailValues)
|
||||
// changed levels
|
||||
bin.WriteVarSize(w, len(s.ChangedIndexLevels))
|
||||
for _, level := range s.ChangedIndexLevels {
|
||||
// index page tail
|
||||
tail := level.IndexPageTail
|
||||
if tail != nil {
|
||||
bin.WriteBool(w, true)
|
||||
bin.WriteUint32(w, tail.PageNo)
|
||||
bin.WriteBool(w, tail.Reused)
|
||||
bin.WriteUint32(w, tail.CRC32)
|
||||
bin.WriteVarSize(w, len(tail.Records)/indexRecordSize)
|
||||
w.Write(tail.Records)
|
||||
} else {
|
||||
bin.WriteBool(w, false)
|
||||
}
|
||||
// full index pages
|
||||
bin.WriteVarSize(w, len(level.IndexPages))
|
||||
for _, p := range level.IndexPages {
|
||||
bin.WriteUint32(w, p.PageNo)
|
||||
bin.WriteBool(w, p.Reused)
|
||||
w.Write(p.Content)
|
||||
}
|
||||
// level tail
|
||||
bin.WriteVarSize(w, len(level.TailRecords)/indexRecordSize)
|
||||
w.Write(level.TailRecords)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MeasuresAppendWithGrowRecord) Parse(r io.Reader) (err error) {
|
||||
s.MetricID, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.DataPageTail.PageNo, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.DataPageTail.Reused, err = bin.ReadBool(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.DataPageTail.PrevPageNo, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.DataPageTail.CRC32, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
offsets, err := bin.ReadByte(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.DataPageTail.TimestampsRewindOffset = int(offsets & 15) // low 4 bits
|
||||
s.DataPageTail.ValuesRewindOffset = int(offsets >> 4) // high 4 bits
|
||||
//
|
||||
s.DataPageTail.Timestamps, err = bin.ReadVarSized(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.DataPageTail.Values, err = bin.ReadVarSized(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// data pages
|
||||
dataPagesQty, err := bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for range dataPagesQty {
|
||||
var p SealedDataPage
|
||||
p.PageNo, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
p.Reused, err = bin.ReadBool(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
p.Content, err = bin.ReadN(r, DataPageSize)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.DataPages = append(s.DataPages, p)
|
||||
}
|
||||
// // data tail
|
||||
s.TailTimestamps, err = bin.ReadVarSized(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.TailValues, err = bin.ReadVarSized(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// changed levels
|
||||
changedIndexLevelsCount, err := bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for range changedIndexLevelsCount {
|
||||
var (
|
||||
hasIndexPageTail bool
|
||||
recordsCount int
|
||||
indexPagesCount int
|
||||
level ChangedIndexLevel
|
||||
)
|
||||
hasIndexPageTail, err = bin.ReadBool(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if hasIndexPageTail {
|
||||
//index page tail
|
||||
var tail IndexPageTail
|
||||
tail.PageNo, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
tail.Reused, err = bin.ReadBool(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
tail.CRC32, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
recordsCount, err = bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
tail.Records, err = bin.ReadN(r, recordsCount*indexRecordSize)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
level.IndexPageTail = &tail
|
||||
}
|
||||
//
|
||||
indexPagesCount, err = bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for range indexPagesCount {
|
||||
var p SealedIndexPage
|
||||
p.PageNo, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
p.Reused, err = bin.ReadBool(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
p.Content, err = bin.ReadN(r, IndexPageSize)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
level.IndexPages = append(level.IndexPages, p)
|
||||
}
|
||||
recordsCount, err = bin.ReadVarSize(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
level.TailRecords, err = bin.ReadN(r, recordsCount*indexRecordSize)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.ChangedIndexLevels = append(s.ChangedIndexLevels, level)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type MeasuresAppendRecord struct {
|
||||
MetricID uint32
|
||||
TimestampsRewindOffset int
|
||||
Timestamps []byte
|
||||
ValuesRewindOffset int
|
||||
Values []byte
|
||||
}
|
||||
|
||||
func (s MeasuresAppendRecord) Pack(w io.Writer) {
|
||||
arr := []byte{
|
||||
CodeMeasuresAppend,
|
||||
0, 0, 0, 0, // metricID
|
||||
0, // offsets
|
||||
}
|
||||
bin.PutUint32(arr[1:], s.MetricID)
|
||||
arr[5] = byte(s.TimestampsRewindOffset) | (byte(s.ValuesRewindOffset) << 4)
|
||||
w.Write(arr)
|
||||
bin.WriteVarSized(w, s.Timestamps)
|
||||
bin.WriteVarSized(w, s.Values)
|
||||
}
|
||||
|
||||
func (s *MeasuresAppendRecord) Parse(r io.Reader) (err error) {
|
||||
s.MetricID, err = bin.ReadUint32(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
offsets, err := bin.ReadByte(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.TimestampsRewindOffset = int(offsets & 15) // low 4 bits
|
||||
s.ValuesRewindOffset = int(offsets >> 4) // high 4 bits
|
||||
//
|
||||
s.Timestamps, err = bin.ReadVarSized(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.Values, err = bin.ReadVarSized(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// fix - add changed pages
|
||||
type DeletedMeasures struct {
|
||||
MetricID uint32
|
||||
FreePageNumbers []uint32
|
||||
type MeasuresDeleteRecord struct {
|
||||
MetricID uint32
|
||||
FreeIndexPages []uint32
|
||||
FreeDataPages []uint32
|
||||
}
|
||||
|
||||
func (s DeletedMeasures) Pack(w io.Writer) {
|
||||
func (s MeasuresDeleteRecord) Pack(w io.Writer) {
|
||||
arr := []byte{
|
||||
CodeMeasuresDelete,
|
||||
0, 0, 0, 0,
|
||||
}
|
||||
bin.PutUint32(arr[1:], s.MetricID)
|
||||
w.Write(arr)
|
||||
bin.WriteVarSize(w, len(s.FreePageNumbers))
|
||||
for _, pageNo := range s.FreePageNumbers {
|
||||
bin.WriteUint32(w, pageNo)
|
||||
}
|
||||
packPageNumbers(w, s.FreeIndexPages)
|
||||
packPageNumbers(w, s.FreeDataPages)
|
||||
}
|
||||
|
||||
func (s *DeletedMeasures) Parse(src io.Reader) (err error) {
|
||||
func (s *MeasuresDeleteRecord) Parse(src io.Reader) (err error) {
|
||||
s.MetricID, err = bin.ReadUint32(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// free pages
|
||||
qty, err := bin.ReadVarSize(src)
|
||||
s.FreeIndexPages, err = parsePageNumbers(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for range qty {
|
||||
var pageNo uint32
|
||||
pageNo, err = bin.ReadUint32(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.FreePageNumbers = append(s.FreePageNumbers, pageNo)
|
||||
s.FreeDataPages, err = parsePageNumbers(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return nil
|
||||
return
|
||||
}
|
||||
|
||||
// type AppendedMeasure struct {
|
||||
@@ -664,3 +628,28 @@ Nb - values payload
|
||||
// }
|
||||
// return nil
|
||||
// }
|
||||
|
||||
// HELPERS
|
||||
|
||||
func packPageNumbers(w io.Writer, pageNumbers []uint32) {
|
||||
bin.WriteVarSize(w, len(pageNumbers))
|
||||
for _, pageNo := range pageNumbers {
|
||||
bin.WriteUint32(w, pageNo)
|
||||
}
|
||||
}
|
||||
|
||||
func parsePageNumbers(src io.Reader) (pageNumbers []uint32, err error) {
|
||||
count, err := bin.ReadVarSize(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for range count {
|
||||
var pageNo uint32
|
||||
pageNo, err = bin.ReadUint32(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
pageNumbers = append(pageNumbers, pageNo)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ const (
|
||||
CodeMetricAdd byte = 1
|
||||
CodeMetricDelete byte = 2
|
||||
CodeMeasuresAppendWithGrow byte = 5
|
||||
CodeMeasuresAppend = 6
|
||||
CodeMeasuresAppend byte = 6
|
||||
CodeMeasuresDelete byte = 7
|
||||
)
|
||||
|
||||
@@ -77,7 +77,7 @@ type Writer struct {
|
||||
indexPageSize int
|
||||
dataPageSize int
|
||||
input []any
|
||||
dataPreparer *DataPreparer
|
||||
dataPreparer *WritePreparer
|
||||
appendToWorkerQueue func(any)
|
||||
//lsn uint32
|
||||
written int64
|
||||
@@ -259,7 +259,7 @@ func (s *Writer) packAndWrite() (err error) {
|
||||
snapshotNumberCh := make(chan int, 1)
|
||||
|
||||
s.appendToWorkerQueue(Changes{
|
||||
Records: prepared.WriteResults,
|
||||
Records: prepared.Commits,
|
||||
SnapshotNumberCh: snapshotNumberCh,
|
||||
FrozenIndexPagesCount: s.indexFreeList.Pages(),
|
||||
IndexPageNumbers: s.indexFreeList.Cached(),
|
||||
@@ -290,7 +290,7 @@ func (s *Writer) packAndWrite() (err error) {
|
||||
}
|
||||
} else {
|
||||
s.appendToWorkerQueue(Changes{
|
||||
Records: prepared.WriteResults,
|
||||
Records: prepared.Commits,
|
||||
})
|
||||
}
|
||||
return nil
|
||||
|
||||
258
storage/write_preparer.go
Normal file
258
storage/write_preparer.go
Normal file
@@ -0,0 +1,258 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
bin "gordenko.dev/dima/bin/little"
|
||||
"gordenko.dev/dima/qb/util"
|
||||
)
|
||||
|
||||
type WritePreparer struct {
|
||||
pagePreparer *PageIngester
|
||||
//dataPagePayloadBound int
|
||||
w *bytes.Buffer
|
||||
indexPages []PageToWrite
|
||||
dataPages []PageToWrite
|
||||
commits []any
|
||||
}
|
||||
|
||||
type WritePreparerOptions struct {
|
||||
MinWALBufferSize int
|
||||
PageIngester *PageIngester
|
||||
}
|
||||
|
||||
func NewWritePreparer(opt WritePreparerOptions) *WritePreparer {
|
||||
//buf := make([]byte, opt.MinWALBufferSize)
|
||||
s := &WritePreparer{
|
||||
pagePreparer: opt.PageIngester,
|
||||
w: bytes.NewBuffer(nil),
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
type PreparedData struct {
|
||||
Packet []byte // пакет для запису в WAL
|
||||
WriteToIndex []PageToWrite
|
||||
WriteToData []PageToWrite
|
||||
Commits []any
|
||||
}
|
||||
|
||||
func (s *WritePreparer) Prepare(input []any) PreparedData {
|
||||
s.w.Write([]byte{
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, // size (max 9 byte)
|
||||
0, 0, 0, 0, // crc32
|
||||
})
|
||||
|
||||
hasher := util.NewHasher()
|
||||
|
||||
w := io.MultiWriter(s.w, hasher)
|
||||
|
||||
// 1. Пакую всі дані в WAL буфер запису
|
||||
for _, untyped := range input {
|
||||
switch x := untyped.(type) {
|
||||
case MetricAdd:
|
||||
rec := MetricAddRecord{
|
||||
MetricID: x.MetricID,
|
||||
MetricType: x.MetricType,
|
||||
FracDigits: x.FracDigits,
|
||||
}
|
||||
rec.Pack(w)
|
||||
s.commits = append(s.commits, MetricAddCommited{
|
||||
MetricID: x.MetricID,
|
||||
ResultCh: x.ResultCh,
|
||||
})
|
||||
|
||||
case MetricDelete:
|
||||
// if len(x.FreeIndexPages) > 0 {
|
||||
// s.indexFreeList.AddPageNumbers(x.FreeIndexPages)
|
||||
// }
|
||||
// if len(x.FreeIndexPages) > 0 {
|
||||
// s.dataFreeList.AddPageNumbers(x.FreeDataPages)
|
||||
// }
|
||||
rec := MetricDeleteRecord{
|
||||
MetricID: x.MetricID,
|
||||
FreeIndexPages: x.FreeIndexPages,
|
||||
FreeDataPages: x.FreeDataPages,
|
||||
}
|
||||
rec.Pack(w)
|
||||
s.commits = append(s.commits, MetricDeleteCommited{
|
||||
MetricID: x.MetricID,
|
||||
ResultCh: x.ResultCh,
|
||||
})
|
||||
|
||||
case MeasuresAppend:
|
||||
rec := MeasuresAppendRecord{
|
||||
MetricID: x.MetricID,
|
||||
TimestampsRewindOffset: x.TimestampsRewindOffset,
|
||||
Timestamps: x.Timestamps,
|
||||
ValuesRewindOffset: x.ValuesRewindOffset,
|
||||
Values: x.Values,
|
||||
}
|
||||
rec.Pack(w)
|
||||
// Дані для Worker
|
||||
s.commits = append(s.commits, MeasuresAppendCommited{
|
||||
MetricID: x.MetricID,
|
||||
ResultCode: x.ResultCode,
|
||||
WrittenCount: x.WrittenCount,
|
||||
ResultCh: x.ResultCh,
|
||||
})
|
||||
|
||||
case MeasuresAppendWithGrow:
|
||||
sealedDataPages, sealedIndexLevels := s.pagePreparer.Ingest(IngestIn{
|
||||
LastPageNo: x.LastPageNo,
|
||||
IndexLevelTails: x.IndexLevelTails,
|
||||
DataPages: x.DataPages,
|
||||
})
|
||||
|
||||
// сторінки для запису в index та data файли
|
||||
for _, p := range sealedDataPages {
|
||||
s.dataPages = append(s.dataPages, PageToWrite{
|
||||
PageNo: p.PageNo,
|
||||
Content: p.Content,
|
||||
})
|
||||
}
|
||||
for _, level := range sealedIndexLevels {
|
||||
for _, p := range level.IndexPages {
|
||||
fmt.Printf("records: %v\n", getIndexRecords(p.Content, 2))
|
||||
s.indexPages = append(s.indexPages, PageToWrite{
|
||||
PageNo: p.PageNo,
|
||||
Content: p.Content,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
// дані для запису в WAL
|
||||
changedIndexPages []ChangedIndexLevel
|
||||
// дані для Воркера
|
||||
indexLevelTails []IndexLevelTail
|
||||
)
|
||||
|
||||
for _, level := range sealedIndexLevels {
|
||||
if len(level.IndexPages) > 0 || level.SkipRecords != level.TailRecordsCount {
|
||||
// на індексному внесені зміни
|
||||
var (
|
||||
indexPageTail *IndexPageTail
|
||||
indexPages []SealedIndexPage
|
||||
)
|
||||
if len(level.IndexPages) > 0 {
|
||||
first := level.IndexPages[0]
|
||||
skipSize := level.SkipRecords * indexRecordSize
|
||||
indexPageTail = &IndexPageTail{
|
||||
PageNo: first.PageNo,
|
||||
Reused: first.Reused,
|
||||
// в WAL файл попадає лише payload
|
||||
Records: first.Content[skipSize : maxRecordsOnIndexPage*indexRecordSize],
|
||||
}
|
||||
indexPages = level.IndexPages[1:]
|
||||
} else {
|
||||
indexPages = level.IndexPages
|
||||
}
|
||||
changedIndexPages = append(changedIndexPages, ChangedIndexLevel{
|
||||
IndexPageTail: indexPageTail,
|
||||
IndexPages: indexPages,
|
||||
// в WAL файл попадає лише payload
|
||||
TailRecords: level.TailRecords[:level.TailRecordsCount*indexRecordSize],
|
||||
})
|
||||
}
|
||||
// Worker-у відправляю повний index
|
||||
indexLevelTails = append(indexLevelTails, IndexLevelTail{
|
||||
// Worker отримує повний буфер
|
||||
Buffer: level.TailRecords,
|
||||
RecordsCount: level.TailRecordsCount,
|
||||
})
|
||||
}
|
||||
|
||||
first := sealedDataPages[0]
|
||||
firstCRC32, _ := bin.GetUint32(first.Content[dataCRC32Idx:])
|
||||
|
||||
rec := MeasuresAppendWithGrowRecord{
|
||||
MetricID: x.MetricID,
|
||||
DataPageTail: DataPageTail{
|
||||
PageNo: first.PageNo,
|
||||
Reused: first.Reused,
|
||||
PrevPageNo: x.LastPageNo,
|
||||
CRC32: firstCRC32,
|
||||
TimestampsRewindOffset: x.TimestampsRewindOffset,
|
||||
Timestamps: x.Timestamps,
|
||||
ValuesRewindOffset: x.ValuesRewindOffset,
|
||||
Values: x.Values,
|
||||
},
|
||||
DataPages: sealedDataPages[1:],
|
||||
ChangedIndexLevels: changedIndexPages,
|
||||
TailTimestamps: x.Timestamps,
|
||||
TailValues: x.Values,
|
||||
}
|
||||
|
||||
rec.Pack(w)
|
||||
// Дані для Worker
|
||||
s.commits = append(s.commits, MeasuresAppendWithGrowCommited{
|
||||
MetricID: x.MetricID,
|
||||
LastPageNo: sealedDataPages[len(sealedDataPages)-1].PageNo,
|
||||
Index: indexLevelTails,
|
||||
ResultCode: x.ResultCode,
|
||||
WrittenCount: x.WrittenCount,
|
||||
ResultCh: x.ResultCh,
|
||||
})
|
||||
|
||||
case MeasuresDelete:
|
||||
// if len(x.FreeIndexPages) > 0 {
|
||||
// s.indexFreeList.AddPageNumbers(x.FreeIndexPages)
|
||||
// }
|
||||
// if len(x.FreeIndexPages) > 0 {
|
||||
// s.dataFreeList.AddPageNumbers(x.FreeDataPages)
|
||||
// }
|
||||
rec := MeasuresDeleteRecord{
|
||||
MetricID: x.MetricID,
|
||||
FreeIndexPages: x.FreeIndexPages,
|
||||
FreeDataPages: x.FreeDataPages,
|
||||
}
|
||||
rec.Pack(w)
|
||||
s.commits = append(s.commits, MeasuresDeleteCommited{
|
||||
MetricID: x.MetricID,
|
||||
ResultCh: x.ResultCh,
|
||||
})
|
||||
//case DeletedMeasuresSince:
|
||||
}
|
||||
}
|
||||
// 2. Додаю розмір пакету і чексуму
|
||||
|
||||
//s.written += int64(len(packet)) + 12
|
||||
|
||||
packet := s.w.Bytes()
|
||||
// write size
|
||||
payloadSize := len(packet) - 13
|
||||
start := 9 - bin.CountVarSize(payloadSize)
|
||||
bin.PutVarSize(packet[start:], payloadSize)
|
||||
// CRC32
|
||||
bin.PutUint32(packet[9:], hasher.Sum32())
|
||||
//
|
||||
return PreparedData{
|
||||
Packet: packet[start:],
|
||||
WriteToIndex: s.indexPages,
|
||||
WriteToData: s.dataPages,
|
||||
Commits: s.commits,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *WritePreparer) Reset() {
|
||||
s.w.Reset()
|
||||
s.indexPages = nil
|
||||
s.dataPages = nil
|
||||
s.commits = nil
|
||||
}
|
||||
|
||||
func getIndexRecords(buf []byte, count int) (list []IndexRecord) {
|
||||
fmt.Printf("ipage: % x\n", buf)
|
||||
i := 0
|
||||
for range count {
|
||||
var rec IndexRecord
|
||||
rec.Timestamp, _ = bin.GetUint32(buf[i:])
|
||||
rec.PageNo, _ = bin.GetUint32(buf[i+4:])
|
||||
list = append(list, rec)
|
||||
i += indexRecordSize
|
||||
}
|
||||
return
|
||||
}
|
||||
Reference in New Issue
Block a user