dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-06-13 08:01:42 +03:00
parent bf22e9a85e
commit aca0252cfd
19 changed files with 2001 additions and 1240 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
database/api.go
View File

@@ -500,22 +500,22 @@ func (s *Database) DeleteMeasures(req proto.DeleteMeasuresReq) uint16 {
case DeleteFromAtreeNotNeeded:
// регистрирую удаление в TransactionLog
s.storage.Append(storage.DeletedMeasures{
s.storage.Append(storage.MeasuresDeleteRecord{
MetricID: req.MetricID,
})
//<-waitCh
case DeleteFromAtreeRequired:
// собираю номера всех data и index страниц метрики (типа запись REDO лога).
pageNumbers, err := s.atree.GetAllPages(req.MetricID)
if err != nil {
qb.Abort(qb.FailedAtreeRequest, err)
}
// регистрирую удаление в TransactionLog
s.storage.Append(storage.DeletedMeasures{
MetricID: req.MetricID,
FreePageNumbers: pageNumbers,
})
// pageNumbers, err := s.atree.GetAllPages(req.MetricID)
// if err != nil {
// qb.Abort(qb.FailedAtreeRequest, err)
// }
// // регистрирую удаление в TransactionLog
// s.storage.Append(storage.DeletedMeasures{
// MetricID: req.MetricID,
// FreePageNumbers: pageNumbers,
// })
//<-waitCh
case NoMetric:

8
database/database_test.go
View File

@@ -98,10 +98,10 @@ func TestComposeHeadIndexPage(t *testing.T) {
},
RecordsCount: 2,
}
head = &storage.HeadIndexPage{
PageNo: 100,
Checksum: 12345,
Records: []byte{},
head = &storage.IndexPageTail{
PageNo: 100,
CRC32: 12345,
Records: []byte{},
}
)
page := composeHeadIndexPage(levelIdx, level, head)

243
database/metric.go
View File

@@ -16,6 +16,11 @@ var (
indexRecordSize = 8
)
type CapturedState struct {
LastTimestamp uint32
LastValue float64
}
type _metric struct {
metricType qb.MetricType
fracDigits byte
@@ -34,20 +39,19 @@ type _metric struct {
capturedState *CapturedState
}
//IndexLevels [][]IndexRec // root - last element
func (s *_metric) LastValue() float64 {
if s.capturedState != nil {
return s.capturedState.LastValue
}
return s.lastValue
}
// func (s *_metric) ReinitBy(timestamp uint32, value float64) {
// s.Timestamps.Renew()
// s.Values.Renew()
// s.Timestamps.Append(timestamp)
// s.Values.Append(value)
// s.Since = timestamp
// s.SinceValue = value
// s.Until = timestamp
// s.UntilValue = value
// }
func (s *_metric) LastTimestamp() uint32 {
if s.capturedState != nil {
return s.capturedState.LastTimestamp
}
return s.timestamps.LastTimestamp()
}
func (s *_metric) DeleteMeasures() {
// s.Timestamps.Renew()
@@ -60,11 +64,6 @@ func (s *_metric) DeleteMeasures() {
// s.UntilValue = 0
}
type CapturedState struct {
LastTimestamp uint32
LastValue float64
}
// func (s *_metric) startAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(storage.AppendedMeasures)) {
func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, tmp []byte, sendToStorage func(any)) {
if s.capturedState != nil {
@@ -92,7 +91,6 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, tmp []byte, send
s.capturedState = &CapturedState{
LastTimestamp: timestamps.LastTimestamp(),
LastValue: s.lastValue,
//LastValue: values.LastValue(),
}
s.timestamps.CaptureState()
@@ -128,7 +126,8 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, tmp []byte, send
// сторінка заповнена
since := s.timestamps.ReplaceSinceWithUntil()
if len(pages) == 0 {
if len(pages) == 0 && idx > 0 {
// idx > 0 required because page may overflows without append any data
headTimestamps = timestamps.Tail(timestampsOffset)
headValues = values.Tail(valuesOffset)
}
@@ -152,7 +151,6 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, tmp []byte, send
timestamps.Append(tReport.RewindOffset, tmp[:tReport.ChangeSize], measure.Timestamp)
values.Append(vReport.RewindOffset, tmp[7:7+vReport.ChangeSize], measure.Value, vReport.Delta)
}
//
s.lastValue = measure.Value
written++
@@ -171,38 +169,38 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, tmp []byte, send
if len(pages) > 0 {
// пишу в storage довгим шляхом через redo файл і запис в data файл
sendToStorage(storage.AppendedMeasuresWithGrow{
MetricID: req.MetricID,
LastPageNo: s.lastPageNo,
TimestampsOffset: timestampsRewindOffset,
Timestamps: headTimestamps,
ValuesOffset: valuesRewindOffset,
Values: headValues,
IndexLevelTails: s.indexLevelTails,
DataPages: pages,
TailTimestamps: timestamps.Tail(0), // payload
TailValues: values.Tail(0),
ResultCode: resultCode,
WrittenCount: written,
ResultCh: req.ResultCh,
sendToStorage(storage.MeasuresAppendWithGrow{
MetricID: req.MetricID,
LastPageNo: s.lastPageNo,
TimestampsRewindOffset: timestampsRewindOffset,
Timestamps: headTimestamps,
ValuesRewindOffset: valuesRewindOffset,
Values: headValues,
IndexLevelTails: s.indexLevelTails,
DataPages: pages,
TailTimestamps: timestamps.Tail(0), // payload
TailValues: values.Tail(0),
ResultCode: resultCode,
WrittenCount: written,
//ResultCh: req.ResultCh,
})
} else {
// короткий шлях - запис лише в storage
sendToStorage(storage.AppendedMeasuresWithGrow{
MetricID: req.MetricID,
TimestampsOffset: timestampsOffset,
ValuesOffset: valuesOffset,
Timestamps: timestamps.Tail(timestampsOffset),
Values: values.Tail(valuesOffset),
ResultCode: resultCode,
WrittenCount: written,
ResultCh: req.ResultCh,
sendToStorage(storage.MeasuresAppend{
MetricID: req.MetricID,
TimestampsRewindOffset: timestampsRewindOffset,
ValuesRewindOffset: valuesRewindOffset,
Timestamps: timestamps.Tail(timestampsOffset),
Values: values.Tail(valuesOffset),
ResultCode: resultCode,
WrittenCount: written,
//ResultCh: req.ResultCh,
})
}
}
func (s *_metric) FinAppendMeasures(rec storage.AppendMeasuresSummary) {
func (s *_metric) FinAppendMeasures(rec storage.MeasuresAppendWithGrowCommited) {
// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
s.capturedState = nil
@@ -217,71 +215,71 @@ func (s *_metric) FinAppendMeasures(rec storage.AppendMeasuresSummary) {
// READ
func (s *_metric) StartRangeScan(req tryRangeScanReq) {
if s.Since == 0 {
req.ResultCh <- rangeScanResult{
ResultCode: QueryDone,
}
return
}
// if s.Since == 0 {
// req.ResultCh <- rangeScanResult{
// ResultCode: QueryDone,
// }
// return
// }
if req.Since > s.Until {
req.ResultCh <- rangeScanResult{
ResultCode: QueryDone,
}
return
}
// if req.Since > s.Until {
// req.ResultCh <- rangeScanResult{
// ResultCode: QueryDone,
// }
// return
// }
if req.Until < s.Since {
if s.RootPageNo > 0 {
req.ResultCh <- rangeScanResult{
ResultCode: UntilNotFound,
RootPageNo: s.RootPageNo,
FracDigits: s.FracDigits,
}
s.RLocks++
return
} else {
req.ResultCh <- rangeScanResult{
ResultCode: QueryDone,
}
return
}
}
// if req.Until < s.Since {
// if s.RootPageNo > 0 {
// req.ResultCh <- rangeScanResult{
// ResultCode: UntilNotFound,
// RootPageNo: s.RootPageNo,
// FracDigits: s.FracDigits,
// }
// s.RLocks++
// return
// } else {
// req.ResultCh <- rangeScanResult{
// ResultCode: QueryDone,
// }
// return
// }
// }
timestampDecompressor := s.timestamps.CreateDecompressor()
valueDecompressor := s.values.CreateDecompressor(s.metricType, s.fracDigits)
// timestampDecompressor := s.timestamps.CreateDecompressor()
// valueDecompressor := s.values.CreateDecompressor(s.metricType, s.fracDigits)
for {
timestamp, done := timestampDecompressor.NextValue()
if done {
break
}
value, done := valueDecompressor.NextValue()
if done {
qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug)
}
if timestamp <= req.Until {
req.ResponseWriter.FeedNoSend(timestamp, value)
if timestamp < req.Since {
req.ResultCh <- rangeScanResult{
ResultCode: QueryDone,
}
return
}
}
}
if s.lastPageNo > 0 {
req.ResultCh <- rangeScanResult{
ResultCode: UntilFound,
LastPageNo: s.lastPageNo,
FracDigits: s.fracDigits,
}
s.RLocks++
} else {
req.ResultCh <- rangeScanResult{
ResultCode: QueryDone,
}
}
// for {
// timestamp, done := timestampDecompressor.NextValue()
// if done {
// break
// }
// value, done := valueDecompressor.NextValue()
// if done {
// qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug)
// }
// if timestamp <= req.Until {
// req.ResponseWriter.FeedNoSend(timestamp, value)
// if timestamp < req.Since {
// req.ResultCh <- rangeScanResult{
// ResultCode: QueryDone,
// }
// return
// }
// }
// }
// if s.lastPageNo > 0 {
// req.ResultCh <- rangeScanResult{
// ResultCode: UntilFound,
// LastPageNo: s.lastPageNo,
// FracDigits: s.fracDigits,
// }
// s.RLocks++
// } else {
// req.ResultCh <- rangeScanResult{
// ResultCode: QueryDone,
// }
// }
}
func (s *_metric) StartFullScan(req tryFullScanReq) {
@@ -384,36 +382,3 @@ func (s *_metric) WriteTo(w io.Writer) (err error) {
}
return
}
// since - 4b
// sinceValue - 8b -
// untilValue - 8b
// time
// if s.state != nil {
// h = s.state.H
// lastUnixtime = s.state.LastUnixtime
// lastDelta = s.state.LastDelta
// payload = s.state.Payload
// }
// return NewTimeDeltaDecompressorFromState(TimeDeltaDecompressorFromStateOptions{
// H: h,
// LastUnixtime: lastUnixtime,
// LastDelta: lastDelta,
// Payload: payload,
// })
// value
// if s.state != nil {
// h = s.state.H
// lastDelta = s.state.LastDelta
// payload = s.state.Payload
// }
// return NewValueDeltaDecompressorFromState(ValueDeltaDecompressorFromStateOptions{
// Coef: s.coef,
// ToFloat64: s.toFloat64,
// H: s.buf[s.pos-1],
// LastDelta: s.lastDelta,
// Payload: s.buf[:s.pos],
// })

12
database/proc.go
View File

@@ -294,7 +294,7 @@ func (s *Database) startDeleteMeasures(metric *_metric, req tryDeleteMeasuresReq
// }
}
func (s *Database) finAppendMeasures(rec storage.AppendMeasuresSummary) {
func (s *Database) finAppendMeasures(rec storage.MeasuresAppendWithGrowCommited) {
metric, ok := s.metrics[rec.MetricID]
if !ok {
qb.Abort(qb.NoMetricBug,
@@ -398,8 +398,8 @@ func (s *Database) tryListCurrentValues(req tryListCurrentValuesReq) {
if ok {
req.ResponseWriter.BufferValue(transform.CurrentValue{
MetricID: metricID,
Timestamp: metric.timestamps.LastTimestamp(),
Value: metric.lastValue,
Timestamp: metric.LastTimestamp(),
Value: metric.LastValue(),
})
}
}
@@ -420,13 +420,13 @@ func (s *Database) applyChanges(req storage.Changes) {
// case storage.AppendedMeasure:
// s.appendMeasure(rec)
case storage.AppendMeasuresSummary:
case storage.MeasuresAppendWithGrowCommited:
s.finAppendMeasures(rec)
// case storage.AppendedMeasureWithOverflowExtended:
// s.appendMeasureAfterOverflow(rec)
case storage.DeletedMeasures:
case storage.MeasuresDeleteRecord:
s.deleteMeasures(rec)
}
}
@@ -555,7 +555,7 @@ func (s *Database) deleteMetric(rec storage.MetricDeleteRecord) {
}
}
func (s *Database) deleteMeasures(rec storage.DeletedMeasures) {
func (s *Database) deleteMeasures(rec storage.MeasuresDeleteRecord) {
metric, ok := s.metrics[rec.MetricID]
if !ok {
qb.Abort(qb.NoMetricBug,

26
database/replay_metric.go
View File

@@ -113,7 +113,7 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec storage.MeasuresAppendWithGrow
// Додати в index level tails недостаючі дані, або замінити
for levelIdx, change := range rec.ChangedIndexLevels {
var (
head = change.HeadIndexPage
head = change.IndexPageTail
newRecordsCount = len(change.TailRecords) / indexRecordSize
)
// 1. набиваю сторінки для перезапису в index файлі
@@ -171,7 +171,7 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec storage.MeasuresAppendWithGrow
// 4. набиваю сторінки для перезапису в data файлі
if isLastPacket {
dataPages = append(dataPages, composeHeadDataPage(s.databuf, rec.HeadDataPage))
dataPages = append(dataPages, composeHeadDataPage(s.databuf, rec.DataPageTail))
for _, x := range rec.DataPages {
dataPages = append(dataPages, storage.PageToWrite{
@@ -182,7 +182,7 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec storage.MeasuresAppendWithGrow
}
// рахую кількість reused дата сторінок
if rec.HeadDataPage.Reused {
if rec.DataPageTail.Reused {
reusedDataPages++
}
for _, x := range rec.DataPages {
@@ -200,7 +200,7 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec storage.MeasuresAppendWithGrow
s.vSize = len(rec.TailValues)
if len(rec.DataPages) == 0 {
s.lastPageNo = rec.HeadDataPage.PageNo
s.lastPageNo = rec.DataPageTail.PageNo
} else {
s.lastPageNo = rec.DataPages[len(rec.DataPages)-1].PageNo
}
@@ -215,7 +215,7 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec storage.MeasuresAppendWithGrow
// HELPERS
func composeHeadIndexPage(levelIdx int, level storage.IndexLevelTail, head *storage.HeadIndexPage) storage.PageToWrite {
func composeHeadIndexPage(levelIdx int, level storage.IndexLevelTail, head *storage.IndexPageTail) storage.PageToWrite {
// розраховую pos, з якого буду дописувати хвіст
pos := level.RecordsCount * indexRecordSize
// створюю копію сторінки
@@ -229,10 +229,10 @@ func composeHeadIndexPage(levelIdx int, level storage.IndexLevelTail, head *stor
ZeroLevel: levelIdx == 0,
})
// перевірка CRC
if calculatedCRC != head.Checksum {
if calculatedCRC != head.CRC32 {
qb.Abort(qb.WALReplayFailed,
fmt.Errorf("calculated CRC %d not equal expected %d of head page %d on index level %d",
calculatedCRC, head.Checksum, head.PageNo, levelIdx))
calculatedCRC, head.CRC32, head.PageNo, levelIdx))
}
return storage.PageToWrite{
PageNo: head.PageNo,
@@ -240,29 +240,29 @@ func composeHeadIndexPage(levelIdx int, level storage.IndexLevelTail, head *stor
}
}
func composeHeadDataPage(databuf []byte, head storage.HeadDataPage) storage.PageToWrite {
func composeHeadDataPage(databuf []byte, head storage.DataPageTail) storage.PageToWrite {
// створюю копію сторінки
var (
page = make([]byte, storage.DataPageSize)
timestampsSize = head.TimestampsOffset - len(head.Timestamps)
timestampsSize = head.TimestampsRewindOffset - len(head.Timestamps)
)
// копіюю поточні дані
copy(page, databuf)
// копіюю нові дані із WAL з урахуванням offset-ів
copy(page[head.ValuesOffset:], head.Values)
copy(page[head.ValuesRewindOffset:], head.Values)
copy(page[len(page)-timestampsSize:], head.Timestamps)
// запечатати сторінку
calculatedCRC := storage.SealDataPage(storage.SealDataPageIn{
Content: page,
PrevPageNo: head.PrevPageNo,
TimestampsSize: timestampsSize,
ValuesSize: head.ValuesOffset + len(head.Values),
ValuesSize: head.ValuesRewindOffset + len(head.Values),
})
// перевірка CRC
if calculatedCRC != head.Checksum {
if calculatedCRC != head.CRC32 {
qb.Abort(qb.WALReplayFailed,
fmt.Errorf("calculated CRC %d not equal expected %d of head data page %d",
calculatedCRC, head.Checksum, head.PageNo))
calculatedCRC, head.CRC32, head.PageNo))
}
return storage.PageToWrite{
PageNo: head.PageNo,