dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-06-12 00:29:02 +03:00
parent ed203195fa
commit dc0410c9e4
8 changed files with 161 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
database/metric.go
View File

@@ -31,6 +31,7 @@ type _metric struct {
RLocks int
WaitQueue []any
indexLevelTails []storage.IndexLevelTail // root - last element
capturedState *CapturedState
}
//IndexLevels [][]IndexRec // root - last element
@@ -59,59 +60,45 @@ func (s *_metric) DeleteMeasures() {
// s.UntilValue = 0
}
type CapturedState struct {
Pages []storage.DataPayload
TimeState qb.TimeDeltaCapturedState
ValueState qb.ValueDeltaCapturedState
}
// func (s *_metric) startAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(storage.AppendedMeasures)) {
func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, tmp []byte, sendToStorage func(any)) {
if s.capturedState != nil {
s.WaitQueue = append(s.WaitQueue, req)
}
var (
timestamps = s.timestamps
values = s.values
dataPages []storage.DataPayload
// офсети на head сторінці
timestampsOffset int
valuesOffset int
//written int
//resultCode byte
written int
resultCode byte
)
s.values.CaptureState()
s.timestamps.CaptureState()
s.capturedState = &CapturedState{
//
TimeState: s.timestamps.CaptureState(),
ValueState: s.values.CaptureState(),
}
for idx, measure := range req.Measures {
// FIX - у випадку помилки треба в транзакції зберегти що помилка, але також зафіксувати скільки елементів збережено.
// якщо idx == 0 - одразу знімаю блокування і нічого не відправляю в storage
if measure.Timestamp <= s.timestamps.LastTimestamp() {
if idx == 0 {
s.values.ForgetCapturedState()
s.timestamps.ForgetCapturedState()
req.ResultCh <- tryAppendMeasuresResult{
ResultCode: ExpiredMeasure,
}
return
}
//resultCode = ExpiredMeasure
//written = idx
resultCode = ExpiredMeasure
break
}
if s.metricType == qb.Cumulative && measure.Value < s.lastValue {
if idx == 0 {
s.values.ForgetCapturedState()
s.timestamps.ForgetCapturedState()
req.ResultCh <- tryAppendMeasuresResult{
ResultCode: NonMonotonicValue,
}
return
}
//resultCode = NonMonotonicValue
//written = idx
resultCode = NonMonotonicValue
break
}
//}
// fix - 1 + 8 bytes
@@ -144,13 +131,13 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, tmp []byte, send
// сторінка заповнена
since := s.timestamps.ReplaceSinceWithUntil()
if len(dataPages) == 0 {
if len(s.capturedState.Pages) == 0 {
// head page FIX
// timestampsPayload =
// valuesPayload =
}
dataPages = append(dataPages, storage.DataPayload{
s.capturedState.Pages = append(s.capturedState.Pages, storage.DataPayload{
Since: since,
Content: s.buffer,
TimestampsSize: timestamps.Size(),
@@ -171,38 +158,54 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, tmp []byte, send
values.Append(vReport.Offset, tmp[7:7+vReport.ChangeSize], measure.Value, vReport.Delta)
//
s.lastValue = measure.Value
written++
}
if written == 0 {
s.capturedState = nil
req.ResultCh <- tryAppendMeasuresResult{
ResultCode: resultCode,
}
return
}
// виділити змінені байти.
// скопіювати. Причому можна скопіювати зрізи chunks
if len(dataPages) > 0 {
if len(s.capturedState.Pages) > 0 {
head := s.capturedState.Pages[0]
databuf := head.Content[:storage.DataPagePayloadSize]
// пишу в storage довгим шляхом через redo файл і запис в data файл
sendToStorage(storage.AppendedMeasures{
MetricID: req.MetricID,
LastPageNo: s.lastPageNo,
TimestampsOffset: timestampsOffset,
Timestamps: databuf[len(databuf)-head.TimestampsSize : len(databuf)-timestampsOffset],
ValuesOffset: valuesOffset,
//Payload: s.payload, // fix - timestamps + values ? or timestamps and values (for WAL)
IndexLevelTails: s.indexLevelTails,
DataPages: dataPages,
Timestamps: nil,
Values: nil,
//ResultCode: resultCode,
//WrittenCount: wri,
ResultCh: nil,
Values: databuf[valuesOffset:head.ValuesSize],
IndexLevelTails: s.indexLevelTails,
DataPages: s.capturedState.Pages,
TailTimestamps: nil, // payload
TailValues: s.buffer[:values.Size()],
ResultCode: resultCode,
WrittenCount: written,
ResultCh: nil,
})
} else {
databuf := s.buffer
if len(s.buffer) == storage.DataPageSize {
databuf = s.buffer[:storage.DataPagePayloadSize]
}
// короткий шлях - запис лише в storage
sendToStorage(storage.AppendedMeasures{
MetricID: req.MetricID,
TimestampsOffset: timestampsOffset,
ValuesOffset: valuesOffset,
Timestamps: nil, // pos - offset
Values: nil,
//ResultCode: resultCode,
//WrittenCount: wri,
ResultCh: nil,
Timestamps: databuf[len(databuf)-timestamps.Size() : len(databuf)-timestampsOffset],
Values: databuf[valuesOffset:values.Size()],
ResultCode: resultCode,
WrittenCount: written,
ResultCh: nil,
})
}
@@ -210,8 +213,7 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, tmp []byte, send
func (s *_metric) FinAppendMeasures(rec storage.AppendMeasuresSummary) {
// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
s.values.ForgetCapturedState()
s.timestamps.ForgetCapturedState()
s.capturedState = nil
if rec.LastPageNo > 0 {
s.lastPageNo = rec.LastPageNo
@@ -367,13 +369,21 @@ func (s *_metric) WriteTo(w io.Writer) (err error) {
if err != nil {
return
}
var (
timestampsCapturedState *qb.TimeDeltaCapturedState
valuesCapturedState *qb.ValueDeltaCapturedState
)
if s.capturedState != nil {
timestampsCapturedState = &s.capturedState.TimeState
valuesCapturedState = &s.capturedState.ValueState
}
// timestamps payload
err = s.timestamps.WritePayloadTo(w)
err = s.timestamps.WritePayloadTo(w, timestampsCapturedState)
if err != nil {
return
}
// values payload
err = s.values.WritePayloadTo(w)
err = s.values.WritePayloadTo(w, valuesCapturedState)
if err != nil {
return
}
@@ -398,3 +408,32 @@ func (s *_metric) WriteTo(w io.Writer) (err error) {
// since - 4b
// sinceValue - 8b -
// untilValue - 8b
// time
// if s.state != nil {
// h = s.state.H
// lastUnixtime = s.state.LastUnixtime
// lastDelta = s.state.LastDelta
// payload = s.state.Payload
// }
// return NewTimeDeltaDecompressorFromState(TimeDeltaDecompressorFromStateOptions{
// H: h,
// LastUnixtime: lastUnixtime,
// LastDelta: lastDelta,
// Payload: payload,
// })
// value
// if s.state != nil {
// h = s.state.H
// lastDelta = s.state.LastDelta
// payload = s.state.Payload
// }
// return NewValueDeltaDecompressorFromState(ValueDeltaDecompressorFromStateOptions{
// Coef: s.coef,
// ToFloat64: s.toFloat64,
// H: s.buf[s.pos-1],
// LastDelta: s.lastDelta,
// Payload: s.buf[:s.pos],
// })

12
database/proc.go
View File

@@ -132,7 +132,7 @@ func (s *Database) processMetricQueue(metricID uint32, metric *_metric) {
s.tryGetMetric(req)
case tryAppendMeasuresReq:
metric.StartAppendMeasures(req, s.storage.Append)
//metric.StartAppendMeasures(req, s.storage.Append) FIX
case tryDeleteMetricReq:
s.startDeleteMetric(metric, req)
@@ -322,7 +322,7 @@ func (s *Database) tryAppendMeasures(req tryAppendMeasuresReq) {
metric.WaitQueue = append(metric.WaitQueue, req)
return
}
metric.StartAppendMeasures(req, s.storage.Append)
//metric.StartAppendMeasures(req, s.storage.Append) FIX
}
type tryRangeScanReq struct {
@@ -472,20 +472,14 @@ func (s *Database) applyAddMetric(rec storage.MetricAddRecord) {
func (s *Database) addMetric(rec storage.MetricAddRecord) {
var (
values qb.ValueCompressor
buffer = make([]byte, minBufferSize)
)
if rec.MetricType == qb.Cumulative {
values = enc.NewCumulativeDeltaCompressor(byte(rec.FracDigits), buffer, 0)
} else {
values = enc.NewInstantDeltaCompressor(byte(rec.FracDigits), buffer, 0)
}
s.metrics[rec.MetricID] = &_metric{
metricType: rec.MetricType,
fracDigits: byte(rec.FracDigits),
buffer: buffer,
timestamps: enc.NewTimeDeltaCompressor(buffer, 0),
values: values,
values: enc.NewValueDeltaCompressor(rec.MetricType, byte(rec.FracDigits), buffer, 0),
}
}

7
database/replay_metric.go
View File

@@ -272,7 +272,6 @@ func composeHeadDataPage(databuf []byte, head storage.HeadDataPage) storage.Page
func (s *ReplayMetric) ToMetric() *_metric {
var (
values qb.ValueCompressor
requiredSpace = s.tSize + s.vSize
)
bufferSize := calculateDataBufferSize(requiredSpace)
@@ -284,11 +283,7 @@ func (s *ReplayMetric) ToMetric() *_metric {
NewBufferSize: bufferSize,
})
}
if s.metricType == qb.Cumulative {
values = enc.NewCumulativeDeltaCompressor(s.fracDigits, s.buf, s.vSize)
} else {
values = enc.NewInstantDeltaCompressor(s.fracDigits, s.buf, s.vSize)
}
values := enc.NewValueDeltaCompressor(s.metricType, s.fracDigits, s.buf, s.vSize)
return &_metric{
metricType: s.metricType,
fracDigits: s.fracDigits,