wp
This commit is contained in:
394
worker/metric.go
Normal file
394
worker/metric.go
Normal file
@@ -0,0 +1,394 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
bin "gordenko.dev/dima/bin/little"
|
||||
"gordenko.dev/dima/qb"
|
||||
"gordenko.dev/dima/qb/storage"
|
||||
)
|
||||
|
||||
// METRIC
|
||||
|
||||
var ErrNoValueBug = errors.New("has timestamp but no value")
|
||||
|
||||
type CapturedState struct {
|
||||
LastTimestamp uint32
|
||||
LastValue float64
|
||||
}
|
||||
|
||||
type Metric struct {
|
||||
metricType qb.MetricType
|
||||
fracDigits byte
|
||||
lastPageNo uint32
|
||||
//SinceValue float64
|
||||
//Since uint32
|
||||
lastValue float64
|
||||
//Until uint32
|
||||
buffer []byte
|
||||
timestamps qb.TimestampCompressor
|
||||
values qb.ValueCompressor
|
||||
XLock bool
|
||||
RLocks int
|
||||
WaitQueue []any
|
||||
indexLevelTails []storage.IndexLevelTail // root - last element
|
||||
capturedState *CapturedState
|
||||
}
|
||||
|
||||
func (s *Metric) LastValue() float64 {
|
||||
if s.capturedState != nil {
|
||||
return s.capturedState.LastValue
|
||||
}
|
||||
return s.lastValue
|
||||
}
|
||||
|
||||
func (s *Metric) LastTimestamp() uint32 {
|
||||
if s.capturedState != nil {
|
||||
return s.capturedState.LastTimestamp
|
||||
}
|
||||
return s.timestamps.LastTimestamp()
|
||||
}
|
||||
|
||||
func (s *Metric) OnMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) {
|
||||
//s.timestamps.Reset()
|
||||
//s.values.Reset()
|
||||
s.XLock = false
|
||||
// s.Timestamps.Renew()
|
||||
// s.Values.Renew()
|
||||
|
||||
// s.LastPageNo = 0
|
||||
// s.Since = 0
|
||||
// s.SinceValue = 0
|
||||
// s.Until = 0
|
||||
s.indexLevelTails = nil
|
||||
s.lastPageNo = 0
|
||||
s.lastValue = 0
|
||||
}
|
||||
|
||||
// func (s *_metric) startAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(storage.AppendedMeasures)) {
|
||||
func (s *Metric) StartAppendMeasures(req AppendMeasuresReq, tmp []byte, saveToStorage func(any)) {
|
||||
if s.capturedState != nil {
|
||||
s.WaitQueue = append(s.WaitQueue, req)
|
||||
}
|
||||
var (
|
||||
timestamps = s.timestamps
|
||||
values = s.values
|
||||
|
||||
// офсети на head сторінці
|
||||
timestampsOffset int
|
||||
timestampsRewindOffset int
|
||||
valuesOffset int
|
||||
valuesRewindOffset int
|
||||
|
||||
headTimestamps []byte
|
||||
headValues []byte
|
||||
|
||||
pages []storage.DataPayload
|
||||
|
||||
written int
|
||||
resultCode byte
|
||||
)
|
||||
|
||||
s.capturedState = &CapturedState{
|
||||
LastTimestamp: timestamps.LastTimestamp(),
|
||||
LastValue: s.lastValue,
|
||||
}
|
||||
|
||||
s.timestamps.CaptureState()
|
||||
s.values.CaptureState()
|
||||
|
||||
for idx, measure := range req.Measures {
|
||||
if measure.Timestamp <= s.timestamps.LastTimestamp() {
|
||||
resultCode = ExpiredMeasure
|
||||
break
|
||||
}
|
||||
if s.metricType == qb.Cumulative && measure.Value < s.lastValue {
|
||||
resultCode = NonMonotonicValue
|
||||
break
|
||||
}
|
||||
|
||||
tReport := timestamps.Evaluate(tmp, measure.Timestamp)
|
||||
vReport := values.Evaluate(tmp, measure.Value)
|
||||
|
||||
totalRequiredSpace := tReport.TotalSpace + vReport.TotalSpace
|
||||
|
||||
if totalRequiredSpace <= len(s.buffer) {
|
||||
// якщо на сторінці є місце
|
||||
timestamps.Append(tReport.RewindOffset, tmp[:tReport.ChangeSize], measure.Timestamp)
|
||||
values.Append(vReport.RewindOffset, tmp[7:7+vReport.ChangeSize], measure.Value, vReport.Delta)
|
||||
|
||||
if idx == 0 {
|
||||
timestampsOffset = tReport.Offset
|
||||
timestampsRewindOffset = tReport.RewindOffset
|
||||
valuesOffset = vReport.Offset
|
||||
valuesRewindOffset = vReport.RewindOffset
|
||||
}
|
||||
} else {
|
||||
// сторінка заповнена
|
||||
since := s.timestamps.ReplaceSinceWithUntil()
|
||||
|
||||
if len(pages) == 0 && idx > 0 {
|
||||
// idx > 0 required because page may overflows without append any data
|
||||
headTimestamps = timestamps.Tail(timestampsOffset)
|
||||
headValues = values.Tail(valuesOffset)
|
||||
}
|
||||
|
||||
pages = append(pages, storage.DataPayload{
|
||||
Since: since,
|
||||
Content: s.buffer,
|
||||
TimestampsSize: timestamps.Size(),
|
||||
ValuesSize: values.Size(),
|
||||
})
|
||||
|
||||
buf := make([]byte, storage.DataPageSize)
|
||||
databuf := buf[:storage.DataPagePayloadSize]
|
||||
|
||||
timestamps.ReplaceBuffer(databuf)
|
||||
values.ReplaceBuffer(databuf)
|
||||
|
||||
// renew
|
||||
s.buffer = buf
|
||||
|
||||
timestamps.Append(tReport.RewindOffset, tmp[:tReport.ChangeSize], measure.Timestamp)
|
||||
values.Append(vReport.RewindOffset, tmp[7:7+vReport.ChangeSize], measure.Value, vReport.Delta)
|
||||
}
|
||||
//
|
||||
s.lastValue = measure.Value
|
||||
written++
|
||||
}
|
||||
|
||||
if written == 0 {
|
||||
s.capturedState = nil
|
||||
req.ResultCh <- AppendMeasuresResult{
|
||||
ResultCode: resultCode,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// виділити змінені байти.
|
||||
// скопіювати. Причому можна скопіювати зрізи chunks
|
||||
|
||||
if len(pages) > 0 {
|
||||
// пишу в storage довгим шляхом через redo файл і запис в data файл
|
||||
saveToStorage(storage.MeasuresAppendWithGrow{
|
||||
MetricID: req.MetricID,
|
||||
LastPageNo: s.lastPageNo,
|
||||
TimestampsRewindOffset: timestampsRewindOffset,
|
||||
Timestamps: headTimestamps,
|
||||
ValuesRewindOffset: valuesRewindOffset,
|
||||
Values: headValues,
|
||||
IndexLevelTails: s.indexLevelTails,
|
||||
DataPages: pages,
|
||||
TailTimestamps: timestamps.Tail(0), // payload
|
||||
TailValues: values.Tail(0),
|
||||
ResultCode: resultCode,
|
||||
WrittenCount: written,
|
||||
//ResultCh: req.ResultCh,
|
||||
})
|
||||
} else {
|
||||
// короткий шлях - запис лише в storage
|
||||
saveToStorage(storage.MeasuresAppend{
|
||||
MetricID: req.MetricID,
|
||||
TimestampsRewindOffset: timestampsRewindOffset,
|
||||
ValuesRewindOffset: valuesRewindOffset,
|
||||
Timestamps: timestamps.Tail(timestampsOffset),
|
||||
Values: values.Tail(valuesOffset),
|
||||
ResultCode: resultCode,
|
||||
WrittenCount: written,
|
||||
//ResultCh: req.ResultCh,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Metric) OnMeasuresAppendCommited(rec storage.MeasuresAppendCommited) {
|
||||
// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
|
||||
s.capturedState = nil
|
||||
s.timestamps.ForgetCapturedState()
|
||||
s.values.ForgetCapturedState()
|
||||
}
|
||||
|
||||
func (s *Metric) OnMeasuresAppendWithGrowCommited(rec storage.MeasuresAppendWithGrowCommited) {
|
||||
// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
|
||||
s.capturedState = nil
|
||||
s.timestamps.ForgetCapturedState()
|
||||
s.values.ForgetCapturedState()
|
||||
|
||||
if rec.LastPageNo > 0 {
|
||||
s.lastPageNo = rec.LastPageNo
|
||||
}
|
||||
// В storage я передав повний індекс. У нього додали елементи (можливо нові рівні).
|
||||
// Тому проста заміна
|
||||
s.indexLevelTails = rec.Index
|
||||
}
|
||||
|
||||
// READ
|
||||
|
||||
func (s *Metric) StartRangeScan(req RangeScanReq) {
|
||||
// if s.Since == 0 {
|
||||
// req.ResultCh <- rangeScanResult{
|
||||
// ResultCode: QueryDone,
|
||||
// }
|
||||
// return
|
||||
// }
|
||||
|
||||
// if req.Since > s.Until {
|
||||
// req.ResultCh <- rangeScanResult{
|
||||
// ResultCode: QueryDone,
|
||||
// }
|
||||
// return
|
||||
// }
|
||||
|
||||
// if req.Until < s.Since {
|
||||
// if s.RootPageNo > 0 {
|
||||
// req.ResultCh <- rangeScanResult{
|
||||
// ResultCode: UntilNotFound,
|
||||
// RootPageNo: s.RootPageNo,
|
||||
// FracDigits: s.FracDigits,
|
||||
// }
|
||||
// s.RLocks++
|
||||
// return
|
||||
// } else {
|
||||
// req.ResultCh <- rangeScanResult{
|
||||
// ResultCode: QueryDone,
|
||||
// }
|
||||
// return
|
||||
// }
|
||||
// }
|
||||
|
||||
// timestampDecompressor := s.timestamps.CreateDecompressor()
|
||||
// valueDecompressor := s.values.CreateDecompressor(s.metricType, s.fracDigits)
|
||||
|
||||
// for {
|
||||
// timestamp, done := timestampDecompressor.NextValue()
|
||||
// if done {
|
||||
// break
|
||||
// }
|
||||
// value, done := valueDecompressor.NextValue()
|
||||
// if done {
|
||||
// qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug)
|
||||
// }
|
||||
// if timestamp <= req.Until {
|
||||
// req.ResponseWriter.FeedNoSend(timestamp, value)
|
||||
// if timestamp < req.Since {
|
||||
// req.ResultCh <- rangeScanResult{
|
||||
// ResultCode: QueryDone,
|
||||
// }
|
||||
// return
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// if s.lastPageNo > 0 {
|
||||
// req.ResultCh <- rangeScanResult{
|
||||
// ResultCode: UntilFound,
|
||||
// LastPageNo: s.lastPageNo,
|
||||
// FracDigits: s.fracDigits,
|
||||
// }
|
||||
// s.RLocks++
|
||||
// } else {
|
||||
// req.ResultCh <- rangeScanResult{
|
||||
// ResultCode: QueryDone,
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
func (s *Metric) StartFullScan(req FullScanReq) {
|
||||
// if s.Since == 0 {
|
||||
// req.ResultCh <- fullScanResult{
|
||||
// ResultCode: QueryDone,
|
||||
// }
|
||||
// return
|
||||
// }
|
||||
|
||||
timestampDecompressor := s.timestamps.CreateDecompressor()
|
||||
valueDecompressor := s.values.CreateDecompressor(s.metricType, s.fracDigits)
|
||||
|
||||
for {
|
||||
timestamp, done := timestampDecompressor.NextValue()
|
||||
if done {
|
||||
break
|
||||
}
|
||||
value, done := valueDecompressor.NextValue()
|
||||
if done {
|
||||
qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug)
|
||||
}
|
||||
req.ResponseWriter.FeedNoSend(timestamp, value)
|
||||
}
|
||||
|
||||
if s.lastPageNo > 0 {
|
||||
req.ResultCh <- FullScanResult{
|
||||
ResultCode: UntilFound,
|
||||
LastPageNo: s.lastPageNo,
|
||||
FracDigits: s.fracDigits,
|
||||
}
|
||||
s.RLocks++
|
||||
} else {
|
||||
req.ResultCh <- FullScanResult{
|
||||
ResultCode: QueryDone,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// індекси
|
||||
// Metric encode format:
|
||||
// metricID - 4b
|
||||
// metricType - 1b
|
||||
// fracDigits - 1b
|
||||
// lastPageNo - 4b
|
||||
// timestamps size - 2b
|
||||
// values size - 2b
|
||||
// timestams payload - Nb
|
||||
// values payload - Nb
|
||||
// index levels count - varsize
|
||||
// [
|
||||
// records qty - varsize
|
||||
// records - Nb
|
||||
// ]
|
||||
|
||||
func (s *Metric) WriteTo(w io.Writer) (err error) {
|
||||
_, err = w.Write([]byte{
|
||||
byte(s.metricType),
|
||||
s.fracDigits,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = bin.WriteUint32(w, s.lastPageNo)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = bin.WriteUint16(w, uint16(s.timestamps.StoredSize()))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = bin.WriteUint16(w, uint16(s.values.StoredSize()))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// timestamps payload
|
||||
err = s.timestamps.WriteStoredTo(w)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// values payload
|
||||
err = s.values.WriteStoredTo(w)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// indexes
|
||||
_, err = bin.WriteVarSize(w, len(s.indexLevelTails))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, level := range s.indexLevelTails {
|
||||
_, err = bin.WriteVarSize(w, level.RecordsCount)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_, err = w.Write(level.Buffer)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
Reference in New Issue
Block a user