484 lines
12 KiB
Go
484 lines
12 KiB
Go
package worker
|
||
|
||
import (
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
|
||
bin "gordenko.dev/dima/bin/little"
|
||
"gordenko.dev/dima/qb"
|
||
"gordenko.dev/dima/qb/inbox"
|
||
"gordenko.dev/dima/qb/storage"
|
||
)
|
||
|
||
// METRIC
|
||
|
||
var ErrNoValueBug = errors.New("has timestamp but no value")
|
||
|
||
type CapturedState struct {
|
||
LastValue float64
|
||
}
|
||
|
||
type Metric struct {
|
||
metricType qb.MetricType
|
||
fracDigits byte
|
||
lastPageNo uint32
|
||
lastValue float64
|
||
buffer []byte
|
||
timestamps qb.TimestampCompressor
|
||
values qb.ValueCompressor
|
||
xLock bool
|
||
rLocks int
|
||
waitQueue []any
|
||
indexLevelTails []storage.IndexLevelTail // root - last element
|
||
capturedState *CapturedState
|
||
}
|
||
|
||
// індекси
|
||
// Metric encode format:
|
||
// metricID - 4b
|
||
// metricType - 1b
|
||
// fracDigits - 1b
|
||
// lastPageNo - 4b
|
||
// timestamps size - 2b
|
||
// values size - 2b
|
||
// timestams payload - Nb
|
||
// values payload - Nb
|
||
// index levels count - varsize
|
||
// [
|
||
// records qty - varsize
|
||
// records - Nb
|
||
// ]
|
||
|
||
func (s *Metric) WriteTo(w io.Writer) (err error) {
|
||
_, err = w.Write([]byte{
|
||
byte(s.metricType),
|
||
s.fracDigits,
|
||
})
|
||
if err != nil {
|
||
return
|
||
}
|
||
err = bin.WriteUint32(w, s.lastPageNo)
|
||
if err != nil {
|
||
return
|
||
}
|
||
err = bin.WriteUint16(w, uint16(s.timestamps.CommitedSize()))
|
||
if err != nil {
|
||
return
|
||
}
|
||
err = bin.WriteUint16(w, uint16(s.values.CommitedSize()))
|
||
if err != nil {
|
||
return
|
||
}
|
||
// timestamps payload
|
||
err = s.timestamps.WriteCommitedTo(w)
|
||
if err != nil {
|
||
return
|
||
}
|
||
// values payload
|
||
err = s.values.WriteCommitedTo(w)
|
||
if err != nil {
|
||
return
|
||
}
|
||
// indexes
|
||
_, err = bin.WriteVarSize(w, len(s.indexLevelTails))
|
||
if err != nil {
|
||
return
|
||
}
|
||
for _, level := range s.indexLevelTails {
|
||
_, err = bin.WriteVarSize(w, level.RecordsCount)
|
||
if err != nil {
|
||
return
|
||
}
|
||
_, err = w.Write(level.Buffer[:level.RecordsCount*storage.IndexRecordSize])
|
||
if err != nil {
|
||
return
|
||
}
|
||
}
|
||
return
|
||
}
|
||
|
||
func (s *Metric) MetricType() qb.MetricType {
|
||
return s.metricType
|
||
}
|
||
|
||
func (s *Metric) FracDigits() byte {
|
||
return s.fracDigits
|
||
}
|
||
|
||
func (s *Metric) LastValue() float64 {
|
||
if s.capturedState != nil {
|
||
return s.capturedState.LastValue
|
||
}
|
||
return s.lastValue
|
||
}
|
||
|
||
// REQUESTS
|
||
|
||
func (s *Metric) ReleaseRLock() {
|
||
s.rLocks--
|
||
if s.rLocks == 0 {
|
||
if len(s.waitQueue) > 0 {
|
||
s.ProcessQueue()
|
||
}
|
||
}
|
||
}
|
||
|
||
// суть у тому що треба запускати запити, пока не зустріну XLock
|
||
func (s *Metric) ProcessQueue(metricID uint32, tmp []byte) {
|
||
if len(s.waitQueue) == 0 {
|
||
return
|
||
}
|
||
for _, untyped := range s.waitQueue {
|
||
switch req := untyped.(type) {
|
||
case RangeScanReq:
|
||
s.StartRangeScan(req)
|
||
case FullScanReq:
|
||
s.StartFullScan(req)
|
||
case GetMetricReq:
|
||
s.GetMetric(req)
|
||
case AppendMeasuresReq:
|
||
metric.AppendMeasures(req, tmp, s.storageInbox)
|
||
case DeleteMetricReq:
|
||
s.DeleteMetric(req)
|
||
case DeleteMeasuresReq:
|
||
metric.DeleteMeasures(req)
|
||
default:
|
||
qb.Abort(qb.UnknownMetricWaitQueueItemBug,
|
||
fmt.Errorf("bug: unknown metric wait queue item type %T", req))
|
||
}
|
||
}
|
||
}
|
||
|
||
func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox *inbox.Inbox) {
|
||
if s.xLock || s.capturedState != nil {
|
||
s.waitQueue = append(s.waitQueue, req)
|
||
return
|
||
}
|
||
var (
|
||
timestamps = s.timestamps
|
||
values = s.values
|
||
|
||
// офсети на head сторінці
|
||
timestampsOffset int
|
||
timestampsRewindOffset int
|
||
valuesOffset int
|
||
valuesRewindOffset int
|
||
|
||
headTimestamps []byte
|
||
headValues []byte
|
||
|
||
pages []storage.DataPayload
|
||
|
||
written int
|
||
resultCode byte = Succeed
|
||
)
|
||
|
||
s.capturedState = &CapturedState{
|
||
LastValue: s.lastValue,
|
||
}
|
||
|
||
s.timestamps.CaptureState()
|
||
s.values.CaptureState()
|
||
|
||
for idx, measure := range req.Measures {
|
||
if measure.Timestamp <= s.timestamps.Until() {
|
||
resultCode = ExpiredMeasure
|
||
break
|
||
}
|
||
if s.metricType == qb.Cumulative && measure.Value < s.lastValue {
|
||
resultCode = NonMonotonicValue
|
||
break
|
||
}
|
||
|
||
tReport := timestamps.Evaluate(tmp[:7], measure.Timestamp)
|
||
//fmt.Printf("tReport: %#v\n", tReport)
|
||
vReport := values.Evaluate(tmp[7:], measure.Value)
|
||
//fmt.Printf("vReport: %#v\n", vReport)
|
||
|
||
totalRequiredSpace := tReport.TotalSpace + vReport.TotalSpace
|
||
|
||
if totalRequiredSpace <= len(s.buffer) {
|
||
// якщо на сторінці є місце
|
||
timestamps.Append(tReport.RewindOffset, tmp[:tReport.ChangeSize], measure.Timestamp)
|
||
values.Append(vReport.RewindOffset, tmp[7:7+vReport.ChangeSize], measure.Value, vReport.Delta)
|
||
|
||
if idx == 0 {
|
||
timestampsOffset = tReport.Offset
|
||
timestampsRewindOffset = tReport.RewindOffset
|
||
valuesOffset = vReport.Offset
|
||
valuesRewindOffset = vReport.RewindOffset
|
||
}
|
||
} else {
|
||
fmt.Println("PAGE FILLED")
|
||
// сторінка заповнена
|
||
since := s.timestamps.ReplaceSinceWithUntil()
|
||
|
||
if len(pages) == 0 && idx > 0 {
|
||
// idx > 0 required because page may overflows without append any data
|
||
headTimestamps = timestamps.Tail(timestampsOffset)
|
||
headValues = values.Tail(valuesOffset)
|
||
}
|
||
|
||
pages = append(pages, storage.DataPayload{
|
||
Since: since,
|
||
Content: s.buffer,
|
||
TimestampsSize: timestamps.Size(),
|
||
ValuesSize: values.Size(),
|
||
})
|
||
|
||
buf := make([]byte, storage.DataPageSize)
|
||
databuf := buf[:storage.DataPagePayloadSize]
|
||
|
||
timestamps.ReplaceBuffer(databuf)
|
||
values.ReplaceBuffer(databuf)
|
||
|
||
// renew
|
||
s.buffer = buf
|
||
|
||
tReport = timestamps.Evaluate(tmp[:7], measure.Timestamp)
|
||
//fmt.Printf("tReport: %#v\n", tReport)
|
||
vReport = values.Evaluate(tmp[7:], measure.Value)
|
||
//fmt.Printf("vReport: %#v\n", vReport)
|
||
|
||
timestamps.Append(tReport.RewindOffset, tmp[:tReport.ChangeSize], measure.Timestamp)
|
||
values.Append(vReport.RewindOffset, tmp[7:7+vReport.ChangeSize], measure.Value, vReport.Delta)
|
||
}
|
||
//
|
||
s.lastValue = measure.Value
|
||
written++
|
||
}
|
||
|
||
if written == 0 {
|
||
s.capturedState = nil
|
||
req.ResultCh <- storage.MeasuresAppendResult{
|
||
ResultCode: resultCode,
|
||
}
|
||
return
|
||
}
|
||
|
||
// виділити змінені байти.
|
||
// скопіювати. Причому можна скопіювати зрізи chunks
|
||
|
||
if len(pages) > 0 {
|
||
// пишу в storage довгим шляхом через redo файл і запис в data файл
|
||
storageInbox.Push(storage.MeasuresAppendWithGrow{
|
||
MetricID: req.MetricID,
|
||
LastPageNo: s.lastPageNo,
|
||
TimestampsRewindOffset: timestampsRewindOffset,
|
||
Timestamps: headTimestamps,
|
||
ValuesRewindOffset: valuesRewindOffset,
|
||
Values: headValues,
|
||
IndexLevelTails: s.indexLevelTails,
|
||
DataPages: pages,
|
||
TailTimestamps: timestamps.Tail(0), // payload
|
||
TailValues: values.Tail(0),
|
||
ResultCode: resultCode,
|
||
WrittenCount: written,
|
||
ResultCh: req.ResultCh,
|
||
})
|
||
} else {
|
||
// короткий шлях - запис лише в storage
|
||
storageInbox.Push(storage.MeasuresAppend{
|
||
MetricID: req.MetricID,
|
||
TimestampsRewindOffset: timestampsRewindOffset,
|
||
ValuesRewindOffset: valuesRewindOffset,
|
||
Timestamps: timestamps.Tail(timestampsOffset),
|
||
Values: values.Tail(valuesOffset),
|
||
ResultCode: resultCode,
|
||
WrittenCount: written,
|
||
ResultCh: req.ResultCh,
|
||
})
|
||
}
|
||
}
|
||
|
||
func (s *Metric) DeleteMeasures(req DeleteMeasuresReq) {
|
||
// if s.xLock || s.capturedState != nil {
|
||
// s.waitQueue = append(s.waitQueue, req)
|
||
// return
|
||
// }
|
||
// since := s.timestamps.FirstTimestamp()
|
||
// until := s.timestamps.LastTimestamp()
|
||
// if since == 0 || (req.Since > 0 && until < req.Since) {
|
||
// req.ResultCh <- NoMeasuresToDelete
|
||
// }
|
||
// if s.RootPageNo > 0 {
|
||
// req.ResultCh <- tryDeleteMeasuresResult{
|
||
// ResultCode: DeleteFromAtreeRequired,
|
||
// RootPageNo: metric.RootPageNo,
|
||
// }
|
||
// } else {
|
||
// req.ResultCh <- tryDeleteMeasuresResult{
|
||
// ResultCode: DeleteFromAtreeNotNeeded,
|
||
// }
|
||
// }
|
||
}
|
||
|
||
func (s *Metric) StartRangeScan(req RangeScanReq) {
|
||
if s.xLock {
|
||
s.waitQueue = append(s.waitQueue, req)
|
||
return
|
||
}
|
||
since := s.timestamps.CommitedSince()
|
||
if since == 0 {
|
||
req.ResultCh <- RangeScanResult{
|
||
ResultCode: QueryDone,
|
||
}
|
||
return
|
||
}
|
||
// range after
|
||
if req.Since > s.timestamps.CommitedUntil() {
|
||
req.ResultCh <- RangeScanResult{
|
||
ResultCode: QueryDone,
|
||
}
|
||
return
|
||
}
|
||
// range before
|
||
if req.Until < since {
|
||
if len(s.indexLevelTails) > 0 {
|
||
pageNo, isDataPage := storage.FindPageOnIndexLevelTails(s.indexLevelTails, req.Until)
|
||
if pageNo > 0 {
|
||
req.ResultCh <- RangeScanResult{
|
||
ResultCode: UntilNotFound,
|
||
FracDigits: s.fracDigits,
|
||
LastPageNo: pageNo,
|
||
IsDataPage: isDataPage,
|
||
}
|
||
s.rLocks++
|
||
return
|
||
} else {
|
||
// range until before 1st measure timestamp
|
||
req.ResultCh <- RangeScanResult{
|
||
ResultCode: QueryDone,
|
||
}
|
||
return
|
||
}
|
||
} else {
|
||
req.ResultCh <- RangeScanResult{
|
||
ResultCode: QueryDone,
|
||
}
|
||
return
|
||
}
|
||
}
|
||
|
||
timestampDecompressor := s.timestamps.CreateDecompressor()
|
||
valueDecompressor := s.values.CreateDecompressor(s.metricType, s.fracDigits)
|
||
|
||
for {
|
||
timestamp, done := timestampDecompressor.NextValue()
|
||
if done {
|
||
break
|
||
}
|
||
value, done := valueDecompressor.NextValue()
|
||
if done {
|
||
qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug)
|
||
}
|
||
if timestamp <= req.Until {
|
||
req.ResponseWriter.FeedNoSend(timestamp, value)
|
||
if timestamp < req.Since {
|
||
req.ResultCh <- RangeScanResult{
|
||
ResultCode: QueryDone,
|
||
}
|
||
return
|
||
}
|
||
}
|
||
}
|
||
if s.lastPageNo > 0 {
|
||
req.ResultCh <- RangeScanResult{
|
||
ResultCode: UntilFound,
|
||
LastPageNo: s.lastPageNo,
|
||
FracDigits: s.fracDigits,
|
||
}
|
||
s.rLocks++
|
||
} else {
|
||
req.ResultCh <- RangeScanResult{
|
||
ResultCode: QueryDone,
|
||
}
|
||
}
|
||
}
|
||
|
||
func (s *Metric) StartFullScan(req FullScanReq) {
|
||
if s.xLock {
|
||
s.waitQueue = append(s.waitQueue, req)
|
||
return
|
||
}
|
||
if s.timestamps.CommitedSize() == 0 {
|
||
req.ResultCh <- FullScanResult{
|
||
ResultCode: QueryDone,
|
||
}
|
||
return
|
||
}
|
||
timestampDecompressor := s.timestamps.CreateDecompressor()
|
||
valueDecompressor := s.values.CreateDecompressor(s.metricType, s.fracDigits)
|
||
for {
|
||
timestamp, done := timestampDecompressor.NextValue()
|
||
if done {
|
||
break
|
||
}
|
||
//fmt.Println("ts:", timestamp)
|
||
value, done := valueDecompressor.NextValue()
|
||
if done {
|
||
qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug)
|
||
}
|
||
//fmt.Println("value:", value)
|
||
req.ResponseWriter.FeedNoSend(timestamp, value)
|
||
}
|
||
|
||
if s.lastPageNo > 0 {
|
||
req.ResultCh <- FullScanResult{
|
||
ResultCode: UntilFound,
|
||
LastPageNo: s.lastPageNo,
|
||
FracDigits: s.fracDigits,
|
||
}
|
||
s.rLocks++
|
||
} else {
|
||
req.ResultCh <- FullScanResult{
|
||
ResultCode: QueryDone,
|
||
}
|
||
}
|
||
}
|
||
|
||
// COMMITS
|
||
|
||
func (s *Metric) OnMeasuresAppendCommited(rec storage.MeasuresAppendCommited) {
|
||
// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
|
||
s.capturedState = nil
|
||
s.timestamps.ForgetCapturedState()
|
||
s.values.ForgetCapturedState()
|
||
|
||
rec.ResultCh <- storage.MeasuresAppendResult{
|
||
ResultCode: rec.ResultCode,
|
||
WrittenCount: rec.WrittenCount,
|
||
}
|
||
}
|
||
|
||
func (s *Metric) OnMeasuresAppendWithGrowCommited(rec storage.MeasuresAppendWithGrowCommited) {
|
||
// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
|
||
s.capturedState = nil
|
||
s.timestamps.ForgetCapturedState()
|
||
s.values.ForgetCapturedState()
|
||
|
||
if rec.LastPageNo > 0 {
|
||
s.lastPageNo = rec.LastPageNo
|
||
}
|
||
// В storage я передав повний індекс. У нього додали елементи (можливо нові рівні).
|
||
// Тому проста заміна
|
||
s.indexLevelTails = rec.Index
|
||
|
||
rec.ResultCh <- storage.MeasuresAppendResult{
|
||
ResultCode: rec.ResultCode,
|
||
WrittenCount: rec.WrittenCount,
|
||
}
|
||
}
|
||
|
||
func (s *Metric) OnMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) {
|
||
//s.timestamps.Reset()
|
||
//s.values.Reset()
|
||
s.xLock = false
|
||
// s.Timestamps.Renew()
|
||
// s.Values.Renew()
|
||
s.indexLevelTails = nil
|
||
s.lastPageNo = 0
|
||
s.lastValue = 0
|
||
}
|