package database import ( "io" bin "gordenko.dev/dima/bin/little" "gordenko.dev/dima/qb" "gordenko.dev/dima/qb/storage" ) // METRIC //const minBufferSize = 1024 var ( indexRecordSize = 8 ) type CapturedState struct { LastTimestamp uint32 LastValue float64 } type _metric struct { metricType qb.MetricType fracDigits byte lastPageNo uint32 //SinceValue float64 //Since uint32 lastValue float64 //Until uint32 buffer []byte timestamps qb.TimestampCompressor values qb.ValueCompressor XLock bool RLocks int WaitQueue []any indexLevelTails []storage.IndexLevelTail // root - last element capturedState *CapturedState } func (s *_metric) LastValue() float64 { if s.capturedState != nil { return s.capturedState.LastValue } return s.lastValue } func (s *_metric) LastTimestamp() uint32 { if s.capturedState != nil { return s.capturedState.LastTimestamp } return s.timestamps.LastTimestamp() } func (s *_metric) OnMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) { //s.timestamps.Reset() //s.values.Reset() s.XLock = false // s.Timestamps.Renew() // s.Values.Renew() // s.LastPageNo = 0 // s.Since = 0 // s.SinceValue = 0 // s.Until = 0 s.indexLevelTails = nil s.lastPageNo = 0 s.lastValue = 0 } // func (s *_metric) startAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(storage.AppendedMeasures)) { func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, tmp []byte, saveToStorage func(any)) { if s.capturedState != nil { s.WaitQueue = append(s.WaitQueue, req) } var ( timestamps = s.timestamps values = s.values // офсети на head сторінці timestampsOffset int timestampsRewindOffset int valuesOffset int valuesRewindOffset int headTimestamps []byte headValues []byte pages []storage.DataPayload written int resultCode byte ) s.capturedState = &CapturedState{ LastTimestamp: timestamps.LastTimestamp(), LastValue: s.lastValue, } s.timestamps.CaptureState() s.values.CaptureState() for idx, measure := range req.Measures { if measure.Timestamp <= s.timestamps.LastTimestamp() { resultCode = ExpiredMeasure break } if s.metricType == qb.Cumulative && measure.Value < s.lastValue { resultCode = NonMonotonicValue break } tReport := timestamps.Evaluate(tmp, measure.Timestamp) vReport := values.Evaluate(tmp, measure.Value) totalRequiredSpace := tReport.TotalSpace + vReport.TotalSpace if totalRequiredSpace <= len(s.buffer) { // якщо на сторінці є місце timestamps.Append(tReport.RewindOffset, tmp[:tReport.ChangeSize], measure.Timestamp) values.Append(vReport.RewindOffset, tmp[7:7+vReport.ChangeSize], measure.Value, vReport.Delta) if idx == 0 { timestampsOffset = tReport.Offset timestampsRewindOffset = tReport.RewindOffset valuesOffset = vReport.Offset valuesRewindOffset = vReport.RewindOffset } } else { // сторінка заповнена since := s.timestamps.ReplaceSinceWithUntil() if len(pages) == 0 && idx > 0 { // idx > 0 required because page may overflows without append any data headTimestamps = timestamps.Tail(timestampsOffset) headValues = values.Tail(valuesOffset) } pages = append(pages, storage.DataPayload{ Since: since, Content: s.buffer, TimestampsSize: timestamps.Size(), ValuesSize: values.Size(), }) buf := make([]byte, storage.DataPageSize) databuf := buf[:storage.DataPagePayloadSize] timestamps.ReplaceBuffer(databuf) values.ReplaceBuffer(databuf) // renew s.buffer = buf timestamps.Append(tReport.RewindOffset, tmp[:tReport.ChangeSize], measure.Timestamp) values.Append(vReport.RewindOffset, tmp[7:7+vReport.ChangeSize], measure.Value, vReport.Delta) } // s.lastValue = measure.Value written++ } if written == 0 { s.capturedState = nil req.ResultCh <- tryAppendMeasuresResult{ ResultCode: resultCode, } return } // виділити змінені байти. // скопіювати. Причому можна скопіювати зрізи chunks if len(pages) > 0 { // пишу в storage довгим шляхом через redo файл і запис в data файл saveToStorage(storage.MeasuresAppendWithGrow{ MetricID: req.MetricID, LastPageNo: s.lastPageNo, TimestampsRewindOffset: timestampsRewindOffset, Timestamps: headTimestamps, ValuesRewindOffset: valuesRewindOffset, Values: headValues, IndexLevelTails: s.indexLevelTails, DataPages: pages, TailTimestamps: timestamps.Tail(0), // payload TailValues: values.Tail(0), ResultCode: resultCode, WrittenCount: written, //ResultCh: req.ResultCh, }) } else { // короткий шлях - запис лише в storage saveToStorage(storage.MeasuresAppend{ MetricID: req.MetricID, TimestampsRewindOffset: timestampsRewindOffset, ValuesRewindOffset: valuesRewindOffset, Timestamps: timestamps.Tail(timestampsOffset), Values: values.Tail(valuesOffset), ResultCode: resultCode, WrittenCount: written, //ResultCh: req.ResultCh, }) } } func (s *_metric) OnMeasuresAppendCommited(rec storage.MeasuresAppendCommited) { // Видаляю state. Оригінальні Timestamps і Values вже мають останню версію s.capturedState = nil s.timestamps.ForgetCapturedState() s.values.ForgetCapturedState() } func (s *_metric) OnMeasuresAppendWithGrowCommited(rec storage.MeasuresAppendWithGrowCommited) { // Видаляю state. Оригінальні Timestamps і Values вже мають останню версію s.capturedState = nil s.timestamps.ForgetCapturedState() s.values.ForgetCapturedState() if rec.LastPageNo > 0 { s.lastPageNo = rec.LastPageNo } // В storage я передав повний індекс. У нього додали елементи (можливо нові рівні). // Тому проста заміна s.indexLevelTails = rec.Index } // READ func (s *_metric) StartRangeScan(req tryRangeScanReq) { // if s.Since == 0 { // req.ResultCh <- rangeScanResult{ // ResultCode: QueryDone, // } // return // } // if req.Since > s.Until { // req.ResultCh <- rangeScanResult{ // ResultCode: QueryDone, // } // return // } // if req.Until < s.Since { // if s.RootPageNo > 0 { // req.ResultCh <- rangeScanResult{ // ResultCode: UntilNotFound, // RootPageNo: s.RootPageNo, // FracDigits: s.FracDigits, // } // s.RLocks++ // return // } else { // req.ResultCh <- rangeScanResult{ // ResultCode: QueryDone, // } // return // } // } // timestampDecompressor := s.timestamps.CreateDecompressor() // valueDecompressor := s.values.CreateDecompressor(s.metricType, s.fracDigits) // for { // timestamp, done := timestampDecompressor.NextValue() // if done { // break // } // value, done := valueDecompressor.NextValue() // if done { // qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug) // } // if timestamp <= req.Until { // req.ResponseWriter.FeedNoSend(timestamp, value) // if timestamp < req.Since { // req.ResultCh <- rangeScanResult{ // ResultCode: QueryDone, // } // return // } // } // } // if s.lastPageNo > 0 { // req.ResultCh <- rangeScanResult{ // ResultCode: UntilFound, // LastPageNo: s.lastPageNo, // FracDigits: s.fracDigits, // } // s.RLocks++ // } else { // req.ResultCh <- rangeScanResult{ // ResultCode: QueryDone, // } // } } func (s *_metric) StartFullScan(req tryFullScanReq) { // if s.Since == 0 { // req.ResultCh <- fullScanResult{ // ResultCode: QueryDone, // } // return // } timestampDecompressor := s.timestamps.CreateDecompressor() valueDecompressor := s.values.CreateDecompressor(s.metricType, s.fracDigits) for { timestamp, done := timestampDecompressor.NextValue() if done { break } value, done := valueDecompressor.NextValue() if done { qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug) } req.ResponseWriter.FeedNoSend(timestamp, value) } if s.lastPageNo > 0 { req.ResultCh <- fullScanResult{ ResultCode: UntilFound, LastPageNo: s.lastPageNo, FracDigits: s.fracDigits, } s.RLocks++ } else { req.ResultCh <- fullScanResult{ ResultCode: QueryDone, } } } // індекси // Metric encode format: // metricID - 4b // metricType - 1b // fracDigits - 1b // lastPageNo - 4b // timestamps size - 2b // values size - 2b // timestams payload - Nb // values payload - Nb // index levels count - varsize // [ // records qty - varsize // records - Nb // ] func (s *_metric) WriteTo(w io.Writer) (err error) { _, err = w.Write([]byte{ byte(s.metricType), s.fracDigits, }) if err != nil { return } err = bin.WriteUint32(w, s.lastPageNo) if err != nil { return } err = bin.WriteUint16(w, uint16(s.timestamps.StoredSize())) if err != nil { return } err = bin.WriteUint16(w, uint16(s.values.StoredSize())) if err != nil { return } // timestamps payload err = s.timestamps.WriteStoredTo(w) if err != nil { return } // values payload err = s.values.WriteStoredTo(w) if err != nil { return } // indexes _, err = bin.WriteVarSize(w, len(s.indexLevelTails)) if err != nil { return } for _, level := range s.indexLevelTails { _, err = bin.WriteVarSize(w, level.RecordsCount) if err != nil { return } _, err = w.Write(level.Buffer) if err != nil { return } } return }