package worker import ( "errors" "fmt" "io" bin "gordenko.dev/dima/bin/little" "gordenko.dev/dima/qb" "gordenko.dev/dima/qb/inbox" "gordenko.dev/dima/qb/storage" ) // METRIC var ErrNoValueBug = errors.New("has timestamp but no value") type CapturedState struct { LastTimestamp uint32 LastValue float64 } type Metric struct { metricType qb.MetricType fracDigits byte lastPageNo uint32 //SinceValue float64 //Since uint32 lastValue float64 //Until uint32 buffer []byte timestamps qb.TimestampCompressor values qb.ValueCompressor xLock bool rLocks int waitQueue []any indexLevelTails []storage.IndexLevelTail // root - last element capturedState *CapturedState } // індекси // Metric encode format: // metricID - 4b // metricType - 1b // fracDigits - 1b // lastPageNo - 4b // timestamps size - 2b // values size - 2b // timestams payload - Nb // values payload - Nb // index levels count - varsize // [ // records qty - varsize // records - Nb // ] func (s *Metric) WriteTo(w io.Writer) (err error) { _, err = w.Write([]byte{ byte(s.metricType), s.fracDigits, }) if err != nil { return } err = bin.WriteUint32(w, s.lastPageNo) if err != nil { return } err = bin.WriteUint16(w, uint16(s.timestamps.CommitedSize())) if err != nil { return } err = bin.WriteUint16(w, uint16(s.values.CommitedSize())) if err != nil { return } // timestamps payload err = s.timestamps.WriteCommitedTo(w) if err != nil { return } // values payload err = s.values.WriteCommitedTo(w) if err != nil { return } // indexes _, err = bin.WriteVarSize(w, len(s.indexLevelTails)) if err != nil { return } for _, level := range s.indexLevelTails { _, err = bin.WriteVarSize(w, level.RecordsCount) if err != nil { return } _, err = w.Write(level.Buffer[:level.RecordsCount*storage.IndexRecordSize]) if err != nil { return } } return } func (s *Metric) MetricType() qb.MetricType { return s.metricType } func (s *Metric) FracDigits() byte { return s.fracDigits } func (s *Metric) LastValue() float64 { if s.capturedState != nil { return s.capturedState.LastValue } return s.lastValue } func (s *Metric) LastTimestamp() uint32 { if s.capturedState != nil { return s.capturedState.LastTimestamp } return s.timestamps.LastTimestamp() } // REQUESTS func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox *inbox.Inbox) { if s.xLock || s.capturedState != nil { s.waitQueue = append(s.waitQueue, req) return } var ( timestamps = s.timestamps values = s.values // офсети на head сторінці timestampsOffset int timestampsRewindOffset int valuesOffset int valuesRewindOffset int headTimestamps []byte headValues []byte pages []storage.DataPayload written int resultCode byte = Succeed ) s.capturedState = &CapturedState{ LastTimestamp: timestamps.LastTimestamp(), LastValue: s.lastValue, } s.timestamps.CaptureState() s.values.CaptureState() for idx, measure := range req.Measures { if measure.Timestamp <= s.timestamps.LastTimestamp() { resultCode = ExpiredMeasure break } if s.metricType == qb.Cumulative && measure.Value < s.lastValue { resultCode = NonMonotonicValue break } tReport := timestamps.Evaluate(tmp[:7], measure.Timestamp) //fmt.Printf("tReport: %#v\n", tReport) vReport := values.Evaluate(tmp[7:], measure.Value) //fmt.Printf("vReport: %#v\n", vReport) totalRequiredSpace := tReport.TotalSpace + vReport.TotalSpace if totalRequiredSpace <= len(s.buffer) { // якщо на сторінці є місце timestamps.Append(tReport.RewindOffset, tmp[:tReport.ChangeSize], measure.Timestamp) values.Append(vReport.RewindOffset, tmp[7:7+vReport.ChangeSize], measure.Value, vReport.Delta) if idx == 0 { timestampsOffset = tReport.Offset timestampsRewindOffset = tReport.RewindOffset valuesOffset = vReport.Offset valuesRewindOffset = vReport.RewindOffset } } else { fmt.Println("PAGE FILLED") // сторінка заповнена since := s.timestamps.ReplaceSinceWithUntil() if len(pages) == 0 && idx > 0 { // idx > 0 required because page may overflows without append any data headTimestamps = timestamps.Tail(timestampsOffset) headValues = values.Tail(valuesOffset) } pages = append(pages, storage.DataPayload{ Since: since, Content: s.buffer, TimestampsSize: timestamps.Size(), ValuesSize: values.Size(), }) buf := make([]byte, storage.DataPageSize) databuf := buf[:storage.DataPagePayloadSize] timestamps.ReplaceBuffer(databuf) values.ReplaceBuffer(databuf) // renew s.buffer = buf tReport = timestamps.Evaluate(tmp[:7], measure.Timestamp) //fmt.Printf("tReport: %#v\n", tReport) vReport = values.Evaluate(tmp[7:], measure.Value) //fmt.Printf("vReport: %#v\n", vReport) timestamps.Append(tReport.RewindOffset, tmp[:tReport.ChangeSize], measure.Timestamp) values.Append(vReport.RewindOffset, tmp[7:7+vReport.ChangeSize], measure.Value, vReport.Delta) } // s.lastValue = measure.Value written++ } if written == 0 { s.capturedState = nil req.ResultCh <- storage.MeasuresAppendResult{ ResultCode: resultCode, } return } // виділити змінені байти. // скопіювати. Причому можна скопіювати зрізи chunks if len(pages) > 0 { // пишу в storage довгим шляхом через redo файл і запис в data файл storageInbox.Push(storage.MeasuresAppendWithGrow{ MetricID: req.MetricID, LastPageNo: s.lastPageNo, TimestampsRewindOffset: timestampsRewindOffset, Timestamps: headTimestamps, ValuesRewindOffset: valuesRewindOffset, Values: headValues, IndexLevelTails: s.indexLevelTails, DataPages: pages, TailTimestamps: timestamps.Tail(0), // payload TailValues: values.Tail(0), ResultCode: resultCode, WrittenCount: written, ResultCh: req.ResultCh, }) } else { // короткий шлях - запис лише в storage storageInbox.Push(storage.MeasuresAppend{ MetricID: req.MetricID, TimestampsRewindOffset: timestampsRewindOffset, ValuesRewindOffset: valuesRewindOffset, Timestamps: timestamps.Tail(timestampsOffset), Values: values.Tail(valuesOffset), ResultCode: resultCode, WrittenCount: written, ResultCh: req.ResultCh, }) } } func (s *Metric) DeleteMeasures(req DeleteMeasuresReq) { if s.xLock || s.capturedState != nil { s.waitQueue = append(s.waitQueue, req) return } since := s.timestamps.FirstTimestamp() until := s.timestamps.LastTimestamp() if since == 0 || (req.Since > 0 && until < req.Since) { req.ResultCh <- NoMeasuresToDelete } // if s.RootPageNo > 0 { // req.ResultCh <- tryDeleteMeasuresResult{ // ResultCode: DeleteFromAtreeRequired, // RootPageNo: metric.RootPageNo, // } // } else { // req.ResultCh <- tryDeleteMeasuresResult{ // ResultCode: DeleteFromAtreeNotNeeded, // } // } } func (s *Metric) StartRangeScan(req RangeScanReq) { if s.xLock { s.waitQueue = append(s.waitQueue, req) return } // if s.timestamps.CommitedSize() == 0 { // req.ResultCh <- RangeScanResult{ // ResultCode: QueryDone, // } // return // } // if req.Since > s.timestamps.LastTimestamp() { // req.ResultCh <- RangeScanResult{ // ResultCode: QueryDone, // } // return // } // if req.Until < s.Since { // if s.RootPageNo > 0 { // req.ResultCh <- RangeScanResult{ // ResultCode: UntilNotFound, // RootPageNo: s.RootPageNo, // FracDigits: s.fracDigits, // } // s.rLocks++ // return // } else { // req.ResultCh <- RangeScanResult{ // ResultCode: QueryDone, // } // return // } // } timestampDecompressor := s.timestamps.CreateDecompressor() valueDecompressor := s.values.CreateDecompressor(s.metricType, s.fracDigits) for { timestamp, done := timestampDecompressor.NextValue() if done { break } value, done := valueDecompressor.NextValue() if done { qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug) } if timestamp <= req.Until { req.ResponseWriter.FeedNoSend(timestamp, value) if timestamp < req.Since { req.ResultCh <- RangeScanResult{ ResultCode: QueryDone, } return } } } if s.lastPageNo > 0 { req.ResultCh <- RangeScanResult{ ResultCode: UntilFound, LastPageNo: s.lastPageNo, FracDigits: s.fracDigits, } s.rLocks++ } else { req.ResultCh <- RangeScanResult{ ResultCode: QueryDone, } } } func (s *Metric) StartFullScan(req FullScanReq) { if s.xLock { s.waitQueue = append(s.waitQueue, req) return } if s.timestamps.CommitedSize() == 0 { req.ResultCh <- FullScanResult{ ResultCode: QueryDone, } return } timestampDecompressor := s.timestamps.CreateDecompressor() valueDecompressor := s.values.CreateDecompressor(s.metricType, s.fracDigits) for { timestamp, done := timestampDecompressor.NextValue() if done { break } //fmt.Println("ts:", timestamp) value, done := valueDecompressor.NextValue() if done { qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug) } //fmt.Println("value:", value) req.ResponseWriter.FeedNoSend(timestamp, value) } if s.lastPageNo > 0 { req.ResultCh <- FullScanResult{ ResultCode: UntilFound, LastPageNo: s.lastPageNo, FracDigits: s.fracDigits, } s.rLocks++ } else { req.ResultCh <- FullScanResult{ ResultCode: QueryDone, } } } // COMMITS func (s *Metric) OnMeasuresAppendCommited(rec storage.MeasuresAppendCommited) { // Видаляю state. Оригінальні Timestamps і Values вже мають останню версію s.capturedState = nil s.timestamps.ForgetCapturedState() s.values.ForgetCapturedState() rec.ResultCh <- storage.MeasuresAppendResult{ ResultCode: rec.ResultCode, WrittenCount: rec.WrittenCount, } } func (s *Metric) OnMeasuresAppendWithGrowCommited(rec storage.MeasuresAppendWithGrowCommited) { // Видаляю state. Оригінальні Timestamps і Values вже мають останню версію s.capturedState = nil s.timestamps.ForgetCapturedState() s.values.ForgetCapturedState() if rec.LastPageNo > 0 { s.lastPageNo = rec.LastPageNo } // В storage я передав повний індекс. У нього додали елементи (можливо нові рівні). // Тому проста заміна s.indexLevelTails = rec.Index rec.ResultCh <- storage.MeasuresAppendResult{ ResultCode: rec.ResultCode, WrittenCount: rec.WrittenCount, } } func (s *Metric) OnMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) { //s.timestamps.Reset() //s.values.Reset() s.xLock = false // s.Timestamps.Renew() // s.Values.Renew() // s.LastPageNo = 0 // s.Since = 0 // s.SinceValue = 0 // s.Until = 0 s.indexLevelTails = nil s.lastPageNo = 0 s.lastValue = 0 }