dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-06-19 07:25:46 +03:00
parent 06823ee547
commit 2d73ce1727
52 changed files with 5173 additions and 547 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
worker/metric.go
View File

@@ -2,12 +2,13 @@ package worker
import (
"errors"
"fmt"
"io"
"time"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/inbox"
"gordenko.dev/dima/qb/proto"
"gordenko.dev/dima/qb/storage"
)
@@ -119,37 +120,53 @@ func (s *Metric) ReleaseRLock() {
s.rLocks--
if s.rLocks == 0 {
if len(s.waitQueue) > 0 {
s.ProcessQueue()
//s.ProcessQueue()
}
}
}
// суть у тому що треба запускати запити, пока не зустріну XLock
func (s *Metric) ProcessQueue(metricID uint32, tmp []byte) {
if len(s.waitQueue) == 0 {
return
type GetMetricResult struct {
MetricType qb.MetricType
FracDigits byte
ResultCode byte
}
type GetMetricReq struct {
MetricID uint32
ResultCh chan GetMetricResult
}
func (s *Metric) GetMetric(req GetMetricReq) {
req.ResultCh <- GetMetricResult{
ResultCode: Succeed,
MetricType: s.metricType,
FracDigits: s.fracDigits,
}
for _, untyped := range s.waitQueue {
switch req := untyped.(type) {
case RangeScanReq:
s.StartRangeScan(req)
case FullScanReq:
s.StartFullScan(req)
case GetMetricReq:
s.GetMetric(req)
case AppendMeasuresReq:
metric.AppendMeasures(req, tmp, s.storageInbox)
case DeleteMetricReq:
s.DeleteMetric(req)
case DeleteMeasuresReq:
metric.DeleteMeasures(req)
default:
qb.Abort(qb.UnknownMetricWaitQueueItemBug,
fmt.Errorf("bug: unknown metric wait queue item type %T", req))
}
}
func (s *Metric) DeleteMetric(req DeleteMetricReq) {
if s.xLock || s.rLocks > 0 || s.capturedState != nil {
s.waitQueue = append(s.waitQueue, req)
} else {
s.xLock = true
// fix - has pages -> do query from goroutine
// else push to storage
// collect all pages, than
// s.storageInbox.Push(storage.MetricDelete{
// MetricID: req.MetricID,
// FreeIndexPages: nil,
// FreeDataPages: nil,
// ResultCh: req.ResultCh,
// })
}
}
type AppendMeasuresReq struct {
MetricID uint32
Measures []proto.Measure
ResultCh chan storage.MeasuresAppendResult
}
func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox *inbox.Inbox) {
if s.xLock || s.capturedState != nil {
s.waitQueue = append(s.waitQueue, req)
@@ -182,7 +199,9 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
s.values.CaptureState()
for idx, measure := range req.Measures {
//fmt.Println(idx, measure.Timestamp)
if measure.Timestamp <= s.timestamps.Until() {
//fmt.Printf("timestamp %d <= until %d\n", measure.Timestamp, s.timestamps.Until())
resultCode = ExpiredMeasure
break
}
@@ -191,6 +210,8 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
break
}
//fmt.Printf(" %d: %s => %.2f\n", idx, formatTime(measure.Timestamp), measure.Value)
tReport := timestamps.Evaluate(tmp[:7], measure.Timestamp)
//fmt.Printf("tReport: %#v\n", tReport)
vReport := values.Evaluate(tmp[7:], measure.Value)
@@ -198,7 +219,7 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
totalRequiredSpace := tReport.TotalSpace + vReport.TotalSpace
if totalRequiredSpace <= len(s.buffer) {
if totalRequiredSpace <= storage.DataPagePayloadSize {
// якщо на сторінці є місце
timestamps.Append(tReport.RewindOffset, tmp[:tReport.ChangeSize], measure.Timestamp)
values.Append(vReport.RewindOffset, tmp[7:7+vReport.ChangeSize], measure.Value, vReport.Delta)
@@ -210,7 +231,8 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
valuesRewindOffset = vReport.RewindOffset
}
} else {
fmt.Println("PAGE FILLED")
//fmt.Println("PAGE FILLED")
//fmt.Println(timestamps.Size() + values.Size())
// сторінка заповнена
since := s.timestamps.ReplaceSinceWithUntil()
@@ -227,6 +249,9 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
ValuesSize: values.Size(),
})
//xxx := timestamps.Tail(0)
//fmt.Printf("PAGE timestamps %d:\n% x\n", len(xxx), xxx)
buf := make([]byte, storage.DataPageSize)
databuf := buf[:storage.DataPagePayloadSize]
@@ -257,10 +282,14 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
return
}
//xxx := timestamps.Tail(0)
//fmt.Printf("PAGE TAIL timestamps %d:\n% x\n", len(xxx), xxx)
// виділити змінені байти.
// скопіювати. Причому можна скопіювати зрізи chunks
if len(pages) > 0 {
//fmt.Printf("WITH GROW")
// пишу в storage довгим шляхом через redo файл і запис в data файл
storageInbox.Push(storage.MeasuresAppendWithGrow{
MetricID: req.MetricID,
@@ -278,6 +307,7 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
ResultCh: req.ResultCh,
})
} else {
//fmt.Printf("SIMPLE")
// короткий шлях - запис лише в storage
storageInbox.Push(storage.MeasuresAppend{
MetricID: req.MetricID,
@@ -292,6 +322,12 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
}
}
type DeleteMeasuresReq struct {
MetricID uint32
Since uint32
ResultCh chan byte
}
func (s *Metric) DeleteMeasures(req DeleteMeasuresReq) {
// if s.xLock || s.capturedState != nil {
// s.waitQueue = append(s.waitQueue, req)
@@ -314,7 +350,23 @@ func (s *Metric) DeleteMeasures(req DeleteMeasuresReq) {
// }
}
func (s *Metric) StartRangeScan(req RangeScanReq) {
type RangeScanResult struct {
ResultCode byte
FracDigits byte
LastPageNo uint32
IsDataPage bool // for UntilNotFound only
}
type RangeScanReq struct {
MetricID uint32
Since uint32
Until uint32
MetricType qb.MetricType
ResponseWriter qb.WorkerMeasureConsumer
ResultCh chan RangeScanResult
}
func (s *Metric) RangeScan(req RangeScanReq) {
if s.xLock {
s.waitQueue = append(s.waitQueue, req)
return
@@ -397,7 +449,23 @@ func (s *Metric) StartRangeScan(req RangeScanReq) {
}
}
func (s *Metric) StartFullScan(req FullScanReq) {
type FullScanResult struct {
ResultCode byte
FracDigits byte
LastPageNo uint32
}
type FullScanReq struct {
MetricID uint32
MetricType qb.MetricType
ResponseWriter qb.WorkerMeasureConsumer
ResultCh chan FullScanResult
}
func (s *Metric) FullScan(req FullScanReq) {
//fmt.Println("lastPageNo:", s.lastPageNo)
//fmt.Printf("index: %#v\n", s.indexLevelTails)
if s.xLock {
s.waitQueue = append(s.waitQueue, req)
return
@@ -410,18 +478,20 @@ func (s *Metric) StartFullScan(req FullScanReq) {
}
timestampDecompressor := s.timestamps.CreateDecompressor()
valueDecompressor := s.values.CreateDecompressor(s.metricType, s.fracDigits)
idx := 0
for {
timestamp, done := timestampDecompressor.NextValue()
if done {
//fmt.Println("DONE")
break
}
//fmt.Println("ts:", timestamp)
value, done := valueDecompressor.NextValue()
if done {
qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug)
}
//fmt.Println("value:", value)
req.ResponseWriter.FeedNoSend(timestamp, value)
//fmt.Printf(" %d: %s => %.2f\n", idx, formatTime(timestamp), value)
idx++
}
if s.lastPageNo > 0 {
@@ -441,6 +511,7 @@ func (s *Metric) StartFullScan(req FullScanReq) {
// COMMITS
func (s *Metric) OnMeasuresAppendCommited(rec storage.MeasuresAppendCommited) {
//fmt.Println("OnMeasuresAppendCommited: written=", rec.WrittenCount)
// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
s.capturedState = nil
s.timestamps.ForgetCapturedState()
@@ -453,6 +524,8 @@ func (s *Metric) OnMeasuresAppendCommited(rec storage.MeasuresAppendCommited) {
}
func (s *Metric) OnMeasuresAppendWithGrowCommited(rec storage.MeasuresAppendWithGrowCommited) {
//fmt.Println("OnMeasuresAppendWithGrowCommited: written=", rec.WrittenCount, ", lastPageNo=", rec.LastPageNo)
//fmt.Printf("index: %#v\n", rec.Index)
// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
s.capturedState = nil
s.timestamps.ForgetCapturedState()
@@ -480,4 +553,16 @@ func (s *Metric) OnMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) {
s.indexLevelTails = nil
s.lastPageNo = 0
s.lastValue = 0
}
// const (
// free
// )
const datetimeLayout = "2006-01-02 15:04:05"
func formatTime(timestamp uint32) string {
tm := time.Unix(int64(timestamp), 0)
return tm.Format(datetimeLayout)
}