dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-06-19 07:25:46 +03:00
parent 06823ee547
commit 2d73ce1727
52 changed files with 5173 additions and 547 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
worker/metric.go
View File

@@ -2,12 +2,13 @@ package worker
import (
"errors"
"fmt"
"io"
"time"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/inbox"
"gordenko.dev/dima/qb/proto"
"gordenko.dev/dima/qb/storage"
)
@@ -119,37 +120,53 @@ func (s *Metric) ReleaseRLock() {
s.rLocks--
if s.rLocks == 0 {
if len(s.waitQueue) > 0 {
s.ProcessQueue()
//s.ProcessQueue()
}
}
}
// суть у тому що треба запускати запити, пока не зустріну XLock
func (s *Metric) ProcessQueue(metricID uint32, tmp []byte) {
if len(s.waitQueue) == 0 {
return
type GetMetricResult struct {
MetricType qb.MetricType
FracDigits byte
ResultCode byte
}
type GetMetricReq struct {
MetricID uint32
ResultCh chan GetMetricResult
}
func (s *Metric) GetMetric(req GetMetricReq) {
req.ResultCh <- GetMetricResult{
ResultCode: Succeed,
MetricType: s.metricType,
FracDigits: s.fracDigits,
}
for _, untyped := range s.waitQueue {
switch req := untyped.(type) {
case RangeScanReq:
s.StartRangeScan(req)
case FullScanReq:
s.StartFullScan(req)
case GetMetricReq:
s.GetMetric(req)
case AppendMeasuresReq:
metric.AppendMeasures(req, tmp, s.storageInbox)
case DeleteMetricReq:
s.DeleteMetric(req)
case DeleteMeasuresReq:
metric.DeleteMeasures(req)
default:
qb.Abort(qb.UnknownMetricWaitQueueItemBug,
fmt.Errorf("bug: unknown metric wait queue item type %T", req))
}
}
func (s *Metric) DeleteMetric(req DeleteMetricReq) {
if s.xLock || s.rLocks > 0 || s.capturedState != nil {
s.waitQueue = append(s.waitQueue, req)
} else {
s.xLock = true
// fix - has pages -> do query from goroutine
// else push to storage
// collect all pages, than
// s.storageInbox.Push(storage.MetricDelete{
// MetricID: req.MetricID,
// FreeIndexPages: nil,
// FreeDataPages: nil,
// ResultCh: req.ResultCh,
// })
}
}
type AppendMeasuresReq struct {
MetricID uint32
Measures []proto.Measure
ResultCh chan storage.MeasuresAppendResult
}
func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox *inbox.Inbox) {
if s.xLock || s.capturedState != nil {
s.waitQueue = append(s.waitQueue, req)
@@ -182,7 +199,9 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
s.values.CaptureState()
for idx, measure := range req.Measures {
//fmt.Println(idx, measure.Timestamp)
if measure.Timestamp <= s.timestamps.Until() {
//fmt.Printf("timestamp %d <= until %d\n", measure.Timestamp, s.timestamps.Until())
resultCode = ExpiredMeasure
break
}
@@ -191,6 +210,8 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
break
}
//fmt.Printf(" %d: %s => %.2f\n", idx, formatTime(measure.Timestamp), measure.Value)
tReport := timestamps.Evaluate(tmp[:7], measure.Timestamp)
//fmt.Printf("tReport: %#v\n", tReport)
vReport := values.Evaluate(tmp[7:], measure.Value)
@@ -198,7 +219,7 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
totalRequiredSpace := tReport.TotalSpace + vReport.TotalSpace
if totalRequiredSpace <= len(s.buffer) {
if totalRequiredSpace <= storage.DataPagePayloadSize {
// якщо на сторінці є місце
timestamps.Append(tReport.RewindOffset, tmp[:tReport.ChangeSize], measure.Timestamp)
values.Append(vReport.RewindOffset, tmp[7:7+vReport.ChangeSize], measure.Value, vReport.Delta)
@@ -210,7 +231,8 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
valuesRewindOffset = vReport.RewindOffset
}
} else {
fmt.Println("PAGE FILLED")
//fmt.Println("PAGE FILLED")
//fmt.Println(timestamps.Size() + values.Size())
// сторінка заповнена
since := s.timestamps.ReplaceSinceWithUntil()
@@ -227,6 +249,9 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
ValuesSize: values.Size(),
})
//xxx := timestamps.Tail(0)
//fmt.Printf("PAGE timestamps %d:\n% x\n", len(xxx), xxx)
buf := make([]byte, storage.DataPageSize)
databuf := buf[:storage.DataPagePayloadSize]
@@ -257,10 +282,14 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
return
}
//xxx := timestamps.Tail(0)
//fmt.Printf("PAGE TAIL timestamps %d:\n% x\n", len(xxx), xxx)
// виділити змінені байти.
// скопіювати. Причому можна скопіювати зрізи chunks
if len(pages) > 0 {
//fmt.Printf("WITH GROW")
// пишу в storage довгим шляхом через redo файл і запис в data файл
storageInbox.Push(storage.MeasuresAppendWithGrow{
MetricID: req.MetricID,
@@ -278,6 +307,7 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
ResultCh: req.ResultCh,
})
} else {
//fmt.Printf("SIMPLE")
// короткий шлях - запис лише в storage
storageInbox.Push(storage.MeasuresAppend{
MetricID: req.MetricID,
@@ -292,6 +322,12 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
}
}
type DeleteMeasuresReq struct {
MetricID uint32
Since uint32
ResultCh chan byte
}
func (s *Metric) DeleteMeasures(req DeleteMeasuresReq) {
// if s.xLock || s.capturedState != nil {
// s.waitQueue = append(s.waitQueue, req)
@@ -314,7 +350,23 @@ func (s *Metric) DeleteMeasures(req DeleteMeasuresReq) {
// }
}
func (s *Metric) StartRangeScan(req RangeScanReq) {
type RangeScanResult struct {
ResultCode byte
FracDigits byte
LastPageNo uint32
IsDataPage bool // for UntilNotFound only
}
type RangeScanReq struct {
MetricID uint32
Since uint32
Until uint32
MetricType qb.MetricType
ResponseWriter qb.WorkerMeasureConsumer
ResultCh chan RangeScanResult
}
func (s *Metric) RangeScan(req RangeScanReq) {
if s.xLock {
s.waitQueue = append(s.waitQueue, req)
return
@@ -397,7 +449,23 @@ func (s *Metric) StartRangeScan(req RangeScanReq) {
}
}
func (s *Metric) StartFullScan(req FullScanReq) {
type FullScanResult struct {
ResultCode byte
FracDigits byte
LastPageNo uint32
}
type FullScanReq struct {
MetricID uint32
MetricType qb.MetricType
ResponseWriter qb.WorkerMeasureConsumer
ResultCh chan FullScanResult
}
func (s *Metric) FullScan(req FullScanReq) {
//fmt.Println("lastPageNo:", s.lastPageNo)
//fmt.Printf("index: %#v\n", s.indexLevelTails)
if s.xLock {
s.waitQueue = append(s.waitQueue, req)
return
@@ -410,18 +478,20 @@ func (s *Metric) StartFullScan(req FullScanReq) {
}
timestampDecompressor := s.timestamps.CreateDecompressor()
valueDecompressor := s.values.CreateDecompressor(s.metricType, s.fracDigits)
idx := 0
for {
timestamp, done := timestampDecompressor.NextValue()
if done {
//fmt.Println("DONE")
break
}
//fmt.Println("ts:", timestamp)
value, done := valueDecompressor.NextValue()
if done {
qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug)
}
//fmt.Println("value:", value)
req.ResponseWriter.FeedNoSend(timestamp, value)
//fmt.Printf(" %d: %s => %.2f\n", idx, formatTime(timestamp), value)
idx++
}
if s.lastPageNo > 0 {
@@ -441,6 +511,7 @@ func (s *Metric) StartFullScan(req FullScanReq) {
// COMMITS
func (s *Metric) OnMeasuresAppendCommited(rec storage.MeasuresAppendCommited) {
//fmt.Println("OnMeasuresAppendCommited: written=", rec.WrittenCount)
// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
s.capturedState = nil
s.timestamps.ForgetCapturedState()
@@ -453,6 +524,8 @@ func (s *Metric) OnMeasuresAppendCommited(rec storage.MeasuresAppendCommited) {
}
func (s *Metric) OnMeasuresAppendWithGrowCommited(rec storage.MeasuresAppendWithGrowCommited) {
//fmt.Println("OnMeasuresAppendWithGrowCommited: written=", rec.WrittenCount, ", lastPageNo=", rec.LastPageNo)
//fmt.Printf("index: %#v\n", rec.Index)
// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
s.capturedState = nil
s.timestamps.ForgetCapturedState()
@@ -480,4 +553,16 @@ func (s *Metric) OnMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) {
s.indexLevelTails = nil
s.lastPageNo = 0
s.lastValue = 0
}
// const (
// free
// )
const datetimeLayout = "2006-01-02 15:04:05"
func formatTime(timestamp uint32) string {
tm := time.Unix(int64(timestamp), 0)
return tm.Format(datetimeLayout)
}

22
worker/snapshot.go
View File

@@ -177,7 +177,7 @@ func ReadSnapshot(filePath string) (out ReadSnapshotOut, err error) {
if err != nil {
return
}
fmt.Println("metricID", metricID)
//fmt.Println("metricID", metricID)
metricType, err = bin.ReadByte(src)
if err != nil {
return
@@ -187,13 +187,13 @@ func ReadSnapshot(filePath string) (out ReadSnapshotOut, err error) {
if err != nil {
return
}
fmt.Println("m.MetricType", m.MetricType)
fmt.Println("m.FracDigits", m.FracDigits)
//fmt.Println("m.MetricType", m.MetricType)
//fmt.Println("m.FracDigits", m.FracDigits)
m.LastPageNo, err = bin.ReadUint32(src)
if err != nil {
return
}
fmt.Println("m.LastPageNo", m.LastPageNo)
//fmt.Println("m.LastPageNo", m.LastPageNo)
m.TimestampsSize, err = bin.ReadUint16AsInt(src)
if err != nil {
return
@@ -202,7 +202,7 @@ func ReadSnapshot(filePath string) (out ReadSnapshotOut, err error) {
if err != nil {
return
}
fmt.Println("m.TimestampsSize", m.TimestampsSize, "m.ValuesSize", m.ValuesSize)
//fmt.Println("m.TimestampsSize", m.TimestampsSize, "m.ValuesSize", m.ValuesSize)
err = bin.ReadNInto(src, buf[storage.DataPagePayloadSize-m.TimestampsSize:storage.DataPagePayloadSize])
if err != nil {
return
@@ -211,13 +211,13 @@ func ReadSnapshot(filePath string) (out ReadSnapshotOut, err error) {
if err != nil {
return
}
fmt.Printf("% x\n", buf)
//fmt.Printf("% x\n", buf)
//
levelsCount, err = bin.ReadVarSize(src)
if err != nil {
return
}
fmt.Println("levels count:", levelsCount)
//fmt.Println("levels count:", levelsCount)
for range levelsCount {
var (
buf = make([]byte, storage.IndexPageSize)
@@ -227,12 +227,12 @@ func ReadSnapshot(filePath string) (out ReadSnapshotOut, err error) {
if err != nil {
return
}
fmt.Println("records count:", count)
//fmt.Println("records count:", count)
err = bin.ReadNInto(src, buf[:count*storage.IndexRecordSize])
if err != nil {
return
}
fmt.Printf("% x\n", buf)
//fmt.Printf("% x\n", buf)
m.IndexLevelTails = append(m.IndexLevelTails, storage.IndexLevelTail{
Buffer: buf,
RecordsCount: count,
@@ -278,7 +278,7 @@ func writeFreePages(dst io.Writer, frozenPagesCount int, pageNumbers []uint32) (
if err != nil {
return
}
fmt.Println("write frozenPagesCount:", frozenPagesCount)
//fmt.Println("write frozenPagesCount:", frozenPagesCount)
_, err = bin.WriteVarSize(dst, len(pageNumbers))
if err != nil {
return
@@ -301,7 +301,7 @@ func readFreePages(src io.Reader) (_ int, _ []uint32, err error) {
if err != nil {
return
}
fmt.Println("read frozenPagesCount:", frozenPagesCount)
//fmt.Println("read frozenPagesCount:", frozenPagesCount)
pageNumbersCount, err := bin.ReadVarSize(src)
if err != nil {
return

145
worker/worker.go
View File

@@ -7,7 +7,6 @@ import (
qb "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/enc"
"gordenko.dev/dima/qb/inbox"
"gordenko.dev/dima/qb/proto"
"gordenko.dev/dima/qb/storage"
"gordenko.dev/dima/qb/transform"
)
@@ -81,6 +80,8 @@ func New(opt Options) *Worker {
values: values,
indexLevelTails: x.IndexLevelTails,
}
//fmt.Printf("index: %#v\n", x.IndexLevelTails)
}
return s
}
@@ -109,25 +110,25 @@ func (s *Worker) doWork() {
for _, untyped := range queue {
switch req := untyped.(type) {
case AppendMeasuresReq:
s.AppendMeasures(req)
s.appendMeasures(req)
case ReleaseRLock:
s.releaseRLock(req.MetricID)
case storage.Changes:
s.applyCommits(req) // all metrics only
case ListCurrentValuesReq:
s.ListCurrentValues(req) // all metrics only
s.listCurrentValues(req) // all metrics only
case RangeScanReq:
s.RangeScan(req)
s.rangeScan(req)
case FullScanReq:
s.FullScan(req)
s.fullScan(req)
case AddMetricReq:
s.AddMetric(req)
s.addMetric(req)
case DeleteMetricReq:
s.DeleteMetric(req)
s.deleteMetric(req)
case DeleteMeasuresReq:
s.DeleteMeasures(req)
s.deleteMeasures(req)
case GetMetricReq:
s.GetMetric(req)
s.getMetric(req)
default:
qb.Abort(qb.UnknownWorkerQueueItemBug,
fmt.Errorf("bug: unknown worker queue item type %T", req))
@@ -149,7 +150,7 @@ type AddMetricReq struct {
ResultCh chan byte
}
func (s *Worker) AddMetric(req AddMetricReq) {
func (s *Worker) addMetric(req AddMetricReq) {
_, ok := s.metrics[req.MetricID]
if ok {
req.ResultCh <- MetricDuplicate
@@ -175,30 +176,14 @@ func (s *Worker) AddMetric(req AddMetricReq) {
})
}
type GetMetricResult struct {
MetricType qb.MetricType
FracDigits byte
ResultCode byte
}
type GetMetricReq struct {
MetricID uint32
ResultCh chan GetMetricResult
}
func (s *Worker) GetMetric(req GetMetricReq) {
func (s *Worker) getMetric(req GetMetricReq) {
metric, ok := s.metrics[req.MetricID]
if ok {
req.ResultCh <- GetMetricResult{
ResultCode: Succeed,
MetricType: metric.MetricType(),
FracDigits: metric.FracDigits(),
}
} else {
if !ok {
req.ResultCh <- GetMetricResult{
ResultCode: NoMetric,
}
}
metric.GetMetric(req)
}
type DeleteMetricReq struct {
@@ -206,32 +191,16 @@ type DeleteMetricReq struct {
ResultCh chan byte
}
func (s *Worker) DeleteMetric(req DeleteMetricReq) {
func (s *Worker) deleteMetric(req DeleteMetricReq) {
metric, ok := s.metrics[req.MetricID]
if !ok {
req.ResultCh <- NoMetric
return
}
if metric.xLock {
metric.waitQueue = append(metric.waitQueue, req)
} else {
// collect all pages, than
s.storageInbox.Push(storage.MetricDelete{
MetricID: req.MetricID,
FreeIndexPages: nil,
FreeDataPages: nil,
ResultCh: req.ResultCh,
})
}
metric.DeleteMetric(req)
}
type DeleteMeasuresReq struct {
MetricID uint32
Since uint32
ResultCh chan byte
}
func (s *Worker) DeleteMeasures(req DeleteMeasuresReq) {
func (s *Worker) deleteMeasures(req DeleteMeasuresReq) {
metric, ok := s.metrics[req.MetricID]
if !ok {
req.ResultCh <- NoMetric
@@ -240,13 +209,7 @@ func (s *Worker) DeleteMeasures(req DeleteMeasuresReq) {
metric.DeleteMeasures(req)
}
type AppendMeasuresReq struct {
MetricID uint32
Measures []proto.Measure
ResultCh chan storage.MeasuresAppendResult
}
func (s *Worker) AppendMeasures(req AppendMeasuresReq) {
func (s *Worker) appendMeasures(req AppendMeasuresReq) {
metric, ok := s.metrics[req.MetricID]
if !ok {
req.ResultCh <- storage.MeasuresAppendResult{
@@ -261,23 +224,7 @@ func (s *Worker) AppendMeasures(req AppendMeasuresReq) {
}
}
type RangeScanResult struct {
ResultCode byte
FracDigits byte
LastPageNo uint32
IsDataPage bool // for UntilNotFound only
}
type RangeScanReq struct {
MetricID uint32
Since uint32
Until uint32
MetricType qb.MetricType
ResponseWriter qb.WorkerMeasureConsumer
ResultCh chan RangeScanResult
}
func (s *Worker) RangeScan(req RangeScanReq) {
func (s *Worker) rangeScan(req RangeScanReq) {
metric, ok := s.metrics[req.MetricID]
if !ok {
req.ResultCh <- RangeScanResult{
@@ -294,24 +241,11 @@ func (s *Worker) RangeScan(req RangeScanReq) {
if metric.xLock {
metric.waitQueue = append(metric.waitQueue, req)
} else {
metric.StartRangeScan(req)
metric.RangeScan(req)
}
}
type FullScanResult struct {
ResultCode byte
FracDigits byte
LastPageNo uint32
}
type FullScanReq struct {
MetricID uint32
MetricType qb.MetricType
ResponseWriter qb.WorkerMeasureConsumer
ResultCh chan FullScanResult
}
func (s *Worker) FullScan(req FullScanReq) {
func (s *Worker) fullScan(req FullScanReq) {
metric, ok := s.metrics[req.MetricID]
if !ok {
req.ResultCh <- FullScanResult{
@@ -328,7 +262,7 @@ func (s *Worker) FullScan(req FullScanReq) {
if metric.xLock {
metric.waitQueue = append(metric.waitQueue, req)
} else {
metric.StartFullScan(req)
metric.FullScan(req)
}
}
@@ -338,7 +272,7 @@ type ListCurrentValuesReq struct {
ResultCh chan struct{}
}
func (s *Worker) ListCurrentValues(req ListCurrentValuesReq) {
func (s *Worker) listCurrentValues(req ListCurrentValuesReq) {
for _, metricID := range req.MetricIDs {
metric, ok := s.metrics[metricID]
if ok {
@@ -478,8 +412,31 @@ func (s *Worker) onMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) {
//s.doAfterReleaseXLock(rec.MetricID, metric)
}
// func (s *Database) doAfterReleaseXLock(metricID uint32, metric *_metric) {
// if len(metric.WaitQueue) > 0 {
// s.processMetricQueue(metricID, metric)
// }
// }
// rLock сумісний із append measures,
// xLock ні з чим
// після xLock - вся черга
// після rLock, якщо capturedState == nil - вся черга, оскільки rLock блокує лише xLock задачі
// після capturedState = nil, якщо перша задача - appendMeasures - беру, інакше перевірка rLock
// суть у тому що треба запускати запити, пока не зустріну XLock
func (s *Worker) ProcessQueue(metric *Metric, tmp []byte, storageInbox *inbox.Inbox) {
// for _, untyped := range metric.waitQueue {
// switch req := untyped.(type) {
// case RangeScanReq:
// metric.RangeScan(req)
// case FullScanReq:
// metric.FullScan(req)
// case GetMetricReq:
// metric.GetMetric(req)
// case AppendMeasuresReq:
// metric.AppendMeasures(req, tmp, s.storageInbox)
// case DeleteMetricReq:
// metric.DeleteMetric(req)
// case DeleteMeasuresReq:
// metric.DeleteMeasures(req)
// default:
// qb.Abort(qb.UnknownMetricWaitQueueItemBug,
// fmt.Errorf("bug: unknown metric wait queue item type %T", req))
// }
// }
}