dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-06-14 23:12:03 +03:00
parent 181031a753
commit b32279deaf
35 changed files with 1395 additions and 1062 deletions
Download Patch File Download Diff File Expand all files Collapse all files

160
worker/worker.go
View File

@@ -19,28 +19,27 @@ import (
// Якийсь прапор поставити для pendingDelete, щоб read операції вставали в чергу
const (
QueryDone = 1
UntilFound = 2
UntilNotFound = 3
RangeFound = 15
NoMeasures = 16
NoMetric = 4
MetricDuplicate = 5
Succeed = 6
NewPage = 7
ExpiredMeasure = 8
NonMonotonicValue = 9
CanAppend = 10
WrongMetricType = 11
NoMeasuresToDelete = 12
DeleteFromAtreeNotNeeded = 13
DeleteFromAtreeRequired = 14
QueryDone = 1
UntilFound = 2
UntilNotFound = 3
RangeFound = 15
NoMeasures = 16
NoMetric = 4
MetricDuplicate = 5
Succeed = 6
ExpiredMeasure = 8
NonMonotonicValue = 9
WrongMetricType = 11
NoMeasuresToDelete = 12
//DeleteFromAtreeNotNeeded = 13
//DeleteFromAtreeRequired = 14
)
type Worker struct {
inbox *inbox.Inbox
storageInbox *inbox.Inbox
dir string
databaseName string
snapshotNumber int
tmp []byte
metrics map[uint32]*Metric
@@ -53,6 +52,7 @@ type Options struct {
StorageInbox *inbox.Inbox
ReplayMetrics map[uint32]*storage.ReplayMetric
Dir string
DatabaseName string
ExitCh chan struct{}
WaitGroup *sync.WaitGroup
}
@@ -62,6 +62,7 @@ func New(opt Options) *Worker {
inbox: opt.Inbox,
storageInbox: opt.StorageInbox,
dir: opt.Dir,
databaseName: opt.DatabaseName,
tmp: make([]byte, 26),
metrics: make(map[uint32]*Metric),
exitCh: opt.ExitCh,
@@ -151,13 +152,13 @@ func (s *Worker) doWork() {
s.applyCommits(req) // all metrics only
case ListCurrentValuesReq:
s.tryListCurrentValues(req) // all metrics only
s.ListCurrentValues(req) // all metrics only
case RangeScanReq:
s.RangeScan(req)
case FullScanReq:
s.tryFullScan(req)
s.FullScan(req)
case AddMetricReq:
s.AddMetric(req)
@@ -196,7 +197,7 @@ func (s *Worker) processMetricQueue(metricID uint32, metric *Metric, tmp []byte)
s.GetMetric(req)
case AppendMeasuresReq:
metric.StartAppendMeasures(req, tmp, s.storageInbox)
metric.AppendMeasures(req, tmp, s.storageInbox)
case DeleteMetricReq:
s.startDeleteMetric(metric, req)
@@ -212,28 +213,37 @@ func (s *Worker) processMetricQueue(metricID uint32, metric *Metric, tmp []byte)
}
type AddMetricReq struct {
MetricID uint32
ResultCh chan byte
MetricID uint32
MetricType qb.MetricType
FracDigits byte
ResultCh chan byte
}
func (s *Worker) AddMetric(req AddMetricReq) {
_, ok := s.metrics[req.MetricID]
if ok {
fmt.Println("add metric duplicate")
req.ResultCh <- MetricDuplicate
return
}
fmt.Println("metric not found. Success")
req.ResultCh <- Succeed // new
// lockEntry, ok := s.metricLockEntries[req.MetricID]
// if ok {
// lockEntry.WaitQueue = append(lockEntry.WaitQueue, req)
// } else {
// s.metricLockEntries[req.MetricID] = &metricLockEntry{
// XLock: true,
// }
// req.ResultCh <- Succeed
// }
var (
buf = make([]byte, storage.DataPageSize)
databuf = buf[:storage.DataPagePayloadSize]
)
s.metrics[req.MetricID] = &Metric{
metricType: req.MetricType,
fracDigits: req.FracDigits,
buffer: buf,
timestamps: enc.NewTimeDeltaCompressor(databuf, 0),
values: enc.NewValueDeltaCompressor(req.MetricType, req.FracDigits, databuf, 0),
xLock: true,
}
s.storageInbox.Push(storage.MetricAdd{
MetricID: req.MetricID,
MetricType: req.MetricType,
FracDigits: req.FracDigits,
ResultCh: req.ResultCh,
})
}
func (s *Worker) processTryAddMetricReqsImmediatelyAfterDelete(reqs []AddMetricReq) {
@@ -379,7 +389,7 @@ func (s *Worker) onMeasuresAppendCommited(rec storage.MeasuresAppendCommited) {
metric, ok := s.metrics[rec.MetricID]
if !ok {
qb.Abort(qb.NoMetricBug,
fmt.Errorf("finAppendMeasures: metric %d not found",
fmt.Errorf("onMeasuresAppendCommited: metric %d not found",
rec.MetricID))
}
metric.OnMeasuresAppendCommited(rec)
@@ -395,30 +405,25 @@ func (s *Worker) onMeasuresAppendWithGrowCommited(rec storage.MeasuresAppendWith
metric.OnMeasuresAppendWithGrowCommited(rec)
}
type AppendMeasuresResult struct {
ResultCode byte
Written int
}
type AppendMeasuresReq struct {
MetricID uint32
Measures []proto.Measure
ResultCh chan AppendMeasuresResult
ResultCh chan storage.MeasuresAppendResult
}
func (s *Worker) AppendMeasures(req AppendMeasuresReq) {
metric, ok := s.metrics[req.MetricID]
if !ok {
req.ResultCh <- AppendMeasuresResult{
req.ResultCh <- storage.MeasuresAppendResult{
ResultCode: NoMetric,
}
return
}
if metric.XLock {
if metric.xLock {
metric.WaitQueue = append(metric.WaitQueue, req)
return
}
metric.StartAppendMeasures(req, s.tmp, s.storageInbox)
metric.AppendMeasures(req, s.tmp, s.storageInbox)
}
type RangeScanResult struct {
@@ -452,7 +457,7 @@ func (s *Worker) RangeScan(req RangeScanReq) {
return
}
if metric.XLock {
if metric.xLock {
metric.WaitQueue = append(metric.WaitQueue, req)
return
}
@@ -473,7 +478,7 @@ type FullScanReq struct {
ResultCh chan FullScanResult
}
func (s *Worker) tryFullScan(req FullScanReq) {
func (s *Worker) FullScan(req FullScanReq) {
metric, ok := s.metrics[req.MetricID]
if !ok {
req.ResultCh <- FullScanResult{
@@ -488,7 +493,7 @@ func (s *Worker) tryFullScan(req FullScanReq) {
return
}
if metric.XLock {
if metric.xLock {
metric.WaitQueue = append(metric.WaitQueue, req)
return
}
@@ -501,7 +506,7 @@ type ListCurrentValuesReq struct {
ResultCh chan struct{}
}
func (s *Worker) tryListCurrentValues(req ListCurrentValuesReq) {
func (s *Worker) ListCurrentValues(req ListCurrentValuesReq) {
for _, metricID := range req.MetricIDs {
metric, ok := s.metrics[metricID]
if ok {
@@ -539,55 +544,32 @@ func (s *Worker) applyCommits(req storage.Changes) {
if req.SnapshotNumberCh != nil {
s.snapshotNumber++
err := WriteSnapshot(WriteSnapshotIn{
SnapshotNumber: s.snapshotNumber,
Dir: s.dir,
WriteBufferSize: 4 * 1024 * 1024, // 1mb
Metrics: s.metrics,
FrozenIndexPagesCount: req.FrozenIndexPagesCount,
IndexPageNumbers: req.IndexPageNumbers,
FrozenDataPagesCount: req.FrozenDataPagesCount,
DataPageNumbers: req.DataPageNumbers,
})
err := writeSnapshot(
qb.GetSnapshotFilePath(s.dir, s.databaseName, s.snapshotNumber),
WriteSnapshotIn{
SnapshotNumber: s.snapshotNumber,
WriteBufferSize: 4 * 1024 * 1024, // 4mb
Metrics: s.metrics,
FrozenIndexPagesCount: req.FrozenIndexPagesCount,
IndexPageNumbers: req.IndexPageNumbers,
FrozenDataPagesCount: req.FrozenDataPagesCount,
DataPageNumbers: req.DataPageNumbers,
})
if err != nil {
qb.Abort(qb.WriteSnapshotFailed, err)
}
req.SnapshotNumberCh <- s.snapshotNumber
}
}
func (s *Worker) onMetricAddCommited(rec storage.MetricAddCommited) {
// fix lock
_, ok := s.metrics[rec.MetricID]
if ok {
metric, ok := s.metrics[rec.MetricID]
if !ok {
qb.Abort(qb.MetricAddedBug,
fmt.Errorf("addMetric: metric %d already added",
rec.MetricID))
}
// if !lockEntry.XLock {
// qb.Abort(qb.NoXLockBug,
// fmt.Errorf("addMetric: xlock not set for the metric %d",
// rec.MetricID))
// }
// lockEntry.XLock = false
// delete(s.metricLockEntries, rec.MetricID)
}
func (s *Worker) addMetric(rec storage.MetricAddRecord) {
var (
buf = make([]byte, storage.DataPageSize)
databuf = buf[:storage.DataPagePayloadSize]
)
s.metrics[rec.MetricID] = &Metric{
metricType: rec.MetricType,
fracDigits: rec.FracDigits,
buffer: buf,
timestamps: enc.NewTimeDeltaCompressor(databuf, 0),
values: enc.NewValueDeltaCompressor(rec.MetricType, rec.FracDigits, databuf, 0),
fmt.Errorf("metric %d not found after commit", rec.MetricID))
}
metric.xLock = false
rec.ResultCh <- Succeed // new
}
func (s *Worker) onMetricDeleteCommited(rec storage.MetricDeleteCommited) {
@@ -598,7 +580,7 @@ func (s *Worker) onMetricDeleteCommited(rec storage.MetricDeleteCommited) {
rec.MetricID))
}
if !metric.XLock {
if !metric.xLock {
qb.Abort(qb.NoXLockBug,
fmt.Errorf("deleteMetric: xlock not set for the metric %d",
rec.MetricID))
@@ -669,13 +651,13 @@ func (s *Worker) onMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) {
rec.MetricID))
}
if !metric.XLock {
if !metric.xLock {
qb.Abort(qb.NoXLockBug,
fmt.Errorf("deleteMeasures: xlock not set for the metric %d",
rec.MetricID))
}
metric.OnMeasuresDeleteCommited(rec)
metric.XLock = false
metric.xLock = false
// FIX add in storage
// if len(rec.FreePageNumbers) > 0 {
// s.freeList.AddPages(rec.FreePageNumbers)