dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
dima
2026-05-10 00:59:47 +00:00
parent 271046231c
commit 9b38586ed8
30 changed files with 3560 additions and 1823 deletions
Download Patch File Download Diff File Expand all files Collapse all files

235
database/api.go
View File

@@ -215,7 +215,7 @@ func (s *Database) AddMetric(req proto.AddMetricReq) uint16 {
switch resultCode {
case Succeed:
waitCh := s.txlog.WriteAddedMetric(txlog.AddedMetric{
waitCh := s.txlog.Append(txlog.AddedMetric{
MetricID: req.MetricID,
MetricType: req.MetricType,
FracDigits: req.FracDigits,
@@ -289,21 +289,18 @@ func (s *Database) DeleteMetric(req proto.DeleteMetricReq) uint16 {
switch result.ResultCode {
case Succeed:
var (
freeDataPages []uint32
freeIndexPages []uint32
freePageNumbers []uint32
)
if result.RootPageNo > 0 {
pageLists, err := s.atree.GetAllPages(result.RootPageNo)
var err error
freePageNumbers, err = s.atree.GetAllPages(result.RootPageNo)
if err != nil {
diploma.Abort(diploma.FailedAtreeRequest, err)
}
freeDataPages = pageLists.DataPages
freeIndexPages = pageLists.IndexPages
}
waitCh := s.txlog.WriteDeletedMetric(txlog.DeletedMetric{
MetricID: req.MetricID,
FreeDataPages: freeDataPages,
FreeIndexPages: freeIndexPages,
waitCh := s.txlog.Append(txlog.DeletedMetric{
MetricID: req.MetricID,
FreePageNumbers: freePageNumbers,
})
<-waitCh
@@ -348,7 +345,7 @@ func (s *Database) AppendMeasure(req proto.AppendMeasureReq) uint16 {
switch result.ResultCode {
case CanAppend:
waitCh := s.txlog.WriteAppendMeasure(txlog.AppendedMeasure{
waitCh := s.txlog.Append(txlog.AppendedMeasure{
MetricID: req.MetricID,
Timestamp: req.Timestamp,
Value: req.Value,
@@ -357,35 +354,58 @@ func (s *Database) AppendMeasure(req proto.AppendMeasureReq) uint16 {
case NewPage:
filled := result.FilledPage
report, err := s.atree.AppendDataPage(atree.AppendDataPageReq{
MetricID: req.MetricID,
Timestamp: req.Timestamp,
Value: req.Value,
Since: filled.Since,
RootPageNo: filled.RootPageNo,
PrevPageNo: filled.PrevPageNo,
var path atree.PathToDataPage
if filled.RootPageNo > 0 {
path, err := s.atree.FindPathToLastPage(filled.RootPageNo)
if err != nil {
// FIX
diploma.Abort(diploma.WriteToAtreeFailed, err)
}
// for _, leg := range path.Legs {
// pagesToRelease = append(pagesToRelease, leg.PageNo)
// }
if path.LastPageNo != filled.PrevPageNo {
diploma.Abort(
diploma.WrongPrevPageNo,
fmt.Errorf("bug: last pageNo %d in tree != prev pageNo %d in _metric",
path.LastPageNo, filled.PrevPageNo),
)
}
}
// report, err := s.atree.AppendDataPages(atree.AppendDataPageReq{
// MetricID: req.MetricID,
// Timestamp: req.Timestamp,
// Value: req.Value,
// Since: filled.Since,
// RootPageNo: filled.RootPageNo,
// PrevPageNo: filled.PrevPageNo,
// TimestampsChunks: filled.TimestampsChunks,
// TimestampsSize: filled.TimestampsSize,
// ValuesChunks: filled.ValuesChunks,
// ValuesSize: filled.ValuesSize,
// })
// if err != nil {
// diploma.Abort(diploma.WriteToAtreeFailed, err)
// }
// _ = report
//waitCh := s.txlog.WriteAppended(path, report)
waitCh := s.txlog.Append(txlog.AppendedPagesReq{
Legs: path.Legs,
MetricID: req.MetricID,
Timestamp: req.Timestamp,
Value: req.Value,
LastPageNo: filled.PrevPageNo,
//Pages: ,
TimestampsChunks: filled.TimestampsChunks,
TimestampsSize: filled.TimestampsSize,
ValuesChunks: filled.ValuesChunks,
ValuesSize: filled.ValuesSize,
})
if err != nil {
diploma.Abort(diploma.WriteToAtreeFailed, err)
}
_ = report
waitCh := s.txlog.WriteAppendedMeasureWithOverflow(
txlog.AppendedMeasureWithOverflow{
// FIX
// MetricID: req.MetricID,
// Timestamp: req.Timestamp,
// Value: req.Value,
// IsDataPageReused: report.IsDataPageReused,
// DataPageNo: report.DataPageNo,
// IsRootChanged: report.IsRootChanged,
// RootPageNo: report.NewRootPageNo,
// ReusedIndexPages: report.ReusedIndexPages,
},
false,
},
)
<-waitCh
@@ -442,7 +462,9 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
until = result.Until
untilValue = result.UntilValue
//
toAppendMeasures []proto.Measure
//toAppendMeasures []proto.Measure
dataPages []atree.NotLinkedDataPage
)
for idx, measure := range req.Measures {
@@ -451,30 +473,32 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
since = measure.Timestamp
} else {
if measure.Timestamp <= until {
if len(toAppendMeasures) > 0 {
waitCh := s.txlog.WriteAppendMeasures(
txlog.AppendedMeasures{
MetricID: req.MetricID,
Measures: toAppendMeasures,
},
false,
)
<-waitCh
}
// пишу в БД що вже встиг закодувати FIX
// if len(toAppendMeasures) > 0 {
// waitCh := s.txlog.WriteAppendMeasures(
// txlog.AppendedMeasures{
// MetricID: req.MetricID,
// Measures: toAppendMeasures,
// },
// false,
// )
// <-waitCh
// }
return proto.ErrExpiredMeasure
}
if result.MetricType == diploma.Cumulative && measure.Value < untilValue {
if len(toAppendMeasures) > 0 {
waitCh := s.txlog.WriteAppendMeasures(
txlog.AppendedMeasures{
MetricID: req.MetricID,
Measures: toAppendMeasures,
},
false,
)
<-waitCh
}
// пишу в БД що вже встиг закодувати FIX
// if len(toAppendMeasures) > 0 {
// waitCh := s.txlog.WriteAppendMeasures(
// txlog.AppendedMeasures{
// MetricID: req.MetricID,
// Measures: toAppendMeasures,
// },
// false,
// )
// <-waitCh
// }
return proto.ErrNonMonotonicValue
}
}
@@ -485,38 +509,40 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
totalSpace := timestamps.Size() + values.Size() + extraSpace
if totalSpace <= atree.DataPagePayloadSize {
// накопичую
timestamps.Append(measure.Timestamp)
values.Append(measure.Value)
toAppendMeasures = append(toAppendMeasures, measure)
//toAppendMeasures = append(toAppendMeasures, measure)
} else {
if len(toAppendMeasures) > 0 {
waitCh := s.txlog.WriteAppendMeasures(
txlog.AppendedMeasures{
MetricID: req.MetricID,
Measures: toAppendMeasures,
},
true,
)
<-waitCh
// сторінка заповнена
// if len(toAppendMeasures) > 0 {
// waitCh := s.txlog.WriteAppendMeasures(
// txlog.AppendedMeasures{
// MetricID: req.MetricID,
// Measures: toAppendMeasures,
// },
// true,
// )
// <-waitCh
toAppendMeasures = nil
}
report, err := s.atree.AppendDataPage(atree.AppendDataPageReq{
MetricID: req.MetricID,
Timestamp: until,
Value: untilValue,
Since: since,
RootPageNo: rootPageNo,
PrevPageNo: prevPageNo,
// toAppendMeasures = nil
// }
pageData := make([]byte, atree.PageSize)
atree.ChunksToNotLinkedDataPage(pageData, atree.ChunksToNotLinkedDataPageReq{
//PrevPageNo: path.LastPageNo,
TimestampsChunks: timestampsBuf.Chunks(),
TimestampsSize: uint16(timestamps.Size()),
ValuesChunks: valuesBuf.Chunks(),
ValuesSize: uint16(values.Size()),
})
if err != nil {
diploma.Abort(diploma.WriteToAtreeFailed, err)
}
_ = report
dataPages = append(dataPages, atree.NotLinkedDataPage{
Since: since,
Data: pageData,
})
// FIX
// prevPageNo = report.DataPageNo
// if report.IsRootChanged {
@@ -540,7 +566,7 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
timestampsBuf = conbuf.New(nil)
valuesBuf = conbuf.New(nil)
timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor(
timestamps = chunkenc.NewReverseTimeDeltaCompressor(
timestampsBuf, 0)
if result.MetricType == diploma.Cumulative {
values = chunkenc.NewReverseCumulativeDeltaCompressor(
@@ -560,16 +586,32 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
untilValue = measure.Value
}
if len(toAppendMeasures) > 0 {
waitCh := s.txlog.WriteAppendMeasures(
txlog.AppendedMeasures{
MetricID: req.MetricID,
Measures: toAppendMeasures,
},
false,
)
<-waitCh
}
waitCh := s.txlog.Append(txlog.AppendedPagesReq{
MetricID: req.MetricID,
Timestamp: until,
Value: untilValue,
LastPageNo: prevPageNo,
//Measures: toAppendMeasures,
},
)
<-waitCh
// report, err := s.atree.AppendDataPage(atree.AppendDataPageReq{})
// if err != nil {
// diploma.Abort(diploma.WriteToAtreeFailed, err)
// }
// _ = report
// if len(toAppendMeasures) > 0 {
// waitCh := s.txlog.WriteAppendMeasures(
// txlog.AppendedMeasures{
// MetricID: req.MetricID,
// Measures: toAppendMeasures,
// },
// false,
// )
// <-waitCh
// }
case NoMetric:
return proto.ErrNoMetric
@@ -602,22 +644,21 @@ func (s *Database) DeleteMeasures(req proto.DeleteMeasuresReq) uint16 {
case DeleteFromAtreeNotNeeded:
// регистрирую удаление в TransactionLog
waitCh := s.txlog.WriteDeletedMeasures(txlog.DeletedMeasures{
waitCh := s.txlog.Append(txlog.DeletedMeasures{
MetricID: req.MetricID,
})
<-waitCh
case DeleteFromAtreeRequired:
// собираю номера всех data и index страниц метрики (типа запись REDO лога).
pageLists, err := s.atree.GetAllPages(req.MetricID)
pageNumbers, err := s.atree.GetAllPages(req.MetricID)
if err != nil {
diploma.Abort(diploma.FailedAtreeRequest, err)
}
// регистрирую удаление в TransactionLog
waitCh := s.txlog.WriteDeletedMeasures(txlog.DeletedMeasures{
MetricID: req.MetricID,
FreeDataPages: pageLists.DataPages,
FreeIndexPages: pageLists.IndexPages,
waitCh := s.txlog.Append(txlog.DeletedMeasures{
MetricID: req.MetricID,
FreePageNumbers: pageNumbers,
})
<-waitCh

53
database/database.go
View File

@@ -109,7 +109,6 @@ func (s *Database) ListenAndServe() (err error) {
s.atree, err = atree.New(atree.Options{
Dir: s.dir,
DatabaseName: s.databaseName,
FreeList: s.freeList,
})
if err != nil {
return fmt.Errorf("atree.New: %s", err)
@@ -169,6 +168,8 @@ func (s *Database) recovery() {
Dir: s.dir,
LogNumber: logNumber,
AppendToWorkerQueue: s.appendJobToWorkerQueue,
FreeList: s.freeList,
Atree: s.atree,
ExitCh: s.exitCh,
WaitGroup: s.waitGroup,
})
@@ -374,11 +375,8 @@ func (s *Database) replayChangesRecord(untyped any) error {
case txlog.DeletedMetric:
delete(s.metrics, rec.MetricID)
if len(rec.FreeDataPages) > 0 {
s.freeList.AddPages(rec.FreeDataPages)
}
if len(rec.FreeIndexPages) > 0 {
s.freeList.AddPages(rec.FreeIndexPages)
if len(rec.FreePageNumbers) > 0 {
s.freeList.AddPages(rec.FreePageNumbers)
}
case txlog.AppendedMeasure:
@@ -413,34 +411,31 @@ func (s *Database) replayChangesRecord(untyped any) error {
}
}
case txlog.AppendedMeasureWithOverflow:
metric, ok := s.metrics[rec.MetricID]
if ok {
metric.ReinitBy(rec.Timestamp, rec.Value)
if rec.IsRootChanged {
metric.RootPageNo = rec.RootPageNo
}
metric.LastPageNo = rec.DataPageNo
// delete free pages
if rec.IsDataPageReused {
s.freeList.DeleteReservedPages([]uint32{
rec.DataPageNo,
})
}
if len(rec.ReusedIndexPages) > 0 {
s.freeList.DeleteReservedPages(rec.ReusedIndexPages)
}
}
// case txlog.AppendedMeasureWithOverflow:
// metric, ok := s.metrics[rec.MetricID]
// if ok {
// metric.ReinitBy(rec.Timestamp, rec.Value)
// if rec.IsRootChanged {
// metric.RootPageNo = rec.RootPageNo
// }
// metric.LastPageNo = rec.DataPageNo
// // delete free pages
// if rec.IsDataPageReused {
// s.freeList.DeleteReservedPages([]uint32{
// rec.DataPageNo,
// })
// }
// if len(rec.ReusedIndexPages) > 0 {
// s.freeList.DeleteReservedPages(rec.ReusedIndexPages)
// }
// }
case txlog.DeletedMeasures:
metric, ok := s.metrics[rec.MetricID]
if ok {
metric.DeleteMeasures()
if len(rec.FreeDataPages) > 0 {
s.freeList.AddPages(rec.FreeDataPages)
}
if len(rec.FreeDataPages) > 0 {
s.freeList.AddPages(rec.FreeIndexPages)
if len(rec.FreePageNumbers) > 0 {
s.freeList.AddPages(rec.FreePageNumbers)
}
}

58
database/metric.go
View File

@@ -2,7 +2,6 @@ package database
import (
octopus "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/chunkenc"
"gordenko.dev/dima/qb/conbuf"
)
@@ -24,43 +23,44 @@ type _metric struct {
}
func (s *_metric) ReinitBy(timestamp uint32, value float64) {
s.TimestampsBuf = conbuf.New(nil)
s.ValuesBuf = conbuf.New(nil)
//
s.Timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor(
s.TimestampsBuf, 0)
// s.TimestampsBuf = conbuf.New(nil)
// s.ValuesBuf = conbuf.New(nil)
// //
// s.Timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor(
// s.TimestampsBuf, 0)
if s.MetricType == octopus.Cumulative {
s.Values = chunkenc.NewReverseCumulativeDeltaCompressor(
s.ValuesBuf, 0, s.FracDigits)
} else {
s.Values = chunkenc.NewReverseInstantDeltaCompressor(
s.ValuesBuf, 0, s.FracDigits)
}
// if s.MetricType == octopus.Cumulative {
// s.Values = chunkenc.NewReverseCumulativeDeltaCompressor(
// s.ValuesBuf, 0, s.FracDigits)
// } else {
// s.Values = chunkenc.NewReverseInstantDeltaCompressor(
// s.ValuesBuf, 0, s.FracDigits)
// }
s.Timestamps.Append(timestamp)
s.Values.Append(value)
// s.Timestamps.Append(timestamp)
// s.Values.Append(value)
s.Since = timestamp
s.SinceValue = value
s.Until = timestamp
s.UntilValue = value
// s.Since = timestamp
// s.SinceValue = value
// s.Until = timestamp
// s.UntilValue = value
}
func (s *_metric) DeleteMeasures() {
s.TimestampsBuf = conbuf.New(nil)
s.ValuesBuf = conbuf.New(nil)
//
s.Timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor(
s.TimestampsBuf, 0)
// fix
// s.Timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor(
// s.TimestampsBuf, 0)
if s.MetricType == octopus.Cumulative {
s.Values = chunkenc.NewReverseCumulativeDeltaCompressor(
s.ValuesBuf, 0, s.FracDigits)
} else {
s.Values = chunkenc.NewReverseInstantDeltaCompressor(
s.ValuesBuf, 0, s.FracDigits)
}
// if s.MetricType == octopus.Cumulative {
// s.Values = chunkenc.NewReverseCumulativeDeltaCompressor(
// s.ValuesBuf, 0, s.FracDigits)
// } else {
// s.Values = chunkenc.NewReverseInstantDeltaCompressor(
// s.ValuesBuf, 0, s.FracDigits)
// }
// END OF FIX
s.RootPageNo = 0
s.LastPageNo = 0

36
database/proc.go
View File

@@ -352,6 +352,11 @@ func (s *Database) tryAppendMeasure(req tryAppendMeasureReq) {
s.startAppendMeasure(metric, req, lockEntry)
}
type ReadBound struct {
ValuesPos int
ValuesLastValue float64
}
func (s *Database) startAppendMeasure(metric *_metric, req tryAppendMeasureReq, lockEntry *metricLockEntry) {
if req.Timestamp <= metric.Until {
req.ResultCh <- tryAppendMeasureResult{
@@ -466,8 +471,7 @@ func (s *Database) appendMeasure(rec txlog.AppendedMeasure) {
}
}
func (s *Database) appendMeasures(extended txlog.AppendedMeasuresExtended) {
rec := extended.Record
func (s *Database) appendMeasures(rec txlog.AppendedMeasures) {
metric, ok := s.metrics[rec.MetricID]
if !ok {
diploma.Abort(diploma.NoMetricBug,
@@ -500,10 +504,8 @@ func (s *Database) appendMeasures(extended txlog.AppendedMeasuresExtended) {
metric.UntilValue = measure.Value
}
if !extended.HoldLock {
lockEntry.XLock = false
s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry)
}
lockEntry.XLock = false
s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry)
}
func (s *Database) appendMeasureAfterOverflow(extended txlog.AppendedMeasureWithOverflowExtended) {
@@ -544,10 +546,8 @@ func (s *Database) appendMeasureAfterOverflow(extended txlog.AppendedMeasureWith
s.freeList.DeleteReservedPages(rec.ReusedIndexPages)
}
if !extended.HoldLock {
lockEntry.XLock = false
s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry)
}
lockEntry.XLock = false
s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry)
}
type tryAppendMeasuresReq struct {
@@ -880,7 +880,7 @@ func (s *Database) applyChanges(req txlog.Changes) {
case txlog.AppendedMeasure:
s.appendMeasure(rec)
case txlog.AppendedMeasuresExtended:
case txlog.AppendedMeasures:
s.appendMeasures(rec)
case txlog.AppendedMeasureWithOverflowExtended:
@@ -1019,11 +1019,8 @@ func (s *Database) deleteMetric(rec txlog.DeletedMetric) {
delete(s.metrics, rec.MetricID)
delete(s.metricLockEntries, rec.MetricID)
if len(rec.FreeDataPages) > 0 {
s.freeList.AddPages(rec.FreeDataPages)
}
if len(rec.FreeIndexPages) > 0 {
s.freeList.AddPages(rec.FreeIndexPages)
if len(rec.FreePageNumbers) > 0 {
s.freeList.AddPages(rec.FreePageNumbers)
}
if len(addMetricReqs) > 0 {
@@ -1053,11 +1050,8 @@ func (s *Database) deleteMeasures(rec txlog.DeletedMeasures) {
}
metric.DeleteMeasures()
lockEntry.XLock = false
if len(rec.FreeDataPages) > 0 {
s.freeList.AddPages(rec.FreeDataPages)
}
if len(rec.FreeDataPages) > 0 {
s.freeList.AddPages(rec.FreeIndexPages)
if len(rec.FreePageNumbers) > 0 {
s.freeList.AddPages(rec.FreePageNumbers)
}
s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry)
}