dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
dima
2026-05-10 19:15:11 +00:00
parent 9b38586ed8
commit ef60296027
5 changed files with 472 additions and 464 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
chunkenc/chunkenc_test.go
View File

@@ -149,9 +149,9 @@ func TestCumdeltaBound(t *testing.T) {
fmt.Printf("%d\n", buf.Chunks()[0]) fmt.Printf("%d\n", buf.Chunks()[0])
////////////////////////// //////////////////////////
bound := c.GetState() c.Lock()
fmt.Printf("bound: pos=%d, h=%d\n", bound.Pos, bound.H) //fmt.Printf("bound: pos=%d, h=%d\n", bound.Pos, bound.H)
c.Append(248) c.Append(248)
c.Append(305) c.Append(305)
@@ -167,8 +167,9 @@ func TestCumdeltaBound(t *testing.T) {
fmt.Println(value, done) fmt.Println(value, done)
} }
boundDecompressor := NewReverseCumulativeDeltaDecompressor(buf, c.Size(), fracDigits) //boundDecompressor := NewReverseCumulativeDeltaDecompressor(buf, c.Size(), fracDigits)
boundDecompressor.RestoreFromBound(bound) //boundDecompressor.RestoreFromBound(bound)
boundDecompressor := c.CreateDecompressor(fracDigits)
fmt.Println("from bound:") fmt.Println("from bound:")
for range 8 { for range 8 {

209
database/api.go
View File

@@ -94,13 +94,13 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e
} }
reply(conn, s.DeleteMetric(req)) reply(conn, s.DeleteMetric(req))
case proto.TypeAppendMeasure: // case proto.TypeAppendMeasure:
req, err := proto.ReadAppendMeasureReq(r) // req, err := proto.ReadAppendMeasureReq(r)
if err != nil { // if err != nil {
return fmt.Errorf("proto.ReadAppendMeasureReq: %s", err) // return fmt.Errorf("proto.ReadAppendMeasureReq: %s", err)
} // }
//fmt.Println("append measure", req.MetricID, conn.RemoteAddr().String()) // //fmt.Println("append measure", req.MetricID, conn.RemoteAddr().String())
reply(conn, s.AppendMeasure(req)) // reply(conn, s.AppendMeasure(req))
case proto.TypeAppendMeasures: case proto.TypeAppendMeasures:
req, err := proto.ReadAppendMeasuresReq(r) req, err := proto.ReadAppendMeasuresReq(r)
@@ -323,120 +323,120 @@ type FilledPage struct {
ValuesSize uint16 ValuesSize uint16
} }
type tryAppendMeasureResult struct { // type tryAppendMeasureResult struct {
MetricID uint32 // MetricID uint32
Timestamp uint32 // Timestamp uint32
Value float64 // Value float64
FilledPage *FilledPage // FilledPage *FilledPage
ResultCode byte // ResultCode byte
} // }
func (s *Database) AppendMeasure(req proto.AppendMeasureReq) uint16 { // func (s *Database) AppendMeasure(req proto.AppendMeasureReq) uint16 {
resultCh := make(chan tryAppendMeasureResult, 1) // resultCh := make(chan tryAppendMeasureResult, 1)
s.appendJobToWorkerQueue(tryAppendMeasureReq{ // s.appendJobToWorkerQueue(tryAppendMeasureReq{
MetricID: req.MetricID, // MetricID: req.MetricID,
Timestamp: req.Timestamp, // Timestamp: req.Timestamp,
Value: req.Value, // Value: req.Value,
ResultCh: resultCh, // ResultCh: resultCh,
}) // })
result := <-resultCh // result := <-resultCh
switch result.ResultCode { // switch result.ResultCode {
case CanAppend: // case CanAppend:
waitCh := s.txlog.Append(txlog.AppendedMeasure{ // waitCh := s.txlog.Append(txlog.AppendedMeasure{
MetricID: req.MetricID, // MetricID: req.MetricID,
Timestamp: req.Timestamp, // Timestamp: req.Timestamp,
Value: req.Value, // Value: req.Value,
}) // })
<-waitCh // <-waitCh
case NewPage: // case NewPage:
filled := result.FilledPage // filled := result.FilledPage
var path atree.PathToDataPage // var path atree.PathToDataPage
if filled.RootPageNo > 0 { // if filled.RootPageNo > 0 {
path, err := s.atree.FindPathToLastPage(filled.RootPageNo) // path, err := s.atree.FindPathToLastPage(filled.RootPageNo)
if err != nil { // if err != nil {
// FIX // // FIX
diploma.Abort(diploma.WriteToAtreeFailed, err) // diploma.Abort(diploma.WriteToAtreeFailed, err)
} // }
// for _, leg := range path.Legs { // // for _, leg := range path.Legs {
// pagesToRelease = append(pagesToRelease, leg.PageNo) // // pagesToRelease = append(pagesToRelease, leg.PageNo)
// } // // }
if path.LastPageNo != filled.PrevPageNo { // if path.LastPageNo != filled.PrevPageNo {
diploma.Abort( // diploma.Abort(
diploma.WrongPrevPageNo, // diploma.WrongPrevPageNo,
fmt.Errorf("bug: last pageNo %d in tree != prev pageNo %d in _metric", // fmt.Errorf("bug: last pageNo %d in tree != prev pageNo %d in _metric",
path.LastPageNo, filled.PrevPageNo), // path.LastPageNo, filled.PrevPageNo),
) // )
} // }
} // }
// report, err := s.atree.AppendDataPages(atree.AppendDataPageReq{ // // report, err := s.atree.AppendDataPages(atree.AppendDataPageReq{
// MetricID: req.MetricID, // // MetricID: req.MetricID,
// Timestamp: req.Timestamp, // // Timestamp: req.Timestamp,
// Value: req.Value, // // Value: req.Value,
// Since: filled.Since, // // Since: filled.Since,
// RootPageNo: filled.RootPageNo, // // RootPageNo: filled.RootPageNo,
// PrevPageNo: filled.PrevPageNo, // // PrevPageNo: filled.PrevPageNo,
// TimestampsChunks: filled.TimestampsChunks, // // TimestampsChunks: filled.TimestampsChunks,
// TimestampsSize: filled.TimestampsSize, // // TimestampsSize: filled.TimestampsSize,
// ValuesChunks: filled.ValuesChunks, // // ValuesChunks: filled.ValuesChunks,
// ValuesSize: filled.ValuesSize, // // ValuesSize: filled.ValuesSize,
// }) // // })
// if err != nil { // // if err != nil {
// diploma.Abort(diploma.WriteToAtreeFailed, err) // // diploma.Abort(diploma.WriteToAtreeFailed, err)
// } // // }
// _ = report // // _ = report
//waitCh := s.txlog.WriteAppended(path, report) // //waitCh := s.txlog.WriteAppended(path, report)
waitCh := s.txlog.Append(txlog.AppendedPagesReq{ // waitCh := s.txlog.Append(txlog.AppendedPagesReq{
Legs: path.Legs, // Legs: path.Legs,
MetricID: req.MetricID, // MetricID: req.MetricID,
Timestamp: req.Timestamp, // Timestamp: req.Timestamp,
Value: req.Value, // Value: req.Value,
LastPageNo: filled.PrevPageNo, // LastPageNo: filled.PrevPageNo,
//Pages: , // //Pages: ,
TimestampsChunks: filled.TimestampsChunks, // TimestampsChunks: filled.TimestampsChunks,
TimestampsSize: filled.TimestampsSize, // TimestampsSize: filled.TimestampsSize,
ValuesChunks: filled.ValuesChunks, // ValuesChunks: filled.ValuesChunks,
ValuesSize: filled.ValuesSize, // ValuesSize: filled.ValuesSize,
}, // },
) // )
<-waitCh // <-waitCh
case NoMetric: // case NoMetric:
return proto.ErrNoMetric // return proto.ErrNoMetric
case ExpiredMeasure: // case ExpiredMeasure:
return proto.ErrExpiredMeasure // return proto.ErrExpiredMeasure
case NonMonotonicValue: // case NonMonotonicValue:
return proto.ErrNonMonotonicValue // return proto.ErrNonMonotonicValue
default: // default:
diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug) // diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug)
} // }
return 0 // return 0
} // }
type tryAppendMeasuresResult struct { type tryAppendMeasuresResult struct {
ResultCode byte ResultCode byte
MetricType diploma.MetricType // MetricType diploma.MetricType
FracDigits byte // FracDigits byte
Since uint32 // Since uint32
Until uint32 // Until uint32
UntilValue float64 // UntilValue float64
RootPageNo uint32 // RootPageNo uint32
PrevPageNo uint32 // PrevPageNo uint32
TimestampsBuf *conbuf.ContinuousBuffer // TimestampsBuf *conbuf.ContinuousBuffer
ValuesBuf *conbuf.ContinuousBuffer // ValuesBuf *conbuf.ContinuousBuffer
Timestamps diploma.TimestampCompressor // Timestamps diploma.TimestampCompressor
Values diploma.ValueCompressor // Values diploma.ValueCompressor
} }
func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 { func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
@@ -444,6 +444,7 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
s.appendJobToWorkerQueue(tryAppendMeasuresReq{ s.appendJobToWorkerQueue(tryAppendMeasuresReq{
MetricID: req.MetricID, MetricID: req.MetricID,
Measures: req.Measures,
ResultCh: resultCh, ResultCh: resultCh,
}) })

6
database/database.go
View File

@@ -38,7 +38,7 @@ type Database struct {
workerQueue []any workerQueue []any
rLocksToRelease []uint32 rLocksToRelease []uint32
metrics map[uint32]*_metric metrics map[uint32]*_metric
metricLockEntries map[uint32]*metricLockEntry //metricLockEntries map[uint32]*metricLockEntry
freeList *freelist.FreeList freeList *freelist.FreeList
dir string dir string
databaseName string databaseName string
@@ -89,7 +89,7 @@ func New(opt Options) (_ *Database, err error) {
dir: opt.Dir, dir: opt.Dir,
databaseName: opt.DatabaseName, databaseName: opt.DatabaseName,
metrics: make(map[uint32]*_metric), metrics: make(map[uint32]*_metric),
metricLockEntries: make(map[uint32]*metricLockEntry), //metricLockEntries: make(map[uint32]*metricLockEntry),
freeList: freelist.New(), freeList: freelist.New(),
tcpPort: opt.TCPPort, tcpPort: opt.TCPPort,
logfile: opt.Logfile, logfile: opt.Logfile,
@@ -369,7 +369,7 @@ func (s *Database) replayChangesRecord(untyped any) error {
FracDigits: byte(rec.FracDigits), FracDigits: byte(rec.FracDigits),
TimestampsBuf: timestampsBuf, TimestampsBuf: timestampsBuf,
ValuesBuf: valuesBuf, ValuesBuf: valuesBuf,
Timestamps: chunkenc.NewReverseTimeDeltaOfDeltaCompressor(timestampsBuf, 0), Timestamps: chunkenc.NewReverseTimeDeltaCompressor(timestampsBuf, 0),
Values: values, Values: values,
} }

510
database/proc.go
View File

@@ -7,6 +7,7 @@ import (
"gordenko.dev/dima/qb/atree" "gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/chunkenc" "gordenko.dev/dima/qb/chunkenc"
"gordenko.dev/dima/qb/conbuf" "gordenko.dev/dima/qb/conbuf"
"gordenko.dev/dima/qb/proto"
"gordenko.dev/dima/qb/transform" "gordenko.dev/dima/qb/transform"
"gordenko.dev/dima/qb/txlog" "gordenko.dev/dima/qb/txlog"
) )
@@ -85,8 +86,8 @@ func (s *Database) DoWork() {
for _, untyped := range workerQueue { for _, untyped := range workerQueue {
switch req := untyped.(type) { switch req := untyped.(type) {
case tryAppendMeasureReq: //case tryAppendMeasureReq:
s.tryAppendMeasure(req) // s.tryAppendMeasure(req)
case tryAppendMeasuresReq: case tryAppendMeasuresReq:
s.tryAppendMeasures(req) s.tryAppendMeasures(req)
@@ -155,8 +156,8 @@ func (s *Database) processMetricQueue(metricID uint32, metric *_metric, lockEntr
} else { } else {
for idx, untyped := range modificationReqs { for idx, untyped := range modificationReqs {
switch req := untyped.(type) { switch req := untyped.(type) {
case tryAppendMeasureReq: //case tryAppendMeasureReq:
s.startAppendMeasure(metric, req, nil) // s.startAppendMeasure(metric, req, nil)
case tryAppendMeasuresReq: case tryAppendMeasuresReq:
s.startAppendMeasures(metric, req, nil) s.startAppendMeasures(metric, req, nil)
@@ -195,16 +196,17 @@ func (s *Database) tryAddMetric(req tryAddMetricReq) {
req.ResultCh <- MetricDuplicate req.ResultCh <- MetricDuplicate
return return
} }
req.ResultCh <- Succeed // new
lockEntry, ok := s.metricLockEntries[req.MetricID] // lockEntry, ok := s.metricLockEntries[req.MetricID]
if ok { // if ok {
lockEntry.WaitQueue = append(lockEntry.WaitQueue, req) // lockEntry.WaitQueue = append(lockEntry.WaitQueue, req)
} else { // } else {
s.metricLockEntries[req.MetricID] = &metricLockEntry{ // s.metricLockEntries[req.MetricID] = &metricLockEntry{
XLock: true, // XLock: true,
} // }
req.ResultCh <- Succeed // req.ResultCh <- Succeed
} // }
} }
func (s *Database) processTryAddMetricReqsImmediatelyAfterDelete(reqs []tryAddMetricReq) { func (s *Database) processTryAddMetricReqsImmediatelyAfterDelete(reqs []tryAddMetricReq) {
@@ -220,10 +222,11 @@ func (s *Database) processTryAddMetricReqsImmediatelyAfterDelete(reqs []tryAddMe
waitQueue = append(waitQueue, req) waitQueue = append(waitQueue, req)
} }
} }
s.metricLockEntries[req.MetricID] = &metricLockEntry{ // FIX
XLock: true, // s.metricLockEntries[req.MetricID] = &metricLockEntry{
WaitQueue: waitQueue, // XLock: true,
} // WaitQueue: waitQueue,
// }
req.ResultCh <- Succeed req.ResultCh <- Succeed
} }
@@ -253,30 +256,32 @@ type tryDeleteMetricReq struct {
} }
func (s *Database) tryDeleteMetric(req tryDeleteMetricReq) { func (s *Database) tryDeleteMetric(req tryDeleteMetricReq) {
metric, ok := s.metrics[req.MetricID] // FIX
if !ok { // metric, ok := s.metrics[req.MetricID]
req.ResultCh <- tryDeleteMetricResult{ // if !ok {
ResultCode: NoMetric, // req.ResultCh <- tryDeleteMetricResult{
} // ResultCode: NoMetric,
return // }
} // return
// }
lockEntry, ok := s.metricLockEntries[req.MetricID] // lockEntry, ok := s.metricLockEntries[req.MetricID]
if ok { // if ok {
lockEntry.WaitQueue = append(lockEntry.WaitQueue, req) // lockEntry.WaitQueue = append(lockEntry.WaitQueue, req)
} else { // } else {
s.startDeleteMetric(metric, req) // s.startDeleteMetric(metric, req)
} // }
} }
func (s *Database) startDeleteMetric(metric *_metric, req tryDeleteMetricReq) { func (s *Database) startDeleteMetric(metric *_metric, req tryDeleteMetricReq) {
s.metricLockEntries[req.MetricID] = &metricLockEntry{ // FIX
XLock: true, // s.metricLockEntries[req.MetricID] = &metricLockEntry{
} // XLock: true,
req.ResultCh <- tryDeleteMetricResult{ // }
ResultCode: Succeed, // req.ResultCh <- tryDeleteMetricResult{
RootPageNo: metric.RootPageNo, // ResultCode: Succeed,
} // RootPageNo: metric.RootPageNo,
// }
} }
type tryDeleteMeasuresReq struct { type tryDeleteMeasuresReq struct {
@@ -286,190 +291,192 @@ type tryDeleteMeasuresReq struct {
} }
func (s *Database) tryDeleteMeasures(req tryDeleteMeasuresReq) { func (s *Database) tryDeleteMeasures(req tryDeleteMeasuresReq) {
metric, ok := s.metrics[req.MetricID] // FIX
if !ok { // metric, ok := s.metrics[req.MetricID]
req.ResultCh <- tryDeleteMeasuresResult{ // if !ok {
ResultCode: NoMetric, // req.ResultCh <- tryDeleteMeasuresResult{
} // ResultCode: NoMetric,
return // }
} // return
// }
if metric.Since == 0 || (req.Since > 0 && metric.Until < req.Since) { // if metric.Since == 0 || (req.Since > 0 && metric.Until < req.Since) {
req.ResultCh <- tryDeleteMeasuresResult{ // req.ResultCh <- tryDeleteMeasuresResult{
ResultCode: NoMeasuresToDelete, // ResultCode: NoMeasuresToDelete,
} // }
} // }
lockEntry, ok := s.metricLockEntries[req.MetricID] // lockEntry, ok := s.metricLockEntries[req.MetricID]
if ok { // if ok {
lockEntry.WaitQueue = append(lockEntry.WaitQueue, req) // lockEntry.WaitQueue = append(lockEntry.WaitQueue, req)
} else { // } else {
s.startDeleteMeasures(metric, req) // s.startDeleteMeasures(metric, req)
} // }
} }
func (s *Database) startDeleteMeasures(metric *_metric, req tryDeleteMeasuresReq) { func (s *Database) startDeleteMeasures(metric *_metric, req tryDeleteMeasuresReq) {
s.metricLockEntries[req.MetricID] = &metricLockEntry{ // FIX
XLock: true, // s.metricLockEntries[req.MetricID] = &metricLockEntry{
} // XLock: true,
// }
if metric.RootPageNo > 0 { // if metric.RootPageNo > 0 {
req.ResultCh <- tryDeleteMeasuresResult{ // req.ResultCh <- tryDeleteMeasuresResult{
ResultCode: DeleteFromAtreeRequired, // ResultCode: DeleteFromAtreeRequired,
RootPageNo: metric.RootPageNo, // RootPageNo: metric.RootPageNo,
} // }
} else { // } else {
req.ResultCh <- tryDeleteMeasuresResult{ // req.ResultCh <- tryDeleteMeasuresResult{
ResultCode: DeleteFromAtreeNotNeeded, // ResultCode: DeleteFromAtreeNotNeeded,
} // }
} // }
} }
type tryAppendMeasureReq struct { // type tryAppendMeasureReq struct {
MetricID uint32 // MetricID uint32
Timestamp uint32 // Timestamp uint32
Value float64 // Value float64
ResultCh chan tryAppendMeasureResult // ResultCh chan tryAppendMeasureResult
} // }
func (s *Database) tryAppendMeasure(req tryAppendMeasureReq) { // func (s *Database) tryAppendMeasure(req tryAppendMeasureReq) {
metric, ok := s.metrics[req.MetricID] // metric, ok := s.metrics[req.MetricID]
if !ok { // if !ok {
req.ResultCh <- tryAppendMeasureResult{ // req.ResultCh <- tryAppendMeasureResult{
MetricID: req.MetricID, // MetricID: req.MetricID,
ResultCode: NoMetric, // ResultCode: NoMetric,
} // }
return // return
} // }
lockEntry, ok := s.metricLockEntries[req.MetricID] // lockEntry, ok := s.metricLockEntries[req.MetricID]
if ok { // if ok {
if lockEntry.XLock { // if lockEntry.XLock {
lockEntry.WaitQueue = append(lockEntry.WaitQueue, req) // lockEntry.WaitQueue = append(lockEntry.WaitQueue, req)
return // return
} // }
} // }
s.startAppendMeasure(metric, req, lockEntry) // s.startAppendMeasure(metric, req, lockEntry)
} // }
type ReadBound struct { type ReadBound struct {
ValuesPos int ValuesPos int
ValuesLastValue float64 ValuesLastValue float64
} }
func (s *Database) startAppendMeasure(metric *_metric, req tryAppendMeasureReq, lockEntry *metricLockEntry) { // func (s *Database) startAppendMeasure(metric *_metric, req tryAppendMeasureReq, lockEntry *metricLockEntry) {
if req.Timestamp <= metric.Until { // if req.Timestamp <= metric.Until {
req.ResultCh <- tryAppendMeasureResult{ // req.ResultCh <- tryAppendMeasureResult{
MetricID: req.MetricID, // MetricID: req.MetricID,
ResultCode: ExpiredMeasure, // ResultCode: ExpiredMeasure,
} // }
return // return
} // }
if metric.MetricType == diploma.Cumulative && req.Value < metric.UntilValue { // if metric.MetricType == diploma.Cumulative && req.Value < metric.UntilValue {
req.ResultCh <- tryAppendMeasureResult{ // req.ResultCh <- tryAppendMeasureResult{
MetricID: req.MetricID, // MetricID: req.MetricID,
ResultCode: NonMonotonicValue, // ResultCode: NonMonotonicValue,
} // }
return // return
} // }
extraSpace := metric.Timestamps.CalcRequiredSpace(req.Timestamp) + // extraSpace := metric.Timestamps.CalcRequiredSpace(req.Timestamp) +
metric.Values.CalcRequiredSpace(req.Value) // metric.Values.CalcRequiredSpace(req.Value)
totalSpace := metric.Timestamps.Size() + metric.Values.Size() + extraSpace // totalSpace := metric.Timestamps.Size() + metric.Values.Size() + extraSpace
if totalSpace <= atree.DataPagePayloadSize { // if totalSpace <= atree.DataPagePayloadSize {
if lockEntry != nil { // if lockEntry != nil {
lockEntry.RLocks++ // lockEntry.RLocks++
} else { // } else {
s.metricLockEntries[req.MetricID] = &metricLockEntry{ // s.metricLockEntries[req.MetricID] = &metricLockEntry{
RLocks: 1, // RLocks: 1,
} // }
} // }
req.ResultCh <- tryAppendMeasureResult{ // req.ResultCh <- tryAppendMeasureResult{
MetricID: req.MetricID, // MetricID: req.MetricID,
Timestamp: req.Timestamp, // Timestamp: req.Timestamp,
Value: req.Value, // Value: req.Value,
ResultCode: CanAppend, // ResultCode: CanAppend,
} // }
} else { // } else {
if lockEntry != nil { // if lockEntry != nil {
if lockEntry.RLocks > 0 { // if lockEntry.RLocks > 0 {
lockEntry.WaitQueue = append(lockEntry.WaitQueue, req) // lockEntry.WaitQueue = append(lockEntry.WaitQueue, req)
return // return
} // }
lockEntry.XLock = true // lockEntry.XLock = true
} else { // } else {
s.metricLockEntries[req.MetricID] = &metricLockEntry{ // s.metricLockEntries[req.MetricID] = &metricLockEntry{
XLock: true, // XLock: true,
} // }
} // }
req.ResultCh <- tryAppendMeasureResult{ // req.ResultCh <- tryAppendMeasureResult{
MetricID: req.MetricID, // MetricID: req.MetricID,
Timestamp: req.Timestamp, // Timestamp: req.Timestamp,
Value: req.Value, // Value: req.Value,
ResultCode: NewPage, // ResultCode: NewPage,
FilledPage: &FilledPage{ // FilledPage: &FilledPage{
Since: metric.Since, // Since: metric.Since,
RootPageNo: metric.RootPageNo, // RootPageNo: metric.RootPageNo,
PrevPageNo: metric.LastPageNo, // PrevPageNo: metric.LastPageNo,
TimestampsChunks: metric.TimestampsBuf.Chunks(), // TimestampsChunks: metric.TimestampsBuf.Chunks(),
TimestampsSize: uint16(metric.Timestamps.Size()), // TimestampsSize: uint16(metric.Timestamps.Size()),
ValuesChunks: metric.ValuesBuf.Chunks(), // ValuesChunks: metric.ValuesBuf.Chunks(),
ValuesSize: uint16(metric.Values.Size()), // ValuesSize: uint16(metric.Values.Size()),
}, // },
} // }
} // }
} // }
func (s *Database) appendMeasure(rec txlog.AppendedMeasure) { // func (s *Database) appendMeasure(rec txlog.AppendedMeasure) {
metric, ok := s.metrics[rec.MetricID] // metric, ok := s.metrics[rec.MetricID]
if !ok { // if !ok {
diploma.Abort(diploma.NoMetricBug, // diploma.Abort(diploma.NoMetricBug,
fmt.Errorf("appendMeasure: metric %d not found", // fmt.Errorf("appendMeasure: metric %d not found",
rec.MetricID)) // rec.MetricID))
} // }
lockEntry, ok := s.metricLockEntries[rec.MetricID] // lockEntry, ok := s.metricLockEntries[rec.MetricID]
if !ok { // if !ok {
diploma.Abort(diploma.NoLockEntryBug, // diploma.Abort(diploma.NoLockEntryBug,
fmt.Errorf("appendMeasure: lockEntry not found for the metric %d", // fmt.Errorf("appendMeasure: lockEntry not found for the metric %d",
rec.MetricID)) // rec.MetricID))
} // }
if lockEntry.XLock { // if lockEntry.XLock {
diploma.Abort(diploma.XLockBug, // diploma.Abort(diploma.XLockBug,
fmt.Errorf("appendMeasure: xlock is set for the metric %d", // fmt.Errorf("appendMeasure: xlock is set for the metric %d",
rec.MetricID)) // rec.MetricID))
} // }
if lockEntry.RLocks <= 0 { // if lockEntry.RLocks <= 0 {
diploma.Abort(diploma.NoRLockBug, // diploma.Abort(diploma.NoRLockBug,
fmt.Errorf("appendMeasure: rlock not set for the metric %d", // fmt.Errorf("appendMeasure: rlock not set for the metric %d",
rec.MetricID)) // rec.MetricID))
} // }
if metric.Since == 0 { // if metric.Since == 0 {
metric.Since = rec.Timestamp // metric.Since = rec.Timestamp
metric.SinceValue = rec.Value // metric.SinceValue = rec.Value
} // }
metric.Timestamps.Append(rec.Timestamp) // metric.Timestamps.Append(rec.Timestamp)
metric.Values.Append(rec.Value) // metric.Values.Append(rec.Value)
metric.Until = rec.Timestamp // metric.Until = rec.Timestamp
metric.UntilValue = rec.Value // metric.UntilValue = rec.Value
lockEntry.RLocks-- // lockEntry.RLocks--
if len(lockEntry.WaitQueue) > 0 { // if len(lockEntry.WaitQueue) > 0 {
s.processMetricQueue(rec.MetricID, metric, lockEntry) // s.processMetricQueue(rec.MetricID, metric, lockEntry)
} else { // } else {
if lockEntry.RLocks == 0 { // if lockEntry.RLocks == 0 {
delete(s.metricLockEntries, rec.MetricID) // delete(s.metricLockEntries, rec.MetricID)
} // }
} // }
} // }
func (s *Database) appendMeasures(rec txlog.AppendedMeasures) { func (s *Database) appendMeasures(rec txlog.AppendedMeasures) {
metric, ok := s.metrics[rec.MetricID] metric, ok := s.metrics[rec.MetricID]
@@ -508,50 +515,51 @@ func (s *Database) appendMeasures(rec txlog.AppendedMeasures) {
s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry) s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry)
} }
func (s *Database) appendMeasureAfterOverflow(extended txlog.AppendedMeasureWithOverflowExtended) { // func (s *Database) appendMeasureAfterOverflow(extended txlog.AppendedMeasureWithOverflowExtended) {
rec := extended.Record // rec := extended.Record
metric, ok := s.metrics[rec.MetricID] // metric, ok := s.metrics[rec.MetricID]
if !ok { // if !ok {
diploma.Abort(diploma.NoMetricBug, // diploma.Abort(diploma.NoMetricBug,
fmt.Errorf("appendMeasureAfterOverflow: metric %d not found", // fmt.Errorf("appendMeasureAfterOverflow: metric %d not found",
rec.MetricID)) // rec.MetricID))
} // }
lockEntry, ok := s.metricLockEntries[rec.MetricID] // lockEntry, ok := s.metricLockEntries[rec.MetricID]
if !ok { // if !ok {
diploma.Abort(diploma.NoLockEntryBug, // diploma.Abort(diploma.NoLockEntryBug,
fmt.Errorf("appendMeasureAfterOverflow: lockEntry not found for the metric %d", // fmt.Errorf("appendMeasureAfterOverflow: lockEntry not found for the metric %d",
rec.MetricID)) // rec.MetricID))
} // }
if !lockEntry.XLock { // if !lockEntry.XLock {
diploma.Abort(diploma.NoXLockBug, // diploma.Abort(diploma.NoXLockBug,
fmt.Errorf("appendMeasureAfterOverflow: xlock not set for the metric %d", // fmt.Errorf("appendMeasureAfterOverflow: xlock not set for the metric %d",
rec.MetricID)) // rec.MetricID))
} // }
metric.ReinitBy(rec.Timestamp, rec.Value) // metric.ReinitBy(rec.Timestamp, rec.Value)
if rec.IsRootChanged { // if rec.IsRootChanged {
metric.RootPageNo = rec.RootPageNo // metric.RootPageNo = rec.RootPageNo
} // }
metric.LastPageNo = rec.DataPageNo // metric.LastPageNo = rec.DataPageNo
if rec.IsDataPageReused { // if rec.IsDataPageReused {
s.freeList.DeleteReservedPages([]uint32{ // s.freeList.DeleteReservedPages([]uint32{
rec.DataPageNo, // rec.DataPageNo,
}) // })
} // }
if len(rec.ReusedIndexPages) > 0 { // if len(rec.ReusedIndexPages) > 0 {
s.freeList.DeleteReservedPages(rec.ReusedIndexPages) // s.freeList.DeleteReservedPages(rec.ReusedIndexPages)
} // }
lockEntry.XLock = false // lockEntry.XLock = false
s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry) // s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry)
} // }
type tryAppendMeasuresReq struct { type tryAppendMeasuresReq struct {
MetricID uint32 MetricID uint32
Measures []proto.Measure
ResultCh chan tryAppendMeasuresResult ResultCh chan tryAppendMeasuresResult
} }
@@ -624,17 +632,17 @@ func (s *Database) startAppendMeasures(metric *_metric, req tryAppendMeasuresReq
req.ResultCh <- tryAppendMeasuresResult{ req.ResultCh <- tryAppendMeasuresResult{
ResultCode: CanAppend, ResultCode: CanAppend,
MetricType: metric.MetricType, // MetricType: metric.MetricType,
FracDigits: metric.FracDigits, // FracDigits: metric.FracDigits,
Since: metric.Since, // Since: metric.Since,
Until: metric.Until, // Until: metric.Until,
UntilValue: metric.UntilValue, // UntilValue: metric.UntilValue,
RootPageNo: metric.RootPageNo, // RootPageNo: metric.RootPageNo,
PrevPageNo: metric.LastPageNo, // PrevPageNo: metric.LastPageNo,
TimestampsBuf: timestampsBuf, // TimestampsBuf: timestampsBuf,
ValuesBuf: valuesBuf, // ValuesBuf: valuesBuf,
Timestamps: timestamps, // Timestamps: timestamps,
Values: values, // Values: values,
} }
} }
@@ -712,7 +720,7 @@ func (*Database) startRangeScan(metric *_metric, req tryRangeScanReq) bool {
} }
} }
timestampDecompressor := chunkenc.NewReverseTimeDeltaOfDeltaDecompressor( timestampDecompressor := chunkenc.NewReverseTimeDeltaDecompressor(
metric.TimestampsBuf, metric.TimestampsBuf,
metric.Timestamps.Size(), metric.Timestamps.Size(),
) )
@@ -809,7 +817,7 @@ func (*Database) startFullScan(metric *_metric, req tryFullScanReq) bool {
return false return false
} }
timestampDecompressor := chunkenc.NewReverseTimeDeltaOfDeltaDecompressor( timestampDecompressor := chunkenc.NewReverseTimeDeltaDecompressor(
metric.TimestampsBuf, metric.TimestampsBuf,
metric.Timestamps.Size(), metric.Timestamps.Size(),
) )
@@ -877,14 +885,14 @@ func (s *Database) applyChanges(req txlog.Changes) {
case txlog.DeletedMetric: case txlog.DeletedMetric:
s.deleteMetric(rec) s.deleteMetric(rec)
case txlog.AppendedMeasure: // case txlog.AppendedMeasure:
s.appendMeasure(rec) // s.appendMeasure(rec)
case txlog.AppendedMeasures: case txlog.AppendedMeasures:
s.appendMeasures(rec) s.appendMeasures(rec)
case txlog.AppendedMeasureWithOverflowExtended: // case txlog.AppendedMeasureWithOverflowExtended:
s.appendMeasureAfterOverflow(rec) // s.appendMeasureAfterOverflow(rec)
case txlog.DeletedMeasures: case txlog.DeletedMeasures:
s.deleteMeasures(rec) s.deleteMeasures(rec)
@@ -942,7 +950,7 @@ func (s *Database) addMetric(rec txlog.AddedMetric) {
FracDigits: byte(rec.FracDigits), FracDigits: byte(rec.FracDigits),
TimestampsBuf: timestampsBuf, TimestampsBuf: timestampsBuf,
ValuesBuf: valuesBuf, ValuesBuf: valuesBuf,
Timestamps: chunkenc.NewReverseTimeDeltaOfDeltaCompressor(timestampsBuf, 0), Timestamps: chunkenc.NewReverseTimeDeltaCompressor(timestampsBuf, 0),
Values: values, Values: values,
} }
@@ -976,11 +984,11 @@ func (s *Database) deleteMetric(rec txlog.DeletedMetric) {
if len(lockEntry.WaitQueue) > 0 { if len(lockEntry.WaitQueue) > 0 {
for _, untyped := range lockEntry.WaitQueue { for _, untyped := range lockEntry.WaitQueue {
switch req := untyped.(type) { switch req := untyped.(type) {
case tryAppendMeasureReq: // case tryAppendMeasureReq:
req.ResultCh <- tryAppendMeasureResult{ // req.ResultCh <- tryAppendMeasureResult{
MetricID: req.MetricID, // MetricID: req.MetricID,
ResultCode: NoMetric, // ResultCode: NoMetric,
} // }
case tryRangeScanReq: case tryRangeScanReq:
req.ResultCh <- rangeScanResult{ req.ResultCh <- rangeScanResult{

148
database/snapshot.go
View File

@@ -8,10 +8,7 @@ import (
"path/filepath" "path/filepath"
octopus "gordenko.dev/dima/qb" octopus "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/bin" "gordenko.dev/dima/qb/bin"
"gordenko.dev/dima/qb/chunkenc"
"gordenko.dev/dima/qb/conbuf"
"gordenko.dev/dima/qb/freelist" "gordenko.dev/dima/qb/freelist"
) )
@@ -167,91 +164,92 @@ func (s *Database) dumpSnapshot(logNumber int) (err error) {
} }
func (s *Database) loadSnapshot(fileName string) (err error) { func (s *Database) loadSnapshot(fileName string) (err error) {
var ( // FIX
hasher = crc32.NewIEEE() // var (
metricsQty int // hasher = crc32.NewIEEE()
header = make([]byte, metricHeaderSize) // metricsQty int
body = make([]byte, atree.PageSize) // header = make([]byte, metricHeaderSize)
) // body = make([]byte, atree.PageSize)
// )
file, err := os.Open(fileName) // file, err := os.Open(fileName)
if err != nil { // if err != nil {
return // return
} // }
src := io.TeeReader(file, hasher) // src := io.TeeReader(file, hasher)
u64, _, err := bin.ReadVarUint64(src) // u64, _, err := bin.ReadVarUint64(src)
if err != nil { // if err != nil {
return // return
} // }
metricsQty = int(u64) // metricsQty = int(u64)
for range metricsQty { // for range metricsQty {
var metric _metric // var metric _metric
err = bin.ReadNInto(src, header) // err = bin.ReadNInto(src, header)
if err != nil { // if err != nil {
return // return
} // }
metricID := bin.GetUint32(header[0:]) // metricID := bin.GetUint32(header[0:])
metric.MetricType = octopus.MetricType(header[4]) // metric.MetricType = octopus.MetricType(header[4])
metric.FracDigits = header[5] // metric.FracDigits = header[5]
metric.RootPageNo = bin.GetUint32(header[6:]) // metric.RootPageNo = bin.GetUint32(header[6:])
metric.LastPageNo = bin.GetUint32(header[10:]) // metric.LastPageNo = bin.GetUint32(header[10:])
metric.Since = bin.GetUint32(header[14:]) // metric.Since = bin.GetUint32(header[14:])
metric.SinceValue = bin.GetFloat64(header[18:]) // metric.SinceValue = bin.GetFloat64(header[18:])
metric.Until = bin.GetUint32(header[26:]) // metric.Until = bin.GetUint32(header[26:])
metric.UntilValue = bin.GetFloat64(header[30:]) // metric.UntilValue = bin.GetFloat64(header[30:])
tSize := bin.GetUint16(header[38:]) // tSize := bin.GetUint16(header[38:])
vSize := bin.GetUint16(header[40:]) // vSize := bin.GetUint16(header[40:])
buf := body[:tSize] // buf := body[:tSize]
err = bin.ReadNInto(src, buf) // err = bin.ReadNInto(src, buf)
if err != nil { // if err != nil {
return // return
} // }
metric.TimestampsBuf = conbuf.NewFromBuffer(buf) // metric.TimestampsBuf = conbuf.NewFromBuffer(buf)
buf = body[:vSize] // buf = body[:vSize]
err = bin.ReadNInto(src, buf) // err = bin.ReadNInto(src, buf)
if err != nil { // if err != nil {
return // return
} // }
metric.ValuesBuf = conbuf.NewFromBuffer(buf) // metric.ValuesBuf = conbuf.NewFromBuffer(buf)
metric.Timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor( // metric.Timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor(
metric.TimestampsBuf, int(tSize)) // metric.TimestampsBuf, int(tSize))
if metric.MetricType == octopus.Cumulative { // if metric.MetricType == octopus.Cumulative {
metric.Values = chunkenc.NewReverseCumulativeDeltaCompressor( // metric.Values = chunkenc.NewReverseCumulativeDeltaCompressor(
metric.ValuesBuf, int(vSize), metric.FracDigits) // metric.ValuesBuf, int(vSize), metric.FracDigits)
} else { // } else {
metric.Values = chunkenc.NewReverseInstantDeltaCompressor( // metric.Values = chunkenc.NewReverseInstantDeltaCompressor(
metric.ValuesBuf, int(vSize), metric.FracDigits) // metric.ValuesBuf, int(vSize), metric.FracDigits)
} // }
s.metrics[metricID] = &metric // s.metrics[metricID] = &metric
} // }
err = restoreFreeList(s.freeList, src) // err = restoreFreeList(s.freeList, src)
if err != nil { // if err != nil {
return fmt.Errorf("restore dataFreeList: %s", err) // return fmt.Errorf("restore dataFreeList: %s", err)
} // }
err = restoreFreeList(s.freeList, src) // err = restoreFreeList(s.freeList, src)
if err != nil { // if err != nil {
return fmt.Errorf("restore indexFreeList: %s", err) // return fmt.Errorf("restore indexFreeList: %s", err)
} // }
calculatedChecksum := hasher.Sum32() // calculatedChecksum := hasher.Sum32()
writtenChecksum, err := bin.ReadUint32(file) // writtenChecksum, err := bin.ReadUint32(file)
if err != nil { // if err != nil {
return // return
} // }
if calculatedChecksum != writtenChecksum { // if calculatedChecksum != writtenChecksum {
return fmt.Errorf("calculated checksum %d not equal written checksum %d", calculatedChecksum, writtenChecksum) // return fmt.Errorf("calculated checksum %d not equal written checksum %d", calculatedChecksum, writtenChecksum)
} // }
return return
} }