dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-05-21 23:38:52 +03:00
parent 793df29e11
commit e7ec6083cb
21 changed files with 1234 additions and 1017 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
database/api.go
View File

@@ -6,7 +6,7 @@ import (
"io"
"net"
diploma "gordenko.dev/dima/qb"
qb "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/bin"
"gordenko.dev/dima/qb/bufreader"
@@ -192,11 +192,11 @@ func (s *Database) AddMetric(req proto.AddMetricReq) uint16 {
if req.MetricID == 0 {
return proto.ErrEmptyMetricID
}
if byte(req.FracDigits) > diploma.MaxFracDigits {
if byte(req.FracDigits) > qb.MaxFracDigits {
return proto.ErrWrongFracDigits
}
switch req.MetricType {
case diploma.Cumulative, diploma.Instant:
case qb.Cumulative, qb.Instant:
// ok
default:
return proto.ErrWrongMetricType
@@ -213,24 +213,24 @@ func (s *Database) AddMetric(req proto.AddMetricReq) uint16 {
switch resultCode {
case Succeed:
waitCh := s.txlog.Append(txlog.AddedMetric{
s.txlog.Append(txlog.AddedMetric{
MetricID: req.MetricID,
MetricType: req.MetricType,
FracDigits: req.FracDigits,
})
<-waitCh
//<-waitCh
case MetricDuplicate:
return proto.ErrDuplicate
default:
diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug)
qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
}
return 0
}
type Metric struct {
MetricType diploma.MetricType
MetricType qb.MetricType
FracDigits byte
ResultCode byte
}
@@ -264,7 +264,7 @@ func (s *Database) GetMetric(conn io.Writer, req proto.GetMetricReq) error {
reply(conn, proto.ErrNoMetric)
default:
diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug)
qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
}
return nil
}
@@ -293,20 +293,20 @@ func (s *Database) DeleteMetric(req proto.DeleteMetricReq) uint16 {
var err error
freePageNumbers, err = s.atree.GetAllPages(result.RootPageNo)
if err != nil {
diploma.Abort(diploma.FailedAtreeRequest, err)
qb.Abort(qb.FailedAtreeRequest, err)
}
}
waitCh := s.txlog.Append(txlog.DeletedMetric{
s.txlog.Append(txlog.DeletedMetric{
MetricID: req.MetricID,
FreePageNumbers: freePageNumbers,
})
<-waitCh
//<-waitCh
case NoMetric:
return proto.ErrNoMetric
default:
diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug)
qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
}
return 0
}
@@ -358,15 +358,15 @@ type FilledPage struct {
// path, err := s.atree.FindPathToLastPage(filled.RootPageNo)
// if err != nil {
// // FIX
// diploma.Abort(diploma.WriteToAtreeFailed, err)
// qb.Abort(qb.WriteToAtreeFailed, err)
// }
// // for _, leg := range path.Legs {
// // pagesToRelease = append(pagesToRelease, leg.PageNo)
// // }
// if path.LastPageNo != filled.PrevPageNo {
// diploma.Abort(
// diploma.WrongPrevPageNo,
// qb.Abort(
// qb.WrongPrevPageNo,
// fmt.Errorf("bug: last pageNo %d in tree != prev pageNo %d in _metric",
// path.LastPageNo, filled.PrevPageNo),
// )
@@ -386,7 +386,7 @@ type FilledPage struct {
// // ValuesSize: filled.ValuesSize,
// // })
// // if err != nil {
// // diploma.Abort(diploma.WriteToAtreeFailed, err)
// // qb.Abort(qb.WriteToAtreeFailed, err)
// // }
// // _ = report
@@ -417,7 +417,7 @@ type FilledPage struct {
// return proto.ErrNonMonotonicValue
// default:
// diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug)
// qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
// }
// return 0
// }
@@ -425,7 +425,7 @@ type FilledPage struct {
type tryAppendMeasuresResult struct {
ResultCode byte
Written int
// MetricType diploma.MetricType
// MetricType qb.MetricType
// FracDigits byte
// Since uint32
// Until uint32
@@ -434,8 +434,8 @@ type tryAppendMeasuresResult struct {
// PrevPageNo uint32
// TimestampsBuf *conbuf.ContinuousBuffer
// ValuesBuf *conbuf.ContinuousBuffer
// Timestamps diploma.TimestampCompressor
// Values diploma.ValueCompressor
// Timestamps qb.TimestampCompressor
// Values qb.ValueCompressor
}
func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
@@ -454,7 +454,7 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
// report, err := s.atree.AppendDataPage(atree.AppendDataPageReq{})
// if err != nil {
// diploma.Abort(diploma.WriteToAtreeFailed, err)
// qb.Abort(qb.WriteToAtreeFailed, err)
// }
// _ = report
@@ -473,7 +473,7 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
return proto.ErrNoMetric
default:
diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug)
qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
}
return 0
}
@@ -500,29 +500,29 @@ func (s *Database) DeleteMeasures(req proto.DeleteMeasuresReq) uint16 {
case DeleteFromAtreeNotNeeded:
// регистрирую удаление в TransactionLog
waitCh := s.txlog.Append(txlog.DeletedMeasures{
s.txlog.Append(txlog.DeletedMeasures{
MetricID: req.MetricID,
})
<-waitCh
//<-waitCh
case DeleteFromAtreeRequired:
// собираю номера всех data и index страниц метрики (типа запись REDO лога).
pageNumbers, err := s.atree.GetAllPages(req.MetricID)
if err != nil {
diploma.Abort(diploma.FailedAtreeRequest, err)
qb.Abort(qb.FailedAtreeRequest, err)
}
// регистрирую удаление в TransactionLog
waitCh := s.txlog.Append(txlog.DeletedMeasures{
s.txlog.Append(txlog.DeletedMeasures{
MetricID: req.MetricID,
FreePageNumbers: pageNumbers,
})
<-waitCh
//<-waitCh
case NoMetric:
return proto.ErrNoMetric
default:
diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug)
qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
}
return 0
}
@@ -540,7 +540,7 @@ func (s *Database) ListAllInstantMeasures(conn net.Conn, req proto.ListAllInstan
return s.fullScan(fullScanReq{
MetricID: req.MetricID,
MetricType: diploma.Instant,
MetricType: qb.Instant,
Conn: conn,
ResponseWriter: responseWriter,
})
@@ -551,7 +551,7 @@ func (s *Database) ListAllCumulativeMeasures(conn io.Writer, req proto.ListAllCu
return s.fullScan(fullScanReq{
MetricID: req.MetricID,
MetricType: diploma.Cumulative,
MetricType: qb.Cumulative,
Conn: conn,
ResponseWriter: responseWriter,
})
@@ -567,7 +567,7 @@ func (s *Database) ListInstantMeasures(conn net.Conn, req proto.ListInstantMeasu
return s.rangeScan(rangeScanReq{
MetricID: req.MetricID,
MetricType: diploma.Instant,
MetricType: qb.Instant,
Since: req.Since,
Until: req.Until - 1,
Conn: conn,
@@ -585,7 +585,7 @@ func (s *Database) ListCumulativeMeasures(conn net.Conn, req proto.ListCumulativ
return s.rangeScan(rangeScanReq{
MetricID: req.MetricID,
MetricType: diploma.Cumulative,
MetricType: qb.Cumulative,
Since: req.Since,
Until: req.Until - 1,
Conn: conn,
@@ -620,7 +620,7 @@ func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriod
return s.rangeScan(rangeScanReq{
MetricID: req.MetricID,
MetricType: diploma.Instant,
MetricType: qb.Instant,
Since: uint32(since.Unix()),
Until: uint32(until.Unix()),
Conn: conn,
@@ -647,7 +647,7 @@ func (s *Database) ListCumulativePeriods(conn net.Conn, req proto.ListCumulative
return s.rangeScan(rangeScanReq{
MetricID: req.MetricID,
MetricType: diploma.Cumulative,
MetricType: qb.Cumulative,
Since: uint32(since.Unix()),
Until: uint32(until.Unix()),
Conn: conn,
@@ -657,7 +657,7 @@ func (s *Database) ListCumulativePeriods(conn net.Conn, req proto.ListCumulative
type rangeScanReq struct {
MetricID uint32
MetricType diploma.MetricType
MetricType qb.MetricType
Since uint32
Until uint32
Conn io.Writer
@@ -720,14 +720,14 @@ func (s *Database) rangeScan(req rangeScanReq) error {
reply(req.Conn, proto.ErrWrongMetricType)
default:
diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug)
qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
}
return nil
}
type fullScanReq struct {
MetricID uint32
MetricType diploma.MetricType
MetricType qb.MetricType
Conn io.Writer
ResponseWriter atree.PeriodsWriter
}
@@ -768,7 +768,7 @@ func (s *Database) fullScan(req fullScanReq) error {
reply(req.Conn, proto.ErrWrongMetricType)
default:
diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug)
qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
}
return nil
}

216
database/database.go
View File

@@ -12,11 +12,9 @@ import (
"sync"
"time"
diploma "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/bin"
"gordenko.dev/dima/qb/chunkenc"
"gordenko.dev/dima/qb/conbuf"
"gordenko.dev/dima/qb/freelist"
"gordenko.dev/dima/qb/recovery"
"gordenko.dev/dima/qb/txlog"
@@ -84,13 +82,25 @@ func New(opt Options) (_ *Database, err error) {
return nil, errors.New("WaitGroup option is required")
}
filePath := filepath.Join(opt.Dir, opt.DatabaseName+".free")
freeListFile, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE, 0666)
if err != nil {
return nil, err
}
freeList, err := freelist.New(freelist.Options{
PageSize: 2048,
File: freeListFile,
})
s := &Database{
workerSignalCh: make(chan struct{}, 1),
dir: opt.Dir,
databaseName: opt.DatabaseName,
metrics: make(map[uint32]*_metric),
//metricLockEntries: make(map[uint32]*metricLockEntry),
freeList: freelist.New(),
freeList: freeList,
tcpPort: opt.TCPPort,
logfile: opt.Logfile,
logger: log.New(opt.Logfile, "", log.LstdFlags),
@@ -143,7 +153,7 @@ func (s *Database) recovery() {
recipe, err := advisor.GetRecipe()
if err != nil {
diploma.Abort(diploma.GetRecoveryRecipeFailed, err)
qb.Abort(qb.GetRecoveryRecipeFailed, err)
}
var logNumber int
@@ -152,13 +162,13 @@ func (s *Database) recovery() {
if recipe.Snapshot != "" {
err = s.loadSnapshot(recipe.Snapshot)
if err != nil {
diploma.Abort(diploma.LoadSnapshotFailed, err)
qb.Abort(qb.LoadSnapshotFailed, err)
}
}
for _, changesFileName := range recipe.Changes {
err = s.replayChanges(changesFileName)
if err != nil {
diploma.Abort(diploma.ReplayChangesFailed, err)
qb.Abort(qb.ReplayChangesFailed, err)
}
}
logNumber = recipe.LogNumber
@@ -174,28 +184,28 @@ func (s *Database) recovery() {
WaitGroup: s.waitGroup,
})
if err != nil {
diploma.Abort(diploma.CreateChangesWriterFailed, err)
qb.Abort(qb.CreateChangesWriterFailed, err)
}
go s.txlog.Run()
// fileNames, err := s.searchREDOFiles()
// if err != nil {
// diploma.Abort(diploma.SearchREDOFilesFailed, err)
// qb.Abort(qb.SearchREDOFilesFailed, err)
// }
// if len(fileNames) > 0 {
// for _, fileName := range fileNames {
// err = s.replayREDOFile(fileName)
// if err != nil {
// diploma.Abort(diploma.ReplayREDOFileFailed, err)
// qb.Abort(qb.ReplayREDOFileFailed, err)
// }
// }
// for _, fileName := range fileNames {
// err = os.Remove(fileName)
// if err != nil {
// diploma.Abort(diploma.RemoveREDOFileFailed, err)
// qb.Abort(qb.RemoveREDOFileFailed, err)
// }
// }
// }
@@ -204,14 +214,14 @@ func (s *Database) recovery() {
if recipe.CompleteSnapshot {
err = s.dumpSnapshot(logNumber)
if err != nil {
diploma.Abort(diploma.DumpSnapshotFailed, err)
qb.Abort(qb.DumpSnapshotFailed, err)
}
}
for _, fileName := range recipe.ToDelete {
err = os.Remove(fileName)
if err != nil {
diploma.Abort(diploma.RemoveRecipeFileFailed, err)
qb.Abort(qb.RemoveRecipeFileFailed, err)
}
}
}
@@ -348,100 +358,100 @@ func (s *Database) replayChanges(fileName string) error {
}
func (s *Database) replayChangesRecord(untyped any) error {
switch rec := untyped.(type) {
case txlog.AddedMetric:
var (
values diploma.ValueCompressor
timestampsBuf = conbuf.New(nil)
valuesBuf = conbuf.New(nil)
)
// switch rec := untyped.(type) {
// case txlog.AddedMetric:
// var (
// values qb.ValueCompressor
// timestampsBuf = conbuf.New(nil)
// valuesBuf = conbuf.New(nil)
// )
if rec.MetricType == diploma.Cumulative {
values = chunkenc.NewReverseCumulativeDeltaCompressor(
valuesBuf, 0, byte(rec.FracDigits))
} else {
values = chunkenc.NewReverseInstantDeltaCompressor(
valuesBuf, 0, byte(rec.FracDigits))
}
s.metrics[rec.MetricID] = &_metric{
MetricType: rec.MetricType,
FracDigits: byte(rec.FracDigits),
TimestampsBuf: timestampsBuf,
ValuesBuf: valuesBuf,
Timestamps: chunkenc.NewReverseTimeDeltaCompressor(timestampsBuf, 0),
Values: values,
}
case txlog.DeletedMetric:
delete(s.metrics, rec.MetricID)
if len(rec.FreePageNumbers) > 0 {
s.freeList.AddPages(rec.FreePageNumbers)
}
case txlog.AppendedMeasure:
metric, ok := s.metrics[rec.MetricID]
if ok {
metric.Timestamps.Append(rec.Timestamp)
metric.Values.Append(rec.Value)
if metric.Since == 0 {
metric.Since = rec.Timestamp
metric.SinceValue = rec.Value
}
metric.Until = rec.Timestamp
metric.UntilValue = rec.Value
}
case txlog.AppendedMeasures:
metric, ok := s.metrics[rec.MetricID]
if ok {
for _, measure := range rec.Measures {
metric.Timestamps.Append(measure.Timestamp)
metric.Values.Append(measure.Value)
if metric.Since == 0 {
metric.Since = measure.Timestamp
metric.SinceValue = measure.Value
}
metric.Until = measure.Timestamp
metric.UntilValue = measure.Value
}
}
// case txlog.AppendedMeasureWithOverflow:
// metric, ok := s.metrics[rec.MetricID]
// if ok {
// metric.ReinitBy(rec.Timestamp, rec.Value)
// if rec.IsRootChanged {
// metric.RootPageNo = rec.RootPageNo
// if rec.MetricType == qb.Cumulative {
// values = chunkenc.NewReverseCumulativeDeltaCompressor(
// valuesBuf, 0, byte(rec.FracDigits))
// } else {
// values = chunkenc.NewReverseInstantDeltaCompressor(
// valuesBuf, 0, byte(rec.FracDigits))
// }
// metric.LastPageNo = rec.DataPageNo
// // delete free pages
// if rec.IsDataPageReused {
// s.freeList.DeleteReservedPages([]uint32{
// rec.DataPageNo,
// })
// s.metrics[rec.MetricID] = &_metric{
// MetricType: rec.MetricType,
// FracDigits: byte(rec.FracDigits),
// TimestampsBuf: timestampsBuf,
// ValuesBuf: valuesBuf,
// Timestamps: chunkenc.NewReverseTimeDeltaCompressor(timestampsBuf, 0),
// Values: values,
// }
// if len(rec.ReusedIndexPages) > 0 {
// s.freeList.DeleteReservedPages(rec.ReusedIndexPages)
// case txlog.DeletedMetric:
// delete(s.metrics, rec.MetricID)
// if len(rec.FreePageNumbers) > 0 {
// s.freeList.AddPages(rec.FreePageNumbers)
// }
// case txlog.AppendedMeasure:
// metric, ok := s.metrics[rec.MetricID]
// if ok {
// metric.Timestamps.Append(rec.Timestamp)
// metric.Values.Append(rec.Value)
// if metric.Since == 0 {
// metric.Since = rec.Timestamp
// metric.SinceValue = rec.Value
// }
// metric.Until = rec.Timestamp
// metric.UntilValue = rec.Value
// }
// case txlog.AppendedMeasures:
// metric, ok := s.metrics[rec.MetricID]
// if ok {
// for _, measure := range rec.Measures {
// metric.Timestamps.Append(measure.Timestamp)
// metric.Values.Append(measure.Value)
// if metric.Since == 0 {
// metric.Since = measure.Timestamp
// metric.SinceValue = measure.Value
// }
// metric.Until = measure.Timestamp
// metric.UntilValue = measure.Value
// }
// }
// // case txlog.AppendedMeasureWithOverflow:
// // metric, ok := s.metrics[rec.MetricID]
// // if ok {
// // metric.ReinitBy(rec.Timestamp, rec.Value)
// // if rec.IsRootChanged {
// // metric.RootPageNo = rec.RootPageNo
// // }
// // metric.LastPageNo = rec.DataPageNo
// // // delete free pages
// // if rec.IsDataPageReused {
// // s.freeList.DeleteReservedPages([]uint32{
// // rec.DataPageNo,
// // })
// // }
// // if len(rec.ReusedIndexPages) > 0 {
// // s.freeList.DeleteReservedPages(rec.ReusedIndexPages)
// // }
// // }
// case txlog.DeletedMeasures:
// metric, ok := s.metrics[rec.MetricID]
// if ok {
// metric.DeleteMeasures()
// if len(rec.FreePageNumbers) > 0 {
// s.freeList.AddPages(rec.FreePageNumbers)
// }
// }
// default:
// qb.Abort(qb.UnknownTxLogRecordTypeBug,
// fmt.Errorf("bug: unknown record type %T in TransactionLog", rec))
// }
case txlog.DeletedMeasures:
metric, ok := s.metrics[rec.MetricID]
if ok {
metric.DeleteMeasures()
if len(rec.FreePageNumbers) > 0 {
s.freeList.AddPages(rec.FreePageNumbers)
}
}
default:
diploma.Abort(diploma.UnknownTxLogRecordTypeBug,
fmt.Errorf("bug: unknown record type %T in TransactionLog", rec))
}
return nil
}

48
database/metric.go
View File

@@ -2,7 +2,6 @@ package database
import (
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/conbuf"
)
// METRIC
@@ -10,59 +9,32 @@ import (
type _metric struct {
MetricType qb.MetricType
FracDigits byte
RootPageNo uint32
LastPageNo uint32
SinceValue float64
Since uint32
UntilValue float64
Until uint32
//TimestampsBuf *conbuf.ContinuousBuffer
//ValuesBuf *conbuf.ContinuousBuffer
Timestamps qb.TimestampCompressor
Values qb.ValueCompressor
}
func (s *_metric) ReinitBy(timestamp uint32, value float64) {
// s.TimestampsBuf = conbuf.New(nil)
// s.ValuesBuf = conbuf.New(nil)
// //
// s.Timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor(
// s.TimestampsBuf, 0)
s.Timestamps.Renew()
s.Values.Renew()
// if s.MetricType == octopus.Cumulative {
// s.Values = chunkenc.NewReverseCumulativeDeltaCompressor(
// s.ValuesBuf, 0, s.FracDigits)
// } else {
// s.Values = chunkenc.NewReverseInstantDeltaCompressor(
// s.ValuesBuf, 0, s.FracDigits)
// }
s.Timestamps.Append(timestamp)
s.Values.Append(value)
// s.Timestamps.Append(timestamp)
// s.Values.Append(value)
// s.Since = timestamp
// s.SinceValue = value
// s.Until = timestamp
// s.UntilValue = value
s.Since = timestamp
s.SinceValue = value
s.Until = timestamp
s.UntilValue = value
}
func (s *_metric) DeleteMeasures() {
s.TimestampsBuf = conbuf.New(nil)
s.ValuesBuf = conbuf.New(nil)
// fix
// s.Timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor(
// s.TimestampsBuf, 0)
s.Timestamps.Renew()
s.Values.Renew()
// if s.MetricType == octopus.Cumulative {
// s.Values = chunkenc.NewReverseCumulativeDeltaCompressor(
// s.ValuesBuf, 0, s.FracDigits)
// } else {
// s.Values = chunkenc.NewReverseInstantDeltaCompressor(
// s.ValuesBuf, 0, s.FracDigits)
// }
// END OF FIX
s.RootPageNo = 0
s.LastPageNo = 0
s.Since = 0
s.SinceValue = 0

284
database/snapshot.go
View File

@@ -7,21 +7,22 @@ import (
"os"
"path/filepath"
octopus "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/bin"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/chunkenc"
"gordenko.dev/dima/qb/conbuf"
"gordenko.dev/dima/qb/freelist"
)
/*
Формат:
//lsn - varuint (останній LSN, що змінив дані у RAM)
metricsQty - varuint
[metric]*
где metric - це:
де metric - це:
metricID - 4b
metricType - 1b
fracDigits - 1b
rootPageNo - 4b
lastPageNo - 4b
since - 4b
sinceValue - 8b
@@ -31,14 +32,14 @@ timestamps size - 2b
values size - 2b
timestams payload - Nb
values payload - Nb
dataFreeList size - varuint
dataFreeList size - varsize
dataFreeList - Nb
indexFreeList size - varuint
indexFreeList size - varsize
indexFreeList - Nb
CRC32 - 4b
*/
const metricHeaderSize = 42
const metricHeaderSize = 38
func (s *Database) dumpSnapshot(logNumber int) (err error) {
var (
@@ -66,49 +67,23 @@ func (s *Database) dumpSnapshot(logNumber int) (err error) {
bin.PutUint32(prefix[0:], metricID)
prefix[4] = byte(metric.MetricType)
prefix[5] = metric.FracDigits
bin.PutUint32(prefix[6:], metric.RootPageNo)
bin.PutUint32(prefix[10:], metric.LastPageNo)
bin.PutUint32(prefix[14:], metric.Since)
bin.PutFloat64(prefix[18:], metric.SinceValue)
bin.PutUint32(prefix[26:], metric.Until)
bin.PutFloat64(prefix[30:], metric.UntilValue)
bin.PutUint16(prefix[38:], uint16(tSize))
bin.PutUint16(prefix[40:], uint16(vSize))
bin.PutUint32(prefix[6:], metric.LastPageNo)
bin.PutUint32(prefix[10:], metric.Since)
bin.PutFloat64(prefix[14:], metric.SinceValue)
bin.PutUint32(prefix[22:], metric.Until)
bin.PutFloat64(prefix[26:], metric.UntilValue)
bin.PutUint16(prefix[34:], uint16(tSize))
bin.PutUint16(prefix[36:], uint16(vSize))
_, err = dst.Write(prefix)
if err != nil {
return
}
// copy timestamps
remaining := tSize
for _, buf := range metric.TimestampsBuf.Chunks() {
if remaining < len(buf) {
buf = buf[:remaining]
}
_, err = dst.Write(buf)
if err != nil {
return
}
remaining -= len(buf)
if remaining == 0 {
break
}
}
writeChunks(dst, metric.Timestamps.Chunks(), tSize)
// copy values
remaining = vSize
for _, buf := range metric.ValuesBuf.Chunks() {
if remaining < len(buf) {
buf = buf[:remaining]
}
_, err = dst.Write(buf)
if err != nil {
return
}
remaining -= len(buf)
if remaining == 0 {
break
}
}
writeChunks(dst, metric.Values.Chunks(), vSize)
}
// free data pages
err = freeListWriteTo(s.freeList, dst)
@@ -133,123 +108,124 @@ func (s *Database) dumpSnapshot(logNumber int) (err error) {
return
}
prevLogNumber := logNumber - 1
prevChanges := filepath.Join(s.dir, fmt.Sprintf("%d.changes", prevLogNumber))
prevSnapshot := filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", prevLogNumber))
// prevLogNumber := logNumber - 1
// prevChanges := filepath.Join(s.dir, fmt.Sprintf("%d.changes", prevLogNumber))
// prevSnapshot := filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", prevLogNumber))
isExist, err := isFileExist(prevChanges)
if err != nil {
return
}
// isExist, err := isFileExist(prevChanges)
// if err != nil {
// return
// }
if isExist {
err = os.Remove(prevChanges)
if err != nil {
octopus.Abort(octopus.DeletePrevChangesFileFailed, err)
}
}
// if isExist {
// err = os.Remove(prevChanges)
// if err != nil {
// qb.Abort(qb.DeletePrevChangesFileFailed, err)
// }
// }
isExist, err = isFileExist(prevSnapshot)
if err != nil {
return
}
// isExist, err = isFileExist(prevSnapshot)
// if err != nil {
// return
// }
if isExist {
err = os.Remove(prevSnapshot)
if err != nil {
octopus.Abort(octopus.DeletePrevSnapshotFileFailed, err)
}
}
// if isExist {
// err = os.Remove(prevSnapshot)
// if err != nil {
// qb.Abort(qb.DeletePrevSnapshotFileFailed, err)
// }
// }
return
}
func (s *Database) loadSnapshot(fileName string) (err error) {
// FIX
// var (
// hasher = crc32.NewIEEE()
// metricsQty int
// header = make([]byte, metricHeaderSize)
// body = make([]byte, atree.PageSize)
// )
var (
hasher = crc32.NewIEEE()
header = make([]byte, metricHeaderSize)
body = make([]byte, atree.PageSize)
)
// file, err := os.Open(fileName)
// if err != nil {
// return
// }
file, err := os.Open(fileName)
if err != nil {
return
}
// src := io.TeeReader(file, hasher)
// u64, _, err := bin.ReadVarUint64(src)
// if err != nil {
// return
// }
// metricsQty = int(u64)
src := io.TeeReader(file, hasher)
metricsQty, err := bin.ReadVarSize(src)
if err != nil {
return
}
// for range metricsQty {
// var metric _metric
// err = bin.ReadNInto(src, header)
// if err != nil {
// return
// }
for range metricsQty {
var metric _metric
err = bin.ReadNInto(src, header)
if err != nil {
return
}
// metricID := bin.GetUint32(header[0:])
// metric.MetricType = octopus.MetricType(header[4])
// metric.FracDigits = header[5]
// metric.RootPageNo = bin.GetUint32(header[6:])
// metric.LastPageNo = bin.GetUint32(header[10:])
// metric.Since = bin.GetUint32(header[14:])
// metric.SinceValue = bin.GetFloat64(header[18:])
// metric.Until = bin.GetUint32(header[26:])
// metric.UntilValue = bin.GetFloat64(header[30:])
// tSize := bin.GetUint16(header[38:])
// vSize := bin.GetUint16(header[40:])
metricID, _ := bin.GetUint32(header[0:])
metric.MetricType = qb.MetricType(header[4])
metric.FracDigits = header[5]
metric.LastPageNo, _ = bin.GetUint32(header[6:])
metric.Since, _ = bin.GetUint32(header[10:])
metric.SinceValue, _ = bin.GetFloat64(header[14:])
metric.Until, _ = bin.GetUint32(header[22:])
metric.UntilValue, _ = bin.GetFloat64(header[26:])
tSize, _ := bin.GetUint16(header[34:])
vSize, _ := bin.GetUint16(header[36:])
// buf := body[:tSize]
// err = bin.ReadNInto(src, buf)
// if err != nil {
// return
// }
// metric.TimestampsBuf = conbuf.NewFromBuffer(buf)
buf := body[:tSize]
err = bin.ReadNInto(src, buf)
if err != nil {
return
}
metric.Timestamps = chunkenc.NewReverseTimeDeltaCompressor(
conbuf.NewFromBuffer(buf),
int(tSize),
)
// buf = body[:vSize]
// err = bin.ReadNInto(src, buf)
// if err != nil {
// return
// }
// metric.ValuesBuf = conbuf.NewFromBuffer(buf)
buf = body[:vSize]
err = bin.ReadNInto(src, buf)
if err != nil {
return
}
// metric.Timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor(
// metric.TimestampsBuf, int(tSize))
if metric.MetricType == qb.Cumulative {
metric.Values = chunkenc.NewReverseCumulativeDeltaCompressor(
conbuf.NewFromBuffer(buf),
int(vSize),
metric.FracDigits,
)
} else {
metric.Values = chunkenc.NewReverseInstantDeltaCompressor(
conbuf.NewFromBuffer(buf),
int(vSize),
metric.FracDigits,
)
}
s.metrics[metricID] = &metric
}
// if metric.MetricType == octopus.Cumulative {
// metric.Values = chunkenc.NewReverseCumulativeDeltaCompressor(
// metric.ValuesBuf, int(vSize), metric.FracDigits)
// } else {
// metric.Values = chunkenc.NewReverseInstantDeltaCompressor(
// metric.ValuesBuf, int(vSize), metric.FracDigits)
// }
// s.metrics[metricID] = &metric
// }
err = restoreFreeList(s.freeList, src)
if err != nil {
return fmt.Errorf("restore dataFreeList: %s", err)
}
// err = restoreFreeList(s.freeList, src)
// if err != nil {
// return fmt.Errorf("restore dataFreeList: %s", err)
// }
err = restoreFreeList(s.freeList, src)
if err != nil {
return fmt.Errorf("restore indexFreeList: %s", err)
}
// err = restoreFreeList(s.freeList, src)
// if err != nil {
// return fmt.Errorf("restore indexFreeList: %s", err)
// }
calculatedChecksum := hasher.Sum32()
// calculatedChecksum := hasher.Sum32()
writtenChecksum, err := bin.ReadUint32(file)
if err != nil {
return
}
// writtenChecksum, err := bin.ReadUint32(file)
// if err != nil {
// return
// }
// if calculatedChecksum != writtenChecksum {
// return fmt.Errorf("calculated checksum %d not equal written checksum %d", calculatedChecksum, writtenChecksum)
// }
if calculatedChecksum != writtenChecksum {
return fmt.Errorf("calculated checksum %d not equal written checksum %d", calculatedChecksum, writtenChecksum)
}
return
}
@@ -258,9 +234,9 @@ func (s *Database) loadSnapshot(fileName string) (err error) {
func freeListWriteTo(freeList *freelist.FreeList, dst io.Writer) error {
serialized, err := freeList.Serialize()
if err != nil {
octopus.Abort(octopus.FailedFreeListSerialize, err)
qb.Abort(qb.FailedFreeListSerialize, err)
}
_, err = bin.WriteVarUint64(dst, uint64(len(serialized)))
_, err = bin.WriteVarSize(dst, len(serialized))
if err != nil {
return err
}
@@ -272,14 +248,32 @@ func freeListWriteTo(freeList *freelist.FreeList, dst io.Writer) error {
}
func restoreFreeList(freeList *freelist.FreeList, src io.Reader) error {
size, _, err := bin.ReadVarUint64(src)
size, err := bin.ReadVarSize(src)
if err != nil {
return err
}
serialized, err := bin.ReadN(src, int(size))
serialized, err := bin.ReadN(src, size)
if err != nil {
return err
}
freeList.Restore(serialized)
return nil
}
func writeChunks(dst io.Writer, chunks [][]byte, size int) (err error) {
remaining := size
for _, buf := range chunks {
if remaining < len(buf) {
buf = buf[:remaining]
}
_, err = dst.Write(buf)
if err != nil {
return
}
remaining -= len(buf)
if remaining == 0 {
break
}
}
return
}