dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tmp

Browse Source
This commit is contained in:
dima
2026-05-31 20:01:28 +00:00
parent e7ec6083cb
commit db5ecd3dfd
28 changed files with 2090 additions and 2155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
database/database.go
View File

@@ -12,11 +12,9 @@ import (
"sync"
"time"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/bin"
"gordenko.dev/dima/qb/freelist"
"gordenko.dev/dima/qb/recovery"
"gordenko.dev/dima/qb/txlog"
)
@@ -24,11 +22,11 @@ func JoinSnapshotFileName(dir string, logNumber int) string {
return filepath.Join(dir, fmt.Sprintf("%d.snapshot", logNumber))
}
type metricLockEntry struct {
XLock bool
RLocks int
WaitQueue []any
}
// type metricLockEntry struct {
// XLock bool
// RLocks int
// WaitQueue []any
// }
type Database struct {
mutex sync.Mutex
@@ -37,16 +35,17 @@ type Database struct {
rLocksToRelease []uint32
metrics map[uint32]*_metric
//metricLockEntries map[uint32]*metricLockEntry
freeList *freelist.FreeList
dir string
databaseName string
txlog *txlog.Writer
atree *atree.Atree
tcpPort int
logfile *os.File
logger *log.Logger
exitCh chan struct{}
waitGroup *sync.WaitGroup
dataFreeList *freelist.FreeList
indexFreeList *freelist.FreeList
dir string
databaseName string
txlog *txlog.Writer
atree *atree.Atree
tcpPort int
logfile *os.File
logger *log.Logger
exitCh chan struct{}
waitGroup *sync.WaitGroup
}
type Options struct {
@@ -82,16 +81,25 @@ func New(opt Options) (_ *Database, err error) {
return nil, errors.New("WaitGroup option is required")
}
filePath := filepath.Join(opt.Dir, opt.DatabaseName+".free")
// deltaFreeListFile, err := os.OpenFile(
// filepath.Join(opt.Dir, opt.DatabaseName+".deltafree"),
// os.O_RDWR|os.O_CREATE,
// 0666,
// )
// if err != nil {
// return nil, err
// }
freeListFile, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE, 0666)
if err != nil {
return nil, err
}
dataFreeList, err := freelist.New(freelist.Options{
PageSize: 2048,
BaseFilePath: filepath.Join(opt.Dir, opt.DatabaseName+".free_data_base"),
DeltaFilePath: filepath.Join(opt.Dir, opt.DatabaseName+".free_data_delta"),
})
freeList, err := freelist.New(freelist.Options{
PageSize: 2048,
File: freeListFile,
indexFreeList, err := freelist.New(freelist.Options{
PageSize: 2048,
BaseFilePath: filepath.Join(opt.Dir, opt.DatabaseName+".free_index_base"),
DeltaFilePath: filepath.Join(opt.Dir, opt.DatabaseName+".free_index_delta"),
})
s := &Database{
@@ -100,12 +108,13 @@ func New(opt Options) (_ *Database, err error) {
databaseName: opt.DatabaseName,
metrics: make(map[uint32]*_metric),
//metricLockEntries: make(map[uint32]*metricLockEntry),
freeList: freeList,
tcpPort: opt.TCPPort,
logfile: opt.Logfile,
logger: log.New(opt.Logfile, "", log.LstdFlags),
exitCh: opt.ExitCh,
waitGroup: opt.WaitGroup,
dataFreeList: dataFreeList,
indexFreeList: indexFreeList,
tcpPort: opt.TCPPort,
logfile: opt.Logfile,
logger: log.New(opt.Logfile, "", log.LstdFlags),
exitCh: opt.ExitCh,
waitGroup: opt.WaitGroup,
}
return s, nil
}
@@ -143,88 +152,89 @@ func (s *Database) ListenAndServe() (err error) {
}
func (s *Database) recovery() {
advisor, err := recovery.NewRecoveryAdvisor(recovery.RecoveryAdvisorOptions{
Dir: s.dir,
VerifySnapshot: s.verifySnapshot,
})
if err != nil {
panic(err)
}
recipe, err := advisor.GetRecipe()
if err != nil {
qb.Abort(qb.GetRecoveryRecipeFailed, err)
}
var logNumber int
if recipe != nil {
if recipe.Snapshot != "" {
err = s.loadSnapshot(recipe.Snapshot)
if err != nil {
qb.Abort(qb.LoadSnapshotFailed, err)
}
}
for _, changesFileName := range recipe.Changes {
err = s.replayChanges(changesFileName)
if err != nil {
qb.Abort(qb.ReplayChangesFailed, err)
}
}
logNumber = recipe.LogNumber
}
s.txlog, err = txlog.NewWriter(txlog.WriterOptions{
Dir: s.dir,
LogNumber: logNumber,
AppendToWorkerQueue: s.appendJobToWorkerQueue,
FreeList: s.freeList,
Atree: s.atree,
ExitCh: s.exitCh,
WaitGroup: s.waitGroup,
})
if err != nil {
qb.Abort(qb.CreateChangesWriterFailed, err)
}
go s.txlog.Run()
// fileNames, err := s.searchREDOFiles()
// FIX
// advisor, err := recovery.NewRecoveryAdvisor(recovery.RecoveryAdvisorOptions{
// Dir: s.dir,
// VerifySnapshot: s.verifySnapshot,
// })
// if err != nil {
// qb.Abort(qb.SearchREDOFilesFailed, err)
// panic(err)
// }
// if len(fileNames) > 0 {
// for _, fileName := range fileNames {
// err = s.replayREDOFile(fileName)
// recipe, err := advisor.GetRecipe()
// if err != nil {
// qb.Abort(qb.GetRecoveryRecipeFailed, err)
// }
// var logNumber int
// if recipe != nil {
// if recipe.Snapshot != "" {
// err = s.loadSnapshot(recipe.Snapshot)
// if err != nil {
// qb.Abort(qb.ReplayREDOFileFailed, err)
// qb.Abort(qb.LoadSnapshotFailed, err)
// }
// }
// for _, changesFileName := range recipe.Changes {
// err = s.replayChanges(changesFileName)
// if err != nil {
// qb.Abort(qb.ReplayChangesFailed, err)
// }
// }
// logNumber = recipe.LogNumber
// }
// s.txlog, err = txlog.NewWriter(txlog.WriterOptions{
// Dir: s.dir,
// LogNumber: logNumber,
// AppendToWorkerQueue: s.appendJobToWorkerQueue,
// FreeList: s.freeList,
// Atree: s.atree,
// ExitCh: s.exitCh,
// WaitGroup: s.waitGroup,
// })
// if err != nil {
// qb.Abort(qb.CreateChangesWriterFailed, err)
// }
// go s.txlog.Run()
// // fileNames, err := s.searchREDOFiles()
// // if err != nil {
// // qb.Abort(qb.SearchREDOFilesFailed, err)
// // }
// // if len(fileNames) > 0 {
// // for _, fileName := range fileNames {
// // err = s.replayREDOFile(fileName)
// // if err != nil {
// // qb.Abort(qb.ReplayREDOFileFailed, err)
// // }
// // }
// // for _, fileName := range fileNames {
// // err = os.Remove(fileName)
// // if err != nil {
// // qb.Abort(qb.RemoveREDOFileFailed, err)
// // }
// // }
// // }
// if recipe != nil {
// if recipe.CompleteSnapshot {
// err = s.dumpSnapshot(logNumber)
// if err != nil {
// qb.Abort(qb.DumpSnapshotFailed, err)
// }
// }
// for _, fileName := range fileNames {
// for _, fileName := range recipe.ToDelete {
// err = os.Remove(fileName)
// if err != nil {
// qb.Abort(qb.RemoveREDOFileFailed, err)
// qb.Abort(qb.RemoveRecipeFileFailed, err)
// }
// }
// }
if recipe != nil {
if recipe.CompleteSnapshot {
err = s.dumpSnapshot(logNumber)
if err != nil {
qb.Abort(qb.DumpSnapshotFailed, err)
}
}
for _, fileName := range recipe.ToDelete {
err = os.Remove(fileName)
if err != nil {
qb.Abort(qb.RemoveRecipeFileFailed, err)
}
}
}
}
// func (s *Database) searchREDOFiles() ([]string, error) {

272
database/metric.go
View File

@@ -2,10 +2,18 @@ package database
import (
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/bin"
"gordenko.dev/dima/qb/txlog"
)
// METRIC
type IndexRec struct {
Since uint32
PageNo uint32
}
type _metric struct {
MetricType qb.MetricType
FracDigits byte
@@ -16,6 +24,12 @@ type _metric struct {
Until uint32
Timestamps qb.TimestampCompressor
Values qb.ValueCompressor
// fix - add index levels
XLock bool
RLocks int
WaitQueue []any
//IndexLevels [][]IndexRec // root - last element
IndexLevels [][]byte // root - last element
}
func (s *_metric) ReinitBy(timestamp uint32, value float64) {
@@ -41,3 +55,261 @@ func (s *_metric) DeleteMeasures() {
s.Until = 0
s.UntilValue = 0
}
// func (s *_metric) startAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(txlog.AppendedMeasures)) {
func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(any)) {
var (
// AppendedMeasures struct {
// MetricID uint32
// TimestampsOffset int // (заповнені одразу)
// ValuesOffset int // (заповнені одразу)
// Timestamps [][]byte
// TimestampsSize int
// Values [][]byte
// ValuesSize int
// IndexLevels []*atree.IndexLevel
// DataPages []*atree.DataPage // fix - prevPageNo for the 1st data page
// }
timestamps = s.Timestamps
values = s.Values
indexLevels []*atree.IndexLevel
dataPages []*atree.DataPage
//written int
//resultCode byte
)
s.Values.Lock()
s.Timestamps.Lock()
for idx, measure := range req.Measures {
if s.Since == 0 {
s.Since = measure.Timestamp
} else {
// FIX - у випадку помилки треба в транзакції зберегти що помилка, але також зафіксувати скільки елементів збережено.
// якщо idx == 0 - одразу знімаю блокування і нічого не відправляю в txlog
if measure.Timestamp <= s.Until {
if idx == 0 {
s.Values.Unlock()
s.Timestamps.Unlock()
req.ResultCh <- tryAppendMeasuresResult{
ResultCode: ExpiredMeasure,
}
return
}
//resultCode = ExpiredMeasure
//written = idx
break
}
if s.MetricType == qb.Cumulative && measure.Value < s.UntilValue {
if idx == 0 {
s.Values.Unlock()
s.Timestamps.Unlock()
req.ResultCh <- tryAppendMeasuresResult{
ResultCode: NonMonotonicValue,
}
return
}
//resultCode = NonMonotonicValue
//written = idx
break
}
}
// fix - 1 + 8 bytes
extraSpace := timestamps.CalcRequiredSpace(measure.Timestamp) +
values.CalcRequiredSpace(measure.Value)
totalSpace := timestamps.Size() + values.Size() + extraSpace
if totalSpace <= atree.DataPagePayloadSize {
// накопичую
timestamps.Append(measure.Timestamp)
values.Append(measure.Value)
} else {
// сторінка заповнена
// prevPageNo - виставляю в txlog, коли забираю номер сторінки із freeList або генерую новий
dataPages = append(dataPages, &atree.DataPage{
PrevPageNo: 0, // ?
LowerTimestamp: s.Since,
Timestamps: timestamps.Chunks(),
TimestampsSize: timestamps.Size(),
Values: values.Chunks(),
ValuesSize: values.Size(),
})
// обнуляються буфери, але state незмінний
timestamps.Renew()
values.Renew()
timestamps.Append(measure.Timestamp)
values.Append(measure.Value)
s.Since = measure.Timestamp
}
s.Until = measure.Timestamp
s.UntilValue = measure.Value
}
// виділити змінені байти.
// скопіювати. Причому можна скопіювати зрізи chunks
if len(dataPages) > 0 {
// пишу в txlog довгим шляхом через redo файл і запис в data файл
for _, unfilled := range s.IndexLevels {
indexLevels = append(indexLevels, &atree.IndexLevel{
Offset: len(unfilled),
LowerTimestamp: bin.GetUint32(unfilled),
Data: unfilled,
})
}
} else {
// короткий шлях - запис лише в txlog
}
sendToStorage(txlog.AppendedMeasures{
MetricID: req.MetricID,
TimestampsOffset: timestamps.Offset(), // state.Pos() з якої позиції дописувати дані на сторінку 0 (при відновленні)
ValuesOffset: values.Offset(), // state.Pos()
Timestamps: timestamps.Chunks(),
TimestampsSize: timestamps.Size(),
Values: values.Chunks(),
ValuesSize: values.Size(),
IndexLevels: indexLevels,
DataPages: dataPages,
})
}
func (s *_metric) FinAppendMeasures(rec txlog.AppendMeasuresSummary) {
// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
s.Values.Unlock()
s.Timestamps.Unlock()
// fix write index levels
// update prev pageNo
// if len(rec.DataPages) > 0 {
// s.LastPageNo = rec.DataPages[len(rec.DataPages)-1].PageNo
// }
if rec.LastPageNo > 0 {
s.LastPageNo = rec.LastPageNo
}
// В txlog я передав повний індекс. У нього додали елементи (можливо нові рівні).
// Тому проста заміна
s.IndexLevels = rec.Index
}
// READ
func (s *_metric) StartRangeScan(req tryRangeScanReq) {
if s.Since == 0 {
req.ResultCh <- rangeScanResult{
ResultCode: QueryDone,
}
return
}
if req.Since > s.Until {
req.ResultCh <- rangeScanResult{
ResultCode: QueryDone,
}
return
}
if req.Until < s.Since {
if s.RootPageNo > 0 {
req.ResultCh <- rangeScanResult{
ResultCode: UntilNotFound,
RootPageNo: s.RootPageNo,
FracDigits: s.FracDigits,
}
s.RLocks++
return
} else {
req.ResultCh <- rangeScanResult{
ResultCode: QueryDone,
}
return
}
}
timestampDecompressor := s.Timestamps.CreateDecompressor()
valueDecompressor := s.Values.CreateDecompressor(s.FracDigits)
for {
timestamp, done := timestampDecompressor.NextValue()
if done {
break
}
value, done := valueDecompressor.NextValue()
if done {
qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug)
}
if timestamp <= req.Until {
req.ResponseWriter.FeedNoSend(timestamp, value)
if timestamp < req.Since {
req.ResultCh <- rangeScanResult{
ResultCode: QueryDone,
}
return
}
}
}
if s.LastPageNo > 0 {
req.ResultCh <- rangeScanResult{
ResultCode: UntilFound,
LastPageNo: s.LastPageNo,
FracDigits: s.FracDigits,
}
s.RLocks++
} else {
req.ResultCh <- rangeScanResult{
ResultCode: QueryDone,
}
}
}
func (s *_metric) StartFullScan(req tryFullScanReq) {
if s.Since == 0 {
req.ResultCh <- fullScanResult{
ResultCode: QueryDone,
}
return
}
timestampDecompressor := s.Timestamps.CreateDecompressor()
valueDecompressor := s.Values.CreateDecompressor(s.FracDigits)
for {
timestamp, done := timestampDecompressor.NextValue()
if done {
break
}
value, done := valueDecompressor.NextValue()
if done {
qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug)
}
req.ResponseWriter.FeedNoSend(timestamp, value)
}
if s.LastPageNo > 0 {
req.ResultCh <- fullScanResult{
ResultCode: UntilFound,
LastPageNo: s.LastPageNo,
FracDigits: s.FracDigits,
}
s.RLocks++
} else {
req.ResultCh <- fullScanResult{
ResultCode: QueryDone,
}
}
}

762
database/proc.go
View File

@@ -12,6 +12,11 @@ import (
"gordenko.dev/dima/qb/txlog"
)
// AddMetric - create metric in map and set Xlock, rwad - append to queue
// DeleteMetric - set Xlock
// DeleteSince - xLock (тупо редкая операция)
// Якийсь прапор поставити для pendingDelete, щоб read операції вставали в чергу
const (
QueryDone = 1
UntilFound = 2
@@ -49,46 +54,33 @@ func (s *Database) DoWork() {
s.mutex.Unlock()
for _, metricID := range rLocksToRelease {
lockEntry, ok := s.metricLockEntries[metricID]
metric, ok := s.metrics[metricID]
if !ok {
qb.Abort(qb.NoLockEntryBug,
fmt.Errorf("drainQueues: lockEntry not found for the metric %d",
metricID))
qb.Abort(qb.NoMetricBug,
fmt.Errorf("drainQueues: metric %d not found", metricID))
}
if lockEntry.XLock {
if metric.XLock {
qb.Abort(qb.XLockBug,
fmt.Errorf("drainQueues: xlock is set for the metric %d",
metricID))
}
if lockEntry.RLocks <= 0 {
if metric.RLocks <= 0 {
qb.Abort(qb.NoRLockBug,
fmt.Errorf("drainQueues: rlock not set for the metric %d",
metricID))
}
lockEntry.RLocks--
metric.RLocks--
if len(lockEntry.WaitQueue) > 0 {
metric, ok := s.metrics[metricID]
if !ok {
qb.Abort(qb.NoMetricBug,
fmt.Errorf("drainQueues: metric %d not found", metricID))
}
s.processMetricQueue(metricID, metric, lockEntry)
} else {
if lockEntry.RLocks == 0 {
delete(s.metricLockEntries, metricID)
}
if len(metric.WaitQueue) > 0 {
s.processMetricQueue(metricID, metric)
}
}
for _, untyped := range workerQueue {
switch req := untyped.(type) {
//case tryAppendMeasureReq:
// s.tryAppendMeasure(req)
case tryAppendMeasuresReq:
s.tryAppendMeasures(req)
@@ -123,64 +115,35 @@ func (s *Database) DoWork() {
}
}
func (s *Database) processMetricQueue(metricID uint32, metric *_metric, lockEntry *metricLockEntry) {
if len(lockEntry.WaitQueue) == 0 {
// суть у тому що треба запускати запити, пока не зустріну XLock
func (s *Database) processMetricQueue(metricID uint32, metric *_metric) {
if len(metric.WaitQueue) == 0 {
return
}
var modificationReqs []any
for _, untyped := range lockEntry.WaitQueue {
var rLockRequired bool
for _, untyped := range metric.WaitQueue {
switch req := untyped.(type) {
case tryRangeScanReq:
rLockRequired = s.startRangeScan(metric, req)
metric.StartRangeScan(req)
case tryFullScanReq:
rLockRequired = s.startFullScan(metric, req)
metric.StartFullScan(req)
case tryGetMetricReq:
s.tryGetMetric(req)
case tryAppendMeasuresReq:
metric.StartAppendMeasures(req, s.txlog.Append)
case tryDeleteMetricReq:
s.startDeleteMetric(metric, req)
case tryDeleteMeasuresReq:
s.startDeleteMeasures(metric, req)
default:
modificationReqs = append(modificationReqs, untyped)
}
if rLockRequired {
lockEntry.RLocks++
}
}
lockEntry.WaitQueue = nil
if lockEntry.RLocks > 0 {
lockEntry.WaitQueue = modificationReqs
} else {
for idx, untyped := range modificationReqs {
switch req := untyped.(type) {
//case tryAppendMeasureReq:
// s.startAppendMeasure(metric, req, nil)
case tryAppendMeasuresReq:
s.startAppendMeasures(metric, req, nil)
case tryDeleteMetricReq:
s.startDeleteMetric(metric, req)
case tryDeleteMeasuresReq:
s.startDeleteMeasures(metric, req)
default:
qb.Abort(qb.UnknownMetricWaitQueueItemBug,
fmt.Errorf("bug: unknown metric wait queue item type %T", req))
}
lockEntry, ok := s.metricLockEntries[metricID]
if ok {
start := idx + 1
if start < len(modificationReqs) {
lockEntry.WaitQueue = append(lockEntry.WaitQueue, modificationReqs[start:]...)
}
break
}
qb.Abort(qb.UnknownMetricWaitQueueItemBug,
fmt.Errorf("bug: unknown metric wait queue item type %T", req))
}
}
}
@@ -332,222 +295,16 @@ func (s *Database) startDeleteMeasures(metric *_metric, req tryDeleteMeasuresReq
// }
}
// type tryAppendMeasureReq struct {
// MetricID uint32
// Timestamp uint32
// Value float64
// ResultCh chan tryAppendMeasureResult
// }
// func (s *Database) tryAppendMeasure(req tryAppendMeasureReq) {
// metric, ok := s.metrics[req.MetricID]
// if !ok {
// req.ResultCh <- tryAppendMeasureResult{
// MetricID: req.MetricID,
// ResultCode: NoMetric,
// }
// return
// }
// lockEntry, ok := s.metricLockEntries[req.MetricID]
// if ok {
// if lockEntry.XLock {
// lockEntry.WaitQueue = append(lockEntry.WaitQueue, req)
// return
// }
// }
// s.startAppendMeasure(metric, req, lockEntry)
// }
type ReadBound struct {
ValuesPos int
ValuesLastValue float64
}
// func (s *Database) startAppendMeasure(metric *_metric, req tryAppendMeasureReq, lockEntry *metricLockEntry) {
// if req.Timestamp <= metric.Until {
// req.ResultCh <- tryAppendMeasureResult{
// MetricID: req.MetricID,
// ResultCode: ExpiredMeasure,
// }
// return
// }
// if metric.MetricType == qb.Cumulative && req.Value < metric.UntilValue {
// req.ResultCh <- tryAppendMeasureResult{
// MetricID: req.MetricID,
// ResultCode: NonMonotonicValue,
// }
// return
// }
// extraSpace := metric.Timestamps.CalcRequiredSpace(req.Timestamp) +
// metric.Values.CalcRequiredSpace(req.Value)
// totalSpace := metric.Timestamps.Size() + metric.Values.Size() + extraSpace
// if totalSpace <= atree.DataPagePayloadSize {
// if lockEntry != nil {
// lockEntry.RLocks++
// } else {
// s.metricLockEntries[req.MetricID] = &metricLockEntry{
// RLocks: 1,
// }
// }
// req.ResultCh <- tryAppendMeasureResult{
// MetricID: req.MetricID,
// Timestamp: req.Timestamp,
// Value: req.Value,
// ResultCode: CanAppend,
// }
// } else {
// if lockEntry != nil {
// if lockEntry.RLocks > 0 {
// lockEntry.WaitQueue = append(lockEntry.WaitQueue, req)
// return
// }
// lockEntry.XLock = true
// } else {
// s.metricLockEntries[req.MetricID] = &metricLockEntry{
// XLock: true,
// }
// }
// req.ResultCh <- tryAppendMeasureResult{
// MetricID: req.MetricID,
// Timestamp: req.Timestamp,
// Value: req.Value,
// ResultCode: NewPage,
// FilledPage: &FilledPage{
// Since: metric.Since,
// RootPageNo: metric.RootPageNo,
// PrevPageNo: metric.LastPageNo,
// TimestampsChunks: metric.TimestampsBuf.Chunks(),
// TimestampsSize: uint16(metric.Timestamps.Size()),
// ValuesChunks: metric.ValuesBuf.Chunks(),
// ValuesSize: uint16(metric.Values.Size()),
// },
// }
// }
// }
// func (s *Database) appendMeasure(rec txlog.AppendedMeasure) {
// metric, ok := s.metrics[rec.MetricID]
// if !ok {
// qb.Abort(qb.NoMetricBug,
// fmt.Errorf("appendMeasure: metric %d not found",
// rec.MetricID))
// }
// lockEntry, ok := s.metricLockEntries[rec.MetricID]
// if !ok {
// qb.Abort(qb.NoLockEntryBug,
// fmt.Errorf("appendMeasure: lockEntry not found for the metric %d",
// rec.MetricID))
// }
// if lockEntry.XLock {
// qb.Abort(qb.XLockBug,
// fmt.Errorf("appendMeasure: xlock is set for the metric %d",
// rec.MetricID))
// }
// if lockEntry.RLocks <= 0 {
// qb.Abort(qb.NoRLockBug,
// fmt.Errorf("appendMeasure: rlock not set for the metric %d",
// rec.MetricID))
// }
// if metric.Since == 0 {
// metric.Since = rec.Timestamp
// metric.SinceValue = rec.Value
// }
// metric.Timestamps.Append(rec.Timestamp)
// metric.Values.Append(rec.Value)
// metric.Until = rec.Timestamp
// metric.UntilValue = rec.Value
// lockEntry.RLocks--
// if len(lockEntry.WaitQueue) > 0 {
// s.processMetricQueue(rec.MetricID, metric, lockEntry)
// } else {
// if lockEntry.RLocks == 0 {
// delete(s.metricLockEntries, rec.MetricID)
// }
// }
// }
func (s *Database) appendMeasures(rec txlog.AppendedMeasures) {
func (s *Database) finAppendMeasures(rec txlog.AppendMeasuresSummary) {
metric, ok := s.metrics[rec.MetricID]
if !ok {
qb.Abort(qb.NoMetricBug,
fmt.Errorf("appendMeasureAfterOverflow: metric %d not found",
fmt.Errorf("finAppendMeasures: metric %d not found",
rec.MetricID))
}
metric.Values.Unlock()
metric.Timestamps.Unlock()
// lockEntry, ok := s.metricLockEntries[rec.MetricID]
// if !ok {
// qb.Abort(qb.NoLockEntryBug,
// fmt.Errorf("appendMeasureAfterOverflow: lockEntry not found for the metric %d",
// rec.MetricID))
// }
// if !lockEntry.XLock {
// qb.Abort(qb.NoXLockBug,
// fmt.Errorf("appendMeasureAfterOverflow: xlock not set for the metric %d",
// rec.MetricID))
// }
// lockEntry.XLock = false
// s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry)
metric.FinAppendMeasures(rec)
}
// func (s *Database) appendMeasureAfterOverflow(extended txlog.AppendedMeasureWithOverflowExtended) {
// rec := extended.Record
// metric, ok := s.metrics[rec.MetricID]
// if !ok {
// qb.Abort(qb.NoMetricBug,
// fmt.Errorf("appendMeasureAfterOverflow: metric %d not found",
// rec.MetricID))
// }
// lockEntry, ok := s.metricLockEntries[rec.MetricID]
// if !ok {
// qb.Abort(qb.NoLockEntryBug,
// fmt.Errorf("appendMeasureAfterOverflow: lockEntry not found for the metric %d",
// rec.MetricID))
// }
// if !lockEntry.XLock {
// qb.Abort(qb.NoXLockBug,
// fmt.Errorf("appendMeasureAfterOverflow: xlock not set for the metric %d",
// rec.MetricID))
// }
// metric.ReinitBy(rec.Timestamp, rec.Value)
// if rec.IsRootChanged {
// metric.RootPageNo = rec.RootPageNo
// }
// metric.LastPageNo = rec.DataPageNo
// if rec.IsDataPageReused {
// s.freeList.DeleteReservedPages([]uint32{
// rec.DataPageNo,
// })
// }
// if len(rec.ReusedIndexPages) > 0 {
// s.freeList.DeleteReservedPages(rec.ReusedIndexPages)
// }
// lockEntry.XLock = false
// s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry)
// }
type tryAppendMeasuresReq struct {
MetricID uint32
Measures []proto.Measure
@@ -562,175 +319,11 @@ func (s *Database) tryAppendMeasures(req tryAppendMeasuresReq) {
}
return
}
lockEntry, ok := s.metricLockEntries[req.MetricID]
if ok {
if lockEntry.XLock {
lockEntry.WaitQueue = append(lockEntry.WaitQueue, req)
return
}
if metric.XLock {
metric.WaitQueue = append(metric.WaitQueue, req)
return
}
s.startAppendMeasures(metric, req, lockEntry)
}
func (s *Database) startAppendMeasures(metric *_metric, req tryAppendMeasuresReq, lockEntry *metricLockEntry) {
var (
timestamps = metric.Timestamps
values = metric.Values
dataPages []atree.NotLinkedDataPage
resultCode byte
written int
)
metric.Values.Lock()
metric.Timestamps.Lock()
for idx, measure := range req.Measures {
if metric.Since == 0 {
metric.Since = measure.Timestamp
} else {
// FIX - у випадку помилки треба в транзакції зберегти що помилка, але також зафіксувати скільки елементів збережено.
// якщо idx == 0 - одразу знімаю блокування і нічого не відправляю в txlog
if measure.Timestamp <= metric.Until {
if idx == 0 {
metric.Values.Unlock()
metric.Timestamps.Unlock()
req.ResultCh <- tryAppendMeasuresResult{
ResultCode: ExpiredMeasure,
}
return
}
resultCode = ExpiredMeasure
written = idx
break
}
if metric.MetricType == qb.Cumulative && measure.Value < metric.UntilValue {
if idx == 0 {
metric.Values.Unlock()
metric.Timestamps.Unlock()
req.ResultCh <- tryAppendMeasuresResult{
ResultCode: NonMonotonicValue,
}
return
}
resultCode = NonMonotonicValue
written = idx
break
}
}
// fix - 1 + 8 bytes
extraSpace := timestamps.CalcRequiredSpace(measure.Timestamp) +
values.CalcRequiredSpace(measure.Value)
totalSpace := timestamps.Size() + values.Size() + extraSpace
if totalSpace <= atree.DataPagePayloadSize {
// накопичую
timestamps.Append(measure.Timestamp)
values.Append(measure.Value)
} else {
// сторінка заповнена
pageData := make([]byte, atree.PageSize)
// prevPageNo - виставляю в txlog, коли забираю номер сторінки із freeList або генерую новий
atree.ChunksToNotLinkedDataPage(pageData, atree.ChunksToNotLinkedDataPageReq{
TimestampsChunks: timestamps.Chunks(),
TimestampsSize: uint16(timestamps.Size()),
ValuesChunks: values.Chunks(),
ValuesSize: uint16(values.Size()),
})
dataPages = append(dataPages, atree.NotLinkedDataPage{
Since: metric.Since,
Data: pageData,
})
// FIX
// prevPageNo = report.DataPageNo
// if report.IsRootChanged {
// rootPageNo = report.NewRootPageNo
// }
// waitCh := s.txlog.WriteAppendedMeasureWithOverflow(
// txlog.AppendedMeasureWithOverflow{
// MetricID: req.MetricID,
// Timestamp: measure.Timestamp,
// Value: measure.Value,
// IsDataPageReused: report.IsDataPageReused,
// DataPageNo: report.DataPageNo,
// IsRootChanged: report.IsRootChanged,
// RootPageNo: report.NewRootPageNo,
// ReusedIndexPages: report.ReusedIndexPages,
// },
// (idx+1) < len(req.Measures),
// )
// <-waitCh
timestamps.Renew()
values.Renew()
timestamps.Append(measure.Timestamp)
values.Append(measure.Value)
metric.Since = measure.Timestamp
}
metric.Until = measure.Timestamp
metric.UntilValue = measure.Value
}
// виділити змінені байти.
// скопіювати. Причому можна скопіювати зрізи chunks
if len(dataPages) > 0 {
// пишу в txlog довгим шляхом через redo файл і запис в data файл
} else {
// короткий шлях - запис лише в txlog
}
// waitCh := s.txlog.Append(txlog.AppendedPagesReq{
// MetricID: req.MetricID,
// Timestamp: until,
// Value: untilValue,
// LastPageNo: prevPageNo,
// //Measures: toAppendMeasures,
// },
// )
// <-waitCh
// if lockEntry != nil {
// if lockEntry.RLocks > 0 {
// lockEntry.WaitQueue = append(lockEntry.WaitQueue, req)
// return
// }
// lockEntry.XLock = true
// } else {
// s.metricLockEntries[req.MetricID] = &metricLockEntry{
// XLock: true,
// }
// }
}
type PartialChange struct {
Data [][]byte
Size int
Offset int
}
type AppendedMeasures struct {
RootPageNo uint32
LastPageNo uint32
DataPages []atree.ChunksToNotLinkedDataPageReq
Values PartialChange
Timestamps PartialChange
ResultCh chan tryAppendMeasuresResult
metric.StartAppendMeasures(req, s.txlog.Append)
}
type tryRangeScanReq struct {
@@ -757,102 +350,12 @@ func (s *Database) tryRangeScan(req tryRangeScanReq) {
return
}
lockEntry, ok := s.metricLockEntries[req.MetricID]
if ok {
if lockEntry.XLock {
lockEntry.WaitQueue = append(lockEntry.WaitQueue, req)
return
}
if metric.XLock {
metric.WaitQueue = append(metric.WaitQueue, req)
return
}
if s.startRangeScan(metric, req) {
if lockEntry != nil {
lockEntry.RLocks++
} else {
s.metricLockEntries[req.MetricID] = &metricLockEntry{
RLocks: 1,
}
}
}
}
func (*Database) startRangeScan(metric *_metric, req tryRangeScanReq) bool {
if metric.Since == 0 {
req.ResultCh <- rangeScanResult{
ResultCode: QueryDone,
}
return false
}
if req.Since > metric.Until {
req.ResultCh <- rangeScanResult{
ResultCode: QueryDone,
}
return false
}
if req.Until < metric.Since {
if metric.RootPageNo > 0 {
req.ResultCh <- rangeScanResult{
ResultCode: UntilNotFound,
RootPageNo: metric.RootPageNo,
FracDigits: metric.FracDigits,
}
return true
} else {
req.ResultCh <- rangeScanResult{
ResultCode: QueryDone,
}
return false
}
}
timestampDecompressor := chunkenc.NewReverseTimeDeltaDecompressor(
metric.TimestampsBuf,
metric.Timestamps.Size(),
)
valueDecompressor := chunkenc.NewReverseInstantDeltaDecompressor(
metric.ValuesBuf,
metric.Values.Size(),
metric.FracDigits,
)
for {
timestamp, done := timestampDecompressor.NextValue()
if done {
break
}
value, done := valueDecompressor.NextValue()
if done {
qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug)
}
if timestamp <= req.Until {
req.ResponseWriter.FeedNoSend(timestamp, value)
if timestamp < req.Since {
req.ResultCh <- rangeScanResult{
ResultCode: QueryDone,
}
return false
}
}
}
if metric.LastPageNo > 0 {
req.ResultCh <- rangeScanResult{
ResultCode: UntilFound,
LastPageNo: metric.LastPageNo,
FracDigits: metric.FracDigits,
}
return true
} else {
req.ResultCh <- rangeScanResult{
ResultCode: QueryDone,
}
return false
}
metric.StartRangeScan(req)
}
type tryFullScanReq struct {
@@ -877,68 +380,11 @@ func (s *Database) tryFullScan(req tryFullScanReq) {
return
}
lockEntry, ok := s.metricLockEntries[req.MetricID]
if ok {
if lockEntry.XLock {
lockEntry.WaitQueue = append(lockEntry.WaitQueue, req)
return
}
}
if s.startFullScan(metric, req) {
if lockEntry != nil {
lockEntry.RLocks++
} else {
s.metricLockEntries[req.MetricID] = &metricLockEntry{
RLocks: 1,
}
}
}
}
func (*Database) startFullScan(metric *_metric, req tryFullScanReq) bool {
if metric.Since == 0 {
req.ResultCh <- fullScanResult{
ResultCode: QueryDone,
}
return false
}
timestampDecompressor := chunkenc.NewReverseTimeDeltaDecompressor(
metric.TimestampsBuf,
metric.Timestamps.Size(),
)
valueDecompressor := chunkenc.NewReverseInstantDeltaDecompressor(
metric.ValuesBuf,
metric.Values.Size(),
metric.FracDigits,
)
for {
timestamp, done := timestampDecompressor.NextValue()
if done {
break
}
value, done := valueDecompressor.NextValue()
if done {
qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug)
}
req.ResponseWriter.FeedNoSend(timestamp, value)
}
if metric.LastPageNo > 0 {
req.ResultCh <- fullScanResult{
ResultCode: UntilFound,
LastPageNo: metric.LastPageNo,
FracDigits: metric.FracDigits,
}
return true
} else {
req.ResultCh <- fullScanResult{
ResultCode: QueryDone,
}
return false
if metric.XLock {
metric.WaitQueue = append(metric.WaitQueue, req)
return
}
metric.StartFullScan(req)
}
type tryListCurrentValuesReq struct {
@@ -975,8 +421,8 @@ func (s *Database) applyChanges(req txlog.Changes) {
// case txlog.AppendedMeasure:
// s.appendMeasure(rec)
case txlog.AppendedMeasures:
s.appendMeasures(rec)
case txlog.AppendMeasuresSummary:
s.finAppendMeasures(rec)
// case txlog.AppendedMeasureWithOverflowExtended:
// s.appendMeasureAfterOverflow(rec)
@@ -986,18 +432,38 @@ func (s *Database) applyChanges(req txlog.Changes) {
}
}
if req.ForceSnapshot || req.ExitWaitGroup != nil {
s.dumpSnapshot(req.LogNumber)
}
close(req.WaitCh)
if req.ExitWaitGroup != nil {
req.ExitWaitGroup.Done()
if req.MetricsCh != nil {
waitCh := make(chan struct{})
req.MetricsCh <- txlog.MetricsState{
Metrics: s.createMetricsState(),
WaitCh: waitCh,
}
// чекаю поки txlog запише снапшот, а отже можна змінювати буфери
<-waitCh
}
}
func (s *Database) createMetricsState() (metrics []txlog.Metric) {
for metricID, metric := range s.metrics {
x := txlog.Metric{
MetricID: metricID,
MetricType: metric.MetricType,
FracDigits: metric.FracDigits,
LastPageNo: metric.LastPageNo,
Since: metric.Since,
SinceValue: metric.SinceValue,
Until: metric.Until, // ?
UntilValue: metric.UntilValue, // ?
}
x.Timestamps, x.TimestampsSize = metric.Timestamps.Snapshot()
x.Values, x.ValuesSize = metric.Values.Snapshot()
metrics = append(metrics, x)
}
return
}
func (s *Database) addMetric(rec txlog.AddedMetric) {
// fix lock
_, ok := s.metrics[rec.MetricID]
if ok {
qb.Abort(qb.MetricAddedBug,
@@ -1005,18 +471,11 @@ func (s *Database) addMetric(rec txlog.AddedMetric) {
rec.MetricID))
}
lockEntry, ok := s.metricLockEntries[rec.MetricID]
if !ok {
qb.Abort(qb.NoLockEntryBug,
fmt.Errorf("addMetric: lockEntry not found for the metric %d",
rec.MetricID))
}
if !lockEntry.XLock {
qb.Abort(qb.NoXLockBug,
fmt.Errorf("addMetric: xlock not set for the metric %d",
rec.MetricID))
}
// if !lockEntry.XLock {
// qb.Abort(qb.NoXLockBug,
// fmt.Errorf("addMetric: xlock not set for the metric %d",
// rec.MetricID))
// }
var (
values qb.ValueCompressor
@@ -1033,12 +492,10 @@ func (s *Database) addMetric(rec txlog.AddedMetric) {
}
s.metrics[rec.MetricID] = &_metric{
MetricType: rec.MetricType,
FracDigits: byte(rec.FracDigits),
TimestampsBuf: timestampsBuf,
ValuesBuf: valuesBuf,
Timestamps: chunkenc.NewReverseTimeDeltaCompressor(timestampsBuf, 0),
Values: values,
MetricType: rec.MetricType,
FracDigits: byte(rec.FracDigits),
Timestamps: chunkenc.NewReverseTimeDeltaCompressor(timestampsBuf, 0),
Values: values,
}
lockEntry.XLock = false
@@ -1046,21 +503,14 @@ func (s *Database) addMetric(rec txlog.AddedMetric) {
}
func (s *Database) deleteMetric(rec txlog.DeletedMetric) {
_, ok := s.metrics[rec.MetricID]
metric, ok := s.metrics[rec.MetricID]
if !ok {
qb.Abort(qb.NoMetricBug,
fmt.Errorf("deleteMetric: metric %d not found",
rec.MetricID))
}
lockEntry, ok := s.metricLockEntries[rec.MetricID]
if !ok {
qb.Abort(qb.NoLockEntryBug,
fmt.Errorf("deleteMetric: lockEntry not found for the metric %d",
rec.MetricID))
}
if !lockEntry.XLock {
if !metric.XLock {
qb.Abort(qb.NoXLockBug,
fmt.Errorf("deleteMetric: xlock not set for the metric %d",
rec.MetricID))
@@ -1068,8 +518,8 @@ func (s *Database) deleteMetric(rec txlog.DeletedMetric) {
var addMetricReqs []tryAddMetricReq
if len(lockEntry.WaitQueue) > 0 {
for _, untyped := range lockEntry.WaitQueue {
if len(metric.WaitQueue) > 0 {
for _, untyped := range metric.WaitQueue {
switch req := untyped.(type) {
// case tryAppendMeasureReq:
// req.ResultCh <- tryAppendMeasureResult{
@@ -1112,11 +562,11 @@ func (s *Database) deleteMetric(rec txlog.DeletedMetric) {
}
}
delete(s.metrics, rec.MetricID)
delete(s.metricLockEntries, rec.MetricID)
if len(rec.FreePageNumbers) > 0 {
s.freeList.AddPages(rec.FreePageNumbers)
}
// ADD in txlog
// if len(rec.FreePageNumbers) > 0 {
// s.freeList.AddPages(rec.FreePageNumbers)
// }
if len(addMetricReqs) > 0 {
s.processTryAddMetricReqsImmediatelyAfterDelete(addMetricReqs)
@@ -1131,30 +581,22 @@ func (s *Database) deleteMeasures(rec txlog.DeletedMeasures) {
rec.MetricID))
}
lockEntry, ok := s.metricLockEntries[rec.MetricID]
if !ok {
qb.Abort(qb.NoLockEntryBug,
fmt.Errorf("deleteMeasures: lockEntry not found for the metric %d",
rec.MetricID))
}
if !lockEntry.XLock {
if !metric.XLock {
qb.Abort(qb.NoXLockBug,
fmt.Errorf("deleteMeasures: xlock not set for the metric %d",
rec.MetricID))
}
metric.DeleteMeasures()
lockEntry.XLock = false
if len(rec.FreePageNumbers) > 0 {
s.freeList.AddPages(rec.FreePageNumbers)
}
s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry)
metric.XLock = false
// FIX add in txlog
// if len(rec.FreePageNumbers) > 0 {
// s.freeList.AddPages(rec.FreePageNumbers)
// }
s.doAfterReleaseXLock(rec.MetricID, metric)
}
func (s *Database) doAfterReleaseXLock(metricID uint32, metric *_metric, lockEntry *metricLockEntry) {
if len(lockEntry.WaitQueue) == 0 {
delete(s.metricLockEntries, metricID)
} else {
s.processMetricQueue(metricID, metric, lockEntry)
func (s *Database) doAfterReleaseXLock(metricID uint32, metric *_metric) {
if len(metric.WaitQueue) > 0 {
s.processMetricQueue(metricID, metric)
}
}

279
database/snapshot.go
View File

@@ -1,279 +0,0 @@
package database
import (
"fmt"
"hash/crc32"
"io"
"os"
"path/filepath"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/chunkenc"
"gordenko.dev/dima/qb/conbuf"
"gordenko.dev/dima/qb/freelist"
)
/*
Формат:
metricsQty - varuint
[metric]*
де metric - це:
metricID - 4b
metricType - 1b
fracDigits - 1b
lastPageNo - 4b
since - 4b
sinceValue - 8b
until - 4b
untilValue - 8b
timestamps size - 2b
values size - 2b
timestams payload - Nb
values payload - Nb
dataFreeList size - varsize
dataFreeList - Nb
indexFreeList size - varsize
indexFreeList - Nb
CRC32 - 4b
*/
const metricHeaderSize = 38
func (s *Database) dumpSnapshot(logNumber int) (err error) {
var (
fileName = filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", logNumber))
hasher = crc32.NewIEEE()
prefix = make([]byte, metricHeaderSize)
)
file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY, 0770)
if err != nil {
return
}
dst := io.MultiWriter(file, hasher)
_, err = bin.WriteVarUint64(dst, uint64(len(s.metrics)))
if err != nil {
return
}
for metricID, metric := range s.metrics {
tSize := metric.Timestamps.Size()
vSize := metric.Values.Size()
bin.PutUint32(prefix[0:], metricID)
prefix[4] = byte(metric.MetricType)
prefix[5] = metric.FracDigits
bin.PutUint32(prefix[6:], metric.LastPageNo)
bin.PutUint32(prefix[10:], metric.Since)
bin.PutFloat64(prefix[14:], metric.SinceValue)
bin.PutUint32(prefix[22:], metric.Until)
bin.PutFloat64(prefix[26:], metric.UntilValue)
bin.PutUint16(prefix[34:], uint16(tSize))
bin.PutUint16(prefix[36:], uint16(vSize))
_, err = dst.Write(prefix)
if err != nil {
return
}
// copy timestamps
writeChunks(dst, metric.Timestamps.Chunks(), tSize)
// copy values
writeChunks(dst, metric.Values.Chunks(), vSize)
}
// free data pages
err = freeListWriteTo(s.freeList, dst)
if err != nil {
return
}
// free index pages
err = freeListWriteTo(s.freeList, dst)
if err != nil {
return
}
bin.WriteUint32(file, hasher.Sum32())
err = file.Sync()
if err != nil {
return
}
err = file.Close()
if err != nil {
return
}
// prevLogNumber := logNumber - 1
// prevChanges := filepath.Join(s.dir, fmt.Sprintf("%d.changes", prevLogNumber))
// prevSnapshot := filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", prevLogNumber))
// isExist, err := isFileExist(prevChanges)
// if err != nil {
// return
// }
// if isExist {
// err = os.Remove(prevChanges)
// if err != nil {
// qb.Abort(qb.DeletePrevChangesFileFailed, err)
// }
// }
// isExist, err = isFileExist(prevSnapshot)
// if err != nil {
// return
// }
// if isExist {
// err = os.Remove(prevSnapshot)
// if err != nil {
// qb.Abort(qb.DeletePrevSnapshotFileFailed, err)
// }
// }
return
}
func (s *Database) loadSnapshot(fileName string) (err error) {
var (
hasher = crc32.NewIEEE()
header = make([]byte, metricHeaderSize)
body = make([]byte, atree.PageSize)
)
file, err := os.Open(fileName)
if err != nil {
return
}
src := io.TeeReader(file, hasher)
metricsQty, err := bin.ReadVarSize(src)
if err != nil {
return
}
for range metricsQty {
var metric _metric
err = bin.ReadNInto(src, header)
if err != nil {
return
}
metricID, _ := bin.GetUint32(header[0:])
metric.MetricType = qb.MetricType(header[4])
metric.FracDigits = header[5]
metric.LastPageNo, _ = bin.GetUint32(header[6:])
metric.Since, _ = bin.GetUint32(header[10:])
metric.SinceValue, _ = bin.GetFloat64(header[14:])
metric.Until, _ = bin.GetUint32(header[22:])
metric.UntilValue, _ = bin.GetFloat64(header[26:])
tSize, _ := bin.GetUint16(header[34:])
vSize, _ := bin.GetUint16(header[36:])
buf := body[:tSize]
err = bin.ReadNInto(src, buf)
if err != nil {
return
}
metric.Timestamps = chunkenc.NewReverseTimeDeltaCompressor(
conbuf.NewFromBuffer(buf),
int(tSize),
)
buf = body[:vSize]
err = bin.ReadNInto(src, buf)
if err != nil {
return
}
if metric.MetricType == qb.Cumulative {
metric.Values = chunkenc.NewReverseCumulativeDeltaCompressor(
conbuf.NewFromBuffer(buf),
int(vSize),
metric.FracDigits,
)
} else {
metric.Values = chunkenc.NewReverseInstantDeltaCompressor(
conbuf.NewFromBuffer(buf),
int(vSize),
metric.FracDigits,
)
}
s.metrics[metricID] = &metric
}
err = restoreFreeList(s.freeList, src)
if err != nil {
return fmt.Errorf("restore dataFreeList: %s", err)
}
err = restoreFreeList(s.freeList, src)
if err != nil {
return fmt.Errorf("restore indexFreeList: %s", err)
}
calculatedChecksum := hasher.Sum32()
writtenChecksum, err := bin.ReadUint32(file)
if err != nil {
return
}
if calculatedChecksum != writtenChecksum {
return fmt.Errorf("calculated checksum %d not equal written checksum %d", calculatedChecksum, writtenChecksum)
}
return
}
// HELPERS
func freeListWriteTo(freeList *freelist.FreeList, dst io.Writer) error {
serialized, err := freeList.Serialize()
if err != nil {
qb.Abort(qb.FailedFreeListSerialize, err)
}
_, err = bin.WriteVarSize(dst, len(serialized))
if err != nil {
return err
}
_, err = dst.Write(serialized)
if err != nil {
return err
}
return nil
}
func restoreFreeList(freeList *freelist.FreeList, src io.Reader) error {
size, err := bin.ReadVarSize(src)
if err != nil {
return err
}
serialized, err := bin.ReadN(src, size)
if err != nil {
return err
}
freeList.Restore(serialized)
return nil
}
func writeChunks(dst io.Writer, chunks [][]byte, size int) (err error) {
remaining := size
for _, buf := range chunks {
if remaining < len(buf) {
buf = buf[:remaining]
}
_, err = dst.Write(buf)
if err != nil {
return
}
remaining -= len(buf)
if remaining == 0 {
break
}
}
return
}