dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
dima
2026-02-23 19:37:05 +00:00
parent 6011535509
commit 271046231c
15 changed files with 948 additions and 816 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
database/api.go
View File

@@ -372,18 +372,19 @@ func (s *Database) AppendMeasure(req proto.AppendMeasureReq) uint16 {
if err != nil {
diploma.Abort(diploma.WriteToAtreeFailed, err)
}
_ = report
waitCh := s.txlog.WriteAppendedMeasureWithOverflow(
txlog.AppendedMeasureWithOverflow{
MetricID: req.MetricID,
Timestamp: req.Timestamp,
Value: req.Value,
IsDataPageReused: report.IsDataPageReused,
DataPageNo: report.DataPageNo,
IsRootChanged: report.IsRootChanged,
RootPageNo: report.NewRootPageNo,
ReusedIndexPages: report.ReusedIndexPages,
// FIX
// MetricID: req.MetricID,
// Timestamp: req.Timestamp,
// Value: req.Value,
// IsDataPageReused: report.IsDataPageReused,
// DataPageNo: report.DataPageNo,
// IsRootChanged: report.IsRootChanged,
// RootPageNo: report.NewRootPageNo,
// ReusedIndexPages: report.ReusedIndexPages,
},
report.FileName,
false,
)
<-waitCh
@@ -445,6 +446,7 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
)
for idx, measure := range req.Measures {
_ = idx
if since == 0 {
since = measure.Timestamp
} else {
@@ -514,26 +516,26 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
if err != nil {
diploma.Abort(diploma.WriteToAtreeFailed, err)
}
prevPageNo = report.DataPageNo
if report.IsRootChanged {
rootPageNo = report.NewRootPageNo
}
waitCh := s.txlog.WriteAppendedMeasureWithOverflow(
txlog.AppendedMeasureWithOverflow{
MetricID: req.MetricID,
Timestamp: measure.Timestamp,
Value: measure.Value,
IsDataPageReused: report.IsDataPageReused,
DataPageNo: report.DataPageNo,
IsRootChanged: report.IsRootChanged,
RootPageNo: report.NewRootPageNo,
ReusedIndexPages: report.ReusedIndexPages,
},
report.FileName,
(idx+1) < len(req.Measures),
)
<-waitCh
_ = report
// FIX
// prevPageNo = report.DataPageNo
// if report.IsRootChanged {
// rootPageNo = report.NewRootPageNo
// }
// waitCh := s.txlog.WriteAppendedMeasureWithOverflow(
// txlog.AppendedMeasureWithOverflow{
// MetricID: req.MetricID,
// Timestamp: measure.Timestamp,
// Value: measure.Value,
// IsDataPageReused: report.IsDataPageReused,
// DataPageNo: report.DataPageNo,
// IsRootChanged: report.IsRootChanged,
// RootPageNo: report.NewRootPageNo,
// ReusedIndexPages: report.ReusedIndexPages,
// },
// (idx+1) < len(req.Measures),
// )
// <-waitCh
timestampsBuf = conbuf.New(nil)
valuesBuf = conbuf.New(nil)

180
database/database.go
View File

@@ -9,13 +9,11 @@ import (
"net"
"os"
"path/filepath"
"regexp"
"sync"
"time"
diploma "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/atree/redo"
"gordenko.dev/dima/qb/bin"
"gordenko.dev/dima/qb/chunkenc"
"gordenko.dev/dima/qb/conbuf"
@@ -41,11 +39,9 @@ type Database struct {
rLocksToRelease []uint32
metrics map[uint32]*_metric
metricLockEntries map[uint32]*metricLockEntry
dataFreeList *freelist.FreeList
indexFreeList *freelist.FreeList
freeList *freelist.FreeList
dir string
databaseName string
redoDir string
txlog *txlog.Writer
atree *atree.Atree
tcpPort int
@@ -92,11 +88,9 @@ func New(opt Options) (_ *Database, err error) {
workerSignalCh: make(chan struct{}, 1),
dir: opt.Dir,
databaseName: opt.DatabaseName,
redoDir: opt.RedoDir,
metrics: make(map[uint32]*_metric),
metricLockEntries: make(map[uint32]*metricLockEntry),
dataFreeList: freelist.New(),
indexFreeList: freelist.New(),
freeList: freelist.New(),
tcpPort: opt.TCPPort,
logfile: opt.Logfile,
logger: log.New(opt.Logfile, "", log.LstdFlags),
@@ -113,11 +107,9 @@ func (s *Database) ListenAndServe() (err error) {
}
s.atree, err = atree.New(atree.Options{
Dir: s.dir,
DatabaseName: s.databaseName,
RedoDir: s.redoDir,
DataFreeList: s.dataFreeList,
IndexFreeList: s.indexFreeList,
Dir: s.dir,
DatabaseName: s.databaseName,
FreeList: s.freeList,
})
if err != nil {
return fmt.Errorf("atree.New: %s", err)
@@ -186,26 +178,26 @@ func (s *Database) recovery() {
}
go s.txlog.Run()
fileNames, err := s.searchREDOFiles()
if err != nil {
diploma.Abort(diploma.SearchREDOFilesFailed, err)
}
// fileNames, err := s.searchREDOFiles()
// if err != nil {
// diploma.Abort(diploma.SearchREDOFilesFailed, err)
// }
if len(fileNames) > 0 {
for _, fileName := range fileNames {
err = s.replayREDOFile(fileName)
if err != nil {
diploma.Abort(diploma.ReplayREDOFileFailed, err)
}
}
// if len(fileNames) > 0 {
// for _, fileName := range fileNames {
// err = s.replayREDOFile(fileName)
// if err != nil {
// diploma.Abort(diploma.ReplayREDOFileFailed, err)
// }
// }
for _, fileName := range fileNames {
err = os.Remove(fileName)
if err != nil {
diploma.Abort(diploma.RemoveREDOFileFailed, err)
}
}
}
// for _, fileName := range fileNames {
// err = os.Remove(fileName)
// if err != nil {
// diploma.Abort(diploma.RemoveREDOFileFailed, err)
// }
// }
// }
if recipe != nil {
if recipe.CompleteSnapshot {
@@ -224,69 +216,69 @@ func (s *Database) recovery() {
}
}
func (s *Database) searchREDOFiles() ([]string, error) {
var (
reREDO = regexp.MustCompile(`a\d+\.redo`)
fileNames []string
)
// func (s *Database) searchREDOFiles() ([]string, error) {
// var (
// reREDO = regexp.MustCompile(`a\d+\.redo`)
// fileNames []string
// )
entries, err := os.ReadDir(s.redoDir)
if err != nil {
return nil, err
}
// entries, err := os.ReadDir(s.redoDir)
// if err != nil {
// return nil, err
// }
for _, entry := range entries {
if entry.Type().IsRegular() {
baseName := entry.Name()
if reREDO.MatchString(baseName) {
fileNames = append(fileNames, filepath.Join(s.redoDir, baseName))
}
}
}
return fileNames, nil
}
// for _, entry := range entries {
// if entry.Type().IsRegular() {
// baseName := entry.Name()
// if reREDO.MatchString(baseName) {
// fileNames = append(fileNames, filepath.Join(s.redoDir, baseName))
// }
// }
// }
// return fileNames, nil
// }
func (s *Database) replayREDOFile(fileName string) error {
redoFile, err := redo.ReadREDOFile(redo.ReadREDOFileReq{
FileName: fileName,
DataPageSize: atree.DataPageSize,
IndexPageSize: atree.IndexPageSize,
})
if err != nil {
return fmt.Errorf("can't read REDO file %s: %s", fileName, err)
}
// func (s *Database) replayREDOFile(fileName string) error {
// redoFile, err := redo.ReadREDOFile(redo.ReadREDOFileReq{
// FileName: fileName,
// DataPageSize: atree.DataPageSize,
// IndexPageSize: atree.IndexPageSize,
// })
// if err != nil {
// return fmt.Errorf("can't read REDO file %s: %s", fileName, err)
// }
metric, ok := s.metrics[redoFile.MetricID]
if !ok {
return fmt.Errorf("has REDOFile, metric %d not found", redoFile.MetricID)
}
// metric, ok := s.metrics[redoFile.MetricID]
// if !ok {
// return fmt.Errorf("has REDOFile, metric %d not found", redoFile.MetricID)
// }
if metric.Until < redoFile.Timestamp {
waitCh := make(chan struct{})
s.atree.ApplyREDO(atree.WriteTask{
DataPage: redoFile.DataPage,
IndexPages: redoFile.IndexPages,
})
<-waitCh
// if metric.Until < redoFile.Timestamp {
// waitCh := make(chan struct{})
// s.atree.ApplyREDO(atree.WriteTask{
// DataPage: redoFile.DataPage,
// IndexPages: redoFile.IndexPages,
// })
// <-waitCh
waitCh = s.txlog.WriteAppendedMeasureWithOverflow(
txlog.AppendedMeasureWithOverflow{
MetricID: redoFile.MetricID,
Timestamp: redoFile.Timestamp,
Value: redoFile.Value,
IsDataPageReused: redoFile.IsDataPageReused,
DataPageNo: redoFile.DataPage.PageNo,
IsRootChanged: redoFile.IsRootChanged,
RootPageNo: redoFile.RootPageNo,
ReusedIndexPages: redoFile.ReusedIndexPages,
},
fileName,
false,
)
<-waitCh
}
return nil
}
// waitCh = s.txlog.WriteAppendedMeasureWithOverflow(
// txlog.AppendedMeasureWithOverflow{
// MetricID: redoFile.MetricID,
// Timestamp: redoFile.Timestamp,
// Value: redoFile.Value,
// IsDataPageReused: redoFile.IsDataPageReused,
// DataPageNo: redoFile.DataPage.PageNo,
// IsRootChanged: redoFile.IsRootChanged,
// RootPageNo: redoFile.RootPageNo,
// ReusedIndexPages: redoFile.ReusedIndexPages,
// },
// fileName,
// false,
// )
// <-waitCh
// }
// return nil
// }
func (s *Database) verifySnapshot(fileName string) (_ bool, err error) {
file, err := os.Open(fileName)
@@ -383,10 +375,10 @@ func (s *Database) replayChangesRecord(untyped any) error {
case txlog.DeletedMetric:
delete(s.metrics, rec.MetricID)
if len(rec.FreeDataPages) > 0 {
s.dataFreeList.AddPages(rec.FreeDataPages)
s.freeList.AddPages(rec.FreeDataPages)
}
if len(rec.FreeIndexPages) > 0 {
s.indexFreeList.AddPages(rec.FreeIndexPages)
s.freeList.AddPages(rec.FreeIndexPages)
}
case txlog.AppendedMeasure:
@@ -431,12 +423,12 @@ func (s *Database) replayChangesRecord(untyped any) error {
metric.LastPageNo = rec.DataPageNo
// delete free pages
if rec.IsDataPageReused {
s.dataFreeList.DeleteReservedPages([]uint32{
s.freeList.DeleteReservedPages([]uint32{
rec.DataPageNo,
})
}
if len(rec.ReusedIndexPages) > 0 {
s.indexFreeList.DeleteReservedPages(rec.ReusedIndexPages)
s.freeList.DeleteReservedPages(rec.ReusedIndexPages)
}
}
@@ -445,10 +437,10 @@ func (s *Database) replayChangesRecord(untyped any) error {
if ok {
metric.DeleteMeasures()
if len(rec.FreeDataPages) > 0 {
s.dataFreeList.AddPages(rec.FreeDataPages)
s.freeList.AddPages(rec.FreeDataPages)
}
if len(rec.FreeDataPages) > 0 {
s.indexFreeList.AddPages(rec.FreeIndexPages)
s.freeList.AddPages(rec.FreeIndexPages)
}
}

12
database/proc.go
View File

@@ -535,13 +535,13 @@ func (s *Database) appendMeasureAfterOverflow(extended txlog.AppendedMeasureWith
metric.LastPageNo = rec.DataPageNo
if rec.IsDataPageReused {
s.dataFreeList.DeleteReservedPages([]uint32{
s.freeList.DeleteReservedPages([]uint32{
rec.DataPageNo,
})
}
if len(rec.ReusedIndexPages) > 0 {
s.indexFreeList.DeleteReservedPages(rec.ReusedIndexPages)
s.freeList.DeleteReservedPages(rec.ReusedIndexPages)
}
if !extended.HoldLock {
@@ -1020,10 +1020,10 @@ func (s *Database) deleteMetric(rec txlog.DeletedMetric) {
delete(s.metricLockEntries, rec.MetricID)
if len(rec.FreeDataPages) > 0 {
s.dataFreeList.AddPages(rec.FreeDataPages)
s.freeList.AddPages(rec.FreeDataPages)
}
if len(rec.FreeIndexPages) > 0 {
s.indexFreeList.AddPages(rec.FreeIndexPages)
s.freeList.AddPages(rec.FreeIndexPages)
}
if len(addMetricReqs) > 0 {
@@ -1054,10 +1054,10 @@ func (s *Database) deleteMeasures(rec txlog.DeletedMeasures) {
metric.DeleteMeasures()
lockEntry.XLock = false
if len(rec.FreeDataPages) > 0 {
s.dataFreeList.AddPages(rec.FreeDataPages)
s.freeList.AddPages(rec.FreeDataPages)
}
if len(rec.FreeDataPages) > 0 {
s.indexFreeList.AddPages(rec.FreeIndexPages)
s.freeList.AddPages(rec.FreeIndexPages)
}
s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry)
}

10
database/snapshot.go
View File

@@ -114,12 +114,12 @@ func (s *Database) dumpSnapshot(logNumber int) (err error) {
}
}
// free data pages
err = freeListWriteTo(s.dataFreeList, dst)
err = freeListWriteTo(s.freeList, dst)
if err != nil {
return
}
// free index pages
err = freeListWriteTo(s.indexFreeList, dst)
err = freeListWriteTo(s.freeList, dst)
if err != nil {
return
}
@@ -171,7 +171,7 @@ func (s *Database) loadSnapshot(fileName string) (err error) {
hasher = crc32.NewIEEE()
metricsQty int
header = make([]byte, metricHeaderSize)
body = make([]byte, atree.DataPageSize)
body = make([]byte, atree.PageSize)
)
file, err := os.Open(fileName)
@@ -232,12 +232,12 @@ func (s *Database) loadSnapshot(fileName string) (err error) {
s.metrics[metricID] = &metric
}
err = restoreFreeList(s.dataFreeList, src)
err = restoreFreeList(s.freeList, src)
if err != nil {
return fmt.Errorf("restore dataFreeList: %s", err)
}
err = restoreFreeList(s.indexFreeList, src)
err = restoreFreeList(s.freeList, src)
if err != nil {
return fmt.Errorf("restore indexFreeList: %s", err)
}