dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
dima
2026-02-23 19:37:05 +00:00
parent 6011535509
commit 271046231c
15 changed files with 948 additions and 816 deletions
Download Patch File Download Diff File Expand all files Collapse all files

180
database/database.go
View File

@@ -9,13 +9,11 @@ import (
"net"
"os"
"path/filepath"
"regexp"
"sync"
"time"
diploma "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/atree/redo"
"gordenko.dev/dima/qb/bin"
"gordenko.dev/dima/qb/chunkenc"
"gordenko.dev/dima/qb/conbuf"
@@ -41,11 +39,9 @@ type Database struct {
rLocksToRelease []uint32
metrics map[uint32]*_metric
metricLockEntries map[uint32]*metricLockEntry
dataFreeList *freelist.FreeList
indexFreeList *freelist.FreeList
freeList *freelist.FreeList
dir string
databaseName string
redoDir string
txlog *txlog.Writer
atree *atree.Atree
tcpPort int
@@ -92,11 +88,9 @@ func New(opt Options) (_ *Database, err error) {
workerSignalCh: make(chan struct{}, 1),
dir: opt.Dir,
databaseName: opt.DatabaseName,
redoDir: opt.RedoDir,
metrics: make(map[uint32]*_metric),
metricLockEntries: make(map[uint32]*metricLockEntry),
dataFreeList: freelist.New(),
indexFreeList: freelist.New(),
freeList: freelist.New(),
tcpPort: opt.TCPPort,
logfile: opt.Logfile,
logger: log.New(opt.Logfile, "", log.LstdFlags),
@@ -113,11 +107,9 @@ func (s *Database) ListenAndServe() (err error) {
}
s.atree, err = atree.New(atree.Options{
Dir: s.dir,
DatabaseName: s.databaseName,
RedoDir: s.redoDir,
DataFreeList: s.dataFreeList,
IndexFreeList: s.indexFreeList,
Dir: s.dir,
DatabaseName: s.databaseName,
FreeList: s.freeList,
})
if err != nil {
return fmt.Errorf("atree.New: %s", err)
@@ -186,26 +178,26 @@ func (s *Database) recovery() {
}
go s.txlog.Run()
fileNames, err := s.searchREDOFiles()
if err != nil {
diploma.Abort(diploma.SearchREDOFilesFailed, err)
}
// fileNames, err := s.searchREDOFiles()
// if err != nil {
// diploma.Abort(diploma.SearchREDOFilesFailed, err)
// }
if len(fileNames) > 0 {
for _, fileName := range fileNames {
err = s.replayREDOFile(fileName)
if err != nil {
diploma.Abort(diploma.ReplayREDOFileFailed, err)
}
}
// if len(fileNames) > 0 {
// for _, fileName := range fileNames {
// err = s.replayREDOFile(fileName)
// if err != nil {
// diploma.Abort(diploma.ReplayREDOFileFailed, err)
// }
// }
for _, fileName := range fileNames {
err = os.Remove(fileName)
if err != nil {
diploma.Abort(diploma.RemoveREDOFileFailed, err)
}
}
}
// for _, fileName := range fileNames {
// err = os.Remove(fileName)
// if err != nil {
// diploma.Abort(diploma.RemoveREDOFileFailed, err)
// }
// }
// }
if recipe != nil {
if recipe.CompleteSnapshot {
@@ -224,69 +216,69 @@ func (s *Database) recovery() {
}
}
func (s *Database) searchREDOFiles() ([]string, error) {
var (
reREDO = regexp.MustCompile(`a\d+\.redo`)
fileNames []string
)
// func (s *Database) searchREDOFiles() ([]string, error) {
// var (
// reREDO = regexp.MustCompile(`a\d+\.redo`)
// fileNames []string
// )
entries, err := os.ReadDir(s.redoDir)
if err != nil {
return nil, err
}
// entries, err := os.ReadDir(s.redoDir)
// if err != nil {
// return nil, err
// }
for _, entry := range entries {
if entry.Type().IsRegular() {
baseName := entry.Name()
if reREDO.MatchString(baseName) {
fileNames = append(fileNames, filepath.Join(s.redoDir, baseName))
}
}
}
return fileNames, nil
}
// for _, entry := range entries {
// if entry.Type().IsRegular() {
// baseName := entry.Name()
// if reREDO.MatchString(baseName) {
// fileNames = append(fileNames, filepath.Join(s.redoDir, baseName))
// }
// }
// }
// return fileNames, nil
// }
func (s *Database) replayREDOFile(fileName string) error {
redoFile, err := redo.ReadREDOFile(redo.ReadREDOFileReq{
FileName: fileName,
DataPageSize: atree.DataPageSize,
IndexPageSize: atree.IndexPageSize,
})
if err != nil {
return fmt.Errorf("can't read REDO file %s: %s", fileName, err)
}
// func (s *Database) replayREDOFile(fileName string) error {
// redoFile, err := redo.ReadREDOFile(redo.ReadREDOFileReq{
// FileName: fileName,
// DataPageSize: atree.DataPageSize,
// IndexPageSize: atree.IndexPageSize,
// })
// if err != nil {
// return fmt.Errorf("can't read REDO file %s: %s", fileName, err)
// }
metric, ok := s.metrics[redoFile.MetricID]
if !ok {
return fmt.Errorf("has REDOFile, metric %d not found", redoFile.MetricID)
}
// metric, ok := s.metrics[redoFile.MetricID]
// if !ok {
// return fmt.Errorf("has REDOFile, metric %d not found", redoFile.MetricID)
// }
if metric.Until < redoFile.Timestamp {
waitCh := make(chan struct{})
s.atree.ApplyREDO(atree.WriteTask{
DataPage: redoFile.DataPage,
IndexPages: redoFile.IndexPages,
})
<-waitCh
// if metric.Until < redoFile.Timestamp {
// waitCh := make(chan struct{})
// s.atree.ApplyREDO(atree.WriteTask{
// DataPage: redoFile.DataPage,
// IndexPages: redoFile.IndexPages,
// })
// <-waitCh
waitCh = s.txlog.WriteAppendedMeasureWithOverflow(
txlog.AppendedMeasureWithOverflow{
MetricID: redoFile.MetricID,
Timestamp: redoFile.Timestamp,
Value: redoFile.Value,
IsDataPageReused: redoFile.IsDataPageReused,
DataPageNo: redoFile.DataPage.PageNo,
IsRootChanged: redoFile.IsRootChanged,
RootPageNo: redoFile.RootPageNo,
ReusedIndexPages: redoFile.ReusedIndexPages,
},
fileName,
false,
)
<-waitCh
}
return nil
}
// waitCh = s.txlog.WriteAppendedMeasureWithOverflow(
// txlog.AppendedMeasureWithOverflow{
// MetricID: redoFile.MetricID,
// Timestamp: redoFile.Timestamp,
// Value: redoFile.Value,
// IsDataPageReused: redoFile.IsDataPageReused,
// DataPageNo: redoFile.DataPage.PageNo,
// IsRootChanged: redoFile.IsRootChanged,
// RootPageNo: redoFile.RootPageNo,
// ReusedIndexPages: redoFile.ReusedIndexPages,
// },
// fileName,
// false,
// )
// <-waitCh
// }
// return nil
// }
func (s *Database) verifySnapshot(fileName string) (_ bool, err error) {
file, err := os.Open(fileName)
@@ -383,10 +375,10 @@ func (s *Database) replayChangesRecord(untyped any) error {
case txlog.DeletedMetric:
delete(s.metrics, rec.MetricID)
if len(rec.FreeDataPages) > 0 {
s.dataFreeList.AddPages(rec.FreeDataPages)
s.freeList.AddPages(rec.FreeDataPages)
}
if len(rec.FreeIndexPages) > 0 {
s.indexFreeList.AddPages(rec.FreeIndexPages)
s.freeList.AddPages(rec.FreeIndexPages)
}
case txlog.AppendedMeasure:
@@ -431,12 +423,12 @@ func (s *Database) replayChangesRecord(untyped any) error {
metric.LastPageNo = rec.DataPageNo
// delete free pages
if rec.IsDataPageReused {
s.dataFreeList.DeleteReservedPages([]uint32{
s.freeList.DeleteReservedPages([]uint32{
rec.DataPageNo,
})
}
if len(rec.ReusedIndexPages) > 0 {
s.indexFreeList.DeleteReservedPages(rec.ReusedIndexPages)
s.freeList.DeleteReservedPages(rec.ReusedIndexPages)
}
}
@@ -445,10 +437,10 @@ func (s *Database) replayChangesRecord(untyped any) error {
if ok {
metric.DeleteMeasures()
if len(rec.FreeDataPages) > 0 {
s.dataFreeList.AddPages(rec.FreeDataPages)
s.freeList.AddPages(rec.FreeDataPages)
}
if len(rec.FreeDataPages) > 0 {
s.indexFreeList.AddPages(rec.FreeIndexPages)
s.freeList.AddPages(rec.FreeIndexPages)
}
}