dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-06-19 07:25:46 +03:00
parent 06823ee547
commit 2d73ce1727
52 changed files with 5173 additions and 547 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
database/api.go
View File

@@ -84,7 +84,7 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e
if err != nil {
return fmt.Errorf("proto.ReadAddMetricReq: %s", err)
}
fmt.Println("ReadAddMetricReq:", req)
//fmt.Println("ReadAddMetricReq:", req)
reply(conn, s.AddMetric(req))
case proto.TypeDeleteMetric:
@@ -534,8 +534,11 @@ func (s *Database) rangeScan(req rangeScanReq) error {
LastPageNo: result.LastPageNo,
Since: req.Since,
})
//s.metricRUnlock(req.MetricID) fix release unlock
s.workerInbox.Push(worker.ReleaseRLock{
MetricID: req.MetricID,
})
if err != nil {
s.logger.Printf("ContinueRangeScan: %s\n", err)
reply(req.Conn, proto.ErrUnexpected)
} else {
req.ResponseWriter.Close()
@@ -550,8 +553,11 @@ func (s *Database) rangeScan(req rangeScanReq) error {
LastPageNo: result.LastPageNo,
IsDataPage: result.IsDataPage,
})
//s.metricRUnlock(req.MetricID) // fix release
s.workerInbox.Push(worker.ReleaseRLock{
MetricID: req.MetricID,
})
if err != nil {
s.logger.Printf("RangeScan: %s\n", err)
reply(req.Conn, proto.ErrUnexpected)
} else {
req.ResponseWriter.Close()
@@ -595,8 +601,11 @@ func (s *Database) fullScan(req fullScanReq) error {
ResponseWriter: req.ResponseWriter,
LastPageNo: result.LastPageNo,
})
//s.worker.AddJobToQueue(req.MetricID) // FIX release rlock
s.workerInbox.Push(worker.ReleaseRLock{
MetricID: req.MetricID,
})
if err != nil {
s.logger.Printf("ContinueFullScan: %s\n", err)
reply(req.Conn, proto.ErrUnexpected)
} else {
req.ResponseWriter.Close()

285
database/database.go
View File

@@ -77,7 +77,7 @@ func New(opt Options) (_ *Database, err error) {
return nil, fmt.Errorf("recovery.Recovery: %s", err)
}
fmt.Printf("%#v\n", recoveryReport)
//fmt.Printf("%#v\n", recoveryReport)
storageInbox := inbox.New()
s.workerInbox = inbox.New()
@@ -180,6 +180,7 @@ type ContinueFullScanReq struct {
}
func (s *Database) ContinueFullScan(req ContinueFullScanReq) error {
//fmt.Printf("ContinueFullScan from page %d\n", req.LastPageNo)
buf, err := s.dataCache.FetchPage(req.LastPageNo)
if err != nil {
return fmt.Errorf("dataCache.FetchPage(%d): %s", req.LastPageNo, err)
@@ -196,6 +197,7 @@ func (s *Database) ContinueFullScan(req ContinueFullScanReq) error {
return err
}
defer treeCursor.Close()
idx := 0
for {
timestamp, value, done, err := treeCursor.Prev()
if err != nil {
@@ -204,7 +206,9 @@ func (s *Database) ContinueFullScan(req ContinueFullScanReq) error {
if done {
return nil
}
//fmt.Printf(" %d: %s => %.2f\n", idx, formatTime(timestamp), value)
req.ResponseWriter.Feed(timestamp, value)
idx++
}
}
@@ -326,15 +330,15 @@ func (s *Database) findDataPage(foundPageNo uint32, timestamp uint32) (uint32, [
}
}
// type PathLeg struct {
// PageNo uint32
// Data []byte
// }
type PathLeg struct {
PageNo uint32
Data []byte
}
// type PathToDataPage struct {
// Legs []PathLeg
// LastPageNo uint32
// }
type PathToDataPage struct {
Legs []PathLeg
LastPageNo uint32
}
// func (s *Database) FindPathToLastPage(rootPageNo uint32) (_ PathToDataPage, err error) {
// // var (
@@ -381,82 +385,215 @@ func (s *Database) findDataPage(foundPageNo uint32, timestamp uint32) (uint32, [
// s.mutex.Unlock()
// }
type Level struct {
Idx int
PageNumbers []uint32
}
func (s *Database) GetAllPages(pageNumbers []uint32) ([]uint32, []uint32, error) {
levels := []*Level{
{
PageNumbers: pageNumbers,
Idx: 0,
},
}
return s.collectPages(levels)
}
func (s *Database) collectPages(levels []*Level) (indexPageNumbers []uint32, dataPageNumbers []uint32, err error) {
var (
buf []byte
pageNumbers []uint32
)
for {
if len(levels) == 0 {
return
}
var (
lastIdx = len(levels) - 1
level = levels[lastIdx]
pageNo = level.PageNumbers[level.Idx]
)
if level.Idx < len(level.PageNumbers) {
buf, err = s.indexCache.FetchPage(pageNo)
if err != nil {
return nil, nil, fmt.Errorf("indexCache.FetchPage(%d): %s", pageNo, err)
}
pageNumbers = storage.ListPageNumbers(buf)
s.indexCache.ReleasePage(pageNo)
if storage.IsZeroLevelPage(buf) {
dataPageNumbers = append(dataPageNumbers, pageNumbers...)
} else {
levels = append(levels, &Level{
PageNumbers: pageNumbers,
})
indexPageNumbers = append(indexPageNumbers, pageNumbers...)
}
level.Idx++
} else {
levels = levels[:lastIdx]
}
}
}
type DeleteSinceReport struct {
IndexPageNumbers []uint32
DataPageNumbers []uint32
DataPage []byte
IndexLevelTails []storage.IndexLevelTail
}
// type Level struct {
// PageNo uint32
// PageData []byte
// Idx int
// ChildQty int
// Idx int
// PageNumbers []uint32
// }
// func (s *Atree) GetAllPages(rootPageNo uint32) (_ []uint32, err error) {
// // var (
// // pageNumbers []uint32
// // levels []*Level
// // )
// fix - find by (since - 1) ?
// // buf, err := s.fetchIndexPage(rootPageNo)
// // if err != nil {
// // return nil, fmt.Errorf("fetchIndexPage(%d): %s", rootPageNo, err)
// // }
// // pageNumbers = append(pageNumbers, rootPageNo)
// [since, ...]
func (s *Database) DeleteSince(since uint32, indexLevelTail storage.IndexLevelTail) (_ DeleteSinceReport, err error) {
var (
indexPageNumbers []uint32
dataPageNumbers []uint32
dataPage []byte
buf []byte
indexLevelTails []storage.IndexLevelTail
foundPageNo uint32
)
// // // if buf[isDataPageNumbersIdx] == 1 {
// // // pageNumbers := listPageNumbers(buf)
// // // dataPages = append(dataPages, pageNumbers...)
result := storage.DeleteSinceOnIndexTail(indexLevelTail, since)
//
indexLevelTails = append(indexLevelTails, storage.IndexLevelTail{
Buffer: indexLevelTail.Buffer,
RecordsCount: result.RecordsCount,
})
levels := []*Level{
{
PageNumbers: result.PageNumbers,
Idx: 1,
},
}
foundPageNo = result.PageNumbers[0]
for {
buf, err = s.indexCache.FetchPage(foundPageNo)
if err != nil {
err = fmt.Errorf("indexCache.FetchPage(%d): %s", foundPageNo, err)
return
}
toReleaseIndexPageNo := foundPageNo
result := storage.DeleteSinceOnIndexPage(buf, since)
s.indexCache.ReleasePage(toReleaseIndexPageNo)
//
foundPageNo = result.PageNumbers[0]
// // // s.releasePage(rootPageNo)
levels = append(levels, &Level{
PageNumbers: result.PageNumbers,
Idx: 1, // 1st - found pageNo
})
// // // return PageLists{
// // // DataPages: dataPages,
// // // IndexPages: indexPages,
// // // }, nil
// // // }
indexLevelTails = append(indexLevelTails, storage.IndexLevelTail{
Buffer: result.Buffer,
RecordsCount: result.RecordsCount,
})
// // childQty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
if storage.IsZeroLevelPage(buf) {
dataPage, err = s.dataCache.FetchPage(foundPageNo)
if err != nil {
err = fmt.Errorf("dataCache.FetchPage(%d): %s", foundPageNo, err)
return
}
dataPageNumbers = append(dataPageNumbers, result.PageNumbers...)
break
} else {
indexPageNumbers = append(indexPageNumbers, result.PageNumbers...)
}
}
indexPageNumbers, dataPageNumbers, err = s.collectPages(levels)
if err != nil {
return
}
return DeleteSinceReport{
IndexPageNumbers: indexPageNumbers,
DataPageNumbers: dataPageNumbers,
DataPage: dataPage,
IndexLevelTails: indexLevelTails,
}, nil
}
// // levels = append(levels, &Level{
// // PageNo: rootPageNo,
// // PageData: buf,
// // Idx: 0,
// // ChildQty: int(childQty),
// // })
// [..., until]
func (s *Database) DeleteUntil(since uint32, indexLevelTail storage.IndexLevelTail) (_ DeleteSinceReport, err error) {
var (
indexPageNumbers []uint32
dataPageNumbers []uint32
dataPage []byte
buf []byte
indexLevelTails []storage.IndexLevelTail
foundPageNo uint32
)
// // for {
// // if len(levels) == 0 {
// // return pageNumbers, nil
// // }
result := storage.DeleteSinceOnIndexTail(indexLevelTail, since)
//
indexLevelTails = append(indexLevelTails, storage.IndexLevelTail{
Buffer: indexLevelTail.Buffer,
RecordsCount: result.RecordsCount,
})
levels := []*Level{
{
PageNumbers: result.PageNumbers,
Idx: 1,
},
}
foundPageNo = result.PageNumbers[0]
for {
buf, err = s.indexCache.FetchPage(foundPageNo)
if err != nil {
err = fmt.Errorf("indexCache.FetchPage(%d): %s", foundPageNo, err)
return
}
toReleaseIndexPageNo := foundPageNo
result := storage.DeleteSinceOnIndexPage(buf, since)
s.indexCache.ReleasePage(toReleaseIndexPageNo)
//
foundPageNo = result.PageNumbers[0]
// // lastIdx := len(levels) - 1
// // level := levels[lastIdx]
levels = append(levels, &Level{
PageNumbers: result.PageNumbers,
Idx: 1, // 1st - found pageNo
})
// // if level.Idx < level.ChildQty {
// // pageNo := getPageNo(level.PageData, level.Idx)
// // level.Idx++
indexLevelTails = append(indexLevelTails, storage.IndexLevelTail{
Buffer: result.Buffer,
RecordsCount: result.RecordsCount,
})
// // var buf []byte
// // buf, err = s.fetchPage(pageNo)
// // if err != nil {
// // return nil, fmt.Errorf("fetchPage(%d): %s", pageNo, err)
// // }
// // pageNumbers = append(pageNumbers, pageNo)
if storage.IsZeroLevelPage(buf) {
dataPage, err = s.dataCache.FetchPage(foundPageNo)
if err != nil {
err = fmt.Errorf("dataCache.FetchPage(%d): %s", foundPageNo, err)
return
}
dataPageNumbers = append(dataPageNumbers, result.PageNumbers...)
break
} else {
indexPageNumbers = append(indexPageNumbers, result.PageNumbers...)
}
}
indexPageNumbers, dataPageNumbers, err = s.collectPages(levels)
if err != nil {
return
}
return DeleteSinceReport{
IndexPageNumbers: indexPageNumbers,
DataPageNumbers: dataPageNumbers,
DataPage: dataPage,
IndexLevelTails: indexLevelTails,
}, nil
}
// // if buf[pageTypeIdx] == PageTypeData {
// // //pageNumbers := listPageNumbers(buf)
// // //dataPages = append(dataPages, pageNumbers...)
// // s.releasePage(pageNo)
// // } else {
// // childQty, _ = bin.GetUint16(buf[indexRecordsQtyIdx:])
// // levels = append(levels, &Level{
// // PageNo: pageNo,
// // PageData: buf,
// // Idx: 0,
// // ChildQty: int(childQty),
// // })
// // }
// // } else {
// // s.releasePage(level.PageNo)
// // levels = levels[:lastIdx]
// // }
// // }
// return
// }
const datetimeLayout = "2006-01-02 15:04:05"
func formatTime(timestamp uint32) string {
tm := time.Unix(int64(timestamp), 0)
return tm.Format(datetimeLayout)
}