dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
dima
2026-06-15 10:47:24 +00:00
parent ca125c99d2
commit 06823ee547
28 changed files with 1243 additions and 1383 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
a
View File

File diff suppressed because one or more lines are too long

224
atree/atree.go
View File

@@ -1,224 +0,0 @@
package atree
import (
"errors"
"os"
"sync"
)
const (
filePerm = 0666
)
type PeriodsWriter interface {
Feed(uint32, float64)
FeedNoSend(uint32, float64)
Close() error
}
type WorkerMeasureConsumer interface {
FeedNoSend(uint32, float64)
}
type AtreeMeasureConsumer interface {
Feed(uint32, float64)
}
//
type _page struct {
PageNo uint32
Buf []byte
ReferenceCount int
}
type Atree struct {
indexFile *os.File
dataFile *os.File
mutex sync.Mutex
pages map[uint32]*_page
pageWaits map[uint32][]chan readResult
pagesToRead []uint32
readSignalCh chan struct{}
}
type Options struct {
IndexFile *os.File
DataFile *os.File
}
func New(opt Options) (*Atree, error) {
if opt.IndexFile == nil {
return nil, errors.New("IndexFile option is required")
}
if opt.DataFile == nil {
return nil, errors.New("DataFile option is required")
}
s := &Atree{
indexFile: opt.IndexFile,
dataFile: opt.DataFile,
pages: make(map[uint32]*_page),
pageWaits: make(map[uint32][]chan readResult),
readSignalCh: make(chan struct{}, 1),
}
return s, nil
}
func (s *Atree) Run() {
//go s.pageWriter()
go s.pageReader()
}
// FIND
func (s *Atree) findDataPage(rootPageNo uint32, timestamp uint32) (uint32, []byte, error) {
// indexPageNo := rootPageNo
// for {
// buf, err := s.fetchIndexPage(indexPageNo)
// if err != nil {
// return 0, nil, fmt.Errorf("fetchIndexPage(%d): %s", indexPageNo, err)
// }
// foundPageNo := findPageNo(buf, timestamp)
// s.releasePage(indexPageNo)
// // fix
// if buf[isLastLevelIdx] == 1 {
// buf, err := s.fetchDataPage(foundPageNo)
// if err != nil {
// return 0, nil, fmt.Errorf("fetchDataPage(%d): %s", foundPageNo, err)
// }
// return foundPageNo, buf, nil
// }
// // вглубь
// indexPageNo = foundPageNo
// }
return 0, nil, nil
}
type PathLeg struct {
PageNo uint32
Data []byte
}
type PathToDataPage struct {
Legs []PathLeg
LastPageNo uint32
}
func (s *Atree) FindPathToLastPage(rootPageNo uint32) (_ PathToDataPage, err error) {
// var (
// pageNo = rootPageNo
// legs []PathLeg
// )
// for {
// var buf []byte
// buf, err = s.fetchIndexPage(pageNo)
// if err != nil {
// err = fmt.Errorf("FetchIndexPage(%d): %s", pageNo, err)
// return
// }
// legs = append(legs, PathLeg{
// PageNo: pageNo,
// Data: buf,
// // childIdx не нужен
// })
// foundPageNo := getLastPageNo(buf)
// // fix
// if buf[isLastLevelIdx] == 1 {
// return PathToDataPage{
// Legs: legs,
// LastPageNo: foundPageNo,
// }, nil
// }
// // вглубь
// pageNo = foundPageNo
// }
return
}
// DELETE
type Level struct {
PageNo uint32
PageData []byte
Idx int
ChildQty int
}
func (s *Atree) GetAllPages(rootPageNo uint32) (_ []uint32, err error) {
// var (
// pageNumbers []uint32
// levels []*Level
// )
// buf, err := s.fetchIndexPage(rootPageNo)
// if err != nil {
// return nil, fmt.Errorf("fetchIndexPage(%d): %s", rootPageNo, err)
// }
// pageNumbers = append(pageNumbers, rootPageNo)
// // if buf[isDataPageNumbersIdx] == 1 {
// // pageNumbers := listPageNumbers(buf)
// // dataPages = append(dataPages, pageNumbers...)
// // s.releasePage(rootPageNo)
// // return PageLists{
// // DataPages: dataPages,
// // IndexPages: indexPages,
// // }, nil
// // }
// childQty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
// levels = append(levels, &Level{
// PageNo: rootPageNo,
// PageData: buf,
// Idx: 0,
// ChildQty: int(childQty),
// })
// for {
// if len(levels) == 0 {
// return pageNumbers, nil
// }
// lastIdx := len(levels) - 1
// level := levels[lastIdx]
// if level.Idx < level.ChildQty {
// pageNo := getPageNo(level.PageData, level.Idx)
// level.Idx++
// var buf []byte
// buf, err = s.fetchPage(pageNo)
// if err != nil {
// return nil, fmt.Errorf("fetchPage(%d): %s", pageNo, err)
// }
// pageNumbers = append(pageNumbers, pageNo)
// if buf[pageTypeIdx] == PageTypeData {
// //pageNumbers := listPageNumbers(buf)
// //dataPages = append(dataPages, pageNumbers...)
// s.releasePage(pageNo)
// } else {
// childQty, _ = bin.GetUint16(buf[indexRecordsQtyIdx:])
// levels = append(levels, &Level{
// PageNo: pageNo,
// PageData: buf,
// Idx: 0,
// ChildQty: int(childQty),
// })
// }
// } else {
// s.releasePage(level.PageNo)
// levels = levels[:lastIdx]
// }
// }
return
}

187
atree/cursor.go
View File

@@ -1,187 +0,0 @@
package atree
import (
"errors"
"fmt"
"gordenko.dev/dima/bin"
"gordenko.dev/dima/qb"
)
type BackwardCursor struct {
metricType qb.MetricType
fracDigits byte
atree *Atree
pageNo uint32
pageData []byte
timestampDecompressor qb.TimestampDecompressor
valueDecompressor qb.ValueDecompressor
}
type BackwardCursorOptions struct {
MetricType qb.MetricType
FracDigits byte
PageNo uint32
PageData []byte
Atree *Atree
}
func NewBackwardCursor(opt BackwardCursorOptions) (*BackwardCursor, error) {
switch opt.MetricType {
case qb.Instant, qb.Cumulative:
// ok
default:
return nil, fmt.Errorf("MetricType option has wrong value: %d", opt.MetricType)
}
if opt.FracDigits > qb.MaxFracDigits {
return nil, errors.New("FracDigits option is required")
}
if opt.Atree == nil {
return nil, errors.New("Atree option is required")
}
if opt.PageNo == 0 {
return nil, errors.New("PageNo option is required")
}
if len(opt.PageData) == 0 {
return nil, errors.New("PageData option is required")
}
s := &BackwardCursor{
metricType: opt.MetricType,
fracDigits: opt.FracDigits,
atree: opt.Atree,
pageNo: opt.PageNo,
pageData: opt.PageData,
}
err := s.makeDecompressors()
if err != nil {
return nil, err
}
return s, nil
}
// timestamp, value, done, error
func (s *BackwardCursor) Prev() (uint32, float64, bool, error) {
var (
timestamp uint32
value float64
//done bool
//err error
)
// timestamp, done = s.timestampDecompressor.NextValue()
// if !done {
// value, done = s.valueDecompressor.NextValue()
// if done {
// return 0, 0, false,
// fmt.Errorf("corrupted data page %d: has timestamp, no value",
// s.pageNo)
// }
// return timestamp, value, false, nil
// }
// prevPageNo, _ := bin.GetUint32(s.pageData[prevPageIdx:])
// if prevPageNo == 0 {
// return 0, 0, true, nil
// }
// s.atree.releasePage(s.pageNo)
// s.pageNo = prevPageNo
// s.pageData, err = s.atree.fetchDataPage(s.pageNo)
// if err != nil {
// return 0, 0, false, fmt.Errorf("atree.fetchDataPage(%d): %s", s.pageNo, err)
// }
// err = s.makeDecompressors()
// if err != nil {
// return 0, 0, false, err
// }
// timestamp, done = s.timestampDecompressor.NextValue()
// if done {
// return 0, 0, false,
// fmt.Errorf("corrupted data page %d: no timestamps",
// s.pageNo)
// }
// value, done = s.valueDecompressor.NextValue()
// if done {
// return 0, 0, false,
// fmt.Errorf("corrupted data page %d: no values",
// s.pageNo)
// }
return timestamp, value, false, nil
}
func (s *BackwardCursor) Close() {
s.atree.releasePage(s.pageNo)
}
// HELPER
func (s *BackwardCursor) makeDecompressors() error {
timestampsSize, _ := bin.GetUint16(s.pageData[timestampsSizeIdx:])
valuesSize, _ := bin.GetUint16(s.pageData[valuesSizeIdx:])
payloadSize := timestampsSize + valuesSize
if payloadSize > dataFooterIdx {
return fmt.Errorf("corrupted data page %d: timestamps + values size %d gt payload size",
s.pageNo, payloadSize)
}
s.timestampDecompressor = enc.NewTimeDeltaDecompressor(
s.pageData[:timestampsSize],
)
vbuf := s.pageData[timestampsSize : timestampsSize+valuesSize]
switch s.metricType {
case qb.Instant:
s.valueDecompressor = enc.NewInstantDeltaDecompressor(
vbuf, s.fracDigits)
case qb.Cumulative:
s.valueDecompressor = enc.NewCumulativeDeltaDecompressor(
vbuf, s.fracDigits)
default:
return fmt.Errorf("bug: wrong metricType %d", s.metricType)
}
return nil
}
// func makeDecompressors(pageData []byte, metricType qb.MetricType, fracDigits byte) (
// qb.TimestampDecompressor, qb.ValueDecompressor, error,
// ) {
// timestampsSize, _ := bin.GetUint16(pageData[timestampsSizeIdx:])
// valuesSize, _ := bin.GetUint16(pageData[valuesSizeIdx:])
// payloadSize := timestampsSize + valuesSize
// if payloadSize > dataFooterIdx {
// return nil, nil, fmt.Errorf("corrupted: timestamps + values size %d > payload size",
// payloadSize)
// }
// timestampDecompressor := enc.NewTimeDeltaDecompressor(
// pageData[:timestampsSize],
// )
// vbuf := pageData[timestampsSize : timestampsSize+valuesSize]
// var valueDecompressor qb.ValueDecompressor
// switch metricType {
// case qb.Instant:
// valueDecompressor = enc.NewInstantDeltaDecompressor(
// vbuf, fracDigits)
// case qb.Cumulative:
// valueDecompressor = enc.NewCumulativeDeltaDecompressor(
// vbuf, fracDigits)
// default:
// return nil, nil, fmt.Errorf("bug: wrong metricType %d", metricType)
// }
//return timestampDecompressor, valueDecompressor, nil
// return nil, nil, nil
// }

129
atree/select.go
View File

@@ -1,129 +0,0 @@
package atree
import (
"fmt"
"gordenko.dev/dima/qb"
)
type ContinueFullScanReq struct {
FracDigits byte
ResponseWriter AtreeMeasureConsumer
LastPageNo uint32
}
func (s *Atree) ContinueFullScan(req ContinueFullScanReq) error {
buf, err := s.fetchDataPage(req.LastPageNo)
if err != nil {
return fmt.Errorf("fetchDataPage(%d): %s", req.LastPageNo, err)
}
treeCursor, err := NewBackwardCursor(BackwardCursorOptions{
PageNo: req.LastPageNo,
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: qb.Instant,
})
if err != nil {
return err
}
defer treeCursor.Close()
for {
timestamp, value, done, err := treeCursor.Prev()
if err != nil {
return err
}
if done {
return nil
}
req.ResponseWriter.Feed(timestamp, value)
}
}
type ContinueRangeScanReq struct {
FracDigits byte
ResponseWriter AtreeMeasureConsumer
LastPageNo uint32
Since uint32
}
func (s *Atree) ContinueRangeScan(req ContinueRangeScanReq) error {
buf, err := s.fetchDataPage(req.LastPageNo)
if err != nil {
return fmt.Errorf("fetchDataPage(%d): %s", req.LastPageNo, err)
}
treeCursor, err := NewBackwardCursor(BackwardCursorOptions{
PageNo: req.LastPageNo,
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: qb.Instant,
})
if err != nil {
return err
}
defer treeCursor.Close()
for {
timestamp, value, done, err := treeCursor.Prev()
if err != nil {
return err
}
if done {
return nil
}
req.ResponseWriter.Feed(timestamp, value)
if timestamp < req.Since {
return nil
}
}
}
type RangeScanReq struct {
FracDigits byte
ResponseWriter AtreeMeasureConsumer
RootPageNo uint32
Since uint32
Until uint32
}
func (s *Atree) RangeScan(req RangeScanReq) error {
pageNo, buf, err := s.findDataPage(req.RootPageNo, req.Until)
if err != nil {
return err
}
cursor, err := NewBackwardCursor(BackwardCursorOptions{
PageNo: pageNo,
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: qb.Instant,
})
if err != nil {
return err
}
defer cursor.Close()
for {
timestamp, value, done, err := cursor.Prev()
if err != nil {
return err
}
if done {
return nil
}
if timestamp <= req.Until {
req.ResponseWriter.Feed(timestamp, value)
if timestamp < req.Since {
// - записи, удовлетворяющие временным рамкам, закончились.
return nil
}
}
}
}

1
b
View File

File diff suppressed because one or more lines are too long

103
database/api.go
View File

@@ -8,10 +8,10 @@ import (
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/bufreader"
"gordenko.dev/dima/qb/proto"
"gordenko.dev/dima/qb/storage"
"gordenko.dev/dima/qb/timeutil"
"gordenko.dev/dima/qb/transform"
"gordenko.dev/dima/qb/worker"
)
@@ -35,7 +35,6 @@ func reply(conn io.Writer, errcode uint16) {
}
bin.PutUint16(answer[1:], errcode)
}
_, err := conn.Write(answer)
if err != nil {
return
@@ -61,7 +60,6 @@ func (s *Database) handleTCPConn(conn net.Conn) {
}
func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (err error) {
//fmt.Println("process request")
messageType, err := r.ReadByte()
if err != nil {
if err != io.EOF {
@@ -71,8 +69,6 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e
}
}
//fmt.Println("messageType:", messageType)
switch messageType {
case proto.TypeGetMetric:
req, err := proto.ReadGetMetricReq(r)
@@ -111,7 +107,6 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e
if err != nil {
return fmt.Errorf("proto.ReadAppendMeasuresReq: %s", err)
}
//fmt.Println("append measure", req.MetricID, conn.RemoteAddr().String())
if err = s.AppendMeasures(conn, req); err != nil {
return fmt.Errorf("AppendMeasures: %s", err)
}
@@ -162,7 +157,6 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e
}
case proto.TypeDeleteMeasures:
//fmt.Println("delete metric")
req, err := proto.ReadDeleteMeasuresReq(r)
if err != nil {
return fmt.Errorf("proto.ReadDeleteMeasuresReq: %s", err)
@@ -183,7 +177,6 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e
if err != nil {
return fmt.Errorf("proto.ReadListAllCumulativeMeasuresReq: %s", err)
}
if err = s.ListAllCumulativeMeasures(conn, req); err != nil {
return fmt.Errorf("ListAllCumulativeMeasures: %s", err)
}
@@ -198,8 +191,6 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e
// API
func (s *Database) AddMetric(req proto.AddMetricReq) uint16 {
//fmt.Println("database.AddMetric")
// Валидация
if req.MetricID == 0 {
return proto.ErrEmptyMetricID
}
@@ -215,8 +206,6 @@ func (s *Database) AddMetric(req proto.AddMetricReq) uint16 {
resultCh := make(chan byte, 1)
fmt.Println("add job")
s.workerInbox.Push(worker.AddMetricReq{
MetricID: req.MetricID,
MetricType: req.MetricType,
@@ -224,20 +213,14 @@ func (s *Database) AddMetric(req proto.AddMetricReq) uint16 {
ResultCh: resultCh,
})
fmt.Println("job added")
resultCode := <-resultCh
switch resultCode {
case worker.Succeed:
fmt.Println("OK")
//
case worker.MetricDuplicate:
//fmt.Println("ErrDuplicate")
return proto.ErrDuplicate
default:
//fmt.Println("ErrWrongResultCodeBug")
qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
}
return 0
@@ -267,10 +250,8 @@ func (s *Database) GetMetric(conn io.Writer, req proto.GetMetricReq) error {
if err != nil {
return err
}
case worker.NoMetric:
reply(conn, proto.ErrNoMetric)
default:
qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
}
@@ -469,12 +450,11 @@ func (s *Database) ListCumulativeMeasures(conn net.Conn, req proto.ListCumulativ
}
func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriodsReq) error {
since, until := timeBoundsOfAggregation(req.Since, req.Until, req.GroupBy, req.FirstHourOfDay)
since, until := timeutil.TimeBoundsOfAggregation(req.Since, req.Until, req.GroupBy, req.FirstHourOfDay)
if since.After(until) {
reply(conn, proto.ErrInvalidRange)
return nil
}
responseWriter, err := transform.NewInstantPeriodsWriter(transform.InstantPeriodsWriterOptions{
Dst: conn,
GroupBy: req.GroupBy,
@@ -485,7 +465,6 @@ func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriod
reply(conn, proto.ErrUnexpected)
return nil
}
return s.rangeScan(rangeScanReq{
MetricID: req.MetricID,
MetricType: qb.Instant,
@@ -497,12 +476,11 @@ func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriod
}
func (s *Database) ListCumulativePeriods(conn net.Conn, req proto.ListCumulativePeriodsReq) error {
since, until := timeBoundsOfAggregation(req.Since, req.Until, req.GroupBy, req.FirstHourOfDay)
since, until := timeutil.TimeBoundsOfAggregation(req.Since, req.Until, req.GroupBy, req.FirstHourOfDay)
if since.After(until) {
reply(conn, proto.ErrInvalidRange)
return nil
}
responseWriter, err := transform.NewCumulativePeriodsWriter(transform.CumulativePeriodsWriterOptions{
Dst: conn,
GroupBy: req.GroupBy,
@@ -512,7 +490,6 @@ func (s *Database) ListCumulativePeriods(conn net.Conn, req proto.ListCumulative
reply(conn, proto.ErrUnexpected)
return nil
}
return s.rangeScan(rangeScanReq{
MetricID: req.MetricID,
MetricType: qb.Cumulative,
@@ -529,7 +506,7 @@ type rangeScanReq struct {
Since uint32
Until uint32
Conn io.Writer
ResponseWriter atree.PeriodsWriter
ResponseWriter qb.PeriodsWriter
}
func (s *Database) rangeScan(req rangeScanReq) error {
@@ -549,44 +526,40 @@ func (s *Database) rangeScan(req rangeScanReq) error {
switch result.ResultCode {
case worker.QueryDone:
req.ResponseWriter.Close()
case worker.UntilFound:
// err := s.atree.ContinueRangeScan(atree.ContinueRangeScanReq{
// FracDigits: result.FracDigits,
// ResponseWriter: req.ResponseWriter,
// LastPageNo: result.LastPageNo,
// Since: req.Since,
// })
//s.metricRUnlock(req.MetricID)
// if err != nil {
// reply(req.Conn, proto.ErrUnexpected)
// } else {
// req.ResponseWriter.Close()
// }
err := s.ContinueRangeScan(ContinueRangeScanReq{
MetricType: req.MetricType,
FracDigits: result.FracDigits,
ResponseWriter: req.ResponseWriter,
LastPageNo: result.LastPageNo,
Since: req.Since,
})
//s.metricRUnlock(req.MetricID) fix release unlock
if err != nil {
reply(req.Conn, proto.ErrUnexpected)
} else {
req.ResponseWriter.Close()
}
case worker.UntilNotFound:
// err := s.atree.RangeScan(atree.RangeScanReq{
// FracDigits: result.FracDigits,
// ResponseWriter: req.ResponseWriter,
// RootPageNo: result.RootPageNo,
// Since: req.Since,
// Until: req.Until,
// })
//s.metricRUnlock(req.MetricID)
// if err != nil {
// reply(req.Conn, proto.ErrUnexpected)
// } else {
// req.ResponseWriter.Close()
// }
err := s.RangeScan(RangeScanReq{
MetricType: req.MetricType,
FracDigits: result.FracDigits,
ResponseWriter: req.ResponseWriter,
Since: req.Since,
Until: req.Until,
LastPageNo: result.LastPageNo,
IsDataPage: result.IsDataPage,
})
//s.metricRUnlock(req.MetricID) // fix release
if err != nil {
reply(req.Conn, proto.ErrUnexpected)
} else {
req.ResponseWriter.Close()
}
case worker.NoMetric:
reply(req.Conn, proto.ErrNoMetric)
case worker.WrongMetricType:
reply(req.Conn, proto.ErrWrongMetricType)
default:
qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
}
@@ -597,7 +570,7 @@ type fullScanReq struct {
MetricID uint32
MetricType qb.MetricType
Conn io.Writer
ResponseWriter atree.PeriodsWriter
ResponseWriter qb.PeriodsWriter
}
func (s *Database) fullScan(req fullScanReq) error {
@@ -614,11 +587,10 @@ func (s *Database) fullScan(req fullScanReq) error {
switch result.ResultCode {
case worker.QueryDone:
fmt.Printf("query done")
req.ResponseWriter.Close()
case worker.UntilFound:
err := s.atree.ContinueFullScan(atree.ContinueFullScanReq{
err := s.ContinueFullScan(ContinueFullScanReq{
MetricType: req.MetricType,
FracDigits: result.FracDigits,
ResponseWriter: req.ResponseWriter,
LastPageNo: result.LastPageNo,
@@ -629,13 +601,10 @@ func (s *Database) fullScan(req fullScanReq) error {
} else {
req.ResponseWriter.Close()
}
case worker.NoMetric:
reply(req.Conn, proto.ErrNoMetric)
case worker.WrongMetricType:
reply(req.Conn, proto.ErrWrongMetricType)
default:
qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
}

360
database/database.go
View File

@@ -10,8 +10,8 @@ import (
"time"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/inbox"
"gordenko.dev/dima/qb/pagecache"
"gordenko.dev/dima/qb/recovery"
"gordenko.dev/dima/qb/storage"
"gordenko.dev/dima/qb/worker"
@@ -24,7 +24,8 @@ type Database struct {
workerInbox *inbox.Inbox
worker *worker.Worker
storage *storage.Writer
atree *atree.Atree
indexCache *pagecache.PageCache
dataCache *pagecache.PageCache
tcpPort int
logfile *os.File
logger *log.Logger
@@ -100,6 +101,24 @@ func New(opt Options) (_ *Database, err error) {
return nil, err
}
s.indexCache, err = pagecache.New(pagecache.Options{
File: indexFile,
PageSize: storage.IndexPageSize,
VerifyPageCRC: storage.VerifyIndexPageCRC32,
})
if err != nil {
return nil, fmt.Errorf("index pagecache.New: %s", err)
}
s.dataCache, err = pagecache.New(pagecache.Options{
File: dataFile,
PageSize: storage.DataPageSize,
VerifyPageCRC: storage.VerifyDataPageCRC32,
})
if err != nil {
return nil, fmt.Errorf("data pagecache.New: %s", err)
}
s.storage, err = storage.NewWriter(storage.WriterOptions{
Inbox: storageInbox,
WorkerInbox: s.workerInbox,
@@ -126,14 +145,6 @@ func New(opt Options) (_ *Database, err error) {
ExitCh: opt.ExitCh,
WaitGroup: opt.WaitGroup,
})
s.atree, err = atree.New(atree.Options{
DataFile: dataFile,
IndexFile: indexFile,
})
if err != nil {
return nil, fmt.Errorf("atree.New: %s", err)
}
return s, nil
}
@@ -146,14 +157,11 @@ func (s *Database) ListenAndServe() (err error) {
//s.waitGroup.Add(1)
go s.storage.Run()
// s.atree.Run()
s.waitGroup.Add(1)
go s.worker.Run()
s.logger.Println("database started")
for {
// Listen for an incoming connection.
conn, err := listener.Accept()
if err != nil {
s.logger.Printf("listener.Accept: %s\n", err)
@@ -164,53 +172,291 @@ func (s *Database) ListenAndServe() (err error) {
}
}
//
type ContinueFullScanReq struct {
MetricType qb.MetricType
FracDigits byte
ResponseWriter qb.AtreeMeasureConsumer
LastPageNo uint32
}
// зробити object?
func (s *Database) ContinueFullScan(req ContinueFullScanReq) error {
buf, err := s.dataCache.FetchPage(req.LastPageNo)
if err != nil {
return fmt.Errorf("dataCache.FetchPage(%d): %s", req.LastPageNo, err)
}
treeCursor, err := storage.NewBackwardCursor(storage.BackwardCursorOptions{
MetricType: req.MetricType,
FracDigits: req.FracDigits,
PageNo: req.LastPageNo,
PageData: buf,
FetchDataPage: s.dataCache.FetchPage,
ReleasePage: s.dataCache.ReleasePage,
})
if err != nil {
return err
}
defer treeCursor.Close()
for {
timestamp, value, done, err := treeCursor.Prev()
if err != nil {
return err
}
if done {
return nil
}
req.ResponseWriter.Feed(timestamp, value)
}
}
// func (s *Database) replayChanges(snapshotNumber int) (err error) {
// snapshot, err := readSnapshot(JoinSnapshotFileName(s.dir, snapshotNumber))
// if err != nil {
// return
// }
type ContinueRangeScanReq struct {
MetricType qb.MetricType
FracDigits byte
ResponseWriter qb.AtreeMeasureConsumer
LastPageNo uint32
Since uint32
}
// return nil
func (s *Database) ContinueRangeScan(req ContinueRangeScanReq) error {
buf, err := s.dataCache.FetchPage(req.LastPageNo)
if err != nil {
return fmt.Errorf("dataCache.FetchPage(%d): %s", req.LastPageNo, err)
}
treeCursor, err := storage.NewBackwardCursor(storage.BackwardCursorOptions{
MetricType: req.MetricType,
FracDigits: req.FracDigits,
PageNo: req.LastPageNo,
PageData: buf,
FetchDataPage: s.dataCache.FetchPage,
ReleasePage: s.dataCache.ReleasePage,
})
if err != nil {
return err
}
defer treeCursor.Close()
for {
timestamp, value, done, err := treeCursor.Prev()
if err != nil {
return err
}
if done {
return nil
}
req.ResponseWriter.Feed(timestamp, value)
if timestamp < req.Since {
return nil
}
}
}
type RangeScanReq struct {
MetricType qb.MetricType
FracDigits byte
ResponseWriter qb.AtreeMeasureConsumer
Since uint32
Until uint32
LastPageNo uint32
IsDataPage bool
}
func (s *Database) RangeScan(req RangeScanReq) error {
var (
pageNo uint32
buf []byte
err error
)
if req.IsDataPage {
pageNo = req.LastPageNo
buf, err = s.dataCache.FetchPage(pageNo)
if err != nil {
return fmt.Errorf("dataCache.FetchPage(%d): %s", pageNo, err)
}
} else {
pageNo, buf, err = s.findDataPage(req.LastPageNo, req.Until)
if err != nil {
return err
}
}
cursor, err := storage.NewBackwardCursor(storage.BackwardCursorOptions{
MetricType: req.MetricType,
FracDigits: req.FracDigits,
PageNo: pageNo,
PageData: buf,
FetchDataPage: s.dataCache.FetchPage,
ReleasePage: s.dataCache.ReleasePage,
})
if err != nil {
return err
}
defer cursor.Close()
for {
timestamp, value, done, err := cursor.Prev()
if err != nil {
return err
}
if done {
return nil
}
if timestamp <= req.Until {
req.ResponseWriter.Feed(timestamp, value)
if timestamp < req.Since {
// - записи, удовлетворяющие временным рамкам, закончились.
return nil
}
}
}
}
// (records payload only, isZeroLevel, timestamp)
func (s *Database) findDataPage(foundPageNo uint32, timestamp uint32) (uint32, []byte, error) {
for {
buf, err := s.indexCache.FetchPage(foundPageNo)
if err != nil {
return 0, nil, fmt.Errorf("fetchIndexPage(%d): %s", foundPageNo, err)
}
toReleaseIndexPageNo := foundPageNo
foundPageNo = storage.FindPageOnIndexPage(buf, timestamp)
s.indexCache.ReleasePage(toReleaseIndexPageNo)
if storage.IsZeroLevelPage(buf) {
buf, err := s.dataCache.FetchPage(foundPageNo)
if err != nil {
return 0, nil, fmt.Errorf("fetchDataPage(%d): %s", foundPageNo, err)
}
return foundPageNo, buf, nil
}
}
}
// type PathLeg struct {
// PageNo uint32
// Data []byte
// }
//func (s *Database) verifySnapshot(fileName string) (_ bool, err error) {
// file, err := os.Open(fileName)
// if err != nil {
// return
// }
// defer file.Close()
// stat, err := file.Stat()
// if err != nil {
// return
// }
// if stat.Size() <= 4 {
// return false, nil
// }
// var (
// payloadSize = stat.Size() - 4
// hash = crc32.NewIEEE()
// )
// _, err = io.CopyN(hash, file, payloadSize)
// if err != nil {
// return
// }
// calculatedCRC := hash.Sum32()
// storedCRC, err := bin.ReadUint32(file)
// if err != nil {
// return
// }
// if storedCRC != calculatedCRC {
// return false, fmt.Errorf("strored CRC %d not equal calculated CRC %d",
// storedCRC, calculatedCRC)
// }
// return true, nil
// type PathToDataPage struct {
// Legs []PathLeg
// LastPageNo uint32
// }
// func (s *Database) FindPathToLastPage(rootPageNo uint32) (_ PathToDataPage, err error) {
// // var (
// // pageNo = rootPageNo
// // legs []PathLeg
// // )
// // for {
// // var buf []byte
// // buf, err = s.fetchIndexPage(pageNo)
// // if err != nil {
// // err = fmt.Errorf("FetchIndexPage(%d): %s", pageNo, err)
// // return
// // }
// // legs = append(legs, PathLeg{
// // PageNo: pageNo,
// // Data: buf,
// // // childIdx не нужен
// // })
// // foundPageNo := getLastPageNo(buf)
// // // fix
// // if buf[isLastLevelIdx] == 1 {
// // return PathToDataPage{
// // Legs: legs,
// // LastPageNo: foundPageNo,
// // }, nil
// // }
// // // вглубь
// // pageNo = foundPageNo
// // }
// return
// }
// DELETE
// func (s *Atree) DeletePages(pageNumbers []uint32) {
// s.mutex.Lock()
// for _, pageNo := range pageNumbers {
// delete(s.pages, pageNo)
// }
// s.mutex.Unlock()
// }
// type Level struct {
// PageNo uint32
// PageData []byte
// Idx int
// ChildQty int
// }
// func (s *Atree) GetAllPages(rootPageNo uint32) (_ []uint32, err error) {
// // var (
// // pageNumbers []uint32
// // levels []*Level
// // )
// // buf, err := s.fetchIndexPage(rootPageNo)
// // if err != nil {
// // return nil, fmt.Errorf("fetchIndexPage(%d): %s", rootPageNo, err)
// // }
// // pageNumbers = append(pageNumbers, rootPageNo)
// // // if buf[isDataPageNumbersIdx] == 1 {
// // // pageNumbers := listPageNumbers(buf)
// // // dataPages = append(dataPages, pageNumbers...)
// // // s.releasePage(rootPageNo)
// // // return PageLists{
// // // DataPages: dataPages,
// // // IndexPages: indexPages,
// // // }, nil
// // // }
// // childQty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
// // levels = append(levels, &Level{
// // PageNo: rootPageNo,
// // PageData: buf,
// // Idx: 0,
// // ChildQty: int(childQty),
// // })
// // for {
// // if len(levels) == 0 {
// // return pageNumbers, nil
// // }
// // lastIdx := len(levels) - 1
// // level := levels[lastIdx]
// // if level.Idx < level.ChildQty {
// // pageNo := getPageNo(level.PageData, level.Idx)
// // level.Idx++
// // var buf []byte
// // buf, err = s.fetchPage(pageNo)
// // if err != nil {
// // return nil, fmt.Errorf("fetchPage(%d): %s", pageNo, err)
// // }
// // pageNumbers = append(pageNumbers, pageNo)
// // if buf[pageTypeIdx] == PageTypeData {
// // //pageNumbers := listPageNumbers(buf)
// // //dataPages = append(dataPages, pageNumbers...)
// // s.releasePage(pageNo)
// // } else {
// // childQty, _ = bin.GetUint16(buf[indexRecordsQtyIdx:])
// // levels = append(levels, &Level{
// // PageNo: pageNo,
// // PageData: buf,
// // Idx: 0,
// // ChildQty: int(childQty),
// // })
// // }
// // } else {
// // s.releasePage(level.PageNo)
// // levels = levels[:lastIdx]
// // }
// // }
// return
// }

24
database/database_test.go
View File

@@ -1,24 +0,0 @@
package database
// func TestComposeHeadIndexPage(t *testing.T) {
// var (
// levelIdx = 0
// level = storage.IndexLevelTail{
// Buffer: []byte{
// 1, 2, 3,
// },
// RecordsCount: 2,
// }
// head = &storage.IndexPageTail{
// PageNo: 100,
// CRC32: 12345,
// Records: []byte{},
// }
// )
// page := composeHeadIndexPage(levelIdx, level, head)
// if page.PageNo != head.PageNo {
// t.Fatalf("PageNo: got %d are not equal expected %v",
// page.PageNo, head.PageNo)
// }
// // fix compare pages
// }

52
database/helpers.go
View File

@@ -1,52 +0,0 @@
package database
import (
"errors"
"io/fs"
"os"
"time"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/proto"
)
func timeBoundsOfAggregation(since, until proto.TimeBound, groupBy qb.GroupBy, firstHourOfDay int) (s time.Time, u time.Time) {
switch groupBy {
case qb.GroupByHour, qb.GroupByDay:
s = time.Date(since.Year, since.Month, since.Day, 0, 0, 0, 0, time.Local)
u = time.Date(until.Year, until.Month, until.Day, 0, 0, 0, 0, time.Local)
case qb.GroupByMonth:
s = time.Date(since.Year, since.Month, 1, 0, 0, 0, 0, time.Local)
u = time.Date(until.Year, until.Month, 1, 0, 0, 0, 0, time.Local)
}
if firstHourOfDay > 0 {
duration := time.Duration(firstHourOfDay) * time.Hour
s = s.Add(duration)
u = u.Add(duration)
}
u = u.Add(-1 * time.Second)
return
}
func isFileExist(fileName string) (bool, error) {
_, err := os.Stat(fileName)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return false, nil
} else {
return false, err
}
} else {
return true, nil
}
}
func correctToFHD(since, until uint32, firstHourOfDay int) (uint32, uint32) {
duration := time.Duration(firstHourOfDay) * time.Hour
since = uint32(time.Unix(int64(since), 0).Add(duration).Unix())
until = uint32(time.Unix(int64(until), 0).Add(duration).Unix())
return since, until
}

25
enc/time_delta.go
View File

@@ -310,10 +310,33 @@ func (s *TimeDeltaCompressor) FirstTimestamp() uint32 {
return timestamp
}
func (s *TimeDeltaCompressor) LastTimestamp() uint32 {
func (s *TimeDeltaCompressor) Until() uint32 {
return s.lastUnixtime
}
func (s *TimeDeltaCompressor) CommitedSince() uint32 {
if s.state == nil {
if s.pos < len(s.buf) {
timestamp, _ := bin.GetUint32(s.buf[len(s.buf)-4:])
return timestamp
}
} else {
payload := s.state.Payload
if len(payload) > 0 {
timestamp, _ := bin.GetUint32(payload[len(payload)-4:])
return timestamp
}
}
return 0
}
func (s *TimeDeltaCompressor) CommitedUntil() uint32 {
if s.state == nil {
return s.lastUnixtime
}
return s.state.LastUnixtime
}
// DECOMPRESSOR
type TimeDeltaDecompressor struct {

4
inbox/inbox.go
View File

@@ -22,11 +22,9 @@ func (s *Inbox) Ready() chan struct{} {
}
func (s *Inbox) Push(x any) {
//fmt.Printf("inbox.Push: %#v\n", x)
s.mutex.Lock()
s.items = append(s.items, x)
s.mutex.Unlock()
//fmt.Printf("inbox.Pushed\n")
select {
case s.signalCh <- struct{}{}:
default:
@@ -34,11 +32,9 @@ func (s *Inbox) Push(x any) {
}
func (s *Inbox) Drain() []any {
//fmt.Printf("inbox.Drain\n")
s.mutex.Lock()
items := s.items
s.items = nil
s.mutex.Unlock()
//fmt.Printf("inbox.Drained: %#v\n", items)
return items
}

112
atree/io.go → pagecache/pagecache.go
View File

@@ -1,12 +1,12 @@
package atree
package pagecache
import (
"errors"
"fmt"
"os"
"sync"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/storage"
"gordenko.dev/dima/qb/util"
)
type readResult struct {
@@ -16,35 +16,52 @@ type readResult struct {
// INDEX PAGES
func (s *Atree) DeletePages(pageNumbers []uint32) {
s.mutex.Lock()
for _, pageNo := range pageNumbers {
delete(s.pages, pageNo)
type _page struct {
PageNo uint32
Buf []byte
ReferenceCount int
}
type PageCache struct {
mutex sync.Mutex
pageSize int
verifyPageCRC func([]byte) error
file *os.File
pages map[uint32]*_page
pageWaits map[uint32][]chan readResult
pagesToRead []uint32
readSignalCh chan struct{}
}
type Options struct {
File *os.File
PageSize int
VerifyPageCRC func([]byte) error
}
func New(opt Options) (*PageCache, error) {
if opt.File == nil {
return nil, errors.New("File option is required")
}
s.mutex.Unlock()
}
func (s *Atree) fetchIndexPage(pageNo uint32) ([]byte, error) {
// buf, err := s.fetchPage(pageNo)
// if err != nil {
// return nil, err
// }
// if buf[pageTypeIdx] != PageTypeIndex {
// return nil, fmt.Errorf("wrong pageType %d instead of %d", buf[pageTypeIdx], PageTypeIndex)
// }
// return buf, nil
return nil, nil
}
func (s *Atree) fetchDataPage(pageNo uint32) ([]byte, error) {
buf, err := s.fetchPage(pageNo)
if err != nil {
return nil, err
if opt.PageSize <= 0 {
return nil, errors.New("PageSize option is required")
}
return buf, nil
if opt.VerifyPageCRC == nil {
return nil, errors.New("VerifyPageCRC option is required")
}
s := &PageCache{
file: opt.File,
pageSize: opt.PageSize,
verifyPageCRC: opt.VerifyPageCRC,
pages: make(map[uint32]*_page),
pageWaits: make(map[uint32][]chan readResult),
readSignalCh: make(chan struct{}, 1),
}
go s.pageReader()
return s, nil
}
func (s *Atree) fetchPage(pageNo uint32) ([]byte, error) {
func (s *PageCache) FetchPage(pageNo uint32) ([]byte, error) {
s.mutex.Lock()
p, ok := s.pages[pageNo]
if ok {
@@ -69,12 +86,12 @@ func (s *Atree) fetchPage(pageNo uint32) ([]byte, error) {
result := <-resultCh
if result.Err == nil {
result.Err = s.verifyCRC(result.Data, storage.DataPageSize)
result.Err = s.verifyPageCRC(result.Data)
}
return result.Data, result.Err
}
func (s *Atree) releasePage(pageNo uint32) {
func (s *PageCache) ReleasePage(pageNo uint32) {
s.mutex.Lock()
defer s.mutex.Unlock()
@@ -93,11 +110,7 @@ func (s *Atree) releasePage(pageNo uint32) {
}
}
// DATA PAGES
// READ
func (s *Atree) pageReader() {
func (s *PageCache) pageReader() {
for {
select {
case <-s.readSignalCh:
@@ -106,7 +119,7 @@ func (s *Atree) pageReader() {
}
}
func (s *Atree) readPages() {
func (s *PageCache) readPages() {
s.mutex.Lock()
if len(s.pagesToRead) == 0 {
s.mutex.Unlock()
@@ -117,17 +130,15 @@ func (s *Atree) readPages() {
s.mutex.Unlock()
for _, pageNo := range pagesToRead {
buf := make([]byte, storage.DataPageSize)
off := int(pageNo-1) * storage.DataPageSize
buf := make([]byte, s.pageSize)
off := int(pageNo-1) * s.pageSize
n, err := s.file.ReadAt(buf, int64(off))
if n != storage.DataPageSize {
err = fmt.Errorf("read %d instead of %d", n, storage.DataPageSize)
if n != s.pageSize {
err = fmt.Errorf("read %d instead of %d", n, s.pageSize)
}
s.mutex.Lock()
resultChannels := s.pageWaits[pageNo]
delete(s.pageWaits, pageNo)
if err != nil {
s.mutex.Unlock()
for _, resultCh := range resultChannels {
@@ -150,18 +161,3 @@ func (s *Atree) readPages() {
}
}
}
// WRITE
func (s *Atree) verifyCRC(data []byte, pageSize int) error {
var (
pos = pageSize - 4
calculatedCRC = util.CalculateCRC32(data[:pos])
storedCRC, _ = bin.GetUint32(data[pos:])
)
if calculatedCRC != storedCRC {
return fmt.Errorf("calculatedCRC %d not equal storedCRC %d",
calculatedCRC, storedCRC)
}
return nil
}

24
qb.go
View File

@@ -24,6 +24,20 @@ const (
AggregateAvg byte = 4
)
type AtreeMeasureConsumer interface {
Feed(uint32, float64)
}
type PeriodsWriter interface {
Feed(uint32, float64)
FeedNoSend(uint32, float64)
Close() error
}
type WorkerMeasureConsumer interface {
FeedNoSend(uint32, float64)
}
type TimestampCompressor interface {
// (tmp, timestamp)
Evaluate([]byte, uint32) TimeEvaluationReport
@@ -37,12 +51,11 @@ type TimestampCompressor interface {
// (offset) => payload
Tail(int) []byte
CreateDecompressor() TimestampDecompressor
//Payload() []byte // для снапшота
// Offset() int
ReplaceBuffer([]byte)
WriteCommitedTo(io.Writer) error
FirstTimestamp() uint32
LastTimestamp() uint32
Until() uint32
CommitedSince() uint32
CommitedUntil() uint32
ReplaceSinceWithUntil() uint32
}
@@ -67,7 +80,6 @@ type ValueCompressor interface {
Append(int, []byte, float64, uint64)
Size() int
CommitedSize() int
//Chunks() [][]byte
//DeleteLast()
CaptureState()
ForgetCapturedState()
@@ -75,8 +87,6 @@ type ValueCompressor interface {
Tail(int) []byte
// fracDigits
CreateDecompressor(MetricType, byte) ValueDecompressor
//Payload() []byte // для снапшота
//Offset() int
ReplaceBuffer([]byte)
WriteCommitedTo(io.Writer) error
LastValue() float64

0
recovery/test.wal_0
View File

0
recovery/test.wal_123
View File

136
storage/cursor.go Normal file
View File

@@ -0,0 +1,136 @@
package storage
import (
"errors"
"fmt"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
)
type BackwardCursor struct {
metricType qb.MetricType
fracDigits byte
fetchDataPage func(uint32) ([]byte, error)
releasePage func(uint32)
pageNo uint32
pageData []byte
timestampDecompressor qb.TimestampDecompressor
valueDecompressor qb.ValueDecompressor
}
type BackwardCursorOptions struct {
MetricType qb.MetricType
FracDigits byte
PageNo uint32
PageData []byte
FetchDataPage func(uint32) ([]byte, error)
ReleasePage func(uint32)
}
func NewBackwardCursor(opt BackwardCursorOptions) (*BackwardCursor, error) {
switch opt.MetricType {
case qb.Instant, qb.Cumulative:
// ok
default:
return nil, fmt.Errorf("MetricType option has wrong value: %d", opt.MetricType)
}
if opt.FracDigits > qb.MaxFracDigits {
return nil, errors.New("FracDigits option is required")
}
if opt.FetchDataPage == nil {
return nil, errors.New("FetchDataPage option is required")
}
if opt.ReleasePage == nil {
return nil, errors.New("ReleasePage option is required")
}
if opt.PageNo == 0 {
return nil, errors.New("PageNo option is required")
}
if len(opt.PageData) == 0 {
return nil, errors.New("PageData option is required")
}
s := &BackwardCursor{
metricType: opt.MetricType,
fracDigits: opt.FracDigits,
fetchDataPage: opt.FetchDataPage,
releasePage: opt.ReleasePage,
pageNo: opt.PageNo,
pageData: opt.PageData,
}
var err error
s.timestampDecompressor, err = CreateTimestampDecompressor(s.pageData)
if err != nil {
return nil, fmt.Errorf("CreateTimestampDecompressor: %s", err)
}
s.valueDecompressor, err = CreateValueDecompressor(s.pageData, s.metricType, s.fracDigits)
if err != nil {
return nil, fmt.Errorf("CreateValueDecompressor: %s", err)
}
return s, nil
}
// timestamp, value, done, error
func (s *BackwardCursor) Prev() (uint32, float64, bool, error) {
var (
timestamp uint32
value float64
done bool
err error
)
timestamp, done = s.timestampDecompressor.NextValue()
if !done {
value, done = s.valueDecompressor.NextValue()
if done {
return 0, 0, false,
fmt.Errorf("corrupted data page %d: has timestamp, no value",
s.pageNo)
}
return timestamp, value, false, nil
}
prevPageNo, _ := bin.GetUint32(s.pageData[prevPageIdx:])
if prevPageNo == 0 {
return 0, 0, true, nil
}
s.releasePage(s.pageNo)
s.pageNo = prevPageNo
s.pageData, err = s.fetchDataPage(s.pageNo)
if err != nil {
return 0, 0, false,
fmt.Errorf("fetchDataPage(%d): %s", s.pageNo, err)
}
s.timestampDecompressor, err = CreateTimestampDecompressor(s.pageData)
if err != nil {
return 0, 0, false,
fmt.Errorf("CreateTimestampDecompressor: %s", err)
}
s.valueDecompressor, err = CreateValueDecompressor(s.pageData, s.metricType, s.fracDigits)
if err != nil {
return 0, 0, false,
fmt.Errorf("CreateValueDecompressor: %s", err)
}
//
timestamp, done = s.timestampDecompressor.NextValue()
if done {
return 0, 0, false,
fmt.Errorf("corrupted data page %d: no timestamps", s.pageNo)
}
value, done = s.valueDecompressor.NextValue()
if done {
return 0, 0, false,
fmt.Errorf("corrupted data page %d: no values", s.pageNo)
}
return timestamp, value, false, nil
}
func (s *BackwardCursor) Close() {
s.releasePage(s.pageNo)
}
// HELPER
//func (s *BackwardCursor) makeDecompressors() error {
//}

107
storage/misc.go
View File

@@ -6,87 +6,50 @@ import (
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/enc"
"gordenko.dev/dima/qb/util"
)
// func (s *BackwardCursor) makeDecompressors() error {
// timestampsSize, _ := bin.GetUint16(s.pageData[timestampsSizeIdx:])
// valuesSize, _ := bin.GetUint16(s.pageData[valuesSizeIdx:])
// payloadSize := timestampsSize + valuesSize
// if payloadSize > dataFooterIdx {
// return fmt.Errorf("corrupted data page %d: timestamps + values size %d gt payload size",
// s.pageNo, payloadSize)
// }
// s.timestampDecompressor = enc.NewTimeDeltaDecompressor(
// s.pageData[:timestampsSize],
// )
// vbuf := s.pageData[timestampsSize : timestampsSize+valuesSize]
// switch s.metricType {
// case qb.Instant:
// s.valueDecompressor = enc.NewInstantDeltaDecompressor(
// vbuf, s.fracDigits)
// case qb.Cumulative:
// s.valueDecompressor = enc.NewCumulativeDeltaDecompressor(
// vbuf, s.fracDigits)
// default:
// return fmt.Errorf("bug: wrong metricType %d", s.metricType)
// }
// return nil
// }
func makeDecompressors(pageData []byte, metricType qb.MetricType, fracDigits byte) (
qb.TimestampDecompressor, qb.ValueDecompressor, error,
) {
valuesSize, _ := bin.GetUint16(pageData[valuesSizeIdx:])
payloadSize := timestampsSize + valuesSize
if payloadSize > dataFooterIdx {
return nil, nil, fmt.Errorf("corrupted: timestamps + values size %d > payload size",
payloadSize)
}
timestampDecompressor := enc.NewTimeDeltaDecompressor(
pageData[:timestampsSize],
)
vbuf := pageData[timestampsSize : timestampsSize+valuesSize]
var valueDecompressor qb.ValueDecompressor
switch metricType {
case qb.Instant:
valueDecompressor = enc.NewInstantDeltaDecompressor(
vbuf, fracDigits)
case qb.Cumulative:
valueDecompressor = enc.NewCumulativeDeltaDecompressor(
vbuf, fracDigits)
default:
return nil, nil, fmt.Errorf("bug: wrong metricType %d", metricType)
}
return timestampDecompressor, valueDecompressor, nil
return nil, nil, nil
}
func CreateTimeDeltaDecompressor(page []byte) qb.TimestampDecompressor {
func CreateTimestampDecompressor(page []byte) (qb.TimestampDecompressor, error) {
size, _ := bin.GetUint16(page[timestampsSizeIdx:])
if size > DataPageFooterSize {
return nil, fmt.Errorf("bug: invalid timestamps size %d", size)
}
pos := DataPagePayloadSize - int(size)
d := enc.NewTimeDeltaDecompressor()
d.RestoreFromEnd(page[pos:DataPagePayloadSize])
return d
return d, nil
}
func CreateValueDeltaDecompressor(page []byte, metricType qb.MetricType, fracDigits byte) qb.ValueDecompressor {
func CreateValueDecompressor(page []byte, metricType qb.MetricType, fracDigits byte) (qb.ValueDecompressor, error) {
size, _ := bin.GetUint16(page[valuesSizeIdx:])
if size > DataPageFooterSize {
return nil, fmt.Errorf("bug: invalid timestamps size %d", size)
}
d := enc.NewValueDeltaDecompressor(metricType, fracDigits)
d.RestoreFromEnd(page[:size])
return d
return d, nil
}
func VerifyDataPageCRC32(data []byte) error {
var (
calculatedCRC = util.CalculateCRC32(data[:dataCRC32Idx])
writtenCRC, _ = bin.GetUint32(data[dataCRC32Idx:])
)
if calculatedCRC != writtenCRC {
return fmt.Errorf("calculated CRC32 %d are not equal written CRC32 %d",
calculatedCRC, writtenCRC)
}
return nil
}
func VerifyIndexPageCRC32(data []byte) error {
var (
calculatedCRC = util.CalculateCRC32(data[:indexCRC32Idx])
writtenCRC, _ = bin.GetUint32(data[indexCRC32Idx:])
)
if calculatedCRC != writtenCRC {
return fmt.Errorf("calculated CRC32 %d are not equal written CRC32 %d",
calculatedCRC, writtenCRC)
}
return nil
}

94
atree/misc.go → storage/navigation.go
View File

@@ -1,4 +1,10 @@
package atree
package storage
import bin "gordenko.dev/dima/bin/little"
const (
PageNoSize = 4
)
type KeyComparator interface {
CompareTo(int) int
@@ -10,19 +16,18 @@ type ValueAtComparator struct {
}
func (s ValueAtComparator) CompareTo(elemIdx int) int {
// var (
// pos = elemIdx * timestampSize
// elem, _ = bin.GetUint32(s.buf[pos:])
// )
var (
pos = elemIdx * IndexRecordSize
elem, _ = bin.GetUint32(s.buf[pos:])
)
// if s.timestamp < elem {
// return -1
// } else if s.timestamp > elem {
// return 1
// } else {
// return 0
// }
return 213131132
if s.timestamp < elem {
return -1
} else if s.timestamp > elem {
return 1
} else {
return 0
}
}
func BinarySearch(qty int, keyComparator KeyComparator) (elemIdx int, isFound bool) {
@@ -61,18 +66,63 @@ func BinarySearch(qty int, keyComparator KeyComparator) (elemIdx int, isFound bo
// return pageNo
// }
// func findPageNo(buf []byte, timestamp uint32) (pageNo uint32) {
// comparator := ValueAtComparator{
// buf: buf,
// timestamp: timestamp,
// }
// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
// elemIdx, _ := BinarySearch(int(qty), comparator)
// pos := indexFooterIdx - (elemIdx+1)*PageNoSize
// pageNo, _ = bin.GetUint32(buf[pos:])
// func GetIndexRecordsSince(buf []byte) (pageNo uint32) {
// pageNo, _ = bin.GetUint32(buf)
// return
// }
func FindPageOnIndexLevelTail(level IndexLevelTail, timestamp uint32) (pageNo uint32) {
comparator := ValueAtComparator{
buf: level.Buffer,
timestamp: timestamp,
}
elemIdx, _ := BinarySearch(level.RecordsCount, comparator)
pos := elemIdx*IndexRecordSize + 4 // timestamp size
pageNo, _ = bin.GetUint32(level.Buffer[pos:])
return
}
// func FindPageOnRecords(records []byte, timestamp uint32) (pageNo uint32) {
// comparator := ValueAtComparator{
// buf: records,
// timestamp: timestamp,
// }
// count := len(records) / IndexRecordSize
// elemIdx, _ := BinarySearch(count, comparator)
// pageNo, _ = bin.GetUint32(records[elemIdx*IndexRecordSize:])
// return
// }
func FindPageOnIndexPage(page []byte, timestamp uint32) (pageNo uint32) {
comparator := ValueAtComparator{
buf: page,
timestamp: timestamp,
}
count, _ := bin.GetUint16(page[indexRecordsCountIdx:])
elemIdx, _ := BinarySearch(int(count), comparator)
pos := elemIdx*IndexRecordSize + 4 // timestamp size
pageNo, _ = bin.GetUint32(page[pos:])
return
}
func IsZeroLevelPage(buf []byte) bool {
return buf[isZeroLevelIdx] == 1
}
// можна перевірити хвости від 0 до ... Перевіряю since кожного хвоста.
// Якщо timestamp >=, шукаю pageNo бінарним пошуком. І сторінка мені однозначно підходить.
// Якщо timestamp <, піднімаюсь вище. Якщо рівнів більше немає - until вказано за межами Range показань.
func FindPageOnIndexLevelTails(levels []IndexLevelTail, timestamp uint32) (pageNo uint32, isDataPage bool) {
for i, level := range levels {
tailSince, _ := bin.GetUint32(level.Buffer)
if timestamp >= tailSince {
pageNo = FindPageOnIndexLevelTail(level, timestamp)
return pageNo, i == 0
}
}
return
}
// func findPageNoIdx(buf []byte, timestamp uint32) (idx int) {
// comparator := ValueAtComparator{
// buf: buf,

4
storage/storage.go
View File

@@ -9,7 +9,7 @@ var (
// data page
DataPageSize = 8192
DataPagePayloadSize int = DataPageSize - DataPageFooterSize
DataPagePayloadSize = DataPageSize - DataPageFooterSize
dataCRC32Idx = DataPageSize - 4
timestampsSizeIdx = DataPageSize - 6
@@ -19,6 +19,7 @@ var (
// index page
IndexPageSize = 1024
IndexPagePayloadSize = IndexPageSize - IndexPageFooterSize
//indexPageIncSize = IndexPageIncSize
indexCRC32Idx = IndexPageSize - 4
indexRecordsCountIdx = IndexPageSize - 6
@@ -28,7 +29,6 @@ var (
// timestampSize = 4
// pairSize = timestampSize + PageNoSize
// indexFooterIdx = indexRecordsQtyIdx
// dataFooterIdx = timestampsSizeIdx
)

71
storage/storage_test.go
View File

@@ -1081,3 +1081,74 @@ func TestReplayMetric(t *testing.T) {
fmt.Printf("%d: % x\n", metric.IndexLevelTails[0].RecordsCount, metric.IndexLevelTails[0].Buffer)
fmt.Printf("%d: % x\n", metric.IndexLevelTails[1].RecordsCount, metric.IndexLevelTails[1].Buffer)
}
// func TestComposeHeadIndexPage(t *testing.T) {
// var (
// levelIdx = 0
// level = storage.IndexLevelTail{
// Buffer: []byte{
// 1, 2, 3,
// },
// RecordsCount: 2,
// }
// head = &storage.IndexPageTail{
// PageNo: 100,
// CRC32: 12345,
// Records: []byte{},
// }
// )
// page := composeHeadIndexPage(levelIdx, level, head)
// if page.PageNo != head.PageNo {
// t.Fatalf("PageNo: got %d are not equal expected %v",
// page.PageNo, head.PageNo)
// }
// // fix compare pages
// }
func TestFindPageOnIndexTails(t *testing.T) {
levels := []IndexLevelTail{
{
Buffer: []byte{
0x64, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, // 100 => 10
0x6e, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, // 110 => 11
},
RecordsCount: 2,
},
{
Buffer: []byte{
0x0a, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // 10 => 1
0x28, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, // 40 => 4
0x46, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, // 70 => 7
},
RecordsCount: 3,
},
}
testCases := []struct {
Timestamp uint32
PageNo uint32
IsDataPage bool
}{
{Timestamp: 9, PageNo: 0, IsDataPage: false},
{Timestamp: 10, PageNo: 1, IsDataPage: false},
{Timestamp: 20, PageNo: 1, IsDataPage: false},
{Timestamp: 40, PageNo: 4, IsDataPage: false},
{Timestamp: 60, PageNo: 4, IsDataPage: false},
{Timestamp: 70, PageNo: 7, IsDataPage: false},
{Timestamp: 90, PageNo: 7, IsDataPage: false},
{Timestamp: 100, PageNo: 10, IsDataPage: true},
{Timestamp: 105, PageNo: 10, IsDataPage: true},
{Timestamp: 110, PageNo: 11, IsDataPage: true},
{Timestamp: 120, PageNo: 11, IsDataPage: true},
}
for _, testCase := range testCases {
pageNo, isDataPage := FindPageOnIndexLevelTails(levels, testCase.Timestamp)
if pageNo != testCase.PageNo {
t.Fatalf("timestamp %d: got pageNo %d are not equal expected %d",
testCase.Timestamp, pageNo, testCase.PageNo)
}
if isDataPage != testCase.IsDataPage {
t.Fatalf("timestamp %d: got isDataPage %t are not equal expected %t",
testCase.Timestamp, isDataPage, testCase.IsDataPage)
}
}
}

3
storage/writer.go
View File

@@ -365,9 +365,6 @@ type PageToWrite struct {
func WriteDataPages(file *os.File, pages []PageToWrite) (err error) {
for _, p := range pages {
fmt.Println("pageNo: %d\n", p.PageNo)
}
for _, p := range pages {
if len(p.Content) != DataPageSize {
return fmt.Errorf("wrong data page size: %d", len(p.Content))
}

28
timeutil/timeutil.go
View File

@@ -1,6 +1,11 @@
package timeutil
import "time"
import (
"time"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/proto"
)
func FirstSecondInPeriod(since time.Time, period string) (_ time.Time) {
y, m, d := since.Date()
@@ -37,3 +42,24 @@ func LastSecondInPeriod(until time.Time, period string) (_ time.Time) {
return until
}
}
func TimeBoundsOfAggregation(since, until proto.TimeBound, groupBy qb.GroupBy, firstHourOfDay int) (s time.Time, u time.Time) {
switch groupBy {
case qb.GroupByHour, qb.GroupByDay:
s = time.Date(since.Year, since.Month, since.Day, 0, 0, 0, 0, time.Local)
u = time.Date(until.Year, until.Month, until.Day, 0, 0, 0, 0, time.Local)
case qb.GroupByMonth:
s = time.Date(since.Year, since.Month, 1, 0, 0, 0, 0, time.Local)
u = time.Date(until.Year, until.Month, 1, 0, 0, 0, 0, time.Local)
}
if firstHourOfDay > 0 {
duration := time.Duration(firstHourOfDay) * time.Hour
s = s.Add(duration)
u = u.Add(duration)
}
u = u.Add(-1 * time.Second)
return
}

22
database/tmp → tmp
View File

@@ -90,4 +90,24 @@
// ValuesSize: s.vSize,
// NewBufferSize: bufferSize,
// })
// }
// }
// func isFileExist(fileName string) (bool, error) {
// _, err := os.Stat(fileName)
// if err != nil {
// if errors.Is(err, fs.ErrNotExist) {
// return false, nil
// } else {
// return false, err
// }
// } else {
// return true, nil
// }
// }
// func correctToFHD(since, until uint32, firstHourOfDay int) (uint32, uint32) {
// duration := time.Duration(firstHourOfDay) * time.Hour
// since = uint32(time.Unix(int64(since), 0).Add(duration).Unix())
// until = uint32(time.Unix(int64(until), 0).Add(duration).Unix())
// return since, until
// }

151
worker/metric.go
View File

@@ -16,18 +16,14 @@ import (
var ErrNoValueBug = errors.New("has timestamp but no value")
type CapturedState struct {
LastTimestamp uint32
LastValue float64
LastValue float64
}
type Metric struct {
metricType qb.MetricType
fracDigits byte
lastPageNo uint32
//SinceValue float64
//Since uint32
lastValue float64
//Until uint32
metricType qb.MetricType
fracDigits byte
lastPageNo uint32
lastValue float64
buffer []byte
timestamps qb.TimestampCompressor
values qb.ValueCompressor
@@ -117,14 +113,42 @@ func (s *Metric) LastValue() float64 {
return s.lastValue
}
func (s *Metric) LastTimestamp() uint32 {
if s.capturedState != nil {
return s.capturedState.LastTimestamp
// REQUESTS
func (s *Metric) ReleaseRLock() {
s.rLocks--
if s.rLocks == 0 {
if len(s.waitQueue) > 0 {
s.ProcessQueue()
}
}
return s.timestamps.LastTimestamp()
}
// REQUESTS
// суть у тому що треба запускати запити, пока не зустріну XLock
func (s *Metric) ProcessQueue(metricID uint32, tmp []byte) {
if len(s.waitQueue) == 0 {
return
}
for _, untyped := range s.waitQueue {
switch req := untyped.(type) {
case RangeScanReq:
s.StartRangeScan(req)
case FullScanReq:
s.StartFullScan(req)
case GetMetricReq:
s.GetMetric(req)
case AppendMeasuresReq:
metric.AppendMeasures(req, tmp, s.storageInbox)
case DeleteMetricReq:
s.DeleteMetric(req)
case DeleteMeasuresReq:
metric.DeleteMeasures(req)
default:
qb.Abort(qb.UnknownMetricWaitQueueItemBug,
fmt.Errorf("bug: unknown metric wait queue item type %T", req))
}
}
}
func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox *inbox.Inbox) {
if s.xLock || s.capturedState != nil {
@@ -151,15 +175,14 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
)
s.capturedState = &CapturedState{
LastTimestamp: timestamps.LastTimestamp(),
LastValue: s.lastValue,
LastValue: s.lastValue,
}
s.timestamps.CaptureState()
s.values.CaptureState()
for idx, measure := range req.Measures {
if measure.Timestamp <= s.timestamps.LastTimestamp() {
if measure.Timestamp <= s.timestamps.Until() {
resultCode = ExpiredMeasure
break
}
@@ -270,15 +293,15 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
}
func (s *Metric) DeleteMeasures(req DeleteMeasuresReq) {
if s.xLock || s.capturedState != nil {
s.waitQueue = append(s.waitQueue, req)
return
}
since := s.timestamps.FirstTimestamp()
until := s.timestamps.LastTimestamp()
if since == 0 || (req.Since > 0 && until < req.Since) {
req.ResultCh <- NoMeasuresToDelete
}
// if s.xLock || s.capturedState != nil {
// s.waitQueue = append(s.waitQueue, req)
// return
// }
// since := s.timestamps.FirstTimestamp()
// until := s.timestamps.LastTimestamp()
// if since == 0 || (req.Since > 0 && until < req.Since) {
// req.ResultCh <- NoMeasuresToDelete
// }
// if s.RootPageNo > 0 {
// req.ResultCh <- tryDeleteMeasuresResult{
// ResultCode: DeleteFromAtreeRequired,
@@ -296,36 +319,47 @@ func (s *Metric) StartRangeScan(req RangeScanReq) {
s.waitQueue = append(s.waitQueue, req)
return
}
// if s.timestamps.CommitedSize() == 0 {
// req.ResultCh <- RangeScanResult{
// ResultCode: QueryDone,
// }
// return
// }
// if req.Since > s.timestamps.LastTimestamp() {
// req.ResultCh <- RangeScanResult{
// ResultCode: QueryDone,
// }
// return
// }
// if req.Until < s.Since {
// if s.RootPageNo > 0 {
// req.ResultCh <- RangeScanResult{
// ResultCode: UntilNotFound,
// RootPageNo: s.RootPageNo,
// FracDigits: s.fracDigits,
// }
// s.rLocks++
// return
// } else {
// req.ResultCh <- RangeScanResult{
// ResultCode: QueryDone,
// }
// return
// }
// }
since := s.timestamps.CommitedSince()
if since == 0 {
req.ResultCh <- RangeScanResult{
ResultCode: QueryDone,
}
return
}
// range after
if req.Since > s.timestamps.CommitedUntil() {
req.ResultCh <- RangeScanResult{
ResultCode: QueryDone,
}
return
}
// range before
if req.Until < since {
if len(s.indexLevelTails) > 0 {
pageNo, isDataPage := storage.FindPageOnIndexLevelTails(s.indexLevelTails, req.Until)
if pageNo > 0 {
req.ResultCh <- RangeScanResult{
ResultCode: UntilNotFound,
FracDigits: s.fracDigits,
LastPageNo: pageNo,
IsDataPage: isDataPage,
}
s.rLocks++
return
} else {
// range until before 1st measure timestamp
req.ResultCh <- RangeScanResult{
ResultCode: QueryDone,
}
return
}
} else {
req.ResultCh <- RangeScanResult{
ResultCode: QueryDone,
}
return
}
}
timestampDecompressor := s.timestamps.CreateDecompressor()
valueDecompressor := s.values.CreateDecompressor(s.metricType, s.fracDigits)
@@ -443,11 +477,6 @@ func (s *Metric) OnMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) {
s.xLock = false
// s.Timestamps.Renew()
// s.Values.Renew()
// s.LastPageNo = 0
// s.Since = 0
// s.SinceValue = 0
// s.Until = 0
s.indexLevelTails = nil
s.lastPageNo = 0
s.lastValue = 0

78
worker/worker.go
View File

@@ -5,7 +5,6 @@ import (
"sync"
qb "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/enc"
"gordenko.dev/dima/qb/inbox"
"gordenko.dev/dima/qb/proto"
@@ -86,15 +85,8 @@ func New(opt Options) *Worker {
return s
}
func (s *Worker) ReleaseRLock(metricID uint32) {
// s.mutex.Lock()
// //s.rLocksToRelease = append(s.rLocksToRelease, metricID)
// s.mutex.Unlock()
// select {
// case s.signalCh <- struct{}{}:
// default:
// }
type ReleaseRLock struct {
MetricID uint32
}
func (s *Worker) Run() {
@@ -114,39 +106,12 @@ func (s *Worker) Run() {
func (s *Worker) doWork() {
queue := s.inbox.Drain()
//rLocksToRelease := s.rLocksToRelease
//s.rLocksToRelease = nil
// for _, metricID := range rLocksToRelease {
// metric, ok := s.metrics[metricID]
// if !ok {
// qb.Abort(qb.NoMetricBug,
// fmt.Errorf("drainQueues: metric %d not found", metricID))
// }
// if metric.XLock {
// qb.Abort(qb.XLockBug,
// fmt.Errorf("drainQueues: xlock is set for the metric %d",
// metricID))
// }
// if metric.RLocks <= 0 {
// qb.Abort(qb.NoRLockBug,
// fmt.Errorf("drainQueues: rlock not set for the metric %d",
// metricID))
// }
// metric.RLocks--
// if len(metric.WaitQueue) > 0 {
// s.processMetricQueue(metricID, metric)
// }
// }
for _, untyped := range queue {
switch req := untyped.(type) {
case AppendMeasuresReq:
s.AppendMeasures(req)
case ReleaseRLock:
s.releaseRLock(req.MetricID)
case storage.Changes:
s.applyCommits(req) // all metrics only
case ListCurrentValuesReq:
@@ -170,29 +135,10 @@ func (s *Worker) doWork() {
}
}
// суть у тому що треба запускати запити, пока не зустріну XLock
func (s *Worker) processMetricQueue(metricID uint32, metric *Metric, tmp []byte) {
if len(metric.waitQueue) == 0 {
return
}
for _, untyped := range metric.waitQueue {
switch req := untyped.(type) {
case RangeScanReq:
metric.StartRangeScan(req)
case FullScanReq:
metric.StartFullScan(req)
case GetMetricReq:
s.GetMetric(req)
case AppendMeasuresReq:
metric.AppendMeasures(req, tmp, s.storageInbox)
case DeleteMetricReq:
s.DeleteMetric(req)
case DeleteMeasuresReq:
metric.DeleteMeasures(req)
default:
qb.Abort(qb.UnknownMetricWaitQueueItemBug,
fmt.Errorf("bug: unknown metric wait queue item type %T", req))
}
func (s *Worker) releaseRLock(metricID uint32) {
metric, ok := s.metrics[metricID]
if ok {
metric.ReleaseRLock()
}
}
@@ -318,8 +264,8 @@ func (s *Worker) AppendMeasures(req AppendMeasuresReq) {
type RangeScanResult struct {
ResultCode byte
FracDigits byte
RootPageNo uint32
LastPageNo uint32
IsDataPage bool // for UntilNotFound only
}
type RangeScanReq struct {
@@ -327,7 +273,7 @@ type RangeScanReq struct {
Since uint32
Until uint32
MetricType qb.MetricType
ResponseWriter atree.WorkerMeasureConsumer
ResponseWriter qb.WorkerMeasureConsumer
ResultCh chan RangeScanResult
}
@@ -361,7 +307,7 @@ type FullScanResult struct {
type FullScanReq struct {
MetricID uint32
MetricType qb.MetricType
ResponseWriter atree.WorkerMeasureConsumer
ResponseWriter qb.WorkerMeasureConsumer
ResultCh chan FullScanResult
}
@@ -398,7 +344,7 @@ func (s *Worker) ListCurrentValues(req ListCurrentValuesReq) {
if ok {
req.ResponseWriter.BufferValue(transform.CurrentValue{
MetricID: metricID,
Timestamp: metric.LastTimestamp(),
Timestamp: metric.timestamps.CommitedUntil(),
Value: metric.LastValue(),
})
}