dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
dima
2026-06-15 10:47:24 +00:00
parent ca125c99d2
commit 06823ee547
28 changed files with 1243 additions and 1383 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
a
View File

File diff suppressed because one or more lines are too long

224
atree/atree.go
View File

@@ -1,224 +0,0 @@
package atree
import (
"errors"
"os"
"sync"
)
const (
filePerm = 0666
)
type PeriodsWriter interface {
Feed(uint32, float64)
FeedNoSend(uint32, float64)
Close() error
}
type WorkerMeasureConsumer interface {
FeedNoSend(uint32, float64)
}
type AtreeMeasureConsumer interface {
Feed(uint32, float64)
}
//
type _page struct {
PageNo uint32
Buf []byte
ReferenceCount int
}
type Atree struct {
indexFile *os.File
dataFile *os.File
mutex sync.Mutex
pages map[uint32]*_page
pageWaits map[uint32][]chan readResult
pagesToRead []uint32
readSignalCh chan struct{}
}
type Options struct {
IndexFile *os.File
DataFile *os.File
}
func New(opt Options) (*Atree, error) {
if opt.IndexFile == nil {
return nil, errors.New("IndexFile option is required")
}
if opt.DataFile == nil {
return nil, errors.New("DataFile option is required")
}
s := &Atree{
indexFile: opt.IndexFile,
dataFile: opt.DataFile,
pages: make(map[uint32]*_page),
pageWaits: make(map[uint32][]chan readResult),
readSignalCh: make(chan struct{}, 1),
}
return s, nil
}
func (s *Atree) Run() {
//go s.pageWriter()
go s.pageReader()
}
// FIND
func (s *Atree) findDataPage(rootPageNo uint32, timestamp uint32) (uint32, []byte, error) {
// indexPageNo := rootPageNo
// for {
// buf, err := s.fetchIndexPage(indexPageNo)
// if err != nil {
// return 0, nil, fmt.Errorf("fetchIndexPage(%d): %s", indexPageNo, err)
// }
// foundPageNo := findPageNo(buf, timestamp)
// s.releasePage(indexPageNo)
// // fix
// if buf[isLastLevelIdx] == 1 {
// buf, err := s.fetchDataPage(foundPageNo)
// if err != nil {
// return 0, nil, fmt.Errorf("fetchDataPage(%d): %s", foundPageNo, err)
// }
// return foundPageNo, buf, nil
// }
// // вглубь
// indexPageNo = foundPageNo
// }
return 0, nil, nil
}
type PathLeg struct {
PageNo uint32
Data []byte
}
type PathToDataPage struct {
Legs []PathLeg
LastPageNo uint32
}
func (s *Atree) FindPathToLastPage(rootPageNo uint32) (_ PathToDataPage, err error) {
// var (
// pageNo = rootPageNo
// legs []PathLeg
// )
// for {
// var buf []byte
// buf, err = s.fetchIndexPage(pageNo)
// if err != nil {
// err = fmt.Errorf("FetchIndexPage(%d): %s", pageNo, err)
// return
// }
// legs = append(legs, PathLeg{
// PageNo: pageNo,
// Data: buf,
// // childIdx не нужен
// })
// foundPageNo := getLastPageNo(buf)
// // fix
// if buf[isLastLevelIdx] == 1 {
// return PathToDataPage{
// Legs: legs,
// LastPageNo: foundPageNo,
// }, nil
// }
// // вглубь
// pageNo = foundPageNo
// }
return
}
// DELETE
type Level struct {
PageNo uint32
PageData []byte
Idx int
ChildQty int
}
func (s *Atree) GetAllPages(rootPageNo uint32) (_ []uint32, err error) {
// var (
// pageNumbers []uint32
// levels []*Level
// )
// buf, err := s.fetchIndexPage(rootPageNo)
// if err != nil {
// return nil, fmt.Errorf("fetchIndexPage(%d): %s", rootPageNo, err)
// }
// pageNumbers = append(pageNumbers, rootPageNo)
// // if buf[isDataPageNumbersIdx] == 1 {
// // pageNumbers := listPageNumbers(buf)
// // dataPages = append(dataPages, pageNumbers...)
// // s.releasePage(rootPageNo)
// // return PageLists{
// // DataPages: dataPages,
// // IndexPages: indexPages,
// // }, nil
// // }
// childQty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
// levels = append(levels, &Level{
// PageNo: rootPageNo,
// PageData: buf,
// Idx: 0,
// ChildQty: int(childQty),
// })
// for {
// if len(levels) == 0 {
// return pageNumbers, nil
// }
// lastIdx := len(levels) - 1
// level := levels[lastIdx]
// if level.Idx < level.ChildQty {
// pageNo := getPageNo(level.PageData, level.Idx)
// level.Idx++
// var buf []byte
// buf, err = s.fetchPage(pageNo)
// if err != nil {
// return nil, fmt.Errorf("fetchPage(%d): %s", pageNo, err)
// }
// pageNumbers = append(pageNumbers, pageNo)
// if buf[pageTypeIdx] == PageTypeData {
// //pageNumbers := listPageNumbers(buf)
// //dataPages = append(dataPages, pageNumbers...)
// s.releasePage(pageNo)
// } else {
// childQty, _ = bin.GetUint16(buf[indexRecordsQtyIdx:])
// levels = append(levels, &Level{
// PageNo: pageNo,
// PageData: buf,
// Idx: 0,
// ChildQty: int(childQty),
// })
// }
// } else {
// s.releasePage(level.PageNo)
// levels = levels[:lastIdx]
// }
// }
return
}

187
atree/cursor.go
View File

@@ -1,187 +0,0 @@
package atree
import (
"errors"
"fmt"
"gordenko.dev/dima/bin"
"gordenko.dev/dima/qb"
)
type BackwardCursor struct {
metricType qb.MetricType
fracDigits byte
atree *Atree
pageNo uint32
pageData []byte
timestampDecompressor qb.TimestampDecompressor
valueDecompressor qb.ValueDecompressor
}
type BackwardCursorOptions struct {
MetricType qb.MetricType
FracDigits byte
PageNo uint32
PageData []byte
Atree *Atree
}
func NewBackwardCursor(opt BackwardCursorOptions) (*BackwardCursor, error) {
switch opt.MetricType {
case qb.Instant, qb.Cumulative:
// ok
default:
return nil, fmt.Errorf("MetricType option has wrong value: %d", opt.MetricType)
}
if opt.FracDigits > qb.MaxFracDigits {
return nil, errors.New("FracDigits option is required")
}
if opt.Atree == nil {
return nil, errors.New("Atree option is required")
}
if opt.PageNo == 0 {
return nil, errors.New("PageNo option is required")
}
if len(opt.PageData) == 0 {
return nil, errors.New("PageData option is required")
}
s := &BackwardCursor{
metricType: opt.MetricType,
fracDigits: opt.FracDigits,
atree: opt.Atree,
pageNo: opt.PageNo,
pageData: opt.PageData,
}
err := s.makeDecompressors()
if err != nil {
return nil, err
}
return s, nil
}
// timestamp, value, done, error
func (s *BackwardCursor) Prev() (uint32, float64, bool, error) {
var (
timestamp uint32
value float64
//done bool
//err error
)
// timestamp, done = s.timestampDecompressor.NextValue()
// if !done {
// value, done = s.valueDecompressor.NextValue()
// if done {
// return 0, 0, false,
// fmt.Errorf("corrupted data page %d: has timestamp, no value",
// s.pageNo)
// }
// return timestamp, value, false, nil
// }
// prevPageNo, _ := bin.GetUint32(s.pageData[prevPageIdx:])
// if prevPageNo == 0 {
// return 0, 0, true, nil
// }
// s.atree.releasePage(s.pageNo)
// s.pageNo = prevPageNo
// s.pageData, err = s.atree.fetchDataPage(s.pageNo)
// if err != nil {
// return 0, 0, false, fmt.Errorf("atree.fetchDataPage(%d): %s", s.pageNo, err)
// }
// err = s.makeDecompressors()
// if err != nil {
// return 0, 0, false, err
// }
// timestamp, done = s.timestampDecompressor.NextValue()
// if done {
// return 0, 0, false,
// fmt.Errorf("corrupted data page %d: no timestamps",
// s.pageNo)
// }
// value, done = s.valueDecompressor.NextValue()
// if done {
// return 0, 0, false,
// fmt.Errorf("corrupted data page %d: no values",
// s.pageNo)
// }
return timestamp, value, false, nil
}
func (s *BackwardCursor) Close() {
s.atree.releasePage(s.pageNo)
}
// HELPER
func (s *BackwardCursor) makeDecompressors() error {
timestampsSize, _ := bin.GetUint16(s.pageData[timestampsSizeIdx:])
valuesSize, _ := bin.GetUint16(s.pageData[valuesSizeIdx:])
payloadSize := timestampsSize + valuesSize
if payloadSize > dataFooterIdx {
return fmt.Errorf("corrupted data page %d: timestamps + values size %d gt payload size",
s.pageNo, payloadSize)
}
s.timestampDecompressor = enc.NewTimeDeltaDecompressor(
s.pageData[:timestampsSize],
)
vbuf := s.pageData[timestampsSize : timestampsSize+valuesSize]
switch s.metricType {
case qb.Instant:
s.valueDecompressor = enc.NewInstantDeltaDecompressor(
vbuf, s.fracDigits)
case qb.Cumulative:
s.valueDecompressor = enc.NewCumulativeDeltaDecompressor(
vbuf, s.fracDigits)
default:
return fmt.Errorf("bug: wrong metricType %d", s.metricType)
}
return nil
}
// func makeDecompressors(pageData []byte, metricType qb.MetricType, fracDigits byte) (
// qb.TimestampDecompressor, qb.ValueDecompressor, error,
// ) {
// timestampsSize, _ := bin.GetUint16(pageData[timestampsSizeIdx:])
// valuesSize, _ := bin.GetUint16(pageData[valuesSizeIdx:])
// payloadSize := timestampsSize + valuesSize
// if payloadSize > dataFooterIdx {
// return nil, nil, fmt.Errorf("corrupted: timestamps + values size %d > payload size",
// payloadSize)
// }
// timestampDecompressor := enc.NewTimeDeltaDecompressor(
// pageData[:timestampsSize],
// )
// vbuf := pageData[timestampsSize : timestampsSize+valuesSize]
// var valueDecompressor qb.ValueDecompressor
// switch metricType {
// case qb.Instant:
// valueDecompressor = enc.NewInstantDeltaDecompressor(
// vbuf, fracDigits)
// case qb.Cumulative:
// valueDecompressor = enc.NewCumulativeDeltaDecompressor(
// vbuf, fracDigits)
// default:
// return nil, nil, fmt.Errorf("bug: wrong metricType %d", metricType)
// }
//return timestampDecompressor, valueDecompressor, nil
// return nil, nil, nil
// }

129
atree/select.go
View File

@@ -1,129 +0,0 @@
package atree
import (
"fmt"
"gordenko.dev/dima/qb"
)
type ContinueFullScanReq struct {
FracDigits byte
ResponseWriter AtreeMeasureConsumer
LastPageNo uint32
}
func (s *Atree) ContinueFullScan(req ContinueFullScanReq) error {
buf, err := s.fetchDataPage(req.LastPageNo)
if err != nil {
return fmt.Errorf("fetchDataPage(%d): %s", req.LastPageNo, err)
}
treeCursor, err := NewBackwardCursor(BackwardCursorOptions{
PageNo: req.LastPageNo,
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: qb.Instant,
})
if err != nil {
return err
}
defer treeCursor.Close()
for {
timestamp, value, done, err := treeCursor.Prev()
if err != nil {
return err
}
if done {
return nil
}
req.ResponseWriter.Feed(timestamp, value)
}
}
type ContinueRangeScanReq struct {
FracDigits byte
ResponseWriter AtreeMeasureConsumer
LastPageNo uint32
Since uint32
}
func (s *Atree) ContinueRangeScan(req ContinueRangeScanReq) error {
buf, err := s.fetchDataPage(req.LastPageNo)
if err != nil {
return fmt.Errorf("fetchDataPage(%d): %s", req.LastPageNo, err)
}
treeCursor, err := NewBackwardCursor(BackwardCursorOptions{
PageNo: req.LastPageNo,
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: qb.Instant,
})
if err != nil {
return err
}
defer treeCursor.Close()
for {
timestamp, value, done, err := treeCursor.Prev()
if err != nil {
return err
}
if done {
return nil
}
req.ResponseWriter.Feed(timestamp, value)
if timestamp < req.Since {
return nil
}
}
}
type RangeScanReq struct {
FracDigits byte
ResponseWriter AtreeMeasureConsumer
RootPageNo uint32
Since uint32
Until uint32
}
func (s *Atree) RangeScan(req RangeScanReq) error {
pageNo, buf, err := s.findDataPage(req.RootPageNo, req.Until)
if err != nil {
return err
}
cursor, err := NewBackwardCursor(BackwardCursorOptions{
PageNo: pageNo,
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: qb.Instant,
})
if err != nil {
return err
}
defer cursor.Close()
for {
timestamp, value, done, err := cursor.Prev()
if err != nil {
return err
}
if done {
return nil
}
if timestamp <= req.Until {
req.ResponseWriter.Feed(timestamp, value)
if timestamp < req.Since {
// - записи, удовлетворяющие временным рамкам, закончились.
return nil
}
}
}
}

1
b
View File

File diff suppressed because one or more lines are too long

103
database/api.go
View File

@@ -8,10 +8,10 @@ import (
bin "gordenko.dev/dima/bin/little" bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb" "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/bufreader" "gordenko.dev/dima/qb/bufreader"
"gordenko.dev/dima/qb/proto" "gordenko.dev/dima/qb/proto"
"gordenko.dev/dima/qb/storage" "gordenko.dev/dima/qb/storage"
"gordenko.dev/dima/qb/timeutil"
"gordenko.dev/dima/qb/transform" "gordenko.dev/dima/qb/transform"
"gordenko.dev/dima/qb/worker" "gordenko.dev/dima/qb/worker"
) )
@@ -35,7 +35,6 @@ func reply(conn io.Writer, errcode uint16) {
} }
bin.PutUint16(answer[1:], errcode) bin.PutUint16(answer[1:], errcode)
} }
_, err := conn.Write(answer) _, err := conn.Write(answer)
if err != nil { if err != nil {
return return
@@ -61,7 +60,6 @@ func (s *Database) handleTCPConn(conn net.Conn) {
} }
func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (err error) { func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (err error) {
//fmt.Println("process request")
messageType, err := r.ReadByte() messageType, err := r.ReadByte()
if err != nil { if err != nil {
if err != io.EOF { if err != io.EOF {
@@ -71,8 +69,6 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e
} }
} }
//fmt.Println("messageType:", messageType)
switch messageType { switch messageType {
case proto.TypeGetMetric: case proto.TypeGetMetric:
req, err := proto.ReadGetMetricReq(r) req, err := proto.ReadGetMetricReq(r)
@@ -111,7 +107,6 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e
if err != nil { if err != nil {
return fmt.Errorf("proto.ReadAppendMeasuresReq: %s", err) return fmt.Errorf("proto.ReadAppendMeasuresReq: %s", err)
} }
//fmt.Println("append measure", req.MetricID, conn.RemoteAddr().String())
if err = s.AppendMeasures(conn, req); err != nil { if err = s.AppendMeasures(conn, req); err != nil {
return fmt.Errorf("AppendMeasures: %s", err) return fmt.Errorf("AppendMeasures: %s", err)
} }
@@ -162,7 +157,6 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e
} }
case proto.TypeDeleteMeasures: case proto.TypeDeleteMeasures:
//fmt.Println("delete metric")
req, err := proto.ReadDeleteMeasuresReq(r) req, err := proto.ReadDeleteMeasuresReq(r)
if err != nil { if err != nil {
return fmt.Errorf("proto.ReadDeleteMeasuresReq: %s", err) return fmt.Errorf("proto.ReadDeleteMeasuresReq: %s", err)
@@ -183,7 +177,6 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e
if err != nil { if err != nil {
return fmt.Errorf("proto.ReadListAllCumulativeMeasuresReq: %s", err) return fmt.Errorf("proto.ReadListAllCumulativeMeasuresReq: %s", err)
} }
if err = s.ListAllCumulativeMeasures(conn, req); err != nil { if err = s.ListAllCumulativeMeasures(conn, req); err != nil {
return fmt.Errorf("ListAllCumulativeMeasures: %s", err) return fmt.Errorf("ListAllCumulativeMeasures: %s", err)
} }
@@ -198,8 +191,6 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e
// API // API
func (s *Database) AddMetric(req proto.AddMetricReq) uint16 { func (s *Database) AddMetric(req proto.AddMetricReq) uint16 {
//fmt.Println("database.AddMetric")
// Валидация
if req.MetricID == 0 { if req.MetricID == 0 {
return proto.ErrEmptyMetricID return proto.ErrEmptyMetricID
} }
@@ -215,8 +206,6 @@ func (s *Database) AddMetric(req proto.AddMetricReq) uint16 {
resultCh := make(chan byte, 1) resultCh := make(chan byte, 1)
fmt.Println("add job")
s.workerInbox.Push(worker.AddMetricReq{ s.workerInbox.Push(worker.AddMetricReq{
MetricID: req.MetricID, MetricID: req.MetricID,
MetricType: req.MetricType, MetricType: req.MetricType,
@@ -224,20 +213,14 @@ func (s *Database) AddMetric(req proto.AddMetricReq) uint16 {
ResultCh: resultCh, ResultCh: resultCh,
}) })
fmt.Println("job added")
resultCode := <-resultCh resultCode := <-resultCh
switch resultCode { switch resultCode {
case worker.Succeed: case worker.Succeed:
fmt.Println("OK") //
case worker.MetricDuplicate: case worker.MetricDuplicate:
//fmt.Println("ErrDuplicate")
return proto.ErrDuplicate return proto.ErrDuplicate
default: default:
//fmt.Println("ErrWrongResultCodeBug")
qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug) qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
} }
return 0 return 0
@@ -267,10 +250,8 @@ func (s *Database) GetMetric(conn io.Writer, req proto.GetMetricReq) error {
if err != nil { if err != nil {
return err return err
} }
case worker.NoMetric: case worker.NoMetric:
reply(conn, proto.ErrNoMetric) reply(conn, proto.ErrNoMetric)
default: default:
qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug) qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
} }
@@ -469,12 +450,11 @@ func (s *Database) ListCumulativeMeasures(conn net.Conn, req proto.ListCumulativ
} }
func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriodsReq) error { func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriodsReq) error {
since, until := timeBoundsOfAggregation(req.Since, req.Until, req.GroupBy, req.FirstHourOfDay) since, until := timeutil.TimeBoundsOfAggregation(req.Since, req.Until, req.GroupBy, req.FirstHourOfDay)
if since.After(until) { if since.After(until) {
reply(conn, proto.ErrInvalidRange) reply(conn, proto.ErrInvalidRange)
return nil return nil
} }
responseWriter, err := transform.NewInstantPeriodsWriter(transform.InstantPeriodsWriterOptions{ responseWriter, err := transform.NewInstantPeriodsWriter(transform.InstantPeriodsWriterOptions{
Dst: conn, Dst: conn,
GroupBy: req.GroupBy, GroupBy: req.GroupBy,
@@ -485,7 +465,6 @@ func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriod
reply(conn, proto.ErrUnexpected) reply(conn, proto.ErrUnexpected)
return nil return nil
} }
return s.rangeScan(rangeScanReq{ return s.rangeScan(rangeScanReq{
MetricID: req.MetricID, MetricID: req.MetricID,
MetricType: qb.Instant, MetricType: qb.Instant,
@@ -497,12 +476,11 @@ func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriod
} }
func (s *Database) ListCumulativePeriods(conn net.Conn, req proto.ListCumulativePeriodsReq) error { func (s *Database) ListCumulativePeriods(conn net.Conn, req proto.ListCumulativePeriodsReq) error {
since, until := timeBoundsOfAggregation(req.Since, req.Until, req.GroupBy, req.FirstHourOfDay) since, until := timeutil.TimeBoundsOfAggregation(req.Since, req.Until, req.GroupBy, req.FirstHourOfDay)
if since.After(until) { if since.After(until) {
reply(conn, proto.ErrInvalidRange) reply(conn, proto.ErrInvalidRange)
return nil return nil
} }
responseWriter, err := transform.NewCumulativePeriodsWriter(transform.CumulativePeriodsWriterOptions{ responseWriter, err := transform.NewCumulativePeriodsWriter(transform.CumulativePeriodsWriterOptions{
Dst: conn, Dst: conn,
GroupBy: req.GroupBy, GroupBy: req.GroupBy,
@@ -512,7 +490,6 @@ func (s *Database) ListCumulativePeriods(conn net.Conn, req proto.ListCumulative
reply(conn, proto.ErrUnexpected) reply(conn, proto.ErrUnexpected)
return nil return nil
} }
return s.rangeScan(rangeScanReq{ return s.rangeScan(rangeScanReq{
MetricID: req.MetricID, MetricID: req.MetricID,
MetricType: qb.Cumulative, MetricType: qb.Cumulative,
@@ -529,7 +506,7 @@ type rangeScanReq struct {
Since uint32 Since uint32
Until uint32 Until uint32
Conn io.Writer Conn io.Writer
ResponseWriter atree.PeriodsWriter ResponseWriter qb.PeriodsWriter
} }
func (s *Database) rangeScan(req rangeScanReq) error { func (s *Database) rangeScan(req rangeScanReq) error {
@@ -549,44 +526,40 @@ func (s *Database) rangeScan(req rangeScanReq) error {
switch result.ResultCode { switch result.ResultCode {
case worker.QueryDone: case worker.QueryDone:
req.ResponseWriter.Close() req.ResponseWriter.Close()
case worker.UntilFound: case worker.UntilFound:
// err := s.atree.ContinueRangeScan(atree.ContinueRangeScanReq{ err := s.ContinueRangeScan(ContinueRangeScanReq{
// FracDigits: result.FracDigits, MetricType: req.MetricType,
// ResponseWriter: req.ResponseWriter, FracDigits: result.FracDigits,
// LastPageNo: result.LastPageNo, ResponseWriter: req.ResponseWriter,
// Since: req.Since, LastPageNo: result.LastPageNo,
// }) Since: req.Since,
//s.metricRUnlock(req.MetricID) })
//s.metricRUnlock(req.MetricID) fix release unlock
// if err != nil { if err != nil {
// reply(req.Conn, proto.ErrUnexpected) reply(req.Conn, proto.ErrUnexpected)
// } else { } else {
// req.ResponseWriter.Close() req.ResponseWriter.Close()
// } }
case worker.UntilNotFound: case worker.UntilNotFound:
// err := s.atree.RangeScan(atree.RangeScanReq{ err := s.RangeScan(RangeScanReq{
// FracDigits: result.FracDigits, MetricType: req.MetricType,
// ResponseWriter: req.ResponseWriter, FracDigits: result.FracDigits,
// RootPageNo: result.RootPageNo, ResponseWriter: req.ResponseWriter,
// Since: req.Since, Since: req.Since,
// Until: req.Until, Until: req.Until,
// }) LastPageNo: result.LastPageNo,
//s.metricRUnlock(req.MetricID) IsDataPage: result.IsDataPage,
})
// if err != nil { //s.metricRUnlock(req.MetricID) // fix release
// reply(req.Conn, proto.ErrUnexpected) if err != nil {
// } else { reply(req.Conn, proto.ErrUnexpected)
// req.ResponseWriter.Close() } else {
// } req.ResponseWriter.Close()
}
case worker.NoMetric: case worker.NoMetric:
reply(req.Conn, proto.ErrNoMetric) reply(req.Conn, proto.ErrNoMetric)
case worker.WrongMetricType: case worker.WrongMetricType:
reply(req.Conn, proto.ErrWrongMetricType) reply(req.Conn, proto.ErrWrongMetricType)
default: default:
qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug) qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
} }
@@ -597,7 +570,7 @@ type fullScanReq struct {
MetricID uint32 MetricID uint32
MetricType qb.MetricType MetricType qb.MetricType
Conn io.Writer Conn io.Writer
ResponseWriter atree.PeriodsWriter ResponseWriter qb.PeriodsWriter
} }
func (s *Database) fullScan(req fullScanReq) error { func (s *Database) fullScan(req fullScanReq) error {
@@ -614,11 +587,10 @@ func (s *Database) fullScan(req fullScanReq) error {
switch result.ResultCode { switch result.ResultCode {
case worker.QueryDone: case worker.QueryDone:
fmt.Printf("query done")
req.ResponseWriter.Close() req.ResponseWriter.Close()
case worker.UntilFound: case worker.UntilFound:
err := s.atree.ContinueFullScan(atree.ContinueFullScanReq{ err := s.ContinueFullScan(ContinueFullScanReq{
MetricType: req.MetricType,
FracDigits: result.FracDigits, FracDigits: result.FracDigits,
ResponseWriter: req.ResponseWriter, ResponseWriter: req.ResponseWriter,
LastPageNo: result.LastPageNo, LastPageNo: result.LastPageNo,
@@ -629,13 +601,10 @@ func (s *Database) fullScan(req fullScanReq) error {
} else { } else {
req.ResponseWriter.Close() req.ResponseWriter.Close()
} }
case worker.NoMetric: case worker.NoMetric:
reply(req.Conn, proto.ErrNoMetric) reply(req.Conn, proto.ErrNoMetric)
case worker.WrongMetricType: case worker.WrongMetricType:
reply(req.Conn, proto.ErrWrongMetricType) reply(req.Conn, proto.ErrWrongMetricType)
default: default:
qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug) qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
} }

360
database/database.go
View File

@@ -10,8 +10,8 @@ import (
"time" "time"
"gordenko.dev/dima/qb" "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/inbox" "gordenko.dev/dima/qb/inbox"
"gordenko.dev/dima/qb/pagecache"
"gordenko.dev/dima/qb/recovery" "gordenko.dev/dima/qb/recovery"
"gordenko.dev/dima/qb/storage" "gordenko.dev/dima/qb/storage"
"gordenko.dev/dima/qb/worker" "gordenko.dev/dima/qb/worker"
@@ -24,7 +24,8 @@ type Database struct {
workerInbox *inbox.Inbox workerInbox *inbox.Inbox
worker *worker.Worker worker *worker.Worker
storage *storage.Writer storage *storage.Writer
atree *atree.Atree indexCache *pagecache.PageCache
dataCache *pagecache.PageCache
tcpPort int tcpPort int
logfile *os.File logfile *os.File
logger *log.Logger logger *log.Logger
@@ -100,6 +101,24 @@ func New(opt Options) (_ *Database, err error) {
return nil, err return nil, err
} }
s.indexCache, err = pagecache.New(pagecache.Options{
File: indexFile,
PageSize: storage.IndexPageSize,
VerifyPageCRC: storage.VerifyIndexPageCRC32,
})
if err != nil {
return nil, fmt.Errorf("index pagecache.New: %s", err)
}
s.dataCache, err = pagecache.New(pagecache.Options{
File: dataFile,
PageSize: storage.DataPageSize,
VerifyPageCRC: storage.VerifyDataPageCRC32,
})
if err != nil {
return nil, fmt.Errorf("data pagecache.New: %s", err)
}
s.storage, err = storage.NewWriter(storage.WriterOptions{ s.storage, err = storage.NewWriter(storage.WriterOptions{
Inbox: storageInbox, Inbox: storageInbox,
WorkerInbox: s.workerInbox, WorkerInbox: s.workerInbox,
@@ -126,14 +145,6 @@ func New(opt Options) (_ *Database, err error) {
ExitCh: opt.ExitCh, ExitCh: opt.ExitCh,
WaitGroup: opt.WaitGroup, WaitGroup: opt.WaitGroup,
}) })
s.atree, err = atree.New(atree.Options{
DataFile: dataFile,
IndexFile: indexFile,
})
if err != nil {
return nil, fmt.Errorf("atree.New: %s", err)
}
return s, nil return s, nil
} }
@@ -146,14 +157,11 @@ func (s *Database) ListenAndServe() (err error) {
//s.waitGroup.Add(1) //s.waitGroup.Add(1)
go s.storage.Run() go s.storage.Run()
// s.atree.Run()
s.waitGroup.Add(1) s.waitGroup.Add(1)
go s.worker.Run() go s.worker.Run()
s.logger.Println("database started") s.logger.Println("database started")
for { for {
// Listen for an incoming connection.
conn, err := listener.Accept() conn, err := listener.Accept()
if err != nil { if err != nil {
s.logger.Printf("listener.Accept: %s\n", err) s.logger.Printf("listener.Accept: %s\n", err)
@@ -164,53 +172,291 @@ func (s *Database) ListenAndServe() (err error) {
} }
} }
// type ContinueFullScanReq struct {
MetricType qb.MetricType
FracDigits byte
ResponseWriter qb.AtreeMeasureConsumer
LastPageNo uint32
}
// зробити object? func (s *Database) ContinueFullScan(req ContinueFullScanReq) error {
buf, err := s.dataCache.FetchPage(req.LastPageNo)
if err != nil {
return fmt.Errorf("dataCache.FetchPage(%d): %s", req.LastPageNo, err)
}
treeCursor, err := storage.NewBackwardCursor(storage.BackwardCursorOptions{
MetricType: req.MetricType,
FracDigits: req.FracDigits,
PageNo: req.LastPageNo,
PageData: buf,
FetchDataPage: s.dataCache.FetchPage,
ReleasePage: s.dataCache.ReleasePage,
})
if err != nil {
return err
}
defer treeCursor.Close()
for {
timestamp, value, done, err := treeCursor.Prev()
if err != nil {
return err
}
if done {
return nil
}
req.ResponseWriter.Feed(timestamp, value)
}
}
// func (s *Database) replayChanges(snapshotNumber int) (err error) { type ContinueRangeScanReq struct {
// snapshot, err := readSnapshot(JoinSnapshotFileName(s.dir, snapshotNumber)) MetricType qb.MetricType
// if err != nil { FracDigits byte
// return ResponseWriter qb.AtreeMeasureConsumer
// } LastPageNo uint32
Since uint32
}
// return nil func (s *Database) ContinueRangeScan(req ContinueRangeScanReq) error {
buf, err := s.dataCache.FetchPage(req.LastPageNo)
if err != nil {
return fmt.Errorf("dataCache.FetchPage(%d): %s", req.LastPageNo, err)
}
treeCursor, err := storage.NewBackwardCursor(storage.BackwardCursorOptions{
MetricType: req.MetricType,
FracDigits: req.FracDigits,
PageNo: req.LastPageNo,
PageData: buf,
FetchDataPage: s.dataCache.FetchPage,
ReleasePage: s.dataCache.ReleasePage,
})
if err != nil {
return err
}
defer treeCursor.Close()
for {
timestamp, value, done, err := treeCursor.Prev()
if err != nil {
return err
}
if done {
return nil
}
req.ResponseWriter.Feed(timestamp, value)
if timestamp < req.Since {
return nil
}
}
}
type RangeScanReq struct {
MetricType qb.MetricType
FracDigits byte
ResponseWriter qb.AtreeMeasureConsumer
Since uint32
Until uint32
LastPageNo uint32
IsDataPage bool
}
func (s *Database) RangeScan(req RangeScanReq) error {
var (
pageNo uint32
buf []byte
err error
)
if req.IsDataPage {
pageNo = req.LastPageNo
buf, err = s.dataCache.FetchPage(pageNo)
if err != nil {
return fmt.Errorf("dataCache.FetchPage(%d): %s", pageNo, err)
}
} else {
pageNo, buf, err = s.findDataPage(req.LastPageNo, req.Until)
if err != nil {
return err
}
}
cursor, err := storage.NewBackwardCursor(storage.BackwardCursorOptions{
MetricType: req.MetricType,
FracDigits: req.FracDigits,
PageNo: pageNo,
PageData: buf,
FetchDataPage: s.dataCache.FetchPage,
ReleasePage: s.dataCache.ReleasePage,
})
if err != nil {
return err
}
defer cursor.Close()
for {
timestamp, value, done, err := cursor.Prev()
if err != nil {
return err
}
if done {
return nil
}
if timestamp <= req.Until {
req.ResponseWriter.Feed(timestamp, value)
if timestamp < req.Since {
// - записи, удовлетворяющие временным рамкам, закончились.
return nil
}
}
}
}
// (records payload only, isZeroLevel, timestamp)
func (s *Database) findDataPage(foundPageNo uint32, timestamp uint32) (uint32, []byte, error) {
for {
buf, err := s.indexCache.FetchPage(foundPageNo)
if err != nil {
return 0, nil, fmt.Errorf("fetchIndexPage(%d): %s", foundPageNo, err)
}
toReleaseIndexPageNo := foundPageNo
foundPageNo = storage.FindPageOnIndexPage(buf, timestamp)
s.indexCache.ReleasePage(toReleaseIndexPageNo)
if storage.IsZeroLevelPage(buf) {
buf, err := s.dataCache.FetchPage(foundPageNo)
if err != nil {
return 0, nil, fmt.Errorf("fetchDataPage(%d): %s", foundPageNo, err)
}
return foundPageNo, buf, nil
}
}
}
// type PathLeg struct {
// PageNo uint32
// Data []byte
// } // }
//func (s *Database) verifySnapshot(fileName string) (_ bool, err error) { // type PathToDataPage struct {
// file, err := os.Open(fileName) // Legs []PathLeg
// if err != nil { // LastPageNo uint32
// return // }
// }
// defer file.Close() // func (s *Database) FindPathToLastPage(rootPageNo uint32) (_ PathToDataPage, err error) {
// // var (
// stat, err := file.Stat() // // pageNo = rootPageNo
// if err != nil { // // legs []PathLeg
// return // // )
// }
// // for {
// if stat.Size() <= 4 { // // var buf []byte
// return false, nil // // buf, err = s.fetchIndexPage(pageNo)
// } // // if err != nil {
// // err = fmt.Errorf("FetchIndexPage(%d): %s", pageNo, err)
// var ( // // return
// payloadSize = stat.Size() - 4 // // }
// hash = crc32.NewIEEE()
// ) // // legs = append(legs, PathLeg{
// // PageNo: pageNo,
// _, err = io.CopyN(hash, file, payloadSize) // // Data: buf,
// if err != nil { // // // childIdx не нужен
// return // // })
// }
// calculatedCRC := hash.Sum32() // // foundPageNo := getLastPageNo(buf)
// storedCRC, err := bin.ReadUint32(file) // // // fix
// if err != nil { // // if buf[isLastLevelIdx] == 1 {
// return // // return PathToDataPage{
// } // // Legs: legs,
// if storedCRC != calculatedCRC { // // LastPageNo: foundPageNo,
// return false, fmt.Errorf("strored CRC %d not equal calculated CRC %d", // // }, nil
// storedCRC, calculatedCRC) // // }
// } // // // вглубь
// return true, nil // // pageNo = foundPageNo
// // }
// return
// }
// DELETE
// func (s *Atree) DeletePages(pageNumbers []uint32) {
// s.mutex.Lock()
// for _, pageNo := range pageNumbers {
// delete(s.pages, pageNo)
// }
// s.mutex.Unlock()
// }
// type Level struct {
// PageNo uint32
// PageData []byte
// Idx int
// ChildQty int
// }
// func (s *Atree) GetAllPages(rootPageNo uint32) (_ []uint32, err error) {
// // var (
// // pageNumbers []uint32
// // levels []*Level
// // )
// // buf, err := s.fetchIndexPage(rootPageNo)
// // if err != nil {
// // return nil, fmt.Errorf("fetchIndexPage(%d): %s", rootPageNo, err)
// // }
// // pageNumbers = append(pageNumbers, rootPageNo)
// // // if buf[isDataPageNumbersIdx] == 1 {
// // // pageNumbers := listPageNumbers(buf)
// // // dataPages = append(dataPages, pageNumbers...)
// // // s.releasePage(rootPageNo)
// // // return PageLists{
// // // DataPages: dataPages,
// // // IndexPages: indexPages,
// // // }, nil
// // // }
// // childQty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
// // levels = append(levels, &Level{
// // PageNo: rootPageNo,
// // PageData: buf,
// // Idx: 0,
// // ChildQty: int(childQty),
// // })
// // for {
// // if len(levels) == 0 {
// // return pageNumbers, nil
// // }
// // lastIdx := len(levels) - 1
// // level := levels[lastIdx]
// // if level.Idx < level.ChildQty {
// // pageNo := getPageNo(level.PageData, level.Idx)
// // level.Idx++
// // var buf []byte
// // buf, err = s.fetchPage(pageNo)
// // if err != nil {
// // return nil, fmt.Errorf("fetchPage(%d): %s", pageNo, err)
// // }
// // pageNumbers = append(pageNumbers, pageNo)
// // if buf[pageTypeIdx] == PageTypeData {
// // //pageNumbers := listPageNumbers(buf)
// // //dataPages = append(dataPages, pageNumbers...)
// // s.releasePage(pageNo)
// // } else {
// // childQty, _ = bin.GetUint16(buf[indexRecordsQtyIdx:])
// // levels = append(levels, &Level{
// // PageNo: pageNo,
// // PageData: buf,
// // Idx: 0,
// // ChildQty: int(childQty),
// // })
// // }
// // } else {
// // s.releasePage(level.PageNo)
// // levels = levels[:lastIdx]
// // }
// // }
// return
// } // }

24
database/database_test.go
View File

@@ -1,24 +0,0 @@
package database
// func TestComposeHeadIndexPage(t *testing.T) {
// var (
// levelIdx = 0
// level = storage.IndexLevelTail{
// Buffer: []byte{
// 1, 2, 3,
// },
// RecordsCount: 2,
// }
// head = &storage.IndexPageTail{
// PageNo: 100,
// CRC32: 12345,
// Records: []byte{},
// }
// )
// page := composeHeadIndexPage(levelIdx, level, head)
// if page.PageNo != head.PageNo {
// t.Fatalf("PageNo: got %d are not equal expected %v",
// page.PageNo, head.PageNo)
// }
// // fix compare pages
// }

52
database/helpers.go
View File

@@ -1,52 +0,0 @@
package database
import (
"errors"
"io/fs"
"os"
"time"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/proto"
)
func timeBoundsOfAggregation(since, until proto.TimeBound, groupBy qb.GroupBy, firstHourOfDay int) (s time.Time, u time.Time) {
switch groupBy {
case qb.GroupByHour, qb.GroupByDay:
s = time.Date(since.Year, since.Month, since.Day, 0, 0, 0, 0, time.Local)
u = time.Date(until.Year, until.Month, until.Day, 0, 0, 0, 0, time.Local)
case qb.GroupByMonth:
s = time.Date(since.Year, since.Month, 1, 0, 0, 0, 0, time.Local)
u = time.Date(until.Year, until.Month, 1, 0, 0, 0, 0, time.Local)
}
if firstHourOfDay > 0 {
duration := time.Duration(firstHourOfDay) * time.Hour
s = s.Add(duration)
u = u.Add(duration)
}
u = u.Add(-1 * time.Second)
return
}
func isFileExist(fileName string) (bool, error) {
_, err := os.Stat(fileName)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return false, nil
} else {
return false, err
}
} else {
return true, nil
}
}
func correctToFHD(since, until uint32, firstHourOfDay int) (uint32, uint32) {
duration := time.Duration(firstHourOfDay) * time.Hour
since = uint32(time.Unix(int64(since), 0).Add(duration).Unix())
until = uint32(time.Unix(int64(until), 0).Add(duration).Unix())
return since, until
}

25
enc/time_delta.go
View File

@@ -310,10 +310,33 @@ func (s *TimeDeltaCompressor) FirstTimestamp() uint32 {
return timestamp return timestamp
} }
func (s *TimeDeltaCompressor) LastTimestamp() uint32 { func (s *TimeDeltaCompressor) Until() uint32 {
return s.lastUnixtime return s.lastUnixtime
} }
func (s *TimeDeltaCompressor) CommitedSince() uint32 {
if s.state == nil {
if s.pos < len(s.buf) {
timestamp, _ := bin.GetUint32(s.buf[len(s.buf)-4:])
return timestamp
}
} else {
payload := s.state.Payload
if len(payload) > 0 {
timestamp, _ := bin.GetUint32(payload[len(payload)-4:])
return timestamp
}
}
return 0
}
func (s *TimeDeltaCompressor) CommitedUntil() uint32 {
if s.state == nil {
return s.lastUnixtime
}
return s.state.LastUnixtime
}
// DECOMPRESSOR // DECOMPRESSOR
type TimeDeltaDecompressor struct { type TimeDeltaDecompressor struct {

4
inbox/inbox.go
View File

@@ -22,11 +22,9 @@ func (s *Inbox) Ready() chan struct{} {
} }
func (s *Inbox) Push(x any) { func (s *Inbox) Push(x any) {
//fmt.Printf("inbox.Push: %#v\n", x)
s.mutex.Lock() s.mutex.Lock()
s.items = append(s.items, x) s.items = append(s.items, x)
s.mutex.Unlock() s.mutex.Unlock()
//fmt.Printf("inbox.Pushed\n")
select { select {
case s.signalCh <- struct{}{}: case s.signalCh <- struct{}{}:
default: default:
@@ -34,11 +32,9 @@ func (s *Inbox) Push(x any) {
} }
func (s *Inbox) Drain() []any { func (s *Inbox) Drain() []any {
//fmt.Printf("inbox.Drain\n")
s.mutex.Lock() s.mutex.Lock()
items := s.items items := s.items
s.items = nil s.items = nil
s.mutex.Unlock() s.mutex.Unlock()
//fmt.Printf("inbox.Drained: %#v\n", items)
return items return items
} }

112
atree/io.go → pagecache/pagecache.go
View File

@@ -1,12 +1,12 @@
package atree package pagecache
import ( import (
"errors"
"fmt" "fmt"
"os"
"sync"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb" "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/storage"
"gordenko.dev/dima/qb/util"
) )
type readResult struct { type readResult struct {
@@ -16,35 +16,52 @@ type readResult struct {
// INDEX PAGES // INDEX PAGES
func (s *Atree) DeletePages(pageNumbers []uint32) { type _page struct {
s.mutex.Lock() PageNo uint32
for _, pageNo := range pageNumbers { Buf []byte
delete(s.pages, pageNo) ReferenceCount int
}
type PageCache struct {
mutex sync.Mutex
pageSize int
verifyPageCRC func([]byte) error
file *os.File
pages map[uint32]*_page
pageWaits map[uint32][]chan readResult
pagesToRead []uint32
readSignalCh chan struct{}
}
type Options struct {
File *os.File
PageSize int
VerifyPageCRC func([]byte) error
}
func New(opt Options) (*PageCache, error) {
if opt.File == nil {
return nil, errors.New("File option is required")
} }
s.mutex.Unlock() if opt.PageSize <= 0 {
} return nil, errors.New("PageSize option is required")
func (s *Atree) fetchIndexPage(pageNo uint32) ([]byte, error) {
// buf, err := s.fetchPage(pageNo)
// if err != nil {
// return nil, err
// }
// if buf[pageTypeIdx] != PageTypeIndex {
// return nil, fmt.Errorf("wrong pageType %d instead of %d", buf[pageTypeIdx], PageTypeIndex)
// }
// return buf, nil
return nil, nil
}
func (s *Atree) fetchDataPage(pageNo uint32) ([]byte, error) {
buf, err := s.fetchPage(pageNo)
if err != nil {
return nil, err
} }
return buf, nil if opt.VerifyPageCRC == nil {
return nil, errors.New("VerifyPageCRC option is required")
}
s := &PageCache{
file: opt.File,
pageSize: opt.PageSize,
verifyPageCRC: opt.VerifyPageCRC,
pages: make(map[uint32]*_page),
pageWaits: make(map[uint32][]chan readResult),
readSignalCh: make(chan struct{}, 1),
}
go s.pageReader()
return s, nil
} }
func (s *Atree) fetchPage(pageNo uint32) ([]byte, error) { func (s *PageCache) FetchPage(pageNo uint32) ([]byte, error) {
s.mutex.Lock() s.mutex.Lock()
p, ok := s.pages[pageNo] p, ok := s.pages[pageNo]
if ok { if ok {
@@ -69,12 +86,12 @@ func (s *Atree) fetchPage(pageNo uint32) ([]byte, error) {
result := <-resultCh result := <-resultCh
if result.Err == nil { if result.Err == nil {
result.Err = s.verifyCRC(result.Data, storage.DataPageSize) result.Err = s.verifyPageCRC(result.Data)
} }
return result.Data, result.Err return result.Data, result.Err
} }
func (s *Atree) releasePage(pageNo uint32) { func (s *PageCache) ReleasePage(pageNo uint32) {
s.mutex.Lock() s.mutex.Lock()
defer s.mutex.Unlock() defer s.mutex.Unlock()
@@ -93,11 +110,7 @@ func (s *Atree) releasePage(pageNo uint32) {
} }
} }
// DATA PAGES func (s *PageCache) pageReader() {
// READ
func (s *Atree) pageReader() {
for { for {
select { select {
case <-s.readSignalCh: case <-s.readSignalCh:
@@ -106,7 +119,7 @@ func (s *Atree) pageReader() {
} }
} }
func (s *Atree) readPages() { func (s *PageCache) readPages() {
s.mutex.Lock() s.mutex.Lock()
if len(s.pagesToRead) == 0 { if len(s.pagesToRead) == 0 {
s.mutex.Unlock() s.mutex.Unlock()
@@ -117,17 +130,15 @@ func (s *Atree) readPages() {
s.mutex.Unlock() s.mutex.Unlock()
for _, pageNo := range pagesToRead { for _, pageNo := range pagesToRead {
buf := make([]byte, storage.DataPageSize) buf := make([]byte, s.pageSize)
off := int(pageNo-1) * storage.DataPageSize off := int(pageNo-1) * s.pageSize
n, err := s.file.ReadAt(buf, int64(off)) n, err := s.file.ReadAt(buf, int64(off))
if n != storage.DataPageSize { if n != s.pageSize {
err = fmt.Errorf("read %d instead of %d", n, storage.DataPageSize) err = fmt.Errorf("read %d instead of %d", n, s.pageSize)
} }
s.mutex.Lock() s.mutex.Lock()
resultChannels := s.pageWaits[pageNo] resultChannels := s.pageWaits[pageNo]
delete(s.pageWaits, pageNo) delete(s.pageWaits, pageNo)
if err != nil { if err != nil {
s.mutex.Unlock() s.mutex.Unlock()
for _, resultCh := range resultChannels { for _, resultCh := range resultChannels {
@@ -150,18 +161,3 @@ func (s *Atree) readPages() {
} }
} }
} }
// WRITE
func (s *Atree) verifyCRC(data []byte, pageSize int) error {
var (
pos = pageSize - 4
calculatedCRC = util.CalculateCRC32(data[:pos])
storedCRC, _ = bin.GetUint32(data[pos:])
)
if calculatedCRC != storedCRC {
return fmt.Errorf("calculatedCRC %d not equal storedCRC %d",
calculatedCRC, storedCRC)
}
return nil
}

24
qb.go
View File

@@ -24,6 +24,20 @@ const (
AggregateAvg byte = 4 AggregateAvg byte = 4
) )
type AtreeMeasureConsumer interface {
Feed(uint32, float64)
}
type PeriodsWriter interface {
Feed(uint32, float64)
FeedNoSend(uint32, float64)
Close() error
}
type WorkerMeasureConsumer interface {
FeedNoSend(uint32, float64)
}
type TimestampCompressor interface { type TimestampCompressor interface {
// (tmp, timestamp) // (tmp, timestamp)
Evaluate([]byte, uint32) TimeEvaluationReport Evaluate([]byte, uint32) TimeEvaluationReport
@@ -37,12 +51,11 @@ type TimestampCompressor interface {
// (offset) => payload // (offset) => payload
Tail(int) []byte Tail(int) []byte
CreateDecompressor() TimestampDecompressor CreateDecompressor() TimestampDecompressor
//Payload() []byte // для снапшота
// Offset() int
ReplaceBuffer([]byte) ReplaceBuffer([]byte)
WriteCommitedTo(io.Writer) error WriteCommitedTo(io.Writer) error
FirstTimestamp() uint32 Until() uint32
LastTimestamp() uint32 CommitedSince() uint32
CommitedUntil() uint32
ReplaceSinceWithUntil() uint32 ReplaceSinceWithUntil() uint32
} }
@@ -67,7 +80,6 @@ type ValueCompressor interface {
Append(int, []byte, float64, uint64) Append(int, []byte, float64, uint64)
Size() int Size() int
CommitedSize() int CommitedSize() int
//Chunks() [][]byte
//DeleteLast() //DeleteLast()
CaptureState() CaptureState()
ForgetCapturedState() ForgetCapturedState()
@@ -75,8 +87,6 @@ type ValueCompressor interface {
Tail(int) []byte Tail(int) []byte
// fracDigits // fracDigits
CreateDecompressor(MetricType, byte) ValueDecompressor CreateDecompressor(MetricType, byte) ValueDecompressor
//Payload() []byte // для снапшота
//Offset() int
ReplaceBuffer([]byte) ReplaceBuffer([]byte)
WriteCommitedTo(io.Writer) error WriteCommitedTo(io.Writer) error
LastValue() float64 LastValue() float64

0
recovery/test.wal_0
View File

0
recovery/test.wal_123
View File

136
storage/cursor.go Normal file
View File

@@ -0,0 +1,136 @@
package storage
import (
"errors"
"fmt"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
)
type BackwardCursor struct {
metricType qb.MetricType
fracDigits byte
fetchDataPage func(uint32) ([]byte, error)
releasePage func(uint32)
pageNo uint32
pageData []byte
timestampDecompressor qb.TimestampDecompressor
valueDecompressor qb.ValueDecompressor
}
type BackwardCursorOptions struct {
MetricType qb.MetricType
FracDigits byte
PageNo uint32
PageData []byte
FetchDataPage func(uint32) ([]byte, error)
ReleasePage func(uint32)
}
func NewBackwardCursor(opt BackwardCursorOptions) (*BackwardCursor, error) {
switch opt.MetricType {
case qb.Instant, qb.Cumulative:
// ok
default:
return nil, fmt.Errorf("MetricType option has wrong value: %d", opt.MetricType)
}
if opt.FracDigits > qb.MaxFracDigits {
return nil, errors.New("FracDigits option is required")
}
if opt.FetchDataPage == nil {
return nil, errors.New("FetchDataPage option is required")
}
if opt.ReleasePage == nil {
return nil, errors.New("ReleasePage option is required")
}
if opt.PageNo == 0 {
return nil, errors.New("PageNo option is required")
}
if len(opt.PageData) == 0 {
return nil, errors.New("PageData option is required")
}
s := &BackwardCursor{
metricType: opt.MetricType,
fracDigits: opt.FracDigits,
fetchDataPage: opt.FetchDataPage,
releasePage: opt.ReleasePage,
pageNo: opt.PageNo,
pageData: opt.PageData,
}
var err error
s.timestampDecompressor, err = CreateTimestampDecompressor(s.pageData)
if err != nil {
return nil, fmt.Errorf("CreateTimestampDecompressor: %s", err)
}
s.valueDecompressor, err = CreateValueDecompressor(s.pageData, s.metricType, s.fracDigits)
if err != nil {
return nil, fmt.Errorf("CreateValueDecompressor: %s", err)
}
return s, nil
}
// timestamp, value, done, error
func (s *BackwardCursor) Prev() (uint32, float64, bool, error) {
var (
timestamp uint32
value float64
done bool
err error
)
timestamp, done = s.timestampDecompressor.NextValue()
if !done {
value, done = s.valueDecompressor.NextValue()
if done {
return 0, 0, false,
fmt.Errorf("corrupted data page %d: has timestamp, no value",
s.pageNo)
}
return timestamp, value, false, nil
}
prevPageNo, _ := bin.GetUint32(s.pageData[prevPageIdx:])
if prevPageNo == 0 {
return 0, 0, true, nil
}
s.releasePage(s.pageNo)
s.pageNo = prevPageNo
s.pageData, err = s.fetchDataPage(s.pageNo)
if err != nil {
return 0, 0, false,
fmt.Errorf("fetchDataPage(%d): %s", s.pageNo, err)
}
s.timestampDecompressor, err = CreateTimestampDecompressor(s.pageData)
if err != nil {
return 0, 0, false,
fmt.Errorf("CreateTimestampDecompressor: %s", err)
}
s.valueDecompressor, err = CreateValueDecompressor(s.pageData, s.metricType, s.fracDigits)
if err != nil {
return 0, 0, false,
fmt.Errorf("CreateValueDecompressor: %s", err)
}
//
timestamp, done = s.timestampDecompressor.NextValue()
if done {
return 0, 0, false,
fmt.Errorf("corrupted data page %d: no timestamps", s.pageNo)
}
value, done = s.valueDecompressor.NextValue()
if done {
return 0, 0, false,
fmt.Errorf("corrupted data page %d: no values", s.pageNo)
}
return timestamp, value, false, nil
}
func (s *BackwardCursor) Close() {
s.releasePage(s.pageNo)
}
// HELPER
//func (s *BackwardCursor) makeDecompressors() error {
//}

107
storage/misc.go
View File

@@ -6,87 +6,50 @@ import (
bin "gordenko.dev/dima/bin/little" bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb" "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/enc" "gordenko.dev/dima/qb/enc"
"gordenko.dev/dima/qb/util"
) )
// func (s *BackwardCursor) makeDecompressors() error { func CreateTimestampDecompressor(page []byte) (qb.TimestampDecompressor, error) {
// timestampsSize, _ := bin.GetUint16(s.pageData[timestampsSizeIdx:])
// valuesSize, _ := bin.GetUint16(s.pageData[valuesSizeIdx:])
// payloadSize := timestampsSize + valuesSize
// if payloadSize > dataFooterIdx {
// return fmt.Errorf("corrupted data page %d: timestamps + values size %d gt payload size",
// s.pageNo, payloadSize)
// }
// s.timestampDecompressor = enc.NewTimeDeltaDecompressor(
// s.pageData[:timestampsSize],
// )
// vbuf := s.pageData[timestampsSize : timestampsSize+valuesSize]
// switch s.metricType {
// case qb.Instant:
// s.valueDecompressor = enc.NewInstantDeltaDecompressor(
// vbuf, s.fracDigits)
// case qb.Cumulative:
// s.valueDecompressor = enc.NewCumulativeDeltaDecompressor(
// vbuf, s.fracDigits)
// default:
// return fmt.Errorf("bug: wrong metricType %d", s.metricType)
// }
// return nil
// }
func makeDecompressors(pageData []byte, metricType qb.MetricType, fracDigits byte) (
qb.TimestampDecompressor, qb.ValueDecompressor, error,
) {
valuesSize, _ := bin.GetUint16(pageData[valuesSizeIdx:])
payloadSize := timestampsSize + valuesSize
if payloadSize > dataFooterIdx {
return nil, nil, fmt.Errorf("corrupted: timestamps + values size %d > payload size",
payloadSize)
}
timestampDecompressor := enc.NewTimeDeltaDecompressor(
pageData[:timestampsSize],
)
vbuf := pageData[timestampsSize : timestampsSize+valuesSize]
var valueDecompressor qb.ValueDecompressor
switch metricType {
case qb.Instant:
valueDecompressor = enc.NewInstantDeltaDecompressor(
vbuf, fracDigits)
case qb.Cumulative:
valueDecompressor = enc.NewCumulativeDeltaDecompressor(
vbuf, fracDigits)
default:
return nil, nil, fmt.Errorf("bug: wrong metricType %d", metricType)
}
return timestampDecompressor, valueDecompressor, nil
return nil, nil, nil
}
func CreateTimeDeltaDecompressor(page []byte) qb.TimestampDecompressor {
size, _ := bin.GetUint16(page[timestampsSizeIdx:]) size, _ := bin.GetUint16(page[timestampsSizeIdx:])
if size > DataPageFooterSize {
return nil, fmt.Errorf("bug: invalid timestamps size %d", size)
}
pos := DataPagePayloadSize - int(size) pos := DataPagePayloadSize - int(size)
d := enc.NewTimeDeltaDecompressor() d := enc.NewTimeDeltaDecompressor()
d.RestoreFromEnd(page[pos:DataPagePayloadSize]) d.RestoreFromEnd(page[pos:DataPagePayloadSize])
return d return d, nil
} }
func CreateValueDeltaDecompressor(page []byte, metricType qb.MetricType, fracDigits byte) qb.ValueDecompressor { func CreateValueDecompressor(page []byte, metricType qb.MetricType, fracDigits byte) (qb.ValueDecompressor, error) {
size, _ := bin.GetUint16(page[valuesSizeIdx:]) size, _ := bin.GetUint16(page[valuesSizeIdx:])
if size > DataPageFooterSize {
return nil, fmt.Errorf("bug: invalid timestamps size %d", size)
}
d := enc.NewValueDeltaDecompressor(metricType, fracDigits) d := enc.NewValueDeltaDecompressor(metricType, fracDigits)
d.RestoreFromEnd(page[:size]) d.RestoreFromEnd(page[:size])
return d return d, nil
}
func VerifyDataPageCRC32(data []byte) error {
var (
calculatedCRC = util.CalculateCRC32(data[:dataCRC32Idx])
writtenCRC, _ = bin.GetUint32(data[dataCRC32Idx:])
)
if calculatedCRC != writtenCRC {
return fmt.Errorf("calculated CRC32 %d are not equal written CRC32 %d",
calculatedCRC, writtenCRC)
}
return nil
}
func VerifyIndexPageCRC32(data []byte) error {
var (
calculatedCRC = util.CalculateCRC32(data[:indexCRC32Idx])
writtenCRC, _ = bin.GetUint32(data[indexCRC32Idx:])
)
if calculatedCRC != writtenCRC {
return fmt.Errorf("calculated CRC32 %d are not equal written CRC32 %d",
calculatedCRC, writtenCRC)
}
return nil
} }

94
atree/misc.go → storage/navigation.go
View File

@@ -1,4 +1,10 @@
package atree package storage
import bin "gordenko.dev/dima/bin/little"
const (
PageNoSize = 4
)
type KeyComparator interface { type KeyComparator interface {
CompareTo(int) int CompareTo(int) int
@@ -10,19 +16,18 @@ type ValueAtComparator struct {
} }
func (s ValueAtComparator) CompareTo(elemIdx int) int { func (s ValueAtComparator) CompareTo(elemIdx int) int {
// var ( var (
// pos = elemIdx * timestampSize pos = elemIdx * IndexRecordSize
// elem, _ = bin.GetUint32(s.buf[pos:]) elem, _ = bin.GetUint32(s.buf[pos:])
// ) )
// if s.timestamp < elem { if s.timestamp < elem {
// return -1 return -1
// } else if s.timestamp > elem { } else if s.timestamp > elem {
// return 1 return 1
// } else { } else {
// return 0 return 0
// } }
return 213131132
} }
func BinarySearch(qty int, keyComparator KeyComparator) (elemIdx int, isFound bool) { func BinarySearch(qty int, keyComparator KeyComparator) (elemIdx int, isFound bool) {
@@ -61,18 +66,63 @@ func BinarySearch(qty int, keyComparator KeyComparator) (elemIdx int, isFound bo
// return pageNo // return pageNo
// } // }
// func findPageNo(buf []byte, timestamp uint32) (pageNo uint32) { // func GetIndexRecordsSince(buf []byte) (pageNo uint32) {
// comparator := ValueAtComparator{ // pageNo, _ = bin.GetUint32(buf)
// buf: buf,
// timestamp: timestamp,
// }
// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
// elemIdx, _ := BinarySearch(int(qty), comparator)
// pos := indexFooterIdx - (elemIdx+1)*PageNoSize
// pageNo, _ = bin.GetUint32(buf[pos:])
// return // return
// } // }
func FindPageOnIndexLevelTail(level IndexLevelTail, timestamp uint32) (pageNo uint32) {
comparator := ValueAtComparator{
buf: level.Buffer,
timestamp: timestamp,
}
elemIdx, _ := BinarySearch(level.RecordsCount, comparator)
pos := elemIdx*IndexRecordSize + 4 // timestamp size
pageNo, _ = bin.GetUint32(level.Buffer[pos:])
return
}
// func FindPageOnRecords(records []byte, timestamp uint32) (pageNo uint32) {
// comparator := ValueAtComparator{
// buf: records,
// timestamp: timestamp,
// }
// count := len(records) / IndexRecordSize
// elemIdx, _ := BinarySearch(count, comparator)
// pageNo, _ = bin.GetUint32(records[elemIdx*IndexRecordSize:])
// return
// }
func FindPageOnIndexPage(page []byte, timestamp uint32) (pageNo uint32) {
comparator := ValueAtComparator{
buf: page,
timestamp: timestamp,
}
count, _ := bin.GetUint16(page[indexRecordsCountIdx:])
elemIdx, _ := BinarySearch(int(count), comparator)
pos := elemIdx*IndexRecordSize + 4 // timestamp size
pageNo, _ = bin.GetUint32(page[pos:])
return
}
func IsZeroLevelPage(buf []byte) bool {
return buf[isZeroLevelIdx] == 1
}
// можна перевірити хвости від 0 до ... Перевіряю since кожного хвоста.
// Якщо timestamp >=, шукаю pageNo бінарним пошуком. І сторінка мені однозначно підходить.
// Якщо timestamp <, піднімаюсь вище. Якщо рівнів більше немає - until вказано за межами Range показань.
func FindPageOnIndexLevelTails(levels []IndexLevelTail, timestamp uint32) (pageNo uint32, isDataPage bool) {
for i, level := range levels {
tailSince, _ := bin.GetUint32(level.Buffer)
if timestamp >= tailSince {
pageNo = FindPageOnIndexLevelTail(level, timestamp)
return pageNo, i == 0
}
}
return
}
// func findPageNoIdx(buf []byte, timestamp uint32) (idx int) { // func findPageNoIdx(buf []byte, timestamp uint32) (idx int) {
// comparator := ValueAtComparator{ // comparator := ValueAtComparator{
// buf: buf, // buf: buf,

4
storage/storage.go
View File

@@ -9,7 +9,7 @@ var (
// data page // data page
DataPageSize = 8192 DataPageSize = 8192
DataPagePayloadSize int = DataPageSize - DataPageFooterSize DataPagePayloadSize = DataPageSize - DataPageFooterSize
dataCRC32Idx = DataPageSize - 4 dataCRC32Idx = DataPageSize - 4
timestampsSizeIdx = DataPageSize - 6 timestampsSizeIdx = DataPageSize - 6
@@ -19,6 +19,7 @@ var (
// index page // index page
IndexPageSize = 1024 IndexPageSize = 1024
IndexPagePayloadSize = IndexPageSize - IndexPageFooterSize
//indexPageIncSize = IndexPageIncSize //indexPageIncSize = IndexPageIncSize
indexCRC32Idx = IndexPageSize - 4 indexCRC32Idx = IndexPageSize - 4
indexRecordsCountIdx = IndexPageSize - 6 indexRecordsCountIdx = IndexPageSize - 6
@@ -28,7 +29,6 @@ var (
// timestampSize = 4 // timestampSize = 4
// pairSize = timestampSize + PageNoSize // pairSize = timestampSize + PageNoSize
// indexFooterIdx = indexRecordsQtyIdx
// dataFooterIdx = timestampsSizeIdx // dataFooterIdx = timestampsSizeIdx
) )

71
storage/storage_test.go
View File

@@ -1081,3 +1081,74 @@ func TestReplayMetric(t *testing.T) {
fmt.Printf("%d: % x\n", metric.IndexLevelTails[0].RecordsCount, metric.IndexLevelTails[0].Buffer) fmt.Printf("%d: % x\n", metric.IndexLevelTails[0].RecordsCount, metric.IndexLevelTails[0].Buffer)
fmt.Printf("%d: % x\n", metric.IndexLevelTails[1].RecordsCount, metric.IndexLevelTails[1].Buffer) fmt.Printf("%d: % x\n", metric.IndexLevelTails[1].RecordsCount, metric.IndexLevelTails[1].Buffer)
} }
// func TestComposeHeadIndexPage(t *testing.T) {
// var (
// levelIdx = 0
// level = storage.IndexLevelTail{
// Buffer: []byte{
// 1, 2, 3,
// },
// RecordsCount: 2,
// }
// head = &storage.IndexPageTail{
// PageNo: 100,
// CRC32: 12345,
// Records: []byte{},
// }
// )
// page := composeHeadIndexPage(levelIdx, level, head)
// if page.PageNo != head.PageNo {
// t.Fatalf("PageNo: got %d are not equal expected %v",
// page.PageNo, head.PageNo)
// }
// // fix compare pages
// }
func TestFindPageOnIndexTails(t *testing.T) {
levels := []IndexLevelTail{
{
Buffer: []byte{
0x64, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, // 100 => 10
0x6e, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, // 110 => 11
},
RecordsCount: 2,
},
{
Buffer: []byte{
0x0a, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // 10 => 1
0x28, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, // 40 => 4
0x46, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, // 70 => 7
},
RecordsCount: 3,
},
}
testCases := []struct {
Timestamp uint32
PageNo uint32
IsDataPage bool
}{
{Timestamp: 9, PageNo: 0, IsDataPage: false},
{Timestamp: 10, PageNo: 1, IsDataPage: false},
{Timestamp: 20, PageNo: 1, IsDataPage: false},
{Timestamp: 40, PageNo: 4, IsDataPage: false},
{Timestamp: 60, PageNo: 4, IsDataPage: false},
{Timestamp: 70, PageNo: 7, IsDataPage: false},
{Timestamp: 90, PageNo: 7, IsDataPage: false},
{Timestamp: 100, PageNo: 10, IsDataPage: true},
{Timestamp: 105, PageNo: 10, IsDataPage: true},
{Timestamp: 110, PageNo: 11, IsDataPage: true},
{Timestamp: 120, PageNo: 11, IsDataPage: true},
}
for _, testCase := range testCases {
pageNo, isDataPage := FindPageOnIndexLevelTails(levels, testCase.Timestamp)
if pageNo != testCase.PageNo {
t.Fatalf("timestamp %d: got pageNo %d are not equal expected %d",
testCase.Timestamp, pageNo, testCase.PageNo)
}
if isDataPage != testCase.IsDataPage {
t.Fatalf("timestamp %d: got isDataPage %t are not equal expected %t",
testCase.Timestamp, isDataPage, testCase.IsDataPage)
}
}
}

3
storage/writer.go
View File

@@ -365,9 +365,6 @@ type PageToWrite struct {
func WriteDataPages(file *os.File, pages []PageToWrite) (err error) { func WriteDataPages(file *os.File, pages []PageToWrite) (err error) {
for _, p := range pages { for _, p := range pages {
fmt.Println("pageNo: %d\n", p.PageNo)
}
for _, p := range pages {
if len(p.Content) != DataPageSize { if len(p.Content) != DataPageSize {
return fmt.Errorf("wrong data page size: %d", len(p.Content)) return fmt.Errorf("wrong data page size: %d", len(p.Content))
} }

28
timeutil/timeutil.go
View File

@@ -1,6 +1,11 @@
package timeutil package timeutil
import "time" import (
"time"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/proto"
)
func FirstSecondInPeriod(since time.Time, period string) (_ time.Time) { func FirstSecondInPeriod(since time.Time, period string) (_ time.Time) {
y, m, d := since.Date() y, m, d := since.Date()
@@ -37,3 +42,24 @@ func LastSecondInPeriod(until time.Time, period string) (_ time.Time) {
return until return until
} }
} }
func TimeBoundsOfAggregation(since, until proto.TimeBound, groupBy qb.GroupBy, firstHourOfDay int) (s time.Time, u time.Time) {
switch groupBy {
case qb.GroupByHour, qb.GroupByDay:
s = time.Date(since.Year, since.Month, since.Day, 0, 0, 0, 0, time.Local)
u = time.Date(until.Year, until.Month, until.Day, 0, 0, 0, 0, time.Local)
case qb.GroupByMonth:
s = time.Date(since.Year, since.Month, 1, 0, 0, 0, 0, time.Local)
u = time.Date(until.Year, until.Month, 1, 0, 0, 0, 0, time.Local)
}
if firstHourOfDay > 0 {
duration := time.Duration(firstHourOfDay) * time.Hour
s = s.Add(duration)
u = u.Add(duration)
}
u = u.Add(-1 * time.Second)
return
}

22
database/tmp → tmp
View File

@@ -90,4 +90,24 @@
// ValuesSize: s.vSize, // ValuesSize: s.vSize,
// NewBufferSize: bufferSize, // NewBufferSize: bufferSize,
// }) // })
// } // }
// func isFileExist(fileName string) (bool, error) {
// _, err := os.Stat(fileName)
// if err != nil {
// if errors.Is(err, fs.ErrNotExist) {
// return false, nil
// } else {
// return false, err
// }
// } else {
// return true, nil
// }
// }
// func correctToFHD(since, until uint32, firstHourOfDay int) (uint32, uint32) {
// duration := time.Duration(firstHourOfDay) * time.Hour
// since = uint32(time.Unix(int64(since), 0).Add(duration).Unix())
// until = uint32(time.Unix(int64(until), 0).Add(duration).Unix())
// return since, until
// }

151
worker/metric.go
View File

@@ -16,18 +16,14 @@ import (
var ErrNoValueBug = errors.New("has timestamp but no value") var ErrNoValueBug = errors.New("has timestamp but no value")
type CapturedState struct { type CapturedState struct {
LastTimestamp uint32 LastValue float64
LastValue float64
} }
type Metric struct { type Metric struct {
metricType qb.MetricType metricType qb.MetricType
fracDigits byte fracDigits byte
lastPageNo uint32 lastPageNo uint32
//SinceValue float64 lastValue float64
//Since uint32
lastValue float64
//Until uint32
buffer []byte buffer []byte
timestamps qb.TimestampCompressor timestamps qb.TimestampCompressor
values qb.ValueCompressor values qb.ValueCompressor
@@ -117,14 +113,42 @@ func (s *Metric) LastValue() float64 {
return s.lastValue return s.lastValue
} }
func (s *Metric) LastTimestamp() uint32 { // REQUESTS
if s.capturedState != nil {
return s.capturedState.LastTimestamp func (s *Metric) ReleaseRLock() {
s.rLocks--
if s.rLocks == 0 {
if len(s.waitQueue) > 0 {
s.ProcessQueue()
}
} }
return s.timestamps.LastTimestamp()
} }
// REQUESTS // суть у тому що треба запускати запити, пока не зустріну XLock
func (s *Metric) ProcessQueue(metricID uint32, tmp []byte) {
if len(s.waitQueue) == 0 {
return
}
for _, untyped := range s.waitQueue {
switch req := untyped.(type) {
case RangeScanReq:
s.StartRangeScan(req)
case FullScanReq:
s.StartFullScan(req)
case GetMetricReq:
s.GetMetric(req)
case AppendMeasuresReq:
metric.AppendMeasures(req, tmp, s.storageInbox)
case DeleteMetricReq:
s.DeleteMetric(req)
case DeleteMeasuresReq:
metric.DeleteMeasures(req)
default:
qb.Abort(qb.UnknownMetricWaitQueueItemBug,
fmt.Errorf("bug: unknown metric wait queue item type %T", req))
}
}
}
func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox *inbox.Inbox) { func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox *inbox.Inbox) {
if s.xLock || s.capturedState != nil { if s.xLock || s.capturedState != nil {
@@ -151,15 +175,14 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
) )
s.capturedState = &CapturedState{ s.capturedState = &CapturedState{
LastTimestamp: timestamps.LastTimestamp(), LastValue: s.lastValue,
LastValue: s.lastValue,
} }
s.timestamps.CaptureState() s.timestamps.CaptureState()
s.values.CaptureState() s.values.CaptureState()
for idx, measure := range req.Measures { for idx, measure := range req.Measures {
if measure.Timestamp <= s.timestamps.LastTimestamp() { if measure.Timestamp <= s.timestamps.Until() {
resultCode = ExpiredMeasure resultCode = ExpiredMeasure
break break
} }
@@ -270,15 +293,15 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
} }
func (s *Metric) DeleteMeasures(req DeleteMeasuresReq) { func (s *Metric) DeleteMeasures(req DeleteMeasuresReq) {
if s.xLock || s.capturedState != nil { // if s.xLock || s.capturedState != nil {
s.waitQueue = append(s.waitQueue, req) // s.waitQueue = append(s.waitQueue, req)
return // return
} // }
since := s.timestamps.FirstTimestamp() // since := s.timestamps.FirstTimestamp()
until := s.timestamps.LastTimestamp() // until := s.timestamps.LastTimestamp()
if since == 0 || (req.Since > 0 && until < req.Since) { // if since == 0 || (req.Since > 0 && until < req.Since) {
req.ResultCh <- NoMeasuresToDelete // req.ResultCh <- NoMeasuresToDelete
} // }
// if s.RootPageNo > 0 { // if s.RootPageNo > 0 {
// req.ResultCh <- tryDeleteMeasuresResult{ // req.ResultCh <- tryDeleteMeasuresResult{
// ResultCode: DeleteFromAtreeRequired, // ResultCode: DeleteFromAtreeRequired,
@@ -296,36 +319,47 @@ func (s *Metric) StartRangeScan(req RangeScanReq) {
s.waitQueue = append(s.waitQueue, req) s.waitQueue = append(s.waitQueue, req)
return return
} }
// if s.timestamps.CommitedSize() == 0 { since := s.timestamps.CommitedSince()
// req.ResultCh <- RangeScanResult{ if since == 0 {
// ResultCode: QueryDone, req.ResultCh <- RangeScanResult{
// } ResultCode: QueryDone,
// return }
// } return
}
// if req.Since > s.timestamps.LastTimestamp() { // range after
// req.ResultCh <- RangeScanResult{ if req.Since > s.timestamps.CommitedUntil() {
// ResultCode: QueryDone, req.ResultCh <- RangeScanResult{
// } ResultCode: QueryDone,
// return }
// } return
}
// if req.Until < s.Since { // range before
// if s.RootPageNo > 0 { if req.Until < since {
// req.ResultCh <- RangeScanResult{ if len(s.indexLevelTails) > 0 {
// ResultCode: UntilNotFound, pageNo, isDataPage := storage.FindPageOnIndexLevelTails(s.indexLevelTails, req.Until)
// RootPageNo: s.RootPageNo, if pageNo > 0 {
// FracDigits: s.fracDigits, req.ResultCh <- RangeScanResult{
// } ResultCode: UntilNotFound,
// s.rLocks++ FracDigits: s.fracDigits,
// return LastPageNo: pageNo,
// } else { IsDataPage: isDataPage,
// req.ResultCh <- RangeScanResult{ }
// ResultCode: QueryDone, s.rLocks++
// } return
// return } else {
// } // range until before 1st measure timestamp
// } req.ResultCh <- RangeScanResult{
ResultCode: QueryDone,
}
return
}
} else {
req.ResultCh <- RangeScanResult{
ResultCode: QueryDone,
}
return
}
}
timestampDecompressor := s.timestamps.CreateDecompressor() timestampDecompressor := s.timestamps.CreateDecompressor()
valueDecompressor := s.values.CreateDecompressor(s.metricType, s.fracDigits) valueDecompressor := s.values.CreateDecompressor(s.metricType, s.fracDigits)
@@ -443,11 +477,6 @@ func (s *Metric) OnMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) {
s.xLock = false s.xLock = false
// s.Timestamps.Renew() // s.Timestamps.Renew()
// s.Values.Renew() // s.Values.Renew()
// s.LastPageNo = 0
// s.Since = 0
// s.SinceValue = 0
// s.Until = 0
s.indexLevelTails = nil s.indexLevelTails = nil
s.lastPageNo = 0 s.lastPageNo = 0
s.lastValue = 0 s.lastValue = 0

78
worker/worker.go
View File

@@ -5,7 +5,6 @@ import (
"sync" "sync"
qb "gordenko.dev/dima/qb" qb "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/enc" "gordenko.dev/dima/qb/enc"
"gordenko.dev/dima/qb/inbox" "gordenko.dev/dima/qb/inbox"
"gordenko.dev/dima/qb/proto" "gordenko.dev/dima/qb/proto"
@@ -86,15 +85,8 @@ func New(opt Options) *Worker {
return s return s
} }
func (s *Worker) ReleaseRLock(metricID uint32) { type ReleaseRLock struct {
// s.mutex.Lock() MetricID uint32
// //s.rLocksToRelease = append(s.rLocksToRelease, metricID)
// s.mutex.Unlock()
// select {
// case s.signalCh <- struct{}{}:
// default:
// }
} }
func (s *Worker) Run() { func (s *Worker) Run() {
@@ -114,39 +106,12 @@ func (s *Worker) Run() {
func (s *Worker) doWork() { func (s *Worker) doWork() {
queue := s.inbox.Drain() queue := s.inbox.Drain()
//rLocksToRelease := s.rLocksToRelease
//s.rLocksToRelease = nil
// for _, metricID := range rLocksToRelease {
// metric, ok := s.metrics[metricID]
// if !ok {
// qb.Abort(qb.NoMetricBug,
// fmt.Errorf("drainQueues: metric %d not found", metricID))
// }
// if metric.XLock {
// qb.Abort(qb.XLockBug,
// fmt.Errorf("drainQueues: xlock is set for the metric %d",
// metricID))
// }
// if metric.RLocks <= 0 {
// qb.Abort(qb.NoRLockBug,
// fmt.Errorf("drainQueues: rlock not set for the metric %d",
// metricID))
// }
// metric.RLocks--
// if len(metric.WaitQueue) > 0 {
// s.processMetricQueue(metricID, metric)
// }
// }
for _, untyped := range queue { for _, untyped := range queue {
switch req := untyped.(type) { switch req := untyped.(type) {
case AppendMeasuresReq: case AppendMeasuresReq:
s.AppendMeasures(req) s.AppendMeasures(req)
case ReleaseRLock:
s.releaseRLock(req.MetricID)
case storage.Changes: case storage.Changes:
s.applyCommits(req) // all metrics only s.applyCommits(req) // all metrics only
case ListCurrentValuesReq: case ListCurrentValuesReq:
@@ -170,29 +135,10 @@ func (s *Worker) doWork() {
} }
} }
// суть у тому що треба запускати запити, пока не зустріну XLock func (s *Worker) releaseRLock(metricID uint32) {
func (s *Worker) processMetricQueue(metricID uint32, metric *Metric, tmp []byte) { metric, ok := s.metrics[metricID]
if len(metric.waitQueue) == 0 { if ok {
return metric.ReleaseRLock()
}
for _, untyped := range metric.waitQueue {
switch req := untyped.(type) {
case RangeScanReq:
metric.StartRangeScan(req)
case FullScanReq:
metric.StartFullScan(req)
case GetMetricReq:
s.GetMetric(req)
case AppendMeasuresReq:
metric.AppendMeasures(req, tmp, s.storageInbox)
case DeleteMetricReq:
s.DeleteMetric(req)
case DeleteMeasuresReq:
metric.DeleteMeasures(req)
default:
qb.Abort(qb.UnknownMetricWaitQueueItemBug,
fmt.Errorf("bug: unknown metric wait queue item type %T", req))
}
} }
} }
@@ -318,8 +264,8 @@ func (s *Worker) AppendMeasures(req AppendMeasuresReq) {
type RangeScanResult struct { type RangeScanResult struct {
ResultCode byte ResultCode byte
FracDigits byte FracDigits byte
RootPageNo uint32
LastPageNo uint32 LastPageNo uint32
IsDataPage bool // for UntilNotFound only
} }
type RangeScanReq struct { type RangeScanReq struct {
@@ -327,7 +273,7 @@ type RangeScanReq struct {
Since uint32 Since uint32
Until uint32 Until uint32
MetricType qb.MetricType MetricType qb.MetricType
ResponseWriter atree.WorkerMeasureConsumer ResponseWriter qb.WorkerMeasureConsumer
ResultCh chan RangeScanResult ResultCh chan RangeScanResult
} }
@@ -361,7 +307,7 @@ type FullScanResult struct {
type FullScanReq struct { type FullScanReq struct {
MetricID uint32 MetricID uint32
MetricType qb.MetricType MetricType qb.MetricType
ResponseWriter atree.WorkerMeasureConsumer ResponseWriter qb.WorkerMeasureConsumer
ResultCh chan FullScanResult ResultCh chan FullScanResult
} }
@@ -398,7 +344,7 @@ func (s *Worker) ListCurrentValues(req ListCurrentValuesReq) {
if ok { if ok {
req.ResponseWriter.BufferValue(transform.CurrentValue{ req.ResponseWriter.BufferValue(transform.CurrentValue{
MetricID: metricID, MetricID: metricID,
Timestamp: metric.LastTimestamp(), Timestamp: metric.timestamps.CommitedUntil(),
Value: metric.LastValue(), Value: metric.LastValue(),
}) })
} }