dima/diploma
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

rc1

Browse Source
This commit is contained in:
Dima Gordenko
2025-06-03 05:04:18 +03:00
parent 0f50873f0f
commit fbb30f31e8
54 changed files with 13234 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

325
atree/aggregate.go Normal file
View File

@@ -0,0 +1,325 @@
package atree
import (
"fmt"
"time"
"gordenko.dev/dima/diploma"
"gordenko.dev/dima/diploma/timeutil"
)
// AGGREGATE
type InstantAggregator struct {
firstHourOfDay int
lastDayOfMonth int
time2period func(uint32) uint32
currentPeriod uint32
since uint32
until uint32
min float64
max float64
total float64
entries int
}
type InstantAggregatorOptions struct {
GroupBy diploma.GroupBy
FirstHourOfDay int
LastDayOfMonth int
}
func NewInstantAggregator(opt InstantAggregatorOptions) (*InstantAggregator, error) {
s := &InstantAggregator{
firstHourOfDay: opt.FirstHourOfDay,
lastDayOfMonth: opt.LastDayOfMonth,
}
switch opt.GroupBy {
case diploma.GroupByHour:
s.time2period = groupByHour
case diploma.GroupByDay:
if s.firstHourOfDay > 0 {
s.time2period = s.groupByDayUsingFHD
} else {
s.time2period = groupByDay
}
case diploma.GroupByMonth:
if s.firstHourOfDay > 0 {
if s.lastDayOfMonth > 0 {
s.time2period = s.groupByMonthUsingFHDAndLDM
} else {
s.time2period = s.groupByMonthUsingFHD
}
} else {
if s.lastDayOfMonth > 0 {
s.time2period = s.groupByMonthUsingLDM
} else {
s.time2period = groupByMonth
}
}
default:
return nil, fmt.Errorf("unknown groupBy %d option", opt.GroupBy)
}
return s, nil
}
// Приходят данные от свежих к старым, тоесть сперва получаю Until.
// return period complete flag
func (s *InstantAggregator) Feed(timestamp uint32, value float64, p *InstantPeriod) bool {
period := s.time2period(timestamp)
//fmt.Printf("feed: %s %v, period: %s\n", time.Unix(int64(timestamp), 0), value, time.Unix(int64(period), 0))
if s.entries == 0 {
s.currentPeriod = period
s.since = timestamp
s.until = timestamp
s.min = value
s.max = value
s.total = value
s.entries = 1
return false
}
if period != s.currentPeriod {
// готовый период
s.FillPeriod(timestamp, p)
s.currentPeriod = period
s.since = timestamp
s.until = timestamp
s.min = value
s.max = value
s.total = value
s.entries = 1
return true
}
if value < s.min {
s.min = value
} else if value > s.max {
s.max = value
}
// для подсчета AVG
s.total += value
s.entries++
// начало периода
s.since = timestamp
return false
}
func (s *InstantAggregator) FillPeriod(prevTimestamp uint32, p *InstantPeriod) bool {
if s.entries == 0 {
return false
}
//fmt.Printf("FillPeriod: %s, prevTimestamp: %s\n", time.Unix(int64(s.currentPeriod), 0), time.Unix(int64(prevTimestamp), 0))
p.Period = s.currentPeriod
if prevTimestamp > 0 {
p.Since = prevTimestamp
} else {
p.Since = s.since
}
p.Until = s.until
p.Min = s.min
p.Max = s.max
p.Avg = s.total / float64(s.entries)
return true
}
func (s *InstantAggregator) groupByDayUsingFHD(timestamp uint32) uint32 {
tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "d")
if tm.Hour() < s.firstHourOfDay {
tm = tm.AddDate(0, 0, -1)
}
return uint32(tm.Unix())
}
func (s *InstantAggregator) groupByMonthUsingFHD(timestamp uint32) uint32 {
tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "m")
if tm.Hour() < s.firstHourOfDay {
tm = tm.AddDate(0, 0, -1)
}
return uint32(tm.Unix())
}
func (s *InstantAggregator) groupByMonthUsingLDM(timestamp uint32) uint32 {
tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "m")
if tm.Day() > s.lastDayOfMonth {
tm = tm.AddDate(0, 1, 0)
}
return uint32(tm.Unix())
}
func (s *InstantAggregator) groupByMonthUsingFHDAndLDM(timestamp uint32) uint32 {
// ВАЖНО!
// Сперва проверяю время.
tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "m")
if tm.Hour() < s.firstHourOfDay {
tm = tm.AddDate(0, 0, -1)
}
if tm.Day() > s.lastDayOfMonth {
tm = tm.AddDate(0, 1, 0)
}
return uint32(tm.Unix())
}
// CUMULATIVE
type CumulativeAggregator struct {
firstHourOfDay int
lastDayOfMonth int
time2period func(uint32) uint32
currentPeriod uint32
since uint32
until uint32
sinceValue float64
untilValue float64
entries int
}
type CumulativeAggregatorOptions struct {
GroupBy diploma.GroupBy
FirstHourOfDay int
LastDayOfMonth int
}
func NewCumulativeAggregator(opt CumulativeAggregatorOptions) (*CumulativeAggregator, error) {
s := &CumulativeAggregator{
firstHourOfDay: opt.FirstHourOfDay,
lastDayOfMonth: opt.LastDayOfMonth,
}
switch opt.GroupBy {
case diploma.GroupByHour:
s.time2period = groupByHour
case diploma.GroupByDay:
if s.firstHourOfDay > 0 {
s.time2period = s.groupByDayUsingFHD
} else {
s.time2period = groupByDay
}
case diploma.GroupByMonth:
if s.firstHourOfDay > 0 {
if s.lastDayOfMonth > 0 {
s.time2period = s.groupByMonthUsingFHDAndLDM
} else {
s.time2period = s.groupByMonthUsingFHD
}
} else {
if s.lastDayOfMonth > 0 {
s.time2period = s.groupByMonthUsingLDM
} else {
s.time2period = groupByMonth
}
}
default:
return nil, fmt.Errorf("unknown groupBy %d option", opt.GroupBy)
}
return s, nil
}
// return period complete flag
func (s *CumulativeAggregator) Feed(timestamp uint32, value float64, p *CumulativePeriod) bool {
period := s.time2period(timestamp)
if s.entries == 0 {
s.currentPeriod = period
s.since = timestamp
s.until = timestamp
s.sinceValue = value
s.untilValue = value
s.entries = 1
return false
}
if period != s.currentPeriod {
// готовый период
s.FillPeriod(timestamp, value, p)
s.currentPeriod = period
s.since = timestamp
s.until = timestamp
s.sinceValue = value
s.untilValue = value
s.entries = 1
return true
}
// начало периода
s.since = timestamp
s.sinceValue = value
s.entries++
return false
}
func (s *CumulativeAggregator) FillPeriod(prevTimestamp uint32, value float64, p *CumulativePeriod) bool {
if s.entries == 0 {
return false
}
p.Period = s.currentPeriod
if prevTimestamp > 0 {
p.Since = prevTimestamp
p.Total = s.untilValue - value
} else {
p.Since = s.since
p.Total = s.untilValue - s.sinceValue
}
p.Until = s.until
p.EndValue = s.untilValue
return true
}
func (s *CumulativeAggregator) groupByDayUsingFHD(timestamp uint32) uint32 {
tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "d")
if tm.Hour() < s.firstHourOfDay {
tm = tm.AddDate(0, 0, -1)
}
return uint32(tm.Unix())
}
func (s *CumulativeAggregator) groupByMonthUsingFHD(timestamp uint32) uint32 {
tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "m")
if tm.Hour() < s.firstHourOfDay {
tm = tm.AddDate(0, 0, -1)
}
return uint32(tm.Unix())
}
func (s *CumulativeAggregator) groupByMonthUsingLDM(timestamp uint32) uint32 {
tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "m")
if tm.Day() > s.lastDayOfMonth {
tm = tm.AddDate(0, 1, 0)
}
return uint32(tm.Unix())
}
func (s *CumulativeAggregator) groupByMonthUsingFHDAndLDM(timestamp uint32) uint32 {
// ВАЖНО!
// Сперва проверяю время.
tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "m")
if tm.Hour() < s.firstHourOfDay {
tm = tm.AddDate(0, 0, -1)
}
if tm.Day() > s.lastDayOfMonth {
tm = tm.AddDate(0, 1, 0)
}
return uint32(tm.Unix())
}
func groupByHour(timestamp uint32) uint32 {
return uint32(timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "h").Unix())
}
func groupByDay(timestamp uint32) uint32 {
return uint32(timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "d").Unix())
}
func groupByMonth(timestamp uint32) uint32 {
return uint32(timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "m").Unix())
}

497
atree/atree.go Normal file
View File

@@ -0,0 +1,497 @@
package atree
import (
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"gordenko.dev/dima/diploma"
"gordenko.dev/dima/diploma/atree/redo"
"gordenko.dev/dima/diploma/bin"
)
const (
filePerm = 0770
// index page
indexRecordsQtyIdx = IndexPageSize - 7
isDataPageNumbersIdx = IndexPageSize - 5
indexCRC32Idx = IndexPageSize - 4
// data page
timestampsSizeIdx = DataPageSize - 12
valuesSizeIdx = DataPageSize - 10
prevPageIdx = DataPageSize - 8
dataCRC32Idx = DataPageSize - 4
timestampSize = 4
pairSize = timestampSize + PageNoSize
indexFooterIdx = indexRecordsQtyIdx
dataFooterIdx = timestampsSizeIdx
DataPageSize = 8192
IndexPageSize = 1024
PageNoSize = 4
//
DataPagePayloadSize int = dataFooterIdx
)
type FreeList interface {
ReservePage() uint32
}
type _page struct {
PageNo uint32
Buf []byte
ReferenceCount int
}
type Atree struct {
redoDir string
indexFreelist FreeList
dataFreelist FreeList
dataFile *os.File
indexFile *os.File
mutex sync.Mutex
allocatedIndexPagesQty uint32
allocatedDataPagesQty uint32
indexPages map[uint32]*_page
dataPages map[uint32]*_page
indexWaits map[uint32][]chan readResult
dataWaits map[uint32][]chan readResult
indexPagesToRead []uint32
dataPagesToRead []uint32
readSignalCh chan struct{}
writeSignalCh chan struct{}
writeTasksQueue []WriteTask
}
type Options struct {
Dir string
RedoDir string
DatabaseName string
DataFreeList FreeList
IndexFreeList FreeList
}
func New(opt Options) (*Atree, error) {
if opt.Dir == "" {
return nil, errors.New("Dir option is required")
}
if opt.RedoDir == "" {
return nil, errors.New("RedoDir option is required")
}
if opt.DatabaseName == "" {
return nil, errors.New("DatabaseName option is required")
}
if opt.DataFreeList == nil {
return nil, errors.New("DataFreeList option is required")
}
if opt.IndexFreeList == nil {
return nil, errors.New("IndexFreeList option is required")
}
// открываю или создаю dbName.data и dbName.index файлы
var (
indexFileName = filepath.Join(opt.Dir, opt.DatabaseName+".index")
dataFileName = filepath.Join(opt.Dir, opt.DatabaseName+".data")
indexFile *os.File
dataFile *os.File
allocatedIndexPagesQty uint32
allocatedDataPagesQty uint32
)
// При создании data файла сразу создается индекс, поэтому корректное
// состояние БД: либо оба файла есть, либо ни одного файла нет.
isIndexExist, err := isFileExist(indexFileName)
if err != nil {
return nil, fmt.Errorf("check index file is exist: %s", err)
}
isDataExist, err := isFileExist(dataFileName)
if err != nil {
return nil, fmt.Errorf("check data file is exist: %s", err)
}
if isIndexExist {
if isDataExist {
// открываю оба файла
indexFile, allocatedIndexPagesQty, err = openFile(indexFileName, IndexPageSize)
if err != nil {
return nil, fmt.Errorf("open index file: %s", err)
}
dataFile, allocatedDataPagesQty, err = openFile(dataFileName, DataPageSize)
if err != nil {
return nil, fmt.Errorf("open data file: %s", err)
}
} else {
// нет data файла
return nil, errors.New("not found data file")
}
} else {
if isDataExist {
// index файла нет
return nil, errors.New("not found index file")
} else {
// нет обоих файлов
indexFile, err = os.OpenFile(indexFileName, os.O_CREATE|os.O_RDWR, filePerm)
if err != nil {
return nil, err
}
dataFile, err = os.OpenFile(dataFileName, os.O_CREATE|os.O_RDWR, filePerm)
if err != nil {
return nil, err
}
}
}
tree := &Atree{
redoDir: opt.RedoDir,
indexFreelist: opt.IndexFreeList,
dataFreelist: opt.DataFreeList,
indexFile: indexFile,
dataFile: dataFile,
allocatedIndexPagesQty: allocatedIndexPagesQty,
allocatedDataPagesQty: allocatedDataPagesQty,
indexPages: make(map[uint32]*_page),
dataPages: make(map[uint32]*_page),
indexWaits: make(map[uint32][]chan readResult),
dataWaits: make(map[uint32][]chan readResult),
readSignalCh: make(chan struct{}, 1),
writeSignalCh: make(chan struct{}, 1),
}
return tree, nil
}
func (s *Atree) Run() {
go s.pageWriter()
go s.pageReader()
}
// FIND
func (s *Atree) findDataPage(rootPageNo uint32, timestamp uint32) (uint32, []byte, error) {
indexPageNo := rootPageNo
for {
buf, err := s.fetchIndexPage(indexPageNo)
if err != nil {
return 0, nil, fmt.Errorf("fetchIndexPage(%d): %s", indexPageNo, err)
}
foundPageNo := findPageNo(buf, timestamp)
s.releaseIndexPage(indexPageNo)
if buf[isDataPageNumbersIdx] == 1 {
buf, err := s.fetchDataPage(foundPageNo)
if err != nil {
return 0, nil, fmt.Errorf("fetchDataPage(%d): %s", foundPageNo, err)
}
return foundPageNo, buf, nil
}
// вглубь
indexPageNo = foundPageNo
}
}
type pathLeg struct {
PageNo uint32
Data []byte
}
type pathToDataPage struct {
Legs []pathLeg
LastPageNo uint32
}
func (s *Atree) findPathToLastPage(rootPageNo uint32) (_ pathToDataPage, err error) {
var (
pageNo = rootPageNo
legs []pathLeg
)
for {
var buf []byte
buf, err = s.fetchIndexPage(pageNo)
if err != nil {
err = fmt.Errorf("FetchIndexPage(%d): %s", pageNo, err)
return
}
legs = append(legs, pathLeg{
PageNo: pageNo,
Data: buf,
// childIdx не нужен
})
foundPageNo := getLastPageNo(buf)
if buf[isDataPageNumbersIdx] == 1 {
return pathToDataPage{
Legs: legs,
LastPageNo: foundPageNo,
}, nil
}
// вглубь
pageNo = foundPageNo
}
}
// APPEND DATA PAGE
type AppendDataPageReq struct {
MetricID uint32
Timestamp uint32
Value float64
Since uint32
RootPageNo uint32
PrevPageNo uint32
TimestampsChunks [][]byte
TimestampsSize uint16
ValuesChunks [][]byte
ValuesSize uint16
}
func (s *Atree) AppendDataPage(req AppendDataPageReq) (_ redo.Report, err error) {
var (
flags byte
dataPagesToRelease []uint32
indexPagesToRelease []uint32
)
newDataPage := s.allocDataPage()
dataPagesToRelease = append(dataPagesToRelease, newDataPage.PageNo)
chunksToDataPage(newDataPage.Data, chunksToDataPageReq{
PrevPageNo: req.PrevPageNo,
TimestampsChunks: req.TimestampsChunks,
TimestampsSize: req.TimestampsSize,
ValuesChunks: req.ValuesChunks,
ValuesSize: req.ValuesSize,
})
redoWriter, err := redo.NewWriter(redo.WriterOptions{
Dir: s.redoDir,
MetricID: req.MetricID,
Timestamp: req.Timestamp,
Value: req.Value,
IsDataPageReused: newDataPage.IsReused,
DataPageNo: newDataPage.PageNo,
Page: newDataPage.Data,
})
if err != nil {
return
}
if req.RootPageNo > 0 {
var path pathToDataPage
path, err = s.findPathToLastPage(req.RootPageNo)
if err != nil {
return
}
for _, leg := range path.Legs {
indexPagesToRelease = append(indexPagesToRelease, leg.PageNo)
}
if path.LastPageNo != req.PrevPageNo {
diploma.Abort(
diploma.WrongPrevPageNo,
fmt.Errorf("bug: last pageNo %d in tree != prev pageNo %d in _metric",
path.LastPageNo, req.PrevPageNo),
)
}
newPageNo := newDataPage.PageNo
lastIdx := len(path.Legs) - 1
for legIdx := lastIdx; legIdx >= 0; legIdx-- {
leg := path.Legs[legIdx]
ok := appendPair(leg.Data, req.Since, newPageNo)
if ok {
err = redoWriter.AppendIndexPage(leg.PageNo, leg.Data, 0)
if err != nil {
return
}
break
}
newIndexPage := s.allocIndexPage()
indexPagesToRelease = append(indexPagesToRelease, newIndexPage.PageNo)
appendPair(newIndexPage.Data, req.Since, newPageNo)
// ставлю мітку що всі pageNo на сторінці - це data pageNo
if legIdx == lastIdx {
newIndexPage.Data[isDataPageNumbersIdx] = 1
}
flags = 0
if newIndexPage.IsReused {
flags |= redo.FlagReused
}
err = redoWriter.AppendIndexPage(newIndexPage.PageNo, newIndexPage.Data, flags)
if err != nil {
return
}
//
newPageNo = newIndexPage.PageNo
if legIdx == 0 {
newRoot := s.allocIndexPage()
indexPagesToRelease = append(indexPagesToRelease, newRoot.PageNo)
appendPair(newRoot.Data, getSince(leg.Data), leg.PageNo) // old rootPageNo
appendPair(newRoot.Data, req.Since, newIndexPage.PageNo)
// Фиксирую новый root в REDO логе
flags = redo.FlagNewRoot
if newRoot.IsReused {
flags |= redo.FlagReused
}
err = redoWriter.AppendIndexPage(newRoot.PageNo, newRoot.Data, flags)
if err != nil {
return
}
break
}
}
} else {
newRoot := s.allocIndexPage()
indexPagesToRelease = append(indexPagesToRelease, newRoot.PageNo)
newRoot.Data[isDataPageNumbersIdx] = 1
appendPair(newRoot.Data, req.Since, newDataPage.PageNo)
flags = redo.FlagNewRoot
if newRoot.IsReused {
flags |= redo.FlagReused
}
err = redoWriter.AppendIndexPage(newRoot.PageNo, newRoot.Data, flags)
if err != nil {
return
}
}
err = redoWriter.Close()
if err != nil {
return
}
// На данний момен схема - наступна. Всі сторінки - data та index - зафіксовані в кеші.
// Отже запис на диск пройде максимально швидко. Після цього ReferenceCount кожної
// сторінки зменшиться на 1. Оскільки на метрику утримується XLock, сторінки мають
// ReferenceCount = 1 (немає інших читачів).
waitCh := make(chan struct{})
task := WriteTask{
WaitCh: waitCh,
DataPage: redo.PageToWrite{
PageNo: newDataPage.PageNo,
Data: newDataPage.Data,
},
IndexPages: redoWriter.IndexPagesToWrite(),
}
s.appendWriteTaskToQueue(task)
<-waitCh
for _, pageNo := range dataPagesToRelease {
s.releaseDataPage(pageNo)
}
for _, pageNo := range indexPagesToRelease {
s.releaseIndexPage(pageNo)
}
return redoWriter.GetReport(), nil
}
// DELETE
type PageLists struct {
DataPages []uint32
IndexPages []uint32
}
type Level struct {
PageNo uint32
PageData []byte
Idx int
ChildQty int
}
func (s *Atree) GetAllPages(rootPageNo uint32) (_ PageLists, err error) {
var (
dataPages []uint32
indexPages []uint32
levels []*Level
)
buf, err := s.fetchIndexPage(rootPageNo)
if err != nil {
err = fmt.Errorf("fetchIndexPage(%d): %s", rootPageNo, err)
return
}
indexPages = append(indexPages, rootPageNo)
if buf[isDataPageNumbersIdx] == 1 {
pageNumbers := listPageNumbers(buf)
dataPages = append(dataPages, pageNumbers...)
s.releaseIndexPage(rootPageNo)
return PageLists{
DataPages: dataPages,
IndexPages: indexPages,
}, nil
}
levels = append(levels, &Level{
PageNo: rootPageNo,
PageData: buf,
Idx: 0,
ChildQty: bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]),
})
for {
if len(levels) == 0 {
return PageLists{
DataPages: dataPages,
IndexPages: indexPages,
}, nil
}
lastIdx := len(levels) - 1
level := levels[lastIdx]
if level.Idx < level.ChildQty {
pageNo := getPageNo(level.PageData, level.Idx)
level.Idx++
var buf []byte
buf, err = s.fetchIndexPage(pageNo)
if err != nil {
err = fmt.Errorf("fetchIndexPage(%d): %s", pageNo, err)
return
}
indexPages = append(indexPages, pageNo)
if buf[isDataPageNumbersIdx] == 1 {
pageNumbers := listPageNumbers(buf)
dataPages = append(dataPages, pageNumbers...)
s.releaseIndexPage(pageNo)
} else {
levels = append(levels, &Level{
PageNo: pageNo,
PageData: buf,
Idx: 0,
ChildQty: bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]),
})
}
} else {
s.releaseIndexPage(level.PageNo)
levels = levels[:lastIdx]
}
}
}

187
atree/cursor.go Normal file
View File

@@ -0,0 +1,187 @@
package atree
import (
"errors"
"fmt"
octopus "gordenko.dev/dima/diploma"
"gordenko.dev/dima/diploma/bin"
"gordenko.dev/dima/diploma/enc"
)
type BackwardCursor struct {
metricType octopus.MetricType
fracDigits byte
atree *Atree
pageNo uint32
pageData []byte
timestampDecompressor octopus.TimestampDecompressor
valueDecompressor octopus.ValueDecompressor
}
type BackwardCursorOptions struct {
MetricType octopus.MetricType
FracDigits byte
PageNo uint32
PageData []byte
Atree *Atree
}
func NewBackwardCursor(opt BackwardCursorOptions) (*BackwardCursor, error) {
switch opt.MetricType {
case octopus.Instant, octopus.Cumulative:
// ok
default:
return nil, fmt.Errorf("MetricType option has wrong value: %d", opt.MetricType)
}
if opt.FracDigits > octopus.MaxFracDigits {
return nil, errors.New("FracDigits option is required")
}
if opt.Atree == nil {
return nil, errors.New("Atree option is required")
}
if opt.PageNo == 0 {
return nil, errors.New("PageNo option is required")
}
if len(opt.PageData) == 0 {
return nil, errors.New("PageData option is required")
}
s := &BackwardCursor{
metricType: opt.MetricType,
fracDigits: opt.FracDigits,
atree: opt.Atree,
pageNo: opt.PageNo,
pageData: opt.PageData,
}
err := s.makeDecompressors()
if err != nil {
return nil, err
}
return s, nil
}
// timestamp, value, done, error
func (s *BackwardCursor) Prev() (uint32, float64, bool, error) {
var (
timestamp uint32
value float64
done bool
err error
)
timestamp, done = s.timestampDecompressor.NextValue()
if !done {
value, done = s.valueDecompressor.NextValue()
if done {
return 0, 0, false,
fmt.Errorf("corrupted data page %d: has timestamp, no value",
s.pageNo)
}
return timestamp, value, false, nil
}
prevPageNo := bin.GetUint32(s.pageData[prevPageIdx:])
if prevPageNo == 0 {
return 0, 0, true, nil
}
s.atree.releaseDataPage(s.pageNo)
s.pageNo = prevPageNo
s.pageData, err = s.atree.fetchDataPage(s.pageNo)
if err != nil {
return 0, 0, false, fmt.Errorf("atree.fetchDataPage(%d): %s", s.pageNo, err)
}
err = s.makeDecompressors()
if err != nil {
return 0, 0, false, err
}
timestamp, done = s.timestampDecompressor.NextValue()
if done {
return 0, 0, false,
fmt.Errorf("corrupted data page %d: no timestamps",
s.pageNo)
}
value, done = s.valueDecompressor.NextValue()
if done {
return 0, 0, false,
fmt.Errorf("corrupted data page %d: no values",
s.pageNo)
}
return timestamp, value, false, nil
}
func (s *BackwardCursor) Close() {
s.atree.releaseDataPage(s.pageNo)
}
// HELPER
func (s *BackwardCursor) makeDecompressors() error {
timestampsPayloadSize := bin.GetUint16(s.pageData[timestampsSizeIdx:])
valuesPayloadSize := bin.GetUint16(s.pageData[valuesSizeIdx:])
payloadSize := timestampsPayloadSize + valuesPayloadSize
if payloadSize > dataFooterIdx {
return fmt.Errorf("corrupted data page %d: timestamps + values size %d gt payload size",
s.pageNo, payloadSize)
}
s.timestampDecompressor = enc.NewReverseTimeDeltaOfDeltaDecompressor(
s.pageData[:timestampsPayloadSize],
)
vbuf := s.pageData[timestampsPayloadSize : timestampsPayloadSize+valuesPayloadSize]
switch s.metricType {
case octopus.Instant:
s.valueDecompressor = enc.NewReverseInstantDeltaDecompressor(
vbuf, s.fracDigits)
case octopus.Cumulative:
s.valueDecompressor = enc.NewReverseCumulativeDeltaDecompressor(
vbuf, s.fracDigits)
default:
return fmt.Errorf("bug: wrong metricType %d", s.metricType)
}
return nil
}
func makeDecompressors(pageData []byte, metricType octopus.MetricType, fracDigits byte) (
octopus.TimestampDecompressor, octopus.ValueDecompressor, error,
) {
timestampsPayloadSize := bin.GetUint16(pageData[timestampsSizeIdx:])
valuesPayloadSize := bin.GetUint16(pageData[valuesSizeIdx:])
payloadSize := timestampsPayloadSize + valuesPayloadSize
if payloadSize > dataFooterIdx {
return nil, nil, fmt.Errorf("corrupted: timestamps + values size %d > payload size",
payloadSize)
}
timestampDecompressor := enc.NewReverseTimeDeltaOfDeltaDecompressor(
pageData[:timestampsPayloadSize],
)
vbuf := pageData[timestampsPayloadSize : timestampsPayloadSize+valuesPayloadSize]
var valueDecompressor octopus.ValueDecompressor
switch metricType {
case octopus.Instant:
valueDecompressor = enc.NewReverseInstantDeltaDecompressor(
vbuf, fracDigits)
case octopus.Cumulative:
valueDecompressor = enc.NewReverseCumulativeDeltaDecompressor(
vbuf, fracDigits)
default:
return nil, nil, fmt.Errorf("bug: wrong metricType %d", metricType)
}
return timestampDecompressor, valueDecompressor, nil
}

430
atree/io.go Normal file
View File

@@ -0,0 +1,430 @@
package atree
import (
"errors"
"fmt"
"hash/crc32"
"io/fs"
"math"
"os"
octopus "gordenko.dev/dima/diploma"
"gordenko.dev/dima/diploma/atree/redo"
"gordenko.dev/dima/diploma/bin"
)
type AllocatedPage struct {
PageNo uint32
Data []byte
IsReused bool
}
type readResult struct {
Data []byte
Err error
}
// INDEX PAGES
func (s *Atree) DeleteIndexPages(pageNumbers []uint32) {
s.mutex.Lock()
for _, pageNo := range pageNumbers {
delete(s.indexPages, pageNo)
}
s.mutex.Unlock()
}
func (s *Atree) fetchIndexPage(pageNo uint32) ([]byte, error) {
s.mutex.Lock()
p, ok := s.indexPages[pageNo]
if ok {
p.ReferenceCount++
s.mutex.Unlock()
return p.Buf, nil
}
resultCh := make(chan readResult, 1)
s.indexWaits[pageNo] = append(s.indexWaits[pageNo], resultCh)
if len(s.indexWaits[pageNo]) == 1 {
s.indexPagesToRead = append(s.indexPagesToRead, pageNo)
s.mutex.Unlock()
select {
case s.readSignalCh <- struct{}{}:
default:
}
} else {
s.mutex.Unlock()
}
result := <-resultCh
if result.Err == nil {
result.Err = s.verifyCRC(result.Data, IndexPageSize)
}
return result.Data, result.Err
}
func (s *Atree) releaseIndexPage(pageNo uint32) {
s.mutex.Lock()
defer s.mutex.Unlock()
p, ok := s.indexPages[pageNo]
if ok {
if p.ReferenceCount > 0 {
p.ReferenceCount--
return
} else {
octopus.Abort(
octopus.ReferenceCountBug,
fmt.Errorf("call releaseIndexPage on page %d with reference count = %d",
pageNo, p.ReferenceCount),
)
}
}
}
func (s *Atree) allocIndexPage() AllocatedPage {
var (
allocated = AllocatedPage{
Data: make([]byte, IndexPageSize),
}
)
allocated.PageNo = s.indexFreelist.ReservePage()
if allocated.PageNo > 0 {
allocated.IsReused = true
s.mutex.Lock()
} else {
s.mutex.Lock()
if s.allocatedIndexPagesQty == math.MaxUint32 {
octopus.Abort(octopus.MaxAtreeSizeExceeded,
errors.New("no space in Atree index"))
}
s.allocatedIndexPagesQty++
allocated.PageNo = s.allocatedIndexPagesQty
}
s.indexPages[allocated.PageNo] = &_page{
PageNo: allocated.PageNo,
Buf: allocated.Data,
ReferenceCount: 1,
}
s.mutex.Unlock()
return allocated
}
// DATA PAGES
func (s *Atree) DeleteDataPages(pageNumbers []uint32) {
s.mutex.Lock()
for _, pageNo := range pageNumbers {
delete(s.dataPages, pageNo)
}
s.mutex.Unlock()
}
func (s *Atree) fetchDataPage(pageNo uint32) ([]byte, error) {
s.mutex.Lock()
p, ok := s.dataPages[pageNo]
if ok {
p.ReferenceCount++
s.mutex.Unlock()
return p.Buf, nil
}
resultCh := make(chan readResult, 1)
s.dataWaits[pageNo] = append(s.dataWaits[pageNo], resultCh)
if len(s.dataWaits[pageNo]) == 1 {
s.dataPagesToRead = append(s.dataPagesToRead, pageNo)
s.mutex.Unlock()
select {
case s.readSignalCh <- struct{}{}:
default:
}
} else {
s.mutex.Unlock()
}
result := <-resultCh
if result.Err == nil {
result.Err = s.verifyCRC(result.Data, DataPageSize)
}
return result.Data, result.Err
}
func (s *Atree) releaseDataPage(pageNo uint32) {
s.mutex.Lock()
defer s.mutex.Unlock()
p, ok := s.dataPages[pageNo]
if ok {
if p.ReferenceCount > 0 {
p.ReferenceCount--
return
} else {
octopus.Abort(
octopus.ReferenceCountBug,
fmt.Errorf("call releaseDataPage on page %d with reference count = %d",
pageNo, p.ReferenceCount),
)
}
}
}
func (s *Atree) allocDataPage() AllocatedPage {
var (
allocated = AllocatedPage{
Data: make([]byte, DataPageSize),
}
)
allocated.PageNo = s.dataFreelist.ReservePage()
if allocated.PageNo > 0 {
allocated.IsReused = true
s.mutex.Lock()
} else {
s.mutex.Lock()
if s.allocatedDataPagesQty == math.MaxUint32 {
octopus.Abort(octopus.MaxAtreeSizeExceeded,
errors.New("no space in Atree index"))
}
s.allocatedDataPagesQty++
allocated.PageNo = s.allocatedDataPagesQty
}
s.dataPages[allocated.PageNo] = &_page{
PageNo: allocated.PageNo,
Buf: allocated.Data,
ReferenceCount: 1,
}
s.mutex.Unlock()
return allocated
}
// READ
func (s *Atree) pageReader() {
for {
select {
case <-s.readSignalCh:
s.readPages()
}
}
}
func (s *Atree) readPages() {
s.mutex.Lock()
if len(s.indexPagesToRead) == 0 && len(s.dataPagesToRead) == 0 {
s.mutex.Unlock()
return
}
indexPagesToRead := s.indexPagesToRead
s.indexPagesToRead = nil
dataPagesToRead := s.dataPagesToRead
s.dataPagesToRead = nil
s.mutex.Unlock()
for _, pageNo := range dataPagesToRead {
buf := make([]byte, DataPageSize)
off := (pageNo - 1) * DataPageSize
n, err := s.dataFile.ReadAt(buf, int64(off))
if n != DataPageSize {
err = fmt.Errorf("read %d instead of %d", n, DataPageSize)
}
s.mutex.Lock()
resultChannels := s.dataWaits[pageNo]
delete(s.dataWaits, pageNo)
if err != nil {
s.mutex.Unlock()
for _, resultCh := range resultChannels {
resultCh <- readResult{
Err: err,
}
}
} else {
s.dataPages[pageNo] = &_page{
PageNo: pageNo,
Buf: buf,
ReferenceCount: len(resultChannels),
}
s.mutex.Unlock()
for _, resultCh := range resultChannels {
resultCh <- readResult{
Data: buf,
}
}
}
}
for _, pageNo := range indexPagesToRead {
buf := make([]byte, IndexPageSize)
off := (pageNo - 1) * IndexPageSize
n, err := s.indexFile.ReadAt(buf, int64(off))
if n != IndexPageSize {
err = fmt.Errorf("read %d instead of %d", n, IndexPageSize)
}
s.mutex.Lock()
resultChannels := s.indexWaits[pageNo]
delete(s.indexWaits, pageNo)
if err != nil {
s.mutex.Unlock()
for _, resultCh := range resultChannels {
resultCh <- readResult{
Err: err,
}
}
} else {
s.indexPages[pageNo] = &_page{
PageNo: pageNo,
Buf: buf,
ReferenceCount: len(resultChannels),
}
s.mutex.Unlock()
for _, resultCh := range resultChannels {
resultCh <- readResult{
Data: buf,
}
}
}
}
}
// WRITE
func (s *Atree) pageWriter() {
for {
select {
case <-s.writeSignalCh:
err := s.writeTasks()
if err != nil {
octopus.Abort(octopus.WriteToAtreeFailed, err)
}
}
}
}
type WriteTask struct {
WaitCh chan struct{}
DataPage redo.PageToWrite
IndexPages []redo.PageToWrite
}
func (s *Atree) appendWriteTaskToQueue(task WriteTask) {
s.mutex.Lock()
s.writeTasksQueue = append(s.writeTasksQueue, task)
s.mutex.Unlock()
select {
case s.writeSignalCh <- struct{}{}:
default:
}
}
func (s *Atree) writeTasks() error {
s.mutex.Lock()
tasks := s.writeTasksQueue
s.writeTasksQueue = nil
s.mutex.Unlock()
for _, task := range tasks {
// data page
p := task.DataPage
if len(p.Data) != DataPageSize {
return fmt.Errorf("wrong data page %d size: %d",
p.PageNo, len(p.Data))
}
off := (p.PageNo - 1) * DataPageSize
n, err := s.dataFile.WriteAt(p.Data, int64(off))
if err != nil {
return err
}
if n != len(p.Data) {
return fmt.Errorf("write %d instead of %d", n, len(p.Data))
}
// index pages
for _, p := range task.IndexPages {
if len(p.Data) != IndexPageSize {
return fmt.Errorf("wrong index page %d size: %d",
p.PageNo, len(p.Data))
}
bin.PutUint32(p.Data[indexCRC32Idx:], crc32.ChecksumIEEE(p.Data[:indexCRC32Idx]))
off := (p.PageNo - 1) * IndexPageSize
n, err := s.indexFile.WriteAt(p.Data, int64(off))
if err != nil {
return err
}
if n != len(p.Data) {
return fmt.Errorf("write %d instead of %d", n, len(p.Data))
}
}
close(task.WaitCh)
}
return nil
}
// IO
func isFileExist(fileName string) (bool, error) {
_, err := os.Stat(fileName)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return false, nil
} else {
return false, err
}
} else {
return true, nil
}
}
func openFile(fileName string, pageSize int) (_ *os.File, _ uint32, err error) {
file, err := os.OpenFile(fileName, os.O_RDWR, filePerm)
if err != nil {
return
}
fi, err := file.Stat()
if err != nil {
return
}
fileSize := fi.Size()
if (fileSize % int64(pageSize)) > 0 {
err = fmt.Errorf("the file size %d is not a multiple of the page size %d",
fileSize, pageSize)
return
}
allocatedPagesQty := fileSize / int64(pageSize)
if allocatedPagesQty > math.MaxUint32 {
err = fmt.Errorf("allocated pages %d is > max pages %d",
allocatedPagesQty, math.MaxUint32)
return
}
return file, uint32(allocatedPagesQty), nil
}
func (s *Atree) ApplyREDO(task WriteTask) {
s.appendWriteTaskToQueue(task)
}
func (s *Atree) verifyCRC(data []byte, pageSize int) error {
var (
pos = pageSize - 4
calculatedCRC = crc32.ChecksumIEEE(data[:pos])
storedCRC = bin.GetUint32(data[pos:])
)
if calculatedCRC != storedCRC {
return fmt.Errorf("calculatedCRC %d not equal storedCRC %d",
calculatedCRC, storedCRC)
}
return nil
}

214
atree/misc.go Normal file
View File

@@ -0,0 +1,214 @@
package atree
import (
"hash/crc32"
"gordenko.dev/dima/diploma/bin"
)
type ValueAtComparator struct {
buf []byte
timestamp uint32
}
func (s ValueAtComparator) CompareTo(elemIdx int) int {
var (
pos = elemIdx * timestampSize
elem = bin.GetUint32(s.buf[pos:])
)
if s.timestamp < elem {
return -1
} else if s.timestamp > elem {
return 1
} else {
return 0
}
}
func BinarySearch(qty int, keyComparator bin.KeyComparator) (elemIdx int, isFound bool) {
if qty == 0 {
return
}
a := 0
b := qty - 1
for {
var (
elemIdx = (b-a)/2 + a
code = keyComparator.CompareTo(elemIdx)
)
if code == 1 {
a = elemIdx + 1
if a > b {
return elemIdx, false // +1
}
} else if code == -1 {
b = elemIdx - 1
if b < a {
if elemIdx == 0 {
return 0, false
} else {
return elemIdx - 1, false
}
}
} else {
return elemIdx, true
}
}
}
type chunksToDataPageReq struct {
PrevPageNo uint32
TimestampsChunks [][]byte
TimestampsSize uint16
ValuesChunks [][]byte
ValuesSize uint16
}
func chunksToDataPage(buf []byte, req chunksToDataPageReq) {
bin.PutUint32(buf[prevPageIdx:], req.PrevPageNo)
bin.PutUint16(buf[timestampsSizeIdx:], req.TimestampsSize)
bin.PutUint16(buf[valuesSizeIdx:], req.ValuesSize)
var (
remainingSize = int(req.TimestampsSize)
pos = 0
)
for _, chunk := range req.TimestampsChunks {
if remainingSize >= len(chunk) {
copy(buf[pos:], chunk)
remainingSize -= len(chunk)
pos += len(chunk)
} else {
copy(buf[pos:], chunk[:remainingSize])
break
}
}
remainingSize = int(req.ValuesSize)
pos = int(req.TimestampsSize)
for _, chunk := range req.ValuesChunks {
if remainingSize >= len(chunk) {
copy(buf[pos:], chunk)
remainingSize -= len(chunk)
pos += len(chunk)
} else {
copy(buf[pos:], chunk[:remainingSize])
break
}
}
bin.PutUint32(buf[dataCRC32Idx:], crc32.ChecksumIEEE(buf[:dataCRC32Idx]))
}
func setPrevPageNo(buf []byte, pageNo uint32) {
bin.PutUint32(buf[prevPageIdx:], pageNo)
}
func getPrevPageNo(buf []byte) uint32 {
return bin.GetUint32(buf[prevPageIdx:])
}
func findPageNo(buf []byte, timestamp uint32) (pageNo uint32) {
var (
qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:])
comparator = ValueAtComparator{
buf: buf,
timestamp: timestamp,
}
)
elemIdx, _ := BinarySearch(qty, comparator)
pos := indexFooterIdx - (elemIdx+1)*PageNoSize
return bin.GetUint32(buf[pos:])
}
func findPageNoIdx(buf []byte, timestamp uint32) (idx int) {
var (
qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:])
comparator = ValueAtComparator{
buf: buf,
timestamp: timestamp,
}
)
elemIdx, _ := BinarySearch(qty, comparator)
return elemIdx
}
func findPageNoForDeleteSince(buf []byte, since uint32) (uint32, int, int) {
var (
qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:])
comparator = ValueAtComparator{
buf: buf,
timestamp: since,
}
)
elemIdx, _ := BinarySearch(qty, comparator)
pos := elemIdx * timestampSize
timestamp := bin.GetUint32(buf[pos:])
if timestamp == since {
if elemIdx == 0 {
return 0, qty, -1
}
elemIdx--
}
pos = indexFooterIdx - (elemIdx+1)*PageNoSize
return bin.GetUint32(buf[pos:]), qty, elemIdx
}
func getLastPageNo(buf []byte) (pageNo uint32) {
qty := bin.GetUint16AsInt(buf[indexRecordsQtyIdx:])
pos := indexFooterIdx - qty*PageNoSize
return bin.GetUint32(buf[pos:])
}
func getPageNo(buf []byte, idx int) (pageNo uint32) {
pos := indexFooterIdx - (idx+1)*PageNoSize
return bin.GetUint32(buf[pos:])
}
func listPageNumbers(buf []byte) (pageNumbers []uint32) {
qty := bin.GetUint16AsInt(buf[indexRecordsQtyIdx:])
pos := indexFooterIdx - PageNoSize
for range qty {
pageNumbers = append(pageNumbers, bin.GetUint32(buf[pos:]))
pos -= PageNoSize
}
return
}
// include since timestamp
func listPageNumbersSince(buf []byte, timestamp uint32) (pageNumbers []uint32) {
var (
qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:])
comparator = ValueAtComparator{
buf: buf,
timestamp: timestamp,
}
)
elemIdx, _ := BinarySearch(qty, comparator)
pos := indexFooterIdx - (elemIdx+1)*PageNoSize
for range qty {
pageNumbers = append(pageNumbers, bin.GetUint32(buf[pos:]))
pos -= PageNoSize
}
return
}
func getSince(buf []byte) uint32 {
return bin.GetUint32(buf[0:])
}
func appendPair(buf []byte, timestamp uint32, pageNo uint32) bool {
qty := bin.GetUint16AsInt(buf[indexRecordsQtyIdx:])
free := indexFooterIdx - qty*pairSize
if free < pairSize {
return false
}
pos := qty * timestampSize
bin.PutUint32(buf[pos:], timestamp)
pos = indexFooterIdx - (qty+1)*PageNoSize
bin.PutUint32(buf[pos:], pageNo)
bin.PutIntAsUint16(buf[indexRecordsQtyIdx:], qty+1)
return true
}

96
atree/redo/reader.go Normal file
View File

@@ -0,0 +1,96 @@
package redo
import (
"fmt"
"hash/crc32"
"io"
"os"
"gordenko.dev/dima/diploma/bin"
)
type REDOFile struct {
MetricID uint32
Timestamp uint32
Value float64
IsDataPageReused bool
DataPage PageToWrite
IsRootChanged bool
RootPageNo uint32
ReusedIndexPages []uint32
IndexPages []PageToWrite
}
type ReadREDOFileReq struct {
FileName string
DataPageSize int
IndexPageSize int
}
func ReadREDOFile(req ReadREDOFileReq) (*REDOFile, error) {
buf, err := os.ReadFile(req.FileName)
if err != nil {
return nil, err
}
if len(buf) < 25 {
return nil, io.EOF
}
var (
end = len(buf) - 4
payload = buf[:end]
checksum = bin.GetUint32(buf[end:])
calculatedChecksum = crc32.ChecksumIEEE(payload)
)
// Помилка чексуми означає що файл або недописаний, або пошкодженний
if checksum != calculatedChecksum {
return nil, fmt.Errorf("written checksum %d not equal calculated checksum %d",
checksum, calculatedChecksum)
}
var (
redoLog = REDOFile{
MetricID: bin.GetUint32(buf[0:]),
Timestamp: bin.GetUint32(buf[4:]),
Value: bin.GetFloat64(buf[8:]),
IsDataPageReused: buf[16] == 1,
DataPage: PageToWrite{
PageNo: bin.GetUint32(buf[17:]),
Data: buf[21 : 21+req.DataPageSize],
},
}
pos = 21 + req.DataPageSize
)
for {
if pos == len(payload) {
return &redoLog, nil
}
if pos > len(payload) {
return nil, io.EOF
}
flags := buf[pos]
item := PageToWrite{
PageNo: bin.GetUint32(buf[pos+1:]),
}
pos += 5 // flags + pageNo
item.Data = buf[pos : pos+req.IndexPageSize]
pos += req.IndexPageSize
redoLog.IndexPages = append(redoLog.IndexPages, item)
if (flags & FlagReused) == FlagReused {
redoLog.ReusedIndexPages = append(redoLog.ReusedIndexPages, item.PageNo)
}
if (flags & FlagNewRoot) == FlagNewRoot {
redoLog.IsRootChanged = true
redoLog.RootPageNo = item.PageNo
}
}
}

207
atree/redo/writer.go Normal file
View File

@@ -0,0 +1,207 @@
package redo
import (
"errors"
"fmt"
"hash"
"hash/crc32"
"os"
"path/filepath"
"gordenko.dev/dima/diploma/bin"
)
const (
FlagReused byte = 1 // сторінка із FreeList
FlagNewRoot byte = 2 // новая страница
)
type PageToWrite struct {
PageNo uint32
Data []byte
}
type Writer struct {
metricID uint32
timestamp uint32
value float64
tmp []byte
fileName string
file *os.File
hasher hash.Hash32
isDataPageReused bool
dataPageNo uint32
isRootChanged bool
newRootPageNo uint32
indexPages []uint32
reusedIndexPages []uint32
indexPagesToWrite []PageToWrite
}
type WriterOptions struct {
Dir string
MetricID uint32
Value float64
Timestamp uint32
IsDataPageReused bool
DataPageNo uint32
Page []byte
}
// dataPage можно записати 1 раз. Щоб не заплутувати інтерфейс - передаю data сторінку
// через Options. Index сторінок може бути від 1 до N, тому виділяю окремий метод
func NewWriter(opt WriterOptions) (*Writer, error) {
if opt.Dir == "" {
return nil, errors.New("Dir option is required")
}
if opt.MetricID == 0 {
return nil, errors.New("MetricID option is required")
}
if opt.DataPageNo == 0 {
return nil, errors.New("DataPageNo option is required")
}
// if len(opt.Page) != octopus.DataPageSize {
// return nil, fmt.Errorf("bug: wrong data page size %d", len(opt.Page))
// }
s := &Writer{
fileName: JoinREDOFileName(opt.Dir, opt.MetricID),
metricID: opt.MetricID,
timestamp: opt.Timestamp,
value: opt.Value,
tmp: make([]byte, 21),
isDataPageReused: opt.IsDataPageReused,
dataPageNo: opt.DataPageNo,
hasher: crc32.NewIEEE(),
}
var err error
s.file, err = os.OpenFile(s.fileName, os.O_CREATE|os.O_WRONLY, 0770)
if err != nil {
return nil, err
}
err = s.init(opt.Page)
if err != nil {
return nil, err
}
return s, nil
}
/*
Формат:
4b metricID
8b value
4b timestamp
1b flags (reused)
4b dataPageNo
8KB dataPage
*/
func (s *Writer) init(dataPage []byte) error {
bin.PutUint32(s.tmp[0:], s.metricID)
bin.PutUint32(s.tmp[4:], s.timestamp)
bin.PutFloat64(s.tmp[8:], s.value)
if s.isDataPageReused {
s.tmp[16] = 1
}
bin.PutUint32(s.tmp[17:], s.dataPageNo)
_, err := s.file.Write(s.tmp)
if err != nil {
return err
}
_, err = s.file.Write(dataPage)
if err != nil {
return err
}
s.hasher.Write(s.tmp)
s.hasher.Write(dataPage)
return nil
}
/*
Формат
1b index page flags
4b indexPageNo
Nb indexPage
*/
func (s *Writer) AppendIndexPage(indexPageNo uint32, indexPage []byte, flags byte) error {
s.tmp[0] = flags
bin.PutUint32(s.tmp[1:], indexPageNo)
_, err := s.file.Write(s.tmp[:5])
if err != nil {
return err
}
_, err = s.file.Write(indexPage)
if err != nil {
return err
}
s.hasher.Write(s.tmp[:5])
s.hasher.Write(indexPage)
s.indexPages = append(s.indexPages, indexPageNo)
if (flags & FlagReused) == FlagReused {
s.reusedIndexPages = append(s.reusedIndexPages, indexPageNo)
}
if (flags & FlagNewRoot) == FlagNewRoot {
s.newRootPageNo = indexPageNo
s.isRootChanged = true
}
s.indexPagesToWrite = append(s.indexPagesToWrite,
PageToWrite{
PageNo: indexPageNo,
Data: indexPage,
})
return nil
}
func (s *Writer) IndexPagesToWrite() []PageToWrite {
return s.indexPagesToWrite
}
func (s *Writer) Close() (err error) {
// финализирую запись
bin.PutUint32(s.tmp, s.hasher.Sum32())
_, err = s.file.Write(s.tmp[:4])
if err != nil {
return err
}
err = s.file.Sync()
if err != nil {
return
}
return s.file.Close()
}
type Report struct {
FileName string
IsDataPageReused bool
DataPageNo uint32
IsRootChanged bool
NewRootPageNo uint32
ReusedIndexPages []uint32
}
func (s *Writer) GetReport() Report {
return Report{
FileName: s.fileName,
IsDataPageReused: s.isDataPageReused,
DataPageNo: s.dataPageNo,
//IndexPages: s.indexPages,
IsRootChanged: s.isRootChanged,
NewRootPageNo: s.newRootPageNo,
ReusedIndexPages: s.reusedIndexPages,
}
}
// HELPERS
func JoinREDOFileName(dir string, metricID uint32) string {
return filepath.Join(dir, fmt.Sprintf("m%d.redo", metricID))
}

619
atree/select.go Normal file
View File

@@ -0,0 +1,619 @@
package atree
import (
"fmt"
octopus "gordenko.dev/dima/diploma"
)
type IterateAllCumulativeByTreeCursorReq struct {
FracDigits byte
PageNo uint32
EndTimestamp uint32
EndValue float64
ResponseWriter *CumulativeMeasureWriter
}
func (s *Atree) IterateAllCumulativeByTreeCursor(req IterateAllCumulativeByTreeCursorReq) error {
buf, err := s.fetchDataPage(req.PageNo)
if err != nil {
return err
}
treeCursor, err := NewBackwardCursor(BackwardCursorOptions{
PageNo: req.PageNo,
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: octopus.Cumulative,
})
if err != nil {
return err
}
defer treeCursor.Close()
var (
endTimestamp = req.EndTimestamp
endValue = req.EndValue
)
for {
timestamp, value, done, err := treeCursor.Prev()
if err != nil {
return err
}
if done {
err := req.ResponseWriter.WriteMeasure(CumulativeMeasure{
Timestamp: endTimestamp,
Value: endValue,
Total: endValue,
})
if err != nil {
return err
}
return nil
}
err = req.ResponseWriter.WriteMeasure(CumulativeMeasure{
Timestamp: endTimestamp,
Value: endValue,
Total: endValue - value,
})
if err != nil {
return err
}
endTimestamp = timestamp
endValue = value
}
}
type ContinueIterateCumulativeByTreeCursorReq struct {
FracDigits byte
Since uint32
Until uint32
LastPageNo uint32
EndTimestamp uint32
EndValue float64
ResponseWriter *CumulativeMeasureWriter
}
func (s *Atree) ContinueIterateCumulativeByTreeCursor(req ContinueIterateCumulativeByTreeCursorReq) error {
buf, err := s.fetchDataPage(req.LastPageNo)
if err != nil {
return fmt.Errorf("fetchDataPage(%d): %s", req.LastPageNo, err)
}
treeCursor, err := NewBackwardCursor(BackwardCursorOptions{
PageNo: req.LastPageNo,
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: octopus.Cumulative,
})
if err != nil {
return err
}
defer treeCursor.Close()
var (
endTimestamp = req.EndTimestamp
endValue = req.EndValue
)
for {
timestamp, value, done, err := treeCursor.Prev()
if err != nil {
return err
}
if done {
err := req.ResponseWriter.WriteMeasure(CumulativeMeasure{
Timestamp: endTimestamp,
Value: endValue,
Total: endValue,
})
if err != nil {
return err
}
return nil
}
if timestamp <= req.Until {
err := req.ResponseWriter.WriteMeasure(CumulativeMeasure{
Timestamp: endTimestamp,
Value: endValue,
Total: endValue - value,
})
if err != nil {
return err
}
if timestamp < req.Since {
return nil
}
} else {
// bug panic
panic("continue cumulative but timestamp > req.Until")
}
}
}
type FindAndIterateCumulativeByTreeCursorReq struct {
FracDigits byte
Since uint32
Until uint32
RootPageNo uint32
ResponseWriter *CumulativeMeasureWriter
}
func (s *Atree) FindAndIterateCumulativeByTreeCursor(req FindAndIterateCumulativeByTreeCursorReq) error {
pageNo, buf, err := s.findDataPage(req.RootPageNo, req.Until)
if err != nil {
return err
}
treeCursor, err := NewBackwardCursor(BackwardCursorOptions{
PageNo: pageNo,
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: octopus.Cumulative,
})
if err != nil {
return err
}
defer treeCursor.Close()
var (
endTimestamp uint32
endValue float64
)
for {
timestamp, value, done, err := treeCursor.Prev()
if err != nil {
return err
}
if done {
if endTimestamp > 0 {
err := req.ResponseWriter.WriteMeasure(CumulativeMeasure{
Timestamp: endTimestamp,
Value: endValue,
Total: endValue,
})
if err != nil {
return err
}
}
return nil
}
if timestamp > req.Until {
continue
}
if endTimestamp > 0 {
err := req.ResponseWriter.WriteMeasure(CumulativeMeasure{
Timestamp: endTimestamp,
Value: endValue,
Total: endValue - value,
})
if err != nil {
return err
}
}
endTimestamp = timestamp
endValue = value
if timestamp < req.Since {
return nil
}
}
}
type IterateAllInstantByTreeCursorReq struct {
FracDigits byte
PageNo uint32
ResponseWriter *InstantMeasureWriter
}
func (s *Atree) IterateAllInstantByTreeCursor(req IterateAllInstantByTreeCursorReq) error {
buf, err := s.fetchDataPage(req.PageNo)
if err != nil {
return err
}
treeCursor, err := NewBackwardCursor(BackwardCursorOptions{
PageNo: req.PageNo,
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: octopus.Instant,
})
if err != nil {
return err
}
defer treeCursor.Close()
for {
timestamp, value, done, err := treeCursor.Prev()
if err != nil {
return err
}
if done {
return nil
}
err = req.ResponseWriter.WriteMeasure(InstantMeasure{
Timestamp: timestamp,
Value: value,
})
if err != nil {
return err
}
}
}
type ContinueIterateInstantByTreeCursorReq struct {
FracDigits byte
Since uint32
Until uint32
LastPageNo uint32
ResponseWriter *InstantMeasureWriter
}
func (s *Atree) ContinueIterateInstantByTreeCursor(req ContinueIterateInstantByTreeCursorReq) error {
buf, err := s.fetchDataPage(req.LastPageNo)
if err != nil {
return fmt.Errorf("fetchDataPage(%d): %s", req.LastPageNo, err)
}
treeCursor, err := NewBackwardCursor(BackwardCursorOptions{
PageNo: req.LastPageNo,
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: octopus.Instant,
})
if err != nil {
return err
}
defer treeCursor.Close()
for {
timestamp, value, done, err := treeCursor.Prev()
if err != nil {
return err
}
if done {
// - записи закончились;
return nil
}
if timestamp > req.Until {
panic("continue instant timestamp > req.Until")
}
if timestamp < req.Since {
return nil
}
err = req.ResponseWriter.WriteMeasure(InstantMeasure{
Timestamp: timestamp,
Value: value,
})
if err != nil {
return err
}
}
}
type FindAndIterateInstantByTreeCursorReq struct {
FracDigits byte
Since uint32
Until uint32
RootPageNo uint32
ResponseWriter *InstantMeasureWriter
}
func (s *Atree) FindAndIterateInstantByTreeCursor(req FindAndIterateInstantByTreeCursorReq) error {
pageNo, buf, err := s.findDataPage(req.RootPageNo, req.Until)
if err != nil {
return err
}
treeCursor, err := NewBackwardCursor(BackwardCursorOptions{
PageNo: pageNo,
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: octopus.Instant,
})
if err != nil {
return err
}
defer treeCursor.Close()
for {
timestamp, value, done, err := treeCursor.Prev()
if err != nil {
return err
}
if done {
return nil
}
if timestamp > req.Until {
continue
}
if timestamp < req.Since {
return nil
}
err = req.ResponseWriter.WriteMeasure(InstantMeasure{
Timestamp: timestamp,
Value: value,
})
if err != nil {
return err
}
}
}
type ContinueCollectInstantPeriodsReq struct {
FracDigits byte
Aggregator *InstantAggregator
ResponseWriter *InstantPeriodsWriter
LastPageNo uint32
Since uint32
Until uint32
}
func (s *Atree) ContinueCollectInstantPeriods(req ContinueCollectInstantPeriodsReq) error {
buf, err := s.fetchDataPage(req.LastPageNo)
if err != nil {
return fmt.Errorf("fetchDataPage(%d): %s", req.LastPageNo, err)
}
treeCursor, err := NewBackwardCursor(BackwardCursorOptions{
PageNo: req.LastPageNo,
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: octopus.Instant,
})
if err != nil {
return err
}
defer treeCursor.Close()
var period InstantPeriod
for {
timestamp, value, done, err := treeCursor.Prev()
if err != nil {
return err
}
if done || timestamp < req.Since {
isCompleted := req.Aggregator.FillPeriod(timestamp, &period)
if isCompleted {
err := req.ResponseWriter.WritePeriod(period)
if err != nil {
return err
}
}
return nil
}
if timestamp <= req.Until {
isCompleted := req.Aggregator.Feed(timestamp, value, &period)
if isCompleted {
err := req.ResponseWriter.WritePeriod(period)
if err != nil {
return err
}
}
}
}
}
type FindInstantPeriodsReq struct {
FracDigits byte
ResponseWriter *InstantPeriodsWriter
RootPageNo uint32
Since uint32
Until uint32
GroupBy octopus.GroupBy
FirstHourOfDay int
LastDayOfMonth int
}
func (s *Atree) FindInstantPeriods(req FindInstantPeriodsReq) error {
pageNo, buf, err := s.findDataPage(req.RootPageNo, req.Until)
if err != nil {
return err
}
aggregator, err := NewInstantAggregator(InstantAggregatorOptions{
GroupBy: req.GroupBy,
FirstHourOfDay: req.FirstHourOfDay,
LastDayOfMonth: req.LastDayOfMonth,
})
if err != nil {
return err
}
cursor, err := NewBackwardCursor(BackwardCursorOptions{
PageNo: pageNo,
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: octopus.Instant,
})
if err != nil {
return err
}
defer cursor.Close()
var period InstantPeriod
for {
timestamp, value, done, err := cursor.Prev()
if err != nil {
return err
}
if done || timestamp < req.Since {
isCompleted := aggregator.FillPeriod(timestamp, &period)
if isCompleted {
err := req.ResponseWriter.WritePeriod(period)
if err != nil {
return err
}
}
return nil
}
if timestamp <= req.Until {
isCompleted := aggregator.Feed(timestamp, value, &period)
if isCompleted {
err := req.ResponseWriter.WritePeriod(period)
if err != nil {
return err
}
}
}
}
}
type FindCumulativePeriodsReq struct {
FracDigits byte
ResponseWriter *CumulativePeriodsWriter
RootPageNo uint32
Since uint32
Until uint32
GroupBy octopus.GroupBy
FirstHourOfDay int
LastDayOfMonth int
}
func (s *Atree) FindCumulativePeriods(req FindCumulativePeriodsReq) error {
pageNo, buf, err := s.findDataPage(req.RootPageNo, req.Until)
if err != nil {
return err
}
aggregator, err := NewCumulativeAggregator(CumulativeAggregatorOptions{
GroupBy: req.GroupBy,
FirstHourOfDay: req.FirstHourOfDay,
LastDayOfMonth: req.LastDayOfMonth,
})
if err != nil {
return err
}
cursor, err := NewBackwardCursor(BackwardCursorOptions{
PageNo: pageNo,
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: octopus.Cumulative,
})
if err != nil {
return err
}
defer cursor.Close()
var period CumulativePeriod
for {
timestamp, value, done, err := cursor.Prev()
if err != nil {
return err
}
if done || timestamp < req.Since {
isCompleted := aggregator.FillPeriod(timestamp, value, &period)
if isCompleted {
err := req.ResponseWriter.WritePeriod(period)
if err != nil {
return err
}
}
return nil
}
if timestamp <= req.Until {
isCompleted := aggregator.Feed(timestamp, value, &period)
if isCompleted {
err := req.ResponseWriter.WritePeriod(period)
if err != nil {
return err
}
}
}
}
}
type ContinueCollectCumulativePeriodsReq struct {
FracDigits byte
Aggregator *CumulativeAggregator
ResponseWriter *CumulativePeriodsWriter
LastPageNo uint32
Since uint32
Until uint32
}
func (s *Atree) ContinueCollectCumulativePeriods(req ContinueCollectCumulativePeriodsReq) error {
buf, err := s.fetchDataPage(req.LastPageNo)
if err != nil {
return fmt.Errorf("fetchDataPage(%d): %s", req.LastPageNo, err)
}
treeCursor, err := NewBackwardCursor(BackwardCursorOptions{
PageNo: req.LastPageNo,
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: octopus.Cumulative,
})
if err != nil {
return err
}
defer treeCursor.Close()
var period CumulativePeriod
for {
timestamp, value, done, err := treeCursor.Prev()
if err != nil {
return err
}
if done || timestamp < req.Since {
isCompleted := req.Aggregator.FillPeriod(timestamp, value, &period)
if isCompleted {
err := req.ResponseWriter.WritePeriod(period)
if err != nil {
return err
}
}
return nil
}
if timestamp <= req.Until {
isCompleted := req.Aggregator.Feed(timestamp, value, &period)
if isCompleted {
err := req.ResponseWriter.WritePeriod(period)
if err != nil {
return err
}
}
}
}
}

306
atree/writers.go Normal file
View File

@@ -0,0 +1,306 @@
package atree
import (
"bytes"
"fmt"
"io"
octopus "gordenko.dev/dima/diploma"
"gordenko.dev/dima/diploma/bin"
"gordenko.dev/dima/diploma/proto"
)
// CURRENT VALUE WRITER
type CurrentValue struct {
MetricID uint32
Timestamp uint32
Value float64
}
type CurrentValueWriter struct {
arr []byte
responder *ChunkedResponder
}
func NewCurrentValueWriter(dst io.Writer) *CurrentValueWriter {
return &CurrentValueWriter{
arr: make([]byte, 16),
responder: NewChunkedResponder(dst),
}
}
func (s *CurrentValueWriter) BufferValue(m CurrentValue) {
bin.PutUint32(s.arr[0:], m.MetricID)
bin.PutUint32(s.arr[4:], m.Timestamp)
bin.PutFloat64(s.arr[8:], m.Value)
s.responder.BufferRecord(s.arr)
}
func (s *CurrentValueWriter) Close() error {
return s.responder.Flush()
}
// INSTANT MEASURE WRITER
type InstantMeasure struct {
Timestamp uint32
Value float64
}
type InstantMeasureWriter struct {
arr []byte
responder *ChunkedResponder
}
func NewInstantMeasureWriter(dst io.Writer) *InstantMeasureWriter {
return &InstantMeasureWriter{
arr: make([]byte, 12),
responder: NewChunkedResponder(dst),
}
}
func (s *InstantMeasureWriter) BufferMeasure(m InstantMeasure) {
bin.PutUint32(s.arr[0:], m.Timestamp)
bin.PutFloat64(s.arr[4:], m.Value)
s.responder.BufferRecord(s.arr)
}
func (s *InstantMeasureWriter) WriteMeasure(m InstantMeasure) error {
bin.PutUint32(s.arr[0:], m.Timestamp)
bin.PutFloat64(s.arr[4:], m.Value)
return s.responder.AppendRecord(s.arr)
}
func (s *InstantMeasureWriter) Close() error {
return s.responder.Flush()
}
// CUMULATIVE MEASURE WRITER
type CumulativeMeasure struct {
Timestamp uint32
Value float64
Total float64
}
type CumulativeMeasureWriter struct {
arr []byte
responder *ChunkedResponder
}
func NewCumulativeMeasureWriter(dst io.Writer) *CumulativeMeasureWriter {
return &CumulativeMeasureWriter{
arr: make([]byte, 20),
responder: NewChunkedResponder(dst),
}
}
func (s *CumulativeMeasureWriter) BufferMeasure(m CumulativeMeasure) {
bin.PutUint32(s.arr[0:], m.Timestamp)
bin.PutFloat64(s.arr[4:], m.Value)
bin.PutFloat64(s.arr[12:], m.Total)
s.responder.BufferRecord(s.arr)
}
func (s *CumulativeMeasureWriter) WriteMeasure(m CumulativeMeasure) error {
bin.PutUint32(s.arr[0:], m.Timestamp)
bin.PutFloat64(s.arr[4:], m.Value)
bin.PutFloat64(s.arr[12:], m.Total)
return s.responder.AppendRecord(s.arr)
}
func (s *CumulativeMeasureWriter) Close() error {
return s.responder.Flush()
}
// INSTANT AGGREGATE WRITER
type InstantPeriodsWriter struct {
aggregateFuncs byte
arr []byte
responder *ChunkedResponder
}
func NewInstantPeriodsWriter(dst io.Writer, aggregateFuncs byte) *InstantPeriodsWriter {
var q int
if (aggregateFuncs & octopus.AggregateMin) == octopus.AggregateMin {
q++
}
if (aggregateFuncs & octopus.AggregateMax) == octopus.AggregateMax {
q++
}
if (aggregateFuncs & octopus.AggregateAvg) == octopus.AggregateAvg {
q++
}
return &InstantPeriodsWriter{
aggregateFuncs: aggregateFuncs,
arr: make([]byte, 12+q*8),
responder: NewChunkedResponder(dst),
}
}
type InstantPeriod struct {
Period uint32
Since uint32
Until uint32
Min float64
Max float64
Avg float64
}
func (s *InstantPeriodsWriter) BufferMeasure(p InstantPeriod) {
s.pack(p)
s.responder.BufferRecord(s.arr)
}
func (s *InstantPeriodsWriter) WritePeriod(p InstantPeriod) error {
s.pack(p)
return s.responder.AppendRecord(s.arr)
}
func (s *InstantPeriodsWriter) Close() error {
return s.responder.Flush()
}
func (s *InstantPeriodsWriter) pack(p InstantPeriod) {
bin.PutUint32(s.arr[0:], p.Period)
bin.PutUint32(s.arr[4:], p.Since)
bin.PutUint32(s.arr[8:], p.Until)
pos := 12
if (s.aggregateFuncs & octopus.AggregateMin) == octopus.AggregateMin {
bin.PutFloat64(s.arr[pos:], p.Min)
pos += 8
}
if (s.aggregateFuncs & octopus.AggregateMax) == octopus.AggregateMax {
bin.PutFloat64(s.arr[pos:], p.Max)
pos += 8
}
if (s.aggregateFuncs & octopus.AggregateAvg) == octopus.AggregateAvg {
bin.PutFloat64(s.arr[pos:], p.Avg)
}
}
// CUMULATIVE AGGREGATE WRITER
type CumulativePeriodsWriter struct {
arr []byte
responder *ChunkedResponder
}
func NewCumulativePeriodsWriter(dst io.Writer) *CumulativePeriodsWriter {
return &CumulativePeriodsWriter{
arr: make([]byte, 28),
responder: NewChunkedResponder(dst),
}
}
type CumulativePeriod struct {
Period uint32
Since uint32
Until uint32
EndValue float64
Total float64
}
func (s *CumulativePeriodsWriter) BufferMeasure(p CumulativePeriod) {
s.pack(p)
s.responder.BufferRecord(s.arr)
}
func (s *CumulativePeriodsWriter) WritePeriod(p CumulativePeriod) error {
s.pack(p)
return s.responder.AppendRecord(s.arr)
}
func (s *CumulativePeriodsWriter) Close() error {
return s.responder.Flush()
}
func (s *CumulativePeriodsWriter) pack(p CumulativePeriod) {
bin.PutUint32(s.arr[0:], p.Period)
bin.PutUint32(s.arr[4:], p.Since)
bin.PutUint32(s.arr[8:], p.Until)
bin.PutFloat64(s.arr[12:], p.EndValue)
bin.PutFloat64(s.arr[20:], p.Total)
}
// CHUNKED RESPONDER
//const headerSize = 3
var endMsg = []byte{
proto.RespEndOfValue, // end of stream
}
type ChunkedResponder struct {
recordsQty int
buf *bytes.Buffer
dst io.Writer
}
func NewChunkedResponder(dst io.Writer) *ChunkedResponder {
s := &ChunkedResponder{
recordsQty: 0,
buf: bytes.NewBuffer(nil),
dst: dst,
}
s.buf.Write([]byte{
proto.RespPartOfValue, // message type
0, 0, 0, 0, // records qty
})
return s
}
func (s *ChunkedResponder) BufferRecord(rec []byte) {
s.buf.Write(rec)
s.recordsQty++
}
func (s *ChunkedResponder) AppendRecord(rec []byte) error {
s.buf.Write(rec)
s.recordsQty++
if s.buf.Len() < 1500 {
return nil
}
if err := s.sendBuffered(); err != nil {
return err
}
s.buf.Write([]byte{
proto.RespPartOfValue, // message type
0, 0, 0, 0, // records qty
})
s.recordsQty = 0
return nil
}
func (s *ChunkedResponder) Flush() error {
if s.recordsQty > 0 {
if err := s.sendBuffered(); err != nil {
return err
}
}
if _, err := s.dst.Write(endMsg); err != nil {
return err
}
return nil
}
func (s *ChunkedResponder) sendBuffered() (err error) {
msg := s.buf.Bytes()
bin.PutUint32(msg[1:], uint32(s.recordsQty))
n, err := s.dst.Write(msg)
if err != nil {
return
}
if n != len(msg) {
return fmt.Errorf("incomplete write %d bytes instead of %d", n, len(msg))
}
s.buf.Reset()
return
}