dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-06-10 06:18:45 +03:00
parent 4089ca40c9
commit 9ec86282ad
47 changed files with 3493 additions and 4502 deletions
Download Patch File Download Diff File Expand all files Collapse all files

330
atree/atree.go
View File

@@ -1,49 +1,12 @@
package atree
import (
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"gordenko.dev/dima/qb/bin"
)
const (
PageTypeData = 1
PageTypeIndex = 2
filePerm = 0770
// common
crc32Idx = PageSize - 4
pageTypeIdx = PageSize - 5
// index page
indexCRC32Idx = IndexPageSize - 4
isLastLevelIdx = IndexPageSize - 5
indexRecordsQtyIdx = IndexPageSize - 7
indexRecordSize = 8
// data page
dataCRC32Idx = DataPageSize - 4
prevPageIdx = DataPageSize - 8
valuesSizeIdx = DataPageSize - 10
timestampsSizeIdx = DataPageSize - 12
timestampSize = 4
pairSize = timestampSize + PageNoSize
indexFooterIdx = indexRecordsQtyIdx
dataFooterIdx = timestampsSizeIdx
PageSize = 8192
DataPageSize = 8192
IndexPageSize = 1024
PageNoSize = 4
//
DataPagePayloadSize int = dataFooterIdx
)
type _page struct {
@@ -70,48 +33,49 @@ type Options struct {
}
func New(opt Options) (*Atree, error) {
if opt.Dir == "" {
return nil, errors.New("Dir option is required")
}
if opt.DatabaseName == "" {
return nil, errors.New("DatabaseName option is required")
}
// открываю или создаю dbName.data и dbName.index файлы
var (
fileName = filepath.Join(opt.Dir, opt.DatabaseName+".db")
file *os.File
allocatedPagesQty uint32
)
// if opt.Dir == "" {
// return nil, errors.New("Dir option is required")
// }
// if opt.DatabaseName == "" {
// return nil, errors.New("DatabaseName option is required")
// }
// // открываю или создаю dbName.data и dbName.index файлы
// var (
// fileName = filepath.Join(opt.Dir, opt.DatabaseName+".db")
// file *os.File
// allocatedPagesQty uint32
// )
// При создании data файла сразу создается индекс, поэтому корректное
// состояние БД: либо оба файла есть, либо ни одного файла нет.
isDataExist, err := isFileExist(fileName)
if err != nil {
return nil, fmt.Errorf("check data file is exist: %s", err)
}
if isDataExist {
file, allocatedPagesQty, err = openFile(fileName, PageSize)
if err != nil {
return nil, fmt.Errorf("open data file: %s", err)
}
} else {
// нет файла
file, err = os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, filePerm)
if err != nil {
return nil, err
}
}
// // При создании data файла сразу создается индекс, поэтому корректное
// // состояние БД: либо оба файла есть, либо ни одного файла нет.
// isDataExist, err := isFileExist(fileName)
// if err != nil {
// return nil, fmt.Errorf("check data file is exist: %s", err)
// }
// if isDataExist {
// file, allocatedPagesQty, err = openFile(fileName, PageSize)
// if err != nil {
// return nil, fmt.Errorf("open data file: %s", err)
// }
// } else {
// // нет файла
// file, err = os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, filePerm)
// if err != nil {
// return nil, err
// }
// }
tree := &Atree{
file: file,
allocatedPagesQty: allocatedPagesQty,
pages: make(map[uint32]*_page),
pageWaits: make(map[uint32][]chan readResult),
readSignalCh: make(chan struct{}, 1),
writeSignalCh: make(chan struct{}, 1),
}
// tree := &Atree{
// file: file,
// allocatedPagesQty: allocatedPagesQty,
// pages: make(map[uint32]*_page),
// pageWaits: make(map[uint32][]chan readResult),
// readSignalCh: make(chan struct{}, 1),
// writeSignalCh: make(chan struct{}, 1),
// }
return tree, nil
// return tree, nil
return nil, nil
}
func (s *Atree) Run() {
@@ -122,27 +86,28 @@ func (s *Atree) Run() {
// FIND
func (s *Atree) findDataPage(rootPageNo uint32, timestamp uint32) (uint32, []byte, error) {
indexPageNo := rootPageNo
for {
buf, err := s.fetchIndexPage(indexPageNo)
if err != nil {
return 0, nil, fmt.Errorf("fetchIndexPage(%d): %s", indexPageNo, err)
}
// indexPageNo := rootPageNo
// for {
// buf, err := s.fetchIndexPage(indexPageNo)
// if err != nil {
// return 0, nil, fmt.Errorf("fetchIndexPage(%d): %s", indexPageNo, err)
// }
foundPageNo := findPageNo(buf, timestamp)
s.releasePage(indexPageNo)
// foundPageNo := findPageNo(buf, timestamp)
// s.releasePage(indexPageNo)
// fix
if buf[isLastLevelIdx] == 1 {
buf, err := s.fetchDataPage(foundPageNo)
if err != nil {
return 0, nil, fmt.Errorf("fetchDataPage(%d): %s", foundPageNo, err)
}
return foundPageNo, buf, nil
}
// вглубь
indexPageNo = foundPageNo
}
// // fix
// if buf[isLastLevelIdx] == 1 {
// buf, err := s.fetchDataPage(foundPageNo)
// if err != nil {
// return 0, nil, fmt.Errorf("fetchDataPage(%d): %s", foundPageNo, err)
// }
// return foundPageNo, buf, nil
// }
// // вглубь
// indexPageNo = foundPageNo
// }
return 0, nil, nil
}
type PathLeg struct {
@@ -156,37 +121,38 @@ type PathToDataPage struct {
}
func (s *Atree) FindPathToLastPage(rootPageNo uint32) (_ PathToDataPage, err error) {
var (
pageNo = rootPageNo
legs []PathLeg
)
// var (
// pageNo = rootPageNo
// legs []PathLeg
// )
for {
var buf []byte
buf, err = s.fetchIndexPage(pageNo)
if err != nil {
err = fmt.Errorf("FetchIndexPage(%d): %s", pageNo, err)
return
}
// for {
// var buf []byte
// buf, err = s.fetchIndexPage(pageNo)
// if err != nil {
// err = fmt.Errorf("FetchIndexPage(%d): %s", pageNo, err)
// return
// }
legs = append(legs, PathLeg{
PageNo: pageNo,
Data: buf,
// childIdx не нужен
})
// legs = append(legs, PathLeg{
// PageNo: pageNo,
// Data: buf,
// // childIdx не нужен
// })
foundPageNo := getLastPageNo(buf)
// foundPageNo := getLastPageNo(buf)
// fix
if buf[isLastLevelIdx] == 1 {
return PathToDataPage{
Legs: legs,
LastPageNo: foundPageNo,
}, nil
}
// вглубь
pageNo = foundPageNo
}
// // fix
// if buf[isLastLevelIdx] == 1 {
// return PathToDataPage{
// Legs: legs,
// LastPageNo: foundPageNo,
// }, nil
// }
// // вглубь
// pageNo = foundPageNo
// }
return
}
// DELETE
@@ -199,72 +165,76 @@ type Level struct {
}
func (s *Atree) GetAllPages(rootPageNo uint32) (_ []uint32, err error) {
var (
pageNumbers []uint32
levels []*Level
)
// var (
// pageNumbers []uint32
// levels []*Level
// )
buf, err := s.fetchIndexPage(rootPageNo)
if err != nil {
return nil, fmt.Errorf("fetchIndexPage(%d): %s", rootPageNo, err)
}
pageNumbers = append(pageNumbers, rootPageNo)
// if buf[isDataPageNumbersIdx] == 1 {
// pageNumbers := listPageNumbers(buf)
// dataPages = append(dataPages, pageNumbers...)
// s.releasePage(rootPageNo)
// return PageLists{
// DataPages: dataPages,
// IndexPages: indexPages,
// }, nil
// buf, err := s.fetchIndexPage(rootPageNo)
// if err != nil {
// return nil, fmt.Errorf("fetchIndexPage(%d): %s", rootPageNo, err)
// }
// pageNumbers = append(pageNumbers, rootPageNo)
levels = append(levels, &Level{
PageNo: rootPageNo,
PageData: buf,
Idx: 0,
ChildQty: bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]),
})
// // if buf[isDataPageNumbersIdx] == 1 {
// // pageNumbers := listPageNumbers(buf)
// // dataPages = append(dataPages, pageNumbers...)
for {
if len(levels) == 0 {
return pageNumbers, nil
}
// // s.releasePage(rootPageNo)
lastIdx := len(levels) - 1
level := levels[lastIdx]
// // return PageLists{
// // DataPages: dataPages,
// // IndexPages: indexPages,
// // }, nil
// // }
if level.Idx < level.ChildQty {
pageNo := getPageNo(level.PageData, level.Idx)
level.Idx++
// childQty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
var buf []byte
buf, err = s.fetchPage(pageNo)
if err != nil {
return nil, fmt.Errorf("fetchPage(%d): %s", pageNo, err)
}
pageNumbers = append(pageNumbers, pageNo)
// levels = append(levels, &Level{
// PageNo: rootPageNo,
// PageData: buf,
// Idx: 0,
// ChildQty: int(childQty),
// })
if buf[pageTypeIdx] == PageTypeData {
//pageNumbers := listPageNumbers(buf)
//dataPages = append(dataPages, pageNumbers...)
s.releasePage(pageNo)
} else {
levels = append(levels, &Level{
PageNo: pageNo,
PageData: buf,
Idx: 0,
ChildQty: bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]),
})
}
} else {
s.releasePage(level.PageNo)
levels = levels[:lastIdx]
}
}
// for {
// if len(levels) == 0 {
// return pageNumbers, nil
// }
// lastIdx := len(levels) - 1
// level := levels[lastIdx]
// if level.Idx < level.ChildQty {
// pageNo := getPageNo(level.PageData, level.Idx)
// level.Idx++
// var buf []byte
// buf, err = s.fetchPage(pageNo)
// if err != nil {
// return nil, fmt.Errorf("fetchPage(%d): %s", pageNo, err)
// }
// pageNumbers = append(pageNumbers, pageNo)
// if buf[pageTypeIdx] == PageTypeData {
// //pageNumbers := listPageNumbers(buf)
// //dataPages = append(dataPages, pageNumbers...)
// s.releasePage(pageNo)
// } else {
// childQty, _ = bin.GetUint16(buf[indexRecordsQtyIdx:])
// levels = append(levels, &Level{
// PageNo: pageNo,
// PageData: buf,
// Idx: 0,
// ChildQty: int(childQty),
// })
// }
// } else {
// s.releasePage(level.PageNo)
// levels = levels[:lastIdx]
// }
// }
return
}
// const (

167
atree/cursor.go
View File

@@ -5,8 +5,6 @@ import (
"fmt"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/bin"
"gordenko.dev/dima/qb/enc"
)
type BackwardCursor struct {
@@ -66,50 +64,50 @@ func (s *BackwardCursor) Prev() (uint32, float64, bool, error) {
var (
timestamp uint32
value float64
done bool
err error
//done bool
//err error
)
timestamp, done = s.timestampDecompressor.NextValue()
if !done {
value, done = s.valueDecompressor.NextValue()
if done {
return 0, 0, false,
fmt.Errorf("corrupted data page %d: has timestamp, no value",
s.pageNo)
}
return timestamp, value, false, nil
}
// timestamp, done = s.timestampDecompressor.NextValue()
// if !done {
// value, done = s.valueDecompressor.NextValue()
// if done {
// return 0, 0, false,
// fmt.Errorf("corrupted data page %d: has timestamp, no value",
// s.pageNo)
// }
// return timestamp, value, false, nil
// }
prevPageNo := bin.GetUint32(s.pageData[prevPageIdx:])
if prevPageNo == 0 {
return 0, 0, true, nil
}
s.atree.releasePage(s.pageNo)
// prevPageNo, _ := bin.GetUint32(s.pageData[prevPageIdx:])
// if prevPageNo == 0 {
// return 0, 0, true, nil
// }
// s.atree.releasePage(s.pageNo)
s.pageNo = prevPageNo
s.pageData, err = s.atree.fetchDataPage(s.pageNo)
if err != nil {
return 0, 0, false, fmt.Errorf("atree.fetchDataPage(%d): %s", s.pageNo, err)
}
// s.pageNo = prevPageNo
// s.pageData, err = s.atree.fetchDataPage(s.pageNo)
// if err != nil {
// return 0, 0, false, fmt.Errorf("atree.fetchDataPage(%d): %s", s.pageNo, err)
// }
err = s.makeDecompressors()
if err != nil {
return 0, 0, false, err
}
// err = s.makeDecompressors()
// if err != nil {
// return 0, 0, false, err
// }
timestamp, done = s.timestampDecompressor.NextValue()
if done {
return 0, 0, false,
fmt.Errorf("corrupted data page %d: no timestamps",
s.pageNo)
}
value, done = s.valueDecompressor.NextValue()
if done {
return 0, 0, false,
fmt.Errorf("corrupted data page %d: no values",
s.pageNo)
}
// timestamp, done = s.timestampDecompressor.NextValue()
// if done {
// return 0, 0, false,
// fmt.Errorf("corrupted data page %d: no timestamps",
// s.pageNo)
// }
// value, done = s.valueDecompressor.NextValue()
// if done {
// return 0, 0, false,
// fmt.Errorf("corrupted data page %d: no values",
// s.pageNo)
// }
return timestamp, value, false, nil
}
@@ -120,68 +118,69 @@ func (s *BackwardCursor) Close() {
// HELPER
func (s *BackwardCursor) makeDecompressors() error {
timestampsPayloadSize := bin.GetUint16(s.pageData[timestampsSizeIdx:])
valuesPayloadSize := bin.GetUint16(s.pageData[valuesSizeIdx:])
// timestampsSize, _ := bin.GetUint16(s.pageData[timestampsSizeIdx:])
// valuesSize, _ := bin.GetUint16(s.pageData[valuesSizeIdx:])
payloadSize := timestampsPayloadSize + valuesPayloadSize
// payloadSize := timestampsSize + valuesSize
if payloadSize > dataFooterIdx {
return fmt.Errorf("corrupted data page %d: timestamps + values size %d gt payload size",
s.pageNo, payloadSize)
}
// if payloadSize > dataFooterIdx {
// return fmt.Errorf("corrupted data page %d: timestamps + values size %d gt payload size",
// s.pageNo, payloadSize)
// }
s.timestampDecompressor = enc.NewTimeDeltaDecompressor(
s.pageData[:timestampsPayloadSize],
)
// s.timestampDecompressor = enc.NewTimeDeltaDecompressor(
// s.pageData[:timestampsSize],
// )
vbuf := s.pageData[timestampsPayloadSize : timestampsPayloadSize+valuesPayloadSize]
// vbuf := s.pageData[timestampsSize : timestampsSize+valuesSize]
switch s.metricType {
case qb.Instant:
s.valueDecompressor = enc.NewInstantDeltaDecompressor(
vbuf, s.fracDigits)
// switch s.metricType {
// case qb.Instant:
// s.valueDecompressor = enc.NewInstantDeltaDecompressor(
// vbuf, s.fracDigits)
case qb.Cumulative:
s.valueDecompressor = enc.NewCumulativeDeltaDecompressor(
vbuf, s.fracDigits)
// case qb.Cumulative:
// s.valueDecompressor = enc.NewCumulativeDeltaDecompressor(
// vbuf, s.fracDigits)
default:
return fmt.Errorf("bug: wrong metricType %d", s.metricType)
}
// default:
// return fmt.Errorf("bug: wrong metricType %d", s.metricType)
// }
return nil
}
func makeDecompressors(pageData []byte, metricType qb.MetricType, fracDigits byte) (
qb.TimestampDecompressor, qb.ValueDecompressor, error,
) {
timestampsPayloadSize := bin.GetUint16(pageData[timestampsSizeIdx:])
valuesPayloadSize := bin.GetUint16(pageData[valuesSizeIdx:])
// timestampsSize, _ := bin.GetUint16(pageData[timestampsSizeIdx:])
// valuesSize, _ := bin.GetUint16(pageData[valuesSizeIdx:])
payloadSize := timestampsPayloadSize + valuesPayloadSize
// payloadSize := timestampsSize + valuesSize
if payloadSize > dataFooterIdx {
return nil, nil, fmt.Errorf("corrupted: timestamps + values size %d > payload size",
payloadSize)
}
// if payloadSize > dataFooterIdx {
// return nil, nil, fmt.Errorf("corrupted: timestamps + values size %d > payload size",
// payloadSize)
// }
timestampDecompressor := enc.NewTimeDeltaDecompressor(
pageData[:timestampsPayloadSize],
)
// timestampDecompressor := enc.NewTimeDeltaDecompressor(
// pageData[:timestampsSize],
// )
vbuf := pageData[timestampsPayloadSize : timestampsPayloadSize+valuesPayloadSize]
// vbuf := pageData[timestampsSize : timestampsSize+valuesSize]
var valueDecompressor qb.ValueDecompressor
switch metricType {
case qb.Instant:
valueDecompressor = enc.NewInstantDeltaDecompressor(
vbuf, fracDigits)
// var valueDecompressor qb.ValueDecompressor
// switch metricType {
// case qb.Instant:
// valueDecompressor = enc.NewInstantDeltaDecompressor(
// vbuf, fracDigits)
case qb.Cumulative:
valueDecompressor = enc.NewCumulativeDeltaDecompressor(
vbuf, fracDigits)
// case qb.Cumulative:
// valueDecompressor = enc.NewCumulativeDeltaDecompressor(
// vbuf, fracDigits)
default:
return nil, nil, fmt.Errorf("bug: wrong metricType %d", metricType)
}
return timestampDecompressor, valueDecompressor, nil
// default:
// return nil, nil, fmt.Errorf("bug: wrong metricType %d", metricType)
// }
//return timestampDecompressor, valueDecompressor, nil
return nil, nil, nil
}

167
atree/io.go
View File

@@ -7,8 +7,8 @@ import (
"math"
"os"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/bin"
"gordenko.dev/dima/qb/util"
)
@@ -35,55 +35,58 @@ func (s *Atree) DeletePages(pageNumbers []uint32) {
}
func (s *Atree) fetchIndexPage(pageNo uint32) ([]byte, error) {
buf, err := s.fetchPage(pageNo)
if err != nil {
return nil, err
}
if buf[pageTypeIdx] != PageTypeIndex {
return nil, fmt.Errorf("wrong pageType %d instead of %d", buf[pageTypeIdx], PageTypeIndex)
}
return buf, nil
// buf, err := s.fetchPage(pageNo)
// if err != nil {
// return nil, err
// }
// if buf[pageTypeIdx] != PageTypeIndex {
// return nil, fmt.Errorf("wrong pageType %d instead of %d", buf[pageTypeIdx], PageTypeIndex)
// }
// return buf, nil
return nil, nil
}
func (s *Atree) fetchDataPage(pageNo uint32) ([]byte, error) {
buf, err := s.fetchPage(pageNo)
if err != nil {
return nil, err
}
if buf[pageTypeIdx] != PageTypeIndex {
return nil, fmt.Errorf("wrong pageType %d instead of %d", buf[pageTypeIdx], PageTypeIndex)
}
return buf, nil
// buf, err := s.fetchPage(pageNo)
// if err != nil {
// return nil, err
// }
// if buf[pageTypeIdx] != PageTypeIndex {
// return nil, fmt.Errorf("wrong pageType %d instead of %d", buf[pageTypeIdx], PageTypeIndex)
// }
// return buf, nil
return nil, nil
}
func (s *Atree) fetchPage(pageNo uint32) ([]byte, error) {
s.mutex.Lock()
p, ok := s.pages[pageNo]
if ok {
p.ReferenceCount++
s.mutex.Unlock()
return p.Buf, nil
}
// s.mutex.Lock()
// p, ok := s.pages[pageNo]
// if ok {
// p.ReferenceCount++
// s.mutex.Unlock()
// return p.Buf, nil
// }
resultCh := make(chan readResult, 1)
s.pageWaits[pageNo] = append(s.pageWaits[pageNo], resultCh)
if len(s.pageWaits[pageNo]) == 1 {
s.pagesToRead = append(s.pagesToRead, pageNo)
s.mutex.Unlock()
// resultCh := make(chan readResult, 1)
// s.pageWaits[pageNo] = append(s.pageWaits[pageNo], resultCh)
// if len(s.pageWaits[pageNo]) == 1 {
// s.pagesToRead = append(s.pagesToRead, pageNo)
// s.mutex.Unlock()
select {
case s.readSignalCh <- struct{}{}:
default:
}
} else {
s.mutex.Unlock()
}
// select {
// case s.readSignalCh <- struct{}{}:
// default:
// }
// } else {
// s.mutex.Unlock()
// }
result := <-resultCh
if result.Err == nil {
result.Err = s.verifyCRC(result.Data, PageSize)
}
return result.Data, result.Err
// result := <-resultCh
// if result.Err == nil {
// result.Err = s.verifyCRC(result.Data, PageSize)
// }
// return result.Data, result.Err
return nil, nil
}
func (s *Atree) releasePage(pageNo uint32) {
@@ -211,49 +214,49 @@ func (s *Atree) pageReader() {
}
func (s *Atree) readPages() {
s.mutex.Lock()
if len(s.pagesToRead) == 0 {
s.mutex.Unlock()
return
}
pagesToRead := s.pagesToRead
s.pagesToRead = nil
s.mutex.Unlock()
// s.mutex.Lock()
// if len(s.pagesToRead) == 0 {
// s.mutex.Unlock()
// return
// }
// pagesToRead := s.pagesToRead
// s.pagesToRead = nil
// s.mutex.Unlock()
for _, pageNo := range pagesToRead {
buf := make([]byte, PageSize)
off := (pageNo - 1) * PageSize
n, err := s.file.ReadAt(buf, int64(off))
if n != PageSize {
err = fmt.Errorf("read %d instead of %d", n, PageSize)
}
// for _, pageNo := range pagesToRead {
// buf := make([]byte, PageSize)
// off := (pageNo - 1) * PageSize
// n, err := s.file.ReadAt(buf, int64(off))
// if n != PageSize {
// err = fmt.Errorf("read %d instead of %d", n, PageSize)
// }
s.mutex.Lock()
resultChannels := s.pageWaits[pageNo]
delete(s.pageWaits, pageNo)
// s.mutex.Lock()
// resultChannels := s.pageWaits[pageNo]
// delete(s.pageWaits, pageNo)
if err != nil {
s.mutex.Unlock()
for _, resultCh := range resultChannels {
resultCh <- readResult{
Err: err,
}
}
} else {
s.pages[pageNo] = &_page{
// fix - page type
PageNo: pageNo,
Buf: buf,
ReferenceCount: len(resultChannels),
}
s.mutex.Unlock()
for _, resultCh := range resultChannels {
resultCh <- readResult{
Data: buf,
}
}
}
}
// if err != nil {
// s.mutex.Unlock()
// for _, resultCh := range resultChannels {
// resultCh <- readResult{
// Err: err,
// }
// }
// } else {
// s.pages[pageNo] = &_page{
// // fix - page type
// PageNo: pageNo,
// Buf: buf,
// ReferenceCount: len(resultChannels),
// }
// s.mutex.Unlock()
// for _, resultCh := range resultChannels {
// resultCh <- readResult{
// Data: buf,
// }
// }
// }
// }
}
// WRITE
@@ -364,7 +367,7 @@ func (s *Atree) verifyCRC(data []byte, pageSize int) error {
var (
pos = pageSize - 4
calculatedCRC = util.CalcChecksum(data[:pos])
storedCRC = bin.GetUint32(data[pos:])
storedCRC, _ = bin.GetUint32(data[pos:])
)
if calculatedCRC != storedCRC {
return fmt.Errorf("calculatedCRC %d not equal storedCRC %d",

230
atree/misc.go
View File

@@ -1,8 +1,8 @@
package atree
import (
"gordenko.dev/dima/qb/bin"
)
type KeyComparator interface {
CompareTo(int) int
}
type ValueAtComparator struct {
buf []byte
@@ -10,21 +10,22 @@ type ValueAtComparator struct {
}
func (s ValueAtComparator) CompareTo(elemIdx int) int {
var (
pos = elemIdx * timestampSize
elem = bin.GetUint32(s.buf[pos:])
)
// var (
// pos = elemIdx * timestampSize
// elem, _ = bin.GetUint32(s.buf[pos:])
// )
if s.timestamp < elem {
return -1
} else if s.timestamp > elem {
return 1
} else {
return 0
}
// if s.timestamp < elem {
// return -1
// } else if s.timestamp > elem {
// return 1
// } else {
// return 0
// }
return 213131132
}
func BinarySearch(qty int, keyComparator bin.KeyComparator) (elemIdx int, isFound bool) {
func BinarySearch(qty int, keyComparator KeyComparator) (elemIdx int, isFound bool) {
if qty == 0 {
return
}
@@ -55,141 +56,94 @@ func BinarySearch(qty int, keyComparator bin.KeyComparator) (elemIdx int, isFoun
}
}
// type ChunksToNotLinkedDataPageReq struct {
// // PrevPageNo uint32
// TimestampsChunks [][]byte
// TimestampsSize uint16
// ValuesChunks [][]byte
// ValuesSize uint16
// func getPrevPageNo(buf []byte) uint32 {
// pageNo, _ := bin.GetUint32(buf[prevPageIdx:])
// return pageNo
// }
// func ChunksToNotLinkedDataPage(buf []byte, req ChunksToNotLinkedDataPageReq) {
// bin.PutUint16(buf[timestampsSizeIdx:], req.TimestampsSize)
// bin.PutUint16(buf[valuesSizeIdx:], req.ValuesSize)
// var (
// remainingSize = int(req.TimestampsSize)
// pos = 0
// )
// for _, chunk := range req.TimestampsChunks {
// if remainingSize >= len(chunk) {
// copy(buf[pos:], chunk)
// remainingSize -= len(chunk)
// pos += len(chunk)
// } else {
// copy(buf[pos:], chunk[:remainingSize])
// break
// }
// func findPageNo(buf []byte, timestamp uint32) (pageNo uint32) {
// comparator := ValueAtComparator{
// buf: buf,
// timestamp: timestamp,
// }
// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
// elemIdx, _ := BinarySearch(int(qty), comparator)
// pos := indexFooterIdx - (elemIdx+1)*PageNoSize
// pageNo, _ = bin.GetUint32(buf[pos:])
// return
// }
// remainingSize = int(req.ValuesSize)
// pos = int(req.TimestampsSize)
// for _, chunk := range req.ValuesChunks {
// if remainingSize >= len(chunk) {
// copy(buf[pos:], chunk)
// remainingSize -= len(chunk)
// pos += len(chunk)
// } else {
// copy(buf[pos:], chunk[:remainingSize])
// break
// }
// func findPageNoIdx(buf []byte, timestamp uint32) (idx int) {
// comparator := ValueAtComparator{
// buf: buf,
// timestamp: timestamp,
// }
// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
// elemIdx, _ := BinarySearch(int(qty), comparator)
// return elemIdx
// }
// func setPrevPageNo(buf []byte, pageNo uint32) {
// bin.PutUint32(buf[prevPageIdx:], pageNo)
// func findPageNoForDeleteSince(buf []byte, since uint32) (uint32, int, int) {
// comparator := ValueAtComparator{
// buf: buf,
// timestamp: since,
// }
// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
// elemIdx, _ := BinarySearch(int(qty), comparator)
// pos := elemIdx * timestampSize
// timestamp, _ := bin.GetUint32(buf[pos:])
// if timestamp == since {
// if elemIdx == 0 {
// return 0, int(qty), -1
// }
// elemIdx--
// }
// pos = indexFooterIdx - (elemIdx+1)*PageNoSize
// pageNo, _ := bin.GetUint32(buf[pos:])
// return pageNo, int(qty), elemIdx
// }
func getPrevPageNo(buf []byte) uint32 {
return bin.GetUint32(buf[prevPageIdx:])
}
// func getLastPageNo(buf []byte) (pageNo uint32) {
// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
// pos := indexFooterIdx - qty*PageNoSize
// pageNo, _ = bin.GetUint32(buf[pos:])
// return
// }
func findPageNo(buf []byte, timestamp uint32) (pageNo uint32) {
var (
qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:])
comparator = ValueAtComparator{
buf: buf,
timestamp: timestamp,
}
)
elemIdx, _ := BinarySearch(qty, comparator)
pos := indexFooterIdx - (elemIdx+1)*PageNoSize
return bin.GetUint32(buf[pos:])
}
// func getPageNo(buf []byte, idx int) (pageNo uint32) {
// pos := indexFooterIdx - (idx+1)*PageNoSize
// pageNo, _ = bin.GetUint32(buf[pos:])
// return
// }
func findPageNoIdx(buf []byte, timestamp uint32) (idx int) {
var (
qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:])
comparator = ValueAtComparator{
buf: buf,
timestamp: timestamp,
}
)
elemIdx, _ := BinarySearch(qty, comparator)
return elemIdx
}
// func listPageNumbers(buf []byte) (pageNumbers []uint32) {
// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
// pos := indexFooterIdx - PageNoSize
// for range qty {
// pageNo, _ := bin.GetUint32(buf[pos:])
// pageNumbers = append(pageNumbers, pageNo)
// pos -= PageNoSize
// }
// return
// }
func findPageNoForDeleteSince(buf []byte, since uint32) (uint32, int, int) {
var (
qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:])
comparator = ValueAtComparator{
buf: buf,
timestamp: since,
}
)
elemIdx, _ := BinarySearch(qty, comparator)
pos := elemIdx * timestampSize
timestamp := bin.GetUint32(buf[pos:])
if timestamp == since {
if elemIdx == 0 {
return 0, qty, -1
}
elemIdx--
}
pos = indexFooterIdx - (elemIdx+1)*PageNoSize
return bin.GetUint32(buf[pos:]), qty, elemIdx
}
func getLastPageNo(buf []byte) (pageNo uint32) {
qty := bin.GetUint16AsInt(buf[indexRecordsQtyIdx:])
pos := indexFooterIdx - qty*PageNoSize
return bin.GetUint32(buf[pos:])
}
func getPageNo(buf []byte, idx int) (pageNo uint32) {
pos := indexFooterIdx - (idx+1)*PageNoSize
return bin.GetUint32(buf[pos:])
}
func listPageNumbers(buf []byte) (pageNumbers []uint32) {
qty := bin.GetUint16AsInt(buf[indexRecordsQtyIdx:])
pos := indexFooterIdx - PageNoSize
for range qty {
pageNumbers = append(pageNumbers, bin.GetUint32(buf[pos:]))
pos -= PageNoSize
}
return
}
// include since timestamp
func listPageNumbersSince(buf []byte, timestamp uint32) (pageNumbers []uint32) {
var (
qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:])
comparator = ValueAtComparator{
buf: buf,
timestamp: timestamp,
}
)
elemIdx, _ := BinarySearch(qty, comparator)
pos := indexFooterIdx - (elemIdx+1)*PageNoSize
for range qty {
pageNumbers = append(pageNumbers, bin.GetUint32(buf[pos:]))
pos -= PageNoSize
}
return
}
// // include since timestamp
// func listPageNumbersSince(buf []byte, timestamp uint32) (pageNumbers []uint32) {
// comparator := ValueAtComparator{
// buf: buf,
// timestamp: timestamp,
// }
// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
// elemIdx, _ := BinarySearch(int(qty), comparator)
// pos := indexFooterIdx - (elemIdx+1)*PageNoSize
// for range qty {
// pageNo, _ := bin.GetUint32(buf[pos:])
// pageNumbers = append(pageNumbers, pageNo)
// pos -= PageNoSize
// }
// return
// }
// func getSince(buf []byte) uint32 {
// return bin.GetUint32(buf[0:])

8
atree/select.go
View File

@@ -3,7 +3,7 @@ package atree
import (
"fmt"
diploma "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb"
)
type ContinueFullScanReq struct {
@@ -23,7 +23,7 @@ func (s *Atree) ContinueFullScan(req ContinueFullScanReq) error {
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: diploma.Instant,
MetricType: qb.Instant,
})
if err != nil {
return err
@@ -60,7 +60,7 @@ func (s *Atree) ContinueRangeScan(req ContinueRangeScanReq) error {
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: diploma.Instant,
MetricType: qb.Instant,
})
if err != nil {
return err
@@ -101,7 +101,7 @@ func (s *Atree) RangeScan(req RangeScanReq) error {
PageData: buf,
Atree: s,
FracDigits: req.FracDigits,
MetricType: diploma.Instant,
MetricType: qb.Instant,
})
if err != nil {
return err

621
bin/bin.go
View File

@@ -1,621 +0,0 @@
package bin
import (
"errors"
"fmt"
"io"
"math"
)
const (
varInt64SignFlag = 0b01000000
maxReadAttempts = 5
)
var (
ErrNoSpace = errors.New("no space")
ErrIncompleteWrite = errors.New("incomplete write")
ErrReadOverflow = errors.New("bin: reader returned 'n' > bufsize")
// ErrNegativeReadCount shows 100% bug in the source reader
ErrNegativeReadCount = errors.New("bin: reader returned negative 'n'")
)
// READ
func ReadUint16(src io.Reader) (num uint16, err error) {
var (
q = 2
arr = make([]byte, q)
)
n, err := src.Read(arr)
if err != nil {
return
}
if n != q {
return 0, fmt.Errorf("read %d bytes only", n)
}
return GetUint16(arr), nil
}
func ReadUint24AsInt(src io.Reader) (num int, err error) {
var (
q = 3
arr = make([]byte, q)
)
n, err := src.Read(arr)
if err != nil {
return
}
if n != q {
return 0, fmt.Errorf("read %d bytes only", n)
}
return GetUint24AsInt(arr), nil
}
func ReadUint32(src io.Reader) (num uint32, err error) {
var (
q = 4
arr = make([]byte, q)
)
n, err := src.Read(arr)
if err != nil {
return
}
if n != q {
return 0, fmt.Errorf("read %d bytes only", n)
}
return GetUint32(arr), nil
}
func ReadUint64(src io.Reader) (num uint64, err error) {
var (
q = 8
arr = make([]byte, q)
)
n, err := src.Read(arr)
if err != nil {
return
}
if n != q {
return 0, fmt.Errorf("read %d bytes only", n)
}
return GetUint64(arr), nil
}
func ReadFloat64(src io.Reader) (num float64, err error) {
var (
q = 8
arr = make([]byte, q)
)
n, err := src.Read(arr)
if err != nil {
return
}
if n != q {
return 0, fmt.Errorf("read %d bytes only", n)
}
return GetFloat64(arr), nil
}
func ReadUnixtime(src io.Reader) (num int64, err error) {
var (
q = 8
arr = make([]byte, q)
)
n, err := src.Read(arr)
if err != nil {
return
}
if n != q {
return 0, fmt.Errorf("read %d bytes only", n)
}
return int64(GetUint64(arr)), nil
}
// READ VAR
func ReadVarUint64(src io.Reader) (num uint64, n int, err error) {
var (
p = make([]byte, 1)
b byte
)
for i := range 8 {
_, err = src.Read(p)
if err != nil {
return
}
n++
b = p[0]
if b >= 128 {
num |= uint64(b&127) << uint(i*7)
return
}
num |= uint64(b) << uint(i*7)
}
_, err = src.Read(p)
if err != nil {
return
}
n++
num |= uint64(p[0]) << 56
return
}
// GET
func GetUint16(arr []byte) uint16 {
return uint16(arr[0]) | (uint16(arr[1]) << 8)
}
func GetUint16AsInt(arr []byte) int {
return int(arr[0]) | (int(arr[1]) << 8)
}
func GetUint24AsInt(arr []byte) int {
return int(arr[0]) | (int(arr[1]) << 8) | (int(arr[2]) << 16)
}
func GetUint32(arr []byte) uint32 {
return uint32(arr[0]) | (uint32(arr[1]) << 8) |
(uint32(arr[2]) << 16) | (uint32(arr[3]) << 24)
}
func GetUint32AsInt64(arr []byte) int64 {
u32 := uint32(arr[0]) | (uint32(arr[1]) << 8) |
(uint32(arr[2]) << 16) | (uint32(arr[3]) << 24)
return int64(u32)
}
func GetUint40(arr []byte) uint64 {
return uint64(arr[0]) | (uint64(arr[1]) << 8) |
(uint64(arr[2]) << 16) | (uint64(arr[3]) << 24) |
(uint64(arr[4]) << 32)
}
func GetUint48(arr []byte) uint64 {
return uint64(arr[0]) | (uint64(arr[1]) << 8) |
(uint64(arr[2]) << 16) | (uint64(arr[3]) << 24) |
(uint64(arr[4]) << 32) | (uint64(arr[5]) << 40)
}
func GetUint64(arr []byte) uint64 {
return uint64(arr[0]) | (uint64(arr[1]) << 8) |
(uint64(arr[2]) << 16) | (uint64(arr[3]) << 24) |
(uint64(arr[4]) << 32) | (uint64(arr[5]) << 40) |
(uint64(arr[6]) << 48) | (uint64(arr[7]) << 56)
}
func GetFloat32(arr []byte) float32 {
return math.Float32frombits(GetUint32(arr))
}
func GetFloat64(arr []byte) float64 {
return math.Float64frombits(GetUint64(arr))
}
func GetUnixtime(arr []byte) int64 {
u32 := uint32(arr[0]) | (uint32(arr[1]) << 8) |
(uint32(arr[2]) << 16) | (uint32(arr[3]) << 24)
return int64(u32)
}
func GetVarUint64(arr []byte) (num uint64, n int, err error) {
var b byte
for i := range 8 {
if i >= len(arr) {
return 0, 0, io.EOF
}
b = arr[i]
if b >= 128 {
num |= uint64(b&127) << uint(i*7)
return num, i + 1, nil
}
num |= uint64(b) << uint(i*7)
}
if len(arr) < 9 {
return 0, 0, io.EOF
}
return num | uint64(arr[8])<<56, 9, nil
}
func ReverseGetVarUint64(arr []byte) (num uint64, n int, err error) {
var (
b byte
j = len(arr) - 1
)
for i := range 8 {
if j < 0 {
return 0, 0, io.EOF
}
b = arr[j]
if b >= 128 {
num |= uint64(b&127) << uint(i*7)
return num, i + 1, nil
}
num |= uint64(b) << uint(i*7)
j--
}
if j < 0 {
return 0, 0, io.EOF
}
return num | uint64(arr[j])<<56, 9, nil
}
func GetVarInt64(arr []byte) (num int64, n int, err error) {
u64, n, err := GetVarUint64(arr)
if err != nil {
return
}
return DecodeZigZag(u64), n, nil
}
func ReverseGetVarInt64(arr []byte) (num int64, n int, err error) {
u64, n, err := ReverseGetVarUint64(arr)
if err != nil {
return
}
return DecodeZigZag(u64), n, nil
}
// PUT
func PutUint16(arr []byte, num uint16) {
arr[0] = byte(num)
arr[1] = byte(num >> 8)
}
func PutIntAsUint16(arr []byte, num int) {
arr[0] = byte(num)
arr[1] = byte(num >> 8)
}
func PutIntAsUint24(arr []byte, num int) {
arr[0] = byte(num)
arr[1] = byte(num >> 8)
arr[2] = byte(num >> 16)
}
func PutUint32(arr []byte, num uint32) {
arr[0] = byte(num)
arr[1] = byte(num >> 8)
arr[2] = byte(num >> 16)
arr[3] = byte(num >> 24)
}
func PutInt64AsUint32(arr []byte, num int64) {
arr[0] = byte(num)
arr[1] = byte(num >> 8)
arr[2] = byte(num >> 16)
arr[3] = byte(num >> 24)
}
func PutUint40(arr []byte, num uint64) {
arr[0] = byte(num)
arr[1] = byte(num >> 8)
arr[2] = byte(num >> 16)
arr[3] = byte(num >> 24)
arr[4] = byte(num >> 32)
}
func PutUint48(arr []byte, num uint64) {
arr[0] = byte(num)
arr[1] = byte(num >> 8)
arr[2] = byte(num >> 16)
arr[3] = byte(num >> 24)
arr[4] = byte(num >> 32)
arr[5] = byte(num >> 40)
}
func PutUint64(arr []byte, num uint64) {
arr[0] = byte(num)
arr[1] = byte(num >> 8)
arr[2] = byte(num >> 16)
arr[3] = byte(num >> 24)
arr[4] = byte(num >> 32)
arr[5] = byte(num >> 40)
arr[6] = byte(num >> 48)
arr[7] = byte(num >> 56)
}
func PutFloat32(arr []byte, num float32) {
PutUint32(arr, math.Float32bits(num))
}
func PutFloat64(arr []byte, num float64) {
PutUint64(arr, math.Float64bits(num))
}
// WRITE
func WriteUint16(dst io.Writer, num uint16) error {
arr := []byte{
byte(num),
byte(num >> 8),
}
n, err := dst.Write(arr)
if err != nil {
return err
}
if n != 2 {
return ErrIncompleteWrite
}
return err
}
func WriteUint32(dst io.Writer, num uint32) error {
arr := []byte{
byte(num),
byte(num >> 8),
byte(num >> 16),
byte(num >> 24),
}
n, err := dst.Write(arr)
if err != nil {
return err
}
if n != 4 {
return ErrIncompleteWrite
}
return err
}
func WriteFloat64(dst io.Writer, num float64) error {
arr := make([]byte, 8)
PutUint64(arr, math.Float64bits(num))
n, err := dst.Write(arr)
if err != nil {
return err
}
if n != 2 {
return ErrIncompleteWrite
}
return err
}
// WRITE VAR
func WriteVarUint64(dst io.Writer, num uint64) (int, error) {
arr := make([]byte, 9)
for i := range 8 {
arr[i] = byte(num & 127)
num >>= 7
if num == 0 {
arr[i] |= 128
size := i + 1
n, err := dst.Write(arr[:size])
if err != nil {
return n, err
}
if n != size {
return n, ErrIncompleteWrite
}
return size, nil
}
}
arr[8] = byte(num)
n, err := dst.Write(arr)
if err != nil {
return n, err
}
if n != 9 {
return n, ErrIncompleteWrite
}
return 9, nil
}
func PutVarUint64(arr []byte, num uint64) (int, error) {
for i := range 8 {
if i >= len(arr) {
return 0, ErrNoSpace
}
arr[i] = byte(num & 127)
num >>= 7
if num == 0 {
arr[i] |= 128
return i + 1, nil
}
}
if len(arr) < 9 {
return 0, ErrNoSpace
}
arr[8] = byte(num)
return 9, nil
}
func ReversePutVarUint64(arr []byte, num uint64) (int, error) {
var tmp [9]byte
for i := range 8 {
tmp[i] = byte(num & 127)
num >>= 7
if num == 0 {
tmp[i] |= 128
n := i + 1
if len(arr) < n {
return 0, ErrNoSpace
}
for j := i; j >= 0; j-- {
arr[i-j] = tmp[j]
}
return n, nil
}
}
tmp[8] = byte(num)
n := 9
if len(arr) < n {
return 0, ErrNoSpace
}
for j := 8; j >= 0; j-- {
arr[8-j] = tmp[j]
}
return n, nil
}
func PutVarUint64AtEnd(arr []byte, num uint64) (int, error) {
var (
tmp [9]byte
n int
)
for i := range 8 {
tmp[i] = byte(num & 127)
num >>= 7
if num == 0 {
tmp[i] |= 128
n = i + 1
break
}
}
if n == 0 {
tmp[8] = byte(num)
n = 9
}
if len(arr) < n {
return 0, ErrNoSpace
}
j := len(arr) - n
for i := range n {
arr[j] = tmp[i]
j++
}
return n, nil
}
func PutVarInt64(arr []byte, x int64) (int, error) {
return PutVarUint64(arr, EncodeZigZag(x))
}
func PutVarInt64AtEnd(arr []byte, x int64) (int, error) {
return PutVarUint64AtEnd(arr, EncodeZigZag(x))
}
func ReversePutVarInt64(arr []byte, x int64) (int, error) {
return ReversePutVarUint64(arr, EncodeZigZag(x))
}
type KeyComparator interface {
CompareTo(int) int
}
func BinarySearch(qty int, keyComparator KeyComparator) (elemIdx int, isFound bool) {
if qty == 0 {
return
}
a := 0
b := qty - 1
for {
var (
elemIdx = (b-a)/2 + a
code = keyComparator.CompareTo(elemIdx)
)
if code == 1 {
a = elemIdx + 1
if a > b {
return elemIdx + 1, false
}
} else if code == -1 {
b = elemIdx - 1
if b < a {
return elemIdx, false
}
} else {
return elemIdx, true
}
}
}
func DeleteReverseArrElem(arr []byte, qty int, elemSize int, idx int) {
dstIdx := len(arr) - idx*elemSize - 1
srcIdx := dstIdx - elemSize
end := len(arr) - qty*elemSize
for ; srcIdx >= end; srcIdx-- {
arr[dstIdx] = arr[srcIdx]
dstIdx--
}
for i := end; i < end+elemSize; i++ {
arr[i] = 0
}
}
// ZigZag
// ZigZag encoding: int64 -> uint64
func EncodeZigZag(x int64) uint64 {
return uint64(x<<1) ^ uint64(x>>63)
}
// ZigZag decoding: uint64 -> int64
func DecodeZigZag(u uint64) int64 {
return int64(u>>1) ^ -(int64(u & 1))
}
func ReadN(r io.Reader, n int) (_ []byte, err error) {
if n < 0 {
err = fmt.Errorf("wrong n=%d", n)
return
}
buf := make([]byte, n)
err = ReadNInto(r, buf)
if err != nil {
return
}
return buf, nil
}
func ReadNInto(r io.Reader, buf []byte) (err error) {
if len(buf) == 0 {
return
}
var q, total, readAttempts int
for readAttempts < maxReadAttempts {
bufsize := len(buf) - total
q, err = r.Read(buf[total:])
if q == bufsize {
return nil
}
if err != nil {
return
}
if q > bufsize {
err = ErrReadOverflow
return
}
if q < 0 {
err = ErrNegativeReadCount
return
}
if q == 0 {
readAttempts++
} else {
total += q
}
}
err = io.ErrNoProgress
return
}
func CalcVarUint64Length(num uint64) int {
for i := range 8 {
num >>= 7
if num == 0 {
return i + 1
}
}
return 9
}
func CalcVarInt64Length(num int64) int {
u64 := EncodeZigZag(num)
for i := range 8 {
u64 >>= 7
if u64 == 0 {
return i + 1
}
}
return 9
}

521
client/client.go
View File

@@ -4,8 +4,8 @@ import (
"fmt"
"net"
diploma "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/bin"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/bufreader"
"gordenko.dev/dima/qb/proto"
)
@@ -33,7 +33,6 @@ func Connect(address string) (*Connection, error) {
if err != nil {
return nil, err
}
return &Connection{
conn: conn,
src: bufreader.New(conn, 1500),
@@ -51,22 +50,19 @@ func (s *Connection) Close() {
func (s *Connection) mustSuccess(reader *bufreader.BufferedReader) (err error) {
code, err := reader.ReadByte()
if err != nil {
return fmt.Errorf("read response code: %s", err)
return
}
switch code {
case proto.RespSuccess:
return nil // ok
case proto.RespError:
return s.onError()
default:
return fmt.Errorf("unknown reponse code %d", code)
}
}
func (s *Connection) AddMetric(req proto.AddMetricReq) error {
func (s *Connection) AddMetric(req proto.AddMetricReq) (err error) {
arr := []byte{
proto.TypeAddMetric,
0, 0, 0, 0, //
@@ -74,9 +70,9 @@ func (s *Connection) AddMetric(req proto.AddMetricReq) error {
byte(req.FracDigits),
}
bin.PutUint32(arr[1:], req.MetricID)
if _, err := s.conn.Write(arr); err != nil {
return err
// req
if _, err = s.conn.Write(arr); err != nil {
return
}
return s.mustSuccess(s.src)
}
@@ -96,52 +92,57 @@ func (s *Connection) AddMetric(req proto.AddMetricReq) error {
// return s.mustSuccess(s.src)
// }
func (s *Connection) GetMetric(metricID uint32) (*proto.Metric, error) {
func (s *Connection) GetMetric(metricID uint32) (_ *proto.Metric, err error) {
arr := []byte{
proto.TypeGetMetric,
0, 0, 0, 0,
}
bin.PutUint32(arr[1:], metricID)
if _, err := s.conn.Write(arr); err != nil {
return nil, err
// req
if _, err = s.conn.Write(arr); err != nil {
return
}
// answer
code, err := s.src.ReadByte()
if err != nil {
return nil, fmt.Errorf("read response code: %s", err)
return
}
switch code {
case proto.RespValue:
arr, err := s.src.ReadN(6)
var (
metric proto.Metric
metricType byte
)
metric.MetricID, err = bin.ReadUint32(s.src)
if err != nil {
return nil, fmt.Errorf("read body: %s", err)
return
}
return &proto.Metric{
MetricID: bin.GetUint32(arr),
MetricType: diploma.MetricType(arr[4]),
FracDigits: int(arr[5]),
}, nil
metricType, err = bin.ReadByte(s.src)
if err != nil {
return
}
metric.MetricType = qb.MetricType(metricType)
metric.FracDigits, err = bin.ReadByteAsInt(s.src)
if err != nil {
return
}
return &metric, nil
case proto.RespError:
return nil, s.onMaybeError()
default:
return nil, fmt.Errorf("unknown reponse code %d", code)
}
}
func (s *Connection) DeleteMetric(metricID uint32) error {
func (s *Connection) DeleteMetric(metricID uint32) (err error) {
arr := []byte{
proto.TypeDeleteMetric,
0, 0, 0, 0, //
}
bin.PutUint32(arr[1:], metricID)
if _, err := s.conn.Write(arr); err != nil {
return err
// req
if _, err = s.conn.Write(arr); err != nil {
return
}
return s.mustSuccess(s.src)
}
@@ -156,9 +157,9 @@ func (s *Connection) AppendMeasure(req proto.AppendMeasureReq) (err error) {
bin.PutUint32(arr[1:], req.MetricID)
bin.PutUint32(arr[5:], req.Timestamp)
bin.PutFloat64(arr[9:], req.Value)
if _, err := s.conn.Write(arr); err != nil {
return err
// req
if _, err = s.conn.Write(arr); err != nil {
return
}
return s.mustSuccess(s.src)
}
@@ -181,11 +182,10 @@ func (s *Connection) AppendMeasures(req proto.AppendMeasuresReq) (err error) {
bin.PutFloat64(arr[pos+4:], measure.Value)
pos += recordSize
}
if _, err := s.conn.Write(arr); err != nil {
return err
// req
if _, err = s.conn.Write(arr); err != nil {
return
}
//fmt.Printf("encode measures: %d\n", len(req.Measures))
return s.mustSuccess(s.src)
}
@@ -213,26 +213,26 @@ func (s *Connection) AppendMeasurePerMetric(list []proto.MetricMeasure) (_ []pro
bin.PutFloat64(arr[pos+8:], item.Value)
pos += recordSize
}
if _, err := s.conn.Write(arr); err != nil {
return nil, err
// req
if _, err = s.conn.Write(arr); err != nil {
return
}
// answer
code, err := s.src.ReadByte()
if err != nil {
return nil, fmt.Errorf("read response code: %s", err)
return
}
switch code {
case proto.RespValue:
var (
qty uint16
count int
appendErrors []proto.AppendError
)
qty, err = bin.ReadUint16(s.src)
count, err = bin.ReadUint16AsInt(s.src)
if err != nil {
return
}
for range qty {
for range count {
var ae proto.AppendError
ae.MetricID, err = bin.ReadUint32(s.src)
if err != nil {
@@ -245,70 +245,26 @@ func (s *Connection) AppendMeasurePerMetric(list []proto.MetricMeasure) (_ []pro
appendErrors = append(appendErrors, ae)
}
return appendErrors, nil
default:
return nil, fmt.Errorf("unknown reponse code %d", code)
}
}
func (s *Connection) ListAllInstantMeasures(metricID uint32) ([]proto.InstantMeasure, error) {
func (s *Connection) ListAllInstantMeasures(metricID uint32) (_ []proto.InstantMeasure, err error) {
arr := []byte{
proto.TypeListAllInstantMeasures,
0, 0, 0, 0, // metricID
}
bin.PutUint32(arr[1:], metricID)
if _, err := s.conn.Write(arr); err != nil {
return nil, err
}
var (
result []proto.InstantMeasure
tmp = make([]byte, 12)
)
for {
code, err := s.src.ReadByte()
if err != nil {
return nil, fmt.Errorf("read response code: %s", err)
}
//fmt.Printf("code: %d\n", code)
switch code {
case proto.RespPartOfValue:
q, err := bin.ReadUint32(s.src)
if err != nil {
return nil, fmt.Errorf("read records qty: %s", err)
}
//fmt.Printf("q: %d\n", q)
for i := range int(q) {
err = bin.ReadNInto(s.src, tmp)
if err != nil {
return nil, fmt.Errorf("read record #%d: %s", i, err)
}
result = append(result, proto.InstantMeasure{
Timestamp: bin.GetUint32(tmp),
Value: bin.GetFloat64(tmp[4:]),
})
}
case proto.RespEndOfValue:
return result, nil
case proto.RespError:
return nil, s.onError()
default:
return nil, fmt.Errorf("unknown reponse code %d", code)
}
// req
if _, err = s.conn.Write(arr); err != nil {
return
}
// answer
return s.readInstantMeasures()
}
func (s *Connection) ListInstantMeasures(req proto.ListInstantMeasuresReq) ([]proto.InstantMeasure, error) {
func (s *Connection) ListInstantMeasures(req proto.ListInstantMeasuresReq) (_ []proto.InstantMeasure, err error) {
arr := []byte{
proto.TypeListInstantMeasures,
0, 0, 0, 0, // metricID
@@ -318,118 +274,66 @@ func (s *Connection) ListInstantMeasures(req proto.ListInstantMeasuresReq) ([]pr
bin.PutUint32(arr[1:], req.MetricID)
bin.PutUint32(arr[5:], req.Since)
bin.PutUint32(arr[9:], req.Until)
if _, err := s.conn.Write(arr); err != nil {
return nil, err
// req
if _, err = s.conn.Write(arr); err != nil {
return
}
// answer
return s.readInstantMeasures()
}
var (
result []proto.InstantMeasure
tmp = make([]byte, 12)
)
func (s *Connection) readInstantMeasures() (_ []proto.InstantMeasure, err error) {
var result []proto.InstantMeasure
for {
code, err := s.src.ReadByte()
var code byte
code, err = bin.ReadByte(s.src)
if err != nil {
return nil, fmt.Errorf("read response code: %s", err)
}
switch code {
case proto.RespPartOfValue:
q, err := bin.ReadUint32(s.src)
var count int
count, err = bin.ReadUint32AsInt(s.src)
if err != nil {
return nil, fmt.Errorf("read records qty: %s", err)
}
for i := range int(q) {
err = bin.ReadNInto(s.src, tmp)
for range count {
var measure proto.InstantMeasure
measure.Timestamp, err = bin.ReadUint32(s.src)
if err != nil {
return nil, fmt.Errorf("read record #%d: %s", i, err)
return
}
result = append(result, proto.InstantMeasure{
Timestamp: bin.GetUint32(tmp),
Value: bin.GetFloat64(tmp[4:]),
})
measure.Value, err = bin.ReadFloat64(s.src)
if err != nil {
return
}
result = append(result, measure)
}
case proto.RespEndOfValue:
return result, nil
case proto.RespError:
return nil, s.onError()
default:
return nil, fmt.Errorf("unknown reponse code %d", code)
}
}
}
func (s *Connection) ListAllCumulativeMeasures(metricID uint32) ([]proto.CumulativeMeasure, error) {
func (s *Connection) ListAllCumulativeMeasures(metricID uint32) (_ []proto.CumulativeMeasure, err error) {
arr := []byte{
proto.TypeListAllCumulativeMeasures,
0, 0, 0, 0, // metricID
}
bin.PutUint32(arr[1:], metricID)
if _, err := s.conn.Write(arr); err != nil {
return nil, err
}
var (
result []proto.CumulativeMeasure
tmp = make([]byte, 20)
)
for {
code, err := s.src.ReadByte()
if err != nil {
return nil, fmt.Errorf("read response code: %s", err)
}
//fmt.Printf("code: %d\n", code)
switch code {
case proto.RespPartOfValue:
q, err := bin.ReadUint32(s.src)
if err != nil {
return nil, fmt.Errorf("read records qty: %s", err)
}
for i := range int(q) {
err = bin.ReadNInto(s.src, tmp)
if err != nil {
return nil, fmt.Errorf("read record #%d: %s", i, err)
}
//fmt.Printf("tmp: %d\n", tmp)
result = append(result, proto.CumulativeMeasure{
Timestamp: bin.GetUint32(tmp),
Value: bin.GetFloat64(tmp[4:]),
Total: bin.GetFloat64(tmp[12:]),
})
// pretty.PPrintln("measure", CumulativeMeasure{
// Timestamp: bin.GetUint32(tmp),
// Value: bin.GetFloat64(tmp[4:]),
// Total: bin.GetFloat64(tmp[12:]),
// })
}
case proto.RespEndOfValue:
return result, nil
case proto.RespError:
return nil, s.onError()
default:
return nil, fmt.Errorf("unknown reponse code %d", code)
}
// req
if _, err = s.conn.Write(arr); err != nil {
return
}
// answer
return s.readCumulativeMeasures()
}
func (s *Connection) ListCumulativeMeasures(req proto.ListCumulativeMeasuresReq) ([]proto.CumulativeMeasure, error) {
func (s *Connection) ListCumulativeMeasures(req proto.ListCumulativeMeasuresReq) (_ []proto.CumulativeMeasure, err error) {
arr := []byte{
proto.TypeListCumulativeMeasures,
0, 0, 0, 0, // metricID
@@ -439,55 +343,56 @@ func (s *Connection) ListCumulativeMeasures(req proto.ListCumulativeMeasuresReq)
bin.PutUint32(arr[1:], req.MetricID)
bin.PutUint32(arr[5:], req.Since)
bin.PutUint32(arr[9:], req.Until)
if _, err := s.conn.Write(arr); err != nil {
return nil, err
// req
if _, err = s.conn.Write(arr); err != nil {
return
}
// answer
return s.readCumulativeMeasures()
}
var (
result []proto.CumulativeMeasure
tmp = make([]byte, 20)
)
func (s *Connection) readCumulativeMeasures() (_ []proto.CumulativeMeasure, err error) {
var result []proto.CumulativeMeasure
for {
code, err := s.src.ReadByte()
var code byte
code, err = bin.ReadByte(s.src)
if err != nil {
return nil, fmt.Errorf("read response code: %s", err)
}
switch code {
case proto.RespPartOfValue:
q, err := bin.ReadUint32(s.src)
var count int
count, err = bin.ReadUint32AsInt(s.src)
if err != nil {
return nil, fmt.Errorf("read records qty: %s", err)
}
for i := range int(q) {
err = bin.ReadNInto(s.src, tmp)
for range count {
var measure proto.CumulativeMeasure
measure.Timestamp, err = bin.ReadUint32(s.src)
if err != nil {
return nil, fmt.Errorf("read record #%d: %s", i, err)
return
}
result = append(result, proto.CumulativeMeasure{
Timestamp: bin.GetUint32(tmp),
Value: bin.GetFloat64(tmp[4:]),
Total: bin.GetFloat64(tmp[12:]),
})
measure.Value, err = bin.ReadFloat64(s.src)
if err != nil {
return
}
measure.Total, err = bin.ReadFloat64(s.src)
if err != nil {
return
}
result = append(result, measure)
}
case proto.RespEndOfValue:
return result, nil
case proto.RespError:
return nil, s.onError()
default:
return nil, fmt.Errorf("unknown reponse code %d", code)
}
}
}
func (s *Connection) ListInstantPeriods(req proto.ListInstantPeriodsReq) ([]proto.InstantPeriod, error) {
func (s *Connection) ListInstantPeriods(req proto.ListInstantPeriodsReq) (_ []proto.InstantPeriod, err error) {
arr := []byte{
proto.TypeListInstantPeriods,
0, 0, 0, 0, // metricID
@@ -504,87 +409,70 @@ func (s *Connection) ListInstantPeriods(req proto.ListInstantPeriodsReq) ([]prot
bin.PutUint16(arr[9:], uint16(req.Until.Year))
arr[11] = byte(req.Until.Month)
arr[12] = byte(req.Until.Day)
if _, err := s.conn.Write(arr); err != nil {
return nil, err
// req
if _, err = s.conn.Write(arr); err != nil {
return
}
var q int
if (req.AggregateFuncs & diploma.AggregateMin) == diploma.AggregateMin {
q++
}
if (req.AggregateFuncs & diploma.AggregateMax) == diploma.AggregateMax {
q++
}
if (req.AggregateFuncs & diploma.AggregateAvg) == diploma.AggregateAvg {
q++
}
var (
result []proto.InstantPeriod
// 12 bytes - period, since, until
// q * 8 bytes - min, max, avg
tmp = make([]byte, 12+q*8)
)
// answer
var result []proto.InstantPeriod
for {
code, err := s.src.ReadByte()
var code byte
code, err = s.src.ReadByte()
if err != nil {
return nil, fmt.Errorf("read response code: %s", err)
return
}
//fmt.Printf("code: %d\n", code)
switch code {
case proto.RespPartOfValue:
q, err := bin.ReadUint32(s.src)
var count int
count, err = bin.ReadUint32AsInt(s.src)
if err != nil {
return nil, fmt.Errorf("read records qty: %s", err)
return
}
for i := range int(q) {
err = bin.ReadNInto(s.src, tmp)
for range count {
var p proto.InstantPeriod
p.Period, err = bin.ReadUint32(s.src)
if err != nil {
return nil, fmt.Errorf("read record #%d: %s", i, err)
return
}
var (
p = proto.InstantPeriod{
Period: bin.GetUint32(tmp[0:]),
Since: bin.GetUint32(tmp[4:]),
Until: bin.GetUint32(tmp[8:]),
p.Since, err = bin.ReadUint32(s.src)
if err != nil {
return
}
p.Until, err = bin.ReadUint32(s.src)
if err != nil {
return
}
if (req.AggregateFuncs & qb.AggregateMin) == qb.AggregateMin {
p.Min, err = bin.ReadFloat64(s.src)
if err != nil {
return
}
// 12 bytes - period, since, until
pos = 12
)
if (req.AggregateFuncs & diploma.AggregateMin) == diploma.AggregateMin {
p.Min = bin.GetFloat64(tmp[pos:])
pos += 8
}
if (req.AggregateFuncs & diploma.AggregateMax) == diploma.AggregateMax {
p.Max = bin.GetFloat64(tmp[pos:])
pos += 8
if (req.AggregateFuncs & qb.AggregateMax) == qb.AggregateMax {
p.Max, err = bin.ReadFloat64(s.src)
if err != nil {
return
}
}
if (req.AggregateFuncs & diploma.AggregateAvg) == diploma.AggregateAvg {
p.Avg = bin.GetFloat64(tmp[pos:])
if (req.AggregateFuncs & qb.AggregateAvg) == qb.AggregateAvg {
p.Avg, err = bin.ReadFloat64(s.src)
if err != nil {
return
}
}
result = append(result, p)
}
case proto.RespEndOfValue:
return result, nil
case proto.RespError:
return nil, s.onError()
default:
return nil, fmt.Errorf("unknown reponse code %d", code)
}
}
}
func (s *Connection) ListCumulativePeriods(req proto.ListCumulativePeriodsReq) ([]proto.CumulativePeriod, error) {
func (s *Connection) ListCumulativePeriods(req proto.ListCumulativePeriodsReq) (_ []proto.CumulativePeriod, err error) {
arr := []byte{
proto.TypeListCumulativePeriods,
0, 0, 0, 0, // metricID
@@ -600,108 +488,107 @@ func (s *Connection) ListCumulativePeriods(req proto.ListCumulativePeriodsReq) (
bin.PutUint16(arr[9:], uint16(req.Until.Year))
arr[11] = byte(req.Until.Month)
arr[12] = byte(req.Until.Day)
if _, err := s.conn.Write(arr); err != nil {
return nil, err
// req
if _, err = s.conn.Write(arr); err != nil {
return
}
var (
result []proto.CumulativePeriod
tmp = make([]byte, 28)
)
// answer
var result []proto.CumulativePeriod
for {
code, err := s.src.ReadByte()
var code byte
code, err = s.src.ReadByte()
if err != nil {
return nil, fmt.Errorf("read response code: %s", err)
return
}
switch code {
case proto.RespPartOfValue:
q, err := bin.ReadUint32(s.src)
var count int
count, err = bin.ReadUint32AsInt(s.src)
if err != nil {
return nil, fmt.Errorf("read records qty: %s", err)
return
}
for i := range int(q) {
err = bin.ReadNInto(s.src, tmp)
for range count {
var p proto.CumulativePeriod
p.Period, err = bin.ReadUint32(s.src)
if err != nil {
return nil, fmt.Errorf("read record #%d: %s", i, err)
return
}
result = append(result, proto.CumulativePeriod{
Period: bin.GetUint32(tmp[0:]),
Since: bin.GetUint32(tmp[4:]),
Until: bin.GetUint32(tmp[8:]),
EndValue: bin.GetFloat64(tmp[12:]),
Total: bin.GetFloat64(tmp[20:]),
})
p.Since, err = bin.ReadUint32(s.src)
if err != nil {
return
}
p.Until, err = bin.ReadUint32(s.src)
if err != nil {
return
}
p.EndValue, err = bin.ReadFloat64(s.src)
if err != nil {
return
}
p.Total, err = bin.ReadFloat64(s.src)
if err != nil {
return
}
result = append(result, p)
}
case proto.RespEndOfValue:
return result, nil
case proto.RespError:
return nil, s.onError()
default:
return nil, fmt.Errorf("unknown reponse code %d", code)
}
}
}
func (s *Connection) ListCurrentValues(metricIDs []uint32) ([]proto.CurrentValue, error) {
func (s *Connection) ListCurrentValues(metricIDs []uint32) (_ []proto.CurrentValue, err error) {
arr := make([]byte, 3+metricKeySize*len(metricIDs))
arr[0] = proto.TypeListCurrentValues
bin.PutUint16(arr[1:], uint16(len(metricIDs)))
off := 3
for _, metricID := range metricIDs {
bin.PutUint32(arr[off:], metricID)
off += metricKeySize
}
if _, err := s.conn.Write(arr); err != nil {
return nil, err
// req
if _, err = s.conn.Write(arr); err != nil {
return
}
var (
result []proto.CurrentValue
tmp = make([]byte, 16)
)
// answer
var result []proto.CurrentValue
for {
code, err := s.src.ReadByte()
var code byte
code, err = s.src.ReadByte()
if err != nil {
return nil, fmt.Errorf("read response code: %s", err)
return
}
switch code {
case proto.RespPartOfValue:
q, err := bin.ReadUint32(s.src)
var count int
count, err = bin.ReadUint32AsInt(s.src)
if err != nil {
return nil, fmt.Errorf("read records qty: %s", err)
return
}
for i := range int(q) {
err = bin.ReadNInto(s.src, tmp)
for range count {
var m proto.CurrentValue
m.MetricID, err = bin.ReadUint32(s.src)
if err != nil {
return nil, fmt.Errorf("read record #%d: %s", i, err)
return
}
result = append(result, proto.CurrentValue{
MetricID: bin.GetUint32(tmp),
Timestamp: bin.GetUint32(tmp[4:]),
Value: bin.GetFloat64(tmp[8:]),
})
m.Timestamp, err = bin.ReadUint32(s.src)
if err != nil {
return
}
m.Value, err = bin.ReadFloat64(s.src)
if err != nil {
return
}
result = append(result, m)
}
case proto.RespEndOfValue:
return result, nil
case proto.RespError:
return nil, s.onError()
default:
return nil, fmt.Errorf("unknown reponse code %d", code)
}
@@ -716,7 +603,7 @@ func (s *Connection) DeleteMeasures(req proto.DeleteMeasuresReq) (err error) {
}
bin.PutUint32(arr[1:], req.MetricID)
bin.PutUint32(arr[5:], req.Since)
// req
if _, err := s.conn.Write(arr); err != nil {
return err
}

460
conbuf/conbuf.go
View File

@@ -1,460 +0,0 @@
package conbuf
import (
"errors"
"fmt"
"io"
"gordenko.dev/dima/qb/bin"
)
const chunkSize = 512
const ChunkSize = 512
var (
ErrOutOfRange = errors.New("out of range")
)
type ContinuousBuffer struct {
chunks [][]byte
}
func NewFromBuffer(buf []byte) *ContinuousBuffer {
var (
chunks [][]byte
copied = 0
)
for copied < len(buf) {
chunk := make([]byte, chunkSize)
end := min(copied+chunkSize, len(buf))
copy(chunk, buf[copied:end])
chunks = append(chunks, chunk)
copied += end - copied
}
return &ContinuousBuffer{
chunks: chunks,
}
}
func New(chunks [][]byte) *ContinuousBuffer {
for i, chunk := range chunks {
if len(chunk) != chunkSize {
panic(fmt.Sprintf("wrong chunk #%d size %d", i, len(chunk)))
}
}
return &ContinuousBuffer{
chunks: chunks,
}
}
func (s *ContinuousBuffer) Chunks() [][]byte {
return s.chunks
}
// [0, pos)
func (s *ContinuousBuffer) GetByte(idx int) byte {
chunkIdx := idx / chunkSize
if chunkIdx >= len(s.chunks) {
panic(ErrOutOfRange)
}
byteIdx := idx % chunkSize
return s.chunks[chunkIdx][byteIdx]
}
func (s *ContinuousBuffer) SetByte(idx int, b byte) {
chunkIdx := idx / chunkSize
if chunkIdx > len(s.chunks) {
panic(ErrOutOfRange)
}
if chunkIdx == len(s.chunks) {
s.chunks = append(s.chunks, make([]byte, chunkSize))
}
byteIdx := idx % chunkSize
s.chunks[chunkIdx][byteIdx] = b
}
func (s *ContinuousBuffer) SetFlag(idx int, flag byte) {
chunkIdx := idx / chunkSize
if chunkIdx > len(s.chunks) {
panic(ErrOutOfRange)
}
if chunkIdx == len(s.chunks) {
s.chunks = append(s.chunks, make([]byte, chunkSize))
}
byteIdx := idx % chunkSize
s.chunks[chunkIdx][byteIdx] |= flag
}
func (s *ContinuousBuffer) UnsetFlag(idx int, flag byte) {
chunkIdx := idx / chunkSize
if chunkIdx > len(s.chunks) {
panic(ErrOutOfRange)
}
if chunkIdx == len(s.chunks) {
s.chunks = append(s.chunks, make([]byte, chunkSize))
}
byteIdx := idx % chunkSize
s.chunks[chunkIdx][byteIdx] &^= flag
}
// [since, until)
func (s *ContinuousBuffer) ShiftOnePosToRight(since int, until int) {
if since < 0 {
panic("since < 0")
}
if since >= until {
panic("since >= until")
}
chunkIdx := until / chunkSize
byteIdx := until % chunkSize
if chunkIdx > len(s.chunks) {
panic(ErrOutOfRange)
}
if chunkIdx == len(s.chunks) {
if byteIdx == 0 {
s.chunks = append(s.chunks, make([]byte, chunkSize))
} else {
panic(ErrOutOfRange)
}
}
var (
qty = until - since
prevChunkIdx int
prevByteIdx int
)
for range qty {
prevChunkIdx = chunkIdx
prevByteIdx = byteIdx - 1
if prevByteIdx < 0 {
prevChunkIdx = chunkIdx - 1
prevByteIdx = chunkSize - 1
}
s.chunks[chunkIdx][byteIdx] = s.chunks[prevChunkIdx][prevByteIdx]
if byteIdx > 0 {
byteIdx--
} else {
chunkIdx--
byteIdx = chunkSize - 1
}
}
}
// [since, until)
func (s *ContinuousBuffer) ShiftOnePosToLeft(since int, until int) {
if since <= 0 {
panic("since <= 0")
}
if since >= until {
panic("since >= until")
}
chunkIdx := since / chunkSize
byteIdx := since % chunkSize
if until > len(s.chunks)*chunkSize {
panic(ErrOutOfRange)
}
var (
qty = until - since
prevChunkIdx int
prevByteIdx int
)
for range qty {
prevChunkIdx = chunkIdx
prevByteIdx = byteIdx - 1
if prevByteIdx < 0 {
prevChunkIdx = chunkIdx - 1
prevByteIdx = chunkSize - 1
}
s.chunks[prevChunkIdx][prevByteIdx] = s.chunks[chunkIdx][byteIdx]
byteIdx++
if byteIdx == chunkSize {
chunkIdx++
byteIdx = 0
}
}
}
func (s *ContinuousBuffer) PutUint32(pos int, num uint32) {
s.CopyTo(pos,
[]byte{
byte(num),
byte(num >> 8),
byte(num >> 16),
byte(num >> 24),
})
}
func (s *ContinuousBuffer) GetUint32(pos int) uint32 {
arr := s.Slice(pos, pos+4)
return uint32(arr[0]) | (uint32(arr[1]) << 8) |
(uint32(arr[2]) << 16) | (uint32(arr[3]) << 24)
}
func (s *ContinuousBuffer) ReversePutVarUint64(pos int, num uint64) int {
var tmp [9]byte
for i := range 8 {
tmp[i] = byte(num & 127)
num >>= 7
if num == 0 {
tmp[i] |= 128
n := i + 1
s.ReverseCopyTo(pos, tmp[:n])
return n
}
}
tmp[8] = byte(num)
n := 9
s.ReverseCopyTo(pos, tmp[:])
return n
}
func (s *ContinuousBuffer) ReverseGetVarUint64(idx int) (num uint64, n int, err error) {
chunkIdx := idx / chunkSize
if chunkIdx >= len(s.chunks) {
panic(ErrOutOfRange)
}
var (
byteIdx = idx % chunkSize
chunk = s.chunks[chunkIdx]
b byte
)
for i := range 8 {
b = chunk[byteIdx]
if b >= 128 {
num |= uint64(b&127) << uint(i*7)
return num, i + 1, nil
}
num |= uint64(b) << uint(i*7)
if byteIdx > 0 {
byteIdx--
} else {
if chunkIdx == 0 {
return 0, 0, io.EOF
} else {
chunkIdx--
chunk = s.chunks[chunkIdx]
byteIdx = chunkSize - 1
}
}
}
return num | uint64(chunk[byteIdx])<<56, 9, nil
}
func (s *ContinuousBuffer) ReversePutVarInt64(pos int, x int64) int {
return s.ReversePutVarUint64(pos, bin.EncodeZigZag(x))
}
func (s *ContinuousBuffer) ReverseGetVarInt64(idx int) (int64, int, error) {
u64, n, err := s.ReverseGetVarUint64(idx)
if err != nil {
return 0, 0, err
}
return bin.DecodeZigZag(u64), n, nil
}
func (s *ContinuousBuffer) PutVarUint64(pos int, num uint64) int {
var tmp [9]byte
for i := range 8 {
tmp[i] = byte(num & 127)
num >>= 7
if num == 0 {
tmp[i] |= 128
n := i + 1
s.CopyTo(pos, tmp[:n])
return n
}
}
tmp[8] = byte(num)
s.CopyTo(pos, tmp[:])
return 9
}
func (s *ContinuousBuffer) GetVarUint64(idx int) (num uint64, n int, err error) {
chunkIdx := idx / chunkSize
if chunkIdx >= len(s.chunks) {
panic(ErrOutOfRange)
}
var (
byteIdx = idx % chunkSize
chunk = s.chunks[chunkIdx]
b byte
)
for i := range 8 {
b = chunk[byteIdx]
if b >= 128 {
num |= uint64(b&127) << uint(i*7)
return num, i + 1, nil
}
num |= uint64(b) << uint(i*7)
byteIdx++
if byteIdx == chunkSize {
chunkIdx++
if chunkIdx == len(s.chunks) {
return 0, 0, io.EOF
}
chunk = s.chunks[chunkIdx]
byteIdx = 0
}
}
return num | uint64(chunk[byteIdx])<<56, 9, nil
}
func (s *ContinuousBuffer) PutVarInt64(idx int, x int64) int {
return s.PutVarUint64(idx, bin.EncodeZigZag(x))
}
func (s *ContinuousBuffer) GetVarInt64(idx int) (int64, int, error) {
u64, n, err := s.GetVarUint64(idx)
if err != nil {
return 0, 0, err
}
return bin.DecodeZigZag(u64), n, nil
}
func (s *ContinuousBuffer) CopyTo(idx int, data []byte) {
chunkIdx := idx / chunkSize
if chunkIdx > len(s.chunks) {
panic(ErrOutOfRange)
}
if chunkIdx == len(s.chunks) {
s.chunks = append(s.chunks, make([]byte, chunkSize))
}
byteIdx := idx % chunkSize
chunk := s.chunks[chunkIdx]
copied := 0
for _, b := range data {
chunk[byteIdx] = b
copied++
byteIdx++
if byteIdx == chunkSize {
byteIdx = 0
chunkIdx++
if chunkIdx == len(s.chunks) {
if copied == len(data) {
return
}
s.chunks = append(s.chunks, make([]byte, chunkSize))
}
chunk = s.chunks[chunkIdx]
}
}
}
func (s *ContinuousBuffer) ReverseCopyTo(idx int, data []byte) {
chunkIdx := idx / chunkSize
if chunkIdx > len(s.chunks) {
panic(ErrOutOfRange)
}
if chunkIdx == len(s.chunks) {
s.chunks = append(s.chunks, make([]byte, chunkSize))
}
byteIdx := idx % chunkSize
chunk := s.chunks[chunkIdx]
copied := 0
for i := len(data) - 1; i >= 0; i-- {
chunk[byteIdx] = data[i]
copied++
byteIdx++
if byteIdx == chunkSize {
byteIdx = 0
chunkIdx++
if chunkIdx == len(s.chunks) {
if copied == len(data) {
return
}
s.chunks = append(s.chunks, make([]byte, chunkSize))
}
chunk = s.chunks[chunkIdx]
}
}
}
// [since, until)
func (s *ContinuousBuffer) Slice(since int, until int) []byte {
if since >= until {
return nil
}
size := len(s.chunks) * chunkSize
if until >= size {
panic(ErrOutOfRange)
}
data := make([]byte, until-since)
chunkIdx := since / chunkSize
byteIdx := since % chunkSize
chunk := s.chunks[chunkIdx]
for i := range len(data) {
data[i] = chunk[byteIdx]
byteIdx++
if byteIdx == chunkSize {
byteIdx = 0
chunkIdx++
chunk = s.chunks[chunkIdx]
}
}
return data
}
func (s *ContinuousBuffer) DecodeRunLength(pos int) (length uint16, n int) {
b1 := s.GetByte(pos)
pos--
if b1 < 128 {
length = uint16(b1)
n = 1
} else {
b2 := s.GetByte(pos)
length = uint16(b1&127) | (uint16(b2) << 7)
n = 2
}
length += 2
return
}
func (s *ContinuousBuffer) Copy() *ContinuousBuffer {
var copies [][]byte
for _, chunk := range s.chunks {
buf := make([]byte, chunkSize)
copy(buf, chunk)
copies = append(copies, buf)
}
return New(copies)
}
// size to copy
func (s *ContinuousBuffer) CopyChunksToOneBuffer(dst []byte, size int) {
pos := 0
for _, chunk := range s.chunks {
if size >= len(chunk) {
copy(dst[pos:], chunk)
size -= len(chunk)
pos += len(chunk)
} else {
copy(dst[pos:], chunk[:size])
return
}
}
}

50
database/api.go
View File

@@ -6,13 +6,13 @@ import (
"io"
"net"
qb "gordenko.dev/dima/qb"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/bin"
"gordenko.dev/dima/qb/bufreader"
"gordenko.dev/dima/qb/proto"
"gordenko.dev/dima/qb/storage"
"gordenko.dev/dima/qb/transform"
"gordenko.dev/dima/qb/txlog"
)
var (
@@ -213,7 +213,7 @@ func (s *Database) AddMetric(req proto.AddMetricReq) uint16 {
switch resultCode {
case Succeed:
s.txlog.Append(txlog.AddedMetric{
s.storage.Append(storage.MetricAddRecord{
MetricID: req.MetricID,
MetricType: req.MetricType,
FracDigits: req.FracDigits,
@@ -286,20 +286,20 @@ func (s *Database) DeleteMetric(req proto.DeleteMetricReq) uint16 {
switch result.ResultCode {
case Succeed:
var (
freePageNumbers []uint32
)
if result.RootPageNo > 0 {
var err error
freePageNumbers, err = s.atree.GetAllPages(result.RootPageNo)
if err != nil {
qb.Abort(qb.FailedAtreeRequest, err)
}
}
s.txlog.Append(txlog.DeletedMetric{
MetricID: req.MetricID,
FreePageNumbers: freePageNumbers,
})
// var (
// //freePageNumbers []uint32
// )
// if result.RootPageNo > 0 {
// var err error
// freePageNumbers, err = s.atree.GetAllPages(result.RootPageNo)
// if err != nil {
// qb.Abort(qb.FailedAtreeRequest, err)
// }
// }
// s.storage.Append(storage.DeletedMetric{
// MetricID: req.MetricID,
// //FreePageNumbers: freePageNumbers, FIX
// })
//<-waitCh
case NoMetric:
@@ -343,7 +343,7 @@ type FilledPage struct {
// switch result.ResultCode {
// case CanAppend:
// waitCh := s.txlog.Append(txlog.AppendedMeasure{
// waitCh := s.storage.Append(storage.AppendedMeasure{
// MetricID: req.MetricID,
// Timestamp: req.Timestamp,
// Value: req.Value,
@@ -390,9 +390,9 @@ type FilledPage struct {
// // }
// // _ = report
// //waitCh := s.txlog.WriteAppended(path, report)
// //waitCh := s.storage.WriteAppended(path, report)
// waitCh := s.txlog.Append(txlog.AppendedPagesReq{
// waitCh := s.storage.Append(storage.AppendedPagesReq{
// Legs: path.Legs,
// MetricID: req.MetricID,
// Timestamp: req.Timestamp,
@@ -459,8 +459,8 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
// _ = report
// if len(toAppendMeasures) > 0 {
// waitCh := s.txlog.WriteAppendMeasures(
// txlog.AppendedMeasures{
// waitCh := s.storage.WriteAppendMeasures(
// storage.AppendedMeasures{
// MetricID: req.MetricID,
// Measures: toAppendMeasures,
// },
@@ -500,7 +500,7 @@ func (s *Database) DeleteMeasures(req proto.DeleteMeasuresReq) uint16 {
case DeleteFromAtreeNotNeeded:
// регистрирую удаление в TransactionLog
s.txlog.Append(txlog.DeletedMeasures{
s.storage.Append(storage.DeletedMeasures{
MetricID: req.MetricID,
})
//<-waitCh
@@ -512,7 +512,7 @@ func (s *Database) DeleteMeasures(req proto.DeleteMeasuresReq) uint16 {
qb.Abort(qb.FailedAtreeRequest, err)
}
// регистрирую удаление в TransactionLog
s.txlog.Append(txlog.DeletedMeasures{
s.storage.Append(storage.DeletedMeasures{
MetricID: req.MetricID,
FreePageNumbers: pageNumbers,
})

62
database/database.go
View File

@@ -12,7 +12,7 @@ import (
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/freelist"
"gordenko.dev/dima/qb/txlog"
"gordenko.dev/dima/qb/storage"
)
func JoinSnapshotFileName(dir string, snapshotNumber int) string {
@@ -41,7 +41,7 @@ type Database struct {
dir string
databaseName string
snapshotNumber int
txlog *txlog.Writer
storage *storage.Writer
atree *atree.Atree
tcpPort int
logfile *os.File
@@ -186,7 +186,7 @@ func (s *Database) recovery() {
// logNumber = recipe.LogNumber
// }
// s.txlog, err = txlog.NewWriter(txlog.WriterOptions{
// s.storage, err = storage.NewWriter(storage.WriterOptions{
// Dir: s.dir,
// LogNumber: logNumber,
// AppendToWorkerQueue: s.appendJobToWorkerQueue,
@@ -199,7 +199,7 @@ func (s *Database) recovery() {
// qb.Abort(qb.CreateChangesWriterFailed, err)
// }
// go s.txlog.Run()
// go s.storage.Run()
// // fileNames, err := s.searchREDOFiles()
// // if err != nil {
@@ -249,7 +249,7 @@ func (s *Database) replayChanges(snapshotNumber int) (err error) {
return
}
walReader, err := txlog.NewReader(txlog.ReaderOptions{
walReader, err := storage.NewWALReader(storage.WALReaderOptions{
FileName: JoinWALFileName(s.dir, snapshotNumber),
BufferSize: 8 * 1024 * 1024,
})
@@ -271,29 +271,21 @@ func (s *Database) replayChanges(snapshotNumber int) (err error) {
if err != nil {
return
}
err = s.storage.WritePagesToAtree(walReplayer.IndexPages(), walReplayer.DataPages())
if err != nil {
return err
}
// FIX free pages sync
return nil
}
func (s *Database) rewritePagesFromLastWALPacket(list []any) error {
// var (
// indexPages []txlog.PageToWrite
// dataPages []txlog.PageToWrite
// )
// for _, untyped := range list {
// switch rec := untyped.(type) {
// case txlog.WALRecordAppendMeasures:
// metric, ok := s.metrics[rec.MetricID]
// if !ok {
// qb.Abort(qb.MetricNotFoundDuringReplay, nil)
// }
// //metric.ReplayAppendedMeasures(rec)
// default:
// qb.Abort(qb.UnknownTxLogRecordTypeBug,
// fmt.Errorf("bug: unknown record type %T in TransactionLog", rec))
// }
// }
return nil
func (s *Database) relayMetricsToMetrics(replayMetrics map[uint32]*ReplayMetric) {
for metricID, x := range replayMetrics {
s.metrics[metricID] = x.ToMetric()
}
}
// FIX
@@ -314,14 +306,14 @@ func calculateDataBufferSize(payloadSize int) int {
return bufferSize
}
}
return atree.DataPageSize
return storage.DataPageSize
}
func allocateBuffers(requiredSpace int) (buf, databuf []byte) {
bufferSize := calculateDataBufferSize(requiredSpace)
buf = make([]byte, bufferSize)
if bufferSize == atree.DataPageSize {
databuf = buf[:atree.DataPagePayloadSize]
if bufferSize == storage.DataPageSize {
databuf = buf[:storage.DataPagePayloadSize]
} else {
databuf = buf
}
@@ -343,6 +335,20 @@ func growBuffers(in growBuffersIn) (buf, databuf []byte) {
return
}
type shrinkBufferIn struct {
Buffer []byte
TimestampsSize int
ValuesSize int
NewBufferSize int
}
func shrinkBuffer(in shrinkBufferIn) []byte {
buf := make([]byte, in.NewBufferSize)
copy(buf, in.Buffer[:in.ValuesSize])
copy(buf[:len(buf)-in.TimestampsSize], in.Buffer[len(in.Buffer)-in.TimestampsSize:])
return buf
}
//func (s *Database) verifySnapshot(fileName string) (_ bool, err error) {
// file, err := os.Open(fileName)
// if err != nil {

24
database/database_test.go
View File

@@ -6,6 +6,7 @@ import (
"testing"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/storage"
)
func TestWriteReadSnapshot(t *testing.T) {
@@ -87,3 +88,26 @@ func cmpMetric(replay *ReplayMetric, origin *_metric) error {
// }
return nil
}
func TestComposeHeadIndexPage(t *testing.T) {
var (
levelIdx = 0
level = storage.IndexLevelTail{
Buffer: []byte{
1, 2, 3,
},
RecordsCount: 2,
}
head = &storage.HeadIndexPage{
PageNo: 100,
Checksum: 12345,
Records: []byte{},
}
)
page := composeHeadIndexPage(levelIdx, level, head)
if page.PageNo != head.PageNo {
t.Fatalf("PageNo: got %d are not equal expected %v",
page.PageNo, head.PageNo)
}
// fix compare pages
}

8
database/helpers.go
View File

@@ -6,17 +6,17 @@ import (
"os"
"time"
diploma "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/proto"
)
func timeBoundsOfAggregation(since, until proto.TimeBound, groupBy diploma.GroupBy, firstHourOfDay int) (s time.Time, u time.Time) {
func timeBoundsOfAggregation(since, until proto.TimeBound, groupBy qb.GroupBy, firstHourOfDay int) (s time.Time, u time.Time) {
switch groupBy {
case diploma.GroupByHour, diploma.GroupByDay:
case qb.GroupByHour, qb.GroupByDay:
s = time.Date(since.Year, since.Month, since.Day, 0, 0, 0, 0, time.Local)
u = time.Date(until.Year, until.Month, until.Day, 0, 0, 0, 0, time.Local)
case diploma.GroupByMonth:
case qb.GroupByMonth:
s = time.Date(since.Year, since.Month, 1, 0, 0, 0, 0, time.Local)
u = time.Date(until.Year, until.Month, 1, 0, 0, 0, 0, time.Local)
}

39
database/metric.go
View File

@@ -5,8 +5,7 @@ import (
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/txlog"
"gordenko.dev/dima/qb/storage"
)
// METRIC
@@ -31,7 +30,7 @@ type _metric struct {
XLock bool
RLocks int
WaitQueue []any
indexLevelTails []txlog.IndexLevelTail // root - last element
indexLevelTails []storage.IndexLevelTail // root - last element
}
//IndexLevels [][]IndexRec // root - last element
@@ -60,7 +59,7 @@ func (s *_metric) DeleteMeasures() {
// s.UntilValue = 0
}
// func (s *_metric) startAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(txlog.AppendedMeasures)) {
// func (s *_metric) startAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(storage.AppendedMeasures)) {
func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(any)) {
var (
// AppendedMeasures struct {
@@ -78,7 +77,7 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage fu
timestamps = s.timestamps
values = s.values
dataPages []txlog.DataPayload
dataPages []storage.DataPayload
//written int
//resultCode byte
@@ -92,7 +91,7 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage fu
// s.Since = measure.Timestamp
// } else {
// FIX - у випадку помилки треба в транзакції зберегти що помилка, але також зафіксувати скільки елементів збережено.
// якщо idx == 0 - одразу знімаю блокування і нічого не відправляю в txlog
// якщо idx == 0 - одразу знімаю блокування і нічого не відправляю в storage
if measure.Timestamp <= s.timestamps.LastTimestamp() {
if idx == 0 {
s.values.ForgetCapturedState()
@@ -136,7 +135,7 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage fu
// накопичую
timestamps.Compress(timestampCompressionWay, measure.Timestamp)
values.Compress(valueCompressionWay, measure.Value)
} else if len(s.buffer) < atree.DataPagePayloadSize {
} else if len(s.buffer) < storage.DataPagePayloadSize {
// allocate bigger buffer
buffer := make([]byte, len(s.buffer)*2)
// copy timestamps
@@ -151,7 +150,7 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage fu
// сторінка заповнена
since := s.timestamps.ReplaceSinceWithUntil()
dataPages = append(dataPages, txlog.DataPayload{
dataPages = append(dataPages, storage.DataPayload{
Since: since,
Content: s.buffer,
TimestampsSize: timestamps.Size(),
@@ -180,8 +179,8 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage fu
// скопіювати. Причому можна скопіювати зрізи chunks
if len(dataPages) > 0 {
// пишу в txlog довгим шляхом через redo файл і запис в data файл
sendToStorage(txlog.AppendedMeasures{
// пишу в storage довгим шляхом через redo файл і запис в data файл
sendToStorage(storage.AppendedMeasures{
MetricID: req.MetricID,
LastPageNo: s.lastPageNo,
TimestampsOffset: timestamps.Offset(), // state.Pos() з якої позиції дописувати дані на сторінку 0 (при відновленні)
@@ -196,12 +195,12 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage fu
ResultCh: nil,
})
} else {
// короткий шлях - запис лише в txlog
// короткий шлях - запис лише в storage
}
}
func (s *_metric) FinAppendMeasures(rec txlog.AppendMeasuresSummary) {
func (s *_metric) FinAppendMeasures(rec storage.AppendMeasuresSummary) {
// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
s.values.ForgetCapturedState()
s.timestamps.ForgetCapturedState()
@@ -209,7 +208,7 @@ func (s *_metric) FinAppendMeasures(rec txlog.AppendMeasuresSummary) {
if rec.LastPageNo > 0 {
s.lastPageNo = rec.LastPageNo
}
// В txlog я передав повний індекс. У нього додали елементи (можливо нові рівні).
// В storage я передав повний індекс. У нього додали елементи (можливо нові рівні).
// Тому проста заміна
s.indexLevelTails = rec.Index
}
@@ -388,20 +387,6 @@ func (s *_metric) WriteTo(w io.Writer) (err error) {
return
}
// var (
// values qb.ValueCompressor
// )
// if s.metricType == qb.Cumulative {
// values = enc.NewCumulativeDeltaCompressor(s.buffer, s.fracDigits)
// } else {
// values = enc.NewInstantDeltaCompressor(s.buffer, s.fracDigits)
// }
// values.RestoreState(valuesSize)
// s.timestamps = enc.NewTimeDeltaCompressor(s.buffer)
// s.timestamps.RestoreState(timestampsSize, until)
// s.values = values
// s.lastValue = s.values.LastValue()
// since - 4b
// sinceValue - 8b -
// untilValue - 8b

46
database/proc.go
View File

@@ -7,8 +7,8 @@ import (
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/enc"
"gordenko.dev/dima/qb/proto"
"gordenko.dev/dima/qb/storage"
"gordenko.dev/dima/qb/transform"
"gordenko.dev/dima/qb/txlog"
)
// AddMetric - create metric in map and set Xlock, rwad - append to queue
@@ -83,7 +83,7 @@ func (s *Database) DoWork() {
case tryAppendMeasuresReq:
s.tryAppendMeasures(req)
case txlog.Changes:
case storage.Changes:
s.applyChanges(req) // all metrics only
case tryListCurrentValuesReq:
@@ -132,7 +132,7 @@ func (s *Database) processMetricQueue(metricID uint32, metric *_metric) {
s.tryGetMetric(req)
case tryAppendMeasuresReq:
metric.StartAppendMeasures(req, s.txlog.Append)
metric.StartAppendMeasures(req, s.storage.Append)
case tryDeleteMetricReq:
s.startDeleteMetric(metric, req)
@@ -294,7 +294,7 @@ func (s *Database) startDeleteMeasures(metric *_metric, req tryDeleteMeasuresReq
// }
}
func (s *Database) finAppendMeasures(rec txlog.AppendMeasuresSummary) {
func (s *Database) finAppendMeasures(rec storage.AppendMeasuresSummary) {
metric, ok := s.metrics[rec.MetricID]
if !ok {
qb.Abort(qb.NoMetricBug,
@@ -322,7 +322,7 @@ func (s *Database) tryAppendMeasures(req tryAppendMeasuresReq) {
metric.WaitQueue = append(metric.WaitQueue, req)
return
}
metric.StartAppendMeasures(req, s.txlog.Append)
metric.StartAppendMeasures(req, s.storage.Append)
}
type tryRangeScanReq struct {
@@ -408,25 +408,25 @@ func (s *Database) tryListCurrentValues(req tryListCurrentValuesReq) {
///////////////////////////////////////////////////////
func (s *Database) applyChanges(req txlog.Changes) {
func (s *Database) applyChanges(req storage.Changes) {
for _, untyped := range req.Records {
switch rec := untyped.(type) {
case txlog.AddedMetric:
case storage.MetricAddRecord:
s.applyAddMetric(rec)
case txlog.DeletedMetric:
case storage.MetricDeleteRecord:
s.deleteMetric(rec)
// case txlog.AppendedMeasure:
// case storage.AppendedMeasure:
// s.appendMeasure(rec)
case txlog.AppendMeasuresSummary:
case storage.AppendMeasuresSummary:
s.finAppendMeasures(rec)
// case txlog.AppendedMeasureWithOverflowExtended:
// case storage.AppendedMeasureWithOverflowExtended:
// s.appendMeasureAfterOverflow(rec)
case txlog.DeletedMeasures:
case storage.DeletedMeasures:
s.deleteMeasures(rec)
}
}
@@ -436,7 +436,7 @@ func (s *Database) applyChanges(req txlog.Changes) {
err := writeSnapshot(writeSnapshotIn{
SnapshotNumber: s.snapshotNumber,
Dir: s.dir,
WriteBufferSize: 1 * 1024 * 1024, // 1mb
WriteBufferSize: 4 * 1024 * 1024, // 1mb
Metrics: s.metrics,
FrozenIndexPagesCount: req.FrozenIndexPagesCount,
IndexPageNumbers: req.IndexPageNumbers,
@@ -451,7 +451,7 @@ func (s *Database) applyChanges(req txlog.Changes) {
}
}
func (s *Database) applyAddMetric(rec txlog.AddedMetric) {
func (s *Database) applyAddMetric(rec storage.MetricAddRecord) {
// fix lock
_, ok := s.metrics[rec.MetricID]
if ok {
@@ -470,28 +470,26 @@ func (s *Database) applyAddMetric(rec txlog.AddedMetric) {
// delete(s.metricLockEntries, rec.MetricID)
}
func (s *Database) addMetric(rec txlog.AddedMetric) {
func (s *Database) addMetric(rec storage.MetricAddRecord) {
var (
values qb.ValueCompressor
buffer = make([]byte, minBufferSize)
)
if rec.MetricType == qb.Cumulative {
values = enc.NewCumulativeDeltaCompressor(buffer, byte(rec.FracDigits))
values = enc.NewCumulativeDeltaCompressor(byte(rec.FracDigits), buffer, 0)
} else {
values = enc.NewInstantDeltaCompressor(buffer, byte(rec.FracDigits))
values = enc.NewInstantDeltaCompressor(byte(rec.FracDigits), buffer, 0)
}
s.metrics[rec.MetricID] = &_metric{
metricType: rec.MetricType,
fracDigits: byte(rec.FracDigits),
buffer: buffer,
timestamps: enc.NewTimeDeltaCompressor(buffer),
timestamps: enc.NewTimeDeltaCompressor(buffer, 0),
values: values,
}
}
func (s *Database) deleteMetric(rec txlog.DeletedMetric) {
func (s *Database) deleteMetric(rec storage.MetricDeleteRecord) {
metric, ok := s.metrics[rec.MetricID]
if !ok {
qb.Abort(qb.NoMetricBug,
@@ -552,7 +550,7 @@ func (s *Database) deleteMetric(rec txlog.DeletedMetric) {
}
delete(s.metrics, rec.MetricID)
// ADD in txlog
// ADD in storage
// if len(rec.FreePageNumbers) > 0 {
// s.freeList.AddPages(rec.FreePageNumbers)
// }
@@ -562,7 +560,7 @@ func (s *Database) deleteMetric(rec txlog.DeletedMetric) {
}
}
func (s *Database) deleteMeasures(rec txlog.DeletedMeasures) {
func (s *Database) deleteMeasures(rec storage.DeletedMeasures) {
metric, ok := s.metrics[rec.MetricID]
if !ok {
qb.Abort(qb.NoMetricBug,
@@ -577,7 +575,7 @@ func (s *Database) deleteMeasures(rec txlog.DeletedMeasures) {
}
metric.DeleteMeasures()
metric.XLock = false
// FIX add in txlog
// FIX add in storage
// if len(rec.FreePageNumbers) > 0 {
// s.freeList.AddPages(rec.FreePageNumbers)
// }

304
database/replay_metric.go
View File

@@ -1,12 +1,13 @@
package database
import (
"fmt"
"io"
bin "gordenko.dev/dima/bin/little"
qb "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/txlog"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/enc"
"gordenko.dev/dima/qb/storage"
)
type ReplayMetric struct {
@@ -17,7 +18,7 @@ type ReplayMetric struct {
databuf []byte // якщо розмір buf == DataPageSize, то databuf буде менше (мінус футер)
tSize int
vSize int
indexLevelTails []txlog.IndexLevelTail // root - last element
indexLevelTails []storage.IndexLevelTail // root - last element
}
func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) {
@@ -57,7 +58,7 @@ func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) {
}
for range levelsCount {
var (
buf = make([]byte, atree.IndexPageSize)
buf = make([]byte, storage.IndexPageSize)
count int
)
count, err = bin.ReadVarSize(r)
@@ -68,7 +69,7 @@ func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) {
if err != nil {
return
}
s.indexLevelTails = append(s.indexLevelTails, txlog.IndexLevelTail{
s.indexLevelTails = append(s.indexLevelTails, storage.IndexLevelTail{
Buffer: buf,
RecordsCount: count,
})
@@ -76,115 +77,256 @@ func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) {
return
}
// func appendNewRecordsToIndexTail(level storage.IndexLevelTail, records []byte) {
// }
// func countReusedIndexPages(changedIndexLevels []storage.ChangedIndexLevel) (count int) {
// return
// }
func (s *ReplayMetric) MeasuresAppend(rec storage.MeasuresAppendRecord) {
timestampsSize := rec.TimestampsOffset - len(rec.Timestamps)
// копіюю нові дані із WAL з урахуванням offset-ів
copy(s.databuf[rec.ValuesOffset:], rec.Values)
copy(s.databuf[len(s.databuf)-timestampsSize:], rec.Timestamps)
//
s.tSize = timestampsSize
s.vSize = rec.ValuesOffset - len(rec.Values)
}
type AppendMeasuresResult struct {
IndexPages []txlog.PageToWrite
DataPages []txlog.PageToWrite
IndexPages []storage.PageToWrite
DataPages []storage.PageToWrite
ReusedIndexPagesCount int
ReusedDataPagesCount int
}
func (s *ReplayMetric) AppendedMeasures(rec txlog.WALRecordAppendMeasures, pagesNeeded bool) (result AppendMeasuresResult) {
func (s *ReplayMetric) MeasuresAppendWithGrow(rec storage.MeasuresAppendWithGrowRecord, isLastPacket bool) (_ AppendMeasuresResult) {
var (
reusedIndexPages int
reusedDataPages int
indexPages []storage.PageToWrite
dataPages []storage.PageToWrite
)
// Додати в index level tails недостаючі дані, або замінити
for levelIdx, change := range rec.ChangedIndexLevels {
var (
head = change.HeadIndexPage
newRecordsCount = len(change.TailRecords) / indexRecordSize
)
// 1. набиваю сторінки для перезапису в index файлі
if isLastPacket {
if head != nil {
indexPages = append(indexPages,
composeHeadIndexPage(levelIdx, s.indexLevelTails[levelIdx], head))
}
for _, x := range change.IndexPages {
indexPages = append(indexPages, storage.PageToWrite{
PageNo: x.PageNo,
Content: x.Content,
})
}
}
// 2. Вношу зміни в поточні індексні хвости
if levelIdx < len(s.indexLevelTails) {
level := s.indexLevelTails[levelIdx]
if change.CompletedIndexPage != nil {
// попередній хвіст індексного рівня перетворився на сторінку,
if head != nil {
// попередній хвіст індексного рівня перетворився на head сторінку,
// отже TailRecords - це новий хвіст
// або копіюю дані, або алокую більший буфер?
copy(level.Buffer, change.TailRecords)
level.RecordsCount = len(change.TailRecords) / indexRecordSize
level.RecordsCount = newRecordsCount
} else {
// TailRecords - це нові дані, які треба додати
pos := level.RecordsCount * indexRecordSize
copy(level.Buffer[pos:], change.TailRecords)
level.RecordsCount += newRecordsCount
}
s.indexLevelTails[levelIdx] = level
} else {
var (
recordsCount = len(change.TailRecords) / indexRecordSize
buffer = make([]byte, atree.IndexPageSize)
)
copy(buffer, change.TailRecords)
s.indexLevelTails = append(s.indexLevelTails, txlog.IndexLevelTail{
Buffer: buffer,
RecordsCount: recordsCount,
// додаю новий індексний рівень
buf := make([]byte, storage.IndexPageSize)
copy(buf, change.TailRecords)
//
s.indexLevelTails = append(s.indexLevelTails, storage.IndexLevelTail{
Buffer: buf,
RecordsCount: newRecordsCount,
})
}
// 3. рахую кількість reused індексних сторінок
if head != nil {
if head.Reused {
reusedIndexPages++
}
}
for _, x := range change.IndexPages {
if x.Reused {
reusedIndexPages++
}
}
}
// 4. набиваю сторінки для перезапису в data файлі
if isLastPacket {
dataPages = append(dataPages, composeHeadDataPage(s.databuf, rec.HeadDataPage))
for _, x := range rec.DataPages {
dataPages = append(dataPages, storage.PageToWrite{
PageNo: x.PageNo,
Content: x.Content,
})
}
}
// Перезаписати в buffer timestamps та values. Якщо розмір буферу недостатній -
// спочатку створити більший
// рахую кількість reused дата сторінок
if rec.HeadDataPage.Reused {
reusedDataPages++
}
for _, x := range rec.DataPages {
if x.Reused {
reusedDataPages++
}
}
// HeadDataPage є полюбому, оскільки це транзакція із мінімум однією заповненою
// data сторінкою. Отже TailTimestamps і TailValues - це нові хвости data рівня.
copy(s.databuf, rec.TailValues)
pos := len(s.databuf) - len(rec.TailTimestamps)
copy(s.databuf[pos:], rec.TailTimestamps)
var (
// CompletedDataPage є полюбому, оскільки це Record із мінімум одною заповненою
// data сторінкою. Отже TailTimestamps і TailValues - це нові хвости data рівня.
s.tSize = len(rec.TailTimestamps)
s.vSize = len(rec.TailValues)
requiredSpace = len(rec.TailTimestamps) + len(rec.TailValues)
databuf []byte
)
if requiredSpace < len(s.buf) {
databuf = s.buf
// fix - update sizes?
if len(rec.DataPages) == 0 {
s.lastPageNo = rec.HeadDataPage.PageNo
} else {
var buf []byte
buf, databuf = allocateBuffers(requiredSpace)
copy(databuf, rec.TailValues)
pos := len(databuf) - len(rec.TailTimestamps)
copy(databuf[pos:], rec.TailTimestamps)
// fix - replace and remeber sizes?
s.buf = buf
s.lastPageNo = rec.DataPages[len(rec.DataPages)-1].PageNo
}
//rec.TailTimestamps
//rec.TailValues
return
return AppendMeasuresResult{
IndexPages: indexPages,
DataPages: dataPages,
ReusedIndexPagesCount: reusedIndexPages,
ReusedDataPagesCount: reusedDataPages,
}
}
func (s *ReplayMetric) ApplyHeadDataPage(p txlog.CompletedDataPage) {
// буфер має бути розміру сторінки
if len(s.buf) != atree.DataPageSize {
s.buf, s.databuf = growBuffers(growBuffersIn{
databuf: s.databuf,
tSize: s.tSize,
vSize: s.vSize,
requiredSpace: atree.DataPageSize,
// HELPERS
func composeHeadIndexPage(levelIdx int, level storage.IndexLevelTail, head *storage.HeadIndexPage) storage.PageToWrite {
// розраховую pos, з якого буду дописувати хвіст
pos := level.RecordsCount * indexRecordSize
// створюю копію сторінки
page := make([]byte, storage.IndexPageSize)
copy(page, level.Buffer[:pos]) // поточні дані
copy(page[pos:], head.Records)
// запечатати сторінку
calculatedCRC := storage.SealIndexPage(storage.SealIndexPageIn{
Content: page,
RecordsCount: level.RecordsCount + len(head.Records)/indexRecordSize,
ZeroLevel: levelIdx == 0,
})
// перевірка CRC
if calculatedCRC != head.Checksum {
qb.Abort(qb.WALReplayFailed,
fmt.Errorf("calculated CRC %d not equal expected %d of head page %d on index level %d",
calculatedCRC, head.Checksum, head.PageNo, levelIdx))
}
return storage.PageToWrite{
PageNo: head.PageNo,
Content: page,
}
}
func composeHeadDataPage(databuf []byte, head storage.HeadDataPage) storage.PageToWrite {
// створюю копію сторінки
var (
page = make([]byte, storage.DataPageSize)
timestampsSize = head.TimestampsOffset - len(head.Timestamps)
)
// копіюю поточні дані
copy(page, databuf)
// копіюю нові дані із WAL з урахуванням offset-ів
copy(page[head.ValuesOffset:], head.Values)
copy(page[len(page)-timestampsSize:], head.Timestamps)
// запечатати сторінку
calculatedCRC := storage.SealDataPage(storage.SealDataPageIn{
Content: page,
PrevPageNo: head.PrevPageNo,
TimestampsSize: timestampsSize,
ValuesSize: head.ValuesOffset + len(head.Values),
})
// перевірка CRC
if calculatedCRC != head.Checksum {
qb.Abort(qb.WALReplayFailed,
fmt.Errorf("calculated CRC %d not equal expected %d of head data page %d",
calculatedCRC, head.Checksum, head.PageNo))
}
return storage.PageToWrite{
PageNo: head.PageNo,
Content: page,
}
}
func (s *ReplayMetric) ToMetric() *_metric {
var (
values qb.ValueCompressor
requiredSpace = s.tSize + s.vSize
)
bufferSize := calculateDataBufferSize(requiredSpace)
if len(s.buf) > bufferSize {
s.buf = shrinkBuffer(shrinkBufferIn{
Buffer: s.buf,
TimestampsSize: s.tSize,
ValuesSize: s.vSize,
NewBufferSize: bufferSize,
})
}
// дописую дані по зміщенню
pos := len(s.databuf) - p.TimestampsOffset - len(p.Timestamps)
copy(s.databuf[pos:], p.Timestamps)
copy(s.databuf[p.ValuesOffset:], p.Values)
//
s.tSize = p.TimestampsOffset + len(p.Timestamps)
s.tSize = p.ValuesOffset + len(p.Values)
// fix - заполнить страницу
// fix - check checksum
if s.metricType == qb.Cumulative {
values = enc.NewCumulativeDeltaCompressor(s.fracDigits, s.buf, s.vSize)
} else {
values = enc.NewInstantDeltaCompressor(s.fracDigits, s.buf, s.vSize)
}
return &_metric{
metricType: s.metricType,
fracDigits: s.fracDigits,
lastPageNo: s.lastPageNo,
lastValue: values.LastValue(),
buffer: s.buf,
timestamps: enc.NewTimeDeltaCompressor(s.buf, s.tSize),
values: values,
indexLevelTails: s.indexLevelTails,
}
}
// func (s *ReplayMetric) ApplyHeadIndexPages(changedIndexLevels txlog.ChangedIndexLevel) {
// func (s *ReplayMetric) ApplyHeadDataPage(p storage.HeadDataPage) {
// // буфер має бути розміру сторінки
// if len(s.buf) != atree.DataPageSize {
// s.buf, s.databuf = growBuffers(growBuffersIn{
// databuf: s.databuf,
// tSize: s.tSize,
// vSize: s.vSize,
// requiredSpace: atree.DataPageSize,
// })
// }
// // дописую дані по зміщенню
// pos := len(s.databuf) - p.TimestampsOffset - len(p.Timestamps)
// copy(s.databuf[pos:], p.Timestamps)
// copy(s.databuf[p.ValuesOffset:], p.Values)
// //
// s.tSize = p.TimestampsOffset + len(p.Timestamps)
// s.tSize = p.ValuesOffset + len(p.Values)
// // fix - заполнить страницу
// // fix - check checksum
// }
// func (s *ReplayMetric) ApplyHeadIndexPages(changedIndexLevels storage.ChangedIndexLevel) {
// for levelIdx, changedLevel := range changedIndexLevels {
// p := changedLevel.CompletedIndexPage
// if p != nil {
// level := s.indexLevelTails[levelIdx]
// copy(level.Buffer[level.RecordsCount*indexRecordSize:], p.Records)
//
// // fix - заполнить страницу
// // fix - check checksum
// s.indexPages = append(s.indexPages, txlog.PageToWrite{
// PageNo: p.PageNo,
// Content: level.Buffer,
// })
// }
// for _, x := range changedLevel.IndexPages {
// s.indexPages = append(s.indexPages, txlog.PageToWrite{
// PageNo: x.PageNo,
// Content: x.Content,
// })
// }
//
// }
// }

6
database/snapshot.go
View File

@@ -8,7 +8,7 @@ import (
"path/filepath"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/storage"
"gordenko.dev/dima/qb/util"
)
@@ -177,11 +177,11 @@ func readSnapshot(fileName string) (out readSnapshotOut, err error) {
for range metricsQty {
var (
metricID uint32
buf = make([]byte, atree.DataPageSize)
buf = make([]byte, storage.DataPageSize)
// для простоти створюю буфер максимального розміру
metric = &ReplayMetric{
buf: buf,
databuf: buf[:atree.DataPagePayloadSize],
databuf: buf[:storage.DataPagePayloadSize],
}
)
metricID, err = bin.ReadUint32(src)

185
database/wal_replayer.go
View File

@@ -4,22 +4,20 @@ import (
"errors"
"fmt"
qb "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/txlog"
"gordenko.dev/dima/qb/storage"
)
type WALReplayer struct {
walReader *txlog.Reader
walReader *storage.WALReader
metrics map[uint32]*ReplayMetric
freeIndexPages []uint32
freeDataPages []uint32
indexPages []txlog.PageToWrite
dataPages []txlog.PageToWrite
indexPages []storage.PageToWrite
dataPages []storage.PageToWrite
}
type WALReplayerOptions struct {
WALReader *txlog.Reader
WALReader *storage.WALReader
Metrics map[uint32]*ReplayMetric // from snapshot
FreeIndexPages []uint32 // from snapshot
FreeDataPages []uint32 // from snapshot
@@ -40,122 +38,80 @@ func NewWALReplayer(opt WALReplayerOptions) (*WALReplayer, error) {
}, nil
}
func (s *WALReplayer) IndexPages() []storage.PageToWrite {
return s.indexPages
}
func (s *WALReplayer) DataPages() []storage.PageToWrite {
return s.dataPages
}
func (s *WALReplayer) Replay() error {
for {
records, isLast, err := s.walReader.NextPacket()
records, isLastPacket, err := s.walReader.NextPacket()
if err != nil {
return err
}
if isLast {
if len(records) > 0 {
var appendRecords []txlog.WALRecordAppendMeasures
for _, untyped := range records {
rec, ok := untyped.(txlog.WALRecordAppendMeasures)
if ok {
appendRecords = append(appendRecords, rec)
}
}
// prepare pages and rewrite
indexPages, dataPages := pagesToWriteFromWALRecords(metrics, appendRecords)
err = s.txlog.WritePagesToAtree(indexPages, dataPages)
if err != nil {
return err
}
}
// для останнього rec - повторюємо запис сторінок
// rewrite write pages
//s.txlog.
//return nil
}
for _, rec := range records {
if err = s.replayWALRecord(rec); err != nil {
if err = s.replayRecord(rec, isLastPacket); err != nil {
return err
}
}
}
}
// func (s *WALReplayer) replayPacket(records []any, isLastPacket bool) {
// for _, untyped := range records {
// switch rec := untyped.(type) {
// case txlog.WALRecordAppendMeasures:
// s.onWALRecordAppendMeasures(rec, isLastPacket)
// }
// }
// return
// }
func (s *WALReplayer) replayWALRecord(untyped any) (err error) {
func (s *WALReplayer) replayRecord(untyped any, isLastPacket bool) (err error) {
switch rec := untyped.(type) {
case txlog.AddedMetric:
if err = s.onAddedMetric(rec); err != nil {
case storage.MeasuresAppendRecord:
if err = s.onMeasuresAppend(rec); err != nil {
return
}
case txlog.DeletedMetric:
if err = s.onDeletedMetric(rec); err != nil {
case storage.MeasuresAppendWithGrowRecord:
if err = s.onMeasuresAppendWithGrow(rec, isLastPacket); err != nil {
return
}
case storage.MetricAddRecord:
if err = s.onMetricAdd(rec); err != nil {
return
}
case storage.MetricDeleteRecord:
if err = s.onMetricDelete(rec); err != nil {
return
}
// case storage.DeletedMeasures:
// metric, ok := s.metrics[rec.MetricID]
// if ok {
// metric.DeleteMeasures()
// if len(rec.FreePageNumbers) > 0 {
// s.freeList.AddPages(rec.FreePageNumbers)
// }
// }
// case txlog.AppendedMeasure:
// metric, ok := s.metrics[rec.MetricID]
// if ok {
// metric.Timestamps.Append(rec.Timestamp)
// metric.Values.Append(rec.Value)
// if metric.Since == 0 {
// metric.Since = rec.Timestamp
// metric.SinceValue = rec.Value
// }
// metric.Until = rec.Timestamp
// metric.UntilValue = rec.Value
// }
case txlog.WALRecordAppendMeasures:
// metric, ok := s.metrics[rec.MetricID]
// if !ok {
// qb.Abort(qb.MetricNotFoundDuringReplay, nil)
// }
//metric.ReplayAppendedMeasures(rec) FIX
// case txlog.DeletedMeasures:
// metric, ok := s.metrics[rec.MetricID]
// if ok {
// metric.DeleteMeasures()
// if len(rec.FreePageNumbers) > 0 {
// s.freeList.AddPages(rec.FreePageNumbers)
// }
// }
default:
qb.Abort(qb.UnknownTxLogRecordTypeBug,
fmt.Errorf("bug: unknown record type %T in TransactionLog", rec))
// default:
// qb.Abort(qb.UnknownstorageRecordTypeBug,
// fmt.Errorf("bug: unknown record type %T in TransactionLog", rec))
}
return nil
}
func (s *WALReplayer) onAddedMetric(rec txlog.AddedMetric) (err error) {
func (s *WALReplayer) onMetricAdd(rec storage.MetricAddRecord) (err error) {
_, ok := s.metrics[rec.MetricID]
if ok {
return fmt.Errorf("metric %d add failed: already added", rec.MetricID)
}
var (
buf = make([]byte, atree.DataPageSize)
buf = make([]byte, storage.DataPageSize)
)
s.metrics[rec.MetricID] = &ReplayMetric{
metricType: rec.MetricType,
fracDigits: byte(rec.FracDigits),
buf: buf,
databuf: buf[:atree.DataPagePayloadSize],
databuf: buf[:storage.DataPagePayloadSize],
}
return
}
func (s *WALReplayer) onDeletedMetric(rec txlog.DeletedMetric) error {
func (s *WALReplayer) onMetricDelete(rec storage.MetricDeleteRecord) error {
_, ok := s.metrics[rec.MetricID]
if !ok {
return fmt.Errorf("metric %d deletion failed: not found", rec.MetricID)
@@ -166,15 +122,21 @@ func (s *WALReplayer) onDeletedMetric(rec txlog.DeletedMetric) error {
return nil
}
func (s *WALReplayer) onAppendMeasures(rec txlog.WALRecordAppendMeasures, isLastPacket bool) {
func (s *WALReplayer) onMeasuresAppend(rec storage.MeasuresAppendRecord) (err error) {
metric, ok := s.metrics[rec.MetricID]
if !ok {
qb.Abort(qb.MetricNotFoundDuringReplay, nil)
return fmt.Errorf("append measures failed: metric %d not found", rec.MetricID)
}
if isLastPacket {
s.collectPagesToWrite(metric, rec)
metric.MeasuresAppend(rec)
return
}
func (s *WALReplayer) onMeasuresAppendWithGrow(rec storage.MeasuresAppendWithGrowRecord, isLastPacket bool) (err error) {
metric, ok := s.metrics[rec.MetricID]
if !ok {
return fmt.Errorf("append measures failed: metric %d not found", rec.MetricID)
}
result := metric.AppendedMeasures(rec, isLastPacket)
result := metric.MeasuresAppendWithGrow(rec, isLastPacket)
//
if isLastPacket {
s.indexPages = append(s.indexPages, result.IndexPages...)
@@ -183,22 +145,33 @@ func (s *WALReplayer) onAppendMeasures(rec txlog.WALRecordAppendMeasures, isLast
//
s.freeIndexPages = s.freeIndexPages[:len(s.freeIndexPages)-result.ReusedIndexPagesCount]
s.freeDataPages = s.freeDataPages[:len(s.freeDataPages)-result.ReusedDataPagesCount]
return
}
func (s *WALReplayer) collectPagesToWrite(metric *ReplayMetric, rec txlog.WALRecordAppendMeasures) {
metric.ApplyHeadDataPage(rec.CompletedDataPage)
//func (s *WALReplayer) collectPagesToWrite(metric *ReplayMetric, rec storage.WALRecordAppendMeasures) {
// metric.ApplyHeadDataPage(rec.CompletedDataPage)
s.dataPages = append(s.dataPages, txlog.PageToWrite{
PageNo: p.PageNo,
Content: metric.buf,
})
// s.dataPages = append(s.dataPages, storage.PageToWrite{
// PageNo: p.PageNo,
// Content: metric.buf,
// })
for _, x := range rec.DataPages {
s.dataPages = append(s.dataPages, txlog.PageToWrite{
PageNo: x.PageNo,
Content: x.Content,
})
}
//metric.ApplyHeadIndexPages(rec.ChangedIndexLevels)
// for _, x := range rec.DataPages {
// s.dataPages = append(s.dataPages, storage.PageToWrite{
// PageNo: x.PageNo,
// Content: x.Content,
// })
// }
//metric.ApplyHeadIndexPages(rec.ChangedIndexLevels)
}
//}
// func (s *WALReplayer) replayPacket(records []any, isLastPacket bool) {
// for _, untyped := range records {
// switch rec := untyped.(type) {
// case storage.WALRecordAppendMeasures:
// s.onWALRecordAppendMeasures(rec, isLastPacket)
// }
// }
// return
// }

20
enc/cumdelta.go
View File

@@ -48,18 +48,15 @@ type CumulativeDeltaCompressor struct {
}
// Після відновлення із снапшота
func NewCumulativeDeltaCompressor(buf []byte, fracDigits byte) *CumulativeDeltaCompressor {
func NewCumulativeDeltaCompressor(fracDigits byte, buf []byte, payloadSize int) *CumulativeDeltaCompressor {
var coef float64 = 1
if fracDigits > 0 {
coef = math.Pow(10, float64(fracDigits))
}
return &CumulativeDeltaCompressor{
s := &CumulativeDeltaCompressor{
buf: buf,
coef: coef,
}
}
func (s *CumulativeDeltaCompressor) RestoreState(payloadSize int) {
if payloadSize > 0 {
s.pos = payloadSize - 1 // завжди показує на h
// base value на початку
@@ -74,16 +71,27 @@ func (s *CumulativeDeltaCompressor) RestoreState(payloadSize int) {
log.Fatalf("bug: get last delta: %s", err)
}
}
return s
}
// func (s *CumulativeDeltaCompressor) RestoreState() {
// }
// func (s *CumulativeDeltaCompressor) Size() int {
// return s.pos
// }
type EvaluationReport struct {
CompressionWay int
TotalSpace int
AdditionalSpace int
Delta uint64
}
func (s *CumulativeDeltaCompressor) Evaluate(value float64) (compressionWay int, requiredSpace int) {
delta := uint64((value-s.baseValue)*s.coef + eps) // fix - if delta 0, no eps
requiredSpace = s.pos
// fix - requiredSpace - all space
if s.pos > 0 {
if s.h < 128 {
// run

18
enc/enc_test.go
View File

@@ -165,10 +165,11 @@ func TestCumulativeDeltaCompressor(t *testing.T) {
var (
fracDigits byte = 1
buf = make([]byte, 8)
payloadSize = 0
compressionWay int
requiredSpace int
)
c := NewCumulativeDeltaCompressor(buf, fracDigits)
c := NewCumulativeDeltaCompressor(fracDigits, buf, payloadSize)
for _, num := range testCase.Nums {
compressionWay, requiredSpace = c.Evaluate(num)
c.Compress(compressionWay, num)
@@ -281,9 +282,10 @@ func TestCumulativeDeltaDecompressorFromState(t *testing.T) {
var (
fracDigits byte = 1
buf = make([]byte, 16)
payloadSize = 0
decodedNums []float64
)
c := NewCumulativeDeltaCompressor(buf, fracDigits)
c := NewCumulativeDeltaCompressor(fracDigits, buf, payloadSize)
for _, num := range testCase.Nums {
compressionWay, _ := c.Evaluate(num)
c.Compress(compressionWay, num)
@@ -524,10 +526,11 @@ func TestInstantDeltaCompressor(t *testing.T) {
var (
fracDigits byte = 1
buf = make([]byte, 8)
payloadSize = 0
compressionWay int
requiredSpace int
)
c := NewInstantDeltaCompressor(buf, fracDigits)
c := NewInstantDeltaCompressor(fracDigits, buf, payloadSize)
for _, num := range testCase.Nums {
compressionWay, requiredSpace = c.Evaluate(num)
c.Compress(compressionWay, num)
@@ -640,9 +643,10 @@ func TestInstantDeltaDecompressorFromState(t *testing.T) {
var (
fracDigits byte = 1
buf = make([]byte, 16)
payloadSize = 0
decodedNums []float64
)
c := NewInstantDeltaCompressor(buf, fracDigits)
c := NewInstantDeltaCompressor(fracDigits, buf, payloadSize)
for _, num := range testCase.Nums {
compressionWay, _ := c.Evaluate(num)
c.Compress(compressionWay, num)
@@ -840,10 +844,11 @@ func TestTimeDeltaCompressor(t *testing.T) {
//fmt.Println("----------")
var (
buf = make([]byte, 8)
payloadSize = 0
compressionWay int
requiredSpace int
)
c := NewTimeDeltaCompressor(buf)
c := NewTimeDeltaCompressor(buf, payloadSize)
for _, num := range testCase.Nums {
compressionWay, requiredSpace = c.Evaluate(num)
c.Compress(compressionWay, num)
@@ -957,9 +962,10 @@ func TestTimeDeltaDecompressorFromState(t *testing.T) {
for caseIdx, testCase := range testCases {
var (
buf = make([]byte, 16)
payloadSize = 0
decodedNums []uint32
)
c := NewTimeDeltaCompressor(buf)
c := NewTimeDeltaCompressor(buf, payloadSize)
for _, num := range testCase.Nums {
compressionWay, _ := c.Evaluate(num)
c.Compress(compressionWay, num)

11
enc/insdelta.go
View File

@@ -19,7 +19,7 @@ type InstantDeltaCompressor struct {
state *InstantDeltaCapturedState
}
func NewInstantDeltaCompressor(buf []byte, fracDigits byte) *InstantDeltaCompressor {
func NewInstantDeltaCompressor(fracDigits byte, buf []byte, payloadSize int) *InstantDeltaCompressor {
var coef float64 = 1
if fracDigits > 0 {
coef = math.Pow(10, float64(fracDigits))
@@ -28,10 +28,6 @@ func NewInstantDeltaCompressor(buf []byte, fracDigits byte) *InstantDeltaCompres
buf: buf,
coef: coef,
}
return s
}
func (s *InstantDeltaCompressor) RestoreState(payloadSize int) {
if payloadSize > 0 {
s.pos = payloadSize - 1 // завжди показує на h
// base value на початку
@@ -46,8 +42,13 @@ func (s *InstantDeltaCompressor) RestoreState(payloadSize int) {
log.Fatalf("bug: get last delta: %s", err)
}
}
return s
}
// func (s *InstantDeltaCompressor) RestoreState() {
// }
// func (s *InstantDeltaCompressor) Size() int {
// return s.pos
// }

15
enc/time_delta.go
View File

@@ -34,15 +34,12 @@ type TimeDeltaCompressor struct {
// а lastDelta і h треба прочитати із payload (якщо є)
// Кодує значення справа наліво функціями TailPut*. Читає значення зліва направо функціями Get*
func NewTimeDeltaCompressor(buf []byte) *TimeDeltaCompressor {
return &TimeDeltaCompressor{
func NewTimeDeltaCompressor(buf []byte, payloadSize int) *TimeDeltaCompressor {
s := &TimeDeltaCompressor{
buf: buf,
}
}
func (s *TimeDeltaCompressor) RestoreState(payloadSize int, lastTimestamp uint32) {
if payloadSize > 0 {
s.lastUnixtime = lastTimestamp
//s.lastUnixtime = lastTimestamp
s.pos = len(s.buf) - payloadSize
if payloadSize > 4 {
s.h = s.buf[s.pos]
@@ -53,8 +50,14 @@ func (s *TimeDeltaCompressor) RestoreState(payloadSize int, lastTimestamp uint32
s.lastDelta = uint32(u64)
}
}
return s
}
// FIX - restire lastUnixtime
// func (s *TimeDeltaCompressor) RestoreState() {
// }
// retrun
func (s *TimeDeltaCompressor) Evaluate(unixtime uint32) (compressionWay int, requiredSpace int) {
requiredSpace = len(s.buf) - s.pos

2
examples/database/main.go
View File

@@ -82,7 +82,7 @@ const mainUsage = `Usage:
`
const helpMessage = `Diploma project. Database. Version: 1.0
const helpMessage = `Gordenko project. Database. Version: 1.0
created by Dmytro Gordenko, 1.e4.kc6@gmail.com
`

38
examples/loadtest/loadtest.go
View File

@@ -7,7 +7,7 @@ import (
"os"
"time"
diploma "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/client"
"gordenko.dev/dima/qb/proto"
)
@@ -15,12 +15,12 @@ import (
// METRICS INFO
type MetricInfo struct {
MetricID uint32 `json:"metricID"`
MetricType diploma.MetricType `json:"metricType"`
FracDigits int `json:"fracDigits"`
Since int64 `json:"since"`
Until int64 `json:"until"`
Qty int `json:"qty"`
MetricID uint32 `json:"metricID"`
MetricType qb.MetricType `json:"metricType"`
FracDigits int `json:"fracDigits"`
Since int64 `json:"since"`
Until int64 `json:"until"`
Qty int `json:"qty"`
}
func readMetricInfo(fileName string) (list []MetricInfo, err error) {
@@ -42,12 +42,12 @@ type QueryRecipe struct {
RangeCode int
Since uint32
Until uint32
GroupBy diploma.GroupBy
GroupBy qb.GroupBy
}
type RandomQueryGenerator struct {
metrics []MetricInfo
groupByOptions []diploma.GroupBy
groupByOptions []qb.GroupBy
listCurrentValuesProbability int
listPeriodsProbability int
timeRangeProbDistribution []int
@@ -86,10 +86,10 @@ func NewRandomQueryGenerator(opt RandomQueryGeneratorOptions) *RandomQueryGenera
return &RandomQueryGenerator{
metrics: opt.Metrics,
groupByOptions: []diploma.GroupBy{
diploma.GroupByHour,
diploma.GroupByDay,
diploma.GroupByMonth,
groupByOptions: []qb.GroupBy{
qb.GroupByHour,
qb.GroupByDay,
qb.GroupByMonth,
},
listCurrentValuesProbability: opt.ListCurrentValuesProbability,
listPeriodsProbability: opt.ListPeriodsProbability,
@@ -108,7 +108,7 @@ func (s *RandomQueryGenerator) GetQueryRecipe() QueryRecipe {
Method: listCurrentValues,
}
} else {
if metric.MetricType == diploma.Cumulative {
if metric.MetricType == qb.Cumulative {
num = rand.Intn(100)
if num < s.listPeriodsProbability {
groupBy := s.groupByOptions[rand.Intn(len(s.groupByOptions))]
@@ -118,10 +118,10 @@ func (s *RandomQueryGenerator) GetQueryRecipe() QueryRecipe {
maxDays = 7
)
if groupBy == diploma.GroupByDay {
if groupBy == qb.GroupByDay {
minDays = 1
maxDays = 30
} else if groupBy == diploma.GroupByMonth {
} else if groupBy == qb.GroupByMonth {
minDays = 1
maxDays = 30
}
@@ -164,10 +164,10 @@ func (s *RandomQueryGenerator) GetQueryRecipe() QueryRecipe {
maxDays = 7
)
if groupBy == diploma.GroupByDay {
if groupBy == qb.GroupByDay {
minDays = 1
maxDays = 30
} else if groupBy == diploma.GroupByMonth {
} else if groupBy == qb.GroupByMonth {
minDays = 1
maxDays = 30
}
@@ -350,7 +350,7 @@ func execQuery(conn *client.Connection, queryGenerator *RandomQueryGenerator, st
Day: until.Day(),
},
GroupBy: recipe.GroupBy,
AggregateFuncs: diploma.AggregateMin | diploma.AggregateMax | diploma.AggregateAvg,
AggregateFuncs: qb.AggregateMin | qb.AggregateMax | qb.AggregateAvg,
})
elapsedTime = time.Since(t1)
stat.ElapsedTime += elapsedTime

2
examples/loadtest/main.go
View File

@@ -203,7 +203,7 @@ const mainUsage = `Usage:
`
const helpMessage = `Diploma project. Load test. Version: 1.0
const helpMessage = `Gordenko project. Load test. Version: 1.0
created by Dmytro Gordenko, 1.e4.kc6@gmail.com
`

2
examples/requests/main.go
View File

@@ -54,7 +54,7 @@ const mainUsage = `Usage:
`
const helpMessage = `Diploma project. Example requests. Version: 1.0
const helpMessage = `Gordenko project. Example requests. Version: 1.0
created by Dmytro Gordenko, 1.e4.kc6@gmail.com
`

24
examples/requests/requests.go
View File

@@ -5,7 +5,7 @@ import (
"log"
"time"
diploma "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/client"
"gordenko.dev/dima/qb/proto"
)
@@ -25,7 +25,7 @@ func sendRequests(conn *client.Connection) {
err = conn.AddMetric(proto.AddMetricReq{
MetricID: instantMetricID,
MetricType: diploma.Instant,
MetricType: qb.Instant,
FracDigits: fracDigits,
})
if err != nil {
@@ -114,8 +114,8 @@ GetMetric:
Month: until.Month(),
Day: until.Day(),
},
GroupBy: diploma.GroupByHour,
AggregateFuncs: diploma.AggregateMin | diploma.AggregateMax | diploma.AggregateAvg,
GroupBy: qb.GroupByHour,
AggregateFuncs: qb.AggregateMin | qb.AggregateMax | qb.AggregateAvg,
})
if err != nil {
log.Fatalf("conn.ListInstantPeriods: %s\n", err)
@@ -143,8 +143,8 @@ GetMetric:
Month: until.Month(),
Day: until.Day(),
},
GroupBy: diploma.GroupByDay,
AggregateFuncs: diploma.AggregateMin | diploma.AggregateMax | diploma.AggregateAvg,
GroupBy: qb.GroupByDay,
AggregateFuncs: qb.AggregateMin | qb.AggregateMax | qb.AggregateAvg,
})
if err != nil {
log.Fatalf("conn.ListInstantPeriods: %s\n", err)
@@ -172,8 +172,8 @@ GetMetric:
Month: until.Month(),
Day: until.Day(),
},
GroupBy: diploma.GroupByMonth,
AggregateFuncs: diploma.AggregateMin | diploma.AggregateMax | diploma.AggregateAvg,
GroupBy: qb.GroupByMonth,
AggregateFuncs: qb.AggregateMin | qb.AggregateMax | qb.AggregateAvg,
})
if err != nil {
log.Fatalf("conn.ListInstantPeriods: %s\n", err)
@@ -208,7 +208,7 @@ GetMetric:
err = conn.AddMetric(proto.AddMetricReq{
MetricID: cumulativeMetricID,
MetricType: diploma.Cumulative,
MetricType: qb.Cumulative,
FracDigits: fracDigits,
})
if err != nil {
@@ -298,7 +298,7 @@ GetMetric:
Month: until.Month(),
Day: until.Day(),
},
GroupBy: diploma.GroupByHour,
GroupBy: qb.GroupByHour,
})
if err != nil {
log.Fatalf("conn.ListCumulativePeriods: %s\n", err)
@@ -326,7 +326,7 @@ GetMetric:
Month: until.Month(),
Day: until.Day(),
},
GroupBy: diploma.GroupByDay,
GroupBy: qb.GroupByDay,
})
if err != nil {
log.Fatalf("conn.ListCumulativePeriods: %s\n", err)
@@ -354,7 +354,7 @@ GetMetric:
Month: until.Month(),
Day: until.Day(),
},
GroupBy: diploma.GroupByMonth,
GroupBy: qb.GroupByMonth,
})
if err != nil {
log.Fatalf("conn.ListCumulativePeriods: %s\n", err)

42
freelist/freelist.go
View File

@@ -1,12 +1,11 @@
package freelist
import (
"bytes"
"fmt"
"io"
"os"
"gordenko.dev/dima/qb/bin"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb/util"
)
@@ -158,12 +157,13 @@ func (s *FreeList) loadLastPage() error {
return fmt.Errorf("read size %d bytes not equal page size %d", n, s.pageSize)
}
// check crc
checksum := bin.GetUint32(buf[0:])
checksum, _ := bin.GetUint32(buf[0:])
if util.CalcChecksum(buf[crcSize:]) != checksum {
return fmt.Errorf("page %d is corrupted", s.basePagesCount)
}
for i := crcSize; i < len(buf); i += ptrSize {
s.free = append(s.free, bin.GetUint32(buf[i:]))
pageNo, _ := bin.GetUint32(buf[i:])
s.free = append(s.free, pageNo)
}
if s.deltaPagesCount > 0 {
s.deltaPagesCount--
@@ -207,25 +207,25 @@ func (s *FreeList) GetPageNumber() (pageNo uint32, err error) {
}
// для відновлення недозаповненнох сторінки із снапшота
func (s *FreeList) Restore(serialized []byte) error {
if (len(serialized) % ptrSize) != 0 {
return fmt.Errorf("wrong size")
}
for i := 0; i < len(serialized); i += ptrSize {
s.free = append(s.free, bin.GetUint32(serialized[i:]))
}
fmt.Println(s.free)
return nil
}
// func (s *FreeList) Restore(serialized []byte) error {
// if (len(serialized) % ptrSize) != 0 {
// return fmt.Errorf("wrong size")
// }
// for i := 0; i < len(serialized); i += ptrSize {
// s.free = append(s.free, bin.GetUint32(serialized[i:]))
// }
// fmt.Println(s.free)
// return nil
// }
// для запису в snapshot
func (s *FreeList) Serialize() ([]byte, error) {
w := bytes.NewBuffer(nil)
for _, pageNo := range s.free {
bin.WriteUint32(w, pageNo)
}
return w.Bytes(), nil
}
// func (s *FreeList) Serialize() ([]byte, error) {
// w := bytes.NewBuffer(nil)
// for _, pageNo := range s.free {
// bin.WriteUint32(w, pageNo)
// }
// return w.Bytes(), nil
// }
func (s *FreeList) Cached() []uint32 {
return s.free

64
freelist/freelist_test.go
View File

@@ -58,46 +58,46 @@ func TestSave(t *testing.T) {
// }
}
func TestDelta(t *testing.T) {
freeList, err := createFreeList()
if err != nil {
t.Fatal(err)
}
// func TestDelta(t *testing.T) {
// freeList, err := createFreeList()
// if err != nil {
// t.Fatal(err)
// }
serialized := []byte{9, 0, 0, 0, 10, 0, 0, 0}
// serialized := []byte{9, 0, 0, 0, 10, 0, 0, 0}
err = freeList.Restore(serialized)
if err != nil {
t.Fatal(err)
}
// err = freeList.Restore(serialized)
// if err != nil {
// t.Fatal(err)
// }
for range 3 {
pageNo, err := freeList.GetPageNumber()
fmt.Println(pageNo, err)
}
// for range 3 {
// pageNo, err := freeList.GetPageNumber()
// fmt.Println(pageNo, err)
// }
err = freeList.AddPageNumbers([]uint32{20, 21, 22})
if err != nil {
t.Fatal(err)
}
// err = freeList.AddPageNumbers([]uint32{20, 21, 22})
// if err != nil {
// t.Fatal(err)
// }
err = freeList.AddPageNumbers([]uint32{23, 24, 25, 26})
if err != nil {
t.Fatal(err)
}
// err = freeList.AddPageNumbers([]uint32{23, 24, 25, 26})
// if err != nil {
// t.Fatal(err)
// }
serialized, err = freeList.Serialize()
if err != nil {
t.Fatal(err)
}
// serialized, err = freeList.Serialize()
// if err != nil {
// t.Fatal(err)
// }
fmt.Println("free serialized:", serialized)
// fmt.Println("free serialized:", serialized)
err = freeList.Merge()
if err != nil {
t.Fatal(err)
}
}
// err = freeList.Merge()
// if err != nil {
// t.Fatal(err)
// }
// }
// func TestGetFromFreeList(t *testing.T) {
// freeList, err := createFreeList()

84
proto/proto.go
View File

@@ -4,8 +4,8 @@ import (
"fmt"
"time"
diploma "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/bin"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/bufreader"
)
@@ -72,7 +72,7 @@ func ErrorCodeToText(code uint16) string {
type Metric struct {
MetricID uint32
MetricType diploma.MetricType
MetricType qb.MetricType
FracDigits int
}
@@ -161,7 +161,7 @@ func ReadListCurrentValuesReq(r *bufreader.BufferedReader) (m ListCurrentValuesR
type AddMetricReq struct {
MetricID uint32
MetricType diploma.MetricType
MetricType qb.MetricType
FracDigits int
}
@@ -175,8 +175,8 @@ func ReadAddMetricReq(r *bufreader.BufferedReader) (m AddMetricReq, err error) {
}
func UnpackAddMetricReq(arr []byte) (m AddMetricReq) {
m.MetricID = bin.GetUint32(arr)
m.MetricType = diploma.MetricType(arr[4])
m.MetricID, _ = bin.GetUint32(arr)
m.MetricType = qb.MetricType(arr[4])
m.FracDigits = int(arr[5])
return
}
@@ -209,8 +209,8 @@ func ReadDeleteMeasuresReq(r *bufreader.BufferedReader) (m DeleteMeasuresReq, er
}
func UnpackDeleteMeasuresReq(arr []byte) (m DeleteMeasuresReq) {
m.MetricID = bin.GetUint32(arr)
m.Since = bin.GetUint32(arr[4:])
m.MetricID, _ = bin.GetUint32(arr)
m.Since, _ = bin.GetUint32(arr[4:])
return
}
@@ -230,9 +230,9 @@ func ReadAppendMeasureReq(r *bufreader.BufferedReader) (m AppendMeasureReq, err
}
func UnpackAppendMeasureReq(arr []byte) (m AppendMeasureReq) {
m.MetricID = bin.GetUint32(arr)
m.Timestamp = bin.GetUint32(arr[4:])
m.Value = bin.GetFloat64(arr[8:])
m.MetricID, _ = bin.GetUint32(arr)
m.Timestamp, _ = bin.GetUint32(arr[4:])
m.Value, _ = bin.GetFloat64(arr[8:])
return
}
@@ -253,8 +253,8 @@ func ReadAppendMeasuresReq(r *bufreader.BufferedReader) (m AppendMeasuresReq, er
return
}
m.MetricID = bin.GetUint32(prefix[0:])
qty := bin.GetUint16(prefix[4:])
m.MetricID, _ = bin.GetUint32(prefix[0:])
qty, _ := bin.GetUint16(prefix[4:])
for i := range int(qty) {
var measure Measure
@@ -280,21 +280,25 @@ type MetricMeasure struct {
}
func ReadAppendMeasurePerMetricReq(r *bufreader.BufferedReader) (measures []MetricMeasure, err error) {
qty, err := bin.ReadUint16(r)
count, err := bin.ReadUint16(r)
if err != nil {
return
}
var tmp = make([]byte, 16)
for range int(qty) {
err = bin.ReadNInto(r, tmp)
for range count {
var m MetricMeasure
m.MetricID, err = bin.ReadUint32(r)
if err != nil {
return
}
measures = append(measures, MetricMeasure{
MetricID: bin.GetUint32(tmp[0:]),
Timestamp: bin.GetUint32(tmp[4:]),
Value: bin.GetFloat64(tmp[8:]),
})
m.Timestamp, err = bin.ReadUint32(r)
if err != nil {
return
}
m.Value, err = bin.ReadFloat64(r)
if err != nil {
return
}
measures = append(measures, m)
}
return
}
@@ -341,9 +345,9 @@ func ReadListInstantMeasuresReq(r *bufreader.BufferedReader) (m ListInstantMeasu
}
func UnpackListInstantMeasuresReq(arr []byte) (m ListInstantMeasuresReq) {
m.MetricID = bin.GetUint32(arr[0:])
m.Since = bin.GetUint32(arr[4:])
m.Until = bin.GetUint32(arr[8:])
m.MetricID, _ = bin.GetUint32(arr[0:])
m.Since, _ = bin.GetUint32(arr[4:])
m.Until, _ = bin.GetUint32(arr[8:])
return
}
@@ -363,9 +367,9 @@ func ReadListCumulativeMeasuresReq(r *bufreader.BufferedReader) (m ListCumulativ
}
func UnpackListCumulativeMeasuresReq(arr []byte) (m ListCumulativeMeasuresReq) {
m.MetricID = bin.GetUint32(arr)
m.Since = bin.GetUint32(arr[4:])
m.Until = bin.GetUint32(arr[8:])
m.MetricID, _ = bin.GetUint32(arr)
m.Since, _ = bin.GetUint32(arr[4:])
m.Until, _ = bin.GetUint32(arr[8:])
return
}
@@ -373,7 +377,7 @@ type ListInstantPeriodsReq struct {
MetricID uint32
Since TimeBound
Until TimeBound
GroupBy diploma.GroupBy
GroupBy qb.GroupBy
AggregateFuncs byte
FirstHourOfDay int
}
@@ -388,18 +392,20 @@ func ReadListInstantPeriodsReq(r *bufreader.BufferedReader) (m ListInstantPeriod
}
func UnpackListInstantPeriodsReq(arr []byte) (m ListInstantPeriodsReq) {
m.MetricID = bin.GetUint32(arr)
m.MetricID, _ = bin.GetUint32(arr)
sinceYear, _ := bin.GetUint16(arr[4:])
untilYear, _ := bin.GetUint16(arr[8:])
m.Since = TimeBound{
Year: int(bin.GetUint16(arr[4:])),
Year: int(sinceYear),
Month: time.Month(arr[6]),
Day: int(arr[7]),
}
m.Until = TimeBound{
Year: int(bin.GetUint16(arr[8:])),
Year: int(untilYear),
Month: time.Month(arr[10]),
Day: int(arr[11]),
}
m.GroupBy = diploma.GroupBy(arr[12])
m.GroupBy = qb.GroupBy(arr[12])
m.AggregateFuncs = arr[13]
m.FirstHourOfDay = int(arr[14])
return
@@ -409,7 +415,7 @@ type ListCumulativePeriodsReq struct {
MetricID uint32
Since TimeBound
Until TimeBound
GroupBy diploma.GroupBy
GroupBy qb.GroupBy
FirstHourOfDay int
}
@@ -423,18 +429,20 @@ func ReadListCumulativePeriodsReq(r *bufreader.BufferedReader) (m ListCumulative
}
func UnpackListCumulativePeriodsReq(arr []byte) (m ListCumulativePeriodsReq) {
m.MetricID = bin.GetUint32(arr[0:])
m.MetricID, _ = bin.GetUint32(arr[0:])
sinceYear, _ := bin.GetUint16(arr[4:])
untilYear, _ := bin.GetUint16(arr[8:])
m.Since = TimeBound{
Year: int(bin.GetUint16(arr[4:])),
Year: int(sinceYear),
Month: time.Month(arr[6]),
Day: int(arr[7]),
}
m.Until = TimeBound{
Year: int(bin.GetUint16(arr[8:])),
Year: int(untilYear),
Month: time.Month(arr[10]),
Day: int(arr[11]),
}
m.GroupBy = diploma.GroupBy(arr[12])
m.GroupBy = qb.GroupBy(arr[12])
m.FirstHourOfDay = int(arr[13])
return
}

4
qb.go
View File

@@ -24,7 +24,6 @@ const (
)
type TimestampCompressor interface {
RestoreState(int, uint32) // (payload size, last timestamp)
// (timestamp) => compressionWay, requiredSpace
Evaluate(uint32) (int, int)
Compress(int, uint32)
@@ -43,7 +42,6 @@ type TimestampCompressor interface {
}
type ValueCompressor interface {
RestoreState(int) // payload size
// (value) => compressionWay, requiredSpace
Evaluate(float64) (int, int)
Compress(int, float64)
@@ -110,6 +108,8 @@ const (
DeletePrevSnapshotFileFailed AbortCode = 35
MetricNotFoundDuringReplay AbortCode = 36
WALReplayFailed AbortCode = 37
)
func Abort(code AbortCode, err error) {

18
txlog/data_preparer.go → storage/data_preparer.go
View File

@@ -1,4 +1,4 @@
package txlog
package storage
import (
"bytes"
@@ -51,7 +51,7 @@ func (s *DataPreparer) Prepare(input []any) PreparedData {
// 1. Пакую всі дані в WAL буфер запису
for _, untyped := range input {
switch x := untyped.(type) {
case AddedMetric:
case MetricAddRecord:
x.Pack(w)
// case DeletedMetric:
// if len(x.FreePageNumbers) > 0 {
@@ -91,11 +91,11 @@ func (s *DataPreparer) Prepare(input []any) PreparedData {
if len(level.IndexPages) == 0 && level.SkipRecords == level.TailRecordsCount {
break
}
var completedIndexPage *CompletedIndexPage
var completedIndexPage *HeadIndexPage
if len(level.IndexPages) > 0 {
first := level.IndexPages[0]
skipSize := level.SkipRecords * indexRecordSize
completedIndexPage = &CompletedIndexPage{
completedIndexPage = &HeadIndexPage{
PageNo: first.PageNo,
Reused: first.Reused,
Checksum: first.Checksum,
@@ -103,9 +103,9 @@ func (s *DataPreparer) Prepare(input []any) PreparedData {
}
}
changedIndexPages = append(changedIndexPages, ChangedIndexLevel{
CompletedIndexPage: completedIndexPage,
IndexPages: level.IndexPages[1:],
TailRecords: level.TailRecords[:level.TailRecordsCount*indexRecordSize],
HeadIndexPage: completedIndexPage,
IndexPages: level.IndexPages[1:],
TailRecords: level.TailRecords[:level.TailRecordsCount*indexRecordSize],
})
indexLevelTails = append(indexLevelTails, IndexLevelTail{
@@ -116,9 +116,9 @@ func (s *DataPreparer) Prepare(input []any) PreparedData {
first := sealResult.DataPages[0]
rec := WALRecordAppendMeasures{
rec := MeasuresAppendWithGrowRecord{
MetricID: x.MetricID,
CompletedDataPage: CompletedDataPage{
HeadDataPage: HeadDataPage{
PageNo: first.PageNo,
Reused: first.Reused,
PrevPageNo: first.PrevPageNo,

2
txlog/helpers.go → storage/helpers.go
View File

@@ -1,4 +1,4 @@
package txlog
package storage
import (
"bytes"

41
txlog/page_preparer.go → storage/page_preparer.go
View File

@@ -1,10 +1,10 @@
package txlog
package storage
import (
"errors"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/bin"
"gordenko.dev/dima/qb/util"
)
@@ -268,7 +268,7 @@ func (s *PagePreparer) SealPages(in SealPagesIn) (result SealResult) {
level.TailRecordsCount = 1
//
upPageNo = pageNo
upTimestamp = bin.GetUint32(filled.Content)
upTimestamp, _ = bin.GetUint32(filled.Content)
levelIdx++
}
}
@@ -277,3 +277,38 @@ func (s *PagePreparer) SealPages(in SealPagesIn) (result SealResult) {
IndexLevels: sealedIndexLevels,
}
}
// HELPERS
type SealDataPageIn struct {
Content []byte
PrevPageNo uint32
TimestampsSize int
ValuesSize int
}
func SealDataPage(in SealDataPageIn) (checksum uint32) {
// bin.PutUint16(in.Content[s.timestampsSizeIdx:], uint16(in.TimestampsSize))
// bin.PutUint16(in.Content[s.valuesSizeIdx:], uint16(in.ValuesSize))
// bin.PutUint32(in.Content[s.prevPageIdx:], in.PrevPageNo)
// checksum = util.CalcChecksum(in.Content[:s.dataChecksumIdx])
// bin.PutUint32(in.Content[s.dataChecksumIdx:], checksum)
return
}
type SealIndexPageIn struct {
Content []byte
RecordsCount int
ZeroLevel bool
}
func SealIndexPage(in SealIndexPageIn) (checksum uint32) {
// bin.PutUint16(in.Content[s.indexRecordsCountIdx:], uint16(in.RecordsCount))
// if in.LastLevel {
// in.Content[s.isZeroLevelIdx] = 1
// }
// checksum = util.CalcChecksum(in.Content[:s.indexChecksumIdx])
// bin.PutUint32(in.Content[s.indexChecksumIdx:], checksum)
return
}

32
storage/storage.go Normal file
View File

@@ -0,0 +1,32 @@
package storage
const (
//PageNoSize = 4
// data page
DataPageSize = 8192
DataPageFooterSize int = 12
DataPagePayloadSize int = DataPageSize - DataPageFooterSize
dataCRC32Idx = DataPageSize - 4
timestampsSizeIdx = DataPageSize - 6
valuesSizeIdx = DataPageSize - 8
prevPageIdx = DataPageSize - 12
// index page
IndexPageSize = 1024
IndexPageFooterSize = 7
indexRecordSize = 8
maxRecordsOnIndexPage = (IndexPageSize - IndexPageFooterSize) / indexRecordSize
//indexPageIncSize = IndexPageIncSize
indexCRC32Idx = IndexPageSize - 4
indexRecordsCountIdx = IndexPageSize - 6
isZeroLevelIdx = IndexPageSize - 7
// timestampSize = 4
// pairSize = timestampSize + PageNoSize
// indexFooterIdx = indexRecordsQtyIdx
// dataFooterIdx = timestampsSizeIdx
)

30
txlog/reader.go → storage/wal_reader.go
View File

@@ -1,4 +1,4 @@
package txlog
package storage
import (
"bufio"
@@ -14,19 +14,19 @@ import (
const readBufferSize = 8 * 1024 * 1024
type Reader struct {
type WALReader struct {
file *os.File
reader *bufio.Reader
current []any
next []any
}
type ReaderOptions struct {
type WALReaderOptions struct {
FileName string
BufferSize int
}
func NewReader(opt ReaderOptions) (*Reader, error) {
func NewWALReader(opt WALReaderOptions) (*WALReader, error) {
if opt.FileName == "" {
return nil, errors.New("FileName option is required")
}
@@ -39,17 +39,17 @@ func NewReader(opt ReaderOptions) (*Reader, error) {
return nil, err
}
return &Reader{
return &WALReader{
file: file,
reader: bufio.NewReaderSize(file, readBufferSize),
}, nil
}
func (s *Reader) Close() {
func (s *WALReader) Close() {
s.file.Close()
}
func (s *Reader) NextPacket() ([]any, bool, error) {
func (s *WALReader) NextPacket() ([]any, bool, error) {
var err error
if s.current == nil {
s.current, err = s.readPacket()
@@ -75,7 +75,7 @@ func (s *Reader) NextPacket() ([]any, bool, error) {
return current, done, nil
}
func (s *Reader) readPacket() (_ []any, err error) {
func (s *WALReader) readPacket() (_ []any, err error) {
payloadSize, err := bin.ReadVarSize(s.reader)
if err != nil {
if err == io.EOF {
@@ -107,7 +107,7 @@ func (s *Reader) readPacket() (_ []any, err error) {
return nil, nil
}
func (s *Reader) parseRecords(body []byte) ([]any, error) {
func (s *WALReader) parseRecords(body []byte) ([]any, error) {
var (
src = bytes.NewBuffer(body)
records []any
@@ -123,15 +123,15 @@ func (s *Reader) parseRecords(body []byte) ([]any, error) {
}
switch recordType {
case CodeAddedMetric:
rec := new(AddedMetric)
case CodeMetricAdd:
rec := new(MetricAddRecord)
if err = rec.Parse(src); err != nil {
return nil, err
}
records = append(records, rec)
case CodeDeletedMetric:
rec := new(DeletedMetric)
case CodeMetricDelete:
rec := new(MetricDeleteRecord)
if err = rec.Parse(src); err != nil {
return nil, err
}
@@ -158,7 +158,7 @@ func (s *Reader) parseRecords(body []byte) ([]any, error) {
// }
// records = append(records, rec)
case CodeDeletedMeasures:
case CodeMeasuresDelete:
rec := new(DeletedMeasures)
if err = rec.Parse(src); err != nil {
return nil, err
@@ -349,7 +349,7 @@ func (s *Reader) parseRecords(body []byte) ([]any, error) {
// return freePages, nil
// }
func (s *Reader) Seek(offset int64) error {
func (s *WALReader) Seek(offset int64) error {
ret, err := s.file.Seek(offset, 0)
if err != nil {
return err

666
storage/wal_records.go Normal file
View File

@@ -0,0 +1,666 @@
package storage
import (
"bytes"
"io"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
)
type HeadDataPage struct {
PageNo uint32
Reused bool
PrevPageNo uint32
Checksum uint32
TimestampsOffset int // offset on 1st page
Timestamps []byte
ValuesOffset int // offset on 1st page
Values []byte
}
type HeadIndexPage struct {
PageNo uint32
Reused bool
Checksum uint32
Records []byte // offset не потрібен, оскільки Records додаються в кінець
}
type ChangedIndexLevel struct {
HeadIndexPage *HeadIndexPage
IndexPages []SealedIndexPage
TailRecords []byte // payload only
}
type MeasuresAppendWithGrowRecord struct {
MetricID uint32
HeadDataPage HeadDataPage
DataPages []SealedDataPage
ChangedIndexLevels []ChangedIndexLevel
TailTimestamps []byte // data level tail
TailValues []byte // data level tail
}
/*
Format appended measures:
1b - tx type
4b - metricID
4b - pageNo
1b - reused
4b - prevPageNo
4b - page crc32
Nb - (varsize) timestamps offset
Nb - (varsize) timestamps tail size on 1st filled data page
Nb - timestamps tail on 1st filled data page
Nb - (varsize) values offset
Nb - (varsize) values tail size on 1st filled data page
Nb - values tail on 1st filled data page
Nb - (varsize) filled data pages qty
[
4b - pageNo
1b - reused
Nb - (data page size) page content
]
Nb - (varsize) timestamps payload size on tail
Nb - timestamps payload
Nb - (varsize) values payload size on tail
Nb - values payload
Nb - (varsize) qty of index levels
[
// NOTE!
// - idx = 0 - zeroLevel
// - skipped = maxRecordsOnIndexPage - records count
4b - pageNo of 1st index page
1b - reused
4b - page crc32
Nb - (varsize) records tail count on 1st filled index page
Nb - records tail on 1st filled index page
Nb - (varsize) qty of level filled pages
[
4b - pageNo
1b - reused
Nb - (index page size) page content
]
Nb - (varsize) size of records level tail
Nb - level tail records
]
*/
func (s MeasuresAppendWithGrowRecord) WriteTo(w *bytes.Buffer) {
w.WriteByte(CodeMeasuresAppendWithGrow)
bin.WriteUint32(w, s.MetricID)
// completed data page
completed := s.HeadDataPage
bin.WriteUint32(w, completed.PageNo)
bin.WriteBool(w, completed.Reused)
bin.WriteUint32(w, completed.PrevPageNo)
bin.WriteUint32(w, completed.Checksum)
bin.WriteVarSize(w, completed.TimestampsOffset)
bin.WriteVarSize(w, len(completed.Timestamps))
w.Write(completed.Timestamps)
bin.WriteVarSize(w, completed.ValuesOffset)
bin.WriteVarSize(w, len(completed.Values))
w.Write(completed.Values)
// data pages
bin.WriteVarSize(w, len(s.DataPages))
for _, p := range s.DataPages {
bin.WriteUint32(w, p.PageNo)
bin.WriteBool(w, p.Reused)
bin.WriteVarSize(w, len(p.Content))
w.Write(p.Content)
}
// data tail
bin.WriteVarSize(w, len(s.TailTimestamps))
w.Write(s.TailTimestamps)
bin.WriteVarSize(w, len(s.TailValues))
w.Write(s.TailValues)
// changed levels
bin.WriteVarSize(w, len(s.ChangedIndexLevels))
for _, level := range s.ChangedIndexLevels {
// completed index page
completed := level.HeadIndexPage
bin.WriteUint32(w, completed.PageNo)
bin.WriteBool(w, completed.Reused)
bin.WriteUint32(w, completed.Checksum)
bin.WriteVarSize(w, len(completed.Records)/indexRecordSize)
w.Write(completed.Records)
// full index pages
for _, p := range level.IndexPages {
bin.WriteUint32(w, p.PageNo)
bin.WriteBool(w, p.Reused)
bin.WriteVarSize(w, len(p.Content))
w.Write(p.Content)
}
// level tail
bin.WriteVarSize(w, len(level.TailRecords)/indexRecordSize)
w.Write(level.TailRecords)
}
}
func (s *MeasuresAppendWithGrowRecord) Parse(r *bytes.Buffer) (err error) {
var (
size int
recordsCount int
)
s.MetricID, err = bin.ReadUint32(r)
if err != nil {
return
}
s.HeadDataPage.PageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
s.HeadDataPage.Reused, err = bin.ReadBool(r)
if err != nil {
return
}
s.HeadDataPage.PrevPageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
s.HeadDataPage.Checksum, err = bin.ReadUint32(r)
if err != nil {
return
}
s.HeadDataPage.TimestampsOffset, err = bin.ReadVarSize(r)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.HeadDataPage.Timestamps, err = bin.ReadN(r, size)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.HeadDataPage.Values, err = bin.ReadN(r, size)
if err != nil {
return
}
// data pages
dataPagesQty, err := bin.ReadVarSize(r)
if err != nil {
return
}
for range dataPagesQty {
var (
p SealedDataPage
)
p.PageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
p.Reused, err = bin.ReadBool(r)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
p.Content, err = bin.ReadN(r, size)
if err != nil {
return
}
s.DataPages = append(s.DataPages, p)
}
// data tail
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.TailTimestamps, err = bin.ReadN(r, size)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.TailValues, err = bin.ReadN(r, size)
if err != nil {
return
}
// changed levels
changedIndexLevelsCount, err := bin.ReadVarSize(r)
if err != nil {
return
}
for range changedIndexLevelsCount {
var (
indexPagesCount int
level ChangedIndexLevel
)
// completed index page
level.HeadIndexPage.PageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
level.HeadIndexPage.Reused, err = bin.ReadBool(r)
if err != nil {
return
}
level.HeadIndexPage.Checksum, err = bin.ReadUint32(r)
if err != nil {
return
}
recordsCount, err = bin.ReadVarSize(r)
if err != nil {
return
}
level.HeadIndexPage.Records, err = bin.ReadN(r, recordsCount*indexRecordSize)
if err != nil {
return
}
//
indexPagesCount, err = bin.ReadVarSize(r)
if err != nil {
return
}
for range indexPagesCount {
var p SealedIndexPage
p.PageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
p.Reused, err = bin.ReadBool(r)
if err != nil {
return
}
p.Checksum, err = bin.ReadUint32(r)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
p.Content, err = bin.ReadN(r, size)
if err != nil {
return
}
level.IndexPages = append(level.IndexPages, p)
}
recordsCount, err = bin.ReadVarSize(r)
if err != nil {
return
}
level.TailRecords, err = bin.ReadN(r, recordsCount*indexRecordSize)
if err != nil {
return
}
s.ChangedIndexLevels = append(s.ChangedIndexLevels, level)
}
return
}
type MeasuresAppendRecord struct {
MetricID uint32
TimestampsOffset int
Timestamps []byte
ValuesOffset int
Values []byte
}
func (s MeasuresAppendRecord) WriteTo(w *bytes.Buffer) {
w.WriteByte(CodeMeasuresAppendWithGrow)
bin.WriteUint32(w, s.MetricID)
bin.WriteVarSize(w, s.TimestampsOffset)
bin.WriteVarSize(w, len(s.Timestamps))
w.Write(s.Timestamps)
bin.WriteVarSize(w, s.ValuesOffset)
bin.WriteVarSize(w, len(s.Values))
w.Write(s.Values)
}
func (s *MeasuresAppendRecord) Parse(r *bytes.Buffer) (err error) {
var size int
s.MetricID, err = bin.ReadUint32(r)
if err != nil {
return
}
s.TimestampsOffset, err = bin.ReadVarSize(r)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.Timestamps, err = bin.ReadN(r, size)
if err != nil {
return
}
s.ValuesOffset, err = bin.ReadVarSize(r)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.Values, err = bin.ReadN(r, size)
if err != nil {
return
}
return
}
type MetricAddRecord struct {
MetricID uint32
MetricType qb.MetricType
FracDigits int
}
func (s MetricAddRecord) Pack(w io.Writer) {
arr := []byte{
CodeMetricAdd,
0, 0, 0, 0, //
byte(s.MetricType),
byte(s.FracDigits),
}
bin.PutUint32(arr[1:], s.MetricID)
w.Write(arr)
}
func (s *MetricAddRecord) Parse(src io.Reader) (err error) {
arr, err := bin.ReadN(src, 6)
if err != nil {
return
}
s.MetricID, _ = bin.GetUint32(arr)
s.MetricType = qb.MetricType(arr[4])
s.FracDigits = int(arr[5])
return nil
}
type MetricDeleteRecord struct {
MetricID uint32
FreeIndexPages []uint32
FreeDataPages []uint32
}
func (s MetricDeleteRecord) Pack(w io.Writer) {
arr := []byte{
CodeMetricDelete,
0, 0, 0, 0, // metricID
}
bin.PutUint32(arr[1:], s.MetricID)
w.Write(arr)
// fix
// bin.WriteVarSize(w, len(s.FreePageNumbers))
// for _, pageNo := range s.FreePageNumbers {
// bin.WriteUint32(w, pageNo)
// }
}
func (s *MetricDeleteRecord) Parse(src io.Reader) (err error) {
s.MetricID, err = bin.ReadUint32(src)
if err != nil {
return
}
// free pages
// fix
// qty, err := bin.ReadVarSize(src)
// if err != nil {
// return
// }
// for range qty {
// var pageNo uint32
// pageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// //s.FreePageNumbers = append(s.FreePageNumbers, pageNo)
// }
return nil
}
// fix - add changed pages
type DeletedMeasures struct {
MetricID uint32
FreePageNumbers []uint32
}
func (s DeletedMeasures) Pack(w io.Writer) {
arr := []byte{
CodeMeasuresDelete,
0, 0, 0, 0,
}
bin.PutUint32(arr[1:], s.MetricID)
w.Write(arr)
bin.WriteVarSize(w, len(s.FreePageNumbers))
for _, pageNo := range s.FreePageNumbers {
bin.WriteUint32(w, pageNo)
}
}
func (s *DeletedMeasures) Parse(src io.Reader) (err error) {
s.MetricID, err = bin.ReadUint32(src)
if err != nil {
return
}
// free pages
qty, err := bin.ReadVarSize(src)
if err != nil {
return
}
for range qty {
var pageNo uint32
pageNo, err = bin.ReadUint32(src)
if err != nil {
return
}
s.FreePageNumbers = append(s.FreePageNumbers, pageNo)
}
return nil
}
// type AppendedMeasure struct {
// MetricID uint32
// Timestamp uint32
// Value float64
// }
// func (s AppendedMeasure) Pack(w io.Writer) {
// arr := []byte{
// CodeAppendedMeasure,
// 0, 0, 0, 0, // metricID
// 0, 0, 0, 0, // timestamp
// 0, 0, 0, 0, 0, 0, 0, 0, // value
// }
// bin.PutUint32(arr[1:], s.MetricID)
// bin.PutUint32(arr[5:], s.Timestamp)
// bin.PutFloat64(arr[9:], s.Value)
// w.Write(arr)
// }
// func (s *AppendedMeasure) Parse(src io.Reader) (err error) {
// arr, err := bin.ReadN(src, 16)
// if err != nil {
// return
// }
// s.MetricID, _ = bin.GetUint32(arr[0:])
// s.Timestamp, _ = bin.GetUint32(arr[4:])
// s.Value, _ = bin.GetFloat64(arr[8:])
// return nil
// }
// type AppendedMeasures struct {
// MetricID uint32
// Measures []proto.Measure
// }
// func (s AppendedMeasures) Pack(w io.Writer) {
// arr := []byte{
// CodeAppendedMeasures,
// 0, 0, 0, 0, // metricID
// 0, 0, // qty
// }
// bin.PutUint32(arr[1:], s.MetricID)
// bin.PutUint16(arr[5:], uint16(len(s.Measures)))
// w.Write(arr)
// for _, measure := range s.Measures {
// bin.WriteUint32(w, measure.Timestamp)
// bin.WriteFloat64(w, measure.Value)
// }
// }
// func (s *AppendedMeasures) Parse(src io.Reader) (err error) {
// s.MetricID, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// qty, err := bin.ReadUint16(src)
// if err != nil {
// return
// }
// for range qty {
// var measure proto.Measure
// measure.Timestamp, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// measure.Value, err = bin.ReadFloat64(src)
// if err != nil {
// return
// }
// s.Measures = append(s.Measures, measure)
// }
// return nil
// }
// type AppendedPagesReq struct {
// Legs []atree.PathLeg
// MetricID uint32
// Timestamp uint32 // last measure
// Value float64 // last measure
// //RootPageNo uint32
// LastPageNo uint32
// Pages []atree.NotLinkedDataPage
// TimestampsChunks [][]byte
// TimestampsSize uint16
// ValuesChunks [][]byte
// ValuesSize uint16
// }
/*
4b metricID
4b timestamp
8b value
4b LastPageNo (0 if no new root page)
4b newRootPageNo (0 if no new root page)
Nb - varsize (pages qty)
[
4b pageNo
1b isReused
Xb page (page size)
]
2b - timestamps size (not filled page)
Nb - timestamps payload
2b - values size (not filled page)
Nb - values payload
*/
// type AppendedPages struct {
// MetricID uint32
// Timestamp uint32 // last measure
// Value float64 // last measure
// NewRootPageNo uint32
// LastPageNo uint32
// Pages []atree.PageToWrite
// TimestampsChunks [][]byte
// TimestampsSize int
// ValuesChunks [][]byte
// ValuesSize int
// }
// func (s *AppendedPages) Pack(w io.Writer) {
// w.Write([]byte{
// CodeAppendedPages,
// })
// bin.WriteUint32(w, s.MetricID)
// bin.WriteUint32(w, s.Timestamp)
// bin.WriteFloat64(w, s.Value)
// bin.WriteUint32(w, s.LastPageNo)
// bin.WriteUint32(w, s.NewRootPageNo)
// //
// bin.WriteVarSize(w, len(s.Pages))
// for _, p := range s.Pages {
// bin.WriteUint32(w, p.PageNo)
// if p.IsReused {
// w.Write([]byte{1})
// } else {
// w.Write([]byte{0})
// }
// w.Write(p.Data)
// }
// // timestamps
// writeChunkedPayloadTo(s.TimestampsChunks, int(s.TimestampsSize), w)
// // values
// writeChunkedPayloadTo(s.ValuesChunks, int(s.ValuesSize), w)
// }
// func (s *AppendedPages) Parse(src io.Reader) (err error) {
// s.MetricID, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// s.Timestamp, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// s.Value, err = bin.ReadFloat64(src)
// if err != nil {
// return
// }
// s.LastPageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// s.NewRootPageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// pagesQty, err := bin.ReadVarSize(src)
// if err != nil {
// return
// }
// for range pagesQty {
// var page atree.PageToWrite
// page.PageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// page.IsReused, err = bin.ReadBool(src)
// if err != nil {
// return
// }
// page.Data, err = bin.ReadN(src, atree.PageSize)
// if err != nil {
// return
// }
// s.Pages = append(s.Pages, page)
// }
// s.TimestampsChunks, s.TimestampsSize, err = readChunkedPayload(src)
// if err != nil {
// return
// }
// s.ValuesChunks, s.ValuesSize, err = readChunkedPayload(src)
// if err != nil {
// return
// }
// return nil
// }

13
txlog/writer.go → storage/wal_writer.go
View File

@@ -1,4 +1,4 @@
package txlog
package storage
import (
"bytes"
@@ -31,12 +31,11 @@ const (
)
const (
CodeAddedMetric byte = 1
CodeDeletedMetric byte = 2
CodeAppendedMeasure byte = 4
CodeAppendedMeasures byte = 5
CodeAppendedPages byte = 6
CodeDeletedMeasures byte = 7
CodeMetricAdd byte = 1
CodeMetricDelete byte = 2
CodeMeasuresAppendWithGrow byte = 5
CodeMeasuresAppend = 6
CodeMeasuresDelete byte = 7
)
type FreeList interface {

48
transform/aggregate.go
View File

@@ -6,8 +6,8 @@ import (
"io"
"time"
diploma "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/bin"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/timeutil"
)
@@ -15,7 +15,7 @@ import (
type InstantPeriodsWriterOptions struct {
Dst io.Writer
GroupBy diploma.GroupBy
GroupBy qb.GroupBy
AggregateFuncs byte
FirstHourOfDay int
}
@@ -24,7 +24,7 @@ type InstantPeriodsWriter struct {
aggregateFuncs byte
arr []byte
responder *ChunkedResponder
groupBy diploma.GroupBy
groupBy qb.GroupBy
firstHourOfDay int
time2period func(uint32) time.Time
currentPeriod time.Time
@@ -45,13 +45,13 @@ func NewInstantPeriodsWriter(opt InstantPeriodsWriterOptions) (*InstantPeriodsWr
}
// Считаю q, чтобы заранее выделить массив для упаковки периодов
var q int
if (opt.AggregateFuncs & diploma.AggregateMin) == diploma.AggregateMin {
if (opt.AggregateFuncs & qb.AggregateMin) == qb.AggregateMin {
q++
}
if (opt.AggregateFuncs & diploma.AggregateMax) == diploma.AggregateMax {
if (opt.AggregateFuncs & qb.AggregateMax) == qb.AggregateMax {
q++
}
if (opt.AggregateFuncs & diploma.AggregateAvg) == diploma.AggregateAvg {
if (opt.AggregateFuncs & qb.AggregateAvg) == qb.AggregateAvg {
q++
}
@@ -70,17 +70,17 @@ func NewInstantPeriodsWriter(opt InstantPeriodsWriterOptions) (*InstantPeriodsWr
}
switch opt.GroupBy {
case diploma.GroupByHour:
case qb.GroupByHour:
s.time2period = groupByHour
case diploma.GroupByDay:
case qb.GroupByDay:
if s.firstHourOfDay > 0 {
s.time2period = s.groupByDayUsingFHD
} else {
s.time2period = groupByDay
}
case diploma.GroupByMonth:
case qb.GroupByMonth:
if s.firstHourOfDay > 0 {
s.time2period = s.groupByMonthUsingFHD
} else {
@@ -174,12 +174,12 @@ func (s *InstantPeriodsWriter) feed(timestamp uint32, value float64, isBuffer bo
func (s *InstantPeriodsWriter) decrementPeriod() {
switch s.groupBy {
case diploma.GroupByHour:
case qb.GroupByHour:
s.currentPeriod = s.currentPeriod.Add(-1 * time.Hour)
//fmt.Println("decrement")
case diploma.GroupByDay:
case qb.GroupByDay:
s.currentPeriod = s.currentPeriod.AddDate(0, 0, -1)
case diploma.GroupByMonth:
case qb.GroupByMonth:
s.currentPeriod = s.currentPeriod.AddDate(0, -1, 0)
}
}
@@ -210,15 +210,15 @@ func (s *InstantPeriodsWriter) packPeriod(timestamp uint32) {
bin.PutUint32(s.arr[8:], s.endTimestamp)
pos := 12
if (s.aggregateFuncs & diploma.AggregateMin) == diploma.AggregateMin {
if (s.aggregateFuncs & qb.AggregateMin) == qb.AggregateMin {
bin.PutFloat64(s.arr[pos:], s.min)
pos += 8
}
if (s.aggregateFuncs & diploma.AggregateMax) == diploma.AggregateMax {
if (s.aggregateFuncs & qb.AggregateMax) == qb.AggregateMax {
bin.PutFloat64(s.arr[pos:], s.max)
pos += 8
}
if (s.aggregateFuncs & diploma.AggregateAvg) == diploma.AggregateAvg {
if (s.aggregateFuncs & qb.AggregateAvg) == qb.AggregateAvg {
bin.PutFloat64(s.arr[pos:], s.total/float64(s.entries))
}
}
@@ -236,7 +236,7 @@ type CumulativePeriodsWriter struct {
responder *ChunkedResponder
firstHourOfDay int
currentPeriod time.Time
groupBy diploma.GroupBy
groupBy qb.GroupBy
time2period func(uint32) time.Time
endTimestamp uint32
endValue float64
@@ -246,7 +246,7 @@ type CumulativePeriodsWriter struct {
type CumulativePeriodsWriterOptions struct {
Dst io.Writer
GroupBy diploma.GroupBy
GroupBy qb.GroupBy
FirstHourOfDay int
}
@@ -271,17 +271,17 @@ func NewCumulativePeriodsWriter(opt CumulativePeriodsWriterOptions) (*Cumulative
}
switch opt.GroupBy {
case diploma.GroupByHour:
case qb.GroupByHour:
s.time2period = groupByHour
case diploma.GroupByDay:
case qb.GroupByDay:
if s.firstHourOfDay > 0 {
s.time2period = s.groupByDayUsingFHD
} else {
s.time2period = groupByDay
}
case diploma.GroupByMonth:
case qb.GroupByMonth:
if s.firstHourOfDay > 0 {
s.time2period = s.groupByMonthUsingFHD
} else {
@@ -360,12 +360,12 @@ func (s *CumulativePeriodsWriter) feed(timestamp uint32, value float64, isBuffer
func (s *CumulativePeriodsWriter) decrementPeriod() {
switch s.groupBy {
case diploma.GroupByHour:
case qb.GroupByHour:
s.currentPeriod = s.currentPeriod.Add(-1 * time.Hour)
//fmt.Println("decrement")
case diploma.GroupByDay:
case qb.GroupByDay:
s.currentPeriod = s.currentPeriod.AddDate(0, 0, -1)
case diploma.GroupByMonth:
case qb.GroupByMonth:
s.currentPeriod = s.currentPeriod.AddDate(0, -1, 0)
}
}

2
transform/raw.go
View File

@@ -3,7 +3,7 @@ package transform
import (
"io"
"gordenko.dev/dima/qb/bin"
bin "gordenko.dev/dima/bin/little"
)
// CURRENT VALUE WRITER

2
transform/responder.go
View File

@@ -5,7 +5,7 @@ import (
"fmt"
"io"
"gordenko.dev/dima/qb/bin"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb/proto"
)

322
txlog/txlog.go
View File

@@ -1,322 +0,0 @@
package txlog
import (
"io"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
)
var (
indexRecordSize = 8
)
type AddedMetric struct {
MetricID uint32
MetricType qb.MetricType
FracDigits int
}
func (s AddedMetric) Pack(w io.Writer) {
arr := []byte{
CodeAddedMetric,
0, 0, 0, 0, //
byte(s.MetricType),
byte(s.FracDigits),
}
bin.PutUint32(arr[1:], s.MetricID)
w.Write(arr)
}
func (s *AddedMetric) Parse(src io.Reader) (err error) {
arr, err := bin.ReadN(src, 6)
if err != nil {
return
}
s.MetricID, _ = bin.GetUint32(arr)
s.MetricType = qb.MetricType(arr[4])
s.FracDigits = int(arr[5])
return nil
}
type DeletedMetric struct {
MetricID uint32
FreeIndexPages []uint32
FreeDataPages []uint32
}
func (s DeletedMetric) Pack(w io.Writer) {
arr := []byte{
CodeDeletedMetric,
0, 0, 0, 0, // metricID
}
bin.PutUint32(arr[1:], s.MetricID)
w.Write(arr)
// fix
// bin.WriteVarSize(w, len(s.FreePageNumbers))
// for _, pageNo := range s.FreePageNumbers {
// bin.WriteUint32(w, pageNo)
// }
}
func (s *DeletedMetric) Parse(src io.Reader) (err error) {
s.MetricID, err = bin.ReadUint32(src)
if err != nil {
return
}
// free pages
// fix
// qty, err := bin.ReadVarSize(src)
// if err != nil {
// return
// }
// for range qty {
// var pageNo uint32
// pageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// //s.FreePageNumbers = append(s.FreePageNumbers, pageNo)
// }
return nil
}
// fix - add changed pages
type DeletedMeasures struct {
MetricID uint32
FreePageNumbers []uint32
}
func (s DeletedMeasures) Pack(w io.Writer) {
arr := []byte{
CodeDeletedMeasures,
0, 0, 0, 0,
}
bin.PutUint32(arr[1:], s.MetricID)
w.Write(arr)
bin.WriteVarSize(w, len(s.FreePageNumbers))
for _, pageNo := range s.FreePageNumbers {
bin.WriteUint32(w, pageNo)
}
}
func (s *DeletedMeasures) Parse(src io.Reader) (err error) {
s.MetricID, err = bin.ReadUint32(src)
if err != nil {
return
}
// free pages
qty, err := bin.ReadVarSize(src)
if err != nil {
return
}
for range qty {
var pageNo uint32
pageNo, err = bin.ReadUint32(src)
if err != nil {
return
}
s.FreePageNumbers = append(s.FreePageNumbers, pageNo)
}
return nil
}
// type AppendedMeasure struct {
// MetricID uint32
// Timestamp uint32
// Value float64
// }
// func (s AppendedMeasure) Pack(w io.Writer) {
// arr := []byte{
// CodeAppendedMeasure,
// 0, 0, 0, 0, // metricID
// 0, 0, 0, 0, // timestamp
// 0, 0, 0, 0, 0, 0, 0, 0, // value
// }
// bin.PutUint32(arr[1:], s.MetricID)
// bin.PutUint32(arr[5:], s.Timestamp)
// bin.PutFloat64(arr[9:], s.Value)
// w.Write(arr)
// }
// func (s *AppendedMeasure) Parse(src io.Reader) (err error) {
// arr, err := bin.ReadN(src, 16)
// if err != nil {
// return
// }
// s.MetricID, _ = bin.GetUint32(arr[0:])
// s.Timestamp, _ = bin.GetUint32(arr[4:])
// s.Value, _ = bin.GetFloat64(arr[8:])
// return nil
// }
// type AppendedMeasures struct {
// MetricID uint32
// Measures []proto.Measure
// }
// func (s AppendedMeasures) Pack(w io.Writer) {
// arr := []byte{
// CodeAppendedMeasures,
// 0, 0, 0, 0, // metricID
// 0, 0, // qty
// }
// bin.PutUint32(arr[1:], s.MetricID)
// bin.PutUint16(arr[5:], uint16(len(s.Measures)))
// w.Write(arr)
// for _, measure := range s.Measures {
// bin.WriteUint32(w, measure.Timestamp)
// bin.WriteFloat64(w, measure.Value)
// }
// }
// func (s *AppendedMeasures) Parse(src io.Reader) (err error) {
// s.MetricID, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// qty, err := bin.ReadUint16(src)
// if err != nil {
// return
// }
// for range qty {
// var measure proto.Measure
// measure.Timestamp, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// measure.Value, err = bin.ReadFloat64(src)
// if err != nil {
// return
// }
// s.Measures = append(s.Measures, measure)
// }
// return nil
// }
// type AppendedPagesReq struct {
// Legs []atree.PathLeg
// MetricID uint32
// Timestamp uint32 // last measure
// Value float64 // last measure
// //RootPageNo uint32
// LastPageNo uint32
// Pages []atree.NotLinkedDataPage
// TimestampsChunks [][]byte
// TimestampsSize uint16
// ValuesChunks [][]byte
// ValuesSize uint16
// }
/*
4b metricID
4b timestamp
8b value
4b LastPageNo (0 if no new root page)
4b newRootPageNo (0 if no new root page)
Nb - varsize (pages qty)
[
4b pageNo
1b isReused
Xb page (page size)
]
2b - timestamps size (not filled page)
Nb - timestamps payload
2b - values size (not filled page)
Nb - values payload
*/
// type AppendedPages struct {
// MetricID uint32
// Timestamp uint32 // last measure
// Value float64 // last measure
// NewRootPageNo uint32
// LastPageNo uint32
// Pages []atree.PageToWrite
// TimestampsChunks [][]byte
// TimestampsSize int
// ValuesChunks [][]byte
// ValuesSize int
// }
// func (s *AppendedPages) Pack(w io.Writer) {
// w.Write([]byte{
// CodeAppendedPages,
// })
// bin.WriteUint32(w, s.MetricID)
// bin.WriteUint32(w, s.Timestamp)
// bin.WriteFloat64(w, s.Value)
// bin.WriteUint32(w, s.LastPageNo)
// bin.WriteUint32(w, s.NewRootPageNo)
// //
// bin.WriteVarSize(w, len(s.Pages))
// for _, p := range s.Pages {
// bin.WriteUint32(w, p.PageNo)
// if p.IsReused {
// w.Write([]byte{1})
// } else {
// w.Write([]byte{0})
// }
// w.Write(p.Data)
// }
// // timestamps
// writeChunkedPayloadTo(s.TimestampsChunks, int(s.TimestampsSize), w)
// // values
// writeChunkedPayloadTo(s.ValuesChunks, int(s.ValuesSize), w)
// }
// func (s *AppendedPages) Parse(src io.Reader) (err error) {
// s.MetricID, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// s.Timestamp, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// s.Value, err = bin.ReadFloat64(src)
// if err != nil {
// return
// }
// s.LastPageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// s.NewRootPageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// pagesQty, err := bin.ReadVarSize(src)
// if err != nil {
// return
// }
// for range pagesQty {
// var page atree.PageToWrite
// page.PageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// page.IsReused, err = bin.ReadBool(src)
// if err != nil {
// return
// }
// page.Data, err = bin.ReadN(src, atree.PageSize)
// if err != nil {
// return
// }
// s.Pages = append(s.Pages, page)
// }
// s.TimestampsChunks, s.TimestampsSize, err = readChunkedPayload(src)
// if err != nil {
// return
// }
// s.ValuesChunks, s.ValuesSize, err = readChunkedPayload(src)
// if err != nil {
// return
// }
// return nil
// }
// PACK

305
txlog/wal_records.go
View File

@@ -1,305 +0,0 @@
package txlog
import (
"bytes"
bin "gordenko.dev/dima/bin/little"
)
type CompletedDataPage struct {
PageNo uint32
Reused bool
PrevPageNo uint32
Checksum uint32
TimestampsOffset int // offset on 1st page
Timestamps []byte
ValuesOffset int // offset on 1st page
Values []byte
}
type CompletedIndexPage struct {
PageNo uint32
Reused bool
Checksum uint32
Records []byte // offset не потрібен, оскільки Records додаються в кінець
}
type ChangedIndexLevel struct {
CompletedIndexPage *CompletedIndexPage
IndexPages []SealedIndexPage
TailRecords []byte // payload only
}
type WALRecordAppendMeasures struct {
MetricID uint32
CompletedDataPage CompletedDataPage
DataPages []SealedDataPage
ChangedIndexLevels []ChangedIndexLevel
TailTimestamps []byte // data level tail
TailValues []byte // data level tail
}
/*
Format appended measures:
1b - tx type
4b - metricID
4b - pageNo
1b - reused
4b - prevPageNo
4b - page crc32
Nb - (varsize) timestamps offset
Nb - (varsize) timestamps tail size on 1st filled data page
Nb - timestamps tail on 1st filled data page
Nb - (varsize) values offset
Nb - (varsize) values tail size on 1st filled data page
Nb - values tail on 1st filled data page
Nb - (varsize) filled data pages qty
[
4b - pageNo
1b - reused
Nb - (data page size) page content
]
Nb - (varsize) timestamps payload size on tail
Nb - timestamps payload
Nb - (varsize) values payload size on tail
Nb - values payload
Nb - (varsize) qty of index levels
[
// NOTE!
// - idx = 0 - zeroLevel
// - skipped = maxRecordsOnIndexPage - records count
4b - pageNo of 1st index page
1b - reused
4b - page crc32
Nb - (varsize) records tail count on 1st filled index page
Nb - records tail on 1st filled index page
Nb - (varsize) qty of level filled pages
[
4b - pageNo
1b - reused
Nb - (index page size) page content
]
Nb - (varsize) size of records level tail
Nb - level tail records
]
*/
func (s WALRecordAppendMeasures) WriteTo(w *bytes.Buffer) {
w.WriteByte(CodeAppendedMeasures)
bin.WriteUint32(w, s.MetricID)
// completed data page
completed := s.CompletedDataPage
bin.WriteUint32(w, completed.PageNo)
bin.WriteBool(w, completed.Reused)
bin.WriteUint32(w, completed.PrevPageNo)
bin.WriteUint32(w, completed.Checksum)
bin.WriteVarSize(w, completed.TimestampsOffset)
bin.WriteVarSize(w, len(completed.Timestamps))
w.Write(completed.Timestamps)
bin.WriteVarSize(w, completed.ValuesOffset)
bin.WriteVarSize(w, len(completed.Values))
w.Write(completed.Values)
// data pages
bin.WriteVarSize(w, len(s.DataPages))
for _, p := range s.DataPages {
bin.WriteUint32(w, p.PageNo)
bin.WriteBool(w, p.Reused)
bin.WriteVarSize(w, len(p.Content))
w.Write(p.Content)
}
// data tail
bin.WriteVarSize(w, len(s.TailTimestamps))
w.Write(s.TailTimestamps)
bin.WriteVarSize(w, len(s.TailValues))
w.Write(s.TailValues)
// changed levels
bin.WriteVarSize(w, len(s.ChangedIndexLevels))
for _, level := range s.ChangedIndexLevels {
// completed index page
completed := level.CompletedIndexPage
bin.WriteUint32(w, completed.PageNo)
bin.WriteBool(w, completed.Reused)
bin.WriteUint32(w, completed.Checksum)
bin.WriteVarSize(w, len(completed.Records)/indexRecordSize)
w.Write(completed.Records)
// full index pages
for _, p := range level.IndexPages {
bin.WriteUint32(w, p.PageNo)
bin.WriteBool(w, p.Reused)
bin.WriteVarSize(w, len(p.Content))
w.Write(p.Content)
}
// level tail
bin.WriteVarSize(w, len(level.TailRecords)/indexRecordSize)
w.Write(level.TailRecords)
}
}
func (s *WALRecordAppendMeasures) Parse(r *bytes.Buffer) (err error) {
var (
size int
recordsCount int
)
s.MetricID, err = bin.ReadUint32(r)
if err != nil {
return
}
s.CompletedDataPage.PageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
s.CompletedDataPage.Reused, err = bin.ReadBool(r)
if err != nil {
return
}
s.CompletedDataPage.PrevPageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
s.CompletedDataPage.Checksum, err = bin.ReadUint32(r)
if err != nil {
return
}
s.CompletedDataPage.TimestampsOffset, err = bin.ReadVarSize(r)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.CompletedDataPage.Timestamps, err = bin.ReadN(r, size)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.CompletedDataPage.Values, err = bin.ReadN(r, size)
if err != nil {
return
}
// data pages
dataPagesQty, err := bin.ReadVarSize(r)
if err != nil {
return
}
for range dataPagesQty {
var (
p SealedDataPage
)
p.PageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
p.Reused, err = bin.ReadBool(r)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
p.Content, err = bin.ReadN(r, size)
if err != nil {
return
}
s.DataPages = append(s.DataPages, p)
}
// data tail
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.TailTimestamps, err = bin.ReadN(r, size)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.TailValues, err = bin.ReadN(r, size)
if err != nil {
return
}
// changed levels
changedIndexLevelsCount, err := bin.ReadVarSize(r)
if err != nil {
return
}
for range changedIndexLevelsCount {
var (
indexPagesCount int
level ChangedIndexLevel
)
// completed index page
level.CompletedIndexPage.PageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
level.CompletedIndexPage.Reused, err = bin.ReadBool(r)
if err != nil {
return
}
level.CompletedIndexPage.Checksum, err = bin.ReadUint32(r)
if err != nil {
return
}
recordsCount, err = bin.ReadVarSize(r)
if err != nil {
return
}
level.CompletedIndexPage.Records, err = bin.ReadN(r, recordsCount*indexRecordSize)
if err != nil {
return
}
//
indexPagesCount, err = bin.ReadVarSize(r)
if err != nil {
return
}
for range indexPagesCount {
var p SealedIndexPage
p.PageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
p.Reused, err = bin.ReadBool(r)
if err != nil {
return
}
p.Checksum, err = bin.ReadUint32(r)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
p.Content, err = bin.ReadN(r, size)
if err != nil {
return
}
level.IndexPages = append(level.IndexPages, p)
}
recordsCount, err = bin.ReadVarSize(r)
if err != nil {
return
}
level.TailRecords, err = bin.ReadN(r, recordsCount*indexRecordSize)
if err != nil {
return
}
s.ChangedIndexLevels = append(s.ChangedIndexLevels, level)
}
return
}