dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-06-10 06:18:45 +03:00
parent 4089ca40c9
commit 9ec86282ad
47 changed files with 3493 additions and 4502 deletions
Download Patch File Download Diff File Expand all files Collapse all files

332
atree/atree.go
View File

@@ -1,49 +1,12 @@
package atree package atree
import ( import (
"errors"
"fmt"
"os" "os"
"path/filepath"
"sync" "sync"
"gordenko.dev/dima/qb/bin"
) )
const ( const (
PageTypeData = 1
PageTypeIndex = 2
filePerm = 0770 filePerm = 0770
// common
crc32Idx = PageSize - 4
pageTypeIdx = PageSize - 5
// index page
indexCRC32Idx = IndexPageSize - 4
isLastLevelIdx = IndexPageSize - 5
indexRecordsQtyIdx = IndexPageSize - 7
indexRecordSize = 8
// data page
dataCRC32Idx = DataPageSize - 4
prevPageIdx = DataPageSize - 8
valuesSizeIdx = DataPageSize - 10
timestampsSizeIdx = DataPageSize - 12
timestampSize = 4
pairSize = timestampSize + PageNoSize
indexFooterIdx = indexRecordsQtyIdx
dataFooterIdx = timestampsSizeIdx
PageSize = 8192
DataPageSize = 8192
IndexPageSize = 1024
PageNoSize = 4
//
DataPagePayloadSize int = dataFooterIdx
) )
type _page struct { type _page struct {
@@ -70,48 +33,49 @@ type Options struct {
} }
func New(opt Options) (*Atree, error) { func New(opt Options) (*Atree, error) {
if opt.Dir == "" { // if opt.Dir == "" {
return nil, errors.New("Dir option is required") // return nil, errors.New("Dir option is required")
} // }
if opt.DatabaseName == "" { // if opt.DatabaseName == "" {
return nil, errors.New("DatabaseName option is required") // return nil, errors.New("DatabaseName option is required")
} // }
// открываю или создаю dbName.data и dbName.index файлы // // открываю или создаю dbName.data и dbName.index файлы
var ( // var (
fileName = filepath.Join(opt.Dir, opt.DatabaseName+".db") // fileName = filepath.Join(opt.Dir, opt.DatabaseName+".db")
file *os.File // file *os.File
allocatedPagesQty uint32 // allocatedPagesQty uint32
) // )
// При создании data файла сразу создается индекс, поэтому корректное // // При создании data файла сразу создается индекс, поэтому корректное
// состояние БД: либо оба файла есть, либо ни одного файла нет. // // состояние БД: либо оба файла есть, либо ни одного файла нет.
isDataExist, err := isFileExist(fileName) // isDataExist, err := isFileExist(fileName)
if err != nil { // if err != nil {
return nil, fmt.Errorf("check data file is exist: %s", err) // return nil, fmt.Errorf("check data file is exist: %s", err)
} // }
if isDataExist { // if isDataExist {
file, allocatedPagesQty, err = openFile(fileName, PageSize) // file, allocatedPagesQty, err = openFile(fileName, PageSize)
if err != nil { // if err != nil {
return nil, fmt.Errorf("open data file: %s", err) // return nil, fmt.Errorf("open data file: %s", err)
} // }
} else { // } else {
// нет файла // // нет файла
file, err = os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, filePerm) // file, err = os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, filePerm)
if err != nil { // if err != nil {
return nil, err // return nil, err
} // }
} // }
tree := &Atree{ // tree := &Atree{
file: file, // file: file,
allocatedPagesQty: allocatedPagesQty, // allocatedPagesQty: allocatedPagesQty,
pages: make(map[uint32]*_page), // pages: make(map[uint32]*_page),
pageWaits: make(map[uint32][]chan readResult), // pageWaits: make(map[uint32][]chan readResult),
readSignalCh: make(chan struct{}, 1), // readSignalCh: make(chan struct{}, 1),
writeSignalCh: make(chan struct{}, 1), // writeSignalCh: make(chan struct{}, 1),
} // }
return tree, nil // return tree, nil
return nil, nil
} }
func (s *Atree) Run() { func (s *Atree) Run() {
@@ -122,27 +86,28 @@ func (s *Atree) Run() {
// FIND // FIND
func (s *Atree) findDataPage(rootPageNo uint32, timestamp uint32) (uint32, []byte, error) { func (s *Atree) findDataPage(rootPageNo uint32, timestamp uint32) (uint32, []byte, error) {
indexPageNo := rootPageNo // indexPageNo := rootPageNo
for { // for {
buf, err := s.fetchIndexPage(indexPageNo) // buf, err := s.fetchIndexPage(indexPageNo)
if err != nil { // if err != nil {
return 0, nil, fmt.Errorf("fetchIndexPage(%d): %s", indexPageNo, err) // return 0, nil, fmt.Errorf("fetchIndexPage(%d): %s", indexPageNo, err)
} // }
foundPageNo := findPageNo(buf, timestamp) // foundPageNo := findPageNo(buf, timestamp)
s.releasePage(indexPageNo) // s.releasePage(indexPageNo)
// fix // // fix
if buf[isLastLevelIdx] == 1 { // if buf[isLastLevelIdx] == 1 {
buf, err := s.fetchDataPage(foundPageNo) // buf, err := s.fetchDataPage(foundPageNo)
if err != nil { // if err != nil {
return 0, nil, fmt.Errorf("fetchDataPage(%d): %s", foundPageNo, err) // return 0, nil, fmt.Errorf("fetchDataPage(%d): %s", foundPageNo, err)
} // }
return foundPageNo, buf, nil // return foundPageNo, buf, nil
} // }
// вглубь // // вглубь
indexPageNo = foundPageNo // indexPageNo = foundPageNo
} // }
return 0, nil, nil
} }
type PathLeg struct { type PathLeg struct {
@@ -156,37 +121,38 @@ type PathToDataPage struct {
} }
func (s *Atree) FindPathToLastPage(rootPageNo uint32) (_ PathToDataPage, err error) { func (s *Atree) FindPathToLastPage(rootPageNo uint32) (_ PathToDataPage, err error) {
var ( // var (
pageNo = rootPageNo // pageNo = rootPageNo
legs []PathLeg // legs []PathLeg
) // )
for { // for {
var buf []byte // var buf []byte
buf, err = s.fetchIndexPage(pageNo) // buf, err = s.fetchIndexPage(pageNo)
if err != nil { // if err != nil {
err = fmt.Errorf("FetchIndexPage(%d): %s", pageNo, err) // err = fmt.Errorf("FetchIndexPage(%d): %s", pageNo, err)
// return
// }
// legs = append(legs, PathLeg{
// PageNo: pageNo,
// Data: buf,
// // childIdx не нужен
// })
// foundPageNo := getLastPageNo(buf)
// // fix
// if buf[isLastLevelIdx] == 1 {
// return PathToDataPage{
// Legs: legs,
// LastPageNo: foundPageNo,
// }, nil
// }
// // вглубь
// pageNo = foundPageNo
// }
return return
}
legs = append(legs, PathLeg{
PageNo: pageNo,
Data: buf,
// childIdx не нужен
})
foundPageNo := getLastPageNo(buf)
// fix
if buf[isLastLevelIdx] == 1 {
return PathToDataPage{
Legs: legs,
LastPageNo: foundPageNo,
}, nil
}
// вглубь
pageNo = foundPageNo
}
} }
// DELETE // DELETE
@@ -199,72 +165,76 @@ type Level struct {
} }
func (s *Atree) GetAllPages(rootPageNo uint32) (_ []uint32, err error) { func (s *Atree) GetAllPages(rootPageNo uint32) (_ []uint32, err error) {
var ( // var (
pageNumbers []uint32 // pageNumbers []uint32
levels []*Level // levels []*Level
) // )
buf, err := s.fetchIndexPage(rootPageNo) // buf, err := s.fetchIndexPage(rootPageNo)
if err != nil { // if err != nil {
return nil, fmt.Errorf("fetchIndexPage(%d): %s", rootPageNo, err) // return nil, fmt.Errorf("fetchIndexPage(%d): %s", rootPageNo, err)
} // }
pageNumbers = append(pageNumbers, rootPageNo) // pageNumbers = append(pageNumbers, rootPageNo)
// if buf[isDataPageNumbersIdx] == 1 { // // if buf[isDataPageNumbersIdx] == 1 {
// pageNumbers := listPageNumbers(buf) // // pageNumbers := listPageNumbers(buf)
// dataPages = append(dataPages, pageNumbers...) // // dataPages = append(dataPages, pageNumbers...)
// s.releasePage(rootPageNo) // // s.releasePage(rootPageNo)
// return PageLists{ // // return PageLists{
// DataPages: dataPages, // // DataPages: dataPages,
// IndexPages: indexPages, // // IndexPages: indexPages,
// }, nil // // }, nil
// // }
// childQty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
// levels = append(levels, &Level{
// PageNo: rootPageNo,
// PageData: buf,
// Idx: 0,
// ChildQty: int(childQty),
// })
// for {
// if len(levels) == 0 {
// return pageNumbers, nil
// } // }
levels = append(levels, &Level{ // lastIdx := len(levels) - 1
PageNo: rootPageNo, // level := levels[lastIdx]
PageData: buf,
Idx: 0,
ChildQty: bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]),
})
for { // if level.Idx < level.ChildQty {
if len(levels) == 0 { // pageNo := getPageNo(level.PageData, level.Idx)
return pageNumbers, nil // level.Idx++
}
lastIdx := len(levels) - 1 // var buf []byte
level := levels[lastIdx] // buf, err = s.fetchPage(pageNo)
// if err != nil {
// return nil, fmt.Errorf("fetchPage(%d): %s", pageNo, err)
// }
// pageNumbers = append(pageNumbers, pageNo)
if level.Idx < level.ChildQty { // if buf[pageTypeIdx] == PageTypeData {
pageNo := getPageNo(level.PageData, level.Idx) // //pageNumbers := listPageNumbers(buf)
level.Idx++ // //dataPages = append(dataPages, pageNumbers...)
// s.releasePage(pageNo)
var buf []byte // } else {
buf, err = s.fetchPage(pageNo) // childQty, _ = bin.GetUint16(buf[indexRecordsQtyIdx:])
if err != nil { // levels = append(levels, &Level{
return nil, fmt.Errorf("fetchPage(%d): %s", pageNo, err) // PageNo: pageNo,
} // PageData: buf,
pageNumbers = append(pageNumbers, pageNo) // Idx: 0,
// ChildQty: int(childQty),
if buf[pageTypeIdx] == PageTypeData { // })
//pageNumbers := listPageNumbers(buf) // }
//dataPages = append(dataPages, pageNumbers...) // } else {
s.releasePage(pageNo) // s.releasePage(level.PageNo)
} else { // levels = levels[:lastIdx]
levels = append(levels, &Level{ // }
PageNo: pageNo, // }
PageData: buf, return
Idx: 0,
ChildQty: bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]),
})
}
} else {
s.releasePage(level.PageNo)
levels = levels[:lastIdx]
}
}
} }
// const ( // const (

167
atree/cursor.go
View File

@@ -5,8 +5,6 @@ import (
"fmt" "fmt"
"gordenko.dev/dima/qb" "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/bin"
"gordenko.dev/dima/qb/enc"
) )
type BackwardCursor struct { type BackwardCursor struct {
@@ -66,50 +64,50 @@ func (s *BackwardCursor) Prev() (uint32, float64, bool, error) {
var ( var (
timestamp uint32 timestamp uint32
value float64 value float64
done bool //done bool
err error //err error
) )
timestamp, done = s.timestampDecompressor.NextValue() // timestamp, done = s.timestampDecompressor.NextValue()
if !done { // if !done {
value, done = s.valueDecompressor.NextValue() // value, done = s.valueDecompressor.NextValue()
if done { // if done {
return 0, 0, false, // return 0, 0, false,
fmt.Errorf("corrupted data page %d: has timestamp, no value", // fmt.Errorf("corrupted data page %d: has timestamp, no value",
s.pageNo) // s.pageNo)
} // }
return timestamp, value, false, nil // return timestamp, value, false, nil
} // }
prevPageNo := bin.GetUint32(s.pageData[prevPageIdx:]) // prevPageNo, _ := bin.GetUint32(s.pageData[prevPageIdx:])
if prevPageNo == 0 { // if prevPageNo == 0 {
return 0, 0, true, nil // return 0, 0, true, nil
} // }
s.atree.releasePage(s.pageNo) // s.atree.releasePage(s.pageNo)
s.pageNo = prevPageNo // s.pageNo = prevPageNo
s.pageData, err = s.atree.fetchDataPage(s.pageNo) // s.pageData, err = s.atree.fetchDataPage(s.pageNo)
if err != nil { // if err != nil {
return 0, 0, false, fmt.Errorf("atree.fetchDataPage(%d): %s", s.pageNo, err) // return 0, 0, false, fmt.Errorf("atree.fetchDataPage(%d): %s", s.pageNo, err)
} // }
err = s.makeDecompressors() // err = s.makeDecompressors()
if err != nil { // if err != nil {
return 0, 0, false, err // return 0, 0, false, err
} // }
timestamp, done = s.timestampDecompressor.NextValue() // timestamp, done = s.timestampDecompressor.NextValue()
if done { // if done {
return 0, 0, false, // return 0, 0, false,
fmt.Errorf("corrupted data page %d: no timestamps", // fmt.Errorf("corrupted data page %d: no timestamps",
s.pageNo) // s.pageNo)
} // }
value, done = s.valueDecompressor.NextValue() // value, done = s.valueDecompressor.NextValue()
if done { // if done {
return 0, 0, false, // return 0, 0, false,
fmt.Errorf("corrupted data page %d: no values", // fmt.Errorf("corrupted data page %d: no values",
s.pageNo) // s.pageNo)
} // }
return timestamp, value, false, nil return timestamp, value, false, nil
} }
@@ -120,68 +118,69 @@ func (s *BackwardCursor) Close() {
// HELPER // HELPER
func (s *BackwardCursor) makeDecompressors() error { func (s *BackwardCursor) makeDecompressors() error {
timestampsPayloadSize := bin.GetUint16(s.pageData[timestampsSizeIdx:]) // timestampsSize, _ := bin.GetUint16(s.pageData[timestampsSizeIdx:])
valuesPayloadSize := bin.GetUint16(s.pageData[valuesSizeIdx:]) // valuesSize, _ := bin.GetUint16(s.pageData[valuesSizeIdx:])
payloadSize := timestampsPayloadSize + valuesPayloadSize // payloadSize := timestampsSize + valuesSize
if payloadSize > dataFooterIdx { // if payloadSize > dataFooterIdx {
return fmt.Errorf("corrupted data page %d: timestamps + values size %d gt payload size", // return fmt.Errorf("corrupted data page %d: timestamps + values size %d gt payload size",
s.pageNo, payloadSize) // s.pageNo, payloadSize)
} // }
s.timestampDecompressor = enc.NewTimeDeltaDecompressor( // s.timestampDecompressor = enc.NewTimeDeltaDecompressor(
s.pageData[:timestampsPayloadSize], // s.pageData[:timestampsSize],
) // )
vbuf := s.pageData[timestampsPayloadSize : timestampsPayloadSize+valuesPayloadSize] // vbuf := s.pageData[timestampsSize : timestampsSize+valuesSize]
switch s.metricType { // switch s.metricType {
case qb.Instant: // case qb.Instant:
s.valueDecompressor = enc.NewInstantDeltaDecompressor( // s.valueDecompressor = enc.NewInstantDeltaDecompressor(
vbuf, s.fracDigits) // vbuf, s.fracDigits)
case qb.Cumulative: // case qb.Cumulative:
s.valueDecompressor = enc.NewCumulativeDeltaDecompressor( // s.valueDecompressor = enc.NewCumulativeDeltaDecompressor(
vbuf, s.fracDigits) // vbuf, s.fracDigits)
default: // default:
return fmt.Errorf("bug: wrong metricType %d", s.metricType) // return fmt.Errorf("bug: wrong metricType %d", s.metricType)
} // }
return nil return nil
} }
func makeDecompressors(pageData []byte, metricType qb.MetricType, fracDigits byte) ( func makeDecompressors(pageData []byte, metricType qb.MetricType, fracDigits byte) (
qb.TimestampDecompressor, qb.ValueDecompressor, error, qb.TimestampDecompressor, qb.ValueDecompressor, error,
) { ) {
timestampsPayloadSize := bin.GetUint16(pageData[timestampsSizeIdx:]) // timestampsSize, _ := bin.GetUint16(pageData[timestampsSizeIdx:])
valuesPayloadSize := bin.GetUint16(pageData[valuesSizeIdx:]) // valuesSize, _ := bin.GetUint16(pageData[valuesSizeIdx:])
payloadSize := timestampsPayloadSize + valuesPayloadSize // payloadSize := timestampsSize + valuesSize
if payloadSize > dataFooterIdx { // if payloadSize > dataFooterIdx {
return nil, nil, fmt.Errorf("corrupted: timestamps + values size %d > payload size", // return nil, nil, fmt.Errorf("corrupted: timestamps + values size %d > payload size",
payloadSize) // payloadSize)
} // }
timestampDecompressor := enc.NewTimeDeltaDecompressor( // timestampDecompressor := enc.NewTimeDeltaDecompressor(
pageData[:timestampsPayloadSize], // pageData[:timestampsSize],
) // )
vbuf := pageData[timestampsPayloadSize : timestampsPayloadSize+valuesPayloadSize] // vbuf := pageData[timestampsSize : timestampsSize+valuesSize]
var valueDecompressor qb.ValueDecompressor // var valueDecompressor qb.ValueDecompressor
switch metricType { // switch metricType {
case qb.Instant: // case qb.Instant:
valueDecompressor = enc.NewInstantDeltaDecompressor( // valueDecompressor = enc.NewInstantDeltaDecompressor(
vbuf, fracDigits) // vbuf, fracDigits)
case qb.Cumulative: // case qb.Cumulative:
valueDecompressor = enc.NewCumulativeDeltaDecompressor( // valueDecompressor = enc.NewCumulativeDeltaDecompressor(
vbuf, fracDigits) // vbuf, fracDigits)
default: // default:
return nil, nil, fmt.Errorf("bug: wrong metricType %d", metricType) // return nil, nil, fmt.Errorf("bug: wrong metricType %d", metricType)
} // }
return timestampDecompressor, valueDecompressor, nil //return timestampDecompressor, valueDecompressor, nil
return nil, nil, nil
} }

167
atree/io.go
View File

@@ -7,8 +7,8 @@ import (
"math" "math"
"os" "os"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb" "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/bin"
"gordenko.dev/dima/qb/util" "gordenko.dev/dima/qb/util"
) )
@@ -35,55 +35,58 @@ func (s *Atree) DeletePages(pageNumbers []uint32) {
} }
func (s *Atree) fetchIndexPage(pageNo uint32) ([]byte, error) { func (s *Atree) fetchIndexPage(pageNo uint32) ([]byte, error) {
buf, err := s.fetchPage(pageNo) // buf, err := s.fetchPage(pageNo)
if err != nil { // if err != nil {
return nil, err // return nil, err
} // }
if buf[pageTypeIdx] != PageTypeIndex { // if buf[pageTypeIdx] != PageTypeIndex {
return nil, fmt.Errorf("wrong pageType %d instead of %d", buf[pageTypeIdx], PageTypeIndex) // return nil, fmt.Errorf("wrong pageType %d instead of %d", buf[pageTypeIdx], PageTypeIndex)
} // }
return buf, nil // return buf, nil
return nil, nil
} }
func (s *Atree) fetchDataPage(pageNo uint32) ([]byte, error) { func (s *Atree) fetchDataPage(pageNo uint32) ([]byte, error) {
buf, err := s.fetchPage(pageNo) // buf, err := s.fetchPage(pageNo)
if err != nil { // if err != nil {
return nil, err // return nil, err
} // }
if buf[pageTypeIdx] != PageTypeIndex { // if buf[pageTypeIdx] != PageTypeIndex {
return nil, fmt.Errorf("wrong pageType %d instead of %d", buf[pageTypeIdx], PageTypeIndex) // return nil, fmt.Errorf("wrong pageType %d instead of %d", buf[pageTypeIdx], PageTypeIndex)
} // }
return buf, nil // return buf, nil
return nil, nil
} }
func (s *Atree) fetchPage(pageNo uint32) ([]byte, error) { func (s *Atree) fetchPage(pageNo uint32) ([]byte, error) {
s.mutex.Lock() // s.mutex.Lock()
p, ok := s.pages[pageNo] // p, ok := s.pages[pageNo]
if ok { // if ok {
p.ReferenceCount++ // p.ReferenceCount++
s.mutex.Unlock() // s.mutex.Unlock()
return p.Buf, nil // return p.Buf, nil
} // }
resultCh := make(chan readResult, 1) // resultCh := make(chan readResult, 1)
s.pageWaits[pageNo] = append(s.pageWaits[pageNo], resultCh) // s.pageWaits[pageNo] = append(s.pageWaits[pageNo], resultCh)
if len(s.pageWaits[pageNo]) == 1 { // if len(s.pageWaits[pageNo]) == 1 {
s.pagesToRead = append(s.pagesToRead, pageNo) // s.pagesToRead = append(s.pagesToRead, pageNo)
s.mutex.Unlock() // s.mutex.Unlock()
select { // select {
case s.readSignalCh <- struct{}{}: // case s.readSignalCh <- struct{}{}:
default: // default:
} // }
} else { // } else {
s.mutex.Unlock() // s.mutex.Unlock()
} // }
result := <-resultCh // result := <-resultCh
if result.Err == nil { // if result.Err == nil {
result.Err = s.verifyCRC(result.Data, PageSize) // result.Err = s.verifyCRC(result.Data, PageSize)
} // }
return result.Data, result.Err // return result.Data, result.Err
return nil, nil
} }
func (s *Atree) releasePage(pageNo uint32) { func (s *Atree) releasePage(pageNo uint32) {
@@ -211,49 +214,49 @@ func (s *Atree) pageReader() {
} }
func (s *Atree) readPages() { func (s *Atree) readPages() {
s.mutex.Lock() // s.mutex.Lock()
if len(s.pagesToRead) == 0 { // if len(s.pagesToRead) == 0 {
s.mutex.Unlock() // s.mutex.Unlock()
return // return
} // }
pagesToRead := s.pagesToRead // pagesToRead := s.pagesToRead
s.pagesToRead = nil // s.pagesToRead = nil
s.mutex.Unlock() // s.mutex.Unlock()
for _, pageNo := range pagesToRead { // for _, pageNo := range pagesToRead {
buf := make([]byte, PageSize) // buf := make([]byte, PageSize)
off := (pageNo - 1) * PageSize // off := (pageNo - 1) * PageSize
n, err := s.file.ReadAt(buf, int64(off)) // n, err := s.file.ReadAt(buf, int64(off))
if n != PageSize { // if n != PageSize {
err = fmt.Errorf("read %d instead of %d", n, PageSize) // err = fmt.Errorf("read %d instead of %d", n, PageSize)
} // }
s.mutex.Lock() // s.mutex.Lock()
resultChannels := s.pageWaits[pageNo] // resultChannels := s.pageWaits[pageNo]
delete(s.pageWaits, pageNo) // delete(s.pageWaits, pageNo)
if err != nil { // if err != nil {
s.mutex.Unlock() // s.mutex.Unlock()
for _, resultCh := range resultChannels { // for _, resultCh := range resultChannels {
resultCh <- readResult{ // resultCh <- readResult{
Err: err, // Err: err,
} // }
} // }
} else { // } else {
s.pages[pageNo] = &_page{ // s.pages[pageNo] = &_page{
// fix - page type // // fix - page type
PageNo: pageNo, // PageNo: pageNo,
Buf: buf, // Buf: buf,
ReferenceCount: len(resultChannels), // ReferenceCount: len(resultChannels),
} // }
s.mutex.Unlock() // s.mutex.Unlock()
for _, resultCh := range resultChannels { // for _, resultCh := range resultChannels {
resultCh <- readResult{ // resultCh <- readResult{
Data: buf, // Data: buf,
} // }
} // }
} // }
} // }
} }
// WRITE // WRITE
@@ -364,7 +367,7 @@ func (s *Atree) verifyCRC(data []byte, pageSize int) error {
var ( var (
pos = pageSize - 4 pos = pageSize - 4
calculatedCRC = util.CalcChecksum(data[:pos]) calculatedCRC = util.CalcChecksum(data[:pos])
storedCRC = bin.GetUint32(data[pos:]) storedCRC, _ = bin.GetUint32(data[pos:])
) )
if calculatedCRC != storedCRC { if calculatedCRC != storedCRC {
return fmt.Errorf("calculatedCRC %d not equal storedCRC %d", return fmt.Errorf("calculatedCRC %d not equal storedCRC %d",

228
atree/misc.go
View File

@@ -1,8 +1,8 @@
package atree package atree
import ( type KeyComparator interface {
"gordenko.dev/dima/qb/bin" CompareTo(int) int
) }
type ValueAtComparator struct { type ValueAtComparator struct {
buf []byte buf []byte
@@ -10,21 +10,22 @@ type ValueAtComparator struct {
} }
func (s ValueAtComparator) CompareTo(elemIdx int) int { func (s ValueAtComparator) CompareTo(elemIdx int) int {
var ( // var (
pos = elemIdx * timestampSize // pos = elemIdx * timestampSize
elem = bin.GetUint32(s.buf[pos:]) // elem, _ = bin.GetUint32(s.buf[pos:])
) // )
if s.timestamp < elem { // if s.timestamp < elem {
return -1 // return -1
} else if s.timestamp > elem { // } else if s.timestamp > elem {
return 1 // return 1
} else { // } else {
return 0 // return 0
} // }
return 213131132
} }
func BinarySearch(qty int, keyComparator bin.KeyComparator) (elemIdx int, isFound bool) { func BinarySearch(qty int, keyComparator KeyComparator) (elemIdx int, isFound bool) {
if qty == 0 { if qty == 0 {
return return
} }
@@ -55,141 +56,94 @@ func BinarySearch(qty int, keyComparator bin.KeyComparator) (elemIdx int, isFoun
} }
} }
// type ChunksToNotLinkedDataPageReq struct { // func getPrevPageNo(buf []byte) uint32 {
// // PrevPageNo uint32 // pageNo, _ := bin.GetUint32(buf[prevPageIdx:])
// TimestampsChunks [][]byte // return pageNo
// TimestampsSize uint16
// ValuesChunks [][]byte
// ValuesSize uint16
// } // }
// func ChunksToNotLinkedDataPage(buf []byte, req ChunksToNotLinkedDataPageReq) { // func findPageNo(buf []byte, timestamp uint32) (pageNo uint32) {
// bin.PutUint16(buf[timestampsSizeIdx:], req.TimestampsSize) // comparator := ValueAtComparator{
// bin.PutUint16(buf[valuesSizeIdx:], req.ValuesSize) // buf: buf,
// timestamp: timestamp,
// var (
// remainingSize = int(req.TimestampsSize)
// pos = 0
// )
// for _, chunk := range req.TimestampsChunks {
// if remainingSize >= len(chunk) {
// copy(buf[pos:], chunk)
// remainingSize -= len(chunk)
// pos += len(chunk)
// } else {
// copy(buf[pos:], chunk[:remainingSize])
// break
// } // }
// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
// elemIdx, _ := BinarySearch(int(qty), comparator)
// pos := indexFooterIdx - (elemIdx+1)*PageNoSize
// pageNo, _ = bin.GetUint32(buf[pos:])
// return
// } // }
// remainingSize = int(req.ValuesSize) // func findPageNoIdx(buf []byte, timestamp uint32) (idx int) {
// pos = int(req.TimestampsSize) // comparator := ValueAtComparator{
// buf: buf,
// for _, chunk := range req.ValuesChunks { // timestamp: timestamp,
// if remainingSize >= len(chunk) {
// copy(buf[pos:], chunk)
// remainingSize -= len(chunk)
// pos += len(chunk)
// } else {
// copy(buf[pos:], chunk[:remainingSize])
// break
// }
// } // }
// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
// elemIdx, _ := BinarySearch(int(qty), comparator)
// return elemIdx
// } // }
// func setPrevPageNo(buf []byte, pageNo uint32) { // func findPageNoForDeleteSince(buf []byte, since uint32) (uint32, int, int) {
// bin.PutUint32(buf[prevPageIdx:], pageNo) // comparator := ValueAtComparator{
// buf: buf,
// timestamp: since,
// }
// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
// elemIdx, _ := BinarySearch(int(qty), comparator)
// pos := elemIdx * timestampSize
// timestamp, _ := bin.GetUint32(buf[pos:])
// if timestamp == since {
// if elemIdx == 0 {
// return 0, int(qty), -1
// }
// elemIdx--
// }
// pos = indexFooterIdx - (elemIdx+1)*PageNoSize
// pageNo, _ := bin.GetUint32(buf[pos:])
// return pageNo, int(qty), elemIdx
// } // }
func getPrevPageNo(buf []byte) uint32 { // func getLastPageNo(buf []byte) (pageNo uint32) {
return bin.GetUint32(buf[prevPageIdx:]) // qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
} // pos := indexFooterIdx - qty*PageNoSize
// pageNo, _ = bin.GetUint32(buf[pos:])
// return
// }
func findPageNo(buf []byte, timestamp uint32) (pageNo uint32) { // func getPageNo(buf []byte, idx int) (pageNo uint32) {
var ( // pos := indexFooterIdx - (idx+1)*PageNoSize
qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]) // pageNo, _ = bin.GetUint32(buf[pos:])
comparator = ValueAtComparator{ // return
buf: buf, // }
timestamp: timestamp,
}
)
elemIdx, _ := BinarySearch(qty, comparator)
pos := indexFooterIdx - (elemIdx+1)*PageNoSize
return bin.GetUint32(buf[pos:])
}
func findPageNoIdx(buf []byte, timestamp uint32) (idx int) { // func listPageNumbers(buf []byte) (pageNumbers []uint32) {
var ( // qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]) // pos := indexFooterIdx - PageNoSize
comparator = ValueAtComparator{ // for range qty {
buf: buf, // pageNo, _ := bin.GetUint32(buf[pos:])
timestamp: timestamp, // pageNumbers = append(pageNumbers, pageNo)
} // pos -= PageNoSize
) // }
elemIdx, _ := BinarySearch(qty, comparator) // return
return elemIdx // }
}
func findPageNoForDeleteSince(buf []byte, since uint32) (uint32, int, int) { // // include since timestamp
var ( // func listPageNumbersSince(buf []byte, timestamp uint32) (pageNumbers []uint32) {
qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]) // comparator := ValueAtComparator{
comparator = ValueAtComparator{ // buf: buf,
buf: buf, // timestamp: timestamp,
timestamp: since, // }
} // qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
) // elemIdx, _ := BinarySearch(int(qty), comparator)
elemIdx, _ := BinarySearch(qty, comparator) // pos := indexFooterIdx - (elemIdx+1)*PageNoSize
pos := elemIdx * timestampSize // for range qty {
timestamp := bin.GetUint32(buf[pos:]) // pageNo, _ := bin.GetUint32(buf[pos:])
// pageNumbers = append(pageNumbers, pageNo)
if timestamp == since { // pos -= PageNoSize
if elemIdx == 0 { // }
return 0, qty, -1 // return
} // }
elemIdx--
}
pos = indexFooterIdx - (elemIdx+1)*PageNoSize
return bin.GetUint32(buf[pos:]), qty, elemIdx
}
func getLastPageNo(buf []byte) (pageNo uint32) {
qty := bin.GetUint16AsInt(buf[indexRecordsQtyIdx:])
pos := indexFooterIdx - qty*PageNoSize
return bin.GetUint32(buf[pos:])
}
func getPageNo(buf []byte, idx int) (pageNo uint32) {
pos := indexFooterIdx - (idx+1)*PageNoSize
return bin.GetUint32(buf[pos:])
}
func listPageNumbers(buf []byte) (pageNumbers []uint32) {
qty := bin.GetUint16AsInt(buf[indexRecordsQtyIdx:])
pos := indexFooterIdx - PageNoSize
for range qty {
pageNumbers = append(pageNumbers, bin.GetUint32(buf[pos:]))
pos -= PageNoSize
}
return
}
// include since timestamp
func listPageNumbersSince(buf []byte, timestamp uint32) (pageNumbers []uint32) {
var (
qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:])
comparator = ValueAtComparator{
buf: buf,
timestamp: timestamp,
}
)
elemIdx, _ := BinarySearch(qty, comparator)
pos := indexFooterIdx - (elemIdx+1)*PageNoSize
for range qty {
pageNumbers = append(pageNumbers, bin.GetUint32(buf[pos:]))
pos -= PageNoSize
}
return
}
// func getSince(buf []byte) uint32 { // func getSince(buf []byte) uint32 {
// return bin.GetUint32(buf[0:]) // return bin.GetUint32(buf[0:])

8
atree/select.go
View File

@@ -3,7 +3,7 @@ package atree
import ( import (
"fmt" "fmt"
diploma "gordenko.dev/dima/qb" "gordenko.dev/dima/qb"
) )
type ContinueFullScanReq struct { type ContinueFullScanReq struct {
@@ -23,7 +23,7 @@ func (s *Atree) ContinueFullScan(req ContinueFullScanReq) error {
PageData: buf, PageData: buf,
Atree: s, Atree: s,
FracDigits: req.FracDigits, FracDigits: req.FracDigits,
MetricType: diploma.Instant, MetricType: qb.Instant,
}) })
if err != nil { if err != nil {
return err return err
@@ -60,7 +60,7 @@ func (s *Atree) ContinueRangeScan(req ContinueRangeScanReq) error {
PageData: buf, PageData: buf,
Atree: s, Atree: s,
FracDigits: req.FracDigits, FracDigits: req.FracDigits,
MetricType: diploma.Instant, MetricType: qb.Instant,
}) })
if err != nil { if err != nil {
return err return err
@@ -101,7 +101,7 @@ func (s *Atree) RangeScan(req RangeScanReq) error {
PageData: buf, PageData: buf,
Atree: s, Atree: s,
FracDigits: req.FracDigits, FracDigits: req.FracDigits,
MetricType: diploma.Instant, MetricType: qb.Instant,
}) })
if err != nil { if err != nil {
return err return err

621
bin/bin.go
View File

@@ -1,621 +0,0 @@
package bin
import (
"errors"
"fmt"
"io"
"math"
)
const (
varInt64SignFlag = 0b01000000
maxReadAttempts = 5
)
var (
ErrNoSpace = errors.New("no space")
ErrIncompleteWrite = errors.New("incomplete write")
ErrReadOverflow = errors.New("bin: reader returned 'n' > bufsize")
// ErrNegativeReadCount shows 100% bug in the source reader
ErrNegativeReadCount = errors.New("bin: reader returned negative 'n'")
)
// READ
func ReadUint16(src io.Reader) (num uint16, err error) {
var (
q = 2
arr = make([]byte, q)
)
n, err := src.Read(arr)
if err != nil {
return
}
if n != q {
return 0, fmt.Errorf("read %d bytes only", n)
}
return GetUint16(arr), nil
}
func ReadUint24AsInt(src io.Reader) (num int, err error) {
var (
q = 3
arr = make([]byte, q)
)
n, err := src.Read(arr)
if err != nil {
return
}
if n != q {
return 0, fmt.Errorf("read %d bytes only", n)
}
return GetUint24AsInt(arr), nil
}
func ReadUint32(src io.Reader) (num uint32, err error) {
var (
q = 4
arr = make([]byte, q)
)
n, err := src.Read(arr)
if err != nil {
return
}
if n != q {
return 0, fmt.Errorf("read %d bytes only", n)
}
return GetUint32(arr), nil
}
func ReadUint64(src io.Reader) (num uint64, err error) {
var (
q = 8
arr = make([]byte, q)
)
n, err := src.Read(arr)
if err != nil {
return
}
if n != q {
return 0, fmt.Errorf("read %d bytes only", n)
}
return GetUint64(arr), nil
}
func ReadFloat64(src io.Reader) (num float64, err error) {
var (
q = 8
arr = make([]byte, q)
)
n, err := src.Read(arr)
if err != nil {
return
}
if n != q {
return 0, fmt.Errorf("read %d bytes only", n)
}
return GetFloat64(arr), nil
}
func ReadUnixtime(src io.Reader) (num int64, err error) {
var (
q = 8
arr = make([]byte, q)
)
n, err := src.Read(arr)
if err != nil {
return
}
if n != q {
return 0, fmt.Errorf("read %d bytes only", n)
}
return int64(GetUint64(arr)), nil
}
// READ VAR
func ReadVarUint64(src io.Reader) (num uint64, n int, err error) {
var (
p = make([]byte, 1)
b byte
)
for i := range 8 {
_, err = src.Read(p)
if err != nil {
return
}
n++
b = p[0]
if b >= 128 {
num |= uint64(b&127) << uint(i*7)
return
}
num |= uint64(b) << uint(i*7)
}
_, err = src.Read(p)
if err != nil {
return
}
n++
num |= uint64(p[0]) << 56
return
}
// GET
func GetUint16(arr []byte) uint16 {
return uint16(arr[0]) | (uint16(arr[1]) << 8)
}
func GetUint16AsInt(arr []byte) int {
return int(arr[0]) | (int(arr[1]) << 8)
}
func GetUint24AsInt(arr []byte) int {
return int(arr[0]) | (int(arr[1]) << 8) | (int(arr[2]) << 16)
}
func GetUint32(arr []byte) uint32 {
return uint32(arr[0]) | (uint32(arr[1]) << 8) |
(uint32(arr[2]) << 16) | (uint32(arr[3]) << 24)
}
func GetUint32AsInt64(arr []byte) int64 {
u32 := uint32(arr[0]) | (uint32(arr[1]) << 8) |
(uint32(arr[2]) << 16) | (uint32(arr[3]) << 24)
return int64(u32)
}
func GetUint40(arr []byte) uint64 {
return uint64(arr[0]) | (uint64(arr[1]) << 8) |
(uint64(arr[2]) << 16) | (uint64(arr[3]) << 24) |
(uint64(arr[4]) << 32)
}
func GetUint48(arr []byte) uint64 {
return uint64(arr[0]) | (uint64(arr[1]) << 8) |
(uint64(arr[2]) << 16) | (uint64(arr[3]) << 24) |
(uint64(arr[4]) << 32) | (uint64(arr[5]) << 40)
}
func GetUint64(arr []byte) uint64 {
return uint64(arr[0]) | (uint64(arr[1]) << 8) |
(uint64(arr[2]) << 16) | (uint64(arr[3]) << 24) |
(uint64(arr[4]) << 32) | (uint64(arr[5]) << 40) |
(uint64(arr[6]) << 48) | (uint64(arr[7]) << 56)
}
func GetFloat32(arr []byte) float32 {
return math.Float32frombits(GetUint32(arr))
}
func GetFloat64(arr []byte) float64 {
return math.Float64frombits(GetUint64(arr))
}
func GetUnixtime(arr []byte) int64 {
u32 := uint32(arr[0]) | (uint32(arr[1]) << 8) |
(uint32(arr[2]) << 16) | (uint32(arr[3]) << 24)
return int64(u32)
}
func GetVarUint64(arr []byte) (num uint64, n int, err error) {
var b byte
for i := range 8 {
if i >= len(arr) {
return 0, 0, io.EOF
}
b = arr[i]
if b >= 128 {
num |= uint64(b&127) << uint(i*7)
return num, i + 1, nil
}
num |= uint64(b) << uint(i*7)
}
if len(arr) < 9 {
return 0, 0, io.EOF
}
return num | uint64(arr[8])<<56, 9, nil
}
func ReverseGetVarUint64(arr []byte) (num uint64, n int, err error) {
var (
b byte
j = len(arr) - 1
)
for i := range 8 {
if j < 0 {
return 0, 0, io.EOF
}
b = arr[j]
if b >= 128 {
num |= uint64(b&127) << uint(i*7)
return num, i + 1, nil
}
num |= uint64(b) << uint(i*7)
j--
}
if j < 0 {
return 0, 0, io.EOF
}
return num | uint64(arr[j])<<56, 9, nil
}
func GetVarInt64(arr []byte) (num int64, n int, err error) {
u64, n, err := GetVarUint64(arr)
if err != nil {
return
}
return DecodeZigZag(u64), n, nil
}
func ReverseGetVarInt64(arr []byte) (num int64, n int, err error) {
u64, n, err := ReverseGetVarUint64(arr)
if err != nil {
return
}
return DecodeZigZag(u64), n, nil
}
// PUT
func PutUint16(arr []byte, num uint16) {
arr[0] = byte(num)
arr[1] = byte(num >> 8)
}
func PutIntAsUint16(arr []byte, num int) {
arr[0] = byte(num)
arr[1] = byte(num >> 8)
}
func PutIntAsUint24(arr []byte, num int) {
arr[0] = byte(num)
arr[1] = byte(num >> 8)
arr[2] = byte(num >> 16)
}
func PutUint32(arr []byte, num uint32) {
arr[0] = byte(num)
arr[1] = byte(num >> 8)
arr[2] = byte(num >> 16)
arr[3] = byte(num >> 24)
}
func PutInt64AsUint32(arr []byte, num int64) {
arr[0] = byte(num)
arr[1] = byte(num >> 8)
arr[2] = byte(num >> 16)
arr[3] = byte(num >> 24)
}
func PutUint40(arr []byte, num uint64) {
arr[0] = byte(num)
arr[1] = byte(num >> 8)
arr[2] = byte(num >> 16)
arr[3] = byte(num >> 24)
arr[4] = byte(num >> 32)
}
func PutUint48(arr []byte, num uint64) {
arr[0] = byte(num)
arr[1] = byte(num >> 8)
arr[2] = byte(num >> 16)
arr[3] = byte(num >> 24)
arr[4] = byte(num >> 32)
arr[5] = byte(num >> 40)
}
func PutUint64(arr []byte, num uint64) {
arr[0] = byte(num)
arr[1] = byte(num >> 8)
arr[2] = byte(num >> 16)
arr[3] = byte(num >> 24)
arr[4] = byte(num >> 32)
arr[5] = byte(num >> 40)
arr[6] = byte(num >> 48)
arr[7] = byte(num >> 56)
}
func PutFloat32(arr []byte, num float32) {
PutUint32(arr, math.Float32bits(num))
}
func PutFloat64(arr []byte, num float64) {
PutUint64(arr, math.Float64bits(num))
}
// WRITE
func WriteUint16(dst io.Writer, num uint16) error {
arr := []byte{
byte(num),
byte(num >> 8),
}
n, err := dst.Write(arr)
if err != nil {
return err
}
if n != 2 {
return ErrIncompleteWrite
}
return err
}
func WriteUint32(dst io.Writer, num uint32) error {
arr := []byte{
byte(num),
byte(num >> 8),
byte(num >> 16),
byte(num >> 24),
}
n, err := dst.Write(arr)
if err != nil {
return err
}
if n != 4 {
return ErrIncompleteWrite
}
return err
}
func WriteFloat64(dst io.Writer, num float64) error {
arr := make([]byte, 8)
PutUint64(arr, math.Float64bits(num))
n, err := dst.Write(arr)
if err != nil {
return err
}
if n != 2 {
return ErrIncompleteWrite
}
return err
}
// WRITE VAR
func WriteVarUint64(dst io.Writer, num uint64) (int, error) {
arr := make([]byte, 9)
for i := range 8 {
arr[i] = byte(num & 127)
num >>= 7
if num == 0 {
arr[i] |= 128
size := i + 1
n, err := dst.Write(arr[:size])
if err != nil {
return n, err
}
if n != size {
return n, ErrIncompleteWrite
}
return size, nil
}
}
arr[8] = byte(num)
n, err := dst.Write(arr)
if err != nil {
return n, err
}
if n != 9 {
return n, ErrIncompleteWrite
}
return 9, nil
}
func PutVarUint64(arr []byte, num uint64) (int, error) {
for i := range 8 {
if i >= len(arr) {
return 0, ErrNoSpace
}
arr[i] = byte(num & 127)
num >>= 7
if num == 0 {
arr[i] |= 128
return i + 1, nil
}
}
if len(arr) < 9 {
return 0, ErrNoSpace
}
arr[8] = byte(num)
return 9, nil
}
func ReversePutVarUint64(arr []byte, num uint64) (int, error) {
var tmp [9]byte
for i := range 8 {
tmp[i] = byte(num & 127)
num >>= 7
if num == 0 {
tmp[i] |= 128
n := i + 1
if len(arr) < n {
return 0, ErrNoSpace
}
for j := i; j >= 0; j-- {
arr[i-j] = tmp[j]
}
return n, nil
}
}
tmp[8] = byte(num)
n := 9
if len(arr) < n {
return 0, ErrNoSpace
}
for j := 8; j >= 0; j-- {
arr[8-j] = tmp[j]
}
return n, nil
}
func PutVarUint64AtEnd(arr []byte, num uint64) (int, error) {
var (
tmp [9]byte
n int
)
for i := range 8 {
tmp[i] = byte(num & 127)
num >>= 7
if num == 0 {
tmp[i] |= 128
n = i + 1
break
}
}
if n == 0 {
tmp[8] = byte(num)
n = 9
}
if len(arr) < n {
return 0, ErrNoSpace
}
j := len(arr) - n
for i := range n {
arr[j] = tmp[i]
j++
}
return n, nil
}
func PutVarInt64(arr []byte, x int64) (int, error) {
return PutVarUint64(arr, EncodeZigZag(x))
}
func PutVarInt64AtEnd(arr []byte, x int64) (int, error) {
return PutVarUint64AtEnd(arr, EncodeZigZag(x))
}
func ReversePutVarInt64(arr []byte, x int64) (int, error) {
return ReversePutVarUint64(arr, EncodeZigZag(x))
}
type KeyComparator interface {
CompareTo(int) int
}
func BinarySearch(qty int, keyComparator KeyComparator) (elemIdx int, isFound bool) {
if qty == 0 {
return
}
a := 0
b := qty - 1
for {
var (
elemIdx = (b-a)/2 + a
code = keyComparator.CompareTo(elemIdx)
)
if code == 1 {
a = elemIdx + 1
if a > b {
return elemIdx + 1, false
}
} else if code == -1 {
b = elemIdx - 1
if b < a {
return elemIdx, false
}
} else {
return elemIdx, true
}
}
}
func DeleteReverseArrElem(arr []byte, qty int, elemSize int, idx int) {
dstIdx := len(arr) - idx*elemSize - 1
srcIdx := dstIdx - elemSize
end := len(arr) - qty*elemSize
for ; srcIdx >= end; srcIdx-- {
arr[dstIdx] = arr[srcIdx]
dstIdx--
}
for i := end; i < end+elemSize; i++ {
arr[i] = 0
}
}
// ZigZag
// ZigZag encoding: int64 -> uint64
func EncodeZigZag(x int64) uint64 {
return uint64(x<<1) ^ uint64(x>>63)
}
// ZigZag decoding: uint64 -> int64
func DecodeZigZag(u uint64) int64 {
return int64(u>>1) ^ -(int64(u & 1))
}
func ReadN(r io.Reader, n int) (_ []byte, err error) {
if n < 0 {
err = fmt.Errorf("wrong n=%d", n)
return
}
buf := make([]byte, n)
err = ReadNInto(r, buf)
if err != nil {
return
}
return buf, nil
}
func ReadNInto(r io.Reader, buf []byte) (err error) {
if len(buf) == 0 {
return
}
var q, total, readAttempts int
for readAttempts < maxReadAttempts {
bufsize := len(buf) - total
q, err = r.Read(buf[total:])
if q == bufsize {
return nil
}
if err != nil {
return
}
if q > bufsize {
err = ErrReadOverflow
return
}
if q < 0 {
err = ErrNegativeReadCount
return
}
if q == 0 {
readAttempts++
} else {
total += q
}
}
err = io.ErrNoProgress
return
}
func CalcVarUint64Length(num uint64) int {
for i := range 8 {
num >>= 7
if num == 0 {
return i + 1
}
}
return 9
}
func CalcVarInt64Length(num int64) int {
u64 := EncodeZigZag(num)
for i := range 8 {
u64 >>= 7
if u64 == 0 {
return i + 1
}
}
return 9
}

535
client/client.go
View File

@@ -4,8 +4,8 @@ import (
"fmt" "fmt"
"net" "net"
diploma "gordenko.dev/dima/qb" bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb/bin" "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/bufreader" "gordenko.dev/dima/qb/bufreader"
"gordenko.dev/dima/qb/proto" "gordenko.dev/dima/qb/proto"
) )
@@ -33,7 +33,6 @@ func Connect(address string) (*Connection, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &Connection{ return &Connection{
conn: conn, conn: conn,
src: bufreader.New(conn, 1500), src: bufreader.New(conn, 1500),
@@ -51,22 +50,19 @@ func (s *Connection) Close() {
func (s *Connection) mustSuccess(reader *bufreader.BufferedReader) (err error) { func (s *Connection) mustSuccess(reader *bufreader.BufferedReader) (err error) {
code, err := reader.ReadByte() code, err := reader.ReadByte()
if err != nil { if err != nil {
return fmt.Errorf("read response code: %s", err) return
} }
switch code { switch code {
case proto.RespSuccess: case proto.RespSuccess:
return nil // ok return nil // ok
case proto.RespError: case proto.RespError:
return s.onError() return s.onError()
default: default:
return fmt.Errorf("unknown reponse code %d", code) return fmt.Errorf("unknown reponse code %d", code)
} }
} }
func (s *Connection) AddMetric(req proto.AddMetricReq) error { func (s *Connection) AddMetric(req proto.AddMetricReq) (err error) {
arr := []byte{ arr := []byte{
proto.TypeAddMetric, proto.TypeAddMetric,
0, 0, 0, 0, // 0, 0, 0, 0, //
@@ -74,9 +70,9 @@ func (s *Connection) AddMetric(req proto.AddMetricReq) error {
byte(req.FracDigits), byte(req.FracDigits),
} }
bin.PutUint32(arr[1:], req.MetricID) bin.PutUint32(arr[1:], req.MetricID)
// req
if _, err := s.conn.Write(arr); err != nil { if _, err = s.conn.Write(arr); err != nil {
return err return
} }
return s.mustSuccess(s.src) return s.mustSuccess(s.src)
} }
@@ -96,52 +92,57 @@ func (s *Connection) AddMetric(req proto.AddMetricReq) error {
// return s.mustSuccess(s.src) // return s.mustSuccess(s.src)
// } // }
func (s *Connection) GetMetric(metricID uint32) (*proto.Metric, error) { func (s *Connection) GetMetric(metricID uint32) (_ *proto.Metric, err error) {
arr := []byte{ arr := []byte{
proto.TypeGetMetric, proto.TypeGetMetric,
0, 0, 0, 0, 0, 0, 0, 0,
} }
bin.PutUint32(arr[1:], metricID) bin.PutUint32(arr[1:], metricID)
// req
if _, err := s.conn.Write(arr); err != nil { if _, err = s.conn.Write(arr); err != nil {
return nil, err return
} }
// answer
code, err := s.src.ReadByte() code, err := s.src.ReadByte()
if err != nil { if err != nil {
return nil, fmt.Errorf("read response code: %s", err) return
} }
switch code { switch code {
case proto.RespValue: case proto.RespValue:
arr, err := s.src.ReadN(6) var (
metric proto.Metric
metricType byte
)
metric.MetricID, err = bin.ReadUint32(s.src)
if err != nil { if err != nil {
return nil, fmt.Errorf("read body: %s", err) return
} }
metricType, err = bin.ReadByte(s.src)
return &proto.Metric{ if err != nil {
MetricID: bin.GetUint32(arr), return
MetricType: diploma.MetricType(arr[4]), }
FracDigits: int(arr[5]), metric.MetricType = qb.MetricType(metricType)
}, nil metric.FracDigits, err = bin.ReadByteAsInt(s.src)
if err != nil {
return
}
return &metric, nil
case proto.RespError: case proto.RespError:
return nil, s.onMaybeError() return nil, s.onMaybeError()
default: default:
return nil, fmt.Errorf("unknown reponse code %d", code) return nil, fmt.Errorf("unknown reponse code %d", code)
} }
} }
func (s *Connection) DeleteMetric(metricID uint32) error { func (s *Connection) DeleteMetric(metricID uint32) (err error) {
arr := []byte{ arr := []byte{
proto.TypeDeleteMetric, proto.TypeDeleteMetric,
0, 0, 0, 0, // 0, 0, 0, 0, //
} }
bin.PutUint32(arr[1:], metricID) bin.PutUint32(arr[1:], metricID)
// req
if _, err := s.conn.Write(arr); err != nil { if _, err = s.conn.Write(arr); err != nil {
return err return
} }
return s.mustSuccess(s.src) return s.mustSuccess(s.src)
} }
@@ -156,9 +157,9 @@ func (s *Connection) AppendMeasure(req proto.AppendMeasureReq) (err error) {
bin.PutUint32(arr[1:], req.MetricID) bin.PutUint32(arr[1:], req.MetricID)
bin.PutUint32(arr[5:], req.Timestamp) bin.PutUint32(arr[5:], req.Timestamp)
bin.PutFloat64(arr[9:], req.Value) bin.PutFloat64(arr[9:], req.Value)
// req
if _, err := s.conn.Write(arr); err != nil { if _, err = s.conn.Write(arr); err != nil {
return err return
} }
return s.mustSuccess(s.src) return s.mustSuccess(s.src)
} }
@@ -181,11 +182,10 @@ func (s *Connection) AppendMeasures(req proto.AppendMeasuresReq) (err error) {
bin.PutFloat64(arr[pos+4:], measure.Value) bin.PutFloat64(arr[pos+4:], measure.Value)
pos += recordSize pos += recordSize
} }
if _, err := s.conn.Write(arr); err != nil { // req
return err if _, err = s.conn.Write(arr); err != nil {
return
} }
//fmt.Printf("encode measures: %d\n", len(req.Measures))
return s.mustSuccess(s.src) return s.mustSuccess(s.src)
} }
@@ -213,26 +213,26 @@ func (s *Connection) AppendMeasurePerMetric(list []proto.MetricMeasure) (_ []pro
bin.PutFloat64(arr[pos+8:], item.Value) bin.PutFloat64(arr[pos+8:], item.Value)
pos += recordSize pos += recordSize
} }
if _, err := s.conn.Write(arr); err != nil { // req
return nil, err if _, err = s.conn.Write(arr); err != nil {
return
} }
// answer
code, err := s.src.ReadByte() code, err := s.src.ReadByte()
if err != nil { if err != nil {
return nil, fmt.Errorf("read response code: %s", err)
}
switch code {
case proto.RespValue:
var (
qty uint16
appendErrors []proto.AppendError
)
qty, err = bin.ReadUint16(s.src)
if err != nil {
return return
} }
for range qty { switch code {
case proto.RespValue:
var (
count int
appendErrors []proto.AppendError
)
count, err = bin.ReadUint16AsInt(s.src)
if err != nil {
return
}
for range count {
var ae proto.AppendError var ae proto.AppendError
ae.MetricID, err = bin.ReadUint32(s.src) ae.MetricID, err = bin.ReadUint32(s.src)
if err != nil { if err != nil {
@@ -245,70 +245,26 @@ func (s *Connection) AppendMeasurePerMetric(list []proto.MetricMeasure) (_ []pro
appendErrors = append(appendErrors, ae) appendErrors = append(appendErrors, ae)
} }
return appendErrors, nil return appendErrors, nil
default: default:
return nil, fmt.Errorf("unknown reponse code %d", code) return nil, fmt.Errorf("unknown reponse code %d", code)
} }
} }
func (s *Connection) ListAllInstantMeasures(metricID uint32) ([]proto.InstantMeasure, error) { func (s *Connection) ListAllInstantMeasures(metricID uint32) (_ []proto.InstantMeasure, err error) {
arr := []byte{ arr := []byte{
proto.TypeListAllInstantMeasures, proto.TypeListAllInstantMeasures,
0, 0, 0, 0, // metricID 0, 0, 0, 0, // metricID
} }
bin.PutUint32(arr[1:], metricID) bin.PutUint32(arr[1:], metricID)
// req
if _, err := s.conn.Write(arr); err != nil { if _, err = s.conn.Write(arr); err != nil {
return nil, err return
}
var (
result []proto.InstantMeasure
tmp = make([]byte, 12)
)
for {
code, err := s.src.ReadByte()
if err != nil {
return nil, fmt.Errorf("read response code: %s", err)
}
//fmt.Printf("code: %d\n", code)
switch code {
case proto.RespPartOfValue:
q, err := bin.ReadUint32(s.src)
if err != nil {
return nil, fmt.Errorf("read records qty: %s", err)
}
//fmt.Printf("q: %d\n", q)
for i := range int(q) {
err = bin.ReadNInto(s.src, tmp)
if err != nil {
return nil, fmt.Errorf("read record #%d: %s", i, err)
}
result = append(result, proto.InstantMeasure{
Timestamp: bin.GetUint32(tmp),
Value: bin.GetFloat64(tmp[4:]),
})
}
case proto.RespEndOfValue:
return result, nil
case proto.RespError:
return nil, s.onError()
default:
return nil, fmt.Errorf("unknown reponse code %d", code)
}
} }
// answer
return s.readInstantMeasures()
} }
func (s *Connection) ListInstantMeasures(req proto.ListInstantMeasuresReq) ([]proto.InstantMeasure, error) { func (s *Connection) ListInstantMeasures(req proto.ListInstantMeasuresReq) (_ []proto.InstantMeasure, err error) {
arr := []byte{ arr := []byte{
proto.TypeListInstantMeasures, proto.TypeListInstantMeasures,
0, 0, 0, 0, // metricID 0, 0, 0, 0, // metricID
@@ -318,118 +274,66 @@ func (s *Connection) ListInstantMeasures(req proto.ListInstantMeasuresReq) ([]pr
bin.PutUint32(arr[1:], req.MetricID) bin.PutUint32(arr[1:], req.MetricID)
bin.PutUint32(arr[5:], req.Since) bin.PutUint32(arr[5:], req.Since)
bin.PutUint32(arr[9:], req.Until) bin.PutUint32(arr[9:], req.Until)
// req
if _, err := s.conn.Write(arr); err != nil { if _, err = s.conn.Write(arr); err != nil {
return nil, err return
} }
// answer
return s.readInstantMeasures()
}
var ( func (s *Connection) readInstantMeasures() (_ []proto.InstantMeasure, err error) {
result []proto.InstantMeasure var result []proto.InstantMeasure
tmp = make([]byte, 12)
)
for { for {
code, err := s.src.ReadByte() var code byte
code, err = bin.ReadByte(s.src)
if err != nil { if err != nil {
return nil, fmt.Errorf("read response code: %s", err) return nil, fmt.Errorf("read response code: %s", err)
} }
switch code { switch code {
case proto.RespPartOfValue: case proto.RespPartOfValue:
q, err := bin.ReadUint32(s.src) var count int
count, err = bin.ReadUint32AsInt(s.src)
if err != nil { if err != nil {
return nil, fmt.Errorf("read records qty: %s", err) return nil, fmt.Errorf("read records qty: %s", err)
} }
for range count {
for i := range int(q) { var measure proto.InstantMeasure
err = bin.ReadNInto(s.src, tmp) measure.Timestamp, err = bin.ReadUint32(s.src)
if err != nil { if err != nil {
return nil, fmt.Errorf("read record #%d: %s", i, err) return
} }
measure.Value, err = bin.ReadFloat64(s.src)
result = append(result, proto.InstantMeasure{ if err != nil {
Timestamp: bin.GetUint32(tmp), return
Value: bin.GetFloat64(tmp[4:]), }
}) result = append(result, measure)
} }
case proto.RespEndOfValue: case proto.RespEndOfValue:
return result, nil return result, nil
case proto.RespError: case proto.RespError:
return nil, s.onError() return nil, s.onError()
default: default:
return nil, fmt.Errorf("unknown reponse code %d", code) return nil, fmt.Errorf("unknown reponse code %d", code)
} }
} }
} }
func (s *Connection) ListAllCumulativeMeasures(metricID uint32) ([]proto.CumulativeMeasure, error) { func (s *Connection) ListAllCumulativeMeasures(metricID uint32) (_ []proto.CumulativeMeasure, err error) {
arr := []byte{ arr := []byte{
proto.TypeListAllCumulativeMeasures, proto.TypeListAllCumulativeMeasures,
0, 0, 0, 0, // metricID 0, 0, 0, 0, // metricID
} }
bin.PutUint32(arr[1:], metricID) bin.PutUint32(arr[1:], metricID)
// req
if _, err := s.conn.Write(arr); err != nil { if _, err = s.conn.Write(arr); err != nil {
return nil, err return
}
var (
result []proto.CumulativeMeasure
tmp = make([]byte, 20)
)
for {
code, err := s.src.ReadByte()
if err != nil {
return nil, fmt.Errorf("read response code: %s", err)
}
//fmt.Printf("code: %d\n", code)
switch code {
case proto.RespPartOfValue:
q, err := bin.ReadUint32(s.src)
if err != nil {
return nil, fmt.Errorf("read records qty: %s", err)
}
for i := range int(q) {
err = bin.ReadNInto(s.src, tmp)
if err != nil {
return nil, fmt.Errorf("read record #%d: %s", i, err)
}
//fmt.Printf("tmp: %d\n", tmp)
result = append(result, proto.CumulativeMeasure{
Timestamp: bin.GetUint32(tmp),
Value: bin.GetFloat64(tmp[4:]),
Total: bin.GetFloat64(tmp[12:]),
})
// pretty.PPrintln("measure", CumulativeMeasure{
// Timestamp: bin.GetUint32(tmp),
// Value: bin.GetFloat64(tmp[4:]),
// Total: bin.GetFloat64(tmp[12:]),
// })
}
case proto.RespEndOfValue:
return result, nil
case proto.RespError:
return nil, s.onError()
default:
return nil, fmt.Errorf("unknown reponse code %d", code)
}
} }
// answer
return s.readCumulativeMeasures()
} }
func (s *Connection) ListCumulativeMeasures(req proto.ListCumulativeMeasuresReq) ([]proto.CumulativeMeasure, error) { func (s *Connection) ListCumulativeMeasures(req proto.ListCumulativeMeasuresReq) (_ []proto.CumulativeMeasure, err error) {
arr := []byte{ arr := []byte{
proto.TypeListCumulativeMeasures, proto.TypeListCumulativeMeasures,
0, 0, 0, 0, // metricID 0, 0, 0, 0, // metricID
@@ -439,55 +343,56 @@ func (s *Connection) ListCumulativeMeasures(req proto.ListCumulativeMeasuresReq)
bin.PutUint32(arr[1:], req.MetricID) bin.PutUint32(arr[1:], req.MetricID)
bin.PutUint32(arr[5:], req.Since) bin.PutUint32(arr[5:], req.Since)
bin.PutUint32(arr[9:], req.Until) bin.PutUint32(arr[9:], req.Until)
// req
if _, err := s.conn.Write(arr); err != nil { if _, err = s.conn.Write(arr); err != nil {
return nil, err return
} }
// answer
return s.readCumulativeMeasures()
}
var ( func (s *Connection) readCumulativeMeasures() (_ []proto.CumulativeMeasure, err error) {
result []proto.CumulativeMeasure var result []proto.CumulativeMeasure
tmp = make([]byte, 20)
)
for { for {
code, err := s.src.ReadByte() var code byte
code, err = bin.ReadByte(s.src)
if err != nil { if err != nil {
return nil, fmt.Errorf("read response code: %s", err) return nil, fmt.Errorf("read response code: %s", err)
} }
switch code { switch code {
case proto.RespPartOfValue: case proto.RespPartOfValue:
q, err := bin.ReadUint32(s.src) var count int
count, err = bin.ReadUint32AsInt(s.src)
if err != nil { if err != nil {
return nil, fmt.Errorf("read records qty: %s", err) return nil, fmt.Errorf("read records qty: %s", err)
} }
for range count {
for i := range int(q) { var measure proto.CumulativeMeasure
err = bin.ReadNInto(s.src, tmp) measure.Timestamp, err = bin.ReadUint32(s.src)
if err != nil { if err != nil {
return nil, fmt.Errorf("read record #%d: %s", i, err) return
} }
measure.Value, err = bin.ReadFloat64(s.src)
result = append(result, proto.CumulativeMeasure{ if err != nil {
Timestamp: bin.GetUint32(tmp), return
Value: bin.GetFloat64(tmp[4:]), }
Total: bin.GetFloat64(tmp[12:]), measure.Total, err = bin.ReadFloat64(s.src)
}) if err != nil {
return
}
result = append(result, measure)
} }
case proto.RespEndOfValue: case proto.RespEndOfValue:
return result, nil return result, nil
case proto.RespError: case proto.RespError:
return nil, s.onError() return nil, s.onError()
default: default:
return nil, fmt.Errorf("unknown reponse code %d", code) return nil, fmt.Errorf("unknown reponse code %d", code)
} }
} }
} }
func (s *Connection) ListInstantPeriods(req proto.ListInstantPeriodsReq) ([]proto.InstantPeriod, error) { func (s *Connection) ListInstantPeriods(req proto.ListInstantPeriodsReq) (_ []proto.InstantPeriod, err error) {
arr := []byte{ arr := []byte{
proto.TypeListInstantPeriods, proto.TypeListInstantPeriods,
0, 0, 0, 0, // metricID 0, 0, 0, 0, // metricID
@@ -504,87 +409,70 @@ func (s *Connection) ListInstantPeriods(req proto.ListInstantPeriodsReq) ([]prot
bin.PutUint16(arr[9:], uint16(req.Until.Year)) bin.PutUint16(arr[9:], uint16(req.Until.Year))
arr[11] = byte(req.Until.Month) arr[11] = byte(req.Until.Month)
arr[12] = byte(req.Until.Day) arr[12] = byte(req.Until.Day)
// req
if _, err := s.conn.Write(arr); err != nil { if _, err = s.conn.Write(arr); err != nil {
return nil, err return
} }
// answer
var q int var result []proto.InstantPeriod
if (req.AggregateFuncs & diploma.AggregateMin) == diploma.AggregateMin {
q++
}
if (req.AggregateFuncs & diploma.AggregateMax) == diploma.AggregateMax {
q++
}
if (req.AggregateFuncs & diploma.AggregateAvg) == diploma.AggregateAvg {
q++
}
var (
result []proto.InstantPeriod
// 12 bytes - period, since, until
// q * 8 bytes - min, max, avg
tmp = make([]byte, 12+q*8)
)
for { for {
code, err := s.src.ReadByte() var code byte
code, err = s.src.ReadByte()
if err != nil { if err != nil {
return nil, fmt.Errorf("read response code: %s", err) return
} }
//fmt.Printf("code: %d\n", code)
switch code { switch code {
case proto.RespPartOfValue: case proto.RespPartOfValue:
q, err := bin.ReadUint32(s.src) var count int
count, err = bin.ReadUint32AsInt(s.src)
if err != nil { if err != nil {
return nil, fmt.Errorf("read records qty: %s", err) return
} }
for range count {
for i := range int(q) { var p proto.InstantPeriod
err = bin.ReadNInto(s.src, tmp) p.Period, err = bin.ReadUint32(s.src)
if err != nil { if err != nil {
return nil, fmt.Errorf("read record #%d: %s", i, err) return
} }
p.Since, err = bin.ReadUint32(s.src)
var ( if err != nil {
p = proto.InstantPeriod{ return
Period: bin.GetUint32(tmp[0:]),
Since: bin.GetUint32(tmp[4:]),
Until: bin.GetUint32(tmp[8:]),
} }
// 12 bytes - period, since, until p.Until, err = bin.ReadUint32(s.src)
pos = 12 if err != nil {
) return
if (req.AggregateFuncs & diploma.AggregateMin) == diploma.AggregateMin {
p.Min = bin.GetFloat64(tmp[pos:])
pos += 8
} }
if (req.AggregateFuncs & diploma.AggregateMax) == diploma.AggregateMax { if (req.AggregateFuncs & qb.AggregateMin) == qb.AggregateMin {
p.Max = bin.GetFloat64(tmp[pos:]) p.Min, err = bin.ReadFloat64(s.src)
pos += 8 if err != nil {
return
}
}
if (req.AggregateFuncs & qb.AggregateMax) == qb.AggregateMax {
p.Max, err = bin.ReadFloat64(s.src)
if err != nil {
return
}
}
if (req.AggregateFuncs & qb.AggregateAvg) == qb.AggregateAvg {
p.Avg, err = bin.ReadFloat64(s.src)
if err != nil {
return
} }
if (req.AggregateFuncs & diploma.AggregateAvg) == diploma.AggregateAvg {
p.Avg = bin.GetFloat64(tmp[pos:])
} }
result = append(result, p) result = append(result, p)
} }
case proto.RespEndOfValue: case proto.RespEndOfValue:
return result, nil return result, nil
case proto.RespError: case proto.RespError:
return nil, s.onError() return nil, s.onError()
default: default:
return nil, fmt.Errorf("unknown reponse code %d", code) return nil, fmt.Errorf("unknown reponse code %d", code)
} }
} }
} }
func (s *Connection) ListCumulativePeriods(req proto.ListCumulativePeriodsReq) ([]proto.CumulativePeriod, error) { func (s *Connection) ListCumulativePeriods(req proto.ListCumulativePeriodsReq) (_ []proto.CumulativePeriod, err error) {
arr := []byte{ arr := []byte{
proto.TypeListCumulativePeriods, proto.TypeListCumulativePeriods,
0, 0, 0, 0, // metricID 0, 0, 0, 0, // metricID
@@ -600,108 +488,107 @@ func (s *Connection) ListCumulativePeriods(req proto.ListCumulativePeriodsReq) (
bin.PutUint16(arr[9:], uint16(req.Until.Year)) bin.PutUint16(arr[9:], uint16(req.Until.Year))
arr[11] = byte(req.Until.Month) arr[11] = byte(req.Until.Month)
arr[12] = byte(req.Until.Day) arr[12] = byte(req.Until.Day)
// req
if _, err := s.conn.Write(arr); err != nil { if _, err = s.conn.Write(arr); err != nil {
return nil, err return
} }
// answer
var ( var result []proto.CumulativePeriod
result []proto.CumulativePeriod
tmp = make([]byte, 28)
)
for { for {
code, err := s.src.ReadByte() var code byte
code, err = s.src.ReadByte()
if err != nil { if err != nil {
return nil, fmt.Errorf("read response code: %s", err) return
} }
switch code { switch code {
case proto.RespPartOfValue: case proto.RespPartOfValue:
q, err := bin.ReadUint32(s.src) var count int
count, err = bin.ReadUint32AsInt(s.src)
if err != nil { if err != nil {
return nil, fmt.Errorf("read records qty: %s", err) return
} }
for range count {
for i := range int(q) { var p proto.CumulativePeriod
err = bin.ReadNInto(s.src, tmp) p.Period, err = bin.ReadUint32(s.src)
if err != nil { if err != nil {
return nil, fmt.Errorf("read record #%d: %s", i, err) return
} }
result = append(result, proto.CumulativePeriod{ p.Since, err = bin.ReadUint32(s.src)
Period: bin.GetUint32(tmp[0:]), if err != nil {
Since: bin.GetUint32(tmp[4:]), return
Until: bin.GetUint32(tmp[8:]), }
EndValue: bin.GetFloat64(tmp[12:]), p.Until, err = bin.ReadUint32(s.src)
Total: bin.GetFloat64(tmp[20:]), if err != nil {
}) return
}
p.EndValue, err = bin.ReadFloat64(s.src)
if err != nil {
return
}
p.Total, err = bin.ReadFloat64(s.src)
if err != nil {
return
}
result = append(result, p)
} }
case proto.RespEndOfValue: case proto.RespEndOfValue:
return result, nil return result, nil
case proto.RespError: case proto.RespError:
return nil, s.onError() return nil, s.onError()
default: default:
return nil, fmt.Errorf("unknown reponse code %d", code) return nil, fmt.Errorf("unknown reponse code %d", code)
} }
} }
} }
func (s *Connection) ListCurrentValues(metricIDs []uint32) ([]proto.CurrentValue, error) { func (s *Connection) ListCurrentValues(metricIDs []uint32) (_ []proto.CurrentValue, err error) {
arr := make([]byte, 3+metricKeySize*len(metricIDs)) arr := make([]byte, 3+metricKeySize*len(metricIDs))
arr[0] = proto.TypeListCurrentValues arr[0] = proto.TypeListCurrentValues
bin.PutUint16(arr[1:], uint16(len(metricIDs))) bin.PutUint16(arr[1:], uint16(len(metricIDs)))
off := 3 off := 3
for _, metricID := range metricIDs { for _, metricID := range metricIDs {
bin.PutUint32(arr[off:], metricID) bin.PutUint32(arr[off:], metricID)
off += metricKeySize off += metricKeySize
} }
// req
if _, err := s.conn.Write(arr); err != nil { if _, err = s.conn.Write(arr); err != nil {
return nil, err return
} }
// answer
var ( var result []proto.CurrentValue
result []proto.CurrentValue
tmp = make([]byte, 16)
)
for { for {
code, err := s.src.ReadByte() var code byte
code, err = s.src.ReadByte()
if err != nil { if err != nil {
return nil, fmt.Errorf("read response code: %s", err) return
} }
switch code { switch code {
case proto.RespPartOfValue: case proto.RespPartOfValue:
q, err := bin.ReadUint32(s.src) var count int
count, err = bin.ReadUint32AsInt(s.src)
if err != nil { if err != nil {
return nil, fmt.Errorf("read records qty: %s", err) return
} }
for range count {
for i := range int(q) { var m proto.CurrentValue
err = bin.ReadNInto(s.src, tmp) m.MetricID, err = bin.ReadUint32(s.src)
if err != nil { if err != nil {
return nil, fmt.Errorf("read record #%d: %s", i, err) return
} }
m.Timestamp, err = bin.ReadUint32(s.src)
result = append(result, proto.CurrentValue{ if err != nil {
MetricID: bin.GetUint32(tmp), return
Timestamp: bin.GetUint32(tmp[4:]), }
Value: bin.GetFloat64(tmp[8:]), m.Value, err = bin.ReadFloat64(s.src)
}) if err != nil {
return
}
result = append(result, m)
} }
case proto.RespEndOfValue: case proto.RespEndOfValue:
return result, nil return result, nil
case proto.RespError: case proto.RespError:
return nil, s.onError() return nil, s.onError()
default: default:
return nil, fmt.Errorf("unknown reponse code %d", code) return nil, fmt.Errorf("unknown reponse code %d", code)
} }
@@ -716,7 +603,7 @@ func (s *Connection) DeleteMeasures(req proto.DeleteMeasuresReq) (err error) {
} }
bin.PutUint32(arr[1:], req.MetricID) bin.PutUint32(arr[1:], req.MetricID)
bin.PutUint32(arr[5:], req.Since) bin.PutUint32(arr[5:], req.Since)
// req
if _, err := s.conn.Write(arr); err != nil { if _, err := s.conn.Write(arr); err != nil {
return err return err
} }

460
conbuf/conbuf.go
View File

@@ -1,460 +0,0 @@
package conbuf
import (
"errors"
"fmt"
"io"
"gordenko.dev/dima/qb/bin"
)
const chunkSize = 512
const ChunkSize = 512
var (
ErrOutOfRange = errors.New("out of range")
)
type ContinuousBuffer struct {
chunks [][]byte
}
func NewFromBuffer(buf []byte) *ContinuousBuffer {
var (
chunks [][]byte
copied = 0
)
for copied < len(buf) {
chunk := make([]byte, chunkSize)
end := min(copied+chunkSize, len(buf))
copy(chunk, buf[copied:end])
chunks = append(chunks, chunk)
copied += end - copied
}
return &ContinuousBuffer{
chunks: chunks,
}
}
func New(chunks [][]byte) *ContinuousBuffer {
for i, chunk := range chunks {
if len(chunk) != chunkSize {
panic(fmt.Sprintf("wrong chunk #%d size %d", i, len(chunk)))
}
}
return &ContinuousBuffer{
chunks: chunks,
}
}
func (s *ContinuousBuffer) Chunks() [][]byte {
return s.chunks
}
// [0, pos)
func (s *ContinuousBuffer) GetByte(idx int) byte {
chunkIdx := idx / chunkSize
if chunkIdx >= len(s.chunks) {
panic(ErrOutOfRange)
}
byteIdx := idx % chunkSize
return s.chunks[chunkIdx][byteIdx]
}
func (s *ContinuousBuffer) SetByte(idx int, b byte) {
chunkIdx := idx / chunkSize
if chunkIdx > len(s.chunks) {
panic(ErrOutOfRange)
}
if chunkIdx == len(s.chunks) {
s.chunks = append(s.chunks, make([]byte, chunkSize))
}
byteIdx := idx % chunkSize
s.chunks[chunkIdx][byteIdx] = b
}
func (s *ContinuousBuffer) SetFlag(idx int, flag byte) {
chunkIdx := idx / chunkSize
if chunkIdx > len(s.chunks) {
panic(ErrOutOfRange)
}
if chunkIdx == len(s.chunks) {
s.chunks = append(s.chunks, make([]byte, chunkSize))
}
byteIdx := idx % chunkSize
s.chunks[chunkIdx][byteIdx] |= flag
}
func (s *ContinuousBuffer) UnsetFlag(idx int, flag byte) {
chunkIdx := idx / chunkSize
if chunkIdx > len(s.chunks) {
panic(ErrOutOfRange)
}
if chunkIdx == len(s.chunks) {
s.chunks = append(s.chunks, make([]byte, chunkSize))
}
byteIdx := idx % chunkSize
s.chunks[chunkIdx][byteIdx] &^= flag
}
// [since, until)
func (s *ContinuousBuffer) ShiftOnePosToRight(since int, until int) {
if since < 0 {
panic("since < 0")
}
if since >= until {
panic("since >= until")
}
chunkIdx := until / chunkSize
byteIdx := until % chunkSize
if chunkIdx > len(s.chunks) {
panic(ErrOutOfRange)
}
if chunkIdx == len(s.chunks) {
if byteIdx == 0 {
s.chunks = append(s.chunks, make([]byte, chunkSize))
} else {
panic(ErrOutOfRange)
}
}
var (
qty = until - since
prevChunkIdx int
prevByteIdx int
)
for range qty {
prevChunkIdx = chunkIdx
prevByteIdx = byteIdx - 1
if prevByteIdx < 0 {
prevChunkIdx = chunkIdx - 1
prevByteIdx = chunkSize - 1
}
s.chunks[chunkIdx][byteIdx] = s.chunks[prevChunkIdx][prevByteIdx]
if byteIdx > 0 {
byteIdx--
} else {
chunkIdx--
byteIdx = chunkSize - 1
}
}
}
// [since, until)
func (s *ContinuousBuffer) ShiftOnePosToLeft(since int, until int) {
if since <= 0 {
panic("since <= 0")
}
if since >= until {
panic("since >= until")
}
chunkIdx := since / chunkSize
byteIdx := since % chunkSize
if until > len(s.chunks)*chunkSize {
panic(ErrOutOfRange)
}
var (
qty = until - since
prevChunkIdx int
prevByteIdx int
)
for range qty {
prevChunkIdx = chunkIdx
prevByteIdx = byteIdx - 1
if prevByteIdx < 0 {
prevChunkIdx = chunkIdx - 1
prevByteIdx = chunkSize - 1
}
s.chunks[prevChunkIdx][prevByteIdx] = s.chunks[chunkIdx][byteIdx]
byteIdx++
if byteIdx == chunkSize {
chunkIdx++
byteIdx = 0
}
}
}
func (s *ContinuousBuffer) PutUint32(pos int, num uint32) {
s.CopyTo(pos,
[]byte{
byte(num),
byte(num >> 8),
byte(num >> 16),
byte(num >> 24),
})
}
func (s *ContinuousBuffer) GetUint32(pos int) uint32 {
arr := s.Slice(pos, pos+4)
return uint32(arr[0]) | (uint32(arr[1]) << 8) |
(uint32(arr[2]) << 16) | (uint32(arr[3]) << 24)
}
func (s *ContinuousBuffer) ReversePutVarUint64(pos int, num uint64) int {
var tmp [9]byte
for i := range 8 {
tmp[i] = byte(num & 127)
num >>= 7
if num == 0 {
tmp[i] |= 128
n := i + 1
s.ReverseCopyTo(pos, tmp[:n])
return n
}
}
tmp[8] = byte(num)
n := 9
s.ReverseCopyTo(pos, tmp[:])
return n
}
func (s *ContinuousBuffer) ReverseGetVarUint64(idx int) (num uint64, n int, err error) {
chunkIdx := idx / chunkSize
if chunkIdx >= len(s.chunks) {
panic(ErrOutOfRange)
}
var (
byteIdx = idx % chunkSize
chunk = s.chunks[chunkIdx]
b byte
)
for i := range 8 {
b = chunk[byteIdx]
if b >= 128 {
num |= uint64(b&127) << uint(i*7)
return num, i + 1, nil
}
num |= uint64(b) << uint(i*7)
if byteIdx > 0 {
byteIdx--
} else {
if chunkIdx == 0 {
return 0, 0, io.EOF
} else {
chunkIdx--
chunk = s.chunks[chunkIdx]
byteIdx = chunkSize - 1
}
}
}
return num | uint64(chunk[byteIdx])<<56, 9, nil
}
func (s *ContinuousBuffer) ReversePutVarInt64(pos int, x int64) int {
return s.ReversePutVarUint64(pos, bin.EncodeZigZag(x))
}
func (s *ContinuousBuffer) ReverseGetVarInt64(idx int) (int64, int, error) {
u64, n, err := s.ReverseGetVarUint64(idx)
if err != nil {
return 0, 0, err
}
return bin.DecodeZigZag(u64), n, nil
}
func (s *ContinuousBuffer) PutVarUint64(pos int, num uint64) int {
var tmp [9]byte
for i := range 8 {
tmp[i] = byte(num & 127)
num >>= 7
if num == 0 {
tmp[i] |= 128
n := i + 1
s.CopyTo(pos, tmp[:n])
return n
}
}
tmp[8] = byte(num)
s.CopyTo(pos, tmp[:])
return 9
}
func (s *ContinuousBuffer) GetVarUint64(idx int) (num uint64, n int, err error) {
chunkIdx := idx / chunkSize
if chunkIdx >= len(s.chunks) {
panic(ErrOutOfRange)
}
var (
byteIdx = idx % chunkSize
chunk = s.chunks[chunkIdx]
b byte
)
for i := range 8 {
b = chunk[byteIdx]
if b >= 128 {
num |= uint64(b&127) << uint(i*7)
return num, i + 1, nil
}
num |= uint64(b) << uint(i*7)
byteIdx++
if byteIdx == chunkSize {
chunkIdx++
if chunkIdx == len(s.chunks) {
return 0, 0, io.EOF
}
chunk = s.chunks[chunkIdx]
byteIdx = 0
}
}
return num | uint64(chunk[byteIdx])<<56, 9, nil
}
func (s *ContinuousBuffer) PutVarInt64(idx int, x int64) int {
return s.PutVarUint64(idx, bin.EncodeZigZag(x))
}
func (s *ContinuousBuffer) GetVarInt64(idx int) (int64, int, error) {
u64, n, err := s.GetVarUint64(idx)
if err != nil {
return 0, 0, err
}
return bin.DecodeZigZag(u64), n, nil
}
func (s *ContinuousBuffer) CopyTo(idx int, data []byte) {
chunkIdx := idx / chunkSize
if chunkIdx > len(s.chunks) {
panic(ErrOutOfRange)
}
if chunkIdx == len(s.chunks) {
s.chunks = append(s.chunks, make([]byte, chunkSize))
}
byteIdx := idx % chunkSize
chunk := s.chunks[chunkIdx]
copied := 0
for _, b := range data {
chunk[byteIdx] = b
copied++
byteIdx++
if byteIdx == chunkSize {
byteIdx = 0
chunkIdx++
if chunkIdx == len(s.chunks) {
if copied == len(data) {
return
}
s.chunks = append(s.chunks, make([]byte, chunkSize))
}
chunk = s.chunks[chunkIdx]
}
}
}
func (s *ContinuousBuffer) ReverseCopyTo(idx int, data []byte) {
chunkIdx := idx / chunkSize
if chunkIdx > len(s.chunks) {
panic(ErrOutOfRange)
}
if chunkIdx == len(s.chunks) {
s.chunks = append(s.chunks, make([]byte, chunkSize))
}
byteIdx := idx % chunkSize
chunk := s.chunks[chunkIdx]
copied := 0
for i := len(data) - 1; i >= 0; i-- {
chunk[byteIdx] = data[i]
copied++
byteIdx++
if byteIdx == chunkSize {
byteIdx = 0
chunkIdx++
if chunkIdx == len(s.chunks) {
if copied == len(data) {
return
}
s.chunks = append(s.chunks, make([]byte, chunkSize))
}
chunk = s.chunks[chunkIdx]
}
}
}
// [since, until)
func (s *ContinuousBuffer) Slice(since int, until int) []byte {
if since >= until {
return nil
}
size := len(s.chunks) * chunkSize
if until >= size {
panic(ErrOutOfRange)
}
data := make([]byte, until-since)
chunkIdx := since / chunkSize
byteIdx := since % chunkSize
chunk := s.chunks[chunkIdx]
for i := range len(data) {
data[i] = chunk[byteIdx]
byteIdx++
if byteIdx == chunkSize {
byteIdx = 0
chunkIdx++
chunk = s.chunks[chunkIdx]
}
}
return data
}
func (s *ContinuousBuffer) DecodeRunLength(pos int) (length uint16, n int) {
b1 := s.GetByte(pos)
pos--
if b1 < 128 {
length = uint16(b1)
n = 1
} else {
b2 := s.GetByte(pos)
length = uint16(b1&127) | (uint16(b2) << 7)
n = 2
}
length += 2
return
}
func (s *ContinuousBuffer) Copy() *ContinuousBuffer {
var copies [][]byte
for _, chunk := range s.chunks {
buf := make([]byte, chunkSize)
copy(buf, chunk)
copies = append(copies, buf)
}
return New(copies)
}
// size to copy
func (s *ContinuousBuffer) CopyChunksToOneBuffer(dst []byte, size int) {
pos := 0
for _, chunk := range s.chunks {
if size >= len(chunk) {
copy(dst[pos:], chunk)
size -= len(chunk)
pos += len(chunk)
} else {
copy(dst[pos:], chunk[:size])
return
}
}
}

50
database/api.go
View File

@@ -6,13 +6,13 @@ import (
"io" "io"
"net" "net"
qb "gordenko.dev/dima/qb" bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree" "gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/bin"
"gordenko.dev/dima/qb/bufreader" "gordenko.dev/dima/qb/bufreader"
"gordenko.dev/dima/qb/proto" "gordenko.dev/dima/qb/proto"
"gordenko.dev/dima/qb/storage"
"gordenko.dev/dima/qb/transform" "gordenko.dev/dima/qb/transform"
"gordenko.dev/dima/qb/txlog"
) )
var ( var (
@@ -213,7 +213,7 @@ func (s *Database) AddMetric(req proto.AddMetricReq) uint16 {
switch resultCode { switch resultCode {
case Succeed: case Succeed:
s.txlog.Append(txlog.AddedMetric{ s.storage.Append(storage.MetricAddRecord{
MetricID: req.MetricID, MetricID: req.MetricID,
MetricType: req.MetricType, MetricType: req.MetricType,
FracDigits: req.FracDigits, FracDigits: req.FracDigits,
@@ -286,20 +286,20 @@ func (s *Database) DeleteMetric(req proto.DeleteMetricReq) uint16 {
switch result.ResultCode { switch result.ResultCode {
case Succeed: case Succeed:
var ( // var (
freePageNumbers []uint32 // //freePageNumbers []uint32
) // )
if result.RootPageNo > 0 { // if result.RootPageNo > 0 {
var err error // var err error
freePageNumbers, err = s.atree.GetAllPages(result.RootPageNo) // freePageNumbers, err = s.atree.GetAllPages(result.RootPageNo)
if err != nil { // if err != nil {
qb.Abort(qb.FailedAtreeRequest, err) // qb.Abort(qb.FailedAtreeRequest, err)
} // }
} // }
s.txlog.Append(txlog.DeletedMetric{ // s.storage.Append(storage.DeletedMetric{
MetricID: req.MetricID, // MetricID: req.MetricID,
FreePageNumbers: freePageNumbers, // //FreePageNumbers: freePageNumbers, FIX
}) // })
//<-waitCh //<-waitCh
case NoMetric: case NoMetric:
@@ -343,7 +343,7 @@ type FilledPage struct {
// switch result.ResultCode { // switch result.ResultCode {
// case CanAppend: // case CanAppend:
// waitCh := s.txlog.Append(txlog.AppendedMeasure{ // waitCh := s.storage.Append(storage.AppendedMeasure{
// MetricID: req.MetricID, // MetricID: req.MetricID,
// Timestamp: req.Timestamp, // Timestamp: req.Timestamp,
// Value: req.Value, // Value: req.Value,
@@ -390,9 +390,9 @@ type FilledPage struct {
// // } // // }
// // _ = report // // _ = report
// //waitCh := s.txlog.WriteAppended(path, report) // //waitCh := s.storage.WriteAppended(path, report)
// waitCh := s.txlog.Append(txlog.AppendedPagesReq{ // waitCh := s.storage.Append(storage.AppendedPagesReq{
// Legs: path.Legs, // Legs: path.Legs,
// MetricID: req.MetricID, // MetricID: req.MetricID,
// Timestamp: req.Timestamp, // Timestamp: req.Timestamp,
@@ -459,8 +459,8 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
// _ = report // _ = report
// if len(toAppendMeasures) > 0 { // if len(toAppendMeasures) > 0 {
// waitCh := s.txlog.WriteAppendMeasures( // waitCh := s.storage.WriteAppendMeasures(
// txlog.AppendedMeasures{ // storage.AppendedMeasures{
// MetricID: req.MetricID, // MetricID: req.MetricID,
// Measures: toAppendMeasures, // Measures: toAppendMeasures,
// }, // },
@@ -500,7 +500,7 @@ func (s *Database) DeleteMeasures(req proto.DeleteMeasuresReq) uint16 {
case DeleteFromAtreeNotNeeded: case DeleteFromAtreeNotNeeded:
// регистрирую удаление в TransactionLog // регистрирую удаление в TransactionLog
s.txlog.Append(txlog.DeletedMeasures{ s.storage.Append(storage.DeletedMeasures{
MetricID: req.MetricID, MetricID: req.MetricID,
}) })
//<-waitCh //<-waitCh
@@ -512,7 +512,7 @@ func (s *Database) DeleteMeasures(req proto.DeleteMeasuresReq) uint16 {
qb.Abort(qb.FailedAtreeRequest, err) qb.Abort(qb.FailedAtreeRequest, err)
} }
// регистрирую удаление в TransactionLog // регистрирую удаление в TransactionLog
s.txlog.Append(txlog.DeletedMeasures{ s.storage.Append(storage.DeletedMeasures{
MetricID: req.MetricID, MetricID: req.MetricID,
FreePageNumbers: pageNumbers, FreePageNumbers: pageNumbers,
}) })

62
database/database.go
View File

@@ -12,7 +12,7 @@ import (
"gordenko.dev/dima/qb/atree" "gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/freelist" "gordenko.dev/dima/qb/freelist"
"gordenko.dev/dima/qb/txlog" "gordenko.dev/dima/qb/storage"
) )
func JoinSnapshotFileName(dir string, snapshotNumber int) string { func JoinSnapshotFileName(dir string, snapshotNumber int) string {
@@ -41,7 +41,7 @@ type Database struct {
dir string dir string
databaseName string databaseName string
snapshotNumber int snapshotNumber int
txlog *txlog.Writer storage *storage.Writer
atree *atree.Atree atree *atree.Atree
tcpPort int tcpPort int
logfile *os.File logfile *os.File
@@ -186,7 +186,7 @@ func (s *Database) recovery() {
// logNumber = recipe.LogNumber // logNumber = recipe.LogNumber
// } // }
// s.txlog, err = txlog.NewWriter(txlog.WriterOptions{ // s.storage, err = storage.NewWriter(storage.WriterOptions{
// Dir: s.dir, // Dir: s.dir,
// LogNumber: logNumber, // LogNumber: logNumber,
// AppendToWorkerQueue: s.appendJobToWorkerQueue, // AppendToWorkerQueue: s.appendJobToWorkerQueue,
@@ -199,7 +199,7 @@ func (s *Database) recovery() {
// qb.Abort(qb.CreateChangesWriterFailed, err) // qb.Abort(qb.CreateChangesWriterFailed, err)
// } // }
// go s.txlog.Run() // go s.storage.Run()
// // fileNames, err := s.searchREDOFiles() // // fileNames, err := s.searchREDOFiles()
// // if err != nil { // // if err != nil {
@@ -249,7 +249,7 @@ func (s *Database) replayChanges(snapshotNumber int) (err error) {
return return
} }
walReader, err := txlog.NewReader(txlog.ReaderOptions{ walReader, err := storage.NewWALReader(storage.WALReaderOptions{
FileName: JoinWALFileName(s.dir, snapshotNumber), FileName: JoinWALFileName(s.dir, snapshotNumber),
BufferSize: 8 * 1024 * 1024, BufferSize: 8 * 1024 * 1024,
}) })
@@ -271,29 +271,21 @@ func (s *Database) replayChanges(snapshotNumber int) (err error) {
if err != nil { if err != nil {
return return
} }
err = s.storage.WritePagesToAtree(walReplayer.IndexPages(), walReplayer.DataPages())
if err != nil {
return err
}
// FIX free pages sync
return nil return nil
} }
func (s *Database) rewritePagesFromLastWALPacket(list []any) error { func (s *Database) relayMetricsToMetrics(replayMetrics map[uint32]*ReplayMetric) {
// var ( for metricID, x := range replayMetrics {
// indexPages []txlog.PageToWrite s.metrics[metricID] = x.ToMetric()
// dataPages []txlog.PageToWrite }
// )
// for _, untyped := range list {
// switch rec := untyped.(type) {
// case txlog.WALRecordAppendMeasures:
// metric, ok := s.metrics[rec.MetricID]
// if !ok {
// qb.Abort(qb.MetricNotFoundDuringReplay, nil)
// }
// //metric.ReplayAppendedMeasures(rec)
// default:
// qb.Abort(qb.UnknownTxLogRecordTypeBug,
// fmt.Errorf("bug: unknown record type %T in TransactionLog", rec))
// }
// }
return nil
} }
// FIX // FIX
@@ -314,14 +306,14 @@ func calculateDataBufferSize(payloadSize int) int {
return bufferSize return bufferSize
} }
} }
return atree.DataPageSize return storage.DataPageSize
} }
func allocateBuffers(requiredSpace int) (buf, databuf []byte) { func allocateBuffers(requiredSpace int) (buf, databuf []byte) {
bufferSize := calculateDataBufferSize(requiredSpace) bufferSize := calculateDataBufferSize(requiredSpace)
buf = make([]byte, bufferSize) buf = make([]byte, bufferSize)
if bufferSize == atree.DataPageSize { if bufferSize == storage.DataPageSize {
databuf = buf[:atree.DataPagePayloadSize] databuf = buf[:storage.DataPagePayloadSize]
} else { } else {
databuf = buf databuf = buf
} }
@@ -343,6 +335,20 @@ func growBuffers(in growBuffersIn) (buf, databuf []byte) {
return return
} }
type shrinkBufferIn struct {
Buffer []byte
TimestampsSize int
ValuesSize int
NewBufferSize int
}
func shrinkBuffer(in shrinkBufferIn) []byte {
buf := make([]byte, in.NewBufferSize)
copy(buf, in.Buffer[:in.ValuesSize])
copy(buf[:len(buf)-in.TimestampsSize], in.Buffer[len(in.Buffer)-in.TimestampsSize:])
return buf
}
//func (s *Database) verifySnapshot(fileName string) (_ bool, err error) { //func (s *Database) verifySnapshot(fileName string) (_ bool, err error) {
// file, err := os.Open(fileName) // file, err := os.Open(fileName)
// if err != nil { // if err != nil {

24
database/database_test.go
View File

@@ -6,6 +6,7 @@ import (
"testing" "testing"
"gordenko.dev/dima/qb" "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/storage"
) )
func TestWriteReadSnapshot(t *testing.T) { func TestWriteReadSnapshot(t *testing.T) {
@@ -87,3 +88,26 @@ func cmpMetric(replay *ReplayMetric, origin *_metric) error {
// } // }
return nil return nil
} }
func TestComposeHeadIndexPage(t *testing.T) {
var (
levelIdx = 0
level = storage.IndexLevelTail{
Buffer: []byte{
1, 2, 3,
},
RecordsCount: 2,
}
head = &storage.HeadIndexPage{
PageNo: 100,
Checksum: 12345,
Records: []byte{},
}
)
page := composeHeadIndexPage(levelIdx, level, head)
if page.PageNo != head.PageNo {
t.Fatalf("PageNo: got %d are not equal expected %v",
page.PageNo, head.PageNo)
}
// fix compare pages
}

8
database/helpers.go
View File

@@ -6,17 +6,17 @@ import (
"os" "os"
"time" "time"
diploma "gordenko.dev/dima/qb" "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/proto" "gordenko.dev/dima/qb/proto"
) )
func timeBoundsOfAggregation(since, until proto.TimeBound, groupBy diploma.GroupBy, firstHourOfDay int) (s time.Time, u time.Time) { func timeBoundsOfAggregation(since, until proto.TimeBound, groupBy qb.GroupBy, firstHourOfDay int) (s time.Time, u time.Time) {
switch groupBy { switch groupBy {
case diploma.GroupByHour, diploma.GroupByDay: case qb.GroupByHour, qb.GroupByDay:
s = time.Date(since.Year, since.Month, since.Day, 0, 0, 0, 0, time.Local) s = time.Date(since.Year, since.Month, since.Day, 0, 0, 0, 0, time.Local)
u = time.Date(until.Year, until.Month, until.Day, 0, 0, 0, 0, time.Local) u = time.Date(until.Year, until.Month, until.Day, 0, 0, 0, 0, time.Local)
case diploma.GroupByMonth: case qb.GroupByMonth:
s = time.Date(since.Year, since.Month, 1, 0, 0, 0, 0, time.Local) s = time.Date(since.Year, since.Month, 1, 0, 0, 0, 0, time.Local)
u = time.Date(until.Year, until.Month, 1, 0, 0, 0, 0, time.Local) u = time.Date(until.Year, until.Month, 1, 0, 0, 0, 0, time.Local)
} }

39
database/metric.go
View File

@@ -5,8 +5,7 @@ import (
bin "gordenko.dev/dima/bin/little" bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb" "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree" "gordenko.dev/dima/qb/storage"
"gordenko.dev/dima/qb/txlog"
) )
// METRIC // METRIC
@@ -31,7 +30,7 @@ type _metric struct {
XLock bool XLock bool
RLocks int RLocks int
WaitQueue []any WaitQueue []any
indexLevelTails []txlog.IndexLevelTail // root - last element indexLevelTails []storage.IndexLevelTail // root - last element
} }
//IndexLevels [][]IndexRec // root - last element //IndexLevels [][]IndexRec // root - last element
@@ -60,7 +59,7 @@ func (s *_metric) DeleteMeasures() {
// s.UntilValue = 0 // s.UntilValue = 0
} }
// func (s *_metric) startAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(txlog.AppendedMeasures)) { // func (s *_metric) startAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(storage.AppendedMeasures)) {
func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(any)) { func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(any)) {
var ( var (
// AppendedMeasures struct { // AppendedMeasures struct {
@@ -78,7 +77,7 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage fu
timestamps = s.timestamps timestamps = s.timestamps
values = s.values values = s.values
dataPages []txlog.DataPayload dataPages []storage.DataPayload
//written int //written int
//resultCode byte //resultCode byte
@@ -92,7 +91,7 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage fu
// s.Since = measure.Timestamp // s.Since = measure.Timestamp
// } else { // } else {
// FIX - у випадку помилки треба в транзакції зберегти що помилка, але також зафіксувати скільки елементів збережено. // FIX - у випадку помилки треба в транзакції зберегти що помилка, але також зафіксувати скільки елементів збережено.
// якщо idx == 0 - одразу знімаю блокування і нічого не відправляю в txlog // якщо idx == 0 - одразу знімаю блокування і нічого не відправляю в storage
if measure.Timestamp <= s.timestamps.LastTimestamp() { if measure.Timestamp <= s.timestamps.LastTimestamp() {
if idx == 0 { if idx == 0 {
s.values.ForgetCapturedState() s.values.ForgetCapturedState()
@@ -136,7 +135,7 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage fu
// накопичую // накопичую
timestamps.Compress(timestampCompressionWay, measure.Timestamp) timestamps.Compress(timestampCompressionWay, measure.Timestamp)
values.Compress(valueCompressionWay, measure.Value) values.Compress(valueCompressionWay, measure.Value)
} else if len(s.buffer) < atree.DataPagePayloadSize { } else if len(s.buffer) < storage.DataPagePayloadSize {
// allocate bigger buffer // allocate bigger buffer
buffer := make([]byte, len(s.buffer)*2) buffer := make([]byte, len(s.buffer)*2)
// copy timestamps // copy timestamps
@@ -151,7 +150,7 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage fu
// сторінка заповнена // сторінка заповнена
since := s.timestamps.ReplaceSinceWithUntil() since := s.timestamps.ReplaceSinceWithUntil()
dataPages = append(dataPages, txlog.DataPayload{ dataPages = append(dataPages, storage.DataPayload{
Since: since, Since: since,
Content: s.buffer, Content: s.buffer,
TimestampsSize: timestamps.Size(), TimestampsSize: timestamps.Size(),
@@ -180,8 +179,8 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage fu
// скопіювати. Причому можна скопіювати зрізи chunks // скопіювати. Причому можна скопіювати зрізи chunks
if len(dataPages) > 0 { if len(dataPages) > 0 {
// пишу в txlog довгим шляхом через redo файл і запис в data файл // пишу в storage довгим шляхом через redo файл і запис в data файл
sendToStorage(txlog.AppendedMeasures{ sendToStorage(storage.AppendedMeasures{
MetricID: req.MetricID, MetricID: req.MetricID,
LastPageNo: s.lastPageNo, LastPageNo: s.lastPageNo,
TimestampsOffset: timestamps.Offset(), // state.Pos() з якої позиції дописувати дані на сторінку 0 (при відновленні) TimestampsOffset: timestamps.Offset(), // state.Pos() з якої позиції дописувати дані на сторінку 0 (при відновленні)
@@ -196,12 +195,12 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage fu
ResultCh: nil, ResultCh: nil,
}) })
} else { } else {
// короткий шлях - запис лише в txlog // короткий шлях - запис лише в storage
} }
} }
func (s *_metric) FinAppendMeasures(rec txlog.AppendMeasuresSummary) { func (s *_metric) FinAppendMeasures(rec storage.AppendMeasuresSummary) {
// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію // Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
s.values.ForgetCapturedState() s.values.ForgetCapturedState()
s.timestamps.ForgetCapturedState() s.timestamps.ForgetCapturedState()
@@ -209,7 +208,7 @@ func (s *_metric) FinAppendMeasures(rec txlog.AppendMeasuresSummary) {
if rec.LastPageNo > 0 { if rec.LastPageNo > 0 {
s.lastPageNo = rec.LastPageNo s.lastPageNo = rec.LastPageNo
} }
// В txlog я передав повний індекс. У нього додали елементи (можливо нові рівні). // В storage я передав повний індекс. У нього додали елементи (можливо нові рівні).
// Тому проста заміна // Тому проста заміна
s.indexLevelTails = rec.Index s.indexLevelTails = rec.Index
} }
@@ -388,20 +387,6 @@ func (s *_metric) WriteTo(w io.Writer) (err error) {
return return
} }
// var (
// values qb.ValueCompressor
// )
// if s.metricType == qb.Cumulative {
// values = enc.NewCumulativeDeltaCompressor(s.buffer, s.fracDigits)
// } else {
// values = enc.NewInstantDeltaCompressor(s.buffer, s.fracDigits)
// }
// values.RestoreState(valuesSize)
// s.timestamps = enc.NewTimeDeltaCompressor(s.buffer)
// s.timestamps.RestoreState(timestampsSize, until)
// s.values = values
// s.lastValue = s.values.LastValue()
// since - 4b // since - 4b
// sinceValue - 8b - // sinceValue - 8b -
// untilValue - 8b // untilValue - 8b

46
database/proc.go
View File

@@ -7,8 +7,8 @@ import (
"gordenko.dev/dima/qb/atree" "gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/enc" "gordenko.dev/dima/qb/enc"
"gordenko.dev/dima/qb/proto" "gordenko.dev/dima/qb/proto"
"gordenko.dev/dima/qb/storage"
"gordenko.dev/dima/qb/transform" "gordenko.dev/dima/qb/transform"
"gordenko.dev/dima/qb/txlog"
) )
// AddMetric - create metric in map and set Xlock, rwad - append to queue // AddMetric - create metric in map and set Xlock, rwad - append to queue
@@ -83,7 +83,7 @@ func (s *Database) DoWork() {
case tryAppendMeasuresReq: case tryAppendMeasuresReq:
s.tryAppendMeasures(req) s.tryAppendMeasures(req)
case txlog.Changes: case storage.Changes:
s.applyChanges(req) // all metrics only s.applyChanges(req) // all metrics only
case tryListCurrentValuesReq: case tryListCurrentValuesReq:
@@ -132,7 +132,7 @@ func (s *Database) processMetricQueue(metricID uint32, metric *_metric) {
s.tryGetMetric(req) s.tryGetMetric(req)
case tryAppendMeasuresReq: case tryAppendMeasuresReq:
metric.StartAppendMeasures(req, s.txlog.Append) metric.StartAppendMeasures(req, s.storage.Append)
case tryDeleteMetricReq: case tryDeleteMetricReq:
s.startDeleteMetric(metric, req) s.startDeleteMetric(metric, req)
@@ -294,7 +294,7 @@ func (s *Database) startDeleteMeasures(metric *_metric, req tryDeleteMeasuresReq
// } // }
} }
func (s *Database) finAppendMeasures(rec txlog.AppendMeasuresSummary) { func (s *Database) finAppendMeasures(rec storage.AppendMeasuresSummary) {
metric, ok := s.metrics[rec.MetricID] metric, ok := s.metrics[rec.MetricID]
if !ok { if !ok {
qb.Abort(qb.NoMetricBug, qb.Abort(qb.NoMetricBug,
@@ -322,7 +322,7 @@ func (s *Database) tryAppendMeasures(req tryAppendMeasuresReq) {
metric.WaitQueue = append(metric.WaitQueue, req) metric.WaitQueue = append(metric.WaitQueue, req)
return return
} }
metric.StartAppendMeasures(req, s.txlog.Append) metric.StartAppendMeasures(req, s.storage.Append)
} }
type tryRangeScanReq struct { type tryRangeScanReq struct {
@@ -408,25 +408,25 @@ func (s *Database) tryListCurrentValues(req tryListCurrentValuesReq) {
/////////////////////////////////////////////////////// ///////////////////////////////////////////////////////
func (s *Database) applyChanges(req txlog.Changes) { func (s *Database) applyChanges(req storage.Changes) {
for _, untyped := range req.Records { for _, untyped := range req.Records {
switch rec := untyped.(type) { switch rec := untyped.(type) {
case txlog.AddedMetric: case storage.MetricAddRecord:
s.applyAddMetric(rec) s.applyAddMetric(rec)
case txlog.DeletedMetric: case storage.MetricDeleteRecord:
s.deleteMetric(rec) s.deleteMetric(rec)
// case txlog.AppendedMeasure: // case storage.AppendedMeasure:
// s.appendMeasure(rec) // s.appendMeasure(rec)
case txlog.AppendMeasuresSummary: case storage.AppendMeasuresSummary:
s.finAppendMeasures(rec) s.finAppendMeasures(rec)
// case txlog.AppendedMeasureWithOverflowExtended: // case storage.AppendedMeasureWithOverflowExtended:
// s.appendMeasureAfterOverflow(rec) // s.appendMeasureAfterOverflow(rec)
case txlog.DeletedMeasures: case storage.DeletedMeasures:
s.deleteMeasures(rec) s.deleteMeasures(rec)
} }
} }
@@ -436,7 +436,7 @@ func (s *Database) applyChanges(req txlog.Changes) {
err := writeSnapshot(writeSnapshotIn{ err := writeSnapshot(writeSnapshotIn{
SnapshotNumber: s.snapshotNumber, SnapshotNumber: s.snapshotNumber,
Dir: s.dir, Dir: s.dir,
WriteBufferSize: 1 * 1024 * 1024, // 1mb WriteBufferSize: 4 * 1024 * 1024, // 1mb
Metrics: s.metrics, Metrics: s.metrics,
FrozenIndexPagesCount: req.FrozenIndexPagesCount, FrozenIndexPagesCount: req.FrozenIndexPagesCount,
IndexPageNumbers: req.IndexPageNumbers, IndexPageNumbers: req.IndexPageNumbers,
@@ -451,7 +451,7 @@ func (s *Database) applyChanges(req txlog.Changes) {
} }
} }
func (s *Database) applyAddMetric(rec txlog.AddedMetric) { func (s *Database) applyAddMetric(rec storage.MetricAddRecord) {
// fix lock // fix lock
_, ok := s.metrics[rec.MetricID] _, ok := s.metrics[rec.MetricID]
if ok { if ok {
@@ -470,28 +470,26 @@ func (s *Database) applyAddMetric(rec txlog.AddedMetric) {
// delete(s.metricLockEntries, rec.MetricID) // delete(s.metricLockEntries, rec.MetricID)
} }
func (s *Database) addMetric(rec txlog.AddedMetric) { func (s *Database) addMetric(rec storage.MetricAddRecord) {
var ( var (
values qb.ValueCompressor values qb.ValueCompressor
buffer = make([]byte, minBufferSize) buffer = make([]byte, minBufferSize)
) )
if rec.MetricType == qb.Cumulative { if rec.MetricType == qb.Cumulative {
values = enc.NewCumulativeDeltaCompressor(buffer, byte(rec.FracDigits)) values = enc.NewCumulativeDeltaCompressor(byte(rec.FracDigits), buffer, 0)
} else { } else {
values = enc.NewInstantDeltaCompressor(buffer, byte(rec.FracDigits)) values = enc.NewInstantDeltaCompressor(byte(rec.FracDigits), buffer, 0)
} }
s.metrics[rec.MetricID] = &_metric{ s.metrics[rec.MetricID] = &_metric{
metricType: rec.MetricType, metricType: rec.MetricType,
fracDigits: byte(rec.FracDigits), fracDigits: byte(rec.FracDigits),
buffer: buffer, buffer: buffer,
timestamps: enc.NewTimeDeltaCompressor(buffer), timestamps: enc.NewTimeDeltaCompressor(buffer, 0),
values: values, values: values,
} }
} }
func (s *Database) deleteMetric(rec txlog.DeletedMetric) { func (s *Database) deleteMetric(rec storage.MetricDeleteRecord) {
metric, ok := s.metrics[rec.MetricID] metric, ok := s.metrics[rec.MetricID]
if !ok { if !ok {
qb.Abort(qb.NoMetricBug, qb.Abort(qb.NoMetricBug,
@@ -552,7 +550,7 @@ func (s *Database) deleteMetric(rec txlog.DeletedMetric) {
} }
delete(s.metrics, rec.MetricID) delete(s.metrics, rec.MetricID)
// ADD in txlog // ADD in storage
// if len(rec.FreePageNumbers) > 0 { // if len(rec.FreePageNumbers) > 0 {
// s.freeList.AddPages(rec.FreePageNumbers) // s.freeList.AddPages(rec.FreePageNumbers)
// } // }
@@ -562,7 +560,7 @@ func (s *Database) deleteMetric(rec txlog.DeletedMetric) {
} }
} }
func (s *Database) deleteMeasures(rec txlog.DeletedMeasures) { func (s *Database) deleteMeasures(rec storage.DeletedMeasures) {
metric, ok := s.metrics[rec.MetricID] metric, ok := s.metrics[rec.MetricID]
if !ok { if !ok {
qb.Abort(qb.NoMetricBug, qb.Abort(qb.NoMetricBug,
@@ -577,7 +575,7 @@ func (s *Database) deleteMeasures(rec txlog.DeletedMeasures) {
} }
metric.DeleteMeasures() metric.DeleteMeasures()
metric.XLock = false metric.XLock = false
// FIX add in txlog // FIX add in storage
// if len(rec.FreePageNumbers) > 0 { // if len(rec.FreePageNumbers) > 0 {
// s.freeList.AddPages(rec.FreePageNumbers) // s.freeList.AddPages(rec.FreePageNumbers)
// } // }

320
database/replay_metric.go
View File

@@ -1,12 +1,13 @@
package database package database
import ( import (
"fmt"
"io" "io"
bin "gordenko.dev/dima/bin/little" bin "gordenko.dev/dima/bin/little"
qb "gordenko.dev/dima/qb" "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree" "gordenko.dev/dima/qb/enc"
"gordenko.dev/dima/qb/txlog" "gordenko.dev/dima/qb/storage"
) )
type ReplayMetric struct { type ReplayMetric struct {
@@ -17,7 +18,7 @@ type ReplayMetric struct {
databuf []byte // якщо розмір buf == DataPageSize, то databuf буде менше (мінус футер) databuf []byte // якщо розмір buf == DataPageSize, то databuf буде менше (мінус футер)
tSize int tSize int
vSize int vSize int
indexLevelTails []txlog.IndexLevelTail // root - last element indexLevelTails []storage.IndexLevelTail // root - last element
} }
func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) { func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) {
@@ -57,7 +58,7 @@ func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) {
} }
for range levelsCount { for range levelsCount {
var ( var (
buf = make([]byte, atree.IndexPageSize) buf = make([]byte, storage.IndexPageSize)
count int count int
) )
count, err = bin.ReadVarSize(r) count, err = bin.ReadVarSize(r)
@@ -68,7 +69,7 @@ func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) {
if err != nil { if err != nil {
return return
} }
s.indexLevelTails = append(s.indexLevelTails, txlog.IndexLevelTail{ s.indexLevelTails = append(s.indexLevelTails, storage.IndexLevelTail{
Buffer: buf, Buffer: buf,
RecordsCount: count, RecordsCount: count,
}) })
@@ -76,115 +77,256 @@ func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) {
return return
} }
// func appendNewRecordsToIndexTail(level storage.IndexLevelTail, records []byte) {
// }
// func countReusedIndexPages(changedIndexLevels []storage.ChangedIndexLevel) (count int) {
// return
// }
func (s *ReplayMetric) MeasuresAppend(rec storage.MeasuresAppendRecord) {
timestampsSize := rec.TimestampsOffset - len(rec.Timestamps)
// копіюю нові дані із WAL з урахуванням offset-ів
copy(s.databuf[rec.ValuesOffset:], rec.Values)
copy(s.databuf[len(s.databuf)-timestampsSize:], rec.Timestamps)
//
s.tSize = timestampsSize
s.vSize = rec.ValuesOffset - len(rec.Values)
}
type AppendMeasuresResult struct { type AppendMeasuresResult struct {
IndexPages []txlog.PageToWrite IndexPages []storage.PageToWrite
DataPages []txlog.PageToWrite DataPages []storage.PageToWrite
ReusedIndexPagesCount int ReusedIndexPagesCount int
ReusedDataPagesCount int ReusedDataPagesCount int
} }
func (s *ReplayMetric) AppendedMeasures(rec txlog.WALRecordAppendMeasures, pagesNeeded bool) (result AppendMeasuresResult) { func (s *ReplayMetric) MeasuresAppendWithGrow(rec storage.MeasuresAppendWithGrowRecord, isLastPacket bool) (_ AppendMeasuresResult) {
var (
reusedIndexPages int
reusedDataPages int
indexPages []storage.PageToWrite
dataPages []storage.PageToWrite
)
// Додати в index level tails недостаючі дані, або замінити // Додати в index level tails недостаючі дані, або замінити
for levelIdx, change := range rec.ChangedIndexLevels { for levelIdx, change := range rec.ChangedIndexLevels {
var (
head = change.HeadIndexPage
newRecordsCount = len(change.TailRecords) / indexRecordSize
)
// 1. набиваю сторінки для перезапису в index файлі
if isLastPacket {
if head != nil {
indexPages = append(indexPages,
composeHeadIndexPage(levelIdx, s.indexLevelTails[levelIdx], head))
}
for _, x := range change.IndexPages {
indexPages = append(indexPages, storage.PageToWrite{
PageNo: x.PageNo,
Content: x.Content,
})
}
}
// 2. Вношу зміни в поточні індексні хвости
if levelIdx < len(s.indexLevelTails) { if levelIdx < len(s.indexLevelTails) {
level := s.indexLevelTails[levelIdx] level := s.indexLevelTails[levelIdx]
if change.CompletedIndexPage != nil { if head != nil {
// попередній хвіст індексного рівня перетворився на сторінку, // попередній хвіст індексного рівня перетворився на head сторінку,
// отже TailRecords - це новий хвіст // отже TailRecords - це новий хвіст
// або копіюю дані, або алокую більший буфер?
copy(level.Buffer, change.TailRecords) copy(level.Buffer, change.TailRecords)
level.RecordsCount = len(change.TailRecords) / indexRecordSize level.RecordsCount = newRecordsCount
} else { } else {
// TailRecords - це нові дані, які треба додати // TailRecords - це нові дані, які треба додати
pos := level.RecordsCount * indexRecordSize pos := level.RecordsCount * indexRecordSize
copy(level.Buffer[pos:], change.TailRecords) copy(level.Buffer[pos:], change.TailRecords)
level.RecordsCount += newRecordsCount
} }
s.indexLevelTails[levelIdx] = level s.indexLevelTails[levelIdx] = level
} else { } else {
var ( // додаю новий індексний рівень
recordsCount = len(change.TailRecords) / indexRecordSize buf := make([]byte, storage.IndexPageSize)
buffer = make([]byte, atree.IndexPageSize) copy(buf, change.TailRecords)
)
copy(buffer, change.TailRecords)
s.indexLevelTails = append(s.indexLevelTails, txlog.IndexLevelTail{
Buffer: buffer,
RecordsCount: recordsCount,
})
}
}
// Перезаписати в buffer timestamps та values. Якщо розмір буферу недостатній -
// спочатку створити більший
var (
// CompletedDataPage є полюбому, оскільки це Record із мінімум одною заповненою
// data сторінкою. Отже TailTimestamps і TailValues - це нові хвости data рівня.
requiredSpace = len(rec.TailTimestamps) + len(rec.TailValues)
databuf []byte
)
if requiredSpace < len(s.buf) {
databuf = s.buf
// fix - update sizes?
} else {
var buf []byte
buf, databuf = allocateBuffers(requiredSpace)
copy(databuf, rec.TailValues)
pos := len(databuf) - len(rec.TailTimestamps)
copy(databuf[pos:], rec.TailTimestamps)
// fix - replace and remeber sizes?
s.buf = buf
}
//rec.TailTimestamps
//rec.TailValues
return
}
func (s *ReplayMetric) ApplyHeadDataPage(p txlog.CompletedDataPage) {
// буфер має бути розміру сторінки
if len(s.buf) != atree.DataPageSize {
s.buf, s.databuf = growBuffers(growBuffersIn{
databuf: s.databuf,
tSize: s.tSize,
vSize: s.vSize,
requiredSpace: atree.DataPageSize,
})
}
// дописую дані по зміщенню
pos := len(s.databuf) - p.TimestampsOffset - len(p.Timestamps)
copy(s.databuf[pos:], p.Timestamps)
copy(s.databuf[p.ValuesOffset:], p.Values)
// //
s.tSize = p.TimestampsOffset + len(p.Timestamps) s.indexLevelTails = append(s.indexLevelTails, storage.IndexLevelTail{
s.tSize = p.ValuesOffset + len(p.Values) Buffer: buf,
RecordsCount: newRecordsCount,
})
}
// fix - заполнить страницу // 3. рахую кількість reused індексних сторінок
// fix - check checksum if head != nil {
if head.Reused {
reusedIndexPages++
}
}
for _, x := range change.IndexPages {
if x.Reused {
reusedIndexPages++
}
}
}
// 4. набиваю сторінки для перезапису в data файлі
if isLastPacket {
dataPages = append(dataPages, composeHeadDataPage(s.databuf, rec.HeadDataPage))
for _, x := range rec.DataPages {
dataPages = append(dataPages, storage.PageToWrite{
PageNo: x.PageNo,
Content: x.Content,
})
}
}
// рахую кількість reused дата сторінок
if rec.HeadDataPage.Reused {
reusedDataPages++
}
for _, x := range rec.DataPages {
if x.Reused {
reusedDataPages++
}
}
// HeadDataPage є полюбому, оскільки це транзакція із мінімум однією заповненою
// data сторінкою. Отже TailTimestamps і TailValues - це нові хвости data рівня.
copy(s.databuf, rec.TailValues)
pos := len(s.databuf) - len(rec.TailTimestamps)
copy(s.databuf[pos:], rec.TailTimestamps)
s.tSize = len(rec.TailTimestamps)
s.vSize = len(rec.TailValues)
if len(rec.DataPages) == 0 {
s.lastPageNo = rec.HeadDataPage.PageNo
} else {
s.lastPageNo = rec.DataPages[len(rec.DataPages)-1].PageNo
}
return AppendMeasuresResult{
IndexPages: indexPages,
DataPages: dataPages,
ReusedIndexPagesCount: reusedIndexPages,
ReusedDataPagesCount: reusedDataPages,
}
} }
// func (s *ReplayMetric) ApplyHeadIndexPages(changedIndexLevels txlog.ChangedIndexLevel) { // HELPERS
// for levelIdx, changedLevel := range changedIndexLevels {
// p := changedLevel.CompletedIndexPage func composeHeadIndexPage(levelIdx int, level storage.IndexLevelTail, head *storage.HeadIndexPage) storage.PageToWrite {
// if p != nil { // розраховую pos, з якого буду дописувати хвіст
// level := s.indexLevelTails[levelIdx] pos := level.RecordsCount * indexRecordSize
// copy(level.Buffer[level.RecordsCount*indexRecordSize:], p.Records) // створюю копію сторінки
page := make([]byte, storage.IndexPageSize)
copy(page, level.Buffer[:pos]) // поточні дані
copy(page[pos:], head.Records)
// запечатати сторінку
calculatedCRC := storage.SealIndexPage(storage.SealIndexPageIn{
Content: page,
RecordsCount: level.RecordsCount + len(head.Records)/indexRecordSize,
ZeroLevel: levelIdx == 0,
})
// перевірка CRC
if calculatedCRC != head.Checksum {
qb.Abort(qb.WALReplayFailed,
fmt.Errorf("calculated CRC %d not equal expected %d of head page %d on index level %d",
calculatedCRC, head.Checksum, head.PageNo, levelIdx))
}
return storage.PageToWrite{
PageNo: head.PageNo,
Content: page,
}
}
func composeHeadDataPage(databuf []byte, head storage.HeadDataPage) storage.PageToWrite {
// створюю копію сторінки
var (
page = make([]byte, storage.DataPageSize)
timestampsSize = head.TimestampsOffset - len(head.Timestamps)
)
// копіюю поточні дані
copy(page, databuf)
// копіюю нові дані із WAL з урахуванням offset-ів
copy(page[head.ValuesOffset:], head.Values)
copy(page[len(page)-timestampsSize:], head.Timestamps)
// запечатати сторінку
calculatedCRC := storage.SealDataPage(storage.SealDataPageIn{
Content: page,
PrevPageNo: head.PrevPageNo,
TimestampsSize: timestampsSize,
ValuesSize: head.ValuesOffset + len(head.Values),
})
// перевірка CRC
if calculatedCRC != head.Checksum {
qb.Abort(qb.WALReplayFailed,
fmt.Errorf("calculated CRC %d not equal expected %d of head data page %d",
calculatedCRC, head.Checksum, head.PageNo))
}
return storage.PageToWrite{
PageNo: head.PageNo,
Content: page,
}
}
func (s *ReplayMetric) ToMetric() *_metric {
var (
values qb.ValueCompressor
requiredSpace = s.tSize + s.vSize
)
bufferSize := calculateDataBufferSize(requiredSpace)
if len(s.buf) > bufferSize {
s.buf = shrinkBuffer(shrinkBufferIn{
Buffer: s.buf,
TimestampsSize: s.tSize,
ValuesSize: s.vSize,
NewBufferSize: bufferSize,
})
}
if s.metricType == qb.Cumulative {
values = enc.NewCumulativeDeltaCompressor(s.fracDigits, s.buf, s.vSize)
} else {
values = enc.NewInstantDeltaCompressor(s.fracDigits, s.buf, s.vSize)
}
return &_metric{
metricType: s.metricType,
fracDigits: s.fracDigits,
lastPageNo: s.lastPageNo,
lastValue: values.LastValue(),
buffer: s.buf,
timestamps: enc.NewTimeDeltaCompressor(s.buf, s.tSize),
values: values,
indexLevelTails: s.indexLevelTails,
}
}
// func (s *ReplayMetric) ApplyHeadDataPage(p storage.HeadDataPage) {
// // буфер має бути розміру сторінки
// if len(s.buf) != atree.DataPageSize {
// s.buf, s.databuf = growBuffers(growBuffersIn{
// databuf: s.databuf,
// tSize: s.tSize,
// vSize: s.vSize,
// requiredSpace: atree.DataPageSize,
// })
// }
// // дописую дані по зміщенню
// pos := len(s.databuf) - p.TimestampsOffset - len(p.Timestamps)
// copy(s.databuf[pos:], p.Timestamps)
// copy(s.databuf[p.ValuesOffset:], p.Values)
// //
// s.tSize = p.TimestampsOffset + len(p.Timestamps)
// s.tSize = p.ValuesOffset + len(p.Values)
// // fix - заполнить страницу // // fix - заполнить страницу
// // fix - check checksum // // fix - check checksum
// s.indexPages = append(s.indexPages, txlog.PageToWrite{
// PageNo: p.PageNo,
// Content: level.Buffer,
// })
// } // }
// for _, x := range changedLevel.IndexPages { // func (s *ReplayMetric) ApplyHeadIndexPages(changedIndexLevels storage.ChangedIndexLevel) {
// s.indexPages = append(s.indexPages, txlog.PageToWrite{ // for levelIdx, changedLevel := range changedIndexLevels {
// PageNo: x.PageNo, //
// Content: x.Content,
// }) //
// }
// } // }
// } // }

6
database/snapshot.go
View File

@@ -8,7 +8,7 @@ import (
"path/filepath" "path/filepath"
bin "gordenko.dev/dima/bin/little" bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb/atree" "gordenko.dev/dima/qb/storage"
"gordenko.dev/dima/qb/util" "gordenko.dev/dima/qb/util"
) )
@@ -177,11 +177,11 @@ func readSnapshot(fileName string) (out readSnapshotOut, err error) {
for range metricsQty { for range metricsQty {
var ( var (
metricID uint32 metricID uint32
buf = make([]byte, atree.DataPageSize) buf = make([]byte, storage.DataPageSize)
// для простоти створюю буфер максимального розміру // для простоти створюю буфер максимального розміру
metric = &ReplayMetric{ metric = &ReplayMetric{
buf: buf, buf: buf,
databuf: buf[:atree.DataPagePayloadSize], databuf: buf[:storage.DataPagePayloadSize],
} }
) )
metricID, err = bin.ReadUint32(src) metricID, err = bin.ReadUint32(src)

171
database/wal_replayer.go
View File

@@ -4,22 +4,20 @@ import (
"errors" "errors"
"fmt" "fmt"
qb "gordenko.dev/dima/qb" "gordenko.dev/dima/qb/storage"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/txlog"
) )
type WALReplayer struct { type WALReplayer struct {
walReader *txlog.Reader walReader *storage.WALReader
metrics map[uint32]*ReplayMetric metrics map[uint32]*ReplayMetric
freeIndexPages []uint32 freeIndexPages []uint32
freeDataPages []uint32 freeDataPages []uint32
indexPages []txlog.PageToWrite indexPages []storage.PageToWrite
dataPages []txlog.PageToWrite dataPages []storage.PageToWrite
} }
type WALReplayerOptions struct { type WALReplayerOptions struct {
WALReader *txlog.Reader WALReader *storage.WALReader
Metrics map[uint32]*ReplayMetric // from snapshot Metrics map[uint32]*ReplayMetric // from snapshot
FreeIndexPages []uint32 // from snapshot FreeIndexPages []uint32 // from snapshot
FreeDataPages []uint32 // from snapshot FreeDataPages []uint32 // from snapshot
@@ -40,89 +38,47 @@ func NewWALReplayer(opt WALReplayerOptions) (*WALReplayer, error) {
}, nil }, nil
} }
func (s *WALReplayer) IndexPages() []storage.PageToWrite {
return s.indexPages
}
func (s *WALReplayer) DataPages() []storage.PageToWrite {
return s.dataPages
}
func (s *WALReplayer) Replay() error { func (s *WALReplayer) Replay() error {
for { for {
records, isLast, err := s.walReader.NextPacket() records, isLastPacket, err := s.walReader.NextPacket()
if err != nil { if err != nil {
return err return err
} }
if isLast {
if len(records) > 0 {
var appendRecords []txlog.WALRecordAppendMeasures
for _, untyped := range records {
rec, ok := untyped.(txlog.WALRecordAppendMeasures)
if ok {
appendRecords = append(appendRecords, rec)
}
}
// prepare pages and rewrite
indexPages, dataPages := pagesToWriteFromWALRecords(metrics, appendRecords)
err = s.txlog.WritePagesToAtree(indexPages, dataPages)
if err != nil {
return err
}
}
// для останнього rec - повторюємо запис сторінок
// rewrite write pages
//s.txlog.
//return nil
}
for _, rec := range records { for _, rec := range records {
if err = s.replayWALRecord(rec); err != nil { if err = s.replayRecord(rec, isLastPacket); err != nil {
return err return err
} }
} }
} }
} }
// func (s *WALReplayer) replayPacket(records []any, isLastPacket bool) { func (s *WALReplayer) replayRecord(untyped any, isLastPacket bool) (err error) {
// for _, untyped := range records {
// switch rec := untyped.(type) {
// case txlog.WALRecordAppendMeasures:
// s.onWALRecordAppendMeasures(rec, isLastPacket)
// }
// }
// return
// }
func (s *WALReplayer) replayWALRecord(untyped any) (err error) {
switch rec := untyped.(type) { switch rec := untyped.(type) {
case txlog.AddedMetric: case storage.MeasuresAppendRecord:
if err = s.onAddedMetric(rec); err != nil { if err = s.onMeasuresAppend(rec); err != nil {
return return
} }
case storage.MeasuresAppendWithGrowRecord:
case txlog.DeletedMetric: if err = s.onMeasuresAppendWithGrow(rec, isLastPacket); err != nil {
if err = s.onDeletedMetric(rec); err != nil {
return return
} }
case storage.MetricAddRecord:
// case txlog.AppendedMeasure: if err = s.onMetricAdd(rec); err != nil {
// metric, ok := s.metrics[rec.MetricID] return
// if ok { }
// metric.Timestamps.Append(rec.Timestamp) case storage.MetricDeleteRecord:
// metric.Values.Append(rec.Value) if err = s.onMetricDelete(rec); err != nil {
return
// if metric.Since == 0 { }
// metric.Since = rec.Timestamp // case storage.DeletedMeasures:
// metric.SinceValue = rec.Value
// }
// metric.Until = rec.Timestamp
// metric.UntilValue = rec.Value
// }
case txlog.WALRecordAppendMeasures:
// metric, ok := s.metrics[rec.MetricID]
// if !ok {
// qb.Abort(qb.MetricNotFoundDuringReplay, nil)
// }
//metric.ReplayAppendedMeasures(rec) FIX
// case txlog.DeletedMeasures:
// metric, ok := s.metrics[rec.MetricID] // metric, ok := s.metrics[rec.MetricID]
// if ok { // if ok {
// metric.DeleteMeasures() // metric.DeleteMeasures()
@@ -131,31 +87,31 @@ func (s *WALReplayer) replayWALRecord(untyped any) (err error) {
// } // }
// } // }
default: // default:
qb.Abort(qb.UnknownTxLogRecordTypeBug, // qb.Abort(qb.UnknownstorageRecordTypeBug,
fmt.Errorf("bug: unknown record type %T in TransactionLog", rec)) // fmt.Errorf("bug: unknown record type %T in TransactionLog", rec))
} }
return nil return nil
} }
func (s *WALReplayer) onAddedMetric(rec txlog.AddedMetric) (err error) { func (s *WALReplayer) onMetricAdd(rec storage.MetricAddRecord) (err error) {
_, ok := s.metrics[rec.MetricID] _, ok := s.metrics[rec.MetricID]
if ok { if ok {
return fmt.Errorf("metric %d add failed: already added", rec.MetricID) return fmt.Errorf("metric %d add failed: already added", rec.MetricID)
} }
var ( var (
buf = make([]byte, atree.DataPageSize) buf = make([]byte, storage.DataPageSize)
) )
s.metrics[rec.MetricID] = &ReplayMetric{ s.metrics[rec.MetricID] = &ReplayMetric{
metricType: rec.MetricType, metricType: rec.MetricType,
fracDigits: byte(rec.FracDigits), fracDigits: byte(rec.FracDigits),
buf: buf, buf: buf,
databuf: buf[:atree.DataPagePayloadSize], databuf: buf[:storage.DataPagePayloadSize],
} }
return return
} }
func (s *WALReplayer) onDeletedMetric(rec txlog.DeletedMetric) error { func (s *WALReplayer) onMetricDelete(rec storage.MetricDeleteRecord) error {
_, ok := s.metrics[rec.MetricID] _, ok := s.metrics[rec.MetricID]
if !ok { if !ok {
return fmt.Errorf("metric %d deletion failed: not found", rec.MetricID) return fmt.Errorf("metric %d deletion failed: not found", rec.MetricID)
@@ -166,15 +122,21 @@ func (s *WALReplayer) onDeletedMetric(rec txlog.DeletedMetric) error {
return nil return nil
} }
func (s *WALReplayer) onAppendMeasures(rec txlog.WALRecordAppendMeasures, isLastPacket bool) { func (s *WALReplayer) onMeasuresAppend(rec storage.MeasuresAppendRecord) (err error) {
metric, ok := s.metrics[rec.MetricID] metric, ok := s.metrics[rec.MetricID]
if !ok { if !ok {
qb.Abort(qb.MetricNotFoundDuringReplay, nil) return fmt.Errorf("append measures failed: metric %d not found", rec.MetricID)
} }
if isLastPacket { metric.MeasuresAppend(rec)
s.collectPagesToWrite(metric, rec) return
}
func (s *WALReplayer) onMeasuresAppendWithGrow(rec storage.MeasuresAppendWithGrowRecord, isLastPacket bool) (err error) {
metric, ok := s.metrics[rec.MetricID]
if !ok {
return fmt.Errorf("append measures failed: metric %d not found", rec.MetricID)
} }
result := metric.AppendedMeasures(rec, isLastPacket) result := metric.MeasuresAppendWithGrow(rec, isLastPacket)
// //
if isLastPacket { if isLastPacket {
s.indexPages = append(s.indexPages, result.IndexPages...) s.indexPages = append(s.indexPages, result.IndexPages...)
@@ -183,22 +145,33 @@ func (s *WALReplayer) onAppendMeasures(rec txlog.WALRecordAppendMeasures, isLast
// //
s.freeIndexPages = s.freeIndexPages[:len(s.freeIndexPages)-result.ReusedIndexPagesCount] s.freeIndexPages = s.freeIndexPages[:len(s.freeIndexPages)-result.ReusedIndexPagesCount]
s.freeDataPages = s.freeDataPages[:len(s.freeDataPages)-result.ReusedDataPagesCount] s.freeDataPages = s.freeDataPages[:len(s.freeDataPages)-result.ReusedDataPagesCount]
return
} }
func (s *WALReplayer) collectPagesToWrite(metric *ReplayMetric, rec txlog.WALRecordAppendMeasures) { //func (s *WALReplayer) collectPagesToWrite(metric *ReplayMetric, rec storage.WALRecordAppendMeasures) {
metric.ApplyHeadDataPage(rec.CompletedDataPage) // metric.ApplyHeadDataPage(rec.CompletedDataPage)
s.dataPages = append(s.dataPages, txlog.PageToWrite{ // s.dataPages = append(s.dataPages, storage.PageToWrite{
PageNo: p.PageNo, // PageNo: p.PageNo,
Content: metric.buf, // Content: metric.buf,
}) // })
for _, x := range rec.DataPages { // for _, x := range rec.DataPages {
s.dataPages = append(s.dataPages, txlog.PageToWrite{ // s.dataPages = append(s.dataPages, storage.PageToWrite{
PageNo: x.PageNo, // PageNo: x.PageNo,
Content: x.Content, // Content: x.Content,
}) // })
} // }
//metric.ApplyHeadIndexPages(rec.ChangedIndexLevels) //metric.ApplyHeadIndexPages(rec.ChangedIndexLevels)
} //}
// func (s *WALReplayer) replayPacket(records []any, isLastPacket bool) {
// for _, untyped := range records {
// switch rec := untyped.(type) {
// case storage.WALRecordAppendMeasures:
// s.onWALRecordAppendMeasures(rec, isLastPacket)
// }
// }
// return
// }

20
enc/cumdelta.go
View File

@@ -48,18 +48,15 @@ type CumulativeDeltaCompressor struct {
} }
// Після відновлення із снапшота // Після відновлення із снапшота
func NewCumulativeDeltaCompressor(buf []byte, fracDigits byte) *CumulativeDeltaCompressor { func NewCumulativeDeltaCompressor(fracDigits byte, buf []byte, payloadSize int) *CumulativeDeltaCompressor {
var coef float64 = 1 var coef float64 = 1
if fracDigits > 0 { if fracDigits > 0 {
coef = math.Pow(10, float64(fracDigits)) coef = math.Pow(10, float64(fracDigits))
} }
return &CumulativeDeltaCompressor{ s := &CumulativeDeltaCompressor{
buf: buf, buf: buf,
coef: coef, coef: coef,
} }
}
func (s *CumulativeDeltaCompressor) RestoreState(payloadSize int) {
if payloadSize > 0 { if payloadSize > 0 {
s.pos = payloadSize - 1 // завжди показує на h s.pos = payloadSize - 1 // завжди показує на h
// base value на початку // base value на початку
@@ -74,16 +71,27 @@ func (s *CumulativeDeltaCompressor) RestoreState(payloadSize int) {
log.Fatalf("bug: get last delta: %s", err) log.Fatalf("bug: get last delta: %s", err)
} }
} }
return s
} }
// func (s *CumulativeDeltaCompressor) RestoreState() {
// }
// func (s *CumulativeDeltaCompressor) Size() int { // func (s *CumulativeDeltaCompressor) Size() int {
// return s.pos // return s.pos
// } // }
type EvaluationReport struct {
CompressionWay int
TotalSpace int
AdditionalSpace int
Delta uint64
}
func (s *CumulativeDeltaCompressor) Evaluate(value float64) (compressionWay int, requiredSpace int) { func (s *CumulativeDeltaCompressor) Evaluate(value float64) (compressionWay int, requiredSpace int) {
delta := uint64((value-s.baseValue)*s.coef + eps) // fix - if delta 0, no eps delta := uint64((value-s.baseValue)*s.coef + eps) // fix - if delta 0, no eps
requiredSpace = s.pos requiredSpace = s.pos
// fix - requiredSpace - all space
if s.pos > 0 { if s.pos > 0 {
if s.h < 128 { if s.h < 128 {
// run // run

18
enc/enc_test.go
View File

@@ -165,10 +165,11 @@ func TestCumulativeDeltaCompressor(t *testing.T) {
var ( var (
fracDigits byte = 1 fracDigits byte = 1
buf = make([]byte, 8) buf = make([]byte, 8)
payloadSize = 0
compressionWay int compressionWay int
requiredSpace int requiredSpace int
) )
c := NewCumulativeDeltaCompressor(buf, fracDigits) c := NewCumulativeDeltaCompressor(fracDigits, buf, payloadSize)
for _, num := range testCase.Nums { for _, num := range testCase.Nums {
compressionWay, requiredSpace = c.Evaluate(num) compressionWay, requiredSpace = c.Evaluate(num)
c.Compress(compressionWay, num) c.Compress(compressionWay, num)
@@ -281,9 +282,10 @@ func TestCumulativeDeltaDecompressorFromState(t *testing.T) {
var ( var (
fracDigits byte = 1 fracDigits byte = 1
buf = make([]byte, 16) buf = make([]byte, 16)
payloadSize = 0
decodedNums []float64 decodedNums []float64
) )
c := NewCumulativeDeltaCompressor(buf, fracDigits) c := NewCumulativeDeltaCompressor(fracDigits, buf, payloadSize)
for _, num := range testCase.Nums { for _, num := range testCase.Nums {
compressionWay, _ := c.Evaluate(num) compressionWay, _ := c.Evaluate(num)
c.Compress(compressionWay, num) c.Compress(compressionWay, num)
@@ -524,10 +526,11 @@ func TestInstantDeltaCompressor(t *testing.T) {
var ( var (
fracDigits byte = 1 fracDigits byte = 1
buf = make([]byte, 8) buf = make([]byte, 8)
payloadSize = 0
compressionWay int compressionWay int
requiredSpace int requiredSpace int
) )
c := NewInstantDeltaCompressor(buf, fracDigits) c := NewInstantDeltaCompressor(fracDigits, buf, payloadSize)
for _, num := range testCase.Nums { for _, num := range testCase.Nums {
compressionWay, requiredSpace = c.Evaluate(num) compressionWay, requiredSpace = c.Evaluate(num)
c.Compress(compressionWay, num) c.Compress(compressionWay, num)
@@ -640,9 +643,10 @@ func TestInstantDeltaDecompressorFromState(t *testing.T) {
var ( var (
fracDigits byte = 1 fracDigits byte = 1
buf = make([]byte, 16) buf = make([]byte, 16)
payloadSize = 0
decodedNums []float64 decodedNums []float64
) )
c := NewInstantDeltaCompressor(buf, fracDigits) c := NewInstantDeltaCompressor(fracDigits, buf, payloadSize)
for _, num := range testCase.Nums { for _, num := range testCase.Nums {
compressionWay, _ := c.Evaluate(num) compressionWay, _ := c.Evaluate(num)
c.Compress(compressionWay, num) c.Compress(compressionWay, num)
@@ -840,10 +844,11 @@ func TestTimeDeltaCompressor(t *testing.T) {
//fmt.Println("----------") //fmt.Println("----------")
var ( var (
buf = make([]byte, 8) buf = make([]byte, 8)
payloadSize = 0
compressionWay int compressionWay int
requiredSpace int requiredSpace int
) )
c := NewTimeDeltaCompressor(buf) c := NewTimeDeltaCompressor(buf, payloadSize)
for _, num := range testCase.Nums { for _, num := range testCase.Nums {
compressionWay, requiredSpace = c.Evaluate(num) compressionWay, requiredSpace = c.Evaluate(num)
c.Compress(compressionWay, num) c.Compress(compressionWay, num)
@@ -957,9 +962,10 @@ func TestTimeDeltaDecompressorFromState(t *testing.T) {
for caseIdx, testCase := range testCases { for caseIdx, testCase := range testCases {
var ( var (
buf = make([]byte, 16) buf = make([]byte, 16)
payloadSize = 0
decodedNums []uint32 decodedNums []uint32
) )
c := NewTimeDeltaCompressor(buf) c := NewTimeDeltaCompressor(buf, payloadSize)
for _, num := range testCase.Nums { for _, num := range testCase.Nums {
compressionWay, _ := c.Evaluate(num) compressionWay, _ := c.Evaluate(num)
c.Compress(compressionWay, num) c.Compress(compressionWay, num)

11
enc/insdelta.go
View File

@@ -19,7 +19,7 @@ type InstantDeltaCompressor struct {
state *InstantDeltaCapturedState state *InstantDeltaCapturedState
} }
func NewInstantDeltaCompressor(buf []byte, fracDigits byte) *InstantDeltaCompressor { func NewInstantDeltaCompressor(fracDigits byte, buf []byte, payloadSize int) *InstantDeltaCompressor {
var coef float64 = 1 var coef float64 = 1
if fracDigits > 0 { if fracDigits > 0 {
coef = math.Pow(10, float64(fracDigits)) coef = math.Pow(10, float64(fracDigits))
@@ -28,10 +28,6 @@ func NewInstantDeltaCompressor(buf []byte, fracDigits byte) *InstantDeltaCompres
buf: buf, buf: buf,
coef: coef, coef: coef,
} }
return s
}
func (s *InstantDeltaCompressor) RestoreState(payloadSize int) {
if payloadSize > 0 { if payloadSize > 0 {
s.pos = payloadSize - 1 // завжди показує на h s.pos = payloadSize - 1 // завжди показує на h
// base value на початку // base value на початку
@@ -46,8 +42,13 @@ func (s *InstantDeltaCompressor) RestoreState(payloadSize int) {
log.Fatalf("bug: get last delta: %s", err) log.Fatalf("bug: get last delta: %s", err)
} }
} }
return s
} }
// func (s *InstantDeltaCompressor) RestoreState() {
// }
// func (s *InstantDeltaCompressor) Size() int { // func (s *InstantDeltaCompressor) Size() int {
// return s.pos // return s.pos
// } // }

15
enc/time_delta.go
View File

@@ -34,15 +34,12 @@ type TimeDeltaCompressor struct {
// а lastDelta і h треба прочитати із payload (якщо є) // а lastDelta і h треба прочитати із payload (якщо є)
// Кодує значення справа наліво функціями TailPut*. Читає значення зліва направо функціями Get* // Кодує значення справа наліво функціями TailPut*. Читає значення зліва направо функціями Get*
func NewTimeDeltaCompressor(buf []byte) *TimeDeltaCompressor { func NewTimeDeltaCompressor(buf []byte, payloadSize int) *TimeDeltaCompressor {
return &TimeDeltaCompressor{ s := &TimeDeltaCompressor{
buf: buf, buf: buf,
} }
}
func (s *TimeDeltaCompressor) RestoreState(payloadSize int, lastTimestamp uint32) {
if payloadSize > 0 { if payloadSize > 0 {
s.lastUnixtime = lastTimestamp //s.lastUnixtime = lastTimestamp
s.pos = len(s.buf) - payloadSize s.pos = len(s.buf) - payloadSize
if payloadSize > 4 { if payloadSize > 4 {
s.h = s.buf[s.pos] s.h = s.buf[s.pos]
@@ -53,8 +50,14 @@ func (s *TimeDeltaCompressor) RestoreState(payloadSize int, lastTimestamp uint32
s.lastDelta = uint32(u64) s.lastDelta = uint32(u64)
} }
} }
return s
} }
// FIX - restire lastUnixtime
// func (s *TimeDeltaCompressor) RestoreState() {
// }
// retrun // retrun
func (s *TimeDeltaCompressor) Evaluate(unixtime uint32) (compressionWay int, requiredSpace int) { func (s *TimeDeltaCompressor) Evaluate(unixtime uint32) (compressionWay int, requiredSpace int) {
requiredSpace = len(s.buf) - s.pos requiredSpace = len(s.buf) - s.pos

2
examples/database/main.go
View File

@@ -82,7 +82,7 @@ const mainUsage = `Usage:
` `
const helpMessage = `Diploma project. Database. Version: 1.0 const helpMessage = `Gordenko project. Database. Version: 1.0
created by Dmytro Gordenko, 1.e4.kc6@gmail.com created by Dmytro Gordenko, 1.e4.kc6@gmail.com
` `

28
examples/loadtest/loadtest.go
View File

@@ -7,7 +7,7 @@ import (
"os" "os"
"time" "time"
diploma "gordenko.dev/dima/qb" "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/client" "gordenko.dev/dima/qb/client"
"gordenko.dev/dima/qb/proto" "gordenko.dev/dima/qb/proto"
) )
@@ -16,7 +16,7 @@ import (
type MetricInfo struct { type MetricInfo struct {
MetricID uint32 `json:"metricID"` MetricID uint32 `json:"metricID"`
MetricType diploma.MetricType `json:"metricType"` MetricType qb.MetricType `json:"metricType"`
FracDigits int `json:"fracDigits"` FracDigits int `json:"fracDigits"`
Since int64 `json:"since"` Since int64 `json:"since"`
Until int64 `json:"until"` Until int64 `json:"until"`
@@ -42,12 +42,12 @@ type QueryRecipe struct {
RangeCode int RangeCode int
Since uint32 Since uint32
Until uint32 Until uint32
GroupBy diploma.GroupBy GroupBy qb.GroupBy
} }
type RandomQueryGenerator struct { type RandomQueryGenerator struct {
metrics []MetricInfo metrics []MetricInfo
groupByOptions []diploma.GroupBy groupByOptions []qb.GroupBy
listCurrentValuesProbability int listCurrentValuesProbability int
listPeriodsProbability int listPeriodsProbability int
timeRangeProbDistribution []int timeRangeProbDistribution []int
@@ -86,10 +86,10 @@ func NewRandomQueryGenerator(opt RandomQueryGeneratorOptions) *RandomQueryGenera
return &RandomQueryGenerator{ return &RandomQueryGenerator{
metrics: opt.Metrics, metrics: opt.Metrics,
groupByOptions: []diploma.GroupBy{ groupByOptions: []qb.GroupBy{
diploma.GroupByHour, qb.GroupByHour,
diploma.GroupByDay, qb.GroupByDay,
diploma.GroupByMonth, qb.GroupByMonth,
}, },
listCurrentValuesProbability: opt.ListCurrentValuesProbability, listCurrentValuesProbability: opt.ListCurrentValuesProbability,
listPeriodsProbability: opt.ListPeriodsProbability, listPeriodsProbability: opt.ListPeriodsProbability,
@@ -108,7 +108,7 @@ func (s *RandomQueryGenerator) GetQueryRecipe() QueryRecipe {
Method: listCurrentValues, Method: listCurrentValues,
} }
} else { } else {
if metric.MetricType == diploma.Cumulative { if metric.MetricType == qb.Cumulative {
num = rand.Intn(100) num = rand.Intn(100)
if num < s.listPeriodsProbability { if num < s.listPeriodsProbability {
groupBy := s.groupByOptions[rand.Intn(len(s.groupByOptions))] groupBy := s.groupByOptions[rand.Intn(len(s.groupByOptions))]
@@ -118,10 +118,10 @@ func (s *RandomQueryGenerator) GetQueryRecipe() QueryRecipe {
maxDays = 7 maxDays = 7
) )
if groupBy == diploma.GroupByDay { if groupBy == qb.GroupByDay {
minDays = 1 minDays = 1
maxDays = 30 maxDays = 30
} else if groupBy == diploma.GroupByMonth { } else if groupBy == qb.GroupByMonth {
minDays = 1 minDays = 1
maxDays = 30 maxDays = 30
} }
@@ -164,10 +164,10 @@ func (s *RandomQueryGenerator) GetQueryRecipe() QueryRecipe {
maxDays = 7 maxDays = 7
) )
if groupBy == diploma.GroupByDay { if groupBy == qb.GroupByDay {
minDays = 1 minDays = 1
maxDays = 30 maxDays = 30
} else if groupBy == diploma.GroupByMonth { } else if groupBy == qb.GroupByMonth {
minDays = 1 minDays = 1
maxDays = 30 maxDays = 30
} }
@@ -350,7 +350,7 @@ func execQuery(conn *client.Connection, queryGenerator *RandomQueryGenerator, st
Day: until.Day(), Day: until.Day(),
}, },
GroupBy: recipe.GroupBy, GroupBy: recipe.GroupBy,
AggregateFuncs: diploma.AggregateMin | diploma.AggregateMax | diploma.AggregateAvg, AggregateFuncs: qb.AggregateMin | qb.AggregateMax | qb.AggregateAvg,
}) })
elapsedTime = time.Since(t1) elapsedTime = time.Since(t1)
stat.ElapsedTime += elapsedTime stat.ElapsedTime += elapsedTime

2
examples/loadtest/main.go
View File

@@ -203,7 +203,7 @@ const mainUsage = `Usage:
` `
const helpMessage = `Diploma project. Load test. Version: 1.0 const helpMessage = `Gordenko project. Load test. Version: 1.0
created by Dmytro Gordenko, 1.e4.kc6@gmail.com created by Dmytro Gordenko, 1.e4.kc6@gmail.com
` `

2
examples/requests/main.go
View File

@@ -54,7 +54,7 @@ const mainUsage = `Usage:
` `
const helpMessage = `Diploma project. Example requests. Version: 1.0 const helpMessage = `Gordenko project. Example requests. Version: 1.0
created by Dmytro Gordenko, 1.e4.kc6@gmail.com created by Dmytro Gordenko, 1.e4.kc6@gmail.com
` `

24
examples/requests/requests.go
View File

@@ -5,7 +5,7 @@ import (
"log" "log"
"time" "time"
diploma "gordenko.dev/dima/qb" "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/client" "gordenko.dev/dima/qb/client"
"gordenko.dev/dima/qb/proto" "gordenko.dev/dima/qb/proto"
) )
@@ -25,7 +25,7 @@ func sendRequests(conn *client.Connection) {
err = conn.AddMetric(proto.AddMetricReq{ err = conn.AddMetric(proto.AddMetricReq{
MetricID: instantMetricID, MetricID: instantMetricID,
MetricType: diploma.Instant, MetricType: qb.Instant,
FracDigits: fracDigits, FracDigits: fracDigits,
}) })
if err != nil { if err != nil {
@@ -114,8 +114,8 @@ GetMetric:
Month: until.Month(), Month: until.Month(),
Day: until.Day(), Day: until.Day(),
}, },
GroupBy: diploma.GroupByHour, GroupBy: qb.GroupByHour,
AggregateFuncs: diploma.AggregateMin | diploma.AggregateMax | diploma.AggregateAvg, AggregateFuncs: qb.AggregateMin | qb.AggregateMax | qb.AggregateAvg,
}) })
if err != nil { if err != nil {
log.Fatalf("conn.ListInstantPeriods: %s\n", err) log.Fatalf("conn.ListInstantPeriods: %s\n", err)
@@ -143,8 +143,8 @@ GetMetric:
Month: until.Month(), Month: until.Month(),
Day: until.Day(), Day: until.Day(),
}, },
GroupBy: diploma.GroupByDay, GroupBy: qb.GroupByDay,
AggregateFuncs: diploma.AggregateMin | diploma.AggregateMax | diploma.AggregateAvg, AggregateFuncs: qb.AggregateMin | qb.AggregateMax | qb.AggregateAvg,
}) })
if err != nil { if err != nil {
log.Fatalf("conn.ListInstantPeriods: %s\n", err) log.Fatalf("conn.ListInstantPeriods: %s\n", err)
@@ -172,8 +172,8 @@ GetMetric:
Month: until.Month(), Month: until.Month(),
Day: until.Day(), Day: until.Day(),
}, },
GroupBy: diploma.GroupByMonth, GroupBy: qb.GroupByMonth,
AggregateFuncs: diploma.AggregateMin | diploma.AggregateMax | diploma.AggregateAvg, AggregateFuncs: qb.AggregateMin | qb.AggregateMax | qb.AggregateAvg,
}) })
if err != nil { if err != nil {
log.Fatalf("conn.ListInstantPeriods: %s\n", err) log.Fatalf("conn.ListInstantPeriods: %s\n", err)
@@ -208,7 +208,7 @@ GetMetric:
err = conn.AddMetric(proto.AddMetricReq{ err = conn.AddMetric(proto.AddMetricReq{
MetricID: cumulativeMetricID, MetricID: cumulativeMetricID,
MetricType: diploma.Cumulative, MetricType: qb.Cumulative,
FracDigits: fracDigits, FracDigits: fracDigits,
}) })
if err != nil { if err != nil {
@@ -298,7 +298,7 @@ GetMetric:
Month: until.Month(), Month: until.Month(),
Day: until.Day(), Day: until.Day(),
}, },
GroupBy: diploma.GroupByHour, GroupBy: qb.GroupByHour,
}) })
if err != nil { if err != nil {
log.Fatalf("conn.ListCumulativePeriods: %s\n", err) log.Fatalf("conn.ListCumulativePeriods: %s\n", err)
@@ -326,7 +326,7 @@ GetMetric:
Month: until.Month(), Month: until.Month(),
Day: until.Day(), Day: until.Day(),
}, },
GroupBy: diploma.GroupByDay, GroupBy: qb.GroupByDay,
}) })
if err != nil { if err != nil {
log.Fatalf("conn.ListCumulativePeriods: %s\n", err) log.Fatalf("conn.ListCumulativePeriods: %s\n", err)
@@ -354,7 +354,7 @@ GetMetric:
Month: until.Month(), Month: until.Month(),
Day: until.Day(), Day: until.Day(),
}, },
GroupBy: diploma.GroupByMonth, GroupBy: qb.GroupByMonth,
}) })
if err != nil { if err != nil {
log.Fatalf("conn.ListCumulativePeriods: %s\n", err) log.Fatalf("conn.ListCumulativePeriods: %s\n", err)

42
freelist/freelist.go
View File

@@ -1,12 +1,11 @@
package freelist package freelist
import ( import (
"bytes"
"fmt" "fmt"
"io" "io"
"os" "os"
"gordenko.dev/dima/qb/bin" bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb/util" "gordenko.dev/dima/qb/util"
) )
@@ -158,12 +157,13 @@ func (s *FreeList) loadLastPage() error {
return fmt.Errorf("read size %d bytes not equal page size %d", n, s.pageSize) return fmt.Errorf("read size %d bytes not equal page size %d", n, s.pageSize)
} }
// check crc // check crc
checksum := bin.GetUint32(buf[0:]) checksum, _ := bin.GetUint32(buf[0:])
if util.CalcChecksum(buf[crcSize:]) != checksum { if util.CalcChecksum(buf[crcSize:]) != checksum {
return fmt.Errorf("page %d is corrupted", s.basePagesCount) return fmt.Errorf("page %d is corrupted", s.basePagesCount)
} }
for i := crcSize; i < len(buf); i += ptrSize { for i := crcSize; i < len(buf); i += ptrSize {
s.free = append(s.free, bin.GetUint32(buf[i:])) pageNo, _ := bin.GetUint32(buf[i:])
s.free = append(s.free, pageNo)
} }
if s.deltaPagesCount > 0 { if s.deltaPagesCount > 0 {
s.deltaPagesCount-- s.deltaPagesCount--
@@ -207,25 +207,25 @@ func (s *FreeList) GetPageNumber() (pageNo uint32, err error) {
} }
// для відновлення недозаповненнох сторінки із снапшота // для відновлення недозаповненнох сторінки із снапшота
func (s *FreeList) Restore(serialized []byte) error { // func (s *FreeList) Restore(serialized []byte) error {
if (len(serialized) % ptrSize) != 0 { // if (len(serialized) % ptrSize) != 0 {
return fmt.Errorf("wrong size") // return fmt.Errorf("wrong size")
} // }
for i := 0; i < len(serialized); i += ptrSize { // for i := 0; i < len(serialized); i += ptrSize {
s.free = append(s.free, bin.GetUint32(serialized[i:])) // s.free = append(s.free, bin.GetUint32(serialized[i:]))
} // }
fmt.Println(s.free) // fmt.Println(s.free)
return nil // return nil
} // }
// для запису в snapshot // для запису в snapshot
func (s *FreeList) Serialize() ([]byte, error) { // func (s *FreeList) Serialize() ([]byte, error) {
w := bytes.NewBuffer(nil) // w := bytes.NewBuffer(nil)
for _, pageNo := range s.free { // for _, pageNo := range s.free {
bin.WriteUint32(w, pageNo) // bin.WriteUint32(w, pageNo)
} // }
return w.Bytes(), nil // return w.Bytes(), nil
} // }
func (s *FreeList) Cached() []uint32 { func (s *FreeList) Cached() []uint32 {
return s.free return s.free

64
freelist/freelist_test.go
View File

@@ -58,46 +58,46 @@ func TestSave(t *testing.T) {
// } // }
} }
func TestDelta(t *testing.T) { // func TestDelta(t *testing.T) {
freeList, err := createFreeList() // freeList, err := createFreeList()
if err != nil { // if err != nil {
t.Fatal(err) // t.Fatal(err)
} // }
serialized := []byte{9, 0, 0, 0, 10, 0, 0, 0} // serialized := []byte{9, 0, 0, 0, 10, 0, 0, 0}
err = freeList.Restore(serialized) // err = freeList.Restore(serialized)
if err != nil { // if err != nil {
t.Fatal(err) // t.Fatal(err)
} // }
for range 3 { // for range 3 {
pageNo, err := freeList.GetPageNumber() // pageNo, err := freeList.GetPageNumber()
fmt.Println(pageNo, err) // fmt.Println(pageNo, err)
} // }
err = freeList.AddPageNumbers([]uint32{20, 21, 22}) // err = freeList.AddPageNumbers([]uint32{20, 21, 22})
if err != nil { // if err != nil {
t.Fatal(err) // t.Fatal(err)
} // }
err = freeList.AddPageNumbers([]uint32{23, 24, 25, 26}) // err = freeList.AddPageNumbers([]uint32{23, 24, 25, 26})
if err != nil { // if err != nil {
t.Fatal(err) // t.Fatal(err)
} // }
serialized, err = freeList.Serialize() // serialized, err = freeList.Serialize()
if err != nil { // if err != nil {
t.Fatal(err) // t.Fatal(err)
} // }
fmt.Println("free serialized:", serialized) // fmt.Println("free serialized:", serialized)
err = freeList.Merge() // err = freeList.Merge()
if err != nil { // if err != nil {
t.Fatal(err) // t.Fatal(err)
} // }
} // }
// func TestGetFromFreeList(t *testing.T) { // func TestGetFromFreeList(t *testing.T) {
// freeList, err := createFreeList() // freeList, err := createFreeList()

84
proto/proto.go
View File

@@ -4,8 +4,8 @@ import (
"fmt" "fmt"
"time" "time"
diploma "gordenko.dev/dima/qb" bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb/bin" "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/bufreader" "gordenko.dev/dima/qb/bufreader"
) )
@@ -72,7 +72,7 @@ func ErrorCodeToText(code uint16) string {
type Metric struct { type Metric struct {
MetricID uint32 MetricID uint32
MetricType diploma.MetricType MetricType qb.MetricType
FracDigits int FracDigits int
} }
@@ -161,7 +161,7 @@ func ReadListCurrentValuesReq(r *bufreader.BufferedReader) (m ListCurrentValuesR
type AddMetricReq struct { type AddMetricReq struct {
MetricID uint32 MetricID uint32
MetricType diploma.MetricType MetricType qb.MetricType
FracDigits int FracDigits int
} }
@@ -175,8 +175,8 @@ func ReadAddMetricReq(r *bufreader.BufferedReader) (m AddMetricReq, err error) {
} }
func UnpackAddMetricReq(arr []byte) (m AddMetricReq) { func UnpackAddMetricReq(arr []byte) (m AddMetricReq) {
m.MetricID = bin.GetUint32(arr) m.MetricID, _ = bin.GetUint32(arr)
m.MetricType = diploma.MetricType(arr[4]) m.MetricType = qb.MetricType(arr[4])
m.FracDigits = int(arr[5]) m.FracDigits = int(arr[5])
return return
} }
@@ -209,8 +209,8 @@ func ReadDeleteMeasuresReq(r *bufreader.BufferedReader) (m DeleteMeasuresReq, er
} }
func UnpackDeleteMeasuresReq(arr []byte) (m DeleteMeasuresReq) { func UnpackDeleteMeasuresReq(arr []byte) (m DeleteMeasuresReq) {
m.MetricID = bin.GetUint32(arr) m.MetricID, _ = bin.GetUint32(arr)
m.Since = bin.GetUint32(arr[4:]) m.Since, _ = bin.GetUint32(arr[4:])
return return
} }
@@ -230,9 +230,9 @@ func ReadAppendMeasureReq(r *bufreader.BufferedReader) (m AppendMeasureReq, err
} }
func UnpackAppendMeasureReq(arr []byte) (m AppendMeasureReq) { func UnpackAppendMeasureReq(arr []byte) (m AppendMeasureReq) {
m.MetricID = bin.GetUint32(arr) m.MetricID, _ = bin.GetUint32(arr)
m.Timestamp = bin.GetUint32(arr[4:]) m.Timestamp, _ = bin.GetUint32(arr[4:])
m.Value = bin.GetFloat64(arr[8:]) m.Value, _ = bin.GetFloat64(arr[8:])
return return
} }
@@ -253,8 +253,8 @@ func ReadAppendMeasuresReq(r *bufreader.BufferedReader) (m AppendMeasuresReq, er
return return
} }
m.MetricID = bin.GetUint32(prefix[0:]) m.MetricID, _ = bin.GetUint32(prefix[0:])
qty := bin.GetUint16(prefix[4:]) qty, _ := bin.GetUint16(prefix[4:])
for i := range int(qty) { for i := range int(qty) {
var measure Measure var measure Measure
@@ -280,21 +280,25 @@ type MetricMeasure struct {
} }
func ReadAppendMeasurePerMetricReq(r *bufreader.BufferedReader) (measures []MetricMeasure, err error) { func ReadAppendMeasurePerMetricReq(r *bufreader.BufferedReader) (measures []MetricMeasure, err error) {
qty, err := bin.ReadUint16(r) count, err := bin.ReadUint16(r)
if err != nil { if err != nil {
return return
} }
var tmp = make([]byte, 16) for range count {
for range int(qty) { var m MetricMeasure
err = bin.ReadNInto(r, tmp) m.MetricID, err = bin.ReadUint32(r)
if err != nil { if err != nil {
return return
} }
measures = append(measures, MetricMeasure{ m.Timestamp, err = bin.ReadUint32(r)
MetricID: bin.GetUint32(tmp[0:]), if err != nil {
Timestamp: bin.GetUint32(tmp[4:]), return
Value: bin.GetFloat64(tmp[8:]), }
}) m.Value, err = bin.ReadFloat64(r)
if err != nil {
return
}
measures = append(measures, m)
} }
return return
} }
@@ -341,9 +345,9 @@ func ReadListInstantMeasuresReq(r *bufreader.BufferedReader) (m ListInstantMeasu
} }
func UnpackListInstantMeasuresReq(arr []byte) (m ListInstantMeasuresReq) { func UnpackListInstantMeasuresReq(arr []byte) (m ListInstantMeasuresReq) {
m.MetricID = bin.GetUint32(arr[0:]) m.MetricID, _ = bin.GetUint32(arr[0:])
m.Since = bin.GetUint32(arr[4:]) m.Since, _ = bin.GetUint32(arr[4:])
m.Until = bin.GetUint32(arr[8:]) m.Until, _ = bin.GetUint32(arr[8:])
return return
} }
@@ -363,9 +367,9 @@ func ReadListCumulativeMeasuresReq(r *bufreader.BufferedReader) (m ListCumulativ
} }
func UnpackListCumulativeMeasuresReq(arr []byte) (m ListCumulativeMeasuresReq) { func UnpackListCumulativeMeasuresReq(arr []byte) (m ListCumulativeMeasuresReq) {
m.MetricID = bin.GetUint32(arr) m.MetricID, _ = bin.GetUint32(arr)
m.Since = bin.GetUint32(arr[4:]) m.Since, _ = bin.GetUint32(arr[4:])
m.Until = bin.GetUint32(arr[8:]) m.Until, _ = bin.GetUint32(arr[8:])
return return
} }
@@ -373,7 +377,7 @@ type ListInstantPeriodsReq struct {
MetricID uint32 MetricID uint32
Since TimeBound Since TimeBound
Until TimeBound Until TimeBound
GroupBy diploma.GroupBy GroupBy qb.GroupBy
AggregateFuncs byte AggregateFuncs byte
FirstHourOfDay int FirstHourOfDay int
} }
@@ -388,18 +392,20 @@ func ReadListInstantPeriodsReq(r *bufreader.BufferedReader) (m ListInstantPeriod
} }
func UnpackListInstantPeriodsReq(arr []byte) (m ListInstantPeriodsReq) { func UnpackListInstantPeriodsReq(arr []byte) (m ListInstantPeriodsReq) {
m.MetricID = bin.GetUint32(arr) m.MetricID, _ = bin.GetUint32(arr)
sinceYear, _ := bin.GetUint16(arr[4:])
untilYear, _ := bin.GetUint16(arr[8:])
m.Since = TimeBound{ m.Since = TimeBound{
Year: int(bin.GetUint16(arr[4:])), Year: int(sinceYear),
Month: time.Month(arr[6]), Month: time.Month(arr[6]),
Day: int(arr[7]), Day: int(arr[7]),
} }
m.Until = TimeBound{ m.Until = TimeBound{
Year: int(bin.GetUint16(arr[8:])), Year: int(untilYear),
Month: time.Month(arr[10]), Month: time.Month(arr[10]),
Day: int(arr[11]), Day: int(arr[11]),
} }
m.GroupBy = diploma.GroupBy(arr[12]) m.GroupBy = qb.GroupBy(arr[12])
m.AggregateFuncs = arr[13] m.AggregateFuncs = arr[13]
m.FirstHourOfDay = int(arr[14]) m.FirstHourOfDay = int(arr[14])
return return
@@ -409,7 +415,7 @@ type ListCumulativePeriodsReq struct {
MetricID uint32 MetricID uint32
Since TimeBound Since TimeBound
Until TimeBound Until TimeBound
GroupBy diploma.GroupBy GroupBy qb.GroupBy
FirstHourOfDay int FirstHourOfDay int
} }
@@ -423,18 +429,20 @@ func ReadListCumulativePeriodsReq(r *bufreader.BufferedReader) (m ListCumulative
} }
func UnpackListCumulativePeriodsReq(arr []byte) (m ListCumulativePeriodsReq) { func UnpackListCumulativePeriodsReq(arr []byte) (m ListCumulativePeriodsReq) {
m.MetricID = bin.GetUint32(arr[0:]) m.MetricID, _ = bin.GetUint32(arr[0:])
sinceYear, _ := bin.GetUint16(arr[4:])
untilYear, _ := bin.GetUint16(arr[8:])
m.Since = TimeBound{ m.Since = TimeBound{
Year: int(bin.GetUint16(arr[4:])), Year: int(sinceYear),
Month: time.Month(arr[6]), Month: time.Month(arr[6]),
Day: int(arr[7]), Day: int(arr[7]),
} }
m.Until = TimeBound{ m.Until = TimeBound{
Year: int(bin.GetUint16(arr[8:])), Year: int(untilYear),
Month: time.Month(arr[10]), Month: time.Month(arr[10]),
Day: int(arr[11]), Day: int(arr[11]),
} }
m.GroupBy = diploma.GroupBy(arr[12]) m.GroupBy = qb.GroupBy(arr[12])
m.FirstHourOfDay = int(arr[13]) m.FirstHourOfDay = int(arr[13])
return return
} }

4
qb.go
View File

@@ -24,7 +24,6 @@ const (
) )
type TimestampCompressor interface { type TimestampCompressor interface {
RestoreState(int, uint32) // (payload size, last timestamp)
// (timestamp) => compressionWay, requiredSpace // (timestamp) => compressionWay, requiredSpace
Evaluate(uint32) (int, int) Evaluate(uint32) (int, int)
Compress(int, uint32) Compress(int, uint32)
@@ -43,7 +42,6 @@ type TimestampCompressor interface {
} }
type ValueCompressor interface { type ValueCompressor interface {
RestoreState(int) // payload size
// (value) => compressionWay, requiredSpace // (value) => compressionWay, requiredSpace
Evaluate(float64) (int, int) Evaluate(float64) (int, int)
Compress(int, float64) Compress(int, float64)
@@ -110,6 +108,8 @@ const (
DeletePrevSnapshotFileFailed AbortCode = 35 DeletePrevSnapshotFileFailed AbortCode = 35
MetricNotFoundDuringReplay AbortCode = 36 MetricNotFoundDuringReplay AbortCode = 36
WALReplayFailed AbortCode = 37
) )
func Abort(code AbortCode, err error) { func Abort(code AbortCode, err error) {

14
txlog/data_preparer.go → storage/data_preparer.go
View File

@@ -1,4 +1,4 @@
package txlog package storage
import ( import (
"bytes" "bytes"
@@ -51,7 +51,7 @@ func (s *DataPreparer) Prepare(input []any) PreparedData {
// 1. Пакую всі дані в WAL буфер запису // 1. Пакую всі дані в WAL буфер запису
for _, untyped := range input { for _, untyped := range input {
switch x := untyped.(type) { switch x := untyped.(type) {
case AddedMetric: case MetricAddRecord:
x.Pack(w) x.Pack(w)
// case DeletedMetric: // case DeletedMetric:
// if len(x.FreePageNumbers) > 0 { // if len(x.FreePageNumbers) > 0 {
@@ -91,11 +91,11 @@ func (s *DataPreparer) Prepare(input []any) PreparedData {
if len(level.IndexPages) == 0 && level.SkipRecords == level.TailRecordsCount { if len(level.IndexPages) == 0 && level.SkipRecords == level.TailRecordsCount {
break break
} }
var completedIndexPage *CompletedIndexPage var completedIndexPage *HeadIndexPage
if len(level.IndexPages) > 0 { if len(level.IndexPages) > 0 {
first := level.IndexPages[0] first := level.IndexPages[0]
skipSize := level.SkipRecords * indexRecordSize skipSize := level.SkipRecords * indexRecordSize
completedIndexPage = &CompletedIndexPage{ completedIndexPage = &HeadIndexPage{
PageNo: first.PageNo, PageNo: first.PageNo,
Reused: first.Reused, Reused: first.Reused,
Checksum: first.Checksum, Checksum: first.Checksum,
@@ -103,7 +103,7 @@ func (s *DataPreparer) Prepare(input []any) PreparedData {
} }
} }
changedIndexPages = append(changedIndexPages, ChangedIndexLevel{ changedIndexPages = append(changedIndexPages, ChangedIndexLevel{
CompletedIndexPage: completedIndexPage, HeadIndexPage: completedIndexPage,
IndexPages: level.IndexPages[1:], IndexPages: level.IndexPages[1:],
TailRecords: level.TailRecords[:level.TailRecordsCount*indexRecordSize], TailRecords: level.TailRecords[:level.TailRecordsCount*indexRecordSize],
}) })
@@ -116,9 +116,9 @@ func (s *DataPreparer) Prepare(input []any) PreparedData {
first := sealResult.DataPages[0] first := sealResult.DataPages[0]
rec := WALRecordAppendMeasures{ rec := MeasuresAppendWithGrowRecord{
MetricID: x.MetricID, MetricID: x.MetricID,
CompletedDataPage: CompletedDataPage{ HeadDataPage: HeadDataPage{
PageNo: first.PageNo, PageNo: first.PageNo,
Reused: first.Reused, Reused: first.Reused,
PrevPageNo: first.PrevPageNo, PrevPageNo: first.PrevPageNo,

2
txlog/helpers.go → storage/helpers.go
View File

@@ -1,4 +1,4 @@
package txlog package storage
import ( import (
"bytes" "bytes"

41
txlog/page_preparer.go → storage/page_preparer.go
View File

@@ -1,10 +1,10 @@
package txlog package storage
import ( import (
"errors" "errors"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb" "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/bin"
"gordenko.dev/dima/qb/util" "gordenko.dev/dima/qb/util"
) )
@@ -268,7 +268,7 @@ func (s *PagePreparer) SealPages(in SealPagesIn) (result SealResult) {
level.TailRecordsCount = 1 level.TailRecordsCount = 1
// //
upPageNo = pageNo upPageNo = pageNo
upTimestamp = bin.GetUint32(filled.Content) upTimestamp, _ = bin.GetUint32(filled.Content)
levelIdx++ levelIdx++
} }
} }
@@ -277,3 +277,38 @@ func (s *PagePreparer) SealPages(in SealPagesIn) (result SealResult) {
IndexLevels: sealedIndexLevels, IndexLevels: sealedIndexLevels,
} }
} }
// HELPERS
type SealDataPageIn struct {
Content []byte
PrevPageNo uint32
TimestampsSize int
ValuesSize int
}
func SealDataPage(in SealDataPageIn) (checksum uint32) {
// bin.PutUint16(in.Content[s.timestampsSizeIdx:], uint16(in.TimestampsSize))
// bin.PutUint16(in.Content[s.valuesSizeIdx:], uint16(in.ValuesSize))
// bin.PutUint32(in.Content[s.prevPageIdx:], in.PrevPageNo)
// checksum = util.CalcChecksum(in.Content[:s.dataChecksumIdx])
// bin.PutUint32(in.Content[s.dataChecksumIdx:], checksum)
return
}
type SealIndexPageIn struct {
Content []byte
RecordsCount int
ZeroLevel bool
}
func SealIndexPage(in SealIndexPageIn) (checksum uint32) {
// bin.PutUint16(in.Content[s.indexRecordsCountIdx:], uint16(in.RecordsCount))
// if in.LastLevel {
// in.Content[s.isZeroLevelIdx] = 1
// }
// checksum = util.CalcChecksum(in.Content[:s.indexChecksumIdx])
// bin.PutUint32(in.Content[s.indexChecksumIdx:], checksum)
return
}

32
storage/storage.go Normal file
View File

@@ -0,0 +1,32 @@
package storage
const (
//PageNoSize = 4
// data page
DataPageSize = 8192
DataPageFooterSize int = 12
DataPagePayloadSize int = DataPageSize - DataPageFooterSize
dataCRC32Idx = DataPageSize - 4
timestampsSizeIdx = DataPageSize - 6
valuesSizeIdx = DataPageSize - 8
prevPageIdx = DataPageSize - 12
// index page
IndexPageSize = 1024
IndexPageFooterSize = 7
indexRecordSize = 8
maxRecordsOnIndexPage = (IndexPageSize - IndexPageFooterSize) / indexRecordSize
//indexPageIncSize = IndexPageIncSize
indexCRC32Idx = IndexPageSize - 4
indexRecordsCountIdx = IndexPageSize - 6
isZeroLevelIdx = IndexPageSize - 7
// timestampSize = 4
// pairSize = timestampSize + PageNoSize
// indexFooterIdx = indexRecordsQtyIdx
// dataFooterIdx = timestampsSizeIdx
)

30
txlog/reader.go → storage/wal_reader.go
View File

@@ -1,4 +1,4 @@
package txlog package storage
import ( import (
"bufio" "bufio"
@@ -14,19 +14,19 @@ import (
const readBufferSize = 8 * 1024 * 1024 const readBufferSize = 8 * 1024 * 1024
type Reader struct { type WALReader struct {
file *os.File file *os.File
reader *bufio.Reader reader *bufio.Reader
current []any current []any
next []any next []any
} }
type ReaderOptions struct { type WALReaderOptions struct {
FileName string FileName string
BufferSize int BufferSize int
} }
func NewReader(opt ReaderOptions) (*Reader, error) { func NewWALReader(opt WALReaderOptions) (*WALReader, error) {
if opt.FileName == "" { if opt.FileName == "" {
return nil, errors.New("FileName option is required") return nil, errors.New("FileName option is required")
} }
@@ -39,17 +39,17 @@ func NewReader(opt ReaderOptions) (*Reader, error) {
return nil, err return nil, err
} }
return &Reader{ return &WALReader{
file: file, file: file,
reader: bufio.NewReaderSize(file, readBufferSize), reader: bufio.NewReaderSize(file, readBufferSize),
}, nil }, nil
} }
func (s *Reader) Close() { func (s *WALReader) Close() {
s.file.Close() s.file.Close()
} }
func (s *Reader) NextPacket() ([]any, bool, error) { func (s *WALReader) NextPacket() ([]any, bool, error) {
var err error var err error
if s.current == nil { if s.current == nil {
s.current, err = s.readPacket() s.current, err = s.readPacket()
@@ -75,7 +75,7 @@ func (s *Reader) NextPacket() ([]any, bool, error) {
return current, done, nil return current, done, nil
} }
func (s *Reader) readPacket() (_ []any, err error) { func (s *WALReader) readPacket() (_ []any, err error) {
payloadSize, err := bin.ReadVarSize(s.reader) payloadSize, err := bin.ReadVarSize(s.reader)
if err != nil { if err != nil {
if err == io.EOF { if err == io.EOF {
@@ -107,7 +107,7 @@ func (s *Reader) readPacket() (_ []any, err error) {
return nil, nil return nil, nil
} }
func (s *Reader) parseRecords(body []byte) ([]any, error) { func (s *WALReader) parseRecords(body []byte) ([]any, error) {
var ( var (
src = bytes.NewBuffer(body) src = bytes.NewBuffer(body)
records []any records []any
@@ -123,15 +123,15 @@ func (s *Reader) parseRecords(body []byte) ([]any, error) {
} }
switch recordType { switch recordType {
case CodeAddedMetric: case CodeMetricAdd:
rec := new(AddedMetric) rec := new(MetricAddRecord)
if err = rec.Parse(src); err != nil { if err = rec.Parse(src); err != nil {
return nil, err return nil, err
} }
records = append(records, rec) records = append(records, rec)
case CodeDeletedMetric: case CodeMetricDelete:
rec := new(DeletedMetric) rec := new(MetricDeleteRecord)
if err = rec.Parse(src); err != nil { if err = rec.Parse(src); err != nil {
return nil, err return nil, err
} }
@@ -158,7 +158,7 @@ func (s *Reader) parseRecords(body []byte) ([]any, error) {
// } // }
// records = append(records, rec) // records = append(records, rec)
case CodeDeletedMeasures: case CodeMeasuresDelete:
rec := new(DeletedMeasures) rec := new(DeletedMeasures)
if err = rec.Parse(src); err != nil { if err = rec.Parse(src); err != nil {
return nil, err return nil, err
@@ -349,7 +349,7 @@ func (s *Reader) parseRecords(body []byte) ([]any, error) {
// return freePages, nil // return freePages, nil
// } // }
func (s *Reader) Seek(offset int64) error { func (s *WALReader) Seek(offset int64) error {
ret, err := s.file.Seek(offset, 0) ret, err := s.file.Seek(offset, 0)
if err != nil { if err != nil {
return err return err

666
storage/wal_records.go Normal file
View File

@@ -0,0 +1,666 @@
package storage
import (
"bytes"
"io"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
)
type HeadDataPage struct {
PageNo uint32
Reused bool
PrevPageNo uint32
Checksum uint32
TimestampsOffset int // offset on 1st page
Timestamps []byte
ValuesOffset int // offset on 1st page
Values []byte
}
type HeadIndexPage struct {
PageNo uint32
Reused bool
Checksum uint32
Records []byte // offset не потрібен, оскільки Records додаються в кінець
}
type ChangedIndexLevel struct {
HeadIndexPage *HeadIndexPage
IndexPages []SealedIndexPage
TailRecords []byte // payload only
}
type MeasuresAppendWithGrowRecord struct {
MetricID uint32
HeadDataPage HeadDataPage
DataPages []SealedDataPage
ChangedIndexLevels []ChangedIndexLevel
TailTimestamps []byte // data level tail
TailValues []byte // data level tail
}
/*
Format appended measures:
1b - tx type
4b - metricID
4b - pageNo
1b - reused
4b - prevPageNo
4b - page crc32
Nb - (varsize) timestamps offset
Nb - (varsize) timestamps tail size on 1st filled data page
Nb - timestamps tail on 1st filled data page
Nb - (varsize) values offset
Nb - (varsize) values tail size on 1st filled data page
Nb - values tail on 1st filled data page
Nb - (varsize) filled data pages qty
[
4b - pageNo
1b - reused
Nb - (data page size) page content
]
Nb - (varsize) timestamps payload size on tail
Nb - timestamps payload
Nb - (varsize) values payload size on tail
Nb - values payload
Nb - (varsize) qty of index levels
[
// NOTE!
// - idx = 0 - zeroLevel
// - skipped = maxRecordsOnIndexPage - records count
4b - pageNo of 1st index page
1b - reused
4b - page crc32
Nb - (varsize) records tail count on 1st filled index page
Nb - records tail on 1st filled index page
Nb - (varsize) qty of level filled pages
[
4b - pageNo
1b - reused
Nb - (index page size) page content
]
Nb - (varsize) size of records level tail
Nb - level tail records
]
*/
func (s MeasuresAppendWithGrowRecord) WriteTo(w *bytes.Buffer) {
w.WriteByte(CodeMeasuresAppendWithGrow)
bin.WriteUint32(w, s.MetricID)
// completed data page
completed := s.HeadDataPage
bin.WriteUint32(w, completed.PageNo)
bin.WriteBool(w, completed.Reused)
bin.WriteUint32(w, completed.PrevPageNo)
bin.WriteUint32(w, completed.Checksum)
bin.WriteVarSize(w, completed.TimestampsOffset)
bin.WriteVarSize(w, len(completed.Timestamps))
w.Write(completed.Timestamps)
bin.WriteVarSize(w, completed.ValuesOffset)
bin.WriteVarSize(w, len(completed.Values))
w.Write(completed.Values)
// data pages
bin.WriteVarSize(w, len(s.DataPages))
for _, p := range s.DataPages {
bin.WriteUint32(w, p.PageNo)
bin.WriteBool(w, p.Reused)
bin.WriteVarSize(w, len(p.Content))
w.Write(p.Content)
}
// data tail
bin.WriteVarSize(w, len(s.TailTimestamps))
w.Write(s.TailTimestamps)
bin.WriteVarSize(w, len(s.TailValues))
w.Write(s.TailValues)
// changed levels
bin.WriteVarSize(w, len(s.ChangedIndexLevels))
for _, level := range s.ChangedIndexLevels {
// completed index page
completed := level.HeadIndexPage
bin.WriteUint32(w, completed.PageNo)
bin.WriteBool(w, completed.Reused)
bin.WriteUint32(w, completed.Checksum)
bin.WriteVarSize(w, len(completed.Records)/indexRecordSize)
w.Write(completed.Records)
// full index pages
for _, p := range level.IndexPages {
bin.WriteUint32(w, p.PageNo)
bin.WriteBool(w, p.Reused)
bin.WriteVarSize(w, len(p.Content))
w.Write(p.Content)
}
// level tail
bin.WriteVarSize(w, len(level.TailRecords)/indexRecordSize)
w.Write(level.TailRecords)
}
}
func (s *MeasuresAppendWithGrowRecord) Parse(r *bytes.Buffer) (err error) {
var (
size int
recordsCount int
)
s.MetricID, err = bin.ReadUint32(r)
if err != nil {
return
}
s.HeadDataPage.PageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
s.HeadDataPage.Reused, err = bin.ReadBool(r)
if err != nil {
return
}
s.HeadDataPage.PrevPageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
s.HeadDataPage.Checksum, err = bin.ReadUint32(r)
if err != nil {
return
}
s.HeadDataPage.TimestampsOffset, err = bin.ReadVarSize(r)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.HeadDataPage.Timestamps, err = bin.ReadN(r, size)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.HeadDataPage.Values, err = bin.ReadN(r, size)
if err != nil {
return
}
// data pages
dataPagesQty, err := bin.ReadVarSize(r)
if err != nil {
return
}
for range dataPagesQty {
var (
p SealedDataPage
)
p.PageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
p.Reused, err = bin.ReadBool(r)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
p.Content, err = bin.ReadN(r, size)
if err != nil {
return
}
s.DataPages = append(s.DataPages, p)
}
// data tail
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.TailTimestamps, err = bin.ReadN(r, size)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.TailValues, err = bin.ReadN(r, size)
if err != nil {
return
}
// changed levels
changedIndexLevelsCount, err := bin.ReadVarSize(r)
if err != nil {
return
}
for range changedIndexLevelsCount {
var (
indexPagesCount int
level ChangedIndexLevel
)
// completed index page
level.HeadIndexPage.PageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
level.HeadIndexPage.Reused, err = bin.ReadBool(r)
if err != nil {
return
}
level.HeadIndexPage.Checksum, err = bin.ReadUint32(r)
if err != nil {
return
}
recordsCount, err = bin.ReadVarSize(r)
if err != nil {
return
}
level.HeadIndexPage.Records, err = bin.ReadN(r, recordsCount*indexRecordSize)
if err != nil {
return
}
//
indexPagesCount, err = bin.ReadVarSize(r)
if err != nil {
return
}
for range indexPagesCount {
var p SealedIndexPage
p.PageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
p.Reused, err = bin.ReadBool(r)
if err != nil {
return
}
p.Checksum, err = bin.ReadUint32(r)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
p.Content, err = bin.ReadN(r, size)
if err != nil {
return
}
level.IndexPages = append(level.IndexPages, p)
}
recordsCount, err = bin.ReadVarSize(r)
if err != nil {
return
}
level.TailRecords, err = bin.ReadN(r, recordsCount*indexRecordSize)
if err != nil {
return
}
s.ChangedIndexLevels = append(s.ChangedIndexLevels, level)
}
return
}
type MeasuresAppendRecord struct {
MetricID uint32
TimestampsOffset int
Timestamps []byte
ValuesOffset int
Values []byte
}
func (s MeasuresAppendRecord) WriteTo(w *bytes.Buffer) {
w.WriteByte(CodeMeasuresAppendWithGrow)
bin.WriteUint32(w, s.MetricID)
bin.WriteVarSize(w, s.TimestampsOffset)
bin.WriteVarSize(w, len(s.Timestamps))
w.Write(s.Timestamps)
bin.WriteVarSize(w, s.ValuesOffset)
bin.WriteVarSize(w, len(s.Values))
w.Write(s.Values)
}
func (s *MeasuresAppendRecord) Parse(r *bytes.Buffer) (err error) {
var size int
s.MetricID, err = bin.ReadUint32(r)
if err != nil {
return
}
s.TimestampsOffset, err = bin.ReadVarSize(r)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.Timestamps, err = bin.ReadN(r, size)
if err != nil {
return
}
s.ValuesOffset, err = bin.ReadVarSize(r)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.Values, err = bin.ReadN(r, size)
if err != nil {
return
}
return
}
type MetricAddRecord struct {
MetricID uint32
MetricType qb.MetricType
FracDigits int
}
func (s MetricAddRecord) Pack(w io.Writer) {
arr := []byte{
CodeMetricAdd,
0, 0, 0, 0, //
byte(s.MetricType),
byte(s.FracDigits),
}
bin.PutUint32(arr[1:], s.MetricID)
w.Write(arr)
}
func (s *MetricAddRecord) Parse(src io.Reader) (err error) {
arr, err := bin.ReadN(src, 6)
if err != nil {
return
}
s.MetricID, _ = bin.GetUint32(arr)
s.MetricType = qb.MetricType(arr[4])
s.FracDigits = int(arr[5])
return nil
}
type MetricDeleteRecord struct {
MetricID uint32
FreeIndexPages []uint32
FreeDataPages []uint32
}
func (s MetricDeleteRecord) Pack(w io.Writer) {
arr := []byte{
CodeMetricDelete,
0, 0, 0, 0, // metricID
}
bin.PutUint32(arr[1:], s.MetricID)
w.Write(arr)
// fix
// bin.WriteVarSize(w, len(s.FreePageNumbers))
// for _, pageNo := range s.FreePageNumbers {
// bin.WriteUint32(w, pageNo)
// }
}
func (s *MetricDeleteRecord) Parse(src io.Reader) (err error) {
s.MetricID, err = bin.ReadUint32(src)
if err != nil {
return
}
// free pages
// fix
// qty, err := bin.ReadVarSize(src)
// if err != nil {
// return
// }
// for range qty {
// var pageNo uint32
// pageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// //s.FreePageNumbers = append(s.FreePageNumbers, pageNo)
// }
return nil
}
// fix - add changed pages
type DeletedMeasures struct {
MetricID uint32
FreePageNumbers []uint32
}
func (s DeletedMeasures) Pack(w io.Writer) {
arr := []byte{
CodeMeasuresDelete,
0, 0, 0, 0,
}
bin.PutUint32(arr[1:], s.MetricID)
w.Write(arr)
bin.WriteVarSize(w, len(s.FreePageNumbers))
for _, pageNo := range s.FreePageNumbers {
bin.WriteUint32(w, pageNo)
}
}
func (s *DeletedMeasures) Parse(src io.Reader) (err error) {
s.MetricID, err = bin.ReadUint32(src)
if err != nil {
return
}
// free pages
qty, err := bin.ReadVarSize(src)
if err != nil {
return
}
for range qty {
var pageNo uint32
pageNo, err = bin.ReadUint32(src)
if err != nil {
return
}
s.FreePageNumbers = append(s.FreePageNumbers, pageNo)
}
return nil
}
// type AppendedMeasure struct {
// MetricID uint32
// Timestamp uint32
// Value float64
// }
// func (s AppendedMeasure) Pack(w io.Writer) {
// arr := []byte{
// CodeAppendedMeasure,
// 0, 0, 0, 0, // metricID
// 0, 0, 0, 0, // timestamp
// 0, 0, 0, 0, 0, 0, 0, 0, // value
// }
// bin.PutUint32(arr[1:], s.MetricID)
// bin.PutUint32(arr[5:], s.Timestamp)
// bin.PutFloat64(arr[9:], s.Value)
// w.Write(arr)
// }
// func (s *AppendedMeasure) Parse(src io.Reader) (err error) {
// arr, err := bin.ReadN(src, 16)
// if err != nil {
// return
// }
// s.MetricID, _ = bin.GetUint32(arr[0:])
// s.Timestamp, _ = bin.GetUint32(arr[4:])
// s.Value, _ = bin.GetFloat64(arr[8:])
// return nil
// }
// type AppendedMeasures struct {
// MetricID uint32
// Measures []proto.Measure
// }
// func (s AppendedMeasures) Pack(w io.Writer) {
// arr := []byte{
// CodeAppendedMeasures,
// 0, 0, 0, 0, // metricID
// 0, 0, // qty
// }
// bin.PutUint32(arr[1:], s.MetricID)
// bin.PutUint16(arr[5:], uint16(len(s.Measures)))
// w.Write(arr)
// for _, measure := range s.Measures {
// bin.WriteUint32(w, measure.Timestamp)
// bin.WriteFloat64(w, measure.Value)
// }
// }
// func (s *AppendedMeasures) Parse(src io.Reader) (err error) {
// s.MetricID, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// qty, err := bin.ReadUint16(src)
// if err != nil {
// return
// }
// for range qty {
// var measure proto.Measure
// measure.Timestamp, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// measure.Value, err = bin.ReadFloat64(src)
// if err != nil {
// return
// }
// s.Measures = append(s.Measures, measure)
// }
// return nil
// }
// type AppendedPagesReq struct {
// Legs []atree.PathLeg
// MetricID uint32
// Timestamp uint32 // last measure
// Value float64 // last measure
// //RootPageNo uint32
// LastPageNo uint32
// Pages []atree.NotLinkedDataPage
// TimestampsChunks [][]byte
// TimestampsSize uint16
// ValuesChunks [][]byte
// ValuesSize uint16
// }
/*
4b metricID
4b timestamp
8b value
4b LastPageNo (0 if no new root page)
4b newRootPageNo (0 if no new root page)
Nb - varsize (pages qty)
[
4b pageNo
1b isReused
Xb page (page size)
]
2b - timestamps size (not filled page)
Nb - timestamps payload
2b - values size (not filled page)
Nb - values payload
*/
// type AppendedPages struct {
// MetricID uint32
// Timestamp uint32 // last measure
// Value float64 // last measure
// NewRootPageNo uint32
// LastPageNo uint32
// Pages []atree.PageToWrite
// TimestampsChunks [][]byte
// TimestampsSize int
// ValuesChunks [][]byte
// ValuesSize int
// }
// func (s *AppendedPages) Pack(w io.Writer) {
// w.Write([]byte{
// CodeAppendedPages,
// })
// bin.WriteUint32(w, s.MetricID)
// bin.WriteUint32(w, s.Timestamp)
// bin.WriteFloat64(w, s.Value)
// bin.WriteUint32(w, s.LastPageNo)
// bin.WriteUint32(w, s.NewRootPageNo)
// //
// bin.WriteVarSize(w, len(s.Pages))
// for _, p := range s.Pages {
// bin.WriteUint32(w, p.PageNo)
// if p.IsReused {
// w.Write([]byte{1})
// } else {
// w.Write([]byte{0})
// }
// w.Write(p.Data)
// }
// // timestamps
// writeChunkedPayloadTo(s.TimestampsChunks, int(s.TimestampsSize), w)
// // values
// writeChunkedPayloadTo(s.ValuesChunks, int(s.ValuesSize), w)
// }
// func (s *AppendedPages) Parse(src io.Reader) (err error) {
// s.MetricID, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// s.Timestamp, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// s.Value, err = bin.ReadFloat64(src)
// if err != nil {
// return
// }
// s.LastPageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// s.NewRootPageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// pagesQty, err := bin.ReadVarSize(src)
// if err != nil {
// return
// }
// for range pagesQty {
// var page atree.PageToWrite
// page.PageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// page.IsReused, err = bin.ReadBool(src)
// if err != nil {
// return
// }
// page.Data, err = bin.ReadN(src, atree.PageSize)
// if err != nil {
// return
// }
// s.Pages = append(s.Pages, page)
// }
// s.TimestampsChunks, s.TimestampsSize, err = readChunkedPayload(src)
// if err != nil {
// return
// }
// s.ValuesChunks, s.ValuesSize, err = readChunkedPayload(src)
// if err != nil {
// return
// }
// return nil
// }

13
txlog/writer.go → storage/wal_writer.go
View File

@@ -1,4 +1,4 @@
package txlog package storage
import ( import (
"bytes" "bytes"
@@ -31,12 +31,11 @@ const (
) )
const ( const (
CodeAddedMetric byte = 1 CodeMetricAdd byte = 1
CodeDeletedMetric byte = 2 CodeMetricDelete byte = 2
CodeAppendedMeasure byte = 4 CodeMeasuresAppendWithGrow byte = 5
CodeAppendedMeasures byte = 5 CodeMeasuresAppend = 6
CodeAppendedPages byte = 6 CodeMeasuresDelete byte = 7
CodeDeletedMeasures byte = 7
) )
type FreeList interface { type FreeList interface {

48
transform/aggregate.go
View File

@@ -6,8 +6,8 @@ import (
"io" "io"
"time" "time"
diploma "gordenko.dev/dima/qb" bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb/bin" "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/timeutil" "gordenko.dev/dima/qb/timeutil"
) )
@@ -15,7 +15,7 @@ import (
type InstantPeriodsWriterOptions struct { type InstantPeriodsWriterOptions struct {
Dst io.Writer Dst io.Writer
GroupBy diploma.GroupBy GroupBy qb.GroupBy
AggregateFuncs byte AggregateFuncs byte
FirstHourOfDay int FirstHourOfDay int
} }
@@ -24,7 +24,7 @@ type InstantPeriodsWriter struct {
aggregateFuncs byte aggregateFuncs byte
arr []byte arr []byte
responder *ChunkedResponder responder *ChunkedResponder
groupBy diploma.GroupBy groupBy qb.GroupBy
firstHourOfDay int firstHourOfDay int
time2period func(uint32) time.Time time2period func(uint32) time.Time
currentPeriod time.Time currentPeriod time.Time
@@ -45,13 +45,13 @@ func NewInstantPeriodsWriter(opt InstantPeriodsWriterOptions) (*InstantPeriodsWr
} }
// Считаю q, чтобы заранее выделить массив для упаковки периодов // Считаю q, чтобы заранее выделить массив для упаковки периодов
var q int var q int
if (opt.AggregateFuncs & diploma.AggregateMin) == diploma.AggregateMin { if (opt.AggregateFuncs & qb.AggregateMin) == qb.AggregateMin {
q++ q++
} }
if (opt.AggregateFuncs & diploma.AggregateMax) == diploma.AggregateMax { if (opt.AggregateFuncs & qb.AggregateMax) == qb.AggregateMax {
q++ q++
} }
if (opt.AggregateFuncs & diploma.AggregateAvg) == diploma.AggregateAvg { if (opt.AggregateFuncs & qb.AggregateAvg) == qb.AggregateAvg {
q++ q++
} }
@@ -70,17 +70,17 @@ func NewInstantPeriodsWriter(opt InstantPeriodsWriterOptions) (*InstantPeriodsWr
} }
switch opt.GroupBy { switch opt.GroupBy {
case diploma.GroupByHour: case qb.GroupByHour:
s.time2period = groupByHour s.time2period = groupByHour
case diploma.GroupByDay: case qb.GroupByDay:
if s.firstHourOfDay > 0 { if s.firstHourOfDay > 0 {
s.time2period = s.groupByDayUsingFHD s.time2period = s.groupByDayUsingFHD
} else { } else {
s.time2period = groupByDay s.time2period = groupByDay
} }
case diploma.GroupByMonth: case qb.GroupByMonth:
if s.firstHourOfDay > 0 { if s.firstHourOfDay > 0 {
s.time2period = s.groupByMonthUsingFHD s.time2period = s.groupByMonthUsingFHD
} else { } else {
@@ -174,12 +174,12 @@ func (s *InstantPeriodsWriter) feed(timestamp uint32, value float64, isBuffer bo
func (s *InstantPeriodsWriter) decrementPeriod() { func (s *InstantPeriodsWriter) decrementPeriod() {
switch s.groupBy { switch s.groupBy {
case diploma.GroupByHour: case qb.GroupByHour:
s.currentPeriod = s.currentPeriod.Add(-1 * time.Hour) s.currentPeriod = s.currentPeriod.Add(-1 * time.Hour)
//fmt.Println("decrement") //fmt.Println("decrement")
case diploma.GroupByDay: case qb.GroupByDay:
s.currentPeriod = s.currentPeriod.AddDate(0, 0, -1) s.currentPeriod = s.currentPeriod.AddDate(0, 0, -1)
case diploma.GroupByMonth: case qb.GroupByMonth:
s.currentPeriod = s.currentPeriod.AddDate(0, -1, 0) s.currentPeriod = s.currentPeriod.AddDate(0, -1, 0)
} }
} }
@@ -210,15 +210,15 @@ func (s *InstantPeriodsWriter) packPeriod(timestamp uint32) {
bin.PutUint32(s.arr[8:], s.endTimestamp) bin.PutUint32(s.arr[8:], s.endTimestamp)
pos := 12 pos := 12
if (s.aggregateFuncs & diploma.AggregateMin) == diploma.AggregateMin { if (s.aggregateFuncs & qb.AggregateMin) == qb.AggregateMin {
bin.PutFloat64(s.arr[pos:], s.min) bin.PutFloat64(s.arr[pos:], s.min)
pos += 8 pos += 8
} }
if (s.aggregateFuncs & diploma.AggregateMax) == diploma.AggregateMax { if (s.aggregateFuncs & qb.AggregateMax) == qb.AggregateMax {
bin.PutFloat64(s.arr[pos:], s.max) bin.PutFloat64(s.arr[pos:], s.max)
pos += 8 pos += 8
} }
if (s.aggregateFuncs & diploma.AggregateAvg) == diploma.AggregateAvg { if (s.aggregateFuncs & qb.AggregateAvg) == qb.AggregateAvg {
bin.PutFloat64(s.arr[pos:], s.total/float64(s.entries)) bin.PutFloat64(s.arr[pos:], s.total/float64(s.entries))
} }
} }
@@ -236,7 +236,7 @@ type CumulativePeriodsWriter struct {
responder *ChunkedResponder responder *ChunkedResponder
firstHourOfDay int firstHourOfDay int
currentPeriod time.Time currentPeriod time.Time
groupBy diploma.GroupBy groupBy qb.GroupBy
time2period func(uint32) time.Time time2period func(uint32) time.Time
endTimestamp uint32 endTimestamp uint32
endValue float64 endValue float64
@@ -246,7 +246,7 @@ type CumulativePeriodsWriter struct {
type CumulativePeriodsWriterOptions struct { type CumulativePeriodsWriterOptions struct {
Dst io.Writer Dst io.Writer
GroupBy diploma.GroupBy GroupBy qb.GroupBy
FirstHourOfDay int FirstHourOfDay int
} }
@@ -271,17 +271,17 @@ func NewCumulativePeriodsWriter(opt CumulativePeriodsWriterOptions) (*Cumulative
} }
switch opt.GroupBy { switch opt.GroupBy {
case diploma.GroupByHour: case qb.GroupByHour:
s.time2period = groupByHour s.time2period = groupByHour
case diploma.GroupByDay: case qb.GroupByDay:
if s.firstHourOfDay > 0 { if s.firstHourOfDay > 0 {
s.time2period = s.groupByDayUsingFHD s.time2period = s.groupByDayUsingFHD
} else { } else {
s.time2period = groupByDay s.time2period = groupByDay
} }
case diploma.GroupByMonth: case qb.GroupByMonth:
if s.firstHourOfDay > 0 { if s.firstHourOfDay > 0 {
s.time2period = s.groupByMonthUsingFHD s.time2period = s.groupByMonthUsingFHD
} else { } else {
@@ -360,12 +360,12 @@ func (s *CumulativePeriodsWriter) feed(timestamp uint32, value float64, isBuffer
func (s *CumulativePeriodsWriter) decrementPeriod() { func (s *CumulativePeriodsWriter) decrementPeriod() {
switch s.groupBy { switch s.groupBy {
case diploma.GroupByHour: case qb.GroupByHour:
s.currentPeriod = s.currentPeriod.Add(-1 * time.Hour) s.currentPeriod = s.currentPeriod.Add(-1 * time.Hour)
//fmt.Println("decrement") //fmt.Println("decrement")
case diploma.GroupByDay: case qb.GroupByDay:
s.currentPeriod = s.currentPeriod.AddDate(0, 0, -1) s.currentPeriod = s.currentPeriod.AddDate(0, 0, -1)
case diploma.GroupByMonth: case qb.GroupByMonth:
s.currentPeriod = s.currentPeriod.AddDate(0, -1, 0) s.currentPeriod = s.currentPeriod.AddDate(0, -1, 0)
} }
} }

2
transform/raw.go
View File

@@ -3,7 +3,7 @@ package transform
import ( import (
"io" "io"
"gordenko.dev/dima/qb/bin" bin "gordenko.dev/dima/bin/little"
) )
// CURRENT VALUE WRITER // CURRENT VALUE WRITER

2
transform/responder.go
View File

@@ -5,7 +5,7 @@ import (
"fmt" "fmt"
"io" "io"
"gordenko.dev/dima/qb/bin" bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb/proto" "gordenko.dev/dima/qb/proto"
) )

322
txlog/txlog.go
View File

@@ -1,322 +0,0 @@
package txlog
import (
"io"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
)
var (
indexRecordSize = 8
)
type AddedMetric struct {
MetricID uint32
MetricType qb.MetricType
FracDigits int
}
func (s AddedMetric) Pack(w io.Writer) {
arr := []byte{
CodeAddedMetric,
0, 0, 0, 0, //
byte(s.MetricType),
byte(s.FracDigits),
}
bin.PutUint32(arr[1:], s.MetricID)
w.Write(arr)
}
func (s *AddedMetric) Parse(src io.Reader) (err error) {
arr, err := bin.ReadN(src, 6)
if err != nil {
return
}
s.MetricID, _ = bin.GetUint32(arr)
s.MetricType = qb.MetricType(arr[4])
s.FracDigits = int(arr[5])
return nil
}
type DeletedMetric struct {
MetricID uint32
FreeIndexPages []uint32
FreeDataPages []uint32
}
func (s DeletedMetric) Pack(w io.Writer) {
arr := []byte{
CodeDeletedMetric,
0, 0, 0, 0, // metricID
}
bin.PutUint32(arr[1:], s.MetricID)
w.Write(arr)
// fix
// bin.WriteVarSize(w, len(s.FreePageNumbers))
// for _, pageNo := range s.FreePageNumbers {
// bin.WriteUint32(w, pageNo)
// }
}
func (s *DeletedMetric) Parse(src io.Reader) (err error) {
s.MetricID, err = bin.ReadUint32(src)
if err != nil {
return
}
// free pages
// fix
// qty, err := bin.ReadVarSize(src)
// if err != nil {
// return
// }
// for range qty {
// var pageNo uint32
// pageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// //s.FreePageNumbers = append(s.FreePageNumbers, pageNo)
// }
return nil
}
// fix - add changed pages
type DeletedMeasures struct {
MetricID uint32
FreePageNumbers []uint32
}
func (s DeletedMeasures) Pack(w io.Writer) {
arr := []byte{
CodeDeletedMeasures,
0, 0, 0, 0,
}
bin.PutUint32(arr[1:], s.MetricID)
w.Write(arr)
bin.WriteVarSize(w, len(s.FreePageNumbers))
for _, pageNo := range s.FreePageNumbers {
bin.WriteUint32(w, pageNo)
}
}
func (s *DeletedMeasures) Parse(src io.Reader) (err error) {
s.MetricID, err = bin.ReadUint32(src)
if err != nil {
return
}
// free pages
qty, err := bin.ReadVarSize(src)
if err != nil {
return
}
for range qty {
var pageNo uint32
pageNo, err = bin.ReadUint32(src)
if err != nil {
return
}
s.FreePageNumbers = append(s.FreePageNumbers, pageNo)
}
return nil
}
// type AppendedMeasure struct {
// MetricID uint32
// Timestamp uint32
// Value float64
// }
// func (s AppendedMeasure) Pack(w io.Writer) {
// arr := []byte{
// CodeAppendedMeasure,
// 0, 0, 0, 0, // metricID
// 0, 0, 0, 0, // timestamp
// 0, 0, 0, 0, 0, 0, 0, 0, // value
// }
// bin.PutUint32(arr[1:], s.MetricID)
// bin.PutUint32(arr[5:], s.Timestamp)
// bin.PutFloat64(arr[9:], s.Value)
// w.Write(arr)
// }
// func (s *AppendedMeasure) Parse(src io.Reader) (err error) {
// arr, err := bin.ReadN(src, 16)
// if err != nil {
// return
// }
// s.MetricID, _ = bin.GetUint32(arr[0:])
// s.Timestamp, _ = bin.GetUint32(arr[4:])
// s.Value, _ = bin.GetFloat64(arr[8:])
// return nil
// }
// type AppendedMeasures struct {
// MetricID uint32
// Measures []proto.Measure
// }
// func (s AppendedMeasures) Pack(w io.Writer) {
// arr := []byte{
// CodeAppendedMeasures,
// 0, 0, 0, 0, // metricID
// 0, 0, // qty
// }
// bin.PutUint32(arr[1:], s.MetricID)
// bin.PutUint16(arr[5:], uint16(len(s.Measures)))
// w.Write(arr)
// for _, measure := range s.Measures {
// bin.WriteUint32(w, measure.Timestamp)
// bin.WriteFloat64(w, measure.Value)
// }
// }
// func (s *AppendedMeasures) Parse(src io.Reader) (err error) {
// s.MetricID, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// qty, err := bin.ReadUint16(src)
// if err != nil {
// return
// }
// for range qty {
// var measure proto.Measure
// measure.Timestamp, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// measure.Value, err = bin.ReadFloat64(src)
// if err != nil {
// return
// }
// s.Measures = append(s.Measures, measure)
// }
// return nil
// }
// type AppendedPagesReq struct {
// Legs []atree.PathLeg
// MetricID uint32
// Timestamp uint32 // last measure
// Value float64 // last measure
// //RootPageNo uint32
// LastPageNo uint32
// Pages []atree.NotLinkedDataPage
// TimestampsChunks [][]byte
// TimestampsSize uint16
// ValuesChunks [][]byte
// ValuesSize uint16
// }
/*
4b metricID
4b timestamp
8b value
4b LastPageNo (0 if no new root page)
4b newRootPageNo (0 if no new root page)
Nb - varsize (pages qty)
[
4b pageNo
1b isReused
Xb page (page size)
]
2b - timestamps size (not filled page)
Nb - timestamps payload
2b - values size (not filled page)
Nb - values payload
*/
// type AppendedPages struct {
// MetricID uint32
// Timestamp uint32 // last measure
// Value float64 // last measure
// NewRootPageNo uint32
// LastPageNo uint32
// Pages []atree.PageToWrite
// TimestampsChunks [][]byte
// TimestampsSize int
// ValuesChunks [][]byte
// ValuesSize int
// }
// func (s *AppendedPages) Pack(w io.Writer) {
// w.Write([]byte{
// CodeAppendedPages,
// })
// bin.WriteUint32(w, s.MetricID)
// bin.WriteUint32(w, s.Timestamp)
// bin.WriteFloat64(w, s.Value)
// bin.WriteUint32(w, s.LastPageNo)
// bin.WriteUint32(w, s.NewRootPageNo)
// //
// bin.WriteVarSize(w, len(s.Pages))
// for _, p := range s.Pages {
// bin.WriteUint32(w, p.PageNo)
// if p.IsReused {
// w.Write([]byte{1})
// } else {
// w.Write([]byte{0})
// }
// w.Write(p.Data)
// }
// // timestamps
// writeChunkedPayloadTo(s.TimestampsChunks, int(s.TimestampsSize), w)
// // values
// writeChunkedPayloadTo(s.ValuesChunks, int(s.ValuesSize), w)
// }
// func (s *AppendedPages) Parse(src io.Reader) (err error) {
// s.MetricID, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// s.Timestamp, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// s.Value, err = bin.ReadFloat64(src)
// if err != nil {
// return
// }
// s.LastPageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// s.NewRootPageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// pagesQty, err := bin.ReadVarSize(src)
// if err != nil {
// return
// }
// for range pagesQty {
// var page atree.PageToWrite
// page.PageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// page.IsReused, err = bin.ReadBool(src)
// if err != nil {
// return
// }
// page.Data, err = bin.ReadN(src, atree.PageSize)
// if err != nil {
// return
// }
// s.Pages = append(s.Pages, page)
// }
// s.TimestampsChunks, s.TimestampsSize, err = readChunkedPayload(src)
// if err != nil {
// return
// }
// s.ValuesChunks, s.ValuesSize, err = readChunkedPayload(src)
// if err != nil {
// return
// }
// return nil
// }
// PACK

305
txlog/wal_records.go
View File

@@ -1,305 +0,0 @@
package txlog
import (
"bytes"
bin "gordenko.dev/dima/bin/little"
)
type CompletedDataPage struct {
PageNo uint32
Reused bool
PrevPageNo uint32
Checksum uint32
TimestampsOffset int // offset on 1st page
Timestamps []byte
ValuesOffset int // offset on 1st page
Values []byte
}
type CompletedIndexPage struct {
PageNo uint32
Reused bool
Checksum uint32
Records []byte // offset не потрібен, оскільки Records додаються в кінець
}
type ChangedIndexLevel struct {
CompletedIndexPage *CompletedIndexPage
IndexPages []SealedIndexPage
TailRecords []byte // payload only
}
type WALRecordAppendMeasures struct {
MetricID uint32
CompletedDataPage CompletedDataPage
DataPages []SealedDataPage
ChangedIndexLevels []ChangedIndexLevel
TailTimestamps []byte // data level tail
TailValues []byte // data level tail
}
/*
Format appended measures:
1b - tx type
4b - metricID
4b - pageNo
1b - reused
4b - prevPageNo
4b - page crc32
Nb - (varsize) timestamps offset
Nb - (varsize) timestamps tail size on 1st filled data page
Nb - timestamps tail on 1st filled data page
Nb - (varsize) values offset
Nb - (varsize) values tail size on 1st filled data page
Nb - values tail on 1st filled data page
Nb - (varsize) filled data pages qty
[
4b - pageNo
1b - reused
Nb - (data page size) page content
]
Nb - (varsize) timestamps payload size on tail
Nb - timestamps payload
Nb - (varsize) values payload size on tail
Nb - values payload
Nb - (varsize) qty of index levels
[
// NOTE!
// - idx = 0 - zeroLevel
// - skipped = maxRecordsOnIndexPage - records count
4b - pageNo of 1st index page
1b - reused
4b - page crc32
Nb - (varsize) records tail count on 1st filled index page
Nb - records tail on 1st filled index page
Nb - (varsize) qty of level filled pages
[
4b - pageNo
1b - reused
Nb - (index page size) page content
]
Nb - (varsize) size of records level tail
Nb - level tail records
]
*/
func (s WALRecordAppendMeasures) WriteTo(w *bytes.Buffer) {
w.WriteByte(CodeAppendedMeasures)
bin.WriteUint32(w, s.MetricID)
// completed data page
completed := s.CompletedDataPage
bin.WriteUint32(w, completed.PageNo)
bin.WriteBool(w, completed.Reused)
bin.WriteUint32(w, completed.PrevPageNo)
bin.WriteUint32(w, completed.Checksum)
bin.WriteVarSize(w, completed.TimestampsOffset)
bin.WriteVarSize(w, len(completed.Timestamps))
w.Write(completed.Timestamps)
bin.WriteVarSize(w, completed.ValuesOffset)
bin.WriteVarSize(w, len(completed.Values))
w.Write(completed.Values)
// data pages
bin.WriteVarSize(w, len(s.DataPages))
for _, p := range s.DataPages {
bin.WriteUint32(w, p.PageNo)
bin.WriteBool(w, p.Reused)
bin.WriteVarSize(w, len(p.Content))
w.Write(p.Content)
}
// data tail
bin.WriteVarSize(w, len(s.TailTimestamps))
w.Write(s.TailTimestamps)
bin.WriteVarSize(w, len(s.TailValues))
w.Write(s.TailValues)
// changed levels
bin.WriteVarSize(w, len(s.ChangedIndexLevels))
for _, level := range s.ChangedIndexLevels {
// completed index page
completed := level.CompletedIndexPage
bin.WriteUint32(w, completed.PageNo)
bin.WriteBool(w, completed.Reused)
bin.WriteUint32(w, completed.Checksum)
bin.WriteVarSize(w, len(completed.Records)/indexRecordSize)
w.Write(completed.Records)
// full index pages
for _, p := range level.IndexPages {
bin.WriteUint32(w, p.PageNo)
bin.WriteBool(w, p.Reused)
bin.WriteVarSize(w, len(p.Content))
w.Write(p.Content)
}
// level tail
bin.WriteVarSize(w, len(level.TailRecords)/indexRecordSize)
w.Write(level.TailRecords)
}
}
func (s *WALRecordAppendMeasures) Parse(r *bytes.Buffer) (err error) {
var (
size int
recordsCount int
)
s.MetricID, err = bin.ReadUint32(r)
if err != nil {
return
}
s.CompletedDataPage.PageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
s.CompletedDataPage.Reused, err = bin.ReadBool(r)
if err != nil {
return
}
s.CompletedDataPage.PrevPageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
s.CompletedDataPage.Checksum, err = bin.ReadUint32(r)
if err != nil {
return
}
s.CompletedDataPage.TimestampsOffset, err = bin.ReadVarSize(r)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.CompletedDataPage.Timestamps, err = bin.ReadN(r, size)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.CompletedDataPage.Values, err = bin.ReadN(r, size)
if err != nil {
return
}
// data pages
dataPagesQty, err := bin.ReadVarSize(r)
if err != nil {
return
}
for range dataPagesQty {
var (
p SealedDataPage
)
p.PageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
p.Reused, err = bin.ReadBool(r)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
p.Content, err = bin.ReadN(r, size)
if err != nil {
return
}
s.DataPages = append(s.DataPages, p)
}
// data tail
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.TailTimestamps, err = bin.ReadN(r, size)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.TailValues, err = bin.ReadN(r, size)
if err != nil {
return
}
// changed levels
changedIndexLevelsCount, err := bin.ReadVarSize(r)
if err != nil {
return
}
for range changedIndexLevelsCount {
var (
indexPagesCount int
level ChangedIndexLevel
)
// completed index page
level.CompletedIndexPage.PageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
level.CompletedIndexPage.Reused, err = bin.ReadBool(r)
if err != nil {
return
}
level.CompletedIndexPage.Checksum, err = bin.ReadUint32(r)
if err != nil {
return
}
recordsCount, err = bin.ReadVarSize(r)
if err != nil {
return
}
level.CompletedIndexPage.Records, err = bin.ReadN(r, recordsCount*indexRecordSize)
if err != nil {
return
}
//
indexPagesCount, err = bin.ReadVarSize(r)
if err != nil {
return
}
for range indexPagesCount {
var p SealedIndexPage
p.PageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
p.Reused, err = bin.ReadBool(r)
if err != nil {
return
}
p.Checksum, err = bin.ReadUint32(r)
if err != nil {
return
}
size, err = bin.ReadVarSize(r)
if err != nil {
return
}
p.Content, err = bin.ReadN(r, size)
if err != nil {
return
}
level.IndexPages = append(level.IndexPages, p)
}
recordsCount, err = bin.ReadVarSize(r)
if err != nil {
return
}
level.TailRecords, err = bin.ReadN(r, recordsCount*indexRecordSize)
if err != nil {
return
}
s.ChangedIndexLevels = append(s.ChangedIndexLevels, level)
}
return
}