From 9ec86282ad184c7b9af0a3b9381e2c8f06bc3e88 Mon Sep 17 00:00:00 2001 From: Dima Gordenko <1.e4.kc6@gmail.com> Date: Wed, 10 Jun 2026 06:18:45 +0300 Subject: [PATCH] wp --- atree/atree.go | 330 ++++++++++----------- atree/cursor.go | 167 ++++++----- atree/io.go | 167 +++++------ atree/misc.go | 230 ++++++--------- atree/select.go | 8 +- bin/bin.go | 621 --------------------------------------- client/client.go | 521 +++++++++++++-------------------- conbuf/conbuf.go | 460 ----------------------------- database/api.go | 50 ++-- database/database.go | 62 ++-- database/database_test.go | 24 ++ database/helpers.go | 8 +- database/metric.go | 39 +-- database/proc.go | 46 ++- database/replay_metric.go | 304 ++++++++++++++----- database/snapshot.go | 6 +- database/wal_replayer.go | 185 +++++------- enc/cumdelta.go | 20 +- enc/enc_test.go | 18 +- enc/insdelta.go | 11 +- enc/time_delta.go | 15 +- examples/database/main.go | 2 +- examples/loadtest/loadtest.go | 38 +-- examples/loadtest/main.go | 2 +- examples/requests/main.go | 2 +- examples/requests/requests.go | 24 +- freelist/freelist.go | 42 +-- freelist/freelist_test.go | 64 ++-- proto/proto.go | 84 +++--- qb.go | 4 +- storage/data_preparer.go | 207 +++++++++++++ storage/helpers.go | 202 +++++++++++++ storage/page_preparer.go | 314 ++++++++++++++++++++ storage/storage.go | 32 ++ storage/wal_reader.go | 362 +++++++++++++++++++++++ storage/wal_records.go | 666 ++++++++++++++++++++++++++++++++++++++++++ storage/wal_writer.go | 464 +++++++++++++++++++++++++++++ transform/aggregate.go | 48 +-- transform/raw.go | 2 +- transform/responder.go | 2 +- txlog/data_preparer.go | 207 ------------- txlog/helpers.go | 202 ------------- txlog/page_preparer.go | 279 ------------------ txlog/reader.go | 362 ----------------------- txlog/txlog.go | 322 -------------------- txlog/wal_records.go | 305 ------------------- txlog/writer.go | 465 ----------------------------- 47 files changed, 3493 insertions(+), 4502 deletions(-) delete mode 100644 bin/bin.go delete mode 100644 conbuf/conbuf.go create mode 100644 storage/data_preparer.go create mode 100644 storage/helpers.go create mode 100644 storage/page_preparer.go create mode 100644 storage/storage.go create mode 100644 storage/wal_reader.go create mode 100644 storage/wal_records.go create mode 100644 storage/wal_writer.go delete mode 100644 txlog/data_preparer.go delete mode 100644 txlog/helpers.go delete mode 100644 txlog/page_preparer.go delete mode 100644 txlog/reader.go delete mode 100644 txlog/txlog.go delete mode 100644 txlog/wal_records.go delete mode 100644 txlog/writer.go diff --git a/atree/atree.go b/atree/atree.go index 369cdf5..ee2dd0f 100644 --- a/atree/atree.go +++ b/atree/atree.go @@ -1,49 +1,12 @@ package atree import ( - "errors" - "fmt" "os" - "path/filepath" "sync" - - "gordenko.dev/dima/qb/bin" ) const ( - PageTypeData = 1 - PageTypeIndex = 2 - filePerm = 0770 - - // common - crc32Idx = PageSize - 4 - pageTypeIdx = PageSize - 5 - - // index page - indexCRC32Idx = IndexPageSize - 4 - isLastLevelIdx = IndexPageSize - 5 - indexRecordsQtyIdx = IndexPageSize - 7 - - indexRecordSize = 8 - - // data page - dataCRC32Idx = DataPageSize - 4 - prevPageIdx = DataPageSize - 8 - valuesSizeIdx = DataPageSize - 10 - timestampsSizeIdx = DataPageSize - 12 - - timestampSize = 4 - pairSize = timestampSize + PageNoSize - indexFooterIdx = indexRecordsQtyIdx - dataFooterIdx = timestampsSizeIdx - - PageSize = 8192 - DataPageSize = 8192 - IndexPageSize = 1024 - PageNoSize = 4 - // - DataPagePayloadSize int = dataFooterIdx ) type _page struct { @@ -70,48 +33,49 @@ type Options struct { } func New(opt Options) (*Atree, error) { - if opt.Dir == "" { - return nil, errors.New("Dir option is required") - } - if opt.DatabaseName == "" { - return nil, errors.New("DatabaseName option is required") - } - // открываю или создаю dbName.data и dbName.index файлы - var ( - fileName = filepath.Join(opt.Dir, opt.DatabaseName+".db") - file *os.File - allocatedPagesQty uint32 - ) + // if opt.Dir == "" { + // return nil, errors.New("Dir option is required") + // } + // if opt.DatabaseName == "" { + // return nil, errors.New("DatabaseName option is required") + // } + // // открываю или создаю dbName.data и dbName.index файлы + // var ( + // fileName = filepath.Join(opt.Dir, opt.DatabaseName+".db") + // file *os.File + // allocatedPagesQty uint32 + // ) - // При создании data файла сразу создается индекс, поэтому корректное - // состояние БД: либо оба файла есть, либо ни одного файла нет. - isDataExist, err := isFileExist(fileName) - if err != nil { - return nil, fmt.Errorf("check data file is exist: %s", err) - } - if isDataExist { - file, allocatedPagesQty, err = openFile(fileName, PageSize) - if err != nil { - return nil, fmt.Errorf("open data file: %s", err) - } - } else { - // нет файла - file, err = os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, filePerm) - if err != nil { - return nil, err - } - } + // // При создании data файла сразу создается индекс, поэтому корректное + // // состояние БД: либо оба файла есть, либо ни одного файла нет. + // isDataExist, err := isFileExist(fileName) + // if err != nil { + // return nil, fmt.Errorf("check data file is exist: %s", err) + // } + // if isDataExist { + // file, allocatedPagesQty, err = openFile(fileName, PageSize) + // if err != nil { + // return nil, fmt.Errorf("open data file: %s", err) + // } + // } else { + // // нет файла + // file, err = os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, filePerm) + // if err != nil { + // return nil, err + // } + // } - tree := &Atree{ - file: file, - allocatedPagesQty: allocatedPagesQty, - pages: make(map[uint32]*_page), - pageWaits: make(map[uint32][]chan readResult), - readSignalCh: make(chan struct{}, 1), - writeSignalCh: make(chan struct{}, 1), - } + // tree := &Atree{ + // file: file, + // allocatedPagesQty: allocatedPagesQty, + // pages: make(map[uint32]*_page), + // pageWaits: make(map[uint32][]chan readResult), + // readSignalCh: make(chan struct{}, 1), + // writeSignalCh: make(chan struct{}, 1), + // } - return tree, nil + // return tree, nil + return nil, nil } func (s *Atree) Run() { @@ -122,27 +86,28 @@ func (s *Atree) Run() { // FIND func (s *Atree) findDataPage(rootPageNo uint32, timestamp uint32) (uint32, []byte, error) { - indexPageNo := rootPageNo - for { - buf, err := s.fetchIndexPage(indexPageNo) - if err != nil { - return 0, nil, fmt.Errorf("fetchIndexPage(%d): %s", indexPageNo, err) - } + // indexPageNo := rootPageNo + // for { + // buf, err := s.fetchIndexPage(indexPageNo) + // if err != nil { + // return 0, nil, fmt.Errorf("fetchIndexPage(%d): %s", indexPageNo, err) + // } - foundPageNo := findPageNo(buf, timestamp) - s.releasePage(indexPageNo) + // foundPageNo := findPageNo(buf, timestamp) + // s.releasePage(indexPageNo) - // fix - if buf[isLastLevelIdx] == 1 { - buf, err := s.fetchDataPage(foundPageNo) - if err != nil { - return 0, nil, fmt.Errorf("fetchDataPage(%d): %s", foundPageNo, err) - } - return foundPageNo, buf, nil - } - // вглубь - indexPageNo = foundPageNo - } + // // fix + // if buf[isLastLevelIdx] == 1 { + // buf, err := s.fetchDataPage(foundPageNo) + // if err != nil { + // return 0, nil, fmt.Errorf("fetchDataPage(%d): %s", foundPageNo, err) + // } + // return foundPageNo, buf, nil + // } + // // вглубь + // indexPageNo = foundPageNo + // } + return 0, nil, nil } type PathLeg struct { @@ -156,37 +121,38 @@ type PathToDataPage struct { } func (s *Atree) FindPathToLastPage(rootPageNo uint32) (_ PathToDataPage, err error) { - var ( - pageNo = rootPageNo - legs []PathLeg - ) + // var ( + // pageNo = rootPageNo + // legs []PathLeg + // ) - for { - var buf []byte - buf, err = s.fetchIndexPage(pageNo) - if err != nil { - err = fmt.Errorf("FetchIndexPage(%d): %s", pageNo, err) - return - } + // for { + // var buf []byte + // buf, err = s.fetchIndexPage(pageNo) + // if err != nil { + // err = fmt.Errorf("FetchIndexPage(%d): %s", pageNo, err) + // return + // } - legs = append(legs, PathLeg{ - PageNo: pageNo, - Data: buf, - // childIdx не нужен - }) + // legs = append(legs, PathLeg{ + // PageNo: pageNo, + // Data: buf, + // // childIdx не нужен + // }) - foundPageNo := getLastPageNo(buf) + // foundPageNo := getLastPageNo(buf) - // fix - if buf[isLastLevelIdx] == 1 { - return PathToDataPage{ - Legs: legs, - LastPageNo: foundPageNo, - }, nil - } - // вглубь - pageNo = foundPageNo - } + // // fix + // if buf[isLastLevelIdx] == 1 { + // return PathToDataPage{ + // Legs: legs, + // LastPageNo: foundPageNo, + // }, nil + // } + // // вглубь + // pageNo = foundPageNo + // } + return } // DELETE @@ -199,72 +165,76 @@ type Level struct { } func (s *Atree) GetAllPages(rootPageNo uint32) (_ []uint32, err error) { - var ( - pageNumbers []uint32 - levels []*Level - ) + // var ( + // pageNumbers []uint32 + // levels []*Level + // ) - buf, err := s.fetchIndexPage(rootPageNo) - if err != nil { - return nil, fmt.Errorf("fetchIndexPage(%d): %s", rootPageNo, err) - } - pageNumbers = append(pageNumbers, rootPageNo) - - // if buf[isDataPageNumbersIdx] == 1 { - // pageNumbers := listPageNumbers(buf) - // dataPages = append(dataPages, pageNumbers...) - - // s.releasePage(rootPageNo) - - // return PageLists{ - // DataPages: dataPages, - // IndexPages: indexPages, - // }, nil + // buf, err := s.fetchIndexPage(rootPageNo) + // if err != nil { + // return nil, fmt.Errorf("fetchIndexPage(%d): %s", rootPageNo, err) // } + // pageNumbers = append(pageNumbers, rootPageNo) - levels = append(levels, &Level{ - PageNo: rootPageNo, - PageData: buf, - Idx: 0, - ChildQty: bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]), - }) + // // if buf[isDataPageNumbersIdx] == 1 { + // // pageNumbers := listPageNumbers(buf) + // // dataPages = append(dataPages, pageNumbers...) - for { - if len(levels) == 0 { - return pageNumbers, nil - } + // // s.releasePage(rootPageNo) - lastIdx := len(levels) - 1 - level := levels[lastIdx] + // // return PageLists{ + // // DataPages: dataPages, + // // IndexPages: indexPages, + // // }, nil + // // } - if level.Idx < level.ChildQty { - pageNo := getPageNo(level.PageData, level.Idx) - level.Idx++ + // childQty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:]) - var buf []byte - buf, err = s.fetchPage(pageNo) - if err != nil { - return nil, fmt.Errorf("fetchPage(%d): %s", pageNo, err) - } - pageNumbers = append(pageNumbers, pageNo) + // levels = append(levels, &Level{ + // PageNo: rootPageNo, + // PageData: buf, + // Idx: 0, + // ChildQty: int(childQty), + // }) - if buf[pageTypeIdx] == PageTypeData { - //pageNumbers := listPageNumbers(buf) - //dataPages = append(dataPages, pageNumbers...) - s.releasePage(pageNo) - } else { - levels = append(levels, &Level{ - PageNo: pageNo, - PageData: buf, - Idx: 0, - ChildQty: bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]), - }) - } - } else { - s.releasePage(level.PageNo) - levels = levels[:lastIdx] - } - } + // for { + // if len(levels) == 0 { + // return pageNumbers, nil + // } + + // lastIdx := len(levels) - 1 + // level := levels[lastIdx] + + // if level.Idx < level.ChildQty { + // pageNo := getPageNo(level.PageData, level.Idx) + // level.Idx++ + + // var buf []byte + // buf, err = s.fetchPage(pageNo) + // if err != nil { + // return nil, fmt.Errorf("fetchPage(%d): %s", pageNo, err) + // } + // pageNumbers = append(pageNumbers, pageNo) + + // if buf[pageTypeIdx] == PageTypeData { + // //pageNumbers := listPageNumbers(buf) + // //dataPages = append(dataPages, pageNumbers...) + // s.releasePage(pageNo) + // } else { + // childQty, _ = bin.GetUint16(buf[indexRecordsQtyIdx:]) + // levels = append(levels, &Level{ + // PageNo: pageNo, + // PageData: buf, + // Idx: 0, + // ChildQty: int(childQty), + // }) + // } + // } else { + // s.releasePage(level.PageNo) + // levels = levels[:lastIdx] + // } + // } + return } // const ( diff --git a/atree/cursor.go b/atree/cursor.go index 083c96b..c85b673 100644 --- a/atree/cursor.go +++ b/atree/cursor.go @@ -5,8 +5,6 @@ import ( "fmt" "gordenko.dev/dima/qb" - "gordenko.dev/dima/qb/bin" - "gordenko.dev/dima/qb/enc" ) type BackwardCursor struct { @@ -66,50 +64,50 @@ func (s *BackwardCursor) Prev() (uint32, float64, bool, error) { var ( timestamp uint32 value float64 - done bool - err error + //done bool + //err error ) - timestamp, done = s.timestampDecompressor.NextValue() - if !done { - value, done = s.valueDecompressor.NextValue() - if done { - return 0, 0, false, - fmt.Errorf("corrupted data page %d: has timestamp, no value", - s.pageNo) - } - return timestamp, value, false, nil - } + // timestamp, done = s.timestampDecompressor.NextValue() + // if !done { + // value, done = s.valueDecompressor.NextValue() + // if done { + // return 0, 0, false, + // fmt.Errorf("corrupted data page %d: has timestamp, no value", + // s.pageNo) + // } + // return timestamp, value, false, nil + // } - prevPageNo := bin.GetUint32(s.pageData[prevPageIdx:]) - if prevPageNo == 0 { - return 0, 0, true, nil - } - s.atree.releasePage(s.pageNo) + // prevPageNo, _ := bin.GetUint32(s.pageData[prevPageIdx:]) + // if prevPageNo == 0 { + // return 0, 0, true, nil + // } + // s.atree.releasePage(s.pageNo) - s.pageNo = prevPageNo - s.pageData, err = s.atree.fetchDataPage(s.pageNo) - if err != nil { - return 0, 0, false, fmt.Errorf("atree.fetchDataPage(%d): %s", s.pageNo, err) - } + // s.pageNo = prevPageNo + // s.pageData, err = s.atree.fetchDataPage(s.pageNo) + // if err != nil { + // return 0, 0, false, fmt.Errorf("atree.fetchDataPage(%d): %s", s.pageNo, err) + // } - err = s.makeDecompressors() - if err != nil { - return 0, 0, false, err - } + // err = s.makeDecompressors() + // if err != nil { + // return 0, 0, false, err + // } - timestamp, done = s.timestampDecompressor.NextValue() - if done { - return 0, 0, false, - fmt.Errorf("corrupted data page %d: no timestamps", - s.pageNo) - } - value, done = s.valueDecompressor.NextValue() - if done { - return 0, 0, false, - fmt.Errorf("corrupted data page %d: no values", - s.pageNo) - } + // timestamp, done = s.timestampDecompressor.NextValue() + // if done { + // return 0, 0, false, + // fmt.Errorf("corrupted data page %d: no timestamps", + // s.pageNo) + // } + // value, done = s.valueDecompressor.NextValue() + // if done { + // return 0, 0, false, + // fmt.Errorf("corrupted data page %d: no values", + // s.pageNo) + // } return timestamp, value, false, nil } @@ -120,68 +118,69 @@ func (s *BackwardCursor) Close() { // HELPER func (s *BackwardCursor) makeDecompressors() error { - timestampsPayloadSize := bin.GetUint16(s.pageData[timestampsSizeIdx:]) - valuesPayloadSize := bin.GetUint16(s.pageData[valuesSizeIdx:]) + // timestampsSize, _ := bin.GetUint16(s.pageData[timestampsSizeIdx:]) + // valuesSize, _ := bin.GetUint16(s.pageData[valuesSizeIdx:]) - payloadSize := timestampsPayloadSize + valuesPayloadSize + // payloadSize := timestampsSize + valuesSize - if payloadSize > dataFooterIdx { - return fmt.Errorf("corrupted data page %d: timestamps + values size %d gt payload size", - s.pageNo, payloadSize) - } + // if payloadSize > dataFooterIdx { + // return fmt.Errorf("corrupted data page %d: timestamps + values size %d gt payload size", + // s.pageNo, payloadSize) + // } - s.timestampDecompressor = enc.NewTimeDeltaDecompressor( - s.pageData[:timestampsPayloadSize], - ) + // s.timestampDecompressor = enc.NewTimeDeltaDecompressor( + // s.pageData[:timestampsSize], + // ) - vbuf := s.pageData[timestampsPayloadSize : timestampsPayloadSize+valuesPayloadSize] + // vbuf := s.pageData[timestampsSize : timestampsSize+valuesSize] - switch s.metricType { - case qb.Instant: - s.valueDecompressor = enc.NewInstantDeltaDecompressor( - vbuf, s.fracDigits) + // switch s.metricType { + // case qb.Instant: + // s.valueDecompressor = enc.NewInstantDeltaDecompressor( + // vbuf, s.fracDigits) - case qb.Cumulative: - s.valueDecompressor = enc.NewCumulativeDeltaDecompressor( - vbuf, s.fracDigits) + // case qb.Cumulative: + // s.valueDecompressor = enc.NewCumulativeDeltaDecompressor( + // vbuf, s.fracDigits) - default: - return fmt.Errorf("bug: wrong metricType %d", s.metricType) - } + // default: + // return fmt.Errorf("bug: wrong metricType %d", s.metricType) + // } return nil } func makeDecompressors(pageData []byte, metricType qb.MetricType, fracDigits byte) ( qb.TimestampDecompressor, qb.ValueDecompressor, error, ) { - timestampsPayloadSize := bin.GetUint16(pageData[timestampsSizeIdx:]) - valuesPayloadSize := bin.GetUint16(pageData[valuesSizeIdx:]) + // timestampsSize, _ := bin.GetUint16(pageData[timestampsSizeIdx:]) + // valuesSize, _ := bin.GetUint16(pageData[valuesSizeIdx:]) - payloadSize := timestampsPayloadSize + valuesPayloadSize + // payloadSize := timestampsSize + valuesSize - if payloadSize > dataFooterIdx { - return nil, nil, fmt.Errorf("corrupted: timestamps + values size %d > payload size", - payloadSize) - } + // if payloadSize > dataFooterIdx { + // return nil, nil, fmt.Errorf("corrupted: timestamps + values size %d > payload size", + // payloadSize) + // } - timestampDecompressor := enc.NewTimeDeltaDecompressor( - pageData[:timestampsPayloadSize], - ) + // timestampDecompressor := enc.NewTimeDeltaDecompressor( + // pageData[:timestampsSize], + // ) - vbuf := pageData[timestampsPayloadSize : timestampsPayloadSize+valuesPayloadSize] + // vbuf := pageData[timestampsSize : timestampsSize+valuesSize] - var valueDecompressor qb.ValueDecompressor - switch metricType { - case qb.Instant: - valueDecompressor = enc.NewInstantDeltaDecompressor( - vbuf, fracDigits) + // var valueDecompressor qb.ValueDecompressor + // switch metricType { + // case qb.Instant: + // valueDecompressor = enc.NewInstantDeltaDecompressor( + // vbuf, fracDigits) - case qb.Cumulative: - valueDecompressor = enc.NewCumulativeDeltaDecompressor( - vbuf, fracDigits) + // case qb.Cumulative: + // valueDecompressor = enc.NewCumulativeDeltaDecompressor( + // vbuf, fracDigits) - default: - return nil, nil, fmt.Errorf("bug: wrong metricType %d", metricType) - } - return timestampDecompressor, valueDecompressor, nil + // default: + // return nil, nil, fmt.Errorf("bug: wrong metricType %d", metricType) + // } + //return timestampDecompressor, valueDecompressor, nil + return nil, nil, nil } diff --git a/atree/io.go b/atree/io.go index 6f7e3f7..c9b5f1f 100644 --- a/atree/io.go +++ b/atree/io.go @@ -7,8 +7,8 @@ import ( "math" "os" + bin "gordenko.dev/dima/bin/little" "gordenko.dev/dima/qb" - "gordenko.dev/dima/qb/bin" "gordenko.dev/dima/qb/util" ) @@ -35,55 +35,58 @@ func (s *Atree) DeletePages(pageNumbers []uint32) { } func (s *Atree) fetchIndexPage(pageNo uint32) ([]byte, error) { - buf, err := s.fetchPage(pageNo) - if err != nil { - return nil, err - } - if buf[pageTypeIdx] != PageTypeIndex { - return nil, fmt.Errorf("wrong pageType %d instead of %d", buf[pageTypeIdx], PageTypeIndex) - } - return buf, nil + // buf, err := s.fetchPage(pageNo) + // if err != nil { + // return nil, err + // } + // if buf[pageTypeIdx] != PageTypeIndex { + // return nil, fmt.Errorf("wrong pageType %d instead of %d", buf[pageTypeIdx], PageTypeIndex) + // } + // return buf, nil + return nil, nil } func (s *Atree) fetchDataPage(pageNo uint32) ([]byte, error) { - buf, err := s.fetchPage(pageNo) - if err != nil { - return nil, err - } - if buf[pageTypeIdx] != PageTypeIndex { - return nil, fmt.Errorf("wrong pageType %d instead of %d", buf[pageTypeIdx], PageTypeIndex) - } - return buf, nil + // buf, err := s.fetchPage(pageNo) + // if err != nil { + // return nil, err + // } + // if buf[pageTypeIdx] != PageTypeIndex { + // return nil, fmt.Errorf("wrong pageType %d instead of %d", buf[pageTypeIdx], PageTypeIndex) + // } + // return buf, nil + return nil, nil } func (s *Atree) fetchPage(pageNo uint32) ([]byte, error) { - s.mutex.Lock() - p, ok := s.pages[pageNo] - if ok { - p.ReferenceCount++ - s.mutex.Unlock() - return p.Buf, nil - } + // s.mutex.Lock() + // p, ok := s.pages[pageNo] + // if ok { + // p.ReferenceCount++ + // s.mutex.Unlock() + // return p.Buf, nil + // } - resultCh := make(chan readResult, 1) - s.pageWaits[pageNo] = append(s.pageWaits[pageNo], resultCh) - if len(s.pageWaits[pageNo]) == 1 { - s.pagesToRead = append(s.pagesToRead, pageNo) - s.mutex.Unlock() + // resultCh := make(chan readResult, 1) + // s.pageWaits[pageNo] = append(s.pageWaits[pageNo], resultCh) + // if len(s.pageWaits[pageNo]) == 1 { + // s.pagesToRead = append(s.pagesToRead, pageNo) + // s.mutex.Unlock() - select { - case s.readSignalCh <- struct{}{}: - default: - } - } else { - s.mutex.Unlock() - } + // select { + // case s.readSignalCh <- struct{}{}: + // default: + // } + // } else { + // s.mutex.Unlock() + // } - result := <-resultCh - if result.Err == nil { - result.Err = s.verifyCRC(result.Data, PageSize) - } - return result.Data, result.Err + // result := <-resultCh + // if result.Err == nil { + // result.Err = s.verifyCRC(result.Data, PageSize) + // } + // return result.Data, result.Err + return nil, nil } func (s *Atree) releasePage(pageNo uint32) { @@ -211,49 +214,49 @@ func (s *Atree) pageReader() { } func (s *Atree) readPages() { - s.mutex.Lock() - if len(s.pagesToRead) == 0 { - s.mutex.Unlock() - return - } - pagesToRead := s.pagesToRead - s.pagesToRead = nil - s.mutex.Unlock() + // s.mutex.Lock() + // if len(s.pagesToRead) == 0 { + // s.mutex.Unlock() + // return + // } + // pagesToRead := s.pagesToRead + // s.pagesToRead = nil + // s.mutex.Unlock() - for _, pageNo := range pagesToRead { - buf := make([]byte, PageSize) - off := (pageNo - 1) * PageSize - n, err := s.file.ReadAt(buf, int64(off)) - if n != PageSize { - err = fmt.Errorf("read %d instead of %d", n, PageSize) - } + // for _, pageNo := range pagesToRead { + // buf := make([]byte, PageSize) + // off := (pageNo - 1) * PageSize + // n, err := s.file.ReadAt(buf, int64(off)) + // if n != PageSize { + // err = fmt.Errorf("read %d instead of %d", n, PageSize) + // } - s.mutex.Lock() - resultChannels := s.pageWaits[pageNo] - delete(s.pageWaits, pageNo) + // s.mutex.Lock() + // resultChannels := s.pageWaits[pageNo] + // delete(s.pageWaits, pageNo) - if err != nil { - s.mutex.Unlock() - for _, resultCh := range resultChannels { - resultCh <- readResult{ - Err: err, - } - } - } else { - s.pages[pageNo] = &_page{ - // fix - page type - PageNo: pageNo, - Buf: buf, - ReferenceCount: len(resultChannels), - } - s.mutex.Unlock() - for _, resultCh := range resultChannels { - resultCh <- readResult{ - Data: buf, - } - } - } - } + // if err != nil { + // s.mutex.Unlock() + // for _, resultCh := range resultChannels { + // resultCh <- readResult{ + // Err: err, + // } + // } + // } else { + // s.pages[pageNo] = &_page{ + // // fix - page type + // PageNo: pageNo, + // Buf: buf, + // ReferenceCount: len(resultChannels), + // } + // s.mutex.Unlock() + // for _, resultCh := range resultChannels { + // resultCh <- readResult{ + // Data: buf, + // } + // } + // } + // } } // WRITE @@ -364,7 +367,7 @@ func (s *Atree) verifyCRC(data []byte, pageSize int) error { var ( pos = pageSize - 4 calculatedCRC = util.CalcChecksum(data[:pos]) - storedCRC = bin.GetUint32(data[pos:]) + storedCRC, _ = bin.GetUint32(data[pos:]) ) if calculatedCRC != storedCRC { return fmt.Errorf("calculatedCRC %d not equal storedCRC %d", diff --git a/atree/misc.go b/atree/misc.go index 17868c8..d8c1fb7 100644 --- a/atree/misc.go +++ b/atree/misc.go @@ -1,8 +1,8 @@ package atree -import ( - "gordenko.dev/dima/qb/bin" -) +type KeyComparator interface { + CompareTo(int) int +} type ValueAtComparator struct { buf []byte @@ -10,21 +10,22 @@ type ValueAtComparator struct { } func (s ValueAtComparator) CompareTo(elemIdx int) int { - var ( - pos = elemIdx * timestampSize - elem = bin.GetUint32(s.buf[pos:]) - ) + // var ( + // pos = elemIdx * timestampSize + // elem, _ = bin.GetUint32(s.buf[pos:]) + // ) - if s.timestamp < elem { - return -1 - } else if s.timestamp > elem { - return 1 - } else { - return 0 - } + // if s.timestamp < elem { + // return -1 + // } else if s.timestamp > elem { + // return 1 + // } else { + // return 0 + // } + return 213131132 } -func BinarySearch(qty int, keyComparator bin.KeyComparator) (elemIdx int, isFound bool) { +func BinarySearch(qty int, keyComparator KeyComparator) (elemIdx int, isFound bool) { if qty == 0 { return } @@ -55,141 +56,94 @@ func BinarySearch(qty int, keyComparator bin.KeyComparator) (elemIdx int, isFoun } } -// type ChunksToNotLinkedDataPageReq struct { -// // PrevPageNo uint32 -// TimestampsChunks [][]byte -// TimestampsSize uint16 -// ValuesChunks [][]byte -// ValuesSize uint16 +// func getPrevPageNo(buf []byte) uint32 { +// pageNo, _ := bin.GetUint32(buf[prevPageIdx:]) +// return pageNo // } -// func ChunksToNotLinkedDataPage(buf []byte, req ChunksToNotLinkedDataPageReq) { -// bin.PutUint16(buf[timestampsSizeIdx:], req.TimestampsSize) -// bin.PutUint16(buf[valuesSizeIdx:], req.ValuesSize) - -// var ( -// remainingSize = int(req.TimestampsSize) -// pos = 0 -// ) -// for _, chunk := range req.TimestampsChunks { -// if remainingSize >= len(chunk) { -// copy(buf[pos:], chunk) -// remainingSize -= len(chunk) -// pos += len(chunk) -// } else { -// copy(buf[pos:], chunk[:remainingSize]) -// break -// } +// func findPageNo(buf []byte, timestamp uint32) (pageNo uint32) { +// comparator := ValueAtComparator{ +// buf: buf, +// timestamp: timestamp, // } +// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:]) +// elemIdx, _ := BinarySearch(int(qty), comparator) +// pos := indexFooterIdx - (elemIdx+1)*PageNoSize +// pageNo, _ = bin.GetUint32(buf[pos:]) +// return +// } -// remainingSize = int(req.ValuesSize) -// pos = int(req.TimestampsSize) - -// for _, chunk := range req.ValuesChunks { -// if remainingSize >= len(chunk) { -// copy(buf[pos:], chunk) -// remainingSize -= len(chunk) -// pos += len(chunk) -// } else { -// copy(buf[pos:], chunk[:remainingSize]) -// break -// } +// func findPageNoIdx(buf []byte, timestamp uint32) (idx int) { +// comparator := ValueAtComparator{ +// buf: buf, +// timestamp: timestamp, // } +// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:]) +// elemIdx, _ := BinarySearch(int(qty), comparator) +// return elemIdx // } -// func setPrevPageNo(buf []byte, pageNo uint32) { -// bin.PutUint32(buf[prevPageIdx:], pageNo) +// func findPageNoForDeleteSince(buf []byte, since uint32) (uint32, int, int) { +// comparator := ValueAtComparator{ +// buf: buf, +// timestamp: since, +// } +// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:]) +// elemIdx, _ := BinarySearch(int(qty), comparator) +// pos := elemIdx * timestampSize +// timestamp, _ := bin.GetUint32(buf[pos:]) + +// if timestamp == since { +// if elemIdx == 0 { +// return 0, int(qty), -1 +// } +// elemIdx-- +// } +// pos = indexFooterIdx - (elemIdx+1)*PageNoSize +// pageNo, _ := bin.GetUint32(buf[pos:]) +// return pageNo, int(qty), elemIdx // } -func getPrevPageNo(buf []byte) uint32 { - return bin.GetUint32(buf[prevPageIdx:]) -} +// func getLastPageNo(buf []byte) (pageNo uint32) { +// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:]) +// pos := indexFooterIdx - qty*PageNoSize +// pageNo, _ = bin.GetUint32(buf[pos:]) +// return +// } -func findPageNo(buf []byte, timestamp uint32) (pageNo uint32) { - var ( - qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]) - comparator = ValueAtComparator{ - buf: buf, - timestamp: timestamp, - } - ) - elemIdx, _ := BinarySearch(qty, comparator) - pos := indexFooterIdx - (elemIdx+1)*PageNoSize - return bin.GetUint32(buf[pos:]) -} +// func getPageNo(buf []byte, idx int) (pageNo uint32) { +// pos := indexFooterIdx - (idx+1)*PageNoSize +// pageNo, _ = bin.GetUint32(buf[pos:]) +// return +// } -func findPageNoIdx(buf []byte, timestamp uint32) (idx int) { - var ( - qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]) - comparator = ValueAtComparator{ - buf: buf, - timestamp: timestamp, - } - ) - elemIdx, _ := BinarySearch(qty, comparator) - return elemIdx -} +// func listPageNumbers(buf []byte) (pageNumbers []uint32) { +// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:]) +// pos := indexFooterIdx - PageNoSize +// for range qty { +// pageNo, _ := bin.GetUint32(buf[pos:]) +// pageNumbers = append(pageNumbers, pageNo) +// pos -= PageNoSize +// } +// return +// } -func findPageNoForDeleteSince(buf []byte, since uint32) (uint32, int, int) { - var ( - qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]) - comparator = ValueAtComparator{ - buf: buf, - timestamp: since, - } - ) - elemIdx, _ := BinarySearch(qty, comparator) - pos := elemIdx * timestampSize - timestamp := bin.GetUint32(buf[pos:]) - - if timestamp == since { - if elemIdx == 0 { - return 0, qty, -1 - } - elemIdx-- - } - pos = indexFooterIdx - (elemIdx+1)*PageNoSize - return bin.GetUint32(buf[pos:]), qty, elemIdx -} - -func getLastPageNo(buf []byte) (pageNo uint32) { - qty := bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]) - pos := indexFooterIdx - qty*PageNoSize - return bin.GetUint32(buf[pos:]) -} - -func getPageNo(buf []byte, idx int) (pageNo uint32) { - pos := indexFooterIdx - (idx+1)*PageNoSize - return bin.GetUint32(buf[pos:]) -} - -func listPageNumbers(buf []byte) (pageNumbers []uint32) { - qty := bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]) - pos := indexFooterIdx - PageNoSize - for range qty { - pageNumbers = append(pageNumbers, bin.GetUint32(buf[pos:])) - pos -= PageNoSize - } - return -} - -// include since timestamp -func listPageNumbersSince(buf []byte, timestamp uint32) (pageNumbers []uint32) { - var ( - qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]) - comparator = ValueAtComparator{ - buf: buf, - timestamp: timestamp, - } - ) - elemIdx, _ := BinarySearch(qty, comparator) - pos := indexFooterIdx - (elemIdx+1)*PageNoSize - for range qty { - pageNumbers = append(pageNumbers, bin.GetUint32(buf[pos:])) - pos -= PageNoSize - } - return -} +// // include since timestamp +// func listPageNumbersSince(buf []byte, timestamp uint32) (pageNumbers []uint32) { +// comparator := ValueAtComparator{ +// buf: buf, +// timestamp: timestamp, +// } +// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:]) +// elemIdx, _ := BinarySearch(int(qty), comparator) +// pos := indexFooterIdx - (elemIdx+1)*PageNoSize +// for range qty { +// pageNo, _ := bin.GetUint32(buf[pos:]) +// pageNumbers = append(pageNumbers, pageNo) +// pos -= PageNoSize +// } +// return +// } // func getSince(buf []byte) uint32 { // return bin.GetUint32(buf[0:]) diff --git a/atree/select.go b/atree/select.go index 339efbb..ddb62d4 100644 --- a/atree/select.go +++ b/atree/select.go @@ -3,7 +3,7 @@ package atree import ( "fmt" - diploma "gordenko.dev/dima/qb" + "gordenko.dev/dima/qb" ) type ContinueFullScanReq struct { @@ -23,7 +23,7 @@ func (s *Atree) ContinueFullScan(req ContinueFullScanReq) error { PageData: buf, Atree: s, FracDigits: req.FracDigits, - MetricType: diploma.Instant, + MetricType: qb.Instant, }) if err != nil { return err @@ -60,7 +60,7 @@ func (s *Atree) ContinueRangeScan(req ContinueRangeScanReq) error { PageData: buf, Atree: s, FracDigits: req.FracDigits, - MetricType: diploma.Instant, + MetricType: qb.Instant, }) if err != nil { return err @@ -101,7 +101,7 @@ func (s *Atree) RangeScan(req RangeScanReq) error { PageData: buf, Atree: s, FracDigits: req.FracDigits, - MetricType: diploma.Instant, + MetricType: qb.Instant, }) if err != nil { return err diff --git a/bin/bin.go b/bin/bin.go deleted file mode 100644 index 1d14836..0000000 --- a/bin/bin.go +++ /dev/null @@ -1,621 +0,0 @@ -package bin - -import ( - "errors" - "fmt" - "io" - "math" -) - -const ( - varInt64SignFlag = 0b01000000 - maxReadAttempts = 5 -) - -var ( - ErrNoSpace = errors.New("no space") - ErrIncompleteWrite = errors.New("incomplete write") - - ErrReadOverflow = errors.New("bin: reader returned 'n' > bufsize") - // ErrNegativeReadCount shows 100% bug in the source reader - ErrNegativeReadCount = errors.New("bin: reader returned negative 'n'") -) - -// READ - -func ReadUint16(src io.Reader) (num uint16, err error) { - var ( - q = 2 - arr = make([]byte, q) - ) - n, err := src.Read(arr) - if err != nil { - return - } - if n != q { - return 0, fmt.Errorf("read %d bytes only", n) - } - return GetUint16(arr), nil -} - -func ReadUint24AsInt(src io.Reader) (num int, err error) { - var ( - q = 3 - arr = make([]byte, q) - ) - n, err := src.Read(arr) - if err != nil { - return - } - if n != q { - return 0, fmt.Errorf("read %d bytes only", n) - } - return GetUint24AsInt(arr), nil -} - -func ReadUint32(src io.Reader) (num uint32, err error) { - var ( - q = 4 - arr = make([]byte, q) - ) - n, err := src.Read(arr) - if err != nil { - return - } - if n != q { - return 0, fmt.Errorf("read %d bytes only", n) - } - return GetUint32(arr), nil -} - -func ReadUint64(src io.Reader) (num uint64, err error) { - var ( - q = 8 - arr = make([]byte, q) - ) - n, err := src.Read(arr) - if err != nil { - return - } - if n != q { - return 0, fmt.Errorf("read %d bytes only", n) - } - return GetUint64(arr), nil -} - -func ReadFloat64(src io.Reader) (num float64, err error) { - var ( - q = 8 - arr = make([]byte, q) - ) - n, err := src.Read(arr) - if err != nil { - return - } - if n != q { - return 0, fmt.Errorf("read %d bytes only", n) - } - return GetFloat64(arr), nil -} - -func ReadUnixtime(src io.Reader) (num int64, err error) { - var ( - q = 8 - arr = make([]byte, q) - ) - n, err := src.Read(arr) - if err != nil { - return - } - if n != q { - return 0, fmt.Errorf("read %d bytes only", n) - } - return int64(GetUint64(arr)), nil -} - -// READ VAR - -func ReadVarUint64(src io.Reader) (num uint64, n int, err error) { - var ( - p = make([]byte, 1) - b byte - ) - for i := range 8 { - _, err = src.Read(p) - if err != nil { - return - } - n++ - b = p[0] - if b >= 128 { - num |= uint64(b&127) << uint(i*7) - return - } - num |= uint64(b) << uint(i*7) - } - _, err = src.Read(p) - if err != nil { - return - } - n++ - num |= uint64(p[0]) << 56 - return -} - -// GET - -func GetUint16(arr []byte) uint16 { - return uint16(arr[0]) | (uint16(arr[1]) << 8) -} - -func GetUint16AsInt(arr []byte) int { - return int(arr[0]) | (int(arr[1]) << 8) -} - -func GetUint24AsInt(arr []byte) int { - return int(arr[0]) | (int(arr[1]) << 8) | (int(arr[2]) << 16) -} - -func GetUint32(arr []byte) uint32 { - return uint32(arr[0]) | (uint32(arr[1]) << 8) | - (uint32(arr[2]) << 16) | (uint32(arr[3]) << 24) -} - -func GetUint32AsInt64(arr []byte) int64 { - u32 := uint32(arr[0]) | (uint32(arr[1]) << 8) | - (uint32(arr[2]) << 16) | (uint32(arr[3]) << 24) - return int64(u32) -} - -func GetUint40(arr []byte) uint64 { - return uint64(arr[0]) | (uint64(arr[1]) << 8) | - (uint64(arr[2]) << 16) | (uint64(arr[3]) << 24) | - (uint64(arr[4]) << 32) -} - -func GetUint48(arr []byte) uint64 { - return uint64(arr[0]) | (uint64(arr[1]) << 8) | - (uint64(arr[2]) << 16) | (uint64(arr[3]) << 24) | - (uint64(arr[4]) << 32) | (uint64(arr[5]) << 40) -} - -func GetUint64(arr []byte) uint64 { - return uint64(arr[0]) | (uint64(arr[1]) << 8) | - (uint64(arr[2]) << 16) | (uint64(arr[3]) << 24) | - (uint64(arr[4]) << 32) | (uint64(arr[5]) << 40) | - (uint64(arr[6]) << 48) | (uint64(arr[7]) << 56) -} - -func GetFloat32(arr []byte) float32 { - return math.Float32frombits(GetUint32(arr)) -} - -func GetFloat64(arr []byte) float64 { - return math.Float64frombits(GetUint64(arr)) -} - -func GetUnixtime(arr []byte) int64 { - u32 := uint32(arr[0]) | (uint32(arr[1]) << 8) | - (uint32(arr[2]) << 16) | (uint32(arr[3]) << 24) - return int64(u32) -} - -func GetVarUint64(arr []byte) (num uint64, n int, err error) { - var b byte - for i := range 8 { - if i >= len(arr) { - return 0, 0, io.EOF - } - b = arr[i] - if b >= 128 { - num |= uint64(b&127) << uint(i*7) - return num, i + 1, nil - } - num |= uint64(b) << uint(i*7) - } - if len(arr) < 9 { - return 0, 0, io.EOF - } - return num | uint64(arr[8])<<56, 9, nil -} - -func ReverseGetVarUint64(arr []byte) (num uint64, n int, err error) { - var ( - b byte - j = len(arr) - 1 - ) - for i := range 8 { - if j < 0 { - return 0, 0, io.EOF - } - b = arr[j] - if b >= 128 { - num |= uint64(b&127) << uint(i*7) - return num, i + 1, nil - } - num |= uint64(b) << uint(i*7) - j-- - } - if j < 0 { - return 0, 0, io.EOF - } - return num | uint64(arr[j])<<56, 9, nil -} - -func GetVarInt64(arr []byte) (num int64, n int, err error) { - u64, n, err := GetVarUint64(arr) - if err != nil { - return - } - return DecodeZigZag(u64), n, nil -} - -func ReverseGetVarInt64(arr []byte) (num int64, n int, err error) { - u64, n, err := ReverseGetVarUint64(arr) - if err != nil { - return - } - return DecodeZigZag(u64), n, nil -} - -// PUT - -func PutUint16(arr []byte, num uint16) { - arr[0] = byte(num) - arr[1] = byte(num >> 8) -} - -func PutIntAsUint16(arr []byte, num int) { - arr[0] = byte(num) - arr[1] = byte(num >> 8) -} - -func PutIntAsUint24(arr []byte, num int) { - arr[0] = byte(num) - arr[1] = byte(num >> 8) - arr[2] = byte(num >> 16) -} - -func PutUint32(arr []byte, num uint32) { - arr[0] = byte(num) - arr[1] = byte(num >> 8) - arr[2] = byte(num >> 16) - arr[3] = byte(num >> 24) -} - -func PutInt64AsUint32(arr []byte, num int64) { - arr[0] = byte(num) - arr[1] = byte(num >> 8) - arr[2] = byte(num >> 16) - arr[3] = byte(num >> 24) -} - -func PutUint40(arr []byte, num uint64) { - arr[0] = byte(num) - arr[1] = byte(num >> 8) - arr[2] = byte(num >> 16) - arr[3] = byte(num >> 24) - arr[4] = byte(num >> 32) -} - -func PutUint48(arr []byte, num uint64) { - arr[0] = byte(num) - arr[1] = byte(num >> 8) - arr[2] = byte(num >> 16) - arr[3] = byte(num >> 24) - arr[4] = byte(num >> 32) - arr[5] = byte(num >> 40) -} - -func PutUint64(arr []byte, num uint64) { - arr[0] = byte(num) - arr[1] = byte(num >> 8) - arr[2] = byte(num >> 16) - arr[3] = byte(num >> 24) - arr[4] = byte(num >> 32) - arr[5] = byte(num >> 40) - arr[6] = byte(num >> 48) - arr[7] = byte(num >> 56) -} - -func PutFloat32(arr []byte, num float32) { - PutUint32(arr, math.Float32bits(num)) -} - -func PutFloat64(arr []byte, num float64) { - PutUint64(arr, math.Float64bits(num)) -} - -// WRITE - -func WriteUint16(dst io.Writer, num uint16) error { - arr := []byte{ - byte(num), - byte(num >> 8), - } - n, err := dst.Write(arr) - if err != nil { - return err - } - if n != 2 { - return ErrIncompleteWrite - } - return err -} - -func WriteUint32(dst io.Writer, num uint32) error { - arr := []byte{ - byte(num), - byte(num >> 8), - byte(num >> 16), - byte(num >> 24), - } - n, err := dst.Write(arr) - if err != nil { - return err - } - if n != 4 { - return ErrIncompleteWrite - } - return err -} - -func WriteFloat64(dst io.Writer, num float64) error { - arr := make([]byte, 8) - PutUint64(arr, math.Float64bits(num)) - n, err := dst.Write(arr) - if err != nil { - return err - } - if n != 2 { - return ErrIncompleteWrite - } - return err -} - -// WRITE VAR - -func WriteVarUint64(dst io.Writer, num uint64) (int, error) { - arr := make([]byte, 9) - for i := range 8 { - arr[i] = byte(num & 127) - num >>= 7 - if num == 0 { - arr[i] |= 128 - size := i + 1 - n, err := dst.Write(arr[:size]) - if err != nil { - return n, err - } - if n != size { - return n, ErrIncompleteWrite - } - return size, nil - } - } - arr[8] = byte(num) - n, err := dst.Write(arr) - if err != nil { - return n, err - } - if n != 9 { - return n, ErrIncompleteWrite - } - return 9, nil -} - -func PutVarUint64(arr []byte, num uint64) (int, error) { - for i := range 8 { - if i >= len(arr) { - return 0, ErrNoSpace - } - arr[i] = byte(num & 127) - num >>= 7 - if num == 0 { - arr[i] |= 128 - return i + 1, nil - } - } - if len(arr) < 9 { - return 0, ErrNoSpace - } - arr[8] = byte(num) - return 9, nil -} - -func ReversePutVarUint64(arr []byte, num uint64) (int, error) { - var tmp [9]byte - for i := range 8 { - tmp[i] = byte(num & 127) - num >>= 7 - if num == 0 { - tmp[i] |= 128 - n := i + 1 - if len(arr) < n { - return 0, ErrNoSpace - } - for j := i; j >= 0; j-- { - arr[i-j] = tmp[j] - } - return n, nil - } - } - tmp[8] = byte(num) - n := 9 - if len(arr) < n { - return 0, ErrNoSpace - } - for j := 8; j >= 0; j-- { - arr[8-j] = tmp[j] - } - return n, nil -} - -func PutVarUint64AtEnd(arr []byte, num uint64) (int, error) { - var ( - tmp [9]byte - n int - ) - for i := range 8 { - tmp[i] = byte(num & 127) - num >>= 7 - if num == 0 { - tmp[i] |= 128 - n = i + 1 - break - } - } - if n == 0 { - tmp[8] = byte(num) - n = 9 - } - if len(arr) < n { - return 0, ErrNoSpace - } - j := len(arr) - n - for i := range n { - arr[j] = tmp[i] - j++ - } - return n, nil -} - -func PutVarInt64(arr []byte, x int64) (int, error) { - return PutVarUint64(arr, EncodeZigZag(x)) -} - -func PutVarInt64AtEnd(arr []byte, x int64) (int, error) { - return PutVarUint64AtEnd(arr, EncodeZigZag(x)) -} - -func ReversePutVarInt64(arr []byte, x int64) (int, error) { - return ReversePutVarUint64(arr, EncodeZigZag(x)) -} - -type KeyComparator interface { - CompareTo(int) int -} - -func BinarySearch(qty int, keyComparator KeyComparator) (elemIdx int, isFound bool) { - if qty == 0 { - return - } - a := 0 - b := qty - 1 - for { - var ( - elemIdx = (b-a)/2 + a - code = keyComparator.CompareTo(elemIdx) - ) - - if code == 1 { - a = elemIdx + 1 - if a > b { - return elemIdx + 1, false - } - } else if code == -1 { - b = elemIdx - 1 - if b < a { - return elemIdx, false - } - } else { - return elemIdx, true - } - } -} - -func DeleteReverseArrElem(arr []byte, qty int, elemSize int, idx int) { - dstIdx := len(arr) - idx*elemSize - 1 - srcIdx := dstIdx - elemSize - - end := len(arr) - qty*elemSize - - for ; srcIdx >= end; srcIdx-- { - arr[dstIdx] = arr[srcIdx] - dstIdx-- - } - - for i := end; i < end+elemSize; i++ { - arr[i] = 0 - } -} - -// ZigZag - -// ZigZag encoding: int64 -> uint64 -func EncodeZigZag(x int64) uint64 { - return uint64(x<<1) ^ uint64(x>>63) -} - -// ZigZag decoding: uint64 -> int64 -func DecodeZigZag(u uint64) int64 { - return int64(u>>1) ^ -(int64(u & 1)) -} - -func ReadN(r io.Reader, n int) (_ []byte, err error) { - if n < 0 { - err = fmt.Errorf("wrong n=%d", n) - return - } - buf := make([]byte, n) - err = ReadNInto(r, buf) - if err != nil { - return - } - return buf, nil -} - -func ReadNInto(r io.Reader, buf []byte) (err error) { - if len(buf) == 0 { - return - } - - var q, total, readAttempts int - - for readAttempts < maxReadAttempts { - bufsize := len(buf) - total - q, err = r.Read(buf[total:]) - if q == bufsize { - return nil - } - if err != nil { - return - } - if q > bufsize { - err = ErrReadOverflow - return - } - if q < 0 { - err = ErrNegativeReadCount - return - } - if q == 0 { - readAttempts++ - } else { - total += q - } - } - err = io.ErrNoProgress - return -} - -func CalcVarUint64Length(num uint64) int { - for i := range 8 { - num >>= 7 - if num == 0 { - return i + 1 - } - } - return 9 -} - -func CalcVarInt64Length(num int64) int { - u64 := EncodeZigZag(num) - for i := range 8 { - u64 >>= 7 - if u64 == 0 { - return i + 1 - } - } - return 9 -} diff --git a/client/client.go b/client/client.go index 6d60024..37c4f23 100644 --- a/client/client.go +++ b/client/client.go @@ -4,8 +4,8 @@ import ( "fmt" "net" - diploma "gordenko.dev/dima/qb" - "gordenko.dev/dima/qb/bin" + bin "gordenko.dev/dima/bin/little" + "gordenko.dev/dima/qb" "gordenko.dev/dima/qb/bufreader" "gordenko.dev/dima/qb/proto" ) @@ -33,7 +33,6 @@ func Connect(address string) (*Connection, error) { if err != nil { return nil, err } - return &Connection{ conn: conn, src: bufreader.New(conn, 1500), @@ -51,22 +50,19 @@ func (s *Connection) Close() { func (s *Connection) mustSuccess(reader *bufreader.BufferedReader) (err error) { code, err := reader.ReadByte() if err != nil { - return fmt.Errorf("read response code: %s", err) + return } - switch code { case proto.RespSuccess: return nil // ok - case proto.RespError: return s.onError() - default: return fmt.Errorf("unknown reponse code %d", code) } } -func (s *Connection) AddMetric(req proto.AddMetricReq) error { +func (s *Connection) AddMetric(req proto.AddMetricReq) (err error) { arr := []byte{ proto.TypeAddMetric, 0, 0, 0, 0, // @@ -74,9 +70,9 @@ func (s *Connection) AddMetric(req proto.AddMetricReq) error { byte(req.FracDigits), } bin.PutUint32(arr[1:], req.MetricID) - - if _, err := s.conn.Write(arr); err != nil { - return err + // req + if _, err = s.conn.Write(arr); err != nil { + return } return s.mustSuccess(s.src) } @@ -96,52 +92,57 @@ func (s *Connection) AddMetric(req proto.AddMetricReq) error { // return s.mustSuccess(s.src) // } -func (s *Connection) GetMetric(metricID uint32) (*proto.Metric, error) { +func (s *Connection) GetMetric(metricID uint32) (_ *proto.Metric, err error) { arr := []byte{ proto.TypeGetMetric, 0, 0, 0, 0, } bin.PutUint32(arr[1:], metricID) - - if _, err := s.conn.Write(arr); err != nil { - return nil, err + // req + if _, err = s.conn.Write(arr); err != nil { + return } - + // answer code, err := s.src.ReadByte() if err != nil { - return nil, fmt.Errorf("read response code: %s", err) + return } - switch code { case proto.RespValue: - arr, err := s.src.ReadN(6) + var ( + metric proto.Metric + metricType byte + ) + metric.MetricID, err = bin.ReadUint32(s.src) if err != nil { - return nil, fmt.Errorf("read body: %s", err) + return } - - return &proto.Metric{ - MetricID: bin.GetUint32(arr), - MetricType: diploma.MetricType(arr[4]), - FracDigits: int(arr[5]), - }, nil - + metricType, err = bin.ReadByte(s.src) + if err != nil { + return + } + metric.MetricType = qb.MetricType(metricType) + metric.FracDigits, err = bin.ReadByteAsInt(s.src) + if err != nil { + return + } + return &metric, nil case proto.RespError: return nil, s.onMaybeError() - default: return nil, fmt.Errorf("unknown reponse code %d", code) } } -func (s *Connection) DeleteMetric(metricID uint32) error { +func (s *Connection) DeleteMetric(metricID uint32) (err error) { arr := []byte{ proto.TypeDeleteMetric, 0, 0, 0, 0, // } bin.PutUint32(arr[1:], metricID) - - if _, err := s.conn.Write(arr); err != nil { - return err + // req + if _, err = s.conn.Write(arr); err != nil { + return } return s.mustSuccess(s.src) } @@ -156,9 +157,9 @@ func (s *Connection) AppendMeasure(req proto.AppendMeasureReq) (err error) { bin.PutUint32(arr[1:], req.MetricID) bin.PutUint32(arr[5:], req.Timestamp) bin.PutFloat64(arr[9:], req.Value) - - if _, err := s.conn.Write(arr); err != nil { - return err + // req + if _, err = s.conn.Write(arr); err != nil { + return } return s.mustSuccess(s.src) } @@ -181,11 +182,10 @@ func (s *Connection) AppendMeasures(req proto.AppendMeasuresReq) (err error) { bin.PutFloat64(arr[pos+4:], measure.Value) pos += recordSize } - if _, err := s.conn.Write(arr); err != nil { - return err + // req + if _, err = s.conn.Write(arr); err != nil { + return } - - //fmt.Printf("encode measures: %d\n", len(req.Measures)) return s.mustSuccess(s.src) } @@ -213,26 +213,26 @@ func (s *Connection) AppendMeasurePerMetric(list []proto.MetricMeasure) (_ []pro bin.PutFloat64(arr[pos+8:], item.Value) pos += recordSize } - if _, err := s.conn.Write(arr); err != nil { - return nil, err + // req + if _, err = s.conn.Write(arr); err != nil { + return } - + // answer code, err := s.src.ReadByte() if err != nil { - return nil, fmt.Errorf("read response code: %s", err) + return } - switch code { case proto.RespValue: var ( - qty uint16 + count int appendErrors []proto.AppendError ) - qty, err = bin.ReadUint16(s.src) + count, err = bin.ReadUint16AsInt(s.src) if err != nil { return } - for range qty { + for range count { var ae proto.AppendError ae.MetricID, err = bin.ReadUint32(s.src) if err != nil { @@ -245,70 +245,26 @@ func (s *Connection) AppendMeasurePerMetric(list []proto.MetricMeasure) (_ []pro appendErrors = append(appendErrors, ae) } return appendErrors, nil - default: return nil, fmt.Errorf("unknown reponse code %d", code) } } -func (s *Connection) ListAllInstantMeasures(metricID uint32) ([]proto.InstantMeasure, error) { +func (s *Connection) ListAllInstantMeasures(metricID uint32) (_ []proto.InstantMeasure, err error) { arr := []byte{ proto.TypeListAllInstantMeasures, 0, 0, 0, 0, // metricID } bin.PutUint32(arr[1:], metricID) - - if _, err := s.conn.Write(arr); err != nil { - return nil, err - } - - var ( - result []proto.InstantMeasure - tmp = make([]byte, 12) - ) - - for { - code, err := s.src.ReadByte() - if err != nil { - return nil, fmt.Errorf("read response code: %s", err) - } - - //fmt.Printf("code: %d\n", code) - - switch code { - case proto.RespPartOfValue: - q, err := bin.ReadUint32(s.src) - if err != nil { - return nil, fmt.Errorf("read records qty: %s", err) - } - - //fmt.Printf("q: %d\n", q) - - for i := range int(q) { - err = bin.ReadNInto(s.src, tmp) - if err != nil { - return nil, fmt.Errorf("read record #%d: %s", i, err) - } - - result = append(result, proto.InstantMeasure{ - Timestamp: bin.GetUint32(tmp), - Value: bin.GetFloat64(tmp[4:]), - }) - } - - case proto.RespEndOfValue: - return result, nil - - case proto.RespError: - return nil, s.onError() - - default: - return nil, fmt.Errorf("unknown reponse code %d", code) - } + // req + if _, err = s.conn.Write(arr); err != nil { + return } + // answer + return s.readInstantMeasures() } -func (s *Connection) ListInstantMeasures(req proto.ListInstantMeasuresReq) ([]proto.InstantMeasure, error) { +func (s *Connection) ListInstantMeasures(req proto.ListInstantMeasuresReq) (_ []proto.InstantMeasure, err error) { arr := []byte{ proto.TypeListInstantMeasures, 0, 0, 0, 0, // metricID @@ -318,118 +274,66 @@ func (s *Connection) ListInstantMeasures(req proto.ListInstantMeasuresReq) ([]pr bin.PutUint32(arr[1:], req.MetricID) bin.PutUint32(arr[5:], req.Since) bin.PutUint32(arr[9:], req.Until) - - if _, err := s.conn.Write(arr); err != nil { - return nil, err + // req + if _, err = s.conn.Write(arr); err != nil { + return } + // answer + return s.readInstantMeasures() +} - var ( - result []proto.InstantMeasure - tmp = make([]byte, 12) - ) - +func (s *Connection) readInstantMeasures() (_ []proto.InstantMeasure, err error) { + var result []proto.InstantMeasure for { - code, err := s.src.ReadByte() + var code byte + code, err = bin.ReadByte(s.src) if err != nil { return nil, fmt.Errorf("read response code: %s", err) } - switch code { case proto.RespPartOfValue: - q, err := bin.ReadUint32(s.src) + var count int + count, err = bin.ReadUint32AsInt(s.src) if err != nil { return nil, fmt.Errorf("read records qty: %s", err) } - - for i := range int(q) { - err = bin.ReadNInto(s.src, tmp) + for range count { + var measure proto.InstantMeasure + measure.Timestamp, err = bin.ReadUint32(s.src) if err != nil { - return nil, fmt.Errorf("read record #%d: %s", i, err) + return } - - result = append(result, proto.InstantMeasure{ - Timestamp: bin.GetUint32(tmp), - Value: bin.GetFloat64(tmp[4:]), - }) + measure.Value, err = bin.ReadFloat64(s.src) + if err != nil { + return + } + result = append(result, measure) } - case proto.RespEndOfValue: return result, nil - case proto.RespError: return nil, s.onError() - default: return nil, fmt.Errorf("unknown reponse code %d", code) } } } -func (s *Connection) ListAllCumulativeMeasures(metricID uint32) ([]proto.CumulativeMeasure, error) { +func (s *Connection) ListAllCumulativeMeasures(metricID uint32) (_ []proto.CumulativeMeasure, err error) { arr := []byte{ proto.TypeListAllCumulativeMeasures, 0, 0, 0, 0, // metricID } bin.PutUint32(arr[1:], metricID) - - if _, err := s.conn.Write(arr); err != nil { - return nil, err - } - - var ( - result []proto.CumulativeMeasure - tmp = make([]byte, 20) - ) - - for { - code, err := s.src.ReadByte() - if err != nil { - return nil, fmt.Errorf("read response code: %s", err) - } - - //fmt.Printf("code: %d\n", code) - - switch code { - case proto.RespPartOfValue: - q, err := bin.ReadUint32(s.src) - if err != nil { - return nil, fmt.Errorf("read records qty: %s", err) - } - - for i := range int(q) { - err = bin.ReadNInto(s.src, tmp) - if err != nil { - return nil, fmt.Errorf("read record #%d: %s", i, err) - } - - //fmt.Printf("tmp: %d\n", tmp) - - result = append(result, proto.CumulativeMeasure{ - Timestamp: bin.GetUint32(tmp), - Value: bin.GetFloat64(tmp[4:]), - Total: bin.GetFloat64(tmp[12:]), - }) - - // pretty.PPrintln("measure", CumulativeMeasure{ - // Timestamp: bin.GetUint32(tmp), - // Value: bin.GetFloat64(tmp[4:]), - // Total: bin.GetFloat64(tmp[12:]), - // }) - } - - case proto.RespEndOfValue: - return result, nil - - case proto.RespError: - return nil, s.onError() - - default: - return nil, fmt.Errorf("unknown reponse code %d", code) - } + // req + if _, err = s.conn.Write(arr); err != nil { + return } + // answer + return s.readCumulativeMeasures() } -func (s *Connection) ListCumulativeMeasures(req proto.ListCumulativeMeasuresReq) ([]proto.CumulativeMeasure, error) { +func (s *Connection) ListCumulativeMeasures(req proto.ListCumulativeMeasuresReq) (_ []proto.CumulativeMeasure, err error) { arr := []byte{ proto.TypeListCumulativeMeasures, 0, 0, 0, 0, // metricID @@ -439,55 +343,56 @@ func (s *Connection) ListCumulativeMeasures(req proto.ListCumulativeMeasuresReq) bin.PutUint32(arr[1:], req.MetricID) bin.PutUint32(arr[5:], req.Since) bin.PutUint32(arr[9:], req.Until) - - if _, err := s.conn.Write(arr); err != nil { - return nil, err + // req + if _, err = s.conn.Write(arr); err != nil { + return } + // answer + return s.readCumulativeMeasures() +} - var ( - result []proto.CumulativeMeasure - tmp = make([]byte, 20) - ) - +func (s *Connection) readCumulativeMeasures() (_ []proto.CumulativeMeasure, err error) { + var result []proto.CumulativeMeasure for { - code, err := s.src.ReadByte() + var code byte + code, err = bin.ReadByte(s.src) if err != nil { return nil, fmt.Errorf("read response code: %s", err) } - switch code { case proto.RespPartOfValue: - q, err := bin.ReadUint32(s.src) + var count int + count, err = bin.ReadUint32AsInt(s.src) if err != nil { return nil, fmt.Errorf("read records qty: %s", err) } - - for i := range int(q) { - err = bin.ReadNInto(s.src, tmp) + for range count { + var measure proto.CumulativeMeasure + measure.Timestamp, err = bin.ReadUint32(s.src) if err != nil { - return nil, fmt.Errorf("read record #%d: %s", i, err) + return } - - result = append(result, proto.CumulativeMeasure{ - Timestamp: bin.GetUint32(tmp), - Value: bin.GetFloat64(tmp[4:]), - Total: bin.GetFloat64(tmp[12:]), - }) + measure.Value, err = bin.ReadFloat64(s.src) + if err != nil { + return + } + measure.Total, err = bin.ReadFloat64(s.src) + if err != nil { + return + } + result = append(result, measure) } - case proto.RespEndOfValue: return result, nil - case proto.RespError: return nil, s.onError() - default: return nil, fmt.Errorf("unknown reponse code %d", code) } } } -func (s *Connection) ListInstantPeriods(req proto.ListInstantPeriodsReq) ([]proto.InstantPeriod, error) { +func (s *Connection) ListInstantPeriods(req proto.ListInstantPeriodsReq) (_ []proto.InstantPeriod, err error) { arr := []byte{ proto.TypeListInstantPeriods, 0, 0, 0, 0, // metricID @@ -504,87 +409,70 @@ func (s *Connection) ListInstantPeriods(req proto.ListInstantPeriodsReq) ([]prot bin.PutUint16(arr[9:], uint16(req.Until.Year)) arr[11] = byte(req.Until.Month) arr[12] = byte(req.Until.Day) - - if _, err := s.conn.Write(arr); err != nil { - return nil, err + // req + if _, err = s.conn.Write(arr); err != nil { + return } - - var q int - if (req.AggregateFuncs & diploma.AggregateMin) == diploma.AggregateMin { - q++ - } - if (req.AggregateFuncs & diploma.AggregateMax) == diploma.AggregateMax { - q++ - } - if (req.AggregateFuncs & diploma.AggregateAvg) == diploma.AggregateAvg { - q++ - } - - var ( - result []proto.InstantPeriod - // 12 bytes - period, since, until - // q * 8 bytes - min, max, avg - tmp = make([]byte, 12+q*8) - ) - + // answer + var result []proto.InstantPeriod for { - code, err := s.src.ReadByte() + var code byte + code, err = s.src.ReadByte() if err != nil { - return nil, fmt.Errorf("read response code: %s", err) + return } - - //fmt.Printf("code: %d\n", code) - switch code { case proto.RespPartOfValue: - q, err := bin.ReadUint32(s.src) + var count int + count, err = bin.ReadUint32AsInt(s.src) if err != nil { - return nil, fmt.Errorf("read records qty: %s", err) + return } - - for i := range int(q) { - err = bin.ReadNInto(s.src, tmp) + for range count { + var p proto.InstantPeriod + p.Period, err = bin.ReadUint32(s.src) if err != nil { - return nil, fmt.Errorf("read record #%d: %s", i, err) + return } - - var ( - p = proto.InstantPeriod{ - Period: bin.GetUint32(tmp[0:]), - Since: bin.GetUint32(tmp[4:]), - Until: bin.GetUint32(tmp[8:]), + p.Since, err = bin.ReadUint32(s.src) + if err != nil { + return + } + p.Until, err = bin.ReadUint32(s.src) + if err != nil { + return + } + if (req.AggregateFuncs & qb.AggregateMin) == qb.AggregateMin { + p.Min, err = bin.ReadFloat64(s.src) + if err != nil { + return } - // 12 bytes - period, since, until - pos = 12 - ) - - if (req.AggregateFuncs & diploma.AggregateMin) == diploma.AggregateMin { - p.Min = bin.GetFloat64(tmp[pos:]) - pos += 8 } - if (req.AggregateFuncs & diploma.AggregateMax) == diploma.AggregateMax { - p.Max = bin.GetFloat64(tmp[pos:]) - pos += 8 + if (req.AggregateFuncs & qb.AggregateMax) == qb.AggregateMax { + p.Max, err = bin.ReadFloat64(s.src) + if err != nil { + return + } } - if (req.AggregateFuncs & diploma.AggregateAvg) == diploma.AggregateAvg { - p.Avg = bin.GetFloat64(tmp[pos:]) + if (req.AggregateFuncs & qb.AggregateAvg) == qb.AggregateAvg { + p.Avg, err = bin.ReadFloat64(s.src) + if err != nil { + return + } } result = append(result, p) } - case proto.RespEndOfValue: return result, nil - case proto.RespError: return nil, s.onError() - default: return nil, fmt.Errorf("unknown reponse code %d", code) } } } -func (s *Connection) ListCumulativePeriods(req proto.ListCumulativePeriodsReq) ([]proto.CumulativePeriod, error) { +func (s *Connection) ListCumulativePeriods(req proto.ListCumulativePeriodsReq) (_ []proto.CumulativePeriod, err error) { arr := []byte{ proto.TypeListCumulativePeriods, 0, 0, 0, 0, // metricID @@ -600,108 +488,107 @@ func (s *Connection) ListCumulativePeriods(req proto.ListCumulativePeriodsReq) ( bin.PutUint16(arr[9:], uint16(req.Until.Year)) arr[11] = byte(req.Until.Month) arr[12] = byte(req.Until.Day) - - if _, err := s.conn.Write(arr); err != nil { - return nil, err + // req + if _, err = s.conn.Write(arr); err != nil { + return } - - var ( - result []proto.CumulativePeriod - tmp = make([]byte, 28) - ) - + // answer + var result []proto.CumulativePeriod for { - code, err := s.src.ReadByte() + var code byte + code, err = s.src.ReadByte() if err != nil { - return nil, fmt.Errorf("read response code: %s", err) + return } - switch code { case proto.RespPartOfValue: - q, err := bin.ReadUint32(s.src) + var count int + count, err = bin.ReadUint32AsInt(s.src) if err != nil { - return nil, fmt.Errorf("read records qty: %s", err) + return } - - for i := range int(q) { - err = bin.ReadNInto(s.src, tmp) + for range count { + var p proto.CumulativePeriod + p.Period, err = bin.ReadUint32(s.src) if err != nil { - return nil, fmt.Errorf("read record #%d: %s", i, err) + return } - result = append(result, proto.CumulativePeriod{ - Period: bin.GetUint32(tmp[0:]), - Since: bin.GetUint32(tmp[4:]), - Until: bin.GetUint32(tmp[8:]), - EndValue: bin.GetFloat64(tmp[12:]), - Total: bin.GetFloat64(tmp[20:]), - }) + p.Since, err = bin.ReadUint32(s.src) + if err != nil { + return + } + p.Until, err = bin.ReadUint32(s.src) + if err != nil { + return + } + p.EndValue, err = bin.ReadFloat64(s.src) + if err != nil { + return + } + p.Total, err = bin.ReadFloat64(s.src) + if err != nil { + return + } + result = append(result, p) } - case proto.RespEndOfValue: return result, nil - case proto.RespError: return nil, s.onError() - default: return nil, fmt.Errorf("unknown reponse code %d", code) } } } -func (s *Connection) ListCurrentValues(metricIDs []uint32) ([]proto.CurrentValue, error) { +func (s *Connection) ListCurrentValues(metricIDs []uint32) (_ []proto.CurrentValue, err error) { arr := make([]byte, 3+metricKeySize*len(metricIDs)) arr[0] = proto.TypeListCurrentValues - bin.PutUint16(arr[1:], uint16(len(metricIDs))) - off := 3 for _, metricID := range metricIDs { bin.PutUint32(arr[off:], metricID) off += metricKeySize } - - if _, err := s.conn.Write(arr); err != nil { - return nil, err + // req + if _, err = s.conn.Write(arr); err != nil { + return } - - var ( - result []proto.CurrentValue - tmp = make([]byte, 16) - ) - + // answer + var result []proto.CurrentValue for { - code, err := s.src.ReadByte() + var code byte + code, err = s.src.ReadByte() if err != nil { - return nil, fmt.Errorf("read response code: %s", err) + return } - switch code { case proto.RespPartOfValue: - q, err := bin.ReadUint32(s.src) + var count int + count, err = bin.ReadUint32AsInt(s.src) if err != nil { - return nil, fmt.Errorf("read records qty: %s", err) + return } - - for i := range int(q) { - err = bin.ReadNInto(s.src, tmp) + for range count { + var m proto.CurrentValue + m.MetricID, err = bin.ReadUint32(s.src) if err != nil { - return nil, fmt.Errorf("read record #%d: %s", i, err) + return } - - result = append(result, proto.CurrentValue{ - MetricID: bin.GetUint32(tmp), - Timestamp: bin.GetUint32(tmp[4:]), - Value: bin.GetFloat64(tmp[8:]), - }) + m.Timestamp, err = bin.ReadUint32(s.src) + if err != nil { + return + } + m.Value, err = bin.ReadFloat64(s.src) + if err != nil { + return + } + result = append(result, m) } - case proto.RespEndOfValue: return result, nil - case proto.RespError: return nil, s.onError() - default: return nil, fmt.Errorf("unknown reponse code %d", code) } @@ -716,7 +603,7 @@ func (s *Connection) DeleteMeasures(req proto.DeleteMeasuresReq) (err error) { } bin.PutUint32(arr[1:], req.MetricID) bin.PutUint32(arr[5:], req.Since) - + // req if _, err := s.conn.Write(arr); err != nil { return err } diff --git a/conbuf/conbuf.go b/conbuf/conbuf.go deleted file mode 100644 index 19215d2..0000000 --- a/conbuf/conbuf.go +++ /dev/null @@ -1,460 +0,0 @@ -package conbuf - -import ( - "errors" - "fmt" - "io" - - "gordenko.dev/dima/qb/bin" -) - -const chunkSize = 512 -const ChunkSize = 512 - -var ( - ErrOutOfRange = errors.New("out of range") -) - -type ContinuousBuffer struct { - chunks [][]byte -} - -func NewFromBuffer(buf []byte) *ContinuousBuffer { - var ( - chunks [][]byte - copied = 0 - ) - - for copied < len(buf) { - chunk := make([]byte, chunkSize) - end := min(copied+chunkSize, len(buf)) - copy(chunk, buf[copied:end]) - chunks = append(chunks, chunk) - copied += end - copied - } - - return &ContinuousBuffer{ - chunks: chunks, - } -} - -func New(chunks [][]byte) *ContinuousBuffer { - for i, chunk := range chunks { - if len(chunk) != chunkSize { - panic(fmt.Sprintf("wrong chunk #%d size %d", i, len(chunk))) - } - } - return &ContinuousBuffer{ - chunks: chunks, - } -} - -func (s *ContinuousBuffer) Chunks() [][]byte { - return s.chunks -} - -// [0, pos) -func (s *ContinuousBuffer) GetByte(idx int) byte { - chunkIdx := idx / chunkSize - if chunkIdx >= len(s.chunks) { - panic(ErrOutOfRange) - } - byteIdx := idx % chunkSize - return s.chunks[chunkIdx][byteIdx] -} - -func (s *ContinuousBuffer) SetByte(idx int, b byte) { - chunkIdx := idx / chunkSize - if chunkIdx > len(s.chunks) { - panic(ErrOutOfRange) - } - if chunkIdx == len(s.chunks) { - s.chunks = append(s.chunks, make([]byte, chunkSize)) - } - byteIdx := idx % chunkSize - s.chunks[chunkIdx][byteIdx] = b -} - -func (s *ContinuousBuffer) SetFlag(idx int, flag byte) { - chunkIdx := idx / chunkSize - if chunkIdx > len(s.chunks) { - panic(ErrOutOfRange) - } - if chunkIdx == len(s.chunks) { - s.chunks = append(s.chunks, make([]byte, chunkSize)) - } - byteIdx := idx % chunkSize - s.chunks[chunkIdx][byteIdx] |= flag -} - -func (s *ContinuousBuffer) UnsetFlag(idx int, flag byte) { - chunkIdx := idx / chunkSize - if chunkIdx > len(s.chunks) { - panic(ErrOutOfRange) - } - if chunkIdx == len(s.chunks) { - s.chunks = append(s.chunks, make([]byte, chunkSize)) - } - byteIdx := idx % chunkSize - s.chunks[chunkIdx][byteIdx] &^= flag -} - -// [since, until) -func (s *ContinuousBuffer) ShiftOnePosToRight(since int, until int) { - if since < 0 { - panic("since < 0") - } - if since >= until { - panic("since >= until") - } - - chunkIdx := until / chunkSize - byteIdx := until % chunkSize - - if chunkIdx > len(s.chunks) { - panic(ErrOutOfRange) - } - - if chunkIdx == len(s.chunks) { - if byteIdx == 0 { - s.chunks = append(s.chunks, make([]byte, chunkSize)) - } else { - panic(ErrOutOfRange) - } - } - - var ( - qty = until - since - prevChunkIdx int - prevByteIdx int - ) - - for range qty { - prevChunkIdx = chunkIdx - prevByteIdx = byteIdx - 1 - if prevByteIdx < 0 { - prevChunkIdx = chunkIdx - 1 - prevByteIdx = chunkSize - 1 - } - - s.chunks[chunkIdx][byteIdx] = s.chunks[prevChunkIdx][prevByteIdx] - - if byteIdx > 0 { - byteIdx-- - } else { - chunkIdx-- - byteIdx = chunkSize - 1 - } - } -} - -// [since, until) -func (s *ContinuousBuffer) ShiftOnePosToLeft(since int, until int) { - if since <= 0 { - panic("since <= 0") - } - if since >= until { - panic("since >= until") - } - - chunkIdx := since / chunkSize - byteIdx := since % chunkSize - - if until > len(s.chunks)*chunkSize { - panic(ErrOutOfRange) - } - - var ( - qty = until - since - prevChunkIdx int - prevByteIdx int - ) - - for range qty { - prevChunkIdx = chunkIdx - prevByteIdx = byteIdx - 1 - if prevByteIdx < 0 { - prevChunkIdx = chunkIdx - 1 - prevByteIdx = chunkSize - 1 - } - - s.chunks[prevChunkIdx][prevByteIdx] = s.chunks[chunkIdx][byteIdx] - - byteIdx++ - if byteIdx == chunkSize { - chunkIdx++ - byteIdx = 0 - } - } -} - -func (s *ContinuousBuffer) PutUint32(pos int, num uint32) { - s.CopyTo(pos, - []byte{ - byte(num), - byte(num >> 8), - byte(num >> 16), - byte(num >> 24), - }) -} - -func (s *ContinuousBuffer) GetUint32(pos int) uint32 { - arr := s.Slice(pos, pos+4) - return uint32(arr[0]) | (uint32(arr[1]) << 8) | - (uint32(arr[2]) << 16) | (uint32(arr[3]) << 24) -} - -func (s *ContinuousBuffer) ReversePutVarUint64(pos int, num uint64) int { - var tmp [9]byte - for i := range 8 { - tmp[i] = byte(num & 127) - num >>= 7 - if num == 0 { - tmp[i] |= 128 - n := i + 1 - s.ReverseCopyTo(pos, tmp[:n]) - return n - } - } - tmp[8] = byte(num) - n := 9 - s.ReverseCopyTo(pos, tmp[:]) - return n -} - -func (s *ContinuousBuffer) ReverseGetVarUint64(idx int) (num uint64, n int, err error) { - chunkIdx := idx / chunkSize - if chunkIdx >= len(s.chunks) { - panic(ErrOutOfRange) - } - - var ( - byteIdx = idx % chunkSize - chunk = s.chunks[chunkIdx] - b byte - ) - - for i := range 8 { - b = chunk[byteIdx] - if b >= 128 { - num |= uint64(b&127) << uint(i*7) - return num, i + 1, nil - } - num |= uint64(b) << uint(i*7) - - if byteIdx > 0 { - byteIdx-- - } else { - if chunkIdx == 0 { - return 0, 0, io.EOF - } else { - chunkIdx-- - chunk = s.chunks[chunkIdx] - byteIdx = chunkSize - 1 - } - } - } - return num | uint64(chunk[byteIdx])<<56, 9, nil -} - -func (s *ContinuousBuffer) ReversePutVarInt64(pos int, x int64) int { - return s.ReversePutVarUint64(pos, bin.EncodeZigZag(x)) -} - -func (s *ContinuousBuffer) ReverseGetVarInt64(idx int) (int64, int, error) { - u64, n, err := s.ReverseGetVarUint64(idx) - if err != nil { - return 0, 0, err - } - return bin.DecodeZigZag(u64), n, nil -} - -func (s *ContinuousBuffer) PutVarUint64(pos int, num uint64) int { - var tmp [9]byte - for i := range 8 { - tmp[i] = byte(num & 127) - num >>= 7 - if num == 0 { - tmp[i] |= 128 - n := i + 1 - s.CopyTo(pos, tmp[:n]) - return n - } - } - tmp[8] = byte(num) - s.CopyTo(pos, tmp[:]) - return 9 -} - -func (s *ContinuousBuffer) GetVarUint64(idx int) (num uint64, n int, err error) { - chunkIdx := idx / chunkSize - if chunkIdx >= len(s.chunks) { - panic(ErrOutOfRange) - } - - var ( - byteIdx = idx % chunkSize - chunk = s.chunks[chunkIdx] - b byte - ) - - for i := range 8 { - b = chunk[byteIdx] - if b >= 128 { - num |= uint64(b&127) << uint(i*7) - return num, i + 1, nil - } - num |= uint64(b) << uint(i*7) - - byteIdx++ - if byteIdx == chunkSize { - chunkIdx++ - if chunkIdx == len(s.chunks) { - return 0, 0, io.EOF - } - chunk = s.chunks[chunkIdx] - byteIdx = 0 - } - } - return num | uint64(chunk[byteIdx])<<56, 9, nil -} - -func (s *ContinuousBuffer) PutVarInt64(idx int, x int64) int { - return s.PutVarUint64(idx, bin.EncodeZigZag(x)) -} - -func (s *ContinuousBuffer) GetVarInt64(idx int) (int64, int, error) { - u64, n, err := s.GetVarUint64(idx) - if err != nil { - return 0, 0, err - } - return bin.DecodeZigZag(u64), n, nil -} - -func (s *ContinuousBuffer) CopyTo(idx int, data []byte) { - chunkIdx := idx / chunkSize - if chunkIdx > len(s.chunks) { - panic(ErrOutOfRange) - } - if chunkIdx == len(s.chunks) { - s.chunks = append(s.chunks, make([]byte, chunkSize)) - } - byteIdx := idx % chunkSize - chunk := s.chunks[chunkIdx] - copied := 0 - - for _, b := range data { - chunk[byteIdx] = b - copied++ - byteIdx++ - if byteIdx == chunkSize { - byteIdx = 0 - chunkIdx++ - if chunkIdx == len(s.chunks) { - if copied == len(data) { - return - } - s.chunks = append(s.chunks, make([]byte, chunkSize)) - } - chunk = s.chunks[chunkIdx] - } - } -} - -func (s *ContinuousBuffer) ReverseCopyTo(idx int, data []byte) { - chunkIdx := idx / chunkSize - if chunkIdx > len(s.chunks) { - panic(ErrOutOfRange) - } - if chunkIdx == len(s.chunks) { - s.chunks = append(s.chunks, make([]byte, chunkSize)) - } - byteIdx := idx % chunkSize - chunk := s.chunks[chunkIdx] - copied := 0 - - for i := len(data) - 1; i >= 0; i-- { - chunk[byteIdx] = data[i] - copied++ - byteIdx++ - if byteIdx == chunkSize { - byteIdx = 0 - chunkIdx++ - if chunkIdx == len(s.chunks) { - if copied == len(data) { - return - } - s.chunks = append(s.chunks, make([]byte, chunkSize)) - } - chunk = s.chunks[chunkIdx] - } - } -} - -// [since, until) -func (s *ContinuousBuffer) Slice(since int, until int) []byte { - if since >= until { - return nil - } - size := len(s.chunks) * chunkSize - if until >= size { - panic(ErrOutOfRange) - } - - data := make([]byte, until-since) - - chunkIdx := since / chunkSize - byteIdx := since % chunkSize - chunk := s.chunks[chunkIdx] - - for i := range len(data) { - data[i] = chunk[byteIdx] - byteIdx++ - if byteIdx == chunkSize { - byteIdx = 0 - chunkIdx++ - chunk = s.chunks[chunkIdx] - } - } - return data -} - -func (s *ContinuousBuffer) DecodeRunLength(pos int) (length uint16, n int) { - b1 := s.GetByte(pos) - pos-- - if b1 < 128 { - length = uint16(b1) - n = 1 - } else { - b2 := s.GetByte(pos) - length = uint16(b1&127) | (uint16(b2) << 7) - n = 2 - } - length += 2 - return -} - -func (s *ContinuousBuffer) Copy() *ContinuousBuffer { - var copies [][]byte - for _, chunk := range s.chunks { - buf := make([]byte, chunkSize) - copy(buf, chunk) - copies = append(copies, buf) - } - return New(copies) -} - -// size to copy -func (s *ContinuousBuffer) CopyChunksToOneBuffer(dst []byte, size int) { - pos := 0 - for _, chunk := range s.chunks { - if size >= len(chunk) { - copy(dst[pos:], chunk) - size -= len(chunk) - pos += len(chunk) - } else { - copy(dst[pos:], chunk[:size]) - return - } - } -} diff --git a/database/api.go b/database/api.go index 378d97a..7f9c247 100644 --- a/database/api.go +++ b/database/api.go @@ -6,13 +6,13 @@ import ( "io" "net" - qb "gordenko.dev/dima/qb" + bin "gordenko.dev/dima/bin/little" + "gordenko.dev/dima/qb" "gordenko.dev/dima/qb/atree" - "gordenko.dev/dima/qb/bin" "gordenko.dev/dima/qb/bufreader" "gordenko.dev/dima/qb/proto" + "gordenko.dev/dima/qb/storage" "gordenko.dev/dima/qb/transform" - "gordenko.dev/dima/qb/txlog" ) var ( @@ -213,7 +213,7 @@ func (s *Database) AddMetric(req proto.AddMetricReq) uint16 { switch resultCode { case Succeed: - s.txlog.Append(txlog.AddedMetric{ + s.storage.Append(storage.MetricAddRecord{ MetricID: req.MetricID, MetricType: req.MetricType, FracDigits: req.FracDigits, @@ -286,20 +286,20 @@ func (s *Database) DeleteMetric(req proto.DeleteMetricReq) uint16 { switch result.ResultCode { case Succeed: - var ( - freePageNumbers []uint32 - ) - if result.RootPageNo > 0 { - var err error - freePageNumbers, err = s.atree.GetAllPages(result.RootPageNo) - if err != nil { - qb.Abort(qb.FailedAtreeRequest, err) - } - } - s.txlog.Append(txlog.DeletedMetric{ - MetricID: req.MetricID, - FreePageNumbers: freePageNumbers, - }) + // var ( + // //freePageNumbers []uint32 + // ) + // if result.RootPageNo > 0 { + // var err error + // freePageNumbers, err = s.atree.GetAllPages(result.RootPageNo) + // if err != nil { + // qb.Abort(qb.FailedAtreeRequest, err) + // } + // } + // s.storage.Append(storage.DeletedMetric{ + // MetricID: req.MetricID, + // //FreePageNumbers: freePageNumbers, FIX + // }) //<-waitCh case NoMetric: @@ -343,7 +343,7 @@ type FilledPage struct { // switch result.ResultCode { // case CanAppend: -// waitCh := s.txlog.Append(txlog.AppendedMeasure{ +// waitCh := s.storage.Append(storage.AppendedMeasure{ // MetricID: req.MetricID, // Timestamp: req.Timestamp, // Value: req.Value, @@ -390,9 +390,9 @@ type FilledPage struct { // // } // // _ = report -// //waitCh := s.txlog.WriteAppended(path, report) +// //waitCh := s.storage.WriteAppended(path, report) -// waitCh := s.txlog.Append(txlog.AppendedPagesReq{ +// waitCh := s.storage.Append(storage.AppendedPagesReq{ // Legs: path.Legs, // MetricID: req.MetricID, // Timestamp: req.Timestamp, @@ -459,8 +459,8 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 { // _ = report // if len(toAppendMeasures) > 0 { - // waitCh := s.txlog.WriteAppendMeasures( - // txlog.AppendedMeasures{ + // waitCh := s.storage.WriteAppendMeasures( + // storage.AppendedMeasures{ // MetricID: req.MetricID, // Measures: toAppendMeasures, // }, @@ -500,7 +500,7 @@ func (s *Database) DeleteMeasures(req proto.DeleteMeasuresReq) uint16 { case DeleteFromAtreeNotNeeded: // регистрирую удаление в TransactionLog - s.txlog.Append(txlog.DeletedMeasures{ + s.storage.Append(storage.DeletedMeasures{ MetricID: req.MetricID, }) //<-waitCh @@ -512,7 +512,7 @@ func (s *Database) DeleteMeasures(req proto.DeleteMeasuresReq) uint16 { qb.Abort(qb.FailedAtreeRequest, err) } // регистрирую удаление в TransactionLog - s.txlog.Append(txlog.DeletedMeasures{ + s.storage.Append(storage.DeletedMeasures{ MetricID: req.MetricID, FreePageNumbers: pageNumbers, }) diff --git a/database/database.go b/database/database.go index 5d97dbb..e4199c7 100644 --- a/database/database.go +++ b/database/database.go @@ -12,7 +12,7 @@ import ( "gordenko.dev/dima/qb/atree" "gordenko.dev/dima/qb/freelist" - "gordenko.dev/dima/qb/txlog" + "gordenko.dev/dima/qb/storage" ) func JoinSnapshotFileName(dir string, snapshotNumber int) string { @@ -41,7 +41,7 @@ type Database struct { dir string databaseName string snapshotNumber int - txlog *txlog.Writer + storage *storage.Writer atree *atree.Atree tcpPort int logfile *os.File @@ -186,7 +186,7 @@ func (s *Database) recovery() { // logNumber = recipe.LogNumber // } - // s.txlog, err = txlog.NewWriter(txlog.WriterOptions{ + // s.storage, err = storage.NewWriter(storage.WriterOptions{ // Dir: s.dir, // LogNumber: logNumber, // AppendToWorkerQueue: s.appendJobToWorkerQueue, @@ -199,7 +199,7 @@ func (s *Database) recovery() { // qb.Abort(qb.CreateChangesWriterFailed, err) // } - // go s.txlog.Run() + // go s.storage.Run() // // fileNames, err := s.searchREDOFiles() // // if err != nil { @@ -249,7 +249,7 @@ func (s *Database) replayChanges(snapshotNumber int) (err error) { return } - walReader, err := txlog.NewReader(txlog.ReaderOptions{ + walReader, err := storage.NewWALReader(storage.WALReaderOptions{ FileName: JoinWALFileName(s.dir, snapshotNumber), BufferSize: 8 * 1024 * 1024, }) @@ -271,29 +271,21 @@ func (s *Database) replayChanges(snapshotNumber int) (err error) { if err != nil { return } + + err = s.storage.WritePagesToAtree(walReplayer.IndexPages(), walReplayer.DataPages()) + if err != nil { + return err + } + + // FIX free pages sync + return nil } -func (s *Database) rewritePagesFromLastWALPacket(list []any) error { - // var ( - // indexPages []txlog.PageToWrite - // dataPages []txlog.PageToWrite - // ) - // for _, untyped := range list { - // switch rec := untyped.(type) { - // case txlog.WALRecordAppendMeasures: - // metric, ok := s.metrics[rec.MetricID] - // if !ok { - // qb.Abort(qb.MetricNotFoundDuringReplay, nil) - // } - // //metric.ReplayAppendedMeasures(rec) - - // default: - // qb.Abort(qb.UnknownTxLogRecordTypeBug, - // fmt.Errorf("bug: unknown record type %T in TransactionLog", rec)) - // } - // } - return nil +func (s *Database) relayMetricsToMetrics(replayMetrics map[uint32]*ReplayMetric) { + for metricID, x := range replayMetrics { + s.metrics[metricID] = x.ToMetric() + } } // FIX @@ -314,14 +306,14 @@ func calculateDataBufferSize(payloadSize int) int { return bufferSize } } - return atree.DataPageSize + return storage.DataPageSize } func allocateBuffers(requiredSpace int) (buf, databuf []byte) { bufferSize := calculateDataBufferSize(requiredSpace) buf = make([]byte, bufferSize) - if bufferSize == atree.DataPageSize { - databuf = buf[:atree.DataPagePayloadSize] + if bufferSize == storage.DataPageSize { + databuf = buf[:storage.DataPagePayloadSize] } else { databuf = buf } @@ -343,6 +335,20 @@ func growBuffers(in growBuffersIn) (buf, databuf []byte) { return } +type shrinkBufferIn struct { + Buffer []byte + TimestampsSize int + ValuesSize int + NewBufferSize int +} + +func shrinkBuffer(in shrinkBufferIn) []byte { + buf := make([]byte, in.NewBufferSize) + copy(buf, in.Buffer[:in.ValuesSize]) + copy(buf[:len(buf)-in.TimestampsSize], in.Buffer[len(in.Buffer)-in.TimestampsSize:]) + return buf +} + //func (s *Database) verifySnapshot(fileName string) (_ bool, err error) { // file, err := os.Open(fileName) // if err != nil { diff --git a/database/database_test.go b/database/database_test.go index 4ea59db..bc25a75 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -6,6 +6,7 @@ import ( "testing" "gordenko.dev/dima/qb" + "gordenko.dev/dima/qb/storage" ) func TestWriteReadSnapshot(t *testing.T) { @@ -87,3 +88,26 @@ func cmpMetric(replay *ReplayMetric, origin *_metric) error { // } return nil } + +func TestComposeHeadIndexPage(t *testing.T) { + var ( + levelIdx = 0 + level = storage.IndexLevelTail{ + Buffer: []byte{ + 1, 2, 3, + }, + RecordsCount: 2, + } + head = &storage.HeadIndexPage{ + PageNo: 100, + Checksum: 12345, + Records: []byte{}, + } + ) + page := composeHeadIndexPage(levelIdx, level, head) + if page.PageNo != head.PageNo { + t.Fatalf("PageNo: got %d are not equal expected %v", + page.PageNo, head.PageNo) + } + // fix compare pages +} diff --git a/database/helpers.go b/database/helpers.go index 6a3a58a..3e9d503 100644 --- a/database/helpers.go +++ b/database/helpers.go @@ -6,17 +6,17 @@ import ( "os" "time" - diploma "gordenko.dev/dima/qb" + "gordenko.dev/dima/qb" "gordenko.dev/dima/qb/proto" ) -func timeBoundsOfAggregation(since, until proto.TimeBound, groupBy diploma.GroupBy, firstHourOfDay int) (s time.Time, u time.Time) { +func timeBoundsOfAggregation(since, until proto.TimeBound, groupBy qb.GroupBy, firstHourOfDay int) (s time.Time, u time.Time) { switch groupBy { - case diploma.GroupByHour, diploma.GroupByDay: + case qb.GroupByHour, qb.GroupByDay: s = time.Date(since.Year, since.Month, since.Day, 0, 0, 0, 0, time.Local) u = time.Date(until.Year, until.Month, until.Day, 0, 0, 0, 0, time.Local) - case diploma.GroupByMonth: + case qb.GroupByMonth: s = time.Date(since.Year, since.Month, 1, 0, 0, 0, 0, time.Local) u = time.Date(until.Year, until.Month, 1, 0, 0, 0, 0, time.Local) } diff --git a/database/metric.go b/database/metric.go index 6479d4a..bbfb8f1 100644 --- a/database/metric.go +++ b/database/metric.go @@ -5,8 +5,7 @@ import ( bin "gordenko.dev/dima/bin/little" "gordenko.dev/dima/qb" - "gordenko.dev/dima/qb/atree" - "gordenko.dev/dima/qb/txlog" + "gordenko.dev/dima/qb/storage" ) // METRIC @@ -31,7 +30,7 @@ type _metric struct { XLock bool RLocks int WaitQueue []any - indexLevelTails []txlog.IndexLevelTail // root - last element + indexLevelTails []storage.IndexLevelTail // root - last element } //IndexLevels [][]IndexRec // root - last element @@ -60,7 +59,7 @@ func (s *_metric) DeleteMeasures() { // s.UntilValue = 0 } -// func (s *_metric) startAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(txlog.AppendedMeasures)) { +// func (s *_metric) startAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(storage.AppendedMeasures)) { func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(any)) { var ( // AppendedMeasures struct { @@ -78,7 +77,7 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage fu timestamps = s.timestamps values = s.values - dataPages []txlog.DataPayload + dataPages []storage.DataPayload //written int //resultCode byte @@ -92,7 +91,7 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage fu // s.Since = measure.Timestamp // } else { // FIX - у випадку помилки треба в транзакції зберегти що помилка, але також зафіксувати скільки елементів збережено. - // якщо idx == 0 - одразу знімаю блокування і нічого не відправляю в txlog + // якщо idx == 0 - одразу знімаю блокування і нічого не відправляю в storage if measure.Timestamp <= s.timestamps.LastTimestamp() { if idx == 0 { s.values.ForgetCapturedState() @@ -136,7 +135,7 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage fu // накопичую timestamps.Compress(timestampCompressionWay, measure.Timestamp) values.Compress(valueCompressionWay, measure.Value) - } else if len(s.buffer) < atree.DataPagePayloadSize { + } else if len(s.buffer) < storage.DataPagePayloadSize { // allocate bigger buffer buffer := make([]byte, len(s.buffer)*2) // copy timestamps @@ -151,7 +150,7 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage fu // сторінка заповнена since := s.timestamps.ReplaceSinceWithUntil() - dataPages = append(dataPages, txlog.DataPayload{ + dataPages = append(dataPages, storage.DataPayload{ Since: since, Content: s.buffer, TimestampsSize: timestamps.Size(), @@ -180,8 +179,8 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage fu // скопіювати. Причому можна скопіювати зрізи chunks if len(dataPages) > 0 { - // пишу в txlog довгим шляхом через redo файл і запис в data файл - sendToStorage(txlog.AppendedMeasures{ + // пишу в storage довгим шляхом через redo файл і запис в data файл + sendToStorage(storage.AppendedMeasures{ MetricID: req.MetricID, LastPageNo: s.lastPageNo, TimestampsOffset: timestamps.Offset(), // state.Pos() з якої позиції дописувати дані на сторінку 0 (при відновленні) @@ -196,12 +195,12 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, sendToStorage fu ResultCh: nil, }) } else { - // короткий шлях - запис лише в txlog + // короткий шлях - запис лише в storage } } -func (s *_metric) FinAppendMeasures(rec txlog.AppendMeasuresSummary) { +func (s *_metric) FinAppendMeasures(rec storage.AppendMeasuresSummary) { // Видаляю state. Оригінальні Timestamps і Values вже мають останню версію s.values.ForgetCapturedState() s.timestamps.ForgetCapturedState() @@ -209,7 +208,7 @@ func (s *_metric) FinAppendMeasures(rec txlog.AppendMeasuresSummary) { if rec.LastPageNo > 0 { s.lastPageNo = rec.LastPageNo } - // В txlog я передав повний індекс. У нього додали елементи (можливо нові рівні). + // В storage я передав повний індекс. У нього додали елементи (можливо нові рівні). // Тому проста заміна s.indexLevelTails = rec.Index } @@ -388,20 +387,6 @@ func (s *_metric) WriteTo(w io.Writer) (err error) { return } -// var ( -// values qb.ValueCompressor -// ) -// if s.metricType == qb.Cumulative { -// values = enc.NewCumulativeDeltaCompressor(s.buffer, s.fracDigits) -// } else { -// values = enc.NewInstantDeltaCompressor(s.buffer, s.fracDigits) -// } -// values.RestoreState(valuesSize) -// s.timestamps = enc.NewTimeDeltaCompressor(s.buffer) -// s.timestamps.RestoreState(timestampsSize, until) -// s.values = values -// s.lastValue = s.values.LastValue() - // since - 4b // sinceValue - 8b - // untilValue - 8b diff --git a/database/proc.go b/database/proc.go index 122ee62..0e5eb7d 100644 --- a/database/proc.go +++ b/database/proc.go @@ -7,8 +7,8 @@ import ( "gordenko.dev/dima/qb/atree" "gordenko.dev/dima/qb/enc" "gordenko.dev/dima/qb/proto" + "gordenko.dev/dima/qb/storage" "gordenko.dev/dima/qb/transform" - "gordenko.dev/dima/qb/txlog" ) // AddMetric - create metric in map and set Xlock, rwad - append to queue @@ -83,7 +83,7 @@ func (s *Database) DoWork() { case tryAppendMeasuresReq: s.tryAppendMeasures(req) - case txlog.Changes: + case storage.Changes: s.applyChanges(req) // all metrics only case tryListCurrentValuesReq: @@ -132,7 +132,7 @@ func (s *Database) processMetricQueue(metricID uint32, metric *_metric) { s.tryGetMetric(req) case tryAppendMeasuresReq: - metric.StartAppendMeasures(req, s.txlog.Append) + metric.StartAppendMeasures(req, s.storage.Append) case tryDeleteMetricReq: s.startDeleteMetric(metric, req) @@ -294,7 +294,7 @@ func (s *Database) startDeleteMeasures(metric *_metric, req tryDeleteMeasuresReq // } } -func (s *Database) finAppendMeasures(rec txlog.AppendMeasuresSummary) { +func (s *Database) finAppendMeasures(rec storage.AppendMeasuresSummary) { metric, ok := s.metrics[rec.MetricID] if !ok { qb.Abort(qb.NoMetricBug, @@ -322,7 +322,7 @@ func (s *Database) tryAppendMeasures(req tryAppendMeasuresReq) { metric.WaitQueue = append(metric.WaitQueue, req) return } - metric.StartAppendMeasures(req, s.txlog.Append) + metric.StartAppendMeasures(req, s.storage.Append) } type tryRangeScanReq struct { @@ -408,25 +408,25 @@ func (s *Database) tryListCurrentValues(req tryListCurrentValuesReq) { /////////////////////////////////////////////////////// -func (s *Database) applyChanges(req txlog.Changes) { +func (s *Database) applyChanges(req storage.Changes) { for _, untyped := range req.Records { switch rec := untyped.(type) { - case txlog.AddedMetric: + case storage.MetricAddRecord: s.applyAddMetric(rec) - case txlog.DeletedMetric: + case storage.MetricDeleteRecord: s.deleteMetric(rec) - // case txlog.AppendedMeasure: + // case storage.AppendedMeasure: // s.appendMeasure(rec) - case txlog.AppendMeasuresSummary: + case storage.AppendMeasuresSummary: s.finAppendMeasures(rec) - // case txlog.AppendedMeasureWithOverflowExtended: + // case storage.AppendedMeasureWithOverflowExtended: // s.appendMeasureAfterOverflow(rec) - case txlog.DeletedMeasures: + case storage.DeletedMeasures: s.deleteMeasures(rec) } } @@ -436,7 +436,7 @@ func (s *Database) applyChanges(req txlog.Changes) { err := writeSnapshot(writeSnapshotIn{ SnapshotNumber: s.snapshotNumber, Dir: s.dir, - WriteBufferSize: 1 * 1024 * 1024, // 1mb + WriteBufferSize: 4 * 1024 * 1024, // 1mb Metrics: s.metrics, FrozenIndexPagesCount: req.FrozenIndexPagesCount, IndexPageNumbers: req.IndexPageNumbers, @@ -451,7 +451,7 @@ func (s *Database) applyChanges(req txlog.Changes) { } } -func (s *Database) applyAddMetric(rec txlog.AddedMetric) { +func (s *Database) applyAddMetric(rec storage.MetricAddRecord) { // fix lock _, ok := s.metrics[rec.MetricID] if ok { @@ -470,28 +470,26 @@ func (s *Database) applyAddMetric(rec txlog.AddedMetric) { // delete(s.metricLockEntries, rec.MetricID) } -func (s *Database) addMetric(rec txlog.AddedMetric) { +func (s *Database) addMetric(rec storage.MetricAddRecord) { var ( values qb.ValueCompressor buffer = make([]byte, minBufferSize) ) - if rec.MetricType == qb.Cumulative { - values = enc.NewCumulativeDeltaCompressor(buffer, byte(rec.FracDigits)) + values = enc.NewCumulativeDeltaCompressor(byte(rec.FracDigits), buffer, 0) } else { - values = enc.NewInstantDeltaCompressor(buffer, byte(rec.FracDigits)) + values = enc.NewInstantDeltaCompressor(byte(rec.FracDigits), buffer, 0) } - s.metrics[rec.MetricID] = &_metric{ metricType: rec.MetricType, fracDigits: byte(rec.FracDigits), buffer: buffer, - timestamps: enc.NewTimeDeltaCompressor(buffer), + timestamps: enc.NewTimeDeltaCompressor(buffer, 0), values: values, } } -func (s *Database) deleteMetric(rec txlog.DeletedMetric) { +func (s *Database) deleteMetric(rec storage.MetricDeleteRecord) { metric, ok := s.metrics[rec.MetricID] if !ok { qb.Abort(qb.NoMetricBug, @@ -552,7 +550,7 @@ func (s *Database) deleteMetric(rec txlog.DeletedMetric) { } delete(s.metrics, rec.MetricID) - // ADD in txlog + // ADD in storage // if len(rec.FreePageNumbers) > 0 { // s.freeList.AddPages(rec.FreePageNumbers) // } @@ -562,7 +560,7 @@ func (s *Database) deleteMetric(rec txlog.DeletedMetric) { } } -func (s *Database) deleteMeasures(rec txlog.DeletedMeasures) { +func (s *Database) deleteMeasures(rec storage.DeletedMeasures) { metric, ok := s.metrics[rec.MetricID] if !ok { qb.Abort(qb.NoMetricBug, @@ -577,7 +575,7 @@ func (s *Database) deleteMeasures(rec txlog.DeletedMeasures) { } metric.DeleteMeasures() metric.XLock = false - // FIX add in txlog + // FIX add in storage // if len(rec.FreePageNumbers) > 0 { // s.freeList.AddPages(rec.FreePageNumbers) // } diff --git a/database/replay_metric.go b/database/replay_metric.go index 8a9fbbc..7d1fb31 100644 --- a/database/replay_metric.go +++ b/database/replay_metric.go @@ -1,12 +1,13 @@ package database import ( + "fmt" "io" bin "gordenko.dev/dima/bin/little" - qb "gordenko.dev/dima/qb" - "gordenko.dev/dima/qb/atree" - "gordenko.dev/dima/qb/txlog" + "gordenko.dev/dima/qb" + "gordenko.dev/dima/qb/enc" + "gordenko.dev/dima/qb/storage" ) type ReplayMetric struct { @@ -17,7 +18,7 @@ type ReplayMetric struct { databuf []byte // якщо розмір buf == DataPageSize, то databuf буде менше (мінус футер) tSize int vSize int - indexLevelTails []txlog.IndexLevelTail // root - last element + indexLevelTails []storage.IndexLevelTail // root - last element } func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) { @@ -57,7 +58,7 @@ func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) { } for range levelsCount { var ( - buf = make([]byte, atree.IndexPageSize) + buf = make([]byte, storage.IndexPageSize) count int ) count, err = bin.ReadVarSize(r) @@ -68,7 +69,7 @@ func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) { if err != nil { return } - s.indexLevelTails = append(s.indexLevelTails, txlog.IndexLevelTail{ + s.indexLevelTails = append(s.indexLevelTails, storage.IndexLevelTail{ Buffer: buf, RecordsCount: count, }) @@ -76,115 +77,256 @@ func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) { return } +// func appendNewRecordsToIndexTail(level storage.IndexLevelTail, records []byte) { + +// } + +// func countReusedIndexPages(changedIndexLevels []storage.ChangedIndexLevel) (count int) { + +// return +// } + +func (s *ReplayMetric) MeasuresAppend(rec storage.MeasuresAppendRecord) { + timestampsSize := rec.TimestampsOffset - len(rec.Timestamps) + // копіюю нові дані із WAL з урахуванням offset-ів + copy(s.databuf[rec.ValuesOffset:], rec.Values) + copy(s.databuf[len(s.databuf)-timestampsSize:], rec.Timestamps) + // + s.tSize = timestampsSize + s.vSize = rec.ValuesOffset - len(rec.Values) +} + type AppendMeasuresResult struct { - IndexPages []txlog.PageToWrite - DataPages []txlog.PageToWrite + IndexPages []storage.PageToWrite + DataPages []storage.PageToWrite ReusedIndexPagesCount int ReusedDataPagesCount int } -func (s *ReplayMetric) AppendedMeasures(rec txlog.WALRecordAppendMeasures, pagesNeeded bool) (result AppendMeasuresResult) { +func (s *ReplayMetric) MeasuresAppendWithGrow(rec storage.MeasuresAppendWithGrowRecord, isLastPacket bool) (_ AppendMeasuresResult) { + var ( + reusedIndexPages int + reusedDataPages int + indexPages []storage.PageToWrite + dataPages []storage.PageToWrite + ) // Додати в index level tails недостаючі дані, або замінити for levelIdx, change := range rec.ChangedIndexLevels { + var ( + head = change.HeadIndexPage + newRecordsCount = len(change.TailRecords) / indexRecordSize + ) + // 1. набиваю сторінки для перезапису в index файлі + if isLastPacket { + if head != nil { + indexPages = append(indexPages, + composeHeadIndexPage(levelIdx, s.indexLevelTails[levelIdx], head)) + } + for _, x := range change.IndexPages { + indexPages = append(indexPages, storage.PageToWrite{ + PageNo: x.PageNo, + Content: x.Content, + }) + } + } + + // 2. Вношу зміни в поточні індексні хвости if levelIdx < len(s.indexLevelTails) { level := s.indexLevelTails[levelIdx] - if change.CompletedIndexPage != nil { - // попередній хвіст індексного рівня перетворився на сторінку, + if head != nil { + // попередній хвіст індексного рівня перетворився на head сторінку, // отже TailRecords - це новий хвіст - // або копіюю дані, або алокую більший буфер? copy(level.Buffer, change.TailRecords) - level.RecordsCount = len(change.TailRecords) / indexRecordSize + level.RecordsCount = newRecordsCount } else { // TailRecords - це нові дані, які треба додати pos := level.RecordsCount * indexRecordSize copy(level.Buffer[pos:], change.TailRecords) + level.RecordsCount += newRecordsCount } s.indexLevelTails[levelIdx] = level } else { - var ( - recordsCount = len(change.TailRecords) / indexRecordSize - buffer = make([]byte, atree.IndexPageSize) - ) - copy(buffer, change.TailRecords) - s.indexLevelTails = append(s.indexLevelTails, txlog.IndexLevelTail{ - Buffer: buffer, - RecordsCount: recordsCount, + // додаю новий індексний рівень + buf := make([]byte, storage.IndexPageSize) + copy(buf, change.TailRecords) + // + s.indexLevelTails = append(s.indexLevelTails, storage.IndexLevelTail{ + Buffer: buf, + RecordsCount: newRecordsCount, + }) + } + + // 3. рахую кількість reused індексних сторінок + if head != nil { + if head.Reused { + reusedIndexPages++ + } + } + for _, x := range change.IndexPages { + if x.Reused { + reusedIndexPages++ + } + } + } + + // 4. набиваю сторінки для перезапису в data файлі + if isLastPacket { + dataPages = append(dataPages, composeHeadDataPage(s.databuf, rec.HeadDataPage)) + + for _, x := range rec.DataPages { + dataPages = append(dataPages, storage.PageToWrite{ + PageNo: x.PageNo, + Content: x.Content, }) } } - // Перезаписати в buffer timestamps та values. Якщо розмір буферу недостатній - - // спочатку створити більший + // рахую кількість reused дата сторінок + if rec.HeadDataPage.Reused { + reusedDataPages++ + } + for _, x := range rec.DataPages { + if x.Reused { + reusedDataPages++ + } + } + // HeadDataPage є полюбому, оскільки це транзакція із мінімум однією заповненою + // data сторінкою. Отже TailTimestamps і TailValues - це нові хвости data рівня. + copy(s.databuf, rec.TailValues) + pos := len(s.databuf) - len(rec.TailTimestamps) + copy(s.databuf[pos:], rec.TailTimestamps) - var ( - // CompletedDataPage є полюбому, оскільки це Record із мінімум одною заповненою - // data сторінкою. Отже TailTimestamps і TailValues - це нові хвости data рівня. + s.tSize = len(rec.TailTimestamps) + s.vSize = len(rec.TailValues) - requiredSpace = len(rec.TailTimestamps) + len(rec.TailValues) - databuf []byte - ) - if requiredSpace < len(s.buf) { - databuf = s.buf - // fix - update sizes? + if len(rec.DataPages) == 0 { + s.lastPageNo = rec.HeadDataPage.PageNo } else { - var buf []byte - buf, databuf = allocateBuffers(requiredSpace) - copy(databuf, rec.TailValues) - pos := len(databuf) - len(rec.TailTimestamps) - copy(databuf[pos:], rec.TailTimestamps) - // fix - replace and remeber sizes? - - s.buf = buf + s.lastPageNo = rec.DataPages[len(rec.DataPages)-1].PageNo } - //rec.TailTimestamps - //rec.TailValues - return + return AppendMeasuresResult{ + IndexPages: indexPages, + DataPages: dataPages, + ReusedIndexPagesCount: reusedIndexPages, + ReusedDataPagesCount: reusedDataPages, + } } -func (s *ReplayMetric) ApplyHeadDataPage(p txlog.CompletedDataPage) { - // буфер має бути розміру сторінки - if len(s.buf) != atree.DataPageSize { - s.buf, s.databuf = growBuffers(growBuffersIn{ - databuf: s.databuf, - tSize: s.tSize, - vSize: s.vSize, - requiredSpace: atree.DataPageSize, +// HELPERS + +func composeHeadIndexPage(levelIdx int, level storage.IndexLevelTail, head *storage.HeadIndexPage) storage.PageToWrite { + // розраховую pos, з якого буду дописувати хвіст + pos := level.RecordsCount * indexRecordSize + // створюю копію сторінки + page := make([]byte, storage.IndexPageSize) + copy(page, level.Buffer[:pos]) // поточні дані + copy(page[pos:], head.Records) + // запечатати сторінку + calculatedCRC := storage.SealIndexPage(storage.SealIndexPageIn{ + Content: page, + RecordsCount: level.RecordsCount + len(head.Records)/indexRecordSize, + ZeroLevel: levelIdx == 0, + }) + // перевірка CRC + if calculatedCRC != head.Checksum { + qb.Abort(qb.WALReplayFailed, + fmt.Errorf("calculated CRC %d not equal expected %d of head page %d on index level %d", + calculatedCRC, head.Checksum, head.PageNo, levelIdx)) + } + return storage.PageToWrite{ + PageNo: head.PageNo, + Content: page, + } +} + +func composeHeadDataPage(databuf []byte, head storage.HeadDataPage) storage.PageToWrite { + // створюю копію сторінки + var ( + page = make([]byte, storage.DataPageSize) + timestampsSize = head.TimestampsOffset - len(head.Timestamps) + ) + // копіюю поточні дані + copy(page, databuf) + // копіюю нові дані із WAL з урахуванням offset-ів + copy(page[head.ValuesOffset:], head.Values) + copy(page[len(page)-timestampsSize:], head.Timestamps) + // запечатати сторінку + calculatedCRC := storage.SealDataPage(storage.SealDataPageIn{ + Content: page, + PrevPageNo: head.PrevPageNo, + TimestampsSize: timestampsSize, + ValuesSize: head.ValuesOffset + len(head.Values), + }) + // перевірка CRC + if calculatedCRC != head.Checksum { + qb.Abort(qb.WALReplayFailed, + fmt.Errorf("calculated CRC %d not equal expected %d of head data page %d", + calculatedCRC, head.Checksum, head.PageNo)) + } + return storage.PageToWrite{ + PageNo: head.PageNo, + Content: page, + } +} + +func (s *ReplayMetric) ToMetric() *_metric { + var ( + values qb.ValueCompressor + requiredSpace = s.tSize + s.vSize + ) + bufferSize := calculateDataBufferSize(requiredSpace) + if len(s.buf) > bufferSize { + s.buf = shrinkBuffer(shrinkBufferIn{ + Buffer: s.buf, + TimestampsSize: s.tSize, + ValuesSize: s.vSize, + NewBufferSize: bufferSize, }) } - // дописую дані по зміщенню - pos := len(s.databuf) - p.TimestampsOffset - len(p.Timestamps) - copy(s.databuf[pos:], p.Timestamps) - copy(s.databuf[p.ValuesOffset:], p.Values) - // - s.tSize = p.TimestampsOffset + len(p.Timestamps) - s.tSize = p.ValuesOffset + len(p.Values) - - // fix - заполнить страницу - // fix - check checksum + if s.metricType == qb.Cumulative { + values = enc.NewCumulativeDeltaCompressor(s.fracDigits, s.buf, s.vSize) + } else { + values = enc.NewInstantDeltaCompressor(s.fracDigits, s.buf, s.vSize) + } + return &_metric{ + metricType: s.metricType, + fracDigits: s.fracDigits, + lastPageNo: s.lastPageNo, + lastValue: values.LastValue(), + buffer: s.buf, + timestamps: enc.NewTimeDeltaCompressor(s.buf, s.tSize), + values: values, + indexLevelTails: s.indexLevelTails, + } } -// func (s *ReplayMetric) ApplyHeadIndexPages(changedIndexLevels txlog.ChangedIndexLevel) { +// func (s *ReplayMetric) ApplyHeadDataPage(p storage.HeadDataPage) { +// // буфер має бути розміру сторінки +// if len(s.buf) != atree.DataPageSize { +// s.buf, s.databuf = growBuffers(growBuffersIn{ +// databuf: s.databuf, +// tSize: s.tSize, +// vSize: s.vSize, +// requiredSpace: atree.DataPageSize, +// }) +// } +// // дописую дані по зміщенню +// pos := len(s.databuf) - p.TimestampsOffset - len(p.Timestamps) +// copy(s.databuf[pos:], p.Timestamps) +// copy(s.databuf[p.ValuesOffset:], p.Values) +// // +// s.tSize = p.TimestampsOffset + len(p.Timestamps) +// s.tSize = p.ValuesOffset + len(p.Values) + +// // fix - заполнить страницу +// // fix - check checksum +// } + +// func (s *ReplayMetric) ApplyHeadIndexPages(changedIndexLevels storage.ChangedIndexLevel) { // for levelIdx, changedLevel := range changedIndexLevels { -// p := changedLevel.CompletedIndexPage -// if p != nil { -// level := s.indexLevelTails[levelIdx] -// copy(level.Buffer[level.RecordsCount*indexRecordSize:], p.Records) +// -// // fix - заполнить страницу -// // fix - check checksum - -// s.indexPages = append(s.indexPages, txlog.PageToWrite{ -// PageNo: p.PageNo, -// Content: level.Buffer, -// }) -// } - -// for _, x := range changedLevel.IndexPages { -// s.indexPages = append(s.indexPages, txlog.PageToWrite{ -// PageNo: x.PageNo, -// Content: x.Content, -// }) -// } +// // } // } diff --git a/database/snapshot.go b/database/snapshot.go index 69d90a7..c2a1cff 100644 --- a/database/snapshot.go +++ b/database/snapshot.go @@ -8,7 +8,7 @@ import ( "path/filepath" bin "gordenko.dev/dima/bin/little" - "gordenko.dev/dima/qb/atree" + "gordenko.dev/dima/qb/storage" "gordenko.dev/dima/qb/util" ) @@ -177,11 +177,11 @@ func readSnapshot(fileName string) (out readSnapshotOut, err error) { for range metricsQty { var ( metricID uint32 - buf = make([]byte, atree.DataPageSize) + buf = make([]byte, storage.DataPageSize) // для простоти створюю буфер максимального розміру metric = &ReplayMetric{ buf: buf, - databuf: buf[:atree.DataPagePayloadSize], + databuf: buf[:storage.DataPagePayloadSize], } ) metricID, err = bin.ReadUint32(src) diff --git a/database/wal_replayer.go b/database/wal_replayer.go index bd4d80c..f332a4e 100644 --- a/database/wal_replayer.go +++ b/database/wal_replayer.go @@ -4,22 +4,20 @@ import ( "errors" "fmt" - qb "gordenko.dev/dima/qb" - "gordenko.dev/dima/qb/atree" - "gordenko.dev/dima/qb/txlog" + "gordenko.dev/dima/qb/storage" ) type WALReplayer struct { - walReader *txlog.Reader + walReader *storage.WALReader metrics map[uint32]*ReplayMetric freeIndexPages []uint32 freeDataPages []uint32 - indexPages []txlog.PageToWrite - dataPages []txlog.PageToWrite + indexPages []storage.PageToWrite + dataPages []storage.PageToWrite } type WALReplayerOptions struct { - WALReader *txlog.Reader + WALReader *storage.WALReader Metrics map[uint32]*ReplayMetric // from snapshot FreeIndexPages []uint32 // from snapshot FreeDataPages []uint32 // from snapshot @@ -40,122 +38,80 @@ func NewWALReplayer(opt WALReplayerOptions) (*WALReplayer, error) { }, nil } +func (s *WALReplayer) IndexPages() []storage.PageToWrite { + return s.indexPages +} + +func (s *WALReplayer) DataPages() []storage.PageToWrite { + return s.dataPages +} + func (s *WALReplayer) Replay() error { for { - records, isLast, err := s.walReader.NextPacket() + records, isLastPacket, err := s.walReader.NextPacket() if err != nil { return err } - - if isLast { - if len(records) > 0 { - var appendRecords []txlog.WALRecordAppendMeasures - for _, untyped := range records { - rec, ok := untyped.(txlog.WALRecordAppendMeasures) - if ok { - appendRecords = append(appendRecords, rec) - } - } - // prepare pages and rewrite - indexPages, dataPages := pagesToWriteFromWALRecords(metrics, appendRecords) - - err = s.txlog.WritePagesToAtree(indexPages, dataPages) - if err != nil { - return err - } - } - // для останнього rec - повторюємо запис сторінок - // rewrite write pages - //s.txlog. - //return nil - } - for _, rec := range records { - if err = s.replayWALRecord(rec); err != nil { + if err = s.replayRecord(rec, isLastPacket); err != nil { return err } } } } -// func (s *WALReplayer) replayPacket(records []any, isLastPacket bool) { -// for _, untyped := range records { -// switch rec := untyped.(type) { -// case txlog.WALRecordAppendMeasures: -// s.onWALRecordAppendMeasures(rec, isLastPacket) -// } -// } -// return -// } - -func (s *WALReplayer) replayWALRecord(untyped any) (err error) { +func (s *WALReplayer) replayRecord(untyped any, isLastPacket bool) (err error) { switch rec := untyped.(type) { - case txlog.AddedMetric: - if err = s.onAddedMetric(rec); err != nil { + case storage.MeasuresAppendRecord: + if err = s.onMeasuresAppend(rec); err != nil { return } - - case txlog.DeletedMetric: - if err = s.onDeletedMetric(rec); err != nil { + case storage.MeasuresAppendWithGrowRecord: + if err = s.onMeasuresAppendWithGrow(rec, isLastPacket); err != nil { return } + case storage.MetricAddRecord: + if err = s.onMetricAdd(rec); err != nil { + return + } + case storage.MetricDeleteRecord: + if err = s.onMetricDelete(rec); err != nil { + return + } + // case storage.DeletedMeasures: + // metric, ok := s.metrics[rec.MetricID] + // if ok { + // metric.DeleteMeasures() + // if len(rec.FreePageNumbers) > 0 { + // s.freeList.AddPages(rec.FreePageNumbers) + // } + // } - // case txlog.AppendedMeasure: - // metric, ok := s.metrics[rec.MetricID] - // if ok { - // metric.Timestamps.Append(rec.Timestamp) - // metric.Values.Append(rec.Value) - - // if metric.Since == 0 { - // metric.Since = rec.Timestamp - // metric.SinceValue = rec.Value - // } - - // metric.Until = rec.Timestamp - // metric.UntilValue = rec.Value - // } - - case txlog.WALRecordAppendMeasures: - // metric, ok := s.metrics[rec.MetricID] - // if !ok { - // qb.Abort(qb.MetricNotFoundDuringReplay, nil) - // } - //metric.ReplayAppendedMeasures(rec) FIX - - // case txlog.DeletedMeasures: - // metric, ok := s.metrics[rec.MetricID] - // if ok { - // metric.DeleteMeasures() - // if len(rec.FreePageNumbers) > 0 { - // s.freeList.AddPages(rec.FreePageNumbers) - // } - // } - - default: - qb.Abort(qb.UnknownTxLogRecordTypeBug, - fmt.Errorf("bug: unknown record type %T in TransactionLog", rec)) + // default: + // qb.Abort(qb.UnknownstorageRecordTypeBug, + // fmt.Errorf("bug: unknown record type %T in TransactionLog", rec)) } return nil } -func (s *WALReplayer) onAddedMetric(rec txlog.AddedMetric) (err error) { +func (s *WALReplayer) onMetricAdd(rec storage.MetricAddRecord) (err error) { _, ok := s.metrics[rec.MetricID] if ok { return fmt.Errorf("metric %d add failed: already added", rec.MetricID) } var ( - buf = make([]byte, atree.DataPageSize) + buf = make([]byte, storage.DataPageSize) ) s.metrics[rec.MetricID] = &ReplayMetric{ metricType: rec.MetricType, fracDigits: byte(rec.FracDigits), buf: buf, - databuf: buf[:atree.DataPagePayloadSize], + databuf: buf[:storage.DataPagePayloadSize], } return } -func (s *WALReplayer) onDeletedMetric(rec txlog.DeletedMetric) error { +func (s *WALReplayer) onMetricDelete(rec storage.MetricDeleteRecord) error { _, ok := s.metrics[rec.MetricID] if !ok { return fmt.Errorf("metric %d deletion failed: not found", rec.MetricID) @@ -166,15 +122,21 @@ func (s *WALReplayer) onDeletedMetric(rec txlog.DeletedMetric) error { return nil } -func (s *WALReplayer) onAppendMeasures(rec txlog.WALRecordAppendMeasures, isLastPacket bool) { +func (s *WALReplayer) onMeasuresAppend(rec storage.MeasuresAppendRecord) (err error) { metric, ok := s.metrics[rec.MetricID] if !ok { - qb.Abort(qb.MetricNotFoundDuringReplay, nil) + return fmt.Errorf("append measures failed: metric %d not found", rec.MetricID) } - if isLastPacket { - s.collectPagesToWrite(metric, rec) + metric.MeasuresAppend(rec) + return +} + +func (s *WALReplayer) onMeasuresAppendWithGrow(rec storage.MeasuresAppendWithGrowRecord, isLastPacket bool) (err error) { + metric, ok := s.metrics[rec.MetricID] + if !ok { + return fmt.Errorf("append measures failed: metric %d not found", rec.MetricID) } - result := metric.AppendedMeasures(rec, isLastPacket) + result := metric.MeasuresAppendWithGrow(rec, isLastPacket) // if isLastPacket { s.indexPages = append(s.indexPages, result.IndexPages...) @@ -183,22 +145,33 @@ func (s *WALReplayer) onAppendMeasures(rec txlog.WALRecordAppendMeasures, isLast // s.freeIndexPages = s.freeIndexPages[:len(s.freeIndexPages)-result.ReusedIndexPagesCount] s.freeDataPages = s.freeDataPages[:len(s.freeDataPages)-result.ReusedDataPagesCount] + return } -func (s *WALReplayer) collectPagesToWrite(metric *ReplayMetric, rec txlog.WALRecordAppendMeasures) { - metric.ApplyHeadDataPage(rec.CompletedDataPage) +//func (s *WALReplayer) collectPagesToWrite(metric *ReplayMetric, rec storage.WALRecordAppendMeasures) { +// metric.ApplyHeadDataPage(rec.CompletedDataPage) - s.dataPages = append(s.dataPages, txlog.PageToWrite{ - PageNo: p.PageNo, - Content: metric.buf, - }) +// s.dataPages = append(s.dataPages, storage.PageToWrite{ +// PageNo: p.PageNo, +// Content: metric.buf, +// }) - for _, x := range rec.DataPages { - s.dataPages = append(s.dataPages, txlog.PageToWrite{ - PageNo: x.PageNo, - Content: x.Content, - }) - } - //metric.ApplyHeadIndexPages(rec.ChangedIndexLevels) +// for _, x := range rec.DataPages { +// s.dataPages = append(s.dataPages, storage.PageToWrite{ +// PageNo: x.PageNo, +// Content: x.Content, +// }) +// } +//metric.ApplyHeadIndexPages(rec.ChangedIndexLevels) -} +//} + +// func (s *WALReplayer) replayPacket(records []any, isLastPacket bool) { +// for _, untyped := range records { +// switch rec := untyped.(type) { +// case storage.WALRecordAppendMeasures: +// s.onWALRecordAppendMeasures(rec, isLastPacket) +// } +// } +// return +// } diff --git a/enc/cumdelta.go b/enc/cumdelta.go index d3fde53..7558063 100644 --- a/enc/cumdelta.go +++ b/enc/cumdelta.go @@ -48,18 +48,15 @@ type CumulativeDeltaCompressor struct { } // Після відновлення із снапшота -func NewCumulativeDeltaCompressor(buf []byte, fracDigits byte) *CumulativeDeltaCompressor { +func NewCumulativeDeltaCompressor(fracDigits byte, buf []byte, payloadSize int) *CumulativeDeltaCompressor { var coef float64 = 1 if fracDigits > 0 { coef = math.Pow(10, float64(fracDigits)) } - return &CumulativeDeltaCompressor{ + s := &CumulativeDeltaCompressor{ buf: buf, coef: coef, } -} - -func (s *CumulativeDeltaCompressor) RestoreState(payloadSize int) { if payloadSize > 0 { s.pos = payloadSize - 1 // завжди показує на h // base value на початку @@ -74,16 +71,27 @@ func (s *CumulativeDeltaCompressor) RestoreState(payloadSize int) { log.Fatalf("bug: get last delta: %s", err) } } + return s } +// func (s *CumulativeDeltaCompressor) RestoreState() { + +// } + // func (s *CumulativeDeltaCompressor) Size() int { // return s.pos // } +type EvaluationReport struct { + CompressionWay int + TotalSpace int + AdditionalSpace int + Delta uint64 +} + func (s *CumulativeDeltaCompressor) Evaluate(value float64) (compressionWay int, requiredSpace int) { delta := uint64((value-s.baseValue)*s.coef + eps) // fix - if delta 0, no eps requiredSpace = s.pos - // fix - requiredSpace - all space if s.pos > 0 { if s.h < 128 { // run diff --git a/enc/enc_test.go b/enc/enc_test.go index ccdebe6..0b49485 100644 --- a/enc/enc_test.go +++ b/enc/enc_test.go @@ -165,10 +165,11 @@ func TestCumulativeDeltaCompressor(t *testing.T) { var ( fracDigits byte = 1 buf = make([]byte, 8) + payloadSize = 0 compressionWay int requiredSpace int ) - c := NewCumulativeDeltaCompressor(buf, fracDigits) + c := NewCumulativeDeltaCompressor(fracDigits, buf, payloadSize) for _, num := range testCase.Nums { compressionWay, requiredSpace = c.Evaluate(num) c.Compress(compressionWay, num) @@ -281,9 +282,10 @@ func TestCumulativeDeltaDecompressorFromState(t *testing.T) { var ( fracDigits byte = 1 buf = make([]byte, 16) + payloadSize = 0 decodedNums []float64 ) - c := NewCumulativeDeltaCompressor(buf, fracDigits) + c := NewCumulativeDeltaCompressor(fracDigits, buf, payloadSize) for _, num := range testCase.Nums { compressionWay, _ := c.Evaluate(num) c.Compress(compressionWay, num) @@ -524,10 +526,11 @@ func TestInstantDeltaCompressor(t *testing.T) { var ( fracDigits byte = 1 buf = make([]byte, 8) + payloadSize = 0 compressionWay int requiredSpace int ) - c := NewInstantDeltaCompressor(buf, fracDigits) + c := NewInstantDeltaCompressor(fracDigits, buf, payloadSize) for _, num := range testCase.Nums { compressionWay, requiredSpace = c.Evaluate(num) c.Compress(compressionWay, num) @@ -640,9 +643,10 @@ func TestInstantDeltaDecompressorFromState(t *testing.T) { var ( fracDigits byte = 1 buf = make([]byte, 16) + payloadSize = 0 decodedNums []float64 ) - c := NewInstantDeltaCompressor(buf, fracDigits) + c := NewInstantDeltaCompressor(fracDigits, buf, payloadSize) for _, num := range testCase.Nums { compressionWay, _ := c.Evaluate(num) c.Compress(compressionWay, num) @@ -840,10 +844,11 @@ func TestTimeDeltaCompressor(t *testing.T) { //fmt.Println("----------") var ( buf = make([]byte, 8) + payloadSize = 0 compressionWay int requiredSpace int ) - c := NewTimeDeltaCompressor(buf) + c := NewTimeDeltaCompressor(buf, payloadSize) for _, num := range testCase.Nums { compressionWay, requiredSpace = c.Evaluate(num) c.Compress(compressionWay, num) @@ -957,9 +962,10 @@ func TestTimeDeltaDecompressorFromState(t *testing.T) { for caseIdx, testCase := range testCases { var ( buf = make([]byte, 16) + payloadSize = 0 decodedNums []uint32 ) - c := NewTimeDeltaCompressor(buf) + c := NewTimeDeltaCompressor(buf, payloadSize) for _, num := range testCase.Nums { compressionWay, _ := c.Evaluate(num) c.Compress(compressionWay, num) diff --git a/enc/insdelta.go b/enc/insdelta.go index 991ad33..a2fc972 100644 --- a/enc/insdelta.go +++ b/enc/insdelta.go @@ -19,7 +19,7 @@ type InstantDeltaCompressor struct { state *InstantDeltaCapturedState } -func NewInstantDeltaCompressor(buf []byte, fracDigits byte) *InstantDeltaCompressor { +func NewInstantDeltaCompressor(fracDigits byte, buf []byte, payloadSize int) *InstantDeltaCompressor { var coef float64 = 1 if fracDigits > 0 { coef = math.Pow(10, float64(fracDigits)) @@ -28,10 +28,6 @@ func NewInstantDeltaCompressor(buf []byte, fracDigits byte) *InstantDeltaCompres buf: buf, coef: coef, } - return s -} - -func (s *InstantDeltaCompressor) RestoreState(payloadSize int) { if payloadSize > 0 { s.pos = payloadSize - 1 // завжди показує на h // base value на початку @@ -46,8 +42,13 @@ func (s *InstantDeltaCompressor) RestoreState(payloadSize int) { log.Fatalf("bug: get last delta: %s", err) } } + return s } +// func (s *InstantDeltaCompressor) RestoreState() { + +// } + // func (s *InstantDeltaCompressor) Size() int { // return s.pos // } diff --git a/enc/time_delta.go b/enc/time_delta.go index adc5f2d..86d0909 100644 --- a/enc/time_delta.go +++ b/enc/time_delta.go @@ -34,15 +34,12 @@ type TimeDeltaCompressor struct { // а lastDelta і h треба прочитати із payload (якщо є) // Кодує значення справа наліво функціями TailPut*. Читає значення зліва направо функціями Get* -func NewTimeDeltaCompressor(buf []byte) *TimeDeltaCompressor { - return &TimeDeltaCompressor{ +func NewTimeDeltaCompressor(buf []byte, payloadSize int) *TimeDeltaCompressor { + s := &TimeDeltaCompressor{ buf: buf, } -} - -func (s *TimeDeltaCompressor) RestoreState(payloadSize int, lastTimestamp uint32) { if payloadSize > 0 { - s.lastUnixtime = lastTimestamp + //s.lastUnixtime = lastTimestamp s.pos = len(s.buf) - payloadSize if payloadSize > 4 { s.h = s.buf[s.pos] @@ -53,8 +50,14 @@ func (s *TimeDeltaCompressor) RestoreState(payloadSize int, lastTimestamp uint32 s.lastDelta = uint32(u64) } } + return s } +// FIX - restire lastUnixtime +// func (s *TimeDeltaCompressor) RestoreState() { + +// } + // retrun func (s *TimeDeltaCompressor) Evaluate(unixtime uint32) (compressionWay int, requiredSpace int) { requiredSpace = len(s.buf) - s.pos diff --git a/examples/database/main.go b/examples/database/main.go index 95873b9..aaa884d 100644 --- a/examples/database/main.go +++ b/examples/database/main.go @@ -82,7 +82,7 @@ const mainUsage = `Usage: ` -const helpMessage = `Diploma project. Database. Version: 1.0 +const helpMessage = `Gordenko project. Database. Version: 1.0 created by Dmytro Gordenko, 1.e4.kc6@gmail.com ` diff --git a/examples/loadtest/loadtest.go b/examples/loadtest/loadtest.go index 58b6287..a89f124 100644 --- a/examples/loadtest/loadtest.go +++ b/examples/loadtest/loadtest.go @@ -7,7 +7,7 @@ import ( "os" "time" - diploma "gordenko.dev/dima/qb" + "gordenko.dev/dima/qb" "gordenko.dev/dima/qb/client" "gordenko.dev/dima/qb/proto" ) @@ -15,12 +15,12 @@ import ( // METRICS INFO type MetricInfo struct { - MetricID uint32 `json:"metricID"` - MetricType diploma.MetricType `json:"metricType"` - FracDigits int `json:"fracDigits"` - Since int64 `json:"since"` - Until int64 `json:"until"` - Qty int `json:"qty"` + MetricID uint32 `json:"metricID"` + MetricType qb.MetricType `json:"metricType"` + FracDigits int `json:"fracDigits"` + Since int64 `json:"since"` + Until int64 `json:"until"` + Qty int `json:"qty"` } func readMetricInfo(fileName string) (list []MetricInfo, err error) { @@ -42,12 +42,12 @@ type QueryRecipe struct { RangeCode int Since uint32 Until uint32 - GroupBy diploma.GroupBy + GroupBy qb.GroupBy } type RandomQueryGenerator struct { metrics []MetricInfo - groupByOptions []diploma.GroupBy + groupByOptions []qb.GroupBy listCurrentValuesProbability int listPeriodsProbability int timeRangeProbDistribution []int @@ -86,10 +86,10 @@ func NewRandomQueryGenerator(opt RandomQueryGeneratorOptions) *RandomQueryGenera return &RandomQueryGenerator{ metrics: opt.Metrics, - groupByOptions: []diploma.GroupBy{ - diploma.GroupByHour, - diploma.GroupByDay, - diploma.GroupByMonth, + groupByOptions: []qb.GroupBy{ + qb.GroupByHour, + qb.GroupByDay, + qb.GroupByMonth, }, listCurrentValuesProbability: opt.ListCurrentValuesProbability, listPeriodsProbability: opt.ListPeriodsProbability, @@ -108,7 +108,7 @@ func (s *RandomQueryGenerator) GetQueryRecipe() QueryRecipe { Method: listCurrentValues, } } else { - if metric.MetricType == diploma.Cumulative { + if metric.MetricType == qb.Cumulative { num = rand.Intn(100) if num < s.listPeriodsProbability { groupBy := s.groupByOptions[rand.Intn(len(s.groupByOptions))] @@ -118,10 +118,10 @@ func (s *RandomQueryGenerator) GetQueryRecipe() QueryRecipe { maxDays = 7 ) - if groupBy == diploma.GroupByDay { + if groupBy == qb.GroupByDay { minDays = 1 maxDays = 30 - } else if groupBy == diploma.GroupByMonth { + } else if groupBy == qb.GroupByMonth { minDays = 1 maxDays = 30 } @@ -164,10 +164,10 @@ func (s *RandomQueryGenerator) GetQueryRecipe() QueryRecipe { maxDays = 7 ) - if groupBy == diploma.GroupByDay { + if groupBy == qb.GroupByDay { minDays = 1 maxDays = 30 - } else if groupBy == diploma.GroupByMonth { + } else if groupBy == qb.GroupByMonth { minDays = 1 maxDays = 30 } @@ -350,7 +350,7 @@ func execQuery(conn *client.Connection, queryGenerator *RandomQueryGenerator, st Day: until.Day(), }, GroupBy: recipe.GroupBy, - AggregateFuncs: diploma.AggregateMin | diploma.AggregateMax | diploma.AggregateAvg, + AggregateFuncs: qb.AggregateMin | qb.AggregateMax | qb.AggregateAvg, }) elapsedTime = time.Since(t1) stat.ElapsedTime += elapsedTime diff --git a/examples/loadtest/main.go b/examples/loadtest/main.go index 94de4a4..fbcb847 100644 --- a/examples/loadtest/main.go +++ b/examples/loadtest/main.go @@ -203,7 +203,7 @@ const mainUsage = `Usage: ` -const helpMessage = `Diploma project. Load test. Version: 1.0 +const helpMessage = `Gordenko project. Load test. Version: 1.0 created by Dmytro Gordenko, 1.e4.kc6@gmail.com ` diff --git a/examples/requests/main.go b/examples/requests/main.go index 9d58c03..b508915 100644 --- a/examples/requests/main.go +++ b/examples/requests/main.go @@ -54,7 +54,7 @@ const mainUsage = `Usage: ` -const helpMessage = `Diploma project. Example requests. Version: 1.0 +const helpMessage = `Gordenko project. Example requests. Version: 1.0 created by Dmytro Gordenko, 1.e4.kc6@gmail.com ` diff --git a/examples/requests/requests.go b/examples/requests/requests.go index 0e4f7f4..191587b 100644 --- a/examples/requests/requests.go +++ b/examples/requests/requests.go @@ -5,7 +5,7 @@ import ( "log" "time" - diploma "gordenko.dev/dima/qb" + "gordenko.dev/dima/qb" "gordenko.dev/dima/qb/client" "gordenko.dev/dima/qb/proto" ) @@ -25,7 +25,7 @@ func sendRequests(conn *client.Connection) { err = conn.AddMetric(proto.AddMetricReq{ MetricID: instantMetricID, - MetricType: diploma.Instant, + MetricType: qb.Instant, FracDigits: fracDigits, }) if err != nil { @@ -114,8 +114,8 @@ GetMetric: Month: until.Month(), Day: until.Day(), }, - GroupBy: diploma.GroupByHour, - AggregateFuncs: diploma.AggregateMin | diploma.AggregateMax | diploma.AggregateAvg, + GroupBy: qb.GroupByHour, + AggregateFuncs: qb.AggregateMin | qb.AggregateMax | qb.AggregateAvg, }) if err != nil { log.Fatalf("conn.ListInstantPeriods: %s\n", err) @@ -143,8 +143,8 @@ GetMetric: Month: until.Month(), Day: until.Day(), }, - GroupBy: diploma.GroupByDay, - AggregateFuncs: diploma.AggregateMin | diploma.AggregateMax | diploma.AggregateAvg, + GroupBy: qb.GroupByDay, + AggregateFuncs: qb.AggregateMin | qb.AggregateMax | qb.AggregateAvg, }) if err != nil { log.Fatalf("conn.ListInstantPeriods: %s\n", err) @@ -172,8 +172,8 @@ GetMetric: Month: until.Month(), Day: until.Day(), }, - GroupBy: diploma.GroupByMonth, - AggregateFuncs: diploma.AggregateMin | diploma.AggregateMax | diploma.AggregateAvg, + GroupBy: qb.GroupByMonth, + AggregateFuncs: qb.AggregateMin | qb.AggregateMax | qb.AggregateAvg, }) if err != nil { log.Fatalf("conn.ListInstantPeriods: %s\n", err) @@ -208,7 +208,7 @@ GetMetric: err = conn.AddMetric(proto.AddMetricReq{ MetricID: cumulativeMetricID, - MetricType: diploma.Cumulative, + MetricType: qb.Cumulative, FracDigits: fracDigits, }) if err != nil { @@ -298,7 +298,7 @@ GetMetric: Month: until.Month(), Day: until.Day(), }, - GroupBy: diploma.GroupByHour, + GroupBy: qb.GroupByHour, }) if err != nil { log.Fatalf("conn.ListCumulativePeriods: %s\n", err) @@ -326,7 +326,7 @@ GetMetric: Month: until.Month(), Day: until.Day(), }, - GroupBy: diploma.GroupByDay, + GroupBy: qb.GroupByDay, }) if err != nil { log.Fatalf("conn.ListCumulativePeriods: %s\n", err) @@ -354,7 +354,7 @@ GetMetric: Month: until.Month(), Day: until.Day(), }, - GroupBy: diploma.GroupByMonth, + GroupBy: qb.GroupByMonth, }) if err != nil { log.Fatalf("conn.ListCumulativePeriods: %s\n", err) diff --git a/freelist/freelist.go b/freelist/freelist.go index a337c13..10dd8e8 100644 --- a/freelist/freelist.go +++ b/freelist/freelist.go @@ -1,12 +1,11 @@ package freelist import ( - "bytes" "fmt" "io" "os" - "gordenko.dev/dima/qb/bin" + bin "gordenko.dev/dima/bin/little" "gordenko.dev/dima/qb/util" ) @@ -158,12 +157,13 @@ func (s *FreeList) loadLastPage() error { return fmt.Errorf("read size %d bytes not equal page size %d", n, s.pageSize) } // check crc - checksum := bin.GetUint32(buf[0:]) + checksum, _ := bin.GetUint32(buf[0:]) if util.CalcChecksum(buf[crcSize:]) != checksum { return fmt.Errorf("page %d is corrupted", s.basePagesCount) } for i := crcSize; i < len(buf); i += ptrSize { - s.free = append(s.free, bin.GetUint32(buf[i:])) + pageNo, _ := bin.GetUint32(buf[i:]) + s.free = append(s.free, pageNo) } if s.deltaPagesCount > 0 { s.deltaPagesCount-- @@ -207,25 +207,25 @@ func (s *FreeList) GetPageNumber() (pageNo uint32, err error) { } // для відновлення недозаповненнох сторінки із снапшота -func (s *FreeList) Restore(serialized []byte) error { - if (len(serialized) % ptrSize) != 0 { - return fmt.Errorf("wrong size") - } - for i := 0; i < len(serialized); i += ptrSize { - s.free = append(s.free, bin.GetUint32(serialized[i:])) - } - fmt.Println(s.free) - return nil -} +// func (s *FreeList) Restore(serialized []byte) error { +// if (len(serialized) % ptrSize) != 0 { +// return fmt.Errorf("wrong size") +// } +// for i := 0; i < len(serialized); i += ptrSize { +// s.free = append(s.free, bin.GetUint32(serialized[i:])) +// } +// fmt.Println(s.free) +// return nil +// } // для запису в snapshot -func (s *FreeList) Serialize() ([]byte, error) { - w := bytes.NewBuffer(nil) - for _, pageNo := range s.free { - bin.WriteUint32(w, pageNo) - } - return w.Bytes(), nil -} +// func (s *FreeList) Serialize() ([]byte, error) { +// w := bytes.NewBuffer(nil) +// for _, pageNo := range s.free { +// bin.WriteUint32(w, pageNo) +// } +// return w.Bytes(), nil +// } func (s *FreeList) Cached() []uint32 { return s.free diff --git a/freelist/freelist_test.go b/freelist/freelist_test.go index 4c6e7e7..205c844 100644 --- a/freelist/freelist_test.go +++ b/freelist/freelist_test.go @@ -58,46 +58,46 @@ func TestSave(t *testing.T) { // } } -func TestDelta(t *testing.T) { - freeList, err := createFreeList() - if err != nil { - t.Fatal(err) - } +// func TestDelta(t *testing.T) { +// freeList, err := createFreeList() +// if err != nil { +// t.Fatal(err) +// } - serialized := []byte{9, 0, 0, 0, 10, 0, 0, 0} +// serialized := []byte{9, 0, 0, 0, 10, 0, 0, 0} - err = freeList.Restore(serialized) - if err != nil { - t.Fatal(err) - } +// err = freeList.Restore(serialized) +// if err != nil { +// t.Fatal(err) +// } - for range 3 { - pageNo, err := freeList.GetPageNumber() - fmt.Println(pageNo, err) - } +// for range 3 { +// pageNo, err := freeList.GetPageNumber() +// fmt.Println(pageNo, err) +// } - err = freeList.AddPageNumbers([]uint32{20, 21, 22}) - if err != nil { - t.Fatal(err) - } +// err = freeList.AddPageNumbers([]uint32{20, 21, 22}) +// if err != nil { +// t.Fatal(err) +// } - err = freeList.AddPageNumbers([]uint32{23, 24, 25, 26}) - if err != nil { - t.Fatal(err) - } +// err = freeList.AddPageNumbers([]uint32{23, 24, 25, 26}) +// if err != nil { +// t.Fatal(err) +// } - serialized, err = freeList.Serialize() - if err != nil { - t.Fatal(err) - } +// serialized, err = freeList.Serialize() +// if err != nil { +// t.Fatal(err) +// } - fmt.Println("free serialized:", serialized) +// fmt.Println("free serialized:", serialized) - err = freeList.Merge() - if err != nil { - t.Fatal(err) - } -} +// err = freeList.Merge() +// if err != nil { +// t.Fatal(err) +// } +// } // func TestGetFromFreeList(t *testing.T) { // freeList, err := createFreeList() diff --git a/proto/proto.go b/proto/proto.go index 40c79cb..ad6c25d 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -4,8 +4,8 @@ import ( "fmt" "time" - diploma "gordenko.dev/dima/qb" - "gordenko.dev/dima/qb/bin" + bin "gordenko.dev/dima/bin/little" + "gordenko.dev/dima/qb" "gordenko.dev/dima/qb/bufreader" ) @@ -72,7 +72,7 @@ func ErrorCodeToText(code uint16) string { type Metric struct { MetricID uint32 - MetricType diploma.MetricType + MetricType qb.MetricType FracDigits int } @@ -161,7 +161,7 @@ func ReadListCurrentValuesReq(r *bufreader.BufferedReader) (m ListCurrentValuesR type AddMetricReq struct { MetricID uint32 - MetricType diploma.MetricType + MetricType qb.MetricType FracDigits int } @@ -175,8 +175,8 @@ func ReadAddMetricReq(r *bufreader.BufferedReader) (m AddMetricReq, err error) { } func UnpackAddMetricReq(arr []byte) (m AddMetricReq) { - m.MetricID = bin.GetUint32(arr) - m.MetricType = diploma.MetricType(arr[4]) + m.MetricID, _ = bin.GetUint32(arr) + m.MetricType = qb.MetricType(arr[4]) m.FracDigits = int(arr[5]) return } @@ -209,8 +209,8 @@ func ReadDeleteMeasuresReq(r *bufreader.BufferedReader) (m DeleteMeasuresReq, er } func UnpackDeleteMeasuresReq(arr []byte) (m DeleteMeasuresReq) { - m.MetricID = bin.GetUint32(arr) - m.Since = bin.GetUint32(arr[4:]) + m.MetricID, _ = bin.GetUint32(arr) + m.Since, _ = bin.GetUint32(arr[4:]) return } @@ -230,9 +230,9 @@ func ReadAppendMeasureReq(r *bufreader.BufferedReader) (m AppendMeasureReq, err } func UnpackAppendMeasureReq(arr []byte) (m AppendMeasureReq) { - m.MetricID = bin.GetUint32(arr) - m.Timestamp = bin.GetUint32(arr[4:]) - m.Value = bin.GetFloat64(arr[8:]) + m.MetricID, _ = bin.GetUint32(arr) + m.Timestamp, _ = bin.GetUint32(arr[4:]) + m.Value, _ = bin.GetFloat64(arr[8:]) return } @@ -253,8 +253,8 @@ func ReadAppendMeasuresReq(r *bufreader.BufferedReader) (m AppendMeasuresReq, er return } - m.MetricID = bin.GetUint32(prefix[0:]) - qty := bin.GetUint16(prefix[4:]) + m.MetricID, _ = bin.GetUint32(prefix[0:]) + qty, _ := bin.GetUint16(prefix[4:]) for i := range int(qty) { var measure Measure @@ -280,21 +280,25 @@ type MetricMeasure struct { } func ReadAppendMeasurePerMetricReq(r *bufreader.BufferedReader) (measures []MetricMeasure, err error) { - qty, err := bin.ReadUint16(r) + count, err := bin.ReadUint16(r) if err != nil { return } - var tmp = make([]byte, 16) - for range int(qty) { - err = bin.ReadNInto(r, tmp) + for range count { + var m MetricMeasure + m.MetricID, err = bin.ReadUint32(r) if err != nil { return } - measures = append(measures, MetricMeasure{ - MetricID: bin.GetUint32(tmp[0:]), - Timestamp: bin.GetUint32(tmp[4:]), - Value: bin.GetFloat64(tmp[8:]), - }) + m.Timestamp, err = bin.ReadUint32(r) + if err != nil { + return + } + m.Value, err = bin.ReadFloat64(r) + if err != nil { + return + } + measures = append(measures, m) } return } @@ -341,9 +345,9 @@ func ReadListInstantMeasuresReq(r *bufreader.BufferedReader) (m ListInstantMeasu } func UnpackListInstantMeasuresReq(arr []byte) (m ListInstantMeasuresReq) { - m.MetricID = bin.GetUint32(arr[0:]) - m.Since = bin.GetUint32(arr[4:]) - m.Until = bin.GetUint32(arr[8:]) + m.MetricID, _ = bin.GetUint32(arr[0:]) + m.Since, _ = bin.GetUint32(arr[4:]) + m.Until, _ = bin.GetUint32(arr[8:]) return } @@ -363,9 +367,9 @@ func ReadListCumulativeMeasuresReq(r *bufreader.BufferedReader) (m ListCumulativ } func UnpackListCumulativeMeasuresReq(arr []byte) (m ListCumulativeMeasuresReq) { - m.MetricID = bin.GetUint32(arr) - m.Since = bin.GetUint32(arr[4:]) - m.Until = bin.GetUint32(arr[8:]) + m.MetricID, _ = bin.GetUint32(arr) + m.Since, _ = bin.GetUint32(arr[4:]) + m.Until, _ = bin.GetUint32(arr[8:]) return } @@ -373,7 +377,7 @@ type ListInstantPeriodsReq struct { MetricID uint32 Since TimeBound Until TimeBound - GroupBy diploma.GroupBy + GroupBy qb.GroupBy AggregateFuncs byte FirstHourOfDay int } @@ -388,18 +392,20 @@ func ReadListInstantPeriodsReq(r *bufreader.BufferedReader) (m ListInstantPeriod } func UnpackListInstantPeriodsReq(arr []byte) (m ListInstantPeriodsReq) { - m.MetricID = bin.GetUint32(arr) + m.MetricID, _ = bin.GetUint32(arr) + sinceYear, _ := bin.GetUint16(arr[4:]) + untilYear, _ := bin.GetUint16(arr[8:]) m.Since = TimeBound{ - Year: int(bin.GetUint16(arr[4:])), + Year: int(sinceYear), Month: time.Month(arr[6]), Day: int(arr[7]), } m.Until = TimeBound{ - Year: int(bin.GetUint16(arr[8:])), + Year: int(untilYear), Month: time.Month(arr[10]), Day: int(arr[11]), } - m.GroupBy = diploma.GroupBy(arr[12]) + m.GroupBy = qb.GroupBy(arr[12]) m.AggregateFuncs = arr[13] m.FirstHourOfDay = int(arr[14]) return @@ -409,7 +415,7 @@ type ListCumulativePeriodsReq struct { MetricID uint32 Since TimeBound Until TimeBound - GroupBy diploma.GroupBy + GroupBy qb.GroupBy FirstHourOfDay int } @@ -423,18 +429,20 @@ func ReadListCumulativePeriodsReq(r *bufreader.BufferedReader) (m ListCumulative } func UnpackListCumulativePeriodsReq(arr []byte) (m ListCumulativePeriodsReq) { - m.MetricID = bin.GetUint32(arr[0:]) + m.MetricID, _ = bin.GetUint32(arr[0:]) + sinceYear, _ := bin.GetUint16(arr[4:]) + untilYear, _ := bin.GetUint16(arr[8:]) m.Since = TimeBound{ - Year: int(bin.GetUint16(arr[4:])), + Year: int(sinceYear), Month: time.Month(arr[6]), Day: int(arr[7]), } m.Until = TimeBound{ - Year: int(bin.GetUint16(arr[8:])), + Year: int(untilYear), Month: time.Month(arr[10]), Day: int(arr[11]), } - m.GroupBy = diploma.GroupBy(arr[12]) + m.GroupBy = qb.GroupBy(arr[12]) m.FirstHourOfDay = int(arr[13]) return } diff --git a/qb.go b/qb.go index eeb2723..8f6f35c 100644 --- a/qb.go +++ b/qb.go @@ -24,7 +24,6 @@ const ( ) type TimestampCompressor interface { - RestoreState(int, uint32) // (payload size, last timestamp) // (timestamp) => compressionWay, requiredSpace Evaluate(uint32) (int, int) Compress(int, uint32) @@ -43,7 +42,6 @@ type TimestampCompressor interface { } type ValueCompressor interface { - RestoreState(int) // payload size // (value) => compressionWay, requiredSpace Evaluate(float64) (int, int) Compress(int, float64) @@ -110,6 +108,8 @@ const ( DeletePrevSnapshotFileFailed AbortCode = 35 MetricNotFoundDuringReplay AbortCode = 36 + + WALReplayFailed AbortCode = 37 ) func Abort(code AbortCode, err error) { diff --git a/storage/data_preparer.go b/storage/data_preparer.go new file mode 100644 index 0000000..e009abd --- /dev/null +++ b/storage/data_preparer.go @@ -0,0 +1,207 @@ +package storage + +import ( + "bytes" + "io" + + bin "gordenko.dev/dima/bin/little" + "gordenko.dev/dima/qb/util" +) + +type DataPreparer struct { + pagePreparer *PagePreparer + //dataPagePayloadBound int + w *bytes.Buffer + indexPages []PageToWrite + dataPages []PageToWrite + writeResults []any +} + +type DataPreparerOptions struct { + MinWALBufferSize int + PagePreparer *PagePreparer +} + +func NewDataPreparer(opt DataPreparerOptions) *DataPreparer { + buf := make([]byte, opt.MinWALBufferSize) + s := &DataPreparer{ + pagePreparer: opt.PagePreparer, + w: bytes.NewBuffer(buf), + } + return s +} + +type PreparedData struct { + Packet []byte // пакет для запису в WAL + WriteToIndex []PageToWrite + WriteToData []PageToWrite + WriteResults []any +} + +func (s *DataPreparer) Prepare(input []any) PreparedData { + s.w.Write([]byte{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, // size (max 9 byte) + 0, 0, 0, 0, // crc32 + }) + + hasher := util.NewHasher() + + w := io.MultiWriter(s.w, hasher) + + // 1. Пакую всі дані в WAL буфер запису + for _, untyped := range input { + switch x := untyped.(type) { + case MetricAddRecord: + x.Pack(w) + // case DeletedMetric: + // if len(x.FreePageNumbers) > 0 { + // s.freeList.AddPageNumbers(x.FreePageNumbers) + // } + case AppendedMeasures: + sealResult := s.pagePreparer.SealPages(SealPagesIn{ + LastPageNo: x.LastPageNo, + IndexLevelTails: x.IndexLevelTails, + DataPages: x.DataPages, + }) + + // сторінки для запису в index та data файли + for _, p := range sealResult.DataPages { + s.dataPages = append(s.dataPages, PageToWrite{ + PageNo: p.PageNo, + Content: p.Content, + }) + } + for _, level := range sealResult.IndexLevels { + for _, p := range level.IndexPages { + s.indexPages = append(s.indexPages, PageToWrite{ + PageNo: p.PageNo, + Content: p.Content, + }) + } + } + + var ( + // дані для запису в WAL + changedIndexPages []ChangedIndexLevel + // дані для Воркера + indexLevelTails []IndexLevelTail + ) + + for _, level := range sealResult.IndexLevels { + if len(level.IndexPages) == 0 && level.SkipRecords == level.TailRecordsCount { + break + } + var completedIndexPage *HeadIndexPage + if len(level.IndexPages) > 0 { + first := level.IndexPages[0] + skipSize := level.SkipRecords * indexRecordSize + completedIndexPage = &HeadIndexPage{ + PageNo: first.PageNo, + Reused: first.Reused, + Checksum: first.Checksum, + Records: first.Content[skipSize:], + } + } + changedIndexPages = append(changedIndexPages, ChangedIndexLevel{ + HeadIndexPage: completedIndexPage, + IndexPages: level.IndexPages[1:], + TailRecords: level.TailRecords[:level.TailRecordsCount*indexRecordSize], + }) + + indexLevelTails = append(indexLevelTails, IndexLevelTail{ + Buffer: level.TailRecords, + RecordsCount: level.TailRecordsCount, + }) + } + + first := sealResult.DataPages[0] + + rec := MeasuresAppendWithGrowRecord{ + MetricID: x.MetricID, + HeadDataPage: HeadDataPage{ + PageNo: first.PageNo, + Reused: first.Reused, + PrevPageNo: first.PrevPageNo, + Checksum: first.Checksum, + TimestampsOffset: x.TimestampsOffset, + Timestamps: x.Timestamps, + ValuesOffset: x.ValuesOffset, + Values: x.Values, + }, + DataPages: sealResult.DataPages[1:], + ChangedIndexLevels: changedIndexPages, + TailTimestamps: x.Timestamps, + TailValues: x.Values, + } + + rec.WriteTo(s.w) + + // Дані для Worker + s.writeResults = append(s.writeResults, AppendMeasuresSummary{ + MetricID: x.MetricID, + LastPageNo: sealResult.DataPages[len(sealResult.DataPages)-1].PageNo, + Index: indexLevelTails, + ResultCode: x.ResultCode, + WrittenCount: x.WrittenCount, + ResultCh: x.ResultCh, + }) + + //case DeletedMeasures: + //case DeletedMeasuresSince: + } + } + // 2. Додаю розмір пакету і чексуму + + //s.written += int64(len(packet)) + 12 + + packet := s.w.Bytes() + // write size + payloadSize := len(packet) - 13 + start := 9 - bin.CountVarSize(payloadSize) + bin.PutVarSize(packet[start:], payloadSize) + // checksum + bin.PutUint32(packet[9:], hasher.Sum32()) + // + return PreparedData{ + Packet: packet[start:], + WriteToIndex: s.indexPages, + WriteToData: s.dataPages, + WriteResults: s.writeResults, + } +} + +func (s *DataPreparer) Reset() { + s.w.Reset() + s.indexPages = nil + s.dataPages = nil + s.writeResults = nil +} + +// MeasuresToWrite +type AppendedMeasures struct { + MetricID uint32 + LastPageNo uint32 + TimestampsOffset int // offset on 1st page + Timestamps []byte + ValuesOffset int // offset on 1st page + Values []byte + IndexLevelTails []IndexLevelTail + DataPages []DataPayload + TailTimestamps []byte // data level tail + TailValues []byte // data level tail + ResultCode int + WrittenCount int + ResultCh chan struct{} +} + +// WRITE RESULTS + +// результат +type AppendMeasuresSummary struct { + MetricID uint32 + LastPageNo uint32 + Index []IndexLevelTail + ResultCode int + WrittenCount int + ResultCh chan struct{} +} diff --git a/storage/helpers.go b/storage/helpers.go new file mode 100644 index 0000000..2c47b00 --- /dev/null +++ b/storage/helpers.go @@ -0,0 +1,202 @@ +package storage + +import ( + "bytes" + + bin "gordenko.dev/dima/bin/little" +) + +const recordSize = 8 +const chunkSize = 24 + +func copyPayloadFromChunks(w *bytes.Buffer, chunks [][]byte, size int, offset int) { + bin.WriteUint16(w, uint16(size)) + var ( + chunkIdx = offset / chunkSize + byteIdx = offset % chunkSize + ) + for _, chunk := range chunks[chunkIdx:] { + available := len(chunk) - byteIdx + if available <= size { + w.Write(chunk[byteIdx:]) + size -= available + if size == 0 { + return + } + byteIdx = 0 + } else { + w.Write(chunk[byteIdx : byteIdx+size]) + return + } + } +} + +///////////////////////////////////////// + +// type DataPage struct { +// PageNo uint32 +// Reused bool +// PrevPageNo uint32 +// Checksum uint32 +// Timestamps []byte +// Values []byte +// } + +// type IndexPage struct { +// PageNo uint32 +// Reused bool +// Checksum uint32 +// Records []byte +// } + +// type IndexLevel struct { +// Pages []IndexPage +// Records []byte // unfilled +// } + +// Для декодінга +// type TxAppendedMeasures struct { +// MetricID uint32 +// DataPages []DataPage +// // offsets потрібні тому що дані не додаються в кінець, а перезаписують кілька +// // останніх байтів unfilled даних. Потрібно для коректного recovery. +// // Якщо є DataPages, то застосовуються для 1-ї Data сторінки, інакше - до timestamps і values (unfilled) +// TimestampsOffset int +// ValuesOffset int +// Timestamps []byte // unfilled +// Values []byte // unfilled +// IndexLevels []IndexLevel +// } + +// func (s *TxAppendedMeasures) Read(r *bytes.Buffer) (err error) { +// s.MetricID, err = bin.ReadUint32(r) +// if err != nil { +// return +// } +// dataPagesQty, err := bin.ReadVarSize(r) +// if err != nil { +// return +// } +// for range dataPagesQty { +// var ( +// p DataPage +// timestampsSize, valuesSize int +// //b byte +// ) +// p.PageNo, err = bin.ReadUint32(r) +// if err != nil { +// return +// } +// p.Reused, err = bin.ReadBool(r) +// if err != nil { +// return +// } +// p.PrevPageNo, err = bin.ReadUint32(r) +// if err != nil { +// return +// } +// p.Checksum, err = bin.ReadUint32(r) +// if err != nil { +// return +// } +// timestampsSize, err = bin.ReadUint16AsInt(r) +// if err != nil { +// return +// } +// p.Timestamps, err = bin.ReadN(r, timestampsSize) +// if err != nil { +// return +// } +// valuesSize, err = bin.ReadUint16AsInt(r) +// if err != nil { +// return +// } +// p.Values, err = bin.ReadN(r, valuesSize) +// if err != nil { +// return +// } +// s.DataPages = append(s.DataPages, p) +// } +// s.TimestampsOffset, err = bin.ReadUint16AsInt(r) +// if err != nil { +// return +// } +// s.ValuesOffset, err = bin.ReadUint16AsInt(r) +// if err != nil { +// return +// } +// var ( +// timestampsSize, valuesSize int +// ) +// timestampsSize, err = bin.ReadVarSize(r) +// if err != nil { +// return +// } +// s.Timestamps, err = bin.ReadN(r, timestampsSize) +// if err != nil { +// return +// } +// valuesSize, err = bin.ReadVarSize(r) +// if err != nil { +// return +// } +// s.Values, err = bin.ReadN(r, valuesSize) +// if err != nil { +// return +// } +// if len(s.DataPages) == 0 { +// return +// } +// indexLevelsQty, err := bin.ReadVarSize(r) +// if err != nil { +// return +// } +// for range indexLevelsQty { +// var ( +// indexPagesQty int +// level IndexLevel +// recordsQty int +// ) +// indexPagesQty, err = bin.ReadVarSize(r) +// if err != nil { +// return +// } +// for range indexPagesQty { +// var ( +// p IndexPage +// recordsQty int +// ) +// p.PageNo, err = bin.ReadUint32(r) +// if err != nil { +// return +// } +// p.Reused, err = bin.ReadBool(r) +// if err != nil { +// return +// } +// p.Checksum, err = bin.ReadUint32(r) +// if err != nil { +// return +// } +// recordsQty, err = bin.ReadVarSize(r) +// if err != nil { +// return +// } +// p.Records, err = bin.ReadN(r, recordsQty*recordSize) +// if err != nil { +// return +// } +// level.Pages = append(level.Pages, p) +// } +// recordsQty, err = bin.ReadVarSize(r) +// if err != nil { +// return +// } +// level.Records, err = bin.ReadN(r, recordsQty*recordSize) +// if err != nil { +// return +// } +// s.IndexLevels = append(s.IndexLevels, level) +// } +// return +// } diff --git a/storage/page_preparer.go b/storage/page_preparer.go new file mode 100644 index 0000000..afa86be --- /dev/null +++ b/storage/page_preparer.go @@ -0,0 +1,314 @@ +package storage + +import ( + "errors" + + bin "gordenko.dev/dima/bin/little" + "gordenko.dev/dima/qb" + "gordenko.dev/dima/qb/util" +) + +// Задача метода: +// - для data сторінок отримати pageNo та reused, задати sizes та prevPageNo, порахувати CRC32 +// - додати пари до index, згенерувати нові index сторінки +// levels []*IndexLevel, dataPages []*DataPage +// fix - reduce leves + +type PagePreparer struct { + dataChecksumIdx int + maxRecordsOnIndexPage int + indexPageIncSize int // кратно indexPageSize + timestampsSizeIdx int + valuesSizeIdx int + prevPageIdx int + indexRecordsCountIdx int + isZeroLevelIdx int + indexChecksumIdx int + getDataPageNumber func() (uint32, bool, error) + getIndexPageNumber func() (uint32, bool, error) +} + +type PagePreparerOptions struct { + IndexPageSize int + IndexPageIncSize int + DataPageSize int + GetDataPageNumber func() (uint32, bool, error) + GetIndexPageNumber func() (uint32, bool, error) +} + +func NewPagePreparer(opt PagePreparerOptions) (*PagePreparer, error) { + if opt.GetIndexPageNumber == nil { + return nil, errors.New("missing required option: GetIndexPageNumber") + } + if opt.GetDataPageNumber == nil { + return nil, errors.New("missing required option: GetDataPageNumber") + } + if (opt.IndexPageSize % opt.IndexPageIncSize) != 0 { + return nil, errors.New("IndexPageIncSize must multiple of IndexPageSize") + } + s := &PagePreparer{ + dataChecksumIdx: opt.DataPageSize - 4, + indexPageIncSize: opt.IndexPageIncSize, + timestampsSizeIdx: opt.DataPageSize - 6, + valuesSizeIdx: opt.DataPageSize - 8, + prevPageIdx: opt.DataPageSize - 12, + indexRecordsCountIdx: opt.IndexPageSize - 6, + isZeroLevelIdx: opt.IndexPageSize - 7, + indexChecksumIdx: opt.IndexPageSize - 4, + getIndexPageNumber: opt.GetIndexPageNumber, + getDataPageNumber: opt.GetDataPageNumber, + } + s.maxRecordsOnIndexPage = (opt.IndexPageSize - 7) / indexRecordSize + return s, nil +} + +type appendIndexRecordIn struct { + Records []byte + RecordsCount int + Timestamp uint32 + PageNo uint32 +} + +func (s *PagePreparer) appendIndexRecord(in appendIndexRecordIn) []byte { + if in.RecordsCount < s.maxRecordsOnIndexPage { + var ( + pos int + buf = in.Records + ) + if in.RecordsCount > 0 { + pos = in.RecordsCount * indexRecordSize + } else { + buf = make([]byte, s.indexPageIncSize) + } + bin.PutUint32(buf[pos:], in.Timestamp) + bin.PutUint32(buf[pos+4:], in.PageNo) + return buf + } + qb.Abort(qb.NoSpaceOnIndexPage, nil) + return nil +} + +type sealDataPageIn struct { + Content []byte + PrevPageNo uint32 + TimestampsSize int + ValuesSize int +} + +func (s *PagePreparer) sealDataPage(in sealDataPageIn) (checksum uint32) { + bin.PutUint16(in.Content[s.timestampsSizeIdx:], uint16(in.TimestampsSize)) + bin.PutUint16(in.Content[s.valuesSizeIdx:], uint16(in.ValuesSize)) + bin.PutUint32(in.Content[s.prevPageIdx:], in.PrevPageNo) + + checksum = util.CalcChecksum(in.Content[:s.dataChecksumIdx]) + bin.PutUint32(in.Content[s.dataChecksumIdx:], checksum) + return +} + +type sealIndexPageIn struct { + Content []byte + RecordsCount int + LastLevel bool +} + +func (s *PagePreparer) sealIndexPage(in sealIndexPageIn) (checksum uint32) { + bin.PutUint16(in.Content[s.indexRecordsCountIdx:], uint16(in.RecordsCount)) + if in.LastLevel { + in.Content[s.isZeroLevelIdx] = 1 + } + checksum = util.CalcChecksum(in.Content[:s.indexChecksumIdx]) + bin.PutUint32(in.Content[s.indexChecksumIdx:], checksum) + return +} + +type IndexLevelTail struct { + Buffer []byte // розмір більший за кількість + RecordsCount int +} + +type DataPayload struct { + Since uint32 + Content []byte + TimestampsSize int + ValuesSize int +} + +type SealPagesIn struct { + LastPageNo uint32 + IndexLevelTails []IndexLevelTail + DataPages []DataPayload +} + +type SealedDataPage struct { + PageNo uint32 + Reused bool + PrevPageNo uint32 + Checksum uint32 + Content []byte +} + +type SealedIndexPage struct { + Content []byte + PageNo uint32 + Reused bool + Checksum uint32 +} + +// SealedIndexLevel - як зрозуміти чи потрібно щось писати в WAL чи index файл, +// чи додавання нових data сторінок не зачепило індексний рівень? +// Якщо є IndexPages - пишемо весь рівень в WAL і готові сторінки в index файл. +// Якщо IndexPages пустий - порівнюємо Offset і len(Payload) - якщо довжина +// більша за змішення - дані додані на незаповнену сторінку рівня, +// отже різницю пишемо в WAL. +type SealedIndexLevel struct { + // Якщо є IndexPages, offset - це зміщення із якого починаються дані першої сторінки, + // які ще не зписані в WAL. + // Якщо IndexPages пустий, offset - це зміщення із якого починаються дані payload, + // які ще не зписані в WAL. + // В payload в будь-якому разі дані останньої індексної сторінки рівня (незаповненої). + SkipRecords int + IndexPages []SealedIndexPage + TailRecords []byte + TailRecordsCount int +} + +type SealResult struct { + DataPages []SealedDataPage + IndexLevels []*SealedIndexLevel +} + +func (s *PagePreparer) SealPages(in SealPagesIn) (result SealResult) { + var ( + sealedDataPages []SealedDataPage + sealedIndexLevels []*SealedIndexLevel + ) + for _, tail := range in.IndexLevelTails { + sealedIndexLevels = append(sealedIndexLevels, &SealedIndexLevel{ + SkipRecords: tail.RecordsCount, + TailRecords: tail.Buffer, + TailRecordsCount: tail.RecordsCount, + }) + } + for dataPageIdx, d := range in.DataPages { + var ( + upTimestamp = d.Since + upPageNo uint32 + levelIdx = 0 + prevPageNo uint32 + ) + if dataPageIdx == 0 { + prevPageNo = in.LastPageNo + } else { + prevPageNo = result.DataPages[dataPageIdx-1].PageNo + } + pageNo, reused, err := s.getDataPageNumber() + if err != nil { + qb.Abort(qb.FailedGetPageNumber, err) + } + sealedDataPages = append(sealedDataPages, SealedDataPage{ + Content: d.Content, + PrevPageNo: prevPageNo, + PageNo: pageNo, + Reused: reused, + Checksum: s.sealDataPage(sealDataPageIn{ + Content: d.Content, + PrevPageNo: prevPageNo, + TimestampsSize: d.TimestampsSize, + ValuesSize: d.ValuesSize, + }), + }) + upPageNo = pageNo + for { + if levelIdx == len(sealedIndexLevels) { + // новий root + sealedIndexLevels = append(sealedIndexLevels, &SealedIndexLevel{ + TailRecords: s.appendIndexRecord(appendIndexRecordIn{ + Records: nil, + Timestamp: upTimestamp, + PageNo: upPageNo, + }), + TailRecordsCount: 1, + }) + break + } + level := sealedIndexLevels[levelIdx] + if level.TailRecordsCount < s.maxRecordsOnIndexPage { + level.TailRecords = s.appendIndexRecord(appendIndexRecordIn{ + Records: level.TailRecords, + Timestamp: upTimestamp, + PageNo: upPageNo, + }) + level.TailRecordsCount++ + break + } + // в буфері немає місця для додавання нової пари, отже це + // заповнена сторінка + pageNo, reused, err := s.getIndexPageNumber() + if err != nil { + qb.Abort(qb.FailedGetPageNumber, err) + } + + filled := SealedIndexPage{ + Content: level.TailRecords, + PageNo: pageNo, + Reused: reused, + Checksum: s.sealIndexPage(sealIndexPageIn{ + Content: level.TailRecords, + RecordsCount: level.TailRecordsCount, + LastLevel: levelIdx == 0, + }), + } + level.IndexPages = append(level.IndexPages, filled) + // новий tail на індексному рівні + level.TailRecords = s.appendIndexRecord(appendIndexRecordIn{ + Records: nil, + Timestamp: upTimestamp, + PageNo: upPageNo, + }) + level.TailRecordsCount = 1 + // + upPageNo = pageNo + upTimestamp, _ = bin.GetUint32(filled.Content) + levelIdx++ + } + } + return SealResult{ + DataPages: sealedDataPages, + IndexLevels: sealedIndexLevels, + } +} + +// HELPERS + +type SealDataPageIn struct { + Content []byte + PrevPageNo uint32 + TimestampsSize int + ValuesSize int +} + +func SealDataPage(in SealDataPageIn) (checksum uint32) { + // bin.PutUint16(in.Content[s.timestampsSizeIdx:], uint16(in.TimestampsSize)) + // bin.PutUint16(in.Content[s.valuesSizeIdx:], uint16(in.ValuesSize)) + // bin.PutUint32(in.Content[s.prevPageIdx:], in.PrevPageNo) + + // checksum = util.CalcChecksum(in.Content[:s.dataChecksumIdx]) + // bin.PutUint32(in.Content[s.dataChecksumIdx:], checksum) + return +} + +type SealIndexPageIn struct { + Content []byte + RecordsCount int + ZeroLevel bool +} + +func SealIndexPage(in SealIndexPageIn) (checksum uint32) { + // bin.PutUint16(in.Content[s.indexRecordsCountIdx:], uint16(in.RecordsCount)) + // if in.LastLevel { + // in.Content[s.isZeroLevelIdx] = 1 + // } + // checksum = util.CalcChecksum(in.Content[:s.indexChecksumIdx]) + // bin.PutUint32(in.Content[s.indexChecksumIdx:], checksum) + return +} diff --git a/storage/storage.go b/storage/storage.go new file mode 100644 index 0000000..98a5aa3 --- /dev/null +++ b/storage/storage.go @@ -0,0 +1,32 @@ +package storage + +const ( + + //PageNoSize = 4 + + // data page + DataPageSize = 8192 + DataPageFooterSize int = 12 + DataPagePayloadSize int = DataPageSize - DataPageFooterSize + + dataCRC32Idx = DataPageSize - 4 + timestampsSizeIdx = DataPageSize - 6 + valuesSizeIdx = DataPageSize - 8 + prevPageIdx = DataPageSize - 12 + + // index page + IndexPageSize = 1024 + IndexPageFooterSize = 7 + indexRecordSize = 8 + maxRecordsOnIndexPage = (IndexPageSize - IndexPageFooterSize) / indexRecordSize + + //indexPageIncSize = IndexPageIncSize + indexCRC32Idx = IndexPageSize - 4 + indexRecordsCountIdx = IndexPageSize - 6 + isZeroLevelIdx = IndexPageSize - 7 + + // timestampSize = 4 + // pairSize = timestampSize + PageNoSize + // indexFooterIdx = indexRecordsQtyIdx + // dataFooterIdx = timestampsSizeIdx +) diff --git a/storage/wal_reader.go b/storage/wal_reader.go new file mode 100644 index 0000000..4aac585 --- /dev/null +++ b/storage/wal_reader.go @@ -0,0 +1,362 @@ +package storage + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "os" + + bin "gordenko.dev/dima/bin/little" + "gordenko.dev/dima/qb/util" +) + +const readBufferSize = 8 * 1024 * 1024 + +type WALReader struct { + file *os.File + reader *bufio.Reader + current []any + next []any +} + +type WALReaderOptions struct { + FileName string + BufferSize int +} + +func NewWALReader(opt WALReaderOptions) (*WALReader, error) { + if opt.FileName == "" { + return nil, errors.New("FileName option is required") + } + if opt.BufferSize <= 0 { + return nil, errors.New("BufferSize option is required") + } + + file, err := os.Open(opt.FileName) + if err != nil { + return nil, err + } + + return &WALReader{ + file: file, + reader: bufio.NewReaderSize(file, readBufferSize), + }, nil +} + +func (s *WALReader) Close() { + s.file.Close() +} + +func (s *WALReader) NextPacket() ([]any, bool, error) { + var err error + if s.current == nil { + s.current, err = s.readPacket() + if err != nil { + return nil, false, err + } + if len(s.current) > 0 { + s.next, err = s.readPacket() + if err != nil { + return nil, false, err + } + } + } + + current := s.current + done := s.next == nil + + s.current = s.next + s.next, err = s.readPacket() + if err != nil { + return nil, false, err + } + return current, done, nil +} + +func (s *WALReader) readPacket() (_ []any, err error) { + payloadSize, err := bin.ReadVarSize(s.reader) + if err != nil { + if err == io.EOF { + return nil, nil + } else { + return + } + } + + storedCRC, err := bin.ReadUint32(s.reader) + if err != nil { + return + } + + body, err := bin.ReadN(s.reader, int(payloadSize)) + if err != nil { + return + } + + hasher := util.NewHasher() + hasher.Write(body) + + calculatedCRC := hasher.Sum32() + + if calculatedCRC == storedCRC { + return s.parseRecords(body) + } + //s.logger.Printf("stored CRC %d != calculated CRC %d", storedCRC, calculatedCRC) + return nil, nil +} + +func (s *WALReader) parseRecords(body []byte) ([]any, error) { + var ( + src = bytes.NewBuffer(body) + records []any + ) + + for { + recordType, err := src.ReadByte() + if err != nil { + if err == io.EOF { + return records, nil + } + return nil, err + } + + switch recordType { + case CodeMetricAdd: + rec := new(MetricAddRecord) + if err = rec.Parse(src); err != nil { + return nil, err + } + records = append(records, rec) + + case CodeMetricDelete: + rec := new(MetricDeleteRecord) + if err = rec.Parse(src); err != nil { + return nil, err + } + records = append(records, rec) + + // case CodeAppendedMeasure: + // rec := new(AppendedMeasure) + // if err = rec.Parse(src); err != nil { + // return nil, err + // } + // records = append(records, rec) + + // case CodeAppendedMeasures: + // rec := new(AppendedMeasures) + // if err = rec.Parse(src); err != nil { + // return nil, err + // } + // records = append(records, rec) + + // case CodeAppendedPages: + // rec := new(AppendedPages) + // if err = rec.Parse(src); err != nil { + // return nil, err + // } + // records = append(records, rec) + + case CodeMeasuresDelete: + rec := new(DeletedMeasures) + if err = rec.Parse(src); err != nil { + return nil, err + } + records = append(records, rec) + + default: + return nil, fmt.Errorf("unknown record type code: %d", recordType) + } + } +} + +// func (s *Reader) readAddedMetric(src *bytes.Buffer) (_ AddedMetric, err error) { +// arr, err := bin.ReadN(src, 6) +// if err != nil { +// return +// } +// return AddedMetric{ +// MetricID: bin.GetUint32(arr), +// MetricType: diploma.MetricType(arr[4]), +// FracDigits: int(arr[5]), +// }, nil +// } + +// func (s *Reader) readDeletedMetric(src *bytes.Buffer) (_ DeletedMetric, err error) { +// var rec DeletedMetric +// rec.MetricID, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// // free data pages +// dataQty, _, err := bin.ReadVarUint64(src) +// if err != nil { +// return +// } +// for range dataQty { +// var pageNo uint32 +// pageNo, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// rec.FreeDataPages = append(rec.FreeDataPages, pageNo) +// } +// // free index pages +// indexQty, _, err := bin.ReadVarUint64(src) +// if err != nil { +// return +// } +// for range indexQty { +// var pageNo uint32 +// pageNo, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// rec.FreeIndexPages = append(rec.FreeIndexPages, pageNo) +// } +// return rec, nil +// } + +// func (s *Reader) readAppendedMeasure(src *bytes.Buffer) (_ AppendedMeasure, err error) { +// arr, err := bin.ReadN(src, 16) +// if err != nil { +// return +// } +// return AppendedMeasure{ +// MetricID: bin.GetUint32(arr[0:]), +// Timestamp: bin.GetUint32(arr[4:]), +// Value: bin.GetFloat64(arr[8:]), +// }, nil +// } + +// func (s *Reader) readAppendedMeasures(src *bytes.Buffer) (_ AppendedMeasures, err error) { +// var rec AppendedMeasures +// rec.MetricID, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// qty, err := bin.ReadUint16(src) +// if err != nil { +// return +// } +// for range qty { +// var measure proto.Measure +// measure.Timestamp, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// measure.Value, err = bin.ReadFloat64(src) +// if err != nil { +// return +// } +// rec.Measures = append(rec.Measures, measure) +// } +// return rec, nil +// } + +// func (s *Reader) readAppendedMeasureWithOverflow(src *bytes.Buffer) (_ AppendedMeasureWithOverflow, err error) { +// var ( +// b byte +// rec AppendedMeasureWithOverflow +// ) +// rec.MetricID, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// rec.Timestamp, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// rec.Value, err = bin.ReadFloat64(src) +// if err != nil { +// return +// } +// b, err = src.ReadByte() +// if err != nil { +// return +// } +// rec.IsDataPageReused = b == 1 + +// rec.DataPageNo, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// b, err = src.ReadByte() +// if err != nil { +// return +// } +// if b == 1 { +// rec.IsRootChanged = true +// rec.RootPageNo, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// } +// // index pages +// indexQty, err := src.ReadByte() +// if err != nil { +// return +// } +// for range indexQty { +// var pageNo uint32 +// pageNo, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// rec.ReusedIndexPages = append(rec.ReusedIndexPages, pageNo) +// } +// return rec, nil +// } + +// func (s *Reader) readDeletedMeasures(src *bytes.Buffer) (_ DeletedMeasures, err error) { +// var ( +// rec DeletedMeasures +// ) +// rec.MetricID, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// // free data pages +// rec.FreeDataPages, err = s.readFreePageNumbers(src) +// if err != nil { +// return +// } +// // free index pages +// rec.FreeIndexPages, err = s.readFreePageNumbers(src) +// if err != nil { +// return +// } +// return rec, nil +// } + +// HELPERS + +// func (s *Reader) readFreePageNumbers(src *bytes.Buffer) ([]uint32, error) { +// var freePages []uint32 +// qty, _, err := bin.ReadVarUint64(src) +// if err != nil { +// return nil, err +// } +// for range qty { +// var pageNo uint32 +// pageNo, err = bin.ReadUint32(src) +// if err != nil { +// return nil, err +// } +// freePages = append(freePages, pageNo) +// } +// return freePages, nil +// } + +func (s *WALReader) Seek(offset int64) error { + ret, err := s.file.Seek(offset, 0) + if err != nil { + return err + } + + if ret != offset { + return fmt.Errorf("ret %d != offset %d", ret, offset) + } + return nil +} diff --git a/storage/wal_records.go b/storage/wal_records.go new file mode 100644 index 0000000..ef6fe1f --- /dev/null +++ b/storage/wal_records.go @@ -0,0 +1,666 @@ +package storage + +import ( + "bytes" + "io" + + bin "gordenko.dev/dima/bin/little" + "gordenko.dev/dima/qb" +) + +type HeadDataPage struct { + PageNo uint32 + Reused bool + PrevPageNo uint32 + Checksum uint32 + TimestampsOffset int // offset on 1st page + Timestamps []byte + ValuesOffset int // offset on 1st page + Values []byte +} + +type HeadIndexPage struct { + PageNo uint32 + Reused bool + Checksum uint32 + Records []byte // offset не потрібен, оскільки Records додаються в кінець +} + +type ChangedIndexLevel struct { + HeadIndexPage *HeadIndexPage + IndexPages []SealedIndexPage + TailRecords []byte // payload only +} + +type MeasuresAppendWithGrowRecord struct { + MetricID uint32 + HeadDataPage HeadDataPage + DataPages []SealedDataPage + ChangedIndexLevels []ChangedIndexLevel + TailTimestamps []byte // data level tail + TailValues []byte // data level tail +} + +/* +Format appended measures: +1b - tx type +4b - metricID +4b - pageNo +1b - reused +4b - prevPageNo +4b - page crc32 +Nb - (varsize) timestamps offset +Nb - (varsize) timestamps tail size on 1st filled data page +Nb - timestamps tail on 1st filled data page +Nb - (varsize) values offset +Nb - (varsize) values tail size on 1st filled data page +Nb - values tail on 1st filled data page + +Nb - (varsize) filled data pages qty +[ + + 4b - pageNo + 1b - reused + Nb - (data page size) page content + +] +Nb - (varsize) timestamps payload size on tail +Nb - timestamps payload +Nb - (varsize) values payload size on tail +Nb - values payload +Nb - (varsize) qty of index levels +[ + + // NOTE! + // - idx = 0 - zeroLevel + // - skipped = maxRecordsOnIndexPage - records count + 4b - pageNo of 1st index page + 1b - reused + 4b - page crc32 + Nb - (varsize) records tail count on 1st filled index page + Nb - records tail on 1st filled index page + + + Nb - (varsize) qty of level filled pages + [ + 4b - pageNo + 1b - reused + Nb - (index page size) page content + ] + Nb - (varsize) size of records level tail + Nb - level tail records + +] +*/ +func (s MeasuresAppendWithGrowRecord) WriteTo(w *bytes.Buffer) { + w.WriteByte(CodeMeasuresAppendWithGrow) + bin.WriteUint32(w, s.MetricID) + // completed data page + completed := s.HeadDataPage + bin.WriteUint32(w, completed.PageNo) + bin.WriteBool(w, completed.Reused) + bin.WriteUint32(w, completed.PrevPageNo) + bin.WriteUint32(w, completed.Checksum) + bin.WriteVarSize(w, completed.TimestampsOffset) + bin.WriteVarSize(w, len(completed.Timestamps)) + w.Write(completed.Timestamps) + bin.WriteVarSize(w, completed.ValuesOffset) + bin.WriteVarSize(w, len(completed.Values)) + w.Write(completed.Values) + // data pages + bin.WriteVarSize(w, len(s.DataPages)) + for _, p := range s.DataPages { + bin.WriteUint32(w, p.PageNo) + bin.WriteBool(w, p.Reused) + bin.WriteVarSize(w, len(p.Content)) + w.Write(p.Content) + } + // data tail + bin.WriteVarSize(w, len(s.TailTimestamps)) + w.Write(s.TailTimestamps) + bin.WriteVarSize(w, len(s.TailValues)) + w.Write(s.TailValues) + // changed levels + bin.WriteVarSize(w, len(s.ChangedIndexLevels)) + for _, level := range s.ChangedIndexLevels { + // completed index page + completed := level.HeadIndexPage + bin.WriteUint32(w, completed.PageNo) + bin.WriteBool(w, completed.Reused) + bin.WriteUint32(w, completed.Checksum) + bin.WriteVarSize(w, len(completed.Records)/indexRecordSize) + w.Write(completed.Records) + // full index pages + for _, p := range level.IndexPages { + bin.WriteUint32(w, p.PageNo) + bin.WriteBool(w, p.Reused) + bin.WriteVarSize(w, len(p.Content)) + w.Write(p.Content) + } + // level tail + bin.WriteVarSize(w, len(level.TailRecords)/indexRecordSize) + w.Write(level.TailRecords) + } +} + +func (s *MeasuresAppendWithGrowRecord) Parse(r *bytes.Buffer) (err error) { + var ( + size int + recordsCount int + ) + s.MetricID, err = bin.ReadUint32(r) + if err != nil { + return + } + s.HeadDataPage.PageNo, err = bin.ReadUint32(r) + if err != nil { + return + } + s.HeadDataPage.Reused, err = bin.ReadBool(r) + if err != nil { + return + } + s.HeadDataPage.PrevPageNo, err = bin.ReadUint32(r) + if err != nil { + return + } + s.HeadDataPage.Checksum, err = bin.ReadUint32(r) + if err != nil { + return + } + s.HeadDataPage.TimestampsOffset, err = bin.ReadVarSize(r) + if err != nil { + return + } + size, err = bin.ReadVarSize(r) + if err != nil { + return + } + s.HeadDataPage.Timestamps, err = bin.ReadN(r, size) + if err != nil { + return + } + size, err = bin.ReadVarSize(r) + if err != nil { + return + } + s.HeadDataPage.Values, err = bin.ReadN(r, size) + if err != nil { + return + } + // data pages + dataPagesQty, err := bin.ReadVarSize(r) + if err != nil { + return + } + for range dataPagesQty { + var ( + p SealedDataPage + ) + p.PageNo, err = bin.ReadUint32(r) + if err != nil { + return + } + p.Reused, err = bin.ReadBool(r) + if err != nil { + return + } + size, err = bin.ReadVarSize(r) + if err != nil { + return + } + p.Content, err = bin.ReadN(r, size) + if err != nil { + return + } + s.DataPages = append(s.DataPages, p) + } + // data tail + size, err = bin.ReadVarSize(r) + if err != nil { + return + } + s.TailTimestamps, err = bin.ReadN(r, size) + if err != nil { + return + } + size, err = bin.ReadVarSize(r) + if err != nil { + return + } + s.TailValues, err = bin.ReadN(r, size) + if err != nil { + return + } + // changed levels + changedIndexLevelsCount, err := bin.ReadVarSize(r) + if err != nil { + return + } + for range changedIndexLevelsCount { + var ( + indexPagesCount int + level ChangedIndexLevel + ) + // completed index page + level.HeadIndexPage.PageNo, err = bin.ReadUint32(r) + if err != nil { + return + } + level.HeadIndexPage.Reused, err = bin.ReadBool(r) + if err != nil { + return + } + level.HeadIndexPage.Checksum, err = bin.ReadUint32(r) + if err != nil { + return + } + recordsCount, err = bin.ReadVarSize(r) + if err != nil { + return + } + level.HeadIndexPage.Records, err = bin.ReadN(r, recordsCount*indexRecordSize) + if err != nil { + return + } + // + indexPagesCount, err = bin.ReadVarSize(r) + if err != nil { + return + } + for range indexPagesCount { + var p SealedIndexPage + p.PageNo, err = bin.ReadUint32(r) + if err != nil { + return + } + p.Reused, err = bin.ReadBool(r) + if err != nil { + return + } + p.Checksum, err = bin.ReadUint32(r) + if err != nil { + return + } + size, err = bin.ReadVarSize(r) + if err != nil { + return + } + p.Content, err = bin.ReadN(r, size) + if err != nil { + return + } + level.IndexPages = append(level.IndexPages, p) + } + recordsCount, err = bin.ReadVarSize(r) + if err != nil { + return + } + level.TailRecords, err = bin.ReadN(r, recordsCount*indexRecordSize) + if err != nil { + return + } + s.ChangedIndexLevels = append(s.ChangedIndexLevels, level) + } + return +} + +type MeasuresAppendRecord struct { + MetricID uint32 + TimestampsOffset int + Timestamps []byte + ValuesOffset int + Values []byte +} + +func (s MeasuresAppendRecord) WriteTo(w *bytes.Buffer) { + w.WriteByte(CodeMeasuresAppendWithGrow) + bin.WriteUint32(w, s.MetricID) + bin.WriteVarSize(w, s.TimestampsOffset) + bin.WriteVarSize(w, len(s.Timestamps)) + w.Write(s.Timestamps) + bin.WriteVarSize(w, s.ValuesOffset) + bin.WriteVarSize(w, len(s.Values)) + w.Write(s.Values) +} + +func (s *MeasuresAppendRecord) Parse(r *bytes.Buffer) (err error) { + var size int + s.MetricID, err = bin.ReadUint32(r) + if err != nil { + return + } + s.TimestampsOffset, err = bin.ReadVarSize(r) + if err != nil { + return + } + size, err = bin.ReadVarSize(r) + if err != nil { + return + } + s.Timestamps, err = bin.ReadN(r, size) + if err != nil { + return + } + s.ValuesOffset, err = bin.ReadVarSize(r) + if err != nil { + return + } + size, err = bin.ReadVarSize(r) + if err != nil { + return + } + s.Values, err = bin.ReadN(r, size) + if err != nil { + return + } + return +} + +type MetricAddRecord struct { + MetricID uint32 + MetricType qb.MetricType + FracDigits int +} + +func (s MetricAddRecord) Pack(w io.Writer) { + arr := []byte{ + CodeMetricAdd, + 0, 0, 0, 0, // + byte(s.MetricType), + byte(s.FracDigits), + } + bin.PutUint32(arr[1:], s.MetricID) + w.Write(arr) +} + +func (s *MetricAddRecord) Parse(src io.Reader) (err error) { + arr, err := bin.ReadN(src, 6) + if err != nil { + return + } + s.MetricID, _ = bin.GetUint32(arr) + s.MetricType = qb.MetricType(arr[4]) + s.FracDigits = int(arr[5]) + return nil +} + +type MetricDeleteRecord struct { + MetricID uint32 + FreeIndexPages []uint32 + FreeDataPages []uint32 +} + +func (s MetricDeleteRecord) Pack(w io.Writer) { + arr := []byte{ + CodeMetricDelete, + 0, 0, 0, 0, // metricID + } + bin.PutUint32(arr[1:], s.MetricID) + w.Write(arr) + // fix + // bin.WriteVarSize(w, len(s.FreePageNumbers)) + // for _, pageNo := range s.FreePageNumbers { + // bin.WriteUint32(w, pageNo) + // } +} + +func (s *MetricDeleteRecord) Parse(src io.Reader) (err error) { + s.MetricID, err = bin.ReadUint32(src) + if err != nil { + return + } + // free pages + // fix + // qty, err := bin.ReadVarSize(src) + // if err != nil { + // return + // } + // for range qty { + // var pageNo uint32 + // pageNo, err = bin.ReadUint32(src) + // if err != nil { + // return + // } + // //s.FreePageNumbers = append(s.FreePageNumbers, pageNo) + // } + return nil +} + +// fix - add changed pages +type DeletedMeasures struct { + MetricID uint32 + FreePageNumbers []uint32 +} + +func (s DeletedMeasures) Pack(w io.Writer) { + arr := []byte{ + CodeMeasuresDelete, + 0, 0, 0, 0, + } + bin.PutUint32(arr[1:], s.MetricID) + w.Write(arr) + bin.WriteVarSize(w, len(s.FreePageNumbers)) + for _, pageNo := range s.FreePageNumbers { + bin.WriteUint32(w, pageNo) + } +} + +func (s *DeletedMeasures) Parse(src io.Reader) (err error) { + s.MetricID, err = bin.ReadUint32(src) + if err != nil { + return + } + // free pages + qty, err := bin.ReadVarSize(src) + if err != nil { + return + } + for range qty { + var pageNo uint32 + pageNo, err = bin.ReadUint32(src) + if err != nil { + return + } + s.FreePageNumbers = append(s.FreePageNumbers, pageNo) + } + return nil +} + +// type AppendedMeasure struct { +// MetricID uint32 +// Timestamp uint32 +// Value float64 +// } + +// func (s AppendedMeasure) Pack(w io.Writer) { +// arr := []byte{ +// CodeAppendedMeasure, +// 0, 0, 0, 0, // metricID +// 0, 0, 0, 0, // timestamp +// 0, 0, 0, 0, 0, 0, 0, 0, // value +// } +// bin.PutUint32(arr[1:], s.MetricID) +// bin.PutUint32(arr[5:], s.Timestamp) +// bin.PutFloat64(arr[9:], s.Value) +// w.Write(arr) +// } + +// func (s *AppendedMeasure) Parse(src io.Reader) (err error) { +// arr, err := bin.ReadN(src, 16) +// if err != nil { +// return +// } +// s.MetricID, _ = bin.GetUint32(arr[0:]) +// s.Timestamp, _ = bin.GetUint32(arr[4:]) +// s.Value, _ = bin.GetFloat64(arr[8:]) +// return nil +// } + +// type AppendedMeasures struct { +// MetricID uint32 +// Measures []proto.Measure +// } + +// func (s AppendedMeasures) Pack(w io.Writer) { +// arr := []byte{ +// CodeAppendedMeasures, +// 0, 0, 0, 0, // metricID +// 0, 0, // qty +// } +// bin.PutUint32(arr[1:], s.MetricID) +// bin.PutUint16(arr[5:], uint16(len(s.Measures))) +// w.Write(arr) +// for _, measure := range s.Measures { +// bin.WriteUint32(w, measure.Timestamp) +// bin.WriteFloat64(w, measure.Value) +// } +// } + +// func (s *AppendedMeasures) Parse(src io.Reader) (err error) { +// s.MetricID, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// qty, err := bin.ReadUint16(src) +// if err != nil { +// return +// } +// for range qty { +// var measure proto.Measure +// measure.Timestamp, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// measure.Value, err = bin.ReadFloat64(src) +// if err != nil { +// return +// } +// s.Measures = append(s.Measures, measure) +// } +// return nil +// } + +// type AppendedPagesReq struct { +// Legs []atree.PathLeg +// MetricID uint32 +// Timestamp uint32 // last measure +// Value float64 // last measure +// //RootPageNo uint32 +// LastPageNo uint32 +// Pages []atree.NotLinkedDataPage +// TimestampsChunks [][]byte +// TimestampsSize uint16 +// ValuesChunks [][]byte +// ValuesSize uint16 +// } + +/* +4b metricID +4b timestamp +8b value +4b LastPageNo (0 if no new root page) +4b newRootPageNo (0 if no new root page) +Nb - varsize (pages qty) +[ + + 4b pageNo + 1b isReused + Xb page (page size) + +] +2b - timestamps size (not filled page) +Nb - timestamps payload +2b - values size (not filled page) +Nb - values payload +*/ +// type AppendedPages struct { +// MetricID uint32 +// Timestamp uint32 // last measure +// Value float64 // last measure +// NewRootPageNo uint32 +// LastPageNo uint32 +// Pages []atree.PageToWrite +// TimestampsChunks [][]byte +// TimestampsSize int +// ValuesChunks [][]byte +// ValuesSize int +// } + +// func (s *AppendedPages) Pack(w io.Writer) { +// w.Write([]byte{ +// CodeAppendedPages, +// }) +// bin.WriteUint32(w, s.MetricID) +// bin.WriteUint32(w, s.Timestamp) +// bin.WriteFloat64(w, s.Value) +// bin.WriteUint32(w, s.LastPageNo) +// bin.WriteUint32(w, s.NewRootPageNo) +// // +// bin.WriteVarSize(w, len(s.Pages)) +// for _, p := range s.Pages { +// bin.WriteUint32(w, p.PageNo) +// if p.IsReused { +// w.Write([]byte{1}) +// } else { +// w.Write([]byte{0}) +// } +// w.Write(p.Data) +// } +// // timestamps +// writeChunkedPayloadTo(s.TimestampsChunks, int(s.TimestampsSize), w) +// // values +// writeChunkedPayloadTo(s.ValuesChunks, int(s.ValuesSize), w) +// } + +// func (s *AppendedPages) Parse(src io.Reader) (err error) { +// s.MetricID, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// s.Timestamp, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// s.Value, err = bin.ReadFloat64(src) +// if err != nil { +// return +// } +// s.LastPageNo, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// s.NewRootPageNo, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// pagesQty, err := bin.ReadVarSize(src) +// if err != nil { +// return +// } +// for range pagesQty { +// var page atree.PageToWrite +// page.PageNo, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// page.IsReused, err = bin.ReadBool(src) +// if err != nil { +// return +// } +// page.Data, err = bin.ReadN(src, atree.PageSize) +// if err != nil { +// return +// } +// s.Pages = append(s.Pages, page) +// } +// s.TimestampsChunks, s.TimestampsSize, err = readChunkedPayload(src) +// if err != nil { +// return +// } +// s.ValuesChunks, s.ValuesSize, err = readChunkedPayload(src) +// if err != nil { +// return +// } +// return nil +// } diff --git a/storage/wal_writer.go b/storage/wal_writer.go new file mode 100644 index 0000000..8b674cf --- /dev/null +++ b/storage/wal_writer.go @@ -0,0 +1,464 @@ +package storage + +import ( + "bytes" + "errors" + "fmt" + "log" + "math" + "os" + "path/filepath" + "sync" + + "gordenko.dev/dima/qb" + "gordenko.dev/dima/qb/atree" + "gordenko.dev/dima/qb/freelist" +) + +const ( + lsnSize = 4 + packetPrefixSize = 12 // 4 lsn + 4 packet length + 4 crc32 + + lengthIdx = 0 + checksumIdx = 4 + lsnIdx = 8 + + filePerm = 0770 + + dumpSnapshotAfterNBytes = 1024 * 1024 * 1024 // 1 GB + + writeBufferSize = 4 * 1024 * 1024 +) + +const ( + CodeMetricAdd byte = 1 + CodeMetricDelete byte = 2 + CodeMeasuresAppendWithGrow byte = 5 + CodeMeasuresAppend = 6 + CodeMeasuresDelete byte = 7 +) + +type FreeList interface { + // використовується в allocPage + GetPageNumber() (uint32, error) +} + +func JoinChangesFileName(dir string, logNumber int) string { + return filepath.Join(dir, fmt.Sprintf("%d.changes", logNumber)) +} + +type MetricsState struct { + //Metrics []Metric + WaitCh chan struct{} +} + +type Changes struct { + Records []any + SnapshotNumberCh chan int // log number + FrozenIndexPagesCount int + IndexPageNumbers []uint32 + FrozenDataPagesCount int + DataPageNumbers []uint32 +} + +type Writer struct { + mutex sync.Mutex + dataPagesCount uint32 + indexPagesCount uint32 + dataFreeList *freelist.FreeList + indexFreeList *freelist.FreeList + atree *atree.Atree + //logNumber int + dir string + w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск + wal *os.File + dataFile *os.File + indexFile *os.File + indexPageSize int + dataPageSize int + input []any + dataPreparer *DataPreparer + appendToWorkerQueue func(any) + //lsn uint32 + written int64 + isExited bool + exitCh chan struct{} + waitGroup *sync.WaitGroup + signalCh chan struct{} +} + +type WriterOptions struct { + IndexPageSize int + DataPageSize int + Dir string + SnapshotNumber int // номер журнала + AppendToWorkerQueue func(any) + DataFreeList *freelist.FreeList + IndexFreeList *freelist.FreeList + Atree *atree.Atree + ExitCh chan struct{} + WaitGroup *sync.WaitGroup +} + +func NewWriter(opt WriterOptions) (*Writer, error) { + if (opt.IndexPageSize % 2) != 0 { + return nil, errors.New("IndexPageSize must be multiple of 2") + } + if (opt.DataPageSize % 2) != 0 { + return nil, errors.New("DataPageSize must be multiple of 2") + } + if opt.Dir == "" { + return nil, errors.New("Dir option is required") + } + if opt.AppendToWorkerQueue == nil { + return nil, errors.New("AppendToWorkerQueue option is required") + } + if opt.DataFreeList == nil { + return nil, errors.New("DataFreeList option is required") + } + if opt.IndexFreeList == nil { + return nil, errors.New("IndexFreeList option is required") + } + if opt.Atree == nil { + return nil, errors.New("Atree option is required") + } + if opt.ExitCh == nil { + return nil, errors.New("ExitCh option is required") + } + if opt.WaitGroup == nil { + return nil, errors.New("WaitGroup option is required") + } + + s := &Writer{ + indexPageSize: opt.IndexPageSize, + dataPageSize: opt.DataPageSize, + dir: opt.Dir, + appendToWorkerQueue: opt.AppendToWorkerQueue, + dataFreeList: opt.DataFreeList, + indexFreeList: opt.IndexFreeList, + atree: opt.Atree, + exitCh: opt.ExitCh, + waitGroup: opt.WaitGroup, + signalCh: make(chan struct{}, 1), + } + + var err error + + if opt.SnapshotNumber <= 0 { + log.Fatalln("FIX FUCK NUMBER") + } + s.wal, err = os.OpenFile( + JoinChangesFileName(opt.Dir, opt.SnapshotNumber), + os.O_APPEND|os.O_WRONLY, + filePerm, + ) + if err != nil { + return nil, err + } + return s, nil +} + +func (s *Writer) Run() { + for { + select { + case <-s.signalCh: + if err := s.packAndWrite(); err != nil { + qb.Abort(qb.FailedWriteToTxLog, err) + } + + case <-s.exitCh: + s.exit() + return + } + } +} + +func (s *Writer) getDataPageNumber() (uint32, bool, error) { + pageNo, err := s.dataFreeList.GetPageNumber() + if err != nil { + return 0, false, err + } + if pageNo > 0 { + return pageNo, true, nil + } + if s.dataPagesCount < math.MaxUint32 { + s.dataPagesCount++ + return s.dataPagesCount, false, nil + } + return 0, false, errors.New("no space") +} + +func (s *Writer) getIndexPageNumber() (uint32, bool, error) { + pageNo, err := s.indexFreeList.GetPageNumber() + if err != nil { + return 0, false, err + } + if pageNo > 0 { + return pageNo, true, nil + } + if s.indexPagesCount < math.MaxUint32 { + s.indexPagesCount++ + return s.indexPagesCount, false, nil + } + return 0, false, errors.New("no space") +} + +func (s *Writer) packAndWrite() (err error) { + s.mutex.Lock() + //isExited := s.isExited + input := s.input + s.input = nil + + // var exitWaitGroup *sync.WaitGroup + // if s.isExited { + // exitWaitGroup = s.waitGroup + // } + s.mutex.Unlock() + + prepared := s.dataPreparer.Prepare(input) + + // 3. Пишу на диск WAL (append) + n, err := s.wal.Write(prepared.Packet) + if err != nil { + return + } + if n != len(prepared.Packet) { + return fmt.Errorf("written %d != total size %d", n, len(prepared.Packet)) + } + if err = s.wal.Sync(); err != nil { + return + } + + // 4. Пишу в atree сторінки + err = s.WritePagesToAtree(prepared.WriteToIndex, prepared.WriteToData) + if err != nil { + return + } + + // 6. відправляю input - worker-у + + var ( + forceSnapshot bool + ) + + if s.written > dumpSnapshotAfterNBytes { + forceSnapshot = true + } + + // if isExited && s.written > 0 { + // forceSnapshot = true + // } + + if forceSnapshot { + if err := s.wal.Close(); err != nil { + return fmt.Errorf("close wal file: %s", err) + } + s.wal = nil + s.written = 0 + + snapshotNumberCh := make(chan int, 1) + + s.appendToWorkerQueue(Changes{ + Records: prepared.WriteResults, + SnapshotNumberCh: snapshotNumberCh, + FrozenIndexPagesCount: s.indexFreeList.Pages(), + IndexPageNumbers: s.indexFreeList.Cached(), + FrozenDataPagesCount: s.dataFreeList.Pages(), + DataPageNumbers: s.dataFreeList.Cached(), + }) + + snapshotNumber := <-snapshotNumberCh + + // копіюю сторінки із delta файла в base файл і потім роблю Truncate + err = s.indexFreeList.Merge() + if err != nil { + return + } + err = s.dataFreeList.Merge() + if err != nil { + return + } + + var err error + s.wal, err = os.OpenFile( + JoinChangesFileName(s.dir, snapshotNumber), + os.O_CREATE|os.O_WRONLY, + filePerm, + ) + if err != nil { + return fmt.Errorf("create new changes file: %s", err) + } + } else { + s.appendToWorkerQueue(Changes{ + Records: prepared.WriteResults, + }) + } + return nil +} + +func (s *Writer) Append(req any) { + s.mutex.Lock() + s.input = append(s.input, req) + s.mutex.Unlock() + + select { + case s.signalCh <- struct{}{}: + default: + } +} + +// func (s *Writer) reset() { +// s.buf.Reset() +// s.buf.Write([]byte{ +// 0, 0, 0, 0, // packet length +// 0, 0, 0, 0, // crc32 +// 0, 0, 0, 0, // lsn + +// }) +// s.pagesToWrite = nil +// s.workerReqs = nil +// s.waitCh = make(chan struct{}) +// } + +// func (s *Writer) flush(w *bytes.Buffer) error { + +// return nil +// } + +func (s *Writer) exit() { + s.mutex.Lock() + s.isExited = true + s.mutex.Unlock() + + if err := s.packAndWrite(); err != nil { + qb.Abort(qb.FailedWriteToTxLog, err) + } +} + +// type IndexPayloadToWrite struct { +// PageNo uint32 +// Payload []byte +// ZeroLevel bool +// } + +type PageToWrite struct { + PageNo uint32 + Content []byte +} + +// writePagesToAtree - записує сторінки в .data та .index файли +func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToWrite) (err error) { + for _, p := range dataPages { + if len(p.Content) != s.dataPageSize { + return fmt.Errorf("wrong data page size: %d", len(p.Content)) + } + var ( + off = int(p.PageNo-1) * s.dataPageSize + n int + ) + n, err = s.dataFile.WriteAt(p.Content, int64(off)) + if err != nil { + return + } + if n != s.dataPageSize { + return fmt.Errorf("write %d instead of %d", n, s.dataPageSize) + } + } + for _, p := range indexPages { + if len(p.Content) != s.indexPageSize { + return fmt.Errorf("wrong index page size: %d", len(p.Content)) + } + var ( + off = int(p.PageNo-1) * s.indexPageSize + n int + ) + n, err = s.indexFile.WriteAt(p.Content, int64(off)) + if err != nil { + return + } + if n != s.indexPageSize { + return fmt.Errorf("write %d instead of %d", n, s.indexPageSize) + } + } + return nil +} + +// API + +// FIX - add +// type DeletedMeasuresSince struct { +// MetricID uint32 +// LastPageNo uint32 +// IsRootChanged bool +// RootPageNo uint32 +// FreeDataPages []uint32 +// FreeIndexPages []uint32 +// TimestampsBuf []byte +// ValuesBuf []byte +// } + +func (s *Writer) sendSignal() { + select { + case s.signalCh <- struct{}{}: + default: + } +} + +// Якщо (Pages) > 0 - незаповнену сторінку кодую в стисненому вигляді. +// Якщо (Pages) = 0 - кодую просто пари timestamp / value +// func (s *Writer) packWriteAppended(w *bytes.Buffer, req AppendedPagesReq) { +// // завантажений path. Отже просто додаємо в дерево data сторінки, створюємо нові індексні +// // без звернення до диску. +// report := s.atree.AppendDataPages(atree.AppendDataPagesReq{ +// LastPageNo: req.LastPageNo, +// Legs: req.Legs, +// DataPages: req.Pages, +// }) + +// m := AppendedPages{ +// MetricID: req.MetricID, +// Timestamp: req.Timestamp, +// Value: req.Value, +// NewRootPageNo: report.NewRootPageNo, +// LastPageNo: report.LastPageNo, +// Pages: report.Pages, +// TimestampsChunks: req.TimestampsChunks, +// TimestampsSize: int(req.TimestampsSize), +// ValuesChunks: req.ValuesChunks, +// ValuesSize: int(req.ValuesSize), +// } +// m.Pack(w) +// } + +// helpers + +// type Metric struct { +// MetricID uint32 +// MetricType qb.MetricType +// FracDigits byte +// LastPageNo uint32 +// Since uint32 +// SinceValue float64 +// Until uint32 +// UntilValue float64 +// Timestamps []byte // payload FIX +// Values []byte // payload FIX add lastDelta + h ? +// } + +// func writeChunks(dst io.Writer, chunks [][]byte, size int) (err error) { +// remaining := size +// for _, buf := range chunks { +// if remaining < len(buf) { +// buf = buf[:remaining] +// } +// _, err = dst.Write(buf) +// if err != nil { +// return +// } +// remaining -= len(buf) +// if remaining == 0 { +// break +// } +// } +// return +// } diff --git a/transform/aggregate.go b/transform/aggregate.go index de3fe86..dffdc66 100644 --- a/transform/aggregate.go +++ b/transform/aggregate.go @@ -6,8 +6,8 @@ import ( "io" "time" - diploma "gordenko.dev/dima/qb" - "gordenko.dev/dima/qb/bin" + bin "gordenko.dev/dima/bin/little" + "gordenko.dev/dima/qb" "gordenko.dev/dima/qb/timeutil" ) @@ -15,7 +15,7 @@ import ( type InstantPeriodsWriterOptions struct { Dst io.Writer - GroupBy diploma.GroupBy + GroupBy qb.GroupBy AggregateFuncs byte FirstHourOfDay int } @@ -24,7 +24,7 @@ type InstantPeriodsWriter struct { aggregateFuncs byte arr []byte responder *ChunkedResponder - groupBy diploma.GroupBy + groupBy qb.GroupBy firstHourOfDay int time2period func(uint32) time.Time currentPeriod time.Time @@ -45,13 +45,13 @@ func NewInstantPeriodsWriter(opt InstantPeriodsWriterOptions) (*InstantPeriodsWr } // Считаю q, чтобы заранее выделить массив для упаковки периодов var q int - if (opt.AggregateFuncs & diploma.AggregateMin) == diploma.AggregateMin { + if (opt.AggregateFuncs & qb.AggregateMin) == qb.AggregateMin { q++ } - if (opt.AggregateFuncs & diploma.AggregateMax) == diploma.AggregateMax { + if (opt.AggregateFuncs & qb.AggregateMax) == qb.AggregateMax { q++ } - if (opt.AggregateFuncs & diploma.AggregateAvg) == diploma.AggregateAvg { + if (opt.AggregateFuncs & qb.AggregateAvg) == qb.AggregateAvg { q++ } @@ -70,17 +70,17 @@ func NewInstantPeriodsWriter(opt InstantPeriodsWriterOptions) (*InstantPeriodsWr } switch opt.GroupBy { - case diploma.GroupByHour: + case qb.GroupByHour: s.time2period = groupByHour - case diploma.GroupByDay: + case qb.GroupByDay: if s.firstHourOfDay > 0 { s.time2period = s.groupByDayUsingFHD } else { s.time2period = groupByDay } - case diploma.GroupByMonth: + case qb.GroupByMonth: if s.firstHourOfDay > 0 { s.time2period = s.groupByMonthUsingFHD } else { @@ -174,12 +174,12 @@ func (s *InstantPeriodsWriter) feed(timestamp uint32, value float64, isBuffer bo func (s *InstantPeriodsWriter) decrementPeriod() { switch s.groupBy { - case diploma.GroupByHour: + case qb.GroupByHour: s.currentPeriod = s.currentPeriod.Add(-1 * time.Hour) //fmt.Println("decrement") - case diploma.GroupByDay: + case qb.GroupByDay: s.currentPeriod = s.currentPeriod.AddDate(0, 0, -1) - case diploma.GroupByMonth: + case qb.GroupByMonth: s.currentPeriod = s.currentPeriod.AddDate(0, -1, 0) } } @@ -210,15 +210,15 @@ func (s *InstantPeriodsWriter) packPeriod(timestamp uint32) { bin.PutUint32(s.arr[8:], s.endTimestamp) pos := 12 - if (s.aggregateFuncs & diploma.AggregateMin) == diploma.AggregateMin { + if (s.aggregateFuncs & qb.AggregateMin) == qb.AggregateMin { bin.PutFloat64(s.arr[pos:], s.min) pos += 8 } - if (s.aggregateFuncs & diploma.AggregateMax) == diploma.AggregateMax { + if (s.aggregateFuncs & qb.AggregateMax) == qb.AggregateMax { bin.PutFloat64(s.arr[pos:], s.max) pos += 8 } - if (s.aggregateFuncs & diploma.AggregateAvg) == diploma.AggregateAvg { + if (s.aggregateFuncs & qb.AggregateAvg) == qb.AggregateAvg { bin.PutFloat64(s.arr[pos:], s.total/float64(s.entries)) } } @@ -236,7 +236,7 @@ type CumulativePeriodsWriter struct { responder *ChunkedResponder firstHourOfDay int currentPeriod time.Time - groupBy diploma.GroupBy + groupBy qb.GroupBy time2period func(uint32) time.Time endTimestamp uint32 endValue float64 @@ -246,7 +246,7 @@ type CumulativePeriodsWriter struct { type CumulativePeriodsWriterOptions struct { Dst io.Writer - GroupBy diploma.GroupBy + GroupBy qb.GroupBy FirstHourOfDay int } @@ -271,17 +271,17 @@ func NewCumulativePeriodsWriter(opt CumulativePeriodsWriterOptions) (*Cumulative } switch opt.GroupBy { - case diploma.GroupByHour: + case qb.GroupByHour: s.time2period = groupByHour - case diploma.GroupByDay: + case qb.GroupByDay: if s.firstHourOfDay > 0 { s.time2period = s.groupByDayUsingFHD } else { s.time2period = groupByDay } - case diploma.GroupByMonth: + case qb.GroupByMonth: if s.firstHourOfDay > 0 { s.time2period = s.groupByMonthUsingFHD } else { @@ -360,12 +360,12 @@ func (s *CumulativePeriodsWriter) feed(timestamp uint32, value float64, isBuffer func (s *CumulativePeriodsWriter) decrementPeriod() { switch s.groupBy { - case diploma.GroupByHour: + case qb.GroupByHour: s.currentPeriod = s.currentPeriod.Add(-1 * time.Hour) //fmt.Println("decrement") - case diploma.GroupByDay: + case qb.GroupByDay: s.currentPeriod = s.currentPeriod.AddDate(0, 0, -1) - case diploma.GroupByMonth: + case qb.GroupByMonth: s.currentPeriod = s.currentPeriod.AddDate(0, -1, 0) } } diff --git a/transform/raw.go b/transform/raw.go index 101447d..4fa9264 100644 --- a/transform/raw.go +++ b/transform/raw.go @@ -3,7 +3,7 @@ package transform import ( "io" - "gordenko.dev/dima/qb/bin" + bin "gordenko.dev/dima/bin/little" ) // CURRENT VALUE WRITER diff --git a/transform/responder.go b/transform/responder.go index 0260ccf..9e3636d 100644 --- a/transform/responder.go +++ b/transform/responder.go @@ -5,7 +5,7 @@ import ( "fmt" "io" - "gordenko.dev/dima/qb/bin" + bin "gordenko.dev/dima/bin/little" "gordenko.dev/dima/qb/proto" ) diff --git a/txlog/data_preparer.go b/txlog/data_preparer.go deleted file mode 100644 index 4da5ddf..0000000 --- a/txlog/data_preparer.go +++ /dev/null @@ -1,207 +0,0 @@ -package txlog - -import ( - "bytes" - "io" - - bin "gordenko.dev/dima/bin/little" - "gordenko.dev/dima/qb/util" -) - -type DataPreparer struct { - pagePreparer *PagePreparer - //dataPagePayloadBound int - w *bytes.Buffer - indexPages []PageToWrite - dataPages []PageToWrite - writeResults []any -} - -type DataPreparerOptions struct { - MinWALBufferSize int - PagePreparer *PagePreparer -} - -func NewDataPreparer(opt DataPreparerOptions) *DataPreparer { - buf := make([]byte, opt.MinWALBufferSize) - s := &DataPreparer{ - pagePreparer: opt.PagePreparer, - w: bytes.NewBuffer(buf), - } - return s -} - -type PreparedData struct { - Packet []byte // пакет для запису в WAL - WriteToIndex []PageToWrite - WriteToData []PageToWrite - WriteResults []any -} - -func (s *DataPreparer) Prepare(input []any) PreparedData { - s.w.Write([]byte{ - 0, 0, 0, 0, 0, 0, 0, 0, 0, // size (max 9 byte) - 0, 0, 0, 0, // crc32 - }) - - hasher := util.NewHasher() - - w := io.MultiWriter(s.w, hasher) - - // 1. Пакую всі дані в WAL буфер запису - for _, untyped := range input { - switch x := untyped.(type) { - case AddedMetric: - x.Pack(w) - // case DeletedMetric: - // if len(x.FreePageNumbers) > 0 { - // s.freeList.AddPageNumbers(x.FreePageNumbers) - // } - case AppendedMeasures: - sealResult := s.pagePreparer.SealPages(SealPagesIn{ - LastPageNo: x.LastPageNo, - IndexLevelTails: x.IndexLevelTails, - DataPages: x.DataPages, - }) - - // сторінки для запису в index та data файли - for _, p := range sealResult.DataPages { - s.dataPages = append(s.dataPages, PageToWrite{ - PageNo: p.PageNo, - Content: p.Content, - }) - } - for _, level := range sealResult.IndexLevels { - for _, p := range level.IndexPages { - s.indexPages = append(s.indexPages, PageToWrite{ - PageNo: p.PageNo, - Content: p.Content, - }) - } - } - - var ( - // дані для запису в WAL - changedIndexPages []ChangedIndexLevel - // дані для Воркера - indexLevelTails []IndexLevelTail - ) - - for _, level := range sealResult.IndexLevels { - if len(level.IndexPages) == 0 && level.SkipRecords == level.TailRecordsCount { - break - } - var completedIndexPage *CompletedIndexPage - if len(level.IndexPages) > 0 { - first := level.IndexPages[0] - skipSize := level.SkipRecords * indexRecordSize - completedIndexPage = &CompletedIndexPage{ - PageNo: first.PageNo, - Reused: first.Reused, - Checksum: first.Checksum, - Records: first.Content[skipSize:], - } - } - changedIndexPages = append(changedIndexPages, ChangedIndexLevel{ - CompletedIndexPage: completedIndexPage, - IndexPages: level.IndexPages[1:], - TailRecords: level.TailRecords[:level.TailRecordsCount*indexRecordSize], - }) - - indexLevelTails = append(indexLevelTails, IndexLevelTail{ - Buffer: level.TailRecords, - RecordsCount: level.TailRecordsCount, - }) - } - - first := sealResult.DataPages[0] - - rec := WALRecordAppendMeasures{ - MetricID: x.MetricID, - CompletedDataPage: CompletedDataPage{ - PageNo: first.PageNo, - Reused: first.Reused, - PrevPageNo: first.PrevPageNo, - Checksum: first.Checksum, - TimestampsOffset: x.TimestampsOffset, - Timestamps: x.Timestamps, - ValuesOffset: x.ValuesOffset, - Values: x.Values, - }, - DataPages: sealResult.DataPages[1:], - ChangedIndexLevels: changedIndexPages, - TailTimestamps: x.Timestamps, - TailValues: x.Values, - } - - rec.WriteTo(s.w) - - // Дані для Worker - s.writeResults = append(s.writeResults, AppendMeasuresSummary{ - MetricID: x.MetricID, - LastPageNo: sealResult.DataPages[len(sealResult.DataPages)-1].PageNo, - Index: indexLevelTails, - ResultCode: x.ResultCode, - WrittenCount: x.WrittenCount, - ResultCh: x.ResultCh, - }) - - //case DeletedMeasures: - //case DeletedMeasuresSince: - } - } - // 2. Додаю розмір пакету і чексуму - - //s.written += int64(len(packet)) + 12 - - packet := s.w.Bytes() - // write size - payloadSize := len(packet) - 13 - start := 9 - bin.CountVarSize(payloadSize) - bin.PutVarSize(packet[start:], payloadSize) - // checksum - bin.PutUint32(packet[9:], hasher.Sum32()) - // - return PreparedData{ - Packet: packet[start:], - WriteToIndex: s.indexPages, - WriteToData: s.dataPages, - WriteResults: s.writeResults, - } -} - -func (s *DataPreparer) Reset() { - s.w.Reset() - s.indexPages = nil - s.dataPages = nil - s.writeResults = nil -} - -// MeasuresToWrite -type AppendedMeasures struct { - MetricID uint32 - LastPageNo uint32 - TimestampsOffset int // offset on 1st page - Timestamps []byte - ValuesOffset int // offset on 1st page - Values []byte - IndexLevelTails []IndexLevelTail - DataPages []DataPayload - TailTimestamps []byte // data level tail - TailValues []byte // data level tail - ResultCode int - WrittenCount int - ResultCh chan struct{} -} - -// WRITE RESULTS - -// результат -type AppendMeasuresSummary struct { - MetricID uint32 - LastPageNo uint32 - Index []IndexLevelTail - ResultCode int - WrittenCount int - ResultCh chan struct{} -} diff --git a/txlog/helpers.go b/txlog/helpers.go deleted file mode 100644 index 975b8c0..0000000 --- a/txlog/helpers.go +++ /dev/null @@ -1,202 +0,0 @@ -package txlog - -import ( - "bytes" - - bin "gordenko.dev/dima/bin/little" -) - -const recordSize = 8 -const chunkSize = 24 - -func copyPayloadFromChunks(w *bytes.Buffer, chunks [][]byte, size int, offset int) { - bin.WriteUint16(w, uint16(size)) - var ( - chunkIdx = offset / chunkSize - byteIdx = offset % chunkSize - ) - for _, chunk := range chunks[chunkIdx:] { - available := len(chunk) - byteIdx - if available <= size { - w.Write(chunk[byteIdx:]) - size -= available - if size == 0 { - return - } - byteIdx = 0 - } else { - w.Write(chunk[byteIdx : byteIdx+size]) - return - } - } -} - -///////////////////////////////////////// - -// type DataPage struct { -// PageNo uint32 -// Reused bool -// PrevPageNo uint32 -// Checksum uint32 -// Timestamps []byte -// Values []byte -// } - -// type IndexPage struct { -// PageNo uint32 -// Reused bool -// Checksum uint32 -// Records []byte -// } - -// type IndexLevel struct { -// Pages []IndexPage -// Records []byte // unfilled -// } - -// Для декодінга -// type TxAppendedMeasures struct { -// MetricID uint32 -// DataPages []DataPage -// // offsets потрібні тому що дані не додаються в кінець, а перезаписують кілька -// // останніх байтів unfilled даних. Потрібно для коректного recovery. -// // Якщо є DataPages, то застосовуються для 1-ї Data сторінки, інакше - до timestamps і values (unfilled) -// TimestampsOffset int -// ValuesOffset int -// Timestamps []byte // unfilled -// Values []byte // unfilled -// IndexLevels []IndexLevel -// } - -// func (s *TxAppendedMeasures) Read(r *bytes.Buffer) (err error) { -// s.MetricID, err = bin.ReadUint32(r) -// if err != nil { -// return -// } -// dataPagesQty, err := bin.ReadVarSize(r) -// if err != nil { -// return -// } -// for range dataPagesQty { -// var ( -// p DataPage -// timestampsSize, valuesSize int -// //b byte -// ) -// p.PageNo, err = bin.ReadUint32(r) -// if err != nil { -// return -// } -// p.Reused, err = bin.ReadBool(r) -// if err != nil { -// return -// } -// p.PrevPageNo, err = bin.ReadUint32(r) -// if err != nil { -// return -// } -// p.Checksum, err = bin.ReadUint32(r) -// if err != nil { -// return -// } -// timestampsSize, err = bin.ReadUint16AsInt(r) -// if err != nil { -// return -// } -// p.Timestamps, err = bin.ReadN(r, timestampsSize) -// if err != nil { -// return -// } -// valuesSize, err = bin.ReadUint16AsInt(r) -// if err != nil { -// return -// } -// p.Values, err = bin.ReadN(r, valuesSize) -// if err != nil { -// return -// } -// s.DataPages = append(s.DataPages, p) -// } -// s.TimestampsOffset, err = bin.ReadUint16AsInt(r) -// if err != nil { -// return -// } -// s.ValuesOffset, err = bin.ReadUint16AsInt(r) -// if err != nil { -// return -// } -// var ( -// timestampsSize, valuesSize int -// ) -// timestampsSize, err = bin.ReadVarSize(r) -// if err != nil { -// return -// } -// s.Timestamps, err = bin.ReadN(r, timestampsSize) -// if err != nil { -// return -// } -// valuesSize, err = bin.ReadVarSize(r) -// if err != nil { -// return -// } -// s.Values, err = bin.ReadN(r, valuesSize) -// if err != nil { -// return -// } -// if len(s.DataPages) == 0 { -// return -// } -// indexLevelsQty, err := bin.ReadVarSize(r) -// if err != nil { -// return -// } -// for range indexLevelsQty { -// var ( -// indexPagesQty int -// level IndexLevel -// recordsQty int -// ) -// indexPagesQty, err = bin.ReadVarSize(r) -// if err != nil { -// return -// } -// for range indexPagesQty { -// var ( -// p IndexPage -// recordsQty int -// ) -// p.PageNo, err = bin.ReadUint32(r) -// if err != nil { -// return -// } -// p.Reused, err = bin.ReadBool(r) -// if err != nil { -// return -// } -// p.Checksum, err = bin.ReadUint32(r) -// if err != nil { -// return -// } -// recordsQty, err = bin.ReadVarSize(r) -// if err != nil { -// return -// } -// p.Records, err = bin.ReadN(r, recordsQty*recordSize) -// if err != nil { -// return -// } -// level.Pages = append(level.Pages, p) -// } -// recordsQty, err = bin.ReadVarSize(r) -// if err != nil { -// return -// } -// level.Records, err = bin.ReadN(r, recordsQty*recordSize) -// if err != nil { -// return -// } -// s.IndexLevels = append(s.IndexLevels, level) -// } -// return -// } diff --git a/txlog/page_preparer.go b/txlog/page_preparer.go deleted file mode 100644 index 26deb4e..0000000 --- a/txlog/page_preparer.go +++ /dev/null @@ -1,279 +0,0 @@ -package txlog - -import ( - "errors" - - "gordenko.dev/dima/qb" - "gordenko.dev/dima/qb/bin" - "gordenko.dev/dima/qb/util" -) - -// Задача метода: -// - для data сторінок отримати pageNo та reused, задати sizes та prevPageNo, порахувати CRC32 -// - додати пари до index, згенерувати нові index сторінки -// levels []*IndexLevel, dataPages []*DataPage -// fix - reduce leves - -type PagePreparer struct { - dataChecksumIdx int - maxRecordsOnIndexPage int - indexPageIncSize int // кратно indexPageSize - timestampsSizeIdx int - valuesSizeIdx int - prevPageIdx int - indexRecordsCountIdx int - isZeroLevelIdx int - indexChecksumIdx int - getDataPageNumber func() (uint32, bool, error) - getIndexPageNumber func() (uint32, bool, error) -} - -type PagePreparerOptions struct { - IndexPageSize int - IndexPageIncSize int - DataPageSize int - GetDataPageNumber func() (uint32, bool, error) - GetIndexPageNumber func() (uint32, bool, error) -} - -func NewPagePreparer(opt PagePreparerOptions) (*PagePreparer, error) { - if opt.GetIndexPageNumber == nil { - return nil, errors.New("missing required option: GetIndexPageNumber") - } - if opt.GetDataPageNumber == nil { - return nil, errors.New("missing required option: GetDataPageNumber") - } - if (opt.IndexPageSize % opt.IndexPageIncSize) != 0 { - return nil, errors.New("IndexPageIncSize must multiple of IndexPageSize") - } - s := &PagePreparer{ - dataChecksumIdx: opt.DataPageSize - 4, - indexPageIncSize: opt.IndexPageIncSize, - timestampsSizeIdx: opt.DataPageSize - 6, - valuesSizeIdx: opt.DataPageSize - 8, - prevPageIdx: opt.DataPageSize - 12, - indexRecordsCountIdx: opt.IndexPageSize - 6, - isZeroLevelIdx: opt.IndexPageSize - 7, - indexChecksumIdx: opt.IndexPageSize - 4, - getIndexPageNumber: opt.GetIndexPageNumber, - getDataPageNumber: opt.GetDataPageNumber, - } - s.maxRecordsOnIndexPage = (opt.IndexPageSize - 7) / indexRecordSize - return s, nil -} - -type appendIndexRecordIn struct { - Records []byte - RecordsCount int - Timestamp uint32 - PageNo uint32 -} - -func (s *PagePreparer) appendIndexRecord(in appendIndexRecordIn) []byte { - if in.RecordsCount < s.maxRecordsOnIndexPage { - var ( - pos int - buf = in.Records - ) - if in.RecordsCount > 0 { - pos = in.RecordsCount * indexRecordSize - } else { - buf = make([]byte, s.indexPageIncSize) - } - bin.PutUint32(buf[pos:], in.Timestamp) - bin.PutUint32(buf[pos+4:], in.PageNo) - return buf - } - qb.Abort(qb.NoSpaceOnIndexPage, nil) - return nil -} - -type sealDataPageIn struct { - Content []byte - PrevPageNo uint32 - TimestampsSize int - ValuesSize int -} - -func (s *PagePreparer) sealDataPage(in sealDataPageIn) (checksum uint32) { - bin.PutUint16(in.Content[s.timestampsSizeIdx:], uint16(in.TimestampsSize)) - bin.PutUint16(in.Content[s.valuesSizeIdx:], uint16(in.ValuesSize)) - bin.PutUint32(in.Content[s.prevPageIdx:], in.PrevPageNo) - - checksum = util.CalcChecksum(in.Content[:s.dataChecksumIdx]) - bin.PutUint32(in.Content[s.dataChecksumIdx:], checksum) - return -} - -type sealIndexPageIn struct { - Content []byte - RecordsCount int - LastLevel bool -} - -func (s *PagePreparer) sealIndexPage(in sealIndexPageIn) (checksum uint32) { - bin.PutUint16(in.Content[s.indexRecordsCountIdx:], uint16(in.RecordsCount)) - if in.LastLevel { - in.Content[s.isZeroLevelIdx] = 1 - } - checksum = util.CalcChecksum(in.Content[:s.indexChecksumIdx]) - bin.PutUint32(in.Content[s.indexChecksumIdx:], checksum) - return -} - -type IndexLevelTail struct { - Buffer []byte // розмір більший за кількість - RecordsCount int -} - -type DataPayload struct { - Since uint32 - Content []byte - TimestampsSize int - ValuesSize int -} - -type SealPagesIn struct { - LastPageNo uint32 - IndexLevelTails []IndexLevelTail - DataPages []DataPayload -} - -type SealedDataPage struct { - PageNo uint32 - Reused bool - PrevPageNo uint32 - Checksum uint32 - Content []byte -} - -type SealedIndexPage struct { - Content []byte - PageNo uint32 - Reused bool - Checksum uint32 -} - -// SealedIndexLevel - як зрозуміти чи потрібно щось писати в WAL чи index файл, -// чи додавання нових data сторінок не зачепило індексний рівень? -// Якщо є IndexPages - пишемо весь рівень в WAL і готові сторінки в index файл. -// Якщо IndexPages пустий - порівнюємо Offset і len(Payload) - якщо довжина -// більша за змішення - дані додані на незаповнену сторінку рівня, -// отже різницю пишемо в WAL. -type SealedIndexLevel struct { - // Якщо є IndexPages, offset - це зміщення із якого починаються дані першої сторінки, - // які ще не зписані в WAL. - // Якщо IndexPages пустий, offset - це зміщення із якого починаються дані payload, - // які ще не зписані в WAL. - // В payload в будь-якому разі дані останньої індексної сторінки рівня (незаповненої). - SkipRecords int - IndexPages []SealedIndexPage - TailRecords []byte - TailRecordsCount int -} - -type SealResult struct { - DataPages []SealedDataPage - IndexLevels []*SealedIndexLevel -} - -func (s *PagePreparer) SealPages(in SealPagesIn) (result SealResult) { - var ( - sealedDataPages []SealedDataPage - sealedIndexLevels []*SealedIndexLevel - ) - for _, tail := range in.IndexLevelTails { - sealedIndexLevels = append(sealedIndexLevels, &SealedIndexLevel{ - SkipRecords: tail.RecordsCount, - TailRecords: tail.Buffer, - TailRecordsCount: tail.RecordsCount, - }) - } - for dataPageIdx, d := range in.DataPages { - var ( - upTimestamp = d.Since - upPageNo uint32 - levelIdx = 0 - prevPageNo uint32 - ) - if dataPageIdx == 0 { - prevPageNo = in.LastPageNo - } else { - prevPageNo = result.DataPages[dataPageIdx-1].PageNo - } - pageNo, reused, err := s.getDataPageNumber() - if err != nil { - qb.Abort(qb.FailedGetPageNumber, err) - } - sealedDataPages = append(sealedDataPages, SealedDataPage{ - Content: d.Content, - PrevPageNo: prevPageNo, - PageNo: pageNo, - Reused: reused, - Checksum: s.sealDataPage(sealDataPageIn{ - Content: d.Content, - PrevPageNo: prevPageNo, - TimestampsSize: d.TimestampsSize, - ValuesSize: d.ValuesSize, - }), - }) - upPageNo = pageNo - for { - if levelIdx == len(sealedIndexLevels) { - // новий root - sealedIndexLevels = append(sealedIndexLevels, &SealedIndexLevel{ - TailRecords: s.appendIndexRecord(appendIndexRecordIn{ - Records: nil, - Timestamp: upTimestamp, - PageNo: upPageNo, - }), - TailRecordsCount: 1, - }) - break - } - level := sealedIndexLevels[levelIdx] - if level.TailRecordsCount < s.maxRecordsOnIndexPage { - level.TailRecords = s.appendIndexRecord(appendIndexRecordIn{ - Records: level.TailRecords, - Timestamp: upTimestamp, - PageNo: upPageNo, - }) - level.TailRecordsCount++ - break - } - // в буфері немає місця для додавання нової пари, отже це - // заповнена сторінка - pageNo, reused, err := s.getIndexPageNumber() - if err != nil { - qb.Abort(qb.FailedGetPageNumber, err) - } - - filled := SealedIndexPage{ - Content: level.TailRecords, - PageNo: pageNo, - Reused: reused, - Checksum: s.sealIndexPage(sealIndexPageIn{ - Content: level.TailRecords, - RecordsCount: level.TailRecordsCount, - LastLevel: levelIdx == 0, - }), - } - level.IndexPages = append(level.IndexPages, filled) - // новий tail на індексному рівні - level.TailRecords = s.appendIndexRecord(appendIndexRecordIn{ - Records: nil, - Timestamp: upTimestamp, - PageNo: upPageNo, - }) - level.TailRecordsCount = 1 - // - upPageNo = pageNo - upTimestamp = bin.GetUint32(filled.Content) - levelIdx++ - } - } - return SealResult{ - DataPages: sealedDataPages, - IndexLevels: sealedIndexLevels, - } -} diff --git a/txlog/reader.go b/txlog/reader.go deleted file mode 100644 index a6ece2e..0000000 --- a/txlog/reader.go +++ /dev/null @@ -1,362 +0,0 @@ -package txlog - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" - "os" - - bin "gordenko.dev/dima/bin/little" - "gordenko.dev/dima/qb/util" -) - -const readBufferSize = 8 * 1024 * 1024 - -type Reader struct { - file *os.File - reader *bufio.Reader - current []any - next []any -} - -type ReaderOptions struct { - FileName string - BufferSize int -} - -func NewReader(opt ReaderOptions) (*Reader, error) { - if opt.FileName == "" { - return nil, errors.New("FileName option is required") - } - if opt.BufferSize <= 0 { - return nil, errors.New("BufferSize option is required") - } - - file, err := os.Open(opt.FileName) - if err != nil { - return nil, err - } - - return &Reader{ - file: file, - reader: bufio.NewReaderSize(file, readBufferSize), - }, nil -} - -func (s *Reader) Close() { - s.file.Close() -} - -func (s *Reader) NextPacket() ([]any, bool, error) { - var err error - if s.current == nil { - s.current, err = s.readPacket() - if err != nil { - return nil, false, err - } - if len(s.current) > 0 { - s.next, err = s.readPacket() - if err != nil { - return nil, false, err - } - } - } - - current := s.current - done := s.next == nil - - s.current = s.next - s.next, err = s.readPacket() - if err != nil { - return nil, false, err - } - return current, done, nil -} - -func (s *Reader) readPacket() (_ []any, err error) { - payloadSize, err := bin.ReadVarSize(s.reader) - if err != nil { - if err == io.EOF { - return nil, nil - } else { - return - } - } - - storedCRC, err := bin.ReadUint32(s.reader) - if err != nil { - return - } - - body, err := bin.ReadN(s.reader, int(payloadSize)) - if err != nil { - return - } - - hasher := util.NewHasher() - hasher.Write(body) - - calculatedCRC := hasher.Sum32() - - if calculatedCRC == storedCRC { - return s.parseRecords(body) - } - //s.logger.Printf("stored CRC %d != calculated CRC %d", storedCRC, calculatedCRC) - return nil, nil -} - -func (s *Reader) parseRecords(body []byte) ([]any, error) { - var ( - src = bytes.NewBuffer(body) - records []any - ) - - for { - recordType, err := src.ReadByte() - if err != nil { - if err == io.EOF { - return records, nil - } - return nil, err - } - - switch recordType { - case CodeAddedMetric: - rec := new(AddedMetric) - if err = rec.Parse(src); err != nil { - return nil, err - } - records = append(records, rec) - - case CodeDeletedMetric: - rec := new(DeletedMetric) - if err = rec.Parse(src); err != nil { - return nil, err - } - records = append(records, rec) - - // case CodeAppendedMeasure: - // rec := new(AppendedMeasure) - // if err = rec.Parse(src); err != nil { - // return nil, err - // } - // records = append(records, rec) - - // case CodeAppendedMeasures: - // rec := new(AppendedMeasures) - // if err = rec.Parse(src); err != nil { - // return nil, err - // } - // records = append(records, rec) - - // case CodeAppendedPages: - // rec := new(AppendedPages) - // if err = rec.Parse(src); err != nil { - // return nil, err - // } - // records = append(records, rec) - - case CodeDeletedMeasures: - rec := new(DeletedMeasures) - if err = rec.Parse(src); err != nil { - return nil, err - } - records = append(records, rec) - - default: - return nil, fmt.Errorf("unknown record type code: %d", recordType) - } - } -} - -// func (s *Reader) readAddedMetric(src *bytes.Buffer) (_ AddedMetric, err error) { -// arr, err := bin.ReadN(src, 6) -// if err != nil { -// return -// } -// return AddedMetric{ -// MetricID: bin.GetUint32(arr), -// MetricType: diploma.MetricType(arr[4]), -// FracDigits: int(arr[5]), -// }, nil -// } - -// func (s *Reader) readDeletedMetric(src *bytes.Buffer) (_ DeletedMetric, err error) { -// var rec DeletedMetric -// rec.MetricID, err = bin.ReadUint32(src) -// if err != nil { -// return -// } -// // free data pages -// dataQty, _, err := bin.ReadVarUint64(src) -// if err != nil { -// return -// } -// for range dataQty { -// var pageNo uint32 -// pageNo, err = bin.ReadUint32(src) -// if err != nil { -// return -// } -// rec.FreeDataPages = append(rec.FreeDataPages, pageNo) -// } -// // free index pages -// indexQty, _, err := bin.ReadVarUint64(src) -// if err != nil { -// return -// } -// for range indexQty { -// var pageNo uint32 -// pageNo, err = bin.ReadUint32(src) -// if err != nil { -// return -// } -// rec.FreeIndexPages = append(rec.FreeIndexPages, pageNo) -// } -// return rec, nil -// } - -// func (s *Reader) readAppendedMeasure(src *bytes.Buffer) (_ AppendedMeasure, err error) { -// arr, err := bin.ReadN(src, 16) -// if err != nil { -// return -// } -// return AppendedMeasure{ -// MetricID: bin.GetUint32(arr[0:]), -// Timestamp: bin.GetUint32(arr[4:]), -// Value: bin.GetFloat64(arr[8:]), -// }, nil -// } - -// func (s *Reader) readAppendedMeasures(src *bytes.Buffer) (_ AppendedMeasures, err error) { -// var rec AppendedMeasures -// rec.MetricID, err = bin.ReadUint32(src) -// if err != nil { -// return -// } -// qty, err := bin.ReadUint16(src) -// if err != nil { -// return -// } -// for range qty { -// var measure proto.Measure -// measure.Timestamp, err = bin.ReadUint32(src) -// if err != nil { -// return -// } -// measure.Value, err = bin.ReadFloat64(src) -// if err != nil { -// return -// } -// rec.Measures = append(rec.Measures, measure) -// } -// return rec, nil -// } - -// func (s *Reader) readAppendedMeasureWithOverflow(src *bytes.Buffer) (_ AppendedMeasureWithOverflow, err error) { -// var ( -// b byte -// rec AppendedMeasureWithOverflow -// ) -// rec.MetricID, err = bin.ReadUint32(src) -// if err != nil { -// return -// } -// rec.Timestamp, err = bin.ReadUint32(src) -// if err != nil { -// return -// } -// rec.Value, err = bin.ReadFloat64(src) -// if err != nil { -// return -// } -// b, err = src.ReadByte() -// if err != nil { -// return -// } -// rec.IsDataPageReused = b == 1 - -// rec.DataPageNo, err = bin.ReadUint32(src) -// if err != nil { -// return -// } -// b, err = src.ReadByte() -// if err != nil { -// return -// } -// if b == 1 { -// rec.IsRootChanged = true -// rec.RootPageNo, err = bin.ReadUint32(src) -// if err != nil { -// return -// } -// } -// // index pages -// indexQty, err := src.ReadByte() -// if err != nil { -// return -// } -// for range indexQty { -// var pageNo uint32 -// pageNo, err = bin.ReadUint32(src) -// if err != nil { -// return -// } -// rec.ReusedIndexPages = append(rec.ReusedIndexPages, pageNo) -// } -// return rec, nil -// } - -// func (s *Reader) readDeletedMeasures(src *bytes.Buffer) (_ DeletedMeasures, err error) { -// var ( -// rec DeletedMeasures -// ) -// rec.MetricID, err = bin.ReadUint32(src) -// if err != nil { -// return -// } -// // free data pages -// rec.FreeDataPages, err = s.readFreePageNumbers(src) -// if err != nil { -// return -// } -// // free index pages -// rec.FreeIndexPages, err = s.readFreePageNumbers(src) -// if err != nil { -// return -// } -// return rec, nil -// } - -// HELPERS - -// func (s *Reader) readFreePageNumbers(src *bytes.Buffer) ([]uint32, error) { -// var freePages []uint32 -// qty, _, err := bin.ReadVarUint64(src) -// if err != nil { -// return nil, err -// } -// for range qty { -// var pageNo uint32 -// pageNo, err = bin.ReadUint32(src) -// if err != nil { -// return nil, err -// } -// freePages = append(freePages, pageNo) -// } -// return freePages, nil -// } - -func (s *Reader) Seek(offset int64) error { - ret, err := s.file.Seek(offset, 0) - if err != nil { - return err - } - - if ret != offset { - return fmt.Errorf("ret %d != offset %d", ret, offset) - } - return nil -} diff --git a/txlog/txlog.go b/txlog/txlog.go deleted file mode 100644 index 03be1fa..0000000 --- a/txlog/txlog.go +++ /dev/null @@ -1,322 +0,0 @@ -package txlog - -import ( - "io" - - bin "gordenko.dev/dima/bin/little" - "gordenko.dev/dima/qb" -) - -var ( - indexRecordSize = 8 -) - -type AddedMetric struct { - MetricID uint32 - MetricType qb.MetricType - FracDigits int -} - -func (s AddedMetric) Pack(w io.Writer) { - arr := []byte{ - CodeAddedMetric, - 0, 0, 0, 0, // - byte(s.MetricType), - byte(s.FracDigits), - } - bin.PutUint32(arr[1:], s.MetricID) - w.Write(arr) -} - -func (s *AddedMetric) Parse(src io.Reader) (err error) { - arr, err := bin.ReadN(src, 6) - if err != nil { - return - } - s.MetricID, _ = bin.GetUint32(arr) - s.MetricType = qb.MetricType(arr[4]) - s.FracDigits = int(arr[5]) - return nil -} - -type DeletedMetric struct { - MetricID uint32 - FreeIndexPages []uint32 - FreeDataPages []uint32 -} - -func (s DeletedMetric) Pack(w io.Writer) { - arr := []byte{ - CodeDeletedMetric, - 0, 0, 0, 0, // metricID - } - bin.PutUint32(arr[1:], s.MetricID) - w.Write(arr) - // fix - // bin.WriteVarSize(w, len(s.FreePageNumbers)) - // for _, pageNo := range s.FreePageNumbers { - // bin.WriteUint32(w, pageNo) - // } -} - -func (s *DeletedMetric) Parse(src io.Reader) (err error) { - s.MetricID, err = bin.ReadUint32(src) - if err != nil { - return - } - // free pages - // fix - // qty, err := bin.ReadVarSize(src) - // if err != nil { - // return - // } - // for range qty { - // var pageNo uint32 - // pageNo, err = bin.ReadUint32(src) - // if err != nil { - // return - // } - // //s.FreePageNumbers = append(s.FreePageNumbers, pageNo) - // } - return nil -} - -// fix - add changed pages -type DeletedMeasures struct { - MetricID uint32 - FreePageNumbers []uint32 -} - -func (s DeletedMeasures) Pack(w io.Writer) { - arr := []byte{ - CodeDeletedMeasures, - 0, 0, 0, 0, - } - bin.PutUint32(arr[1:], s.MetricID) - w.Write(arr) - bin.WriteVarSize(w, len(s.FreePageNumbers)) - for _, pageNo := range s.FreePageNumbers { - bin.WriteUint32(w, pageNo) - } -} - -func (s *DeletedMeasures) Parse(src io.Reader) (err error) { - s.MetricID, err = bin.ReadUint32(src) - if err != nil { - return - } - // free pages - qty, err := bin.ReadVarSize(src) - if err != nil { - return - } - for range qty { - var pageNo uint32 - pageNo, err = bin.ReadUint32(src) - if err != nil { - return - } - s.FreePageNumbers = append(s.FreePageNumbers, pageNo) - } - return nil -} - -// type AppendedMeasure struct { -// MetricID uint32 -// Timestamp uint32 -// Value float64 -// } - -// func (s AppendedMeasure) Pack(w io.Writer) { -// arr := []byte{ -// CodeAppendedMeasure, -// 0, 0, 0, 0, // metricID -// 0, 0, 0, 0, // timestamp -// 0, 0, 0, 0, 0, 0, 0, 0, // value -// } -// bin.PutUint32(arr[1:], s.MetricID) -// bin.PutUint32(arr[5:], s.Timestamp) -// bin.PutFloat64(arr[9:], s.Value) -// w.Write(arr) -// } - -// func (s *AppendedMeasure) Parse(src io.Reader) (err error) { -// arr, err := bin.ReadN(src, 16) -// if err != nil { -// return -// } -// s.MetricID, _ = bin.GetUint32(arr[0:]) -// s.Timestamp, _ = bin.GetUint32(arr[4:]) -// s.Value, _ = bin.GetFloat64(arr[8:]) -// return nil -// } - -// type AppendedMeasures struct { -// MetricID uint32 -// Measures []proto.Measure -// } - -// func (s AppendedMeasures) Pack(w io.Writer) { -// arr := []byte{ -// CodeAppendedMeasures, -// 0, 0, 0, 0, // metricID -// 0, 0, // qty -// } -// bin.PutUint32(arr[1:], s.MetricID) -// bin.PutUint16(arr[5:], uint16(len(s.Measures))) -// w.Write(arr) -// for _, measure := range s.Measures { -// bin.WriteUint32(w, measure.Timestamp) -// bin.WriteFloat64(w, measure.Value) -// } -// } - -// func (s *AppendedMeasures) Parse(src io.Reader) (err error) { -// s.MetricID, err = bin.ReadUint32(src) -// if err != nil { -// return -// } -// qty, err := bin.ReadUint16(src) -// if err != nil { -// return -// } -// for range qty { -// var measure proto.Measure -// measure.Timestamp, err = bin.ReadUint32(src) -// if err != nil { -// return -// } -// measure.Value, err = bin.ReadFloat64(src) -// if err != nil { -// return -// } -// s.Measures = append(s.Measures, measure) -// } -// return nil -// } - -// type AppendedPagesReq struct { -// Legs []atree.PathLeg -// MetricID uint32 -// Timestamp uint32 // last measure -// Value float64 // last measure -// //RootPageNo uint32 -// LastPageNo uint32 -// Pages []atree.NotLinkedDataPage -// TimestampsChunks [][]byte -// TimestampsSize uint16 -// ValuesChunks [][]byte -// ValuesSize uint16 -// } - -/* -4b metricID -4b timestamp -8b value -4b LastPageNo (0 if no new root page) -4b newRootPageNo (0 if no new root page) -Nb - varsize (pages qty) -[ - - 4b pageNo - 1b isReused - Xb page (page size) - -] -2b - timestamps size (not filled page) -Nb - timestamps payload -2b - values size (not filled page) -Nb - values payload -*/ -// type AppendedPages struct { -// MetricID uint32 -// Timestamp uint32 // last measure -// Value float64 // last measure -// NewRootPageNo uint32 -// LastPageNo uint32 -// Pages []atree.PageToWrite -// TimestampsChunks [][]byte -// TimestampsSize int -// ValuesChunks [][]byte -// ValuesSize int -// } - -// func (s *AppendedPages) Pack(w io.Writer) { -// w.Write([]byte{ -// CodeAppendedPages, -// }) -// bin.WriteUint32(w, s.MetricID) -// bin.WriteUint32(w, s.Timestamp) -// bin.WriteFloat64(w, s.Value) -// bin.WriteUint32(w, s.LastPageNo) -// bin.WriteUint32(w, s.NewRootPageNo) -// // -// bin.WriteVarSize(w, len(s.Pages)) -// for _, p := range s.Pages { -// bin.WriteUint32(w, p.PageNo) -// if p.IsReused { -// w.Write([]byte{1}) -// } else { -// w.Write([]byte{0}) -// } -// w.Write(p.Data) -// } -// // timestamps -// writeChunkedPayloadTo(s.TimestampsChunks, int(s.TimestampsSize), w) -// // values -// writeChunkedPayloadTo(s.ValuesChunks, int(s.ValuesSize), w) -// } - -// func (s *AppendedPages) Parse(src io.Reader) (err error) { -// s.MetricID, err = bin.ReadUint32(src) -// if err != nil { -// return -// } -// s.Timestamp, err = bin.ReadUint32(src) -// if err != nil { -// return -// } -// s.Value, err = bin.ReadFloat64(src) -// if err != nil { -// return -// } -// s.LastPageNo, err = bin.ReadUint32(src) -// if err != nil { -// return -// } -// s.NewRootPageNo, err = bin.ReadUint32(src) -// if err != nil { -// return -// } -// pagesQty, err := bin.ReadVarSize(src) -// if err != nil { -// return -// } -// for range pagesQty { -// var page atree.PageToWrite -// page.PageNo, err = bin.ReadUint32(src) -// if err != nil { -// return -// } -// page.IsReused, err = bin.ReadBool(src) -// if err != nil { -// return -// } -// page.Data, err = bin.ReadN(src, atree.PageSize) -// if err != nil { -// return -// } -// s.Pages = append(s.Pages, page) -// } -// s.TimestampsChunks, s.TimestampsSize, err = readChunkedPayload(src) -// if err != nil { -// return -// } -// s.ValuesChunks, s.ValuesSize, err = readChunkedPayload(src) -// if err != nil { -// return -// } -// return nil -// } - -// PACK diff --git a/txlog/wal_records.go b/txlog/wal_records.go deleted file mode 100644 index abf491d..0000000 --- a/txlog/wal_records.go +++ /dev/null @@ -1,305 +0,0 @@ -package txlog - -import ( - "bytes" - - bin "gordenko.dev/dima/bin/little" -) - -type CompletedDataPage struct { - PageNo uint32 - Reused bool - PrevPageNo uint32 - Checksum uint32 - TimestampsOffset int // offset on 1st page - Timestamps []byte - ValuesOffset int // offset on 1st page - Values []byte -} - -type CompletedIndexPage struct { - PageNo uint32 - Reused bool - Checksum uint32 - Records []byte // offset не потрібен, оскільки Records додаються в кінець -} - -type ChangedIndexLevel struct { - CompletedIndexPage *CompletedIndexPage - IndexPages []SealedIndexPage - TailRecords []byte // payload only -} - -type WALRecordAppendMeasures struct { - MetricID uint32 - CompletedDataPage CompletedDataPage - DataPages []SealedDataPage - ChangedIndexLevels []ChangedIndexLevel - TailTimestamps []byte // data level tail - TailValues []byte // data level tail -} - -/* -Format appended measures: -1b - tx type -4b - metricID -4b - pageNo -1b - reused -4b - prevPageNo -4b - page crc32 -Nb - (varsize) timestamps offset -Nb - (varsize) timestamps tail size on 1st filled data page -Nb - timestamps tail on 1st filled data page -Nb - (varsize) values offset -Nb - (varsize) values tail size on 1st filled data page -Nb - values tail on 1st filled data page - -Nb - (varsize) filled data pages qty -[ - - 4b - pageNo - 1b - reused - Nb - (data page size) page content - -] -Nb - (varsize) timestamps payload size on tail -Nb - timestamps payload -Nb - (varsize) values payload size on tail -Nb - values payload -Nb - (varsize) qty of index levels -[ - - // NOTE! - // - idx = 0 - zeroLevel - // - skipped = maxRecordsOnIndexPage - records count - 4b - pageNo of 1st index page - 1b - reused - 4b - page crc32 - Nb - (varsize) records tail count on 1st filled index page - Nb - records tail on 1st filled index page - - - Nb - (varsize) qty of level filled pages - [ - 4b - pageNo - 1b - reused - Nb - (index page size) page content - ] - Nb - (varsize) size of records level tail - Nb - level tail records - -] -*/ -func (s WALRecordAppendMeasures) WriteTo(w *bytes.Buffer) { - w.WriteByte(CodeAppendedMeasures) - bin.WriteUint32(w, s.MetricID) - // completed data page - completed := s.CompletedDataPage - bin.WriteUint32(w, completed.PageNo) - bin.WriteBool(w, completed.Reused) - bin.WriteUint32(w, completed.PrevPageNo) - bin.WriteUint32(w, completed.Checksum) - bin.WriteVarSize(w, completed.TimestampsOffset) - bin.WriteVarSize(w, len(completed.Timestamps)) - w.Write(completed.Timestamps) - bin.WriteVarSize(w, completed.ValuesOffset) - bin.WriteVarSize(w, len(completed.Values)) - w.Write(completed.Values) - // data pages - bin.WriteVarSize(w, len(s.DataPages)) - for _, p := range s.DataPages { - bin.WriteUint32(w, p.PageNo) - bin.WriteBool(w, p.Reused) - bin.WriteVarSize(w, len(p.Content)) - w.Write(p.Content) - } - // data tail - bin.WriteVarSize(w, len(s.TailTimestamps)) - w.Write(s.TailTimestamps) - bin.WriteVarSize(w, len(s.TailValues)) - w.Write(s.TailValues) - // changed levels - bin.WriteVarSize(w, len(s.ChangedIndexLevels)) - for _, level := range s.ChangedIndexLevels { - // completed index page - completed := level.CompletedIndexPage - bin.WriteUint32(w, completed.PageNo) - bin.WriteBool(w, completed.Reused) - bin.WriteUint32(w, completed.Checksum) - bin.WriteVarSize(w, len(completed.Records)/indexRecordSize) - w.Write(completed.Records) - // full index pages - for _, p := range level.IndexPages { - bin.WriteUint32(w, p.PageNo) - bin.WriteBool(w, p.Reused) - bin.WriteVarSize(w, len(p.Content)) - w.Write(p.Content) - } - // level tail - bin.WriteVarSize(w, len(level.TailRecords)/indexRecordSize) - w.Write(level.TailRecords) - } -} - -func (s *WALRecordAppendMeasures) Parse(r *bytes.Buffer) (err error) { - var ( - size int - recordsCount int - ) - s.MetricID, err = bin.ReadUint32(r) - if err != nil { - return - } - s.CompletedDataPage.PageNo, err = bin.ReadUint32(r) - if err != nil { - return - } - s.CompletedDataPage.Reused, err = bin.ReadBool(r) - if err != nil { - return - } - s.CompletedDataPage.PrevPageNo, err = bin.ReadUint32(r) - if err != nil { - return - } - s.CompletedDataPage.Checksum, err = bin.ReadUint32(r) - if err != nil { - return - } - s.CompletedDataPage.TimestampsOffset, err = bin.ReadVarSize(r) - if err != nil { - return - } - size, err = bin.ReadVarSize(r) - if err != nil { - return - } - s.CompletedDataPage.Timestamps, err = bin.ReadN(r, size) - if err != nil { - return - } - size, err = bin.ReadVarSize(r) - if err != nil { - return - } - s.CompletedDataPage.Values, err = bin.ReadN(r, size) - if err != nil { - return - } - // data pages - dataPagesQty, err := bin.ReadVarSize(r) - if err != nil { - return - } - for range dataPagesQty { - var ( - p SealedDataPage - ) - p.PageNo, err = bin.ReadUint32(r) - if err != nil { - return - } - p.Reused, err = bin.ReadBool(r) - if err != nil { - return - } - size, err = bin.ReadVarSize(r) - if err != nil { - return - } - p.Content, err = bin.ReadN(r, size) - if err != nil { - return - } - s.DataPages = append(s.DataPages, p) - } - // data tail - size, err = bin.ReadVarSize(r) - if err != nil { - return - } - s.TailTimestamps, err = bin.ReadN(r, size) - if err != nil { - return - } - size, err = bin.ReadVarSize(r) - if err != nil { - return - } - s.TailValues, err = bin.ReadN(r, size) - if err != nil { - return - } - // changed levels - changedIndexLevelsCount, err := bin.ReadVarSize(r) - if err != nil { - return - } - for range changedIndexLevelsCount { - var ( - indexPagesCount int - level ChangedIndexLevel - ) - // completed index page - level.CompletedIndexPage.PageNo, err = bin.ReadUint32(r) - if err != nil { - return - } - level.CompletedIndexPage.Reused, err = bin.ReadBool(r) - if err != nil { - return - } - level.CompletedIndexPage.Checksum, err = bin.ReadUint32(r) - if err != nil { - return - } - recordsCount, err = bin.ReadVarSize(r) - if err != nil { - return - } - level.CompletedIndexPage.Records, err = bin.ReadN(r, recordsCount*indexRecordSize) - if err != nil { - return - } - // - indexPagesCount, err = bin.ReadVarSize(r) - if err != nil { - return - } - for range indexPagesCount { - var p SealedIndexPage - - p.PageNo, err = bin.ReadUint32(r) - if err != nil { - return - } - p.Reused, err = bin.ReadBool(r) - if err != nil { - return - } - p.Checksum, err = bin.ReadUint32(r) - if err != nil { - return - } - size, err = bin.ReadVarSize(r) - if err != nil { - return - } - p.Content, err = bin.ReadN(r, size) - if err != nil { - return - } - level.IndexPages = append(level.IndexPages, p) - } - recordsCount, err = bin.ReadVarSize(r) - if err != nil { - return - } - level.TailRecords, err = bin.ReadN(r, recordsCount*indexRecordSize) - if err != nil { - return - } - s.ChangedIndexLevels = append(s.ChangedIndexLevels, level) - } - return -} diff --git a/txlog/writer.go b/txlog/writer.go deleted file mode 100644 index 567e80c..0000000 --- a/txlog/writer.go +++ /dev/null @@ -1,465 +0,0 @@ -package txlog - -import ( - "bytes" - "errors" - "fmt" - "log" - "math" - "os" - "path/filepath" - "sync" - - "gordenko.dev/dima/qb" - "gordenko.dev/dima/qb/atree" - "gordenko.dev/dima/qb/freelist" -) - -const ( - lsnSize = 4 - packetPrefixSize = 12 // 4 lsn + 4 packet length + 4 crc32 - - lengthIdx = 0 - checksumIdx = 4 - lsnIdx = 8 - - filePerm = 0770 - - dumpSnapshotAfterNBytes = 1024 * 1024 * 1024 // 1 GB - - writeBufferSize = 4 * 1024 * 1024 -) - -const ( - CodeAddedMetric byte = 1 - CodeDeletedMetric byte = 2 - CodeAppendedMeasure byte = 4 - CodeAppendedMeasures byte = 5 - CodeAppendedPages byte = 6 - CodeDeletedMeasures byte = 7 -) - -type FreeList interface { - // використовується в allocPage - GetPageNumber() (uint32, error) -} - -func JoinChangesFileName(dir string, logNumber int) string { - return filepath.Join(dir, fmt.Sprintf("%d.changes", logNumber)) -} - -type MetricsState struct { - //Metrics []Metric - WaitCh chan struct{} -} - -type Changes struct { - Records []any - SnapshotNumberCh chan int // log number - FrozenIndexPagesCount int - IndexPageNumbers []uint32 - FrozenDataPagesCount int - DataPageNumbers []uint32 -} - -type Writer struct { - mutex sync.Mutex - dataPagesCount uint32 - indexPagesCount uint32 - dataFreeList *freelist.FreeList - indexFreeList *freelist.FreeList - atree *atree.Atree - //logNumber int - dir string - w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск - wal *os.File - dataFile *os.File - indexFile *os.File - indexPageSize int - dataPageSize int - input []any - dataPreparer *DataPreparer - appendToWorkerQueue func(any) - //lsn uint32 - written int64 - isExited bool - exitCh chan struct{} - waitGroup *sync.WaitGroup - signalCh chan struct{} -} - -type WriterOptions struct { - IndexPageSize int - DataPageSize int - Dir string - SnapshotNumber int // номер журнала - AppendToWorkerQueue func(any) - DataFreeList *freelist.FreeList - IndexFreeList *freelist.FreeList - Atree *atree.Atree - ExitCh chan struct{} - WaitGroup *sync.WaitGroup -} - -func NewWriter(opt WriterOptions) (*Writer, error) { - if (opt.IndexPageSize % 2) != 0 { - return nil, errors.New("IndexPageSize must be multiple of 2") - } - if (opt.DataPageSize % 2) != 0 { - return nil, errors.New("DataPageSize must be multiple of 2") - } - if opt.Dir == "" { - return nil, errors.New("Dir option is required") - } - if opt.AppendToWorkerQueue == nil { - return nil, errors.New("AppendToWorkerQueue option is required") - } - if opt.DataFreeList == nil { - return nil, errors.New("DataFreeList option is required") - } - if opt.IndexFreeList == nil { - return nil, errors.New("IndexFreeList option is required") - } - if opt.Atree == nil { - return nil, errors.New("Atree option is required") - } - if opt.ExitCh == nil { - return nil, errors.New("ExitCh option is required") - } - if opt.WaitGroup == nil { - return nil, errors.New("WaitGroup option is required") - } - - s := &Writer{ - indexPageSize: opt.IndexPageSize, - dataPageSize: opt.DataPageSize, - dir: opt.Dir, - appendToWorkerQueue: opt.AppendToWorkerQueue, - dataFreeList: opt.DataFreeList, - indexFreeList: opt.IndexFreeList, - atree: opt.Atree, - exitCh: opt.ExitCh, - waitGroup: opt.WaitGroup, - signalCh: make(chan struct{}, 1), - } - - var err error - - if opt.SnapshotNumber <= 0 { - log.Fatalln("FIX FUCK NUMBER") - } - s.wal, err = os.OpenFile( - JoinChangesFileName(opt.Dir, opt.SnapshotNumber), - os.O_APPEND|os.O_WRONLY, - filePerm, - ) - if err != nil { - return nil, err - } - return s, nil -} - -func (s *Writer) Run() { - for { - select { - case <-s.signalCh: - if err := s.packAndWrite(); err != nil { - qb.Abort(qb.FailedWriteToTxLog, err) - } - - case <-s.exitCh: - s.exit() - return - } - } -} - -func (s *Writer) getDataPageNumber() (uint32, bool, error) { - pageNo, err := s.dataFreeList.GetPageNumber() - if err != nil { - return 0, false, err - } - if pageNo > 0 { - return pageNo, true, nil - } - if s.dataPagesCount < math.MaxUint32 { - s.dataPagesCount++ - return s.dataPagesCount, false, nil - } - return 0, false, errors.New("no space") -} - -func (s *Writer) getIndexPageNumber() (uint32, bool, error) { - pageNo, err := s.indexFreeList.GetPageNumber() - if err != nil { - return 0, false, err - } - if pageNo > 0 { - return pageNo, true, nil - } - if s.indexPagesCount < math.MaxUint32 { - s.indexPagesCount++ - return s.indexPagesCount, false, nil - } - return 0, false, errors.New("no space") -} - -func (s *Writer) packAndWrite() (err error) { - s.mutex.Lock() - //isExited := s.isExited - input := s.input - s.input = nil - - // var exitWaitGroup *sync.WaitGroup - // if s.isExited { - // exitWaitGroup = s.waitGroup - // } - s.mutex.Unlock() - - prepared := s.dataPreparer.Prepare(input) - - // 3. Пишу на диск WAL (append) - n, err := s.wal.Write(prepared.Packet) - if err != nil { - return - } - if n != len(prepared.Packet) { - return fmt.Errorf("written %d != total size %d", n, len(prepared.Packet)) - } - if err = s.wal.Sync(); err != nil { - return - } - - // 4. Пишу в atree сторінки - err = s.WritePagesToAtree(prepared.WriteToIndex, prepared.WriteToData) - if err != nil { - return - } - - // 6. відправляю input - worker-у - - var ( - forceSnapshot bool - ) - - if s.written > dumpSnapshotAfterNBytes { - forceSnapshot = true - } - - // if isExited && s.written > 0 { - // forceSnapshot = true - // } - - if forceSnapshot { - if err := s.wal.Close(); err != nil { - return fmt.Errorf("close wal file: %s", err) - } - s.wal = nil - s.written = 0 - - snapshotNumberCh := make(chan int, 1) - - s.appendToWorkerQueue(Changes{ - Records: prepared.WriteResults, - SnapshotNumberCh: snapshotNumberCh, - FrozenIndexPagesCount: s.indexFreeList.Pages(), - IndexPageNumbers: s.indexFreeList.Cached(), - FrozenDataPagesCount: s.dataFreeList.Pages(), - DataPageNumbers: s.dataFreeList.Cached(), - }) - - snapshotNumber := <-snapshotNumberCh - - // копіюю сторінки із delta файла в base файл і потім роблю Truncate - err = s.indexFreeList.Merge() - if err != nil { - return - } - err = s.dataFreeList.Merge() - if err != nil { - return - } - - var err error - s.wal, err = os.OpenFile( - JoinChangesFileName(s.dir, snapshotNumber), - os.O_CREATE|os.O_WRONLY, - filePerm, - ) - if err != nil { - return fmt.Errorf("create new changes file: %s", err) - } - } else { - s.appendToWorkerQueue(Changes{ - Records: prepared.WriteResults, - }) - } - return nil -} - -func (s *Writer) Append(req any) { - s.mutex.Lock() - s.input = append(s.input, req) - s.mutex.Unlock() - - select { - case s.signalCh <- struct{}{}: - default: - } -} - -// func (s *Writer) reset() { -// s.buf.Reset() -// s.buf.Write([]byte{ -// 0, 0, 0, 0, // packet length -// 0, 0, 0, 0, // crc32 -// 0, 0, 0, 0, // lsn - -// }) -// s.pagesToWrite = nil -// s.workerReqs = nil -// s.waitCh = make(chan struct{}) -// } - -// func (s *Writer) flush(w *bytes.Buffer) error { - -// return nil -// } - -func (s *Writer) exit() { - s.mutex.Lock() - s.isExited = true - s.mutex.Unlock() - - if err := s.packAndWrite(); err != nil { - qb.Abort(qb.FailedWriteToTxLog, err) - } -} - -// type IndexPayloadToWrite struct { -// PageNo uint32 -// Payload []byte -// ZeroLevel bool -// } - -type PageToWrite struct { - PageNo uint32 - Content []byte -} - -// writePagesToAtree - записує сторінки в .data та .index файли -func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToWrite) (err error) { - for _, p := range dataPages { - if len(p.Content) != s.dataPageSize { - return fmt.Errorf("wrong data page size: %d", len(p.Content)) - } - var ( - off = int(p.PageNo-1) * s.dataPageSize - n int - ) - n, err = s.dataFile.WriteAt(p.Content, int64(off)) - if err != nil { - return - } - if n != s.dataPageSize { - return fmt.Errorf("write %d instead of %d", n, s.dataPageSize) - } - } - for _, p := range indexPages { - if len(p.Content) != s.indexPageSize { - return fmt.Errorf("wrong index page size: %d", len(p.Content)) - } - var ( - off = int(p.PageNo-1) * s.indexPageSize - n int - ) - n, err = s.indexFile.WriteAt(p.Content, int64(off)) - if err != nil { - return - } - if n != s.indexPageSize { - return fmt.Errorf("write %d instead of %d", n, s.indexPageSize) - } - } - return nil -} - -// API - -// FIX - add -// type DeletedMeasuresSince struct { -// MetricID uint32 -// LastPageNo uint32 -// IsRootChanged bool -// RootPageNo uint32 -// FreeDataPages []uint32 -// FreeIndexPages []uint32 -// TimestampsBuf []byte -// ValuesBuf []byte -// } - -func (s *Writer) sendSignal() { - select { - case s.signalCh <- struct{}{}: - default: - } -} - -// Якщо (Pages) > 0 - незаповнену сторінку кодую в стисненому вигляді. -// Якщо (Pages) = 0 - кодую просто пари timestamp / value -// func (s *Writer) packWriteAppended(w *bytes.Buffer, req AppendedPagesReq) { -// // завантажений path. Отже просто додаємо в дерево data сторінки, створюємо нові індексні -// // без звернення до диску. -// report := s.atree.AppendDataPages(atree.AppendDataPagesReq{ -// LastPageNo: req.LastPageNo, -// Legs: req.Legs, -// DataPages: req.Pages, -// }) - -// m := AppendedPages{ -// MetricID: req.MetricID, -// Timestamp: req.Timestamp, -// Value: req.Value, -// NewRootPageNo: report.NewRootPageNo, -// LastPageNo: report.LastPageNo, -// Pages: report.Pages, -// TimestampsChunks: req.TimestampsChunks, -// TimestampsSize: int(req.TimestampsSize), -// ValuesChunks: req.ValuesChunks, -// ValuesSize: int(req.ValuesSize), -// } -// m.Pack(w) -// } - -// helpers - -// type Metric struct { -// MetricID uint32 -// MetricType qb.MetricType -// FracDigits byte -// LastPageNo uint32 -// Since uint32 -// SinceValue float64 -// Until uint32 -// UntilValue float64 -// Timestamps []byte // payload FIX -// Values []byte // payload FIX add lastDelta + h ? -// } - -// func writeChunks(dst io.Writer, chunks [][]byte, size int) (err error) { -// remaining := size -// for _, buf := range chunks { -// if remaining < len(buf) { -// buf = buf[:remaining] -// } -// _, err = dst.Write(buf) -// if err != nil { -// return -// } -// remaining -= len(buf) -// if remaining == 0 { -// break -// } -// } -// return -// }