diff --git a/atree/atree.go b/atree/atree.go index 3d6bf02..3280648 100644 --- a/atree/atree.go +++ b/atree/atree.go @@ -8,7 +8,6 @@ import ( "path/filepath" "sync" - diploma "gordenko.dev/dima/qb" "gordenko.dev/dima/qb/bin" ) @@ -19,8 +18,8 @@ const ( filePerm = 0770 // common - crc32Idx = PageSize - 4 - pageType = PageSize - 5 + crc32Idx = PageSize - 4 + pageTypeIdx = PageSize - 5 // index page indexRecordsQtyIdx = PageSize - 8 @@ -73,16 +72,16 @@ type _page struct { } type Atree struct { - freelist FreeList file *os.File mutex sync.Mutex + freelist FreeList allocatedPagesQty uint32 pages map[uint32]*_page pageWaits map[uint32][]chan readResult pagesToRead []uint32 readSignalCh chan struct{} writeSignalCh chan struct{} - writeTasksQueue []WriteTask + //writeTasksQueue []WriteTask } type Options struct { @@ -131,7 +130,7 @@ func New(opt Options) (*Atree, error) { } tree := &Atree{ - //freelist: opt.FreeList, + freelist: opt.FreeList, file: file, allocatedPagesQty: allocatedPagesQty, pages: make(map[uint32]*_page), @@ -144,7 +143,7 @@ func New(opt Options) (*Atree, error) { } func (s *Atree) Run() { - go s.pageWriter() + //go s.pageWriter() go s.pageReader() } @@ -161,6 +160,7 @@ func (s *Atree) findDataPage(rootPageNo uint32, timestamp uint32) (uint32, []byt foundPageNo := findPageNo(buf, timestamp) s.releasePage(indexPageNo) + // fix if buf[isDataPageNumbersIdx] == 1 { buf, err := s.fetchDataPage(foundPageNo) if err != nil { @@ -173,20 +173,20 @@ func (s *Atree) findDataPage(rootPageNo uint32, timestamp uint32) (uint32, []byt } } -type pathLeg struct { +type PathLeg struct { PageNo uint32 Data []byte } -type pathToDataPage struct { - Legs []pathLeg +type PathToDataPage struct { + Legs []PathLeg LastPageNo uint32 } -func (s *Atree) findPathToLastPage(rootPageNo uint32) (_ pathToDataPage, err error) { +func (s *Atree) FindPathToLastPage(rootPageNo uint32) (_ PathToDataPage, err error) { var ( pageNo = rootPageNo - legs []pathLeg + legs []PathLeg ) for { @@ -197,7 +197,7 @@ func (s *Atree) findPathToLastPage(rootPageNo uint32) (_ pathToDataPage, err err return } - legs = append(legs, pathLeg{ + legs = append(legs, PathLeg{ PageNo: pageNo, Data: buf, // childIdx не нужен @@ -205,8 +205,9 @@ func (s *Atree) findPathToLastPage(rootPageNo uint32) (_ pathToDataPage, err err foundPageNo := getLastPageNo(buf) + // fix if buf[isDataPageNumbersIdx] == 1 { - return pathToDataPage{ + return PathToDataPage{ Legs: legs, LastPageNo: foundPageNo, }, nil @@ -218,26 +219,19 @@ func (s *Atree) findPathToLastPage(rootPageNo uint32) (_ pathToDataPage, err err // APPEND DATA PAGE +//MetricID uint32 +//Timestamp uint32 +//Value float64 +//RootPageNo uint32 +//PrevPageNo uint32 + type AppendDataPageReq struct { - MetricID uint32 - Timestamp uint32 - Value float64 Since uint32 - RootPageNo uint32 - PrevPageNo uint32 TimestampsChunks [][]byte TimestampsSize uint16 ValuesChunks [][]byte ValuesSize uint16 } -type Report struct { - //IsDataPageReused bool - //DataPageNo uint32 - IsRootChanged bool - NewRootPageNo uint32 - //ReusedIndexPages []uint32 - Pages []PageToWrite -} // type ChangedPage struct { // PageNo uint32 @@ -253,186 +247,165 @@ type Report struct { // Тому що є проблема із відновленням FreeList після збою, якщо з нього будуть паралельно // забирати та добавляти номери сторінок інші потоки. // Це буде працювати, якщо додавання в txlog і маніпуляції із freeList будуть відбуватись в одному потоці -func (s *Atree) AppendDataPage(req AppendDataPageReq) (_ Report, err error) { +// NotLinkedDataPage - це data сторінка із payload, але без встановленого prevPageNo та без розрахованого CRC32 +type NotLinkedDataPage struct { + Since uint32 + Data []byte +} + +func (s NotLinkedDataPage) SetPrevPageNo(prevPageNo uint32) { + bin.PutUint32(s.Data[prevPageIdx:], prevPageNo) + bin.PutUint32(s.Data[crc32Idx:], calcChecksum(s.Data[:crc32Idx])) +} + +type AppendDataPagesReq struct { + LastPageNo uint32 + Legs []PathLeg + DataPages []NotLinkedDataPage +} + +type Report struct { + NewRootPageNo uint32 + LastPageNo uint32 + Pages []PageToWrite +} + +// Можливо atree не треба блокувати при читанні. Я можу створити копії index сторінок, +// а потім під 1 мутексом замінити в pages. +func (s *Atree) AppendDataPages(req AppendDataPagesReq) Report { var ( - pagesToRelease []uint32 - report Report + //pagesToRelease []uint32 + newRootPageNo uint32 + lastPageNo = req.LastPageNo + pages []PageToWrite + legs = req.Legs + changed = make(map[uint32]PathLeg) ) - newDataPage := s.allocPage() - pagesToRelease = append(pagesToRelease, newDataPage.PageNo) + for _, p := range req.DataPages { + newDataPageNo, isReused := s.allocPageNumber() // alloc only number + // pagesToRelease = append(pagesToRelease, newDataPage.PageNo) - chunksToDataPage(newDataPage.Data, chunksToDataPageReq{ - PrevPageNo: req.PrevPageNo, - TimestampsChunks: req.TimestampsChunks, - TimestampsSize: req.TimestampsSize, - ValuesChunks: req.ValuesChunks, - ValuesSize: req.ValuesSize, - }) + // set prevPageNo + p.SetPrevPageNo(lastPageNo) - report.Pages = append(report.Pages, PageToWrite{ - PageNo: newDataPage.PageNo, - Data: newDataPage.Data, - IsReused: newDataPage.IsReused, - }) - - // redoWriter, err := NewWriter(WriterOptions{ - // MetricID: req.MetricID, - // Timestamp: req.Timestamp, - // Value: req.Value, - // IsDataPageReused: newDataPage.IsReused, - // DataPageNo: newDataPage.PageNo, - // Page: newDataPage.Data, - // }) - // if err != nil { - // return - // } - - if req.RootPageNo > 0 { - var path pathToDataPage - path, err = s.findPathToLastPage(req.RootPageNo) - if err != nil { - return - } - for _, leg := range path.Legs { - pagesToRelease = append(pagesToRelease, leg.PageNo) - } - - if path.LastPageNo != req.PrevPageNo { - diploma.Abort( - diploma.WrongPrevPageNo, - fmt.Errorf("bug: last pageNo %d in tree != prev pageNo %d in _metric", - path.LastPageNo, req.PrevPageNo), - ) - } - - newPageNo := newDataPage.PageNo - lastIdx := len(path.Legs) - 1 - - for legIdx := lastIdx; legIdx >= 0; legIdx-- { - leg := path.Legs[legIdx] - - ok := appendPair(leg.Data, req.Since, newPageNo) - if ok { - // index - report.Pages = append(report.Pages, PageToWrite{ - PageNo: leg.PageNo, - Data: leg.Data, - }) - // err = redoWriter.AppendIndexPage(leg.PageNo, leg.Data, 0) - // if err != nil { - // return - // } - break - } - - newIndexPage := s.allocPage() - pagesToRelease = append(pagesToRelease, newIndexPage.PageNo) - appendPair(newIndexPage.Data, req.Since, newPageNo) - // ставлю мітку що всі pageNo на сторінці - це data pageNo - if legIdx == lastIdx { - newIndexPage.Data[isDataPageNumbersIdx] = 1 - } - - // flags = 0 - // if newIndexPage.IsReused { - // flags |= FlagReused - // } - report.Pages = append(report.Pages, PageToWrite{ - PageNo: newIndexPage.PageNo, - Data: newIndexPage.Data, - IsReused: newIndexPage.IsReused, - }) - // err = redoWriter.AppendIndexPage(newIndexPage.PageNo, newIndexPage.Data, flags) - // if err != nil { - // return - // } - // - newPageNo = newIndexPage.PageNo - - if legIdx == 0 { - newRoot := s.allocPage() - pagesToRelease = append(pagesToRelease, newRoot.PageNo) - appendPair(newRoot.Data, getSince(leg.Data), leg.PageNo) // old rootPageNo - appendPair(newRoot.Data, req.Since, newIndexPage.PageNo) - - // Фиксирую новый root в REDO логе - - report.Pages = append(report.Pages, PageToWrite{ - PageNo: newRoot.PageNo, - Data: newRoot.Data, - IsReused: newRoot.IsReused, - }) - report.NewRootPageNo = newRoot.PageNo - // flags = FlagNewRoot - // if newRoot.IsReused { - // flags |= FlagReused - // } - // err = redoWriter.AppendIndexPage(newRoot.PageNo, newRoot.Data, flags) - // if err != nil { - // return - // } - break - } - } - } else { - newRoot := s.allocPage() - pagesToRelease = append(pagesToRelease, newRoot.PageNo) - newRoot.Data[isDataPageNumbersIdx] = 1 - appendPair(newRoot.Data, req.Since, newDataPage.PageNo) - - report.Pages = append(report.Pages, PageToWrite{ - PageNo: newRoot.PageNo, - Data: newRoot.Data, - IsReused: newRoot.IsReused, + pages = append(pages, PageToWrite{ + PageNo: newDataPageNo, + Data: p.Data, + IsReused: isReused, }) - report.NewRootPageNo = newRoot.PageNo - // flags = FlagNewRoot - // if newRoot.IsReused { - // flags |= FlagReused - // } - // err = redoWriter.AppendIndexPage(newRoot.PageNo, newRoot.Data, flags) - // if err != nil { - // return + + // FIX - після додавання index page потрібно модифікувати path, + // або будувати щось типу дерева знизу вверх + + if len(legs) > 0 { + newPageNo := newDataPageNo + lastIdx := len(legs) - 1 + + for legIdx := lastIdx; legIdx >= 0; legIdx-- { + leg := req.Legs[legIdx] + ok := appendPair(leg.Data, p.Since, newPageNo) + if ok { + // index FIX + // потрібно запам'ятати змінені сторінки, але їх можуть змінювати + // кілька ітерацій, тому додавати в Pages не можна. + // на індексній сторінці достатньо місця. Запис вставлено. + changed[leg.PageNo] = leg + break + } + // на індексній сторінці НЕ достатньо місця. Створюю нову. + newIndexPage := s.allocPage() + //pagesToRelease = append(pagesToRelease, newIndexPage.PageNo) + appendPair(newIndexPage.Data, p.Since, newPageNo) + // ставлю мітку що всі pageNo на сторінці - це data pageNo + // fix - єдина оптимізація від існування isDataPageNumbersIdx - getAllPages не завантажує data pages + // if legIdx == lastIdx { + // newIndexPage.Data[isDataPageNumbersIdx] = 1 + // } + pages = append(pages, PageToWrite{ + PageNo: newIndexPage.PageNo, + Data: newIndexPage.Data, + IsReused: newIndexPage.IsReused, + }) + // замінюю крок в path на новий + legs[legIdx] = PathLeg{ + PageNo: newIndexPage.PageNo, + Data: newIndexPage.Data, + } + // + newPageNo = newIndexPage.PageNo + + if legIdx == 0 { + newRoot := s.allocPage() + //pagesToRelease = append(pagesToRelease, newRoot.PageNo) + appendPair(newRoot.Data, getSince(leg.Data), leg.PageNo) // old rootPageNo + appendPair(newRoot.Data, p.Since, newIndexPage.PageNo) + + // Фиксирую новый root в REDO логе + pages = append(pages, PageToWrite{ + PageNo: newRoot.PageNo, + Data: newRoot.Data, + IsReused: newRoot.IsReused, + }) + newRootPageNo = newRoot.PageNo + // додаю в початок списку кроків нову root сторінку + legs = append([]PathLeg{ + { + PageNo: newRoot.PageNo, + Data: newRoot.Data, + }, + }, legs...) + break + } + } + } else { + // індексну root сторінку створюю одразу для першої data сторінки, + // root data сторінки, як в B+Tree не буває. + newRoot := s.allocPage() + //pagesToRelease = append(pagesToRelease, newRoot.PageNo) + //newRoot.Data[isDataPageNumbersIdx] = 1 + appendPair(newRoot.Data, p.Since, newDataPageNo) + + pages = append(pages, PageToWrite{ + PageNo: newRoot.PageNo, + Data: newRoot.Data, + IsReused: newRoot.IsReused, + }) + newRootPageNo = newRoot.PageNo + // додаю в початок списку кроків нову root сторінку + legs = append([]PathLeg{ + { + PageNo: newRoot.PageNo, + Data: newRoot.Data, + }, + }, legs...) + } + + // На данний момен схема - наступна. Всі сторінки - data та index - зафіксовані в кеші. + // Отже запис на диск пройде максимально швидко. Після цього ReferenceCount кожної + // сторінки зменшиться на 1. Оскільки на метрику утримується XLock, сторінки мають + // ReferenceCount = 1 (немає інших читачів). + // for _, pageNo := range indexPagesToRelease { + // s.releasePage(pageNo) // } + lastPageNo = newDataPageNo + } + for _, leg := range changed { + // fix recalc checksum + pages = append(pages, PageToWrite{ + PageNo: leg.PageNo, + Data: leg.Data, + }) + } + return Report{ + NewRootPageNo: newRootPageNo, + LastPageNo: lastPageNo, + Pages: pages, } - - // err = redoWriter.Close() - // if err != nil { - // return - // } - - // На данний момен схема - наступна. Всі сторінки - data та index - зафіксовані в кеші. - // Отже запис на диск пройде максимально швидко. Після цього ReferenceCount кожної - // сторінки зменшиться на 1. Оскільки на метрику утримується XLock, сторінки мають - // ReferenceCount = 1 (немає інших читачів). - // waitCh := make(chan struct{}) - - // task := WriteTask{ - // WaitCh: waitCh, - // Pages: report.Pages, - // } - - // s.appendWriteTaskToQueue(task) - - // <-waitCh - - // for _, pageNo := range dataPagesToRelease { - // s.releasePage(pageNo) - // } - // for _, pageNo := range indexPagesToRelease { - // s.releasePage(pageNo) - // } - return report, nil } // DELETE -type PageLists struct { - DataPages []uint32 - IndexPages []uint32 -} - type Level struct { PageNo uint32 PageData []byte @@ -440,31 +413,29 @@ type Level struct { ChildQty int } -func (s *Atree) GetAllPages(rootPageNo uint32) (_ PageLists, err error) { +func (s *Atree) GetAllPages(rootPageNo uint32) (_ []uint32, err error) { var ( - dataPages []uint32 - indexPages []uint32 - levels []*Level + pageNumbers []uint32 + levels []*Level ) buf, err := s.fetchIndexPage(rootPageNo) if err != nil { - err = fmt.Errorf("fetchIndexPage(%d): %s", rootPageNo, err) - return + return nil, fmt.Errorf("fetchIndexPage(%d): %s", rootPageNo, err) } - indexPages = append(indexPages, rootPageNo) + pageNumbers = append(pageNumbers, rootPageNo) - if buf[isDataPageNumbersIdx] == 1 { - pageNumbers := listPageNumbers(buf) - dataPages = append(dataPages, pageNumbers...) + // if buf[isDataPageNumbersIdx] == 1 { + // pageNumbers := listPageNumbers(buf) + // dataPages = append(dataPages, pageNumbers...) - s.releasePage(rootPageNo) + // s.releasePage(rootPageNo) - return PageLists{ - DataPages: dataPages, - IndexPages: indexPages, - }, nil - } + // return PageLists{ + // DataPages: dataPages, + // IndexPages: indexPages, + // }, nil + // } levels = append(levels, &Level{ PageNo: rootPageNo, @@ -475,10 +446,7 @@ func (s *Atree) GetAllPages(rootPageNo uint32) (_ PageLists, err error) { for { if len(levels) == 0 { - return PageLists{ - DataPages: dataPages, - IndexPages: indexPages, - }, nil + return pageNumbers, nil } lastIdx := len(levels) - 1 @@ -489,17 +457,15 @@ func (s *Atree) GetAllPages(rootPageNo uint32) (_ PageLists, err error) { level.Idx++ var buf []byte - buf, err = s.fetchIndexPage(pageNo) + buf, err = s.fetchPage(pageNo) if err != nil { - err = fmt.Errorf("fetchIndexPage(%d): %s", pageNo, err) - return + return nil, fmt.Errorf("fetchPage(%d): %s", pageNo, err) } - indexPages = append(indexPages, pageNo) - - if buf[isDataPageNumbersIdx] == 1 { - pageNumbers := listPageNumbers(buf) - dataPages = append(dataPages, pageNumbers...) + pageNumbers = append(pageNumbers, pageNo) + if buf[pageTypeIdx] == PageTypeData { + //pageNumbers := listPageNumbers(buf) + //dataPages = append(dataPages, pageNumbers...) s.releasePage(pageNo) } else { levels = append(levels, &Level{ diff --git a/atree/io.go b/atree/io.go index 43251d3..714036f 100644 --- a/atree/io.go +++ b/atree/io.go @@ -34,20 +34,31 @@ func (s *Atree) DeletePages(pageNumbers []uint32) { } func (s *Atree) fetchIndexPage(pageNo uint32) ([]byte, error) { - return s.fetchPage(pageNo, PageTypeIndex) + buf, err := s.fetchPage(pageNo) + if err != nil { + return nil, err + } + if buf[pageTypeIdx] != PageTypeIndex { + return nil, fmt.Errorf("wrong pageType %d instead of %d", buf[pageTypeIdx], PageTypeIndex) + } + return buf, nil } func (s *Atree) fetchDataPage(pageNo uint32) ([]byte, error) { - return s.fetchPage(pageNo, PageTypeData) + buf, err := s.fetchPage(pageNo) + if err != nil { + return nil, err + } + if buf[pageTypeIdx] != PageTypeIndex { + return nil, fmt.Errorf("wrong pageType %d instead of %d", buf[pageTypeIdx], PageTypeIndex) + } + return buf, nil } -func (s *Atree) fetchPage(pageNo uint32, pageTypeIdx byte) ([]byte, error) { +func (s *Atree) fetchPage(pageNo uint32) ([]byte, error) { s.mutex.Lock() p, ok := s.pages[pageNo] if ok { - if p.Buf[pageTypeIdx] != pageTypeIdx { - return nil, fmt.Errorf("wrong pageType %d instead of %d", p.Buf[pageTypeIdx], pageTypeIdx) - } p.ReferenceCount++ s.mutex.Unlock() return p.Buf, nil @@ -93,23 +104,28 @@ func (s *Atree) releasePage(pageNo uint32) { } } +func (s *Atree) allocPageNumber() (uint32, bool) { + // fix + return 0, true +} + func (s *Atree) allocPage() AllocatedPage { var ( allocated = AllocatedPage{ Data: make([]byte, PageSize), } ) - allocated.PageNo = s.freelist.ReservePage() + //allocated.PageNo = s.freelist.ReservePage() s.mutex.Lock() - if allocated.PageNo > 0 { - allocated.IsReused = true - } else { - if s.allocatedPagesQty == math.MaxUint32 { - qb.Abort(qb.MaxAtreeSizeExceeded, errors.New("no space in Atree index")) - } - s.allocatedPagesQty++ - allocated.PageNo = s.allocatedPagesQty - } + // if allocated.PageNo > 0 { + // allocated.IsReused = true + // } else { + // if s.allocatedPagesQty == math.MaxUint32 { + // qb.Abort(qb.MaxAtreeSizeExceeded, errors.New("no space in Atree index")) + // } + // s.allocatedPagesQty++ + // allocated.PageNo = s.allocatedPagesQty + // } // s.pages[allocated.PageNo] = &_page{ // // fix pageType @@ -241,61 +257,61 @@ func (s *Atree) readPages() { // WRITE -func (s *Atree) pageWriter() { - for { - select { - case <-s.writeSignalCh: - err := s.writeTasks() - if err != nil { - qb.Abort(qb.WriteToAtreeFailed, err) - } - } - } -} +// func (s *Atree) pageWriter() { +// for { +// select { +// case <-s.writeSignalCh: +// err := s.writeTasks() +// if err != nil { +// qb.Abort(qb.WriteToAtreeFailed, err) +// } +// } +// } +// } -type WriteTask struct { - WaitCh chan struct{} - Pages []PageToWrite -} +// type WriteTask struct { +// WaitCh chan struct{} +// Pages []PageToWrite +// } -func (s *Atree) appendWriteTaskToQueue(task WriteTask) { - s.mutex.Lock() - s.writeTasksQueue = append(s.writeTasksQueue, task) - s.mutex.Unlock() +// func (s *Atree) appendWriteTaskToQueue(task WriteTask) { +// s.mutex.Lock() +// s.writeTasksQueue = append(s.writeTasksQueue, task) +// s.mutex.Unlock() - select { - case s.writeSignalCh <- struct{}{}: - default: - } -} +// select { +// case s.writeSignalCh <- struct{}{}: +// default: +// } +// } -func (s *Atree) writeTasks() error { - s.mutex.Lock() - tasks := s.writeTasksQueue - s.writeTasksQueue = nil - s.mutex.Unlock() +// func (s *Atree) writeTasks() error { +// s.mutex.Lock() +// tasks := s.writeTasksQueue +// s.writeTasksQueue = nil +// s.mutex.Unlock() - for _, task := range tasks { - for _, p := range task.Pages { - if len(p.Data) != PageSize { - return fmt.Errorf("wrong page %d size: %d", - p.PageNo, len(p.Data)) - } - bin.PutUint32(p.Data[crc32Idx:], calcChecksum(p.Data[:crc32Idx])) +// for _, task := range tasks { +// for _, p := range task.Pages { +// if len(p.Data) != PageSize { +// return fmt.Errorf("wrong page %d size: %d", +// p.PageNo, len(p.Data)) +// } +// bin.PutUint32(p.Data[crc32Idx:], calcChecksum(p.Data[:crc32Idx])) - off := (p.PageNo - 1) * PageSize - n, err := s.file.WriteAt(p.Data, int64(off)) - if err != nil { - return err - } - if n != len(p.Data) { - return fmt.Errorf("write %d instead of %d", n, len(p.Data)) - } - } - close(task.WaitCh) - } - return nil -} +// off := (p.PageNo - 1) * PageSize +// n, err := s.file.WriteAt(p.Data, int64(off)) +// if err != nil { +// return err +// } +// if n != len(p.Data) { +// return fmt.Errorf("write %d instead of %d", n, len(p.Data)) +// } +// } +// close(task.WaitCh) +// } +// return nil +// } // IO @@ -339,9 +355,9 @@ func openFile(fileName string, pageSize int) (_ *os.File, _ uint32, err error) { return file, uint32(allocatedPagesQty), nil } -func (s *Atree) ApplyREDO(task WriteTask) { - s.appendWriteTaskToQueue(task) -} +// func (s *Atree) ApplyREDO(task WriteTask) { +// s.appendWriteTaskToQueue(task) +// } func (s *Atree) verifyCRC(data []byte, pageSize int) error { var ( diff --git a/atree/misc.go b/atree/misc.go index b15826c..fe73400 100644 --- a/atree/misc.go +++ b/atree/misc.go @@ -55,16 +55,15 @@ func BinarySearch(qty int, keyComparator bin.KeyComparator) (elemIdx int, isFoun } } -type chunksToDataPageReq struct { - PrevPageNo uint32 +type ChunksToNotLinkedDataPageReq struct { + // PrevPageNo uint32 TimestampsChunks [][]byte TimestampsSize uint16 ValuesChunks [][]byte ValuesSize uint16 } -func chunksToDataPage(buf []byte, req chunksToDataPageReq) { - bin.PutUint32(buf[prevPageIdx:], req.PrevPageNo) +func ChunksToNotLinkedDataPage(buf []byte, req ChunksToNotLinkedDataPageReq) { bin.PutUint16(buf[timestampsSizeIdx:], req.TimestampsSize) bin.PutUint16(buf[valuesSizeIdx:], req.ValuesSize) @@ -96,7 +95,6 @@ func chunksToDataPage(buf []byte, req chunksToDataPageReq) { break } } - bin.PutUint32(buf[crc32Idx:], calcChecksum(buf[:crc32Idx])) } func setPrevPageNo(buf []byte, pageNo uint32) { diff --git a/chunkenc/chunckenc.go b/chunkenc/chunkenc.go similarity index 60% rename from chunkenc/chunckenc.go rename to chunkenc/chunkenc.go index 3973ca9..0432cca 100644 --- a/chunkenc/chunckenc.go +++ b/chunkenc/chunkenc.go @@ -1,3 +1,5 @@ package chunkenc const eps = 0.000001 + +const flagLiteral = 128 diff --git a/chunkenc/chunkenc_test.go b/chunkenc/chunkenc_test.go new file mode 100644 index 0000000..7d56f2f --- /dev/null +++ b/chunkenc/chunkenc_test.go @@ -0,0 +1,298 @@ +package chunkenc + +import ( + "fmt" + "testing" + + "gordenko.dev/dima/qb/conbuf" +) + +func TestCumdelta(t *testing.T) { + var ( + fracDigits byte = 0 + buf = conbuf.New(nil) + value float64 + done bool + ) + c := NewReverseCumulativeDeltaCompressor(buf, 0, fracDigits) + // c.Append(1.55) + // c.Append(23) + // c.Append(23) + // c.Append(23) + // c.Append(23.5) + + c.Append(130) + c.Append(191) + + c.Append(248) + c.Append(305) + + //fmt.Printf("pos: %d\n", c.pos) + //fmt.Printf("%d\n", buf.Chunks()[0]) + + d := NewReverseCumulativeDeltaDecompressor(buf, c.Size(), fracDigits) + + for range 8 { + value, done = d.NextValue() + fmt.Println(value, done) + } +} + +func TestInsdelta(t *testing.T) { + var ( + fracDigits byte = 2 + buf = conbuf.New(nil) + value float64 + done bool + ) + c := NewReverseInstantDeltaCompressor(buf, 0, fracDigits) + c.Append(-1.55) + c.Append(23) + + //fmt.Printf("pos: %d\n", c.pos) + //fmt.Printf("%d\n", buf.Chunks()[0]) + + c.Append(23) + c.Append(23) + + c.Append(-23.5) + + //fmt.Printf("pos: %d\n", c.pos) + //fmt.Printf("%d\n", buf.Chunks()[0]) + + d := NewReverseInstantDeltaDecompressor(buf, c.Size(), fracDigits) + + for range 8 { + value, done = d.NextValue() + fmt.Println(value, done) + } +} + +func TestTimeDelta(t *testing.T) { + var ( + buf = conbuf.New(nil) + value uint32 + done bool + ) + c := NewReverseTimeDeltaCompressor(buf, 0) + c.Append(10) + + //c.Append(131) + + //fmt.Printf("pos: %d\n", c.pos) + //fmt.Printf("%d\n", buf.Chunks()[0]) + + c.Append(70) + + c.Append(130) + c.Append(191) + + c.Append(248) + c.Append(305) + c.Append(330) + c.Append(390) + c.Append(450) + + // fmt.Printf("pos: %d\n", c.pos) + // fmt.Printf("%d\n", buf.Chunks()[0]) + + //c.Sync() + + bound := c.GetState() + + fmt.Println("AFTER SYNC") + fmt.Printf("pos: %d\n", c.pos) + chunks := buf.Chunks() + if len(chunks) > 0 { + fmt.Printf("%d\n", chunks[0]) + } + + d := NewReverseTimeDeltaDecompressor(buf, c.Size()) + d.RestoreFromBound(bound) + //d.RestoreFromEnd() + + for range 12 { + value, done = d.NextValue() + fmt.Println(value, done) + } +} + +func TestCumdeltaBound(t *testing.T) { + var ( + fracDigits byte = 0 + buf = conbuf.New(nil) + value float64 + done bool + ) + c := NewReverseCumulativeDeltaCompressor(buf, 0, fracDigits) + // c.Append(1.55) + // c.Append(23) + // c.Append(23) + // c.Append(23) + // c.Append(23.5) + c.Append(10) + c.Append(70) + + c.Append(130) + + // fmt.Printf("pos: %d\n", c.pos) + // fmt.Printf("%d\n", buf.Chunks()[0]) + + c.Append(191) + + // fmt.Printf("pos: %d\n", c.pos) + // fmt.Printf("%d\n", buf.Chunks()[0]) + + c.Append(191) + + fmt.Printf("pos: %d\n", c.pos) + fmt.Printf("%d\n", buf.Chunks()[0]) + ////////////////////////// + + bound := c.GetState() + + fmt.Printf("bound: pos=%d, h=%d\n", bound.Pos, bound.H) + + c.Append(248) + c.Append(305) + + fmt.Printf("pos: %d\n", c.pos) + fmt.Printf("%d\n", buf.Chunks()[0]) + + d := NewReverseCumulativeDeltaDecompressor(buf, c.Size(), fracDigits) + d.RestoreFromEnd() + + for range 8 { + value, done = d.NextValue() + fmt.Println(value, done) + } + + boundDecompressor := NewReverseCumulativeDeltaDecompressor(buf, c.Size(), fracDigits) + boundDecompressor.RestoreFromBound(bound) + + fmt.Println("from bound:") + for range 8 { + value, done = boundDecompressor.NextValue() + fmt.Println(value, done) + } +} + +func TestInsdeltaBound(t *testing.T) { + var ( + fracDigits byte = 0 + buf = conbuf.New(nil) + value float64 + done bool + ) + c := NewReverseInstantDeltaCompressor(buf, 0, fracDigits) + // c.Append(1.55) + // c.Append(23) + // c.Append(23) + // c.Append(23) + // c.Append(23.5) + c.Append(10) + c.Append(70) + + c.Append(130) + + // fmt.Printf("pos: %d\n", c.pos) + // fmt.Printf("%d\n", buf.Chunks()[0]) + + c.Append(191) + + // fmt.Printf("pos: %d\n", c.pos) + // fmt.Printf("%d\n", buf.Chunks()[0]) + + c.Append(191) + + fmt.Printf("pos: %d\n", c.pos) + fmt.Printf("%d\n", buf.Chunks()[0]) + ////////////////////////// + + bound := c.GetState() + + fmt.Printf("bound: pos=%d, h=%d\n", bound.Pos, bound.H) + + c.Append(248) + c.Append(305) + + fmt.Printf("pos: %d\n", c.pos) + fmt.Printf("%d\n", buf.Chunks()[0]) + + d := NewReverseInstantDeltaDecompressor(buf, c.Size(), fracDigits) + d.RestoreFromEnd() + + for range 8 { + value, done = d.NextValue() + fmt.Println(value, done) + } + + boundDecompressor := NewReverseInstantDeltaDecompressor(buf, c.Size(), fracDigits) + boundDecompressor.RestoreFromBound(bound) + + fmt.Println("from bound:") + for range 8 { + value, done = boundDecompressor.NextValue() + fmt.Println(value, done) + } +} + +func TestInsdeltaBound2(t *testing.T) { + var ( + fracDigits byte = 0 + buf = conbuf.New(nil) + value float64 + done bool + ) + c := NewReverseInstantDeltaCompressor(buf, 0, fracDigits) + // c.Append(1.55) + // c.Append(23) + // c.Append(23) + // c.Append(23) + // c.Append(23.5) + c.Append(305) + c.Append(248) + + c.Append(191) + + // fmt.Printf("pos: %d\n", c.pos) + // fmt.Printf("%d\n", buf.Chunks()[0]) + + c.Append(191) + + // fmt.Printf("pos: %d\n", c.pos) + // fmt.Printf("%d\n", buf.Chunks()[0]) + + c.Append(130) + + fmt.Printf("pos: %d\n", c.pos) + fmt.Printf("%d\n", buf.Chunks()[0]) + ////////////////////////// + + bound := c.GetState() + + fmt.Printf("bound: pos=%d, h=%d\n", bound.Pos, bound.H) + + c.Append(70) + c.Append(10) + + fmt.Printf("pos: %d\n", c.pos) + fmt.Printf("%d\n", buf.Chunks()[0]) + + d := NewReverseInstantDeltaDecompressor(buf, c.Size(), fracDigits) + d.RestoreFromEnd() + + for range 8 { + value, done = d.NextValue() + fmt.Println(value, done) + } + + boundDecompressor := NewReverseInstantDeltaDecompressor(buf, c.Size(), fracDigits) + boundDecompressor.RestoreFromBound(bound) + + fmt.Println("from bound:") + for range 8 { + value, done = boundDecompressor.NextValue() + fmt.Println(value, done) + } +} diff --git a/chunkenc/cumdelta.go b/chunkenc/cumdelta.go index 22ebe1b..3df8889 100644 --- a/chunkenc/cumdelta.go +++ b/chunkenc/cumdelta.go @@ -2,22 +2,49 @@ package chunkenc import ( "fmt" + "log" "math" - "gordenko.dev/dima/qb/bin" "gordenko.dev/dima/qb/conbuf" ) -// REVERSE +// COMPRESSOR REVERSE + +/* +Формат: +base value (var u64) +delta (var u64) +q (qty of previous deltas; msb=0 - series, msb=1 - non series, low 7 bits - qty 1..128) +delta +delta +delta +q + +Дельта рахується від base value. +Декодування у зворотньому порядку. + +Після base value слідують run або literal блоки. +Run блок - це delta + header byte в кінці. +Literal блок - це від одної до N дельт + header byte в кінці. +Спочатку створюється literal блок. +Якщо для останної дельти додається дублікат, literal блок модифікується - +лічильник зменшується до 1. А остання дельта переміщюється в новий run блок. +Причому лічильник 0 - означає 2 елементи. Приклад: +До: +v1 v2 v3 h-byte(literal, 3) <- v3 +Після: +v1 v2 h-byte(literal, 2) v3 h-byte(run, 2) +*/ type ReverseCumulativeDeltaCompressor struct { - buf *conbuf.ContinuousBuffer - coef float64 - pos int - firstValue float64 - lastDelta uint64 - length uint16 - numIdx int + buf *conbuf.ContinuousBuffer + coef float64 + pos int + baseValue float64 + lastDelta uint64 + lastDeltaSize int + h byte + state *CumulativeDeltaBound } func NewReverseCumulativeDeltaCompressor(buf *conbuf.ContinuousBuffer, size int, fracDigits byte) *ReverseCumulativeDeltaCompressor { @@ -27,236 +54,166 @@ func NewReverseCumulativeDeltaCompressor(buf *conbuf.ContinuousBuffer, size int, } s := &ReverseCumulativeDeltaCompressor{ buf: buf, - pos: size, + pos: size, // перший вільний байт coef: coef, } if size > 0 { - s.restoreState() + u64, _, err := s.buf.GetVarUint64(0) + if err != nil { + log.Fatalf("bug: get base value: %s", err) + } + s.baseValue = float64(u64) / s.coef + s.h = s.buf.GetByte(s.pos - 1) + s.lastDelta, s.lastDeltaSize, err = s.buf.ReverseGetVarUint64(s.pos - 2) + if err != nil { + log.Fatalf("bug: get last delta: %s", err) + } } return s } -func (s *ReverseCumulativeDeltaCompressor) restoreState() { - u64, n, err := s.buf.GetVarUint64(0) - if err != nil { - panic(fmt.Sprintf("bug: get first value: %s", err)) - } - s.firstValue = float64(u64) / s.coef - - if s.pos > n { - pos := s.pos - 1 - idxOf8 := uint(8 - s.buf.GetByte(pos)) - pos-- - s8 := s.buf.GetByte(pos) - pos-- - - var n int - s.lastDelta, n, err = s.buf.ReverseGetVarUint64(pos) - if err != nil { - panic(fmt.Sprintf("bug: get last delta: %s", err)) - } - pos -= n - s.numIdx = pos + 1 - - var flag byte = 1 << idxOf8 - if (s8 & flag) == flag { - s.length, _ = s.buf.DecodeRunLength(pos) - } - } -} - func (s *ReverseCumulativeDeltaCompressor) Size() int { return s.pos } -func (s *ReverseCumulativeDeltaCompressor) CalcRequiredSpace(value float64) int { - if s.pos == 0 { - n := bin.CalcVarUint64Length(uint64(value * s.coef)) - return n + 3 - } - delta := uint64((value-s.firstValue)*s.coef + eps) - if delta == s.lastDelta { - if s.length == 0 { - return 1 - } else { - newLength := s.length + 1 - if newLength < 130 { - return 0 - } else if newLength == 130 { - return 1 - } else { - if newLength < 32769 { - return 0 - } else { - n := bin.CalcVarUint64Length(delta) - n += 2 - s8q := s.buf.GetByte(s.pos - 1) - if s8q == 8 { - n -= 1 - } else { - n -= 2 - } - return n - } - } - } - } else { - n := bin.CalcVarUint64Length(delta) - n += 2 - s8q := s.buf.GetByte(s.pos - 1) - if s8q == 8 { - n -= 1 - } else { - n -= 2 - } - return n - } -} +//func (s *ReverseCumulativeDeltaCompressor) CalcRequiredSpace(value float64) int { +//} func (s *ReverseCumulativeDeltaCompressor) Append(value float64) { if s.pos == 0 { - n := s.buf.PutVarUint64(s.pos, uint64(value*s.coef)) - s.pos += n - s.firstValue = value - s.encodeNewDelta(0, 0, 1) + // base value + s.pos += s.buf.PutVarUint64(s.pos, uint64(value*s.coef)) + s.baseValue = value + s.appendNewLiteral(0) } else { - delta := uint64((value-s.firstValue)*s.coef + eps) + delta := uint64((value-s.baseValue)*s.coef + eps) if delta == s.lastDelta { - if s.length == 0 { - s.length = 2 - s.shiftOnePosToRight() - s.buf.SetByte(s.numIdx-1, 0) - s8q := s.buf.GetByte(s.pos - 1) - s.buf.SetFlag(s.pos-2, 1<<(8-s8q)) - } else { - s.length++ - if s.length < 130 { - s.buf.SetByte(s.numIdx-1, byte(s.length-2)) - } else if s.length == 130 { - s.shiftOnePosToRight() - s.encode2bLength() + if s.h < 128 { + // run блок - отже треба збільшити лічильник + if s.h < 127 { + // increase counter + s.h++ + s.buf.SetByte(s.pos-1, s.h) } else { - if s.length < 32769 { - s.encode2bLength() - } else { - s.appendNewDelta(delta) - } + // не можу збільшити - буде переповнення. Додаю новий literal блок + // counter overflow + s.appendNewLiteral(delta) + } + } else { + // literal блок. + // Якщо в ньому лише одне значення - перетворюю його на run блок. + // Інакше забираю останнє значення із literal блока і додаю новий run блок. + q := s.h & 127 + if q == 0 { // 1 кодується як 0 + s.convertLiteralToRun() + } else { + // забираю останнє значення із literal блоку щоб зробити run блок + s.convertLastFromLiteralToRun() } } } else { - s.appendNewDelta(delta) + if s.h < 127 { // fix 128? + // end of run + s.appendNewLiteral(delta) + } else { + if s.h < 255 { + // encode value from pos - 1, then append h byte + s.appendDeltaToLiteral(delta) + } else { + // overflowed - encode new + s.appendNewLiteral(delta) + } + } } } } -func (s *ReverseCumulativeDeltaCompressor) appendNewDelta(delta uint64) { - s.length = 0 - - s8 := s.buf.GetByte(s.pos - 2) - s8q := s.buf.GetByte(s.pos - 1) - - if s8q == 8 { - s.pos -= 1 - s8 = 0 - s8q = 1 - } else { - s.pos -= 2 - s8q++ - } - - s.encodeNewDelta(delta, s8, s8q) +func (s *ReverseCumulativeDeltaCompressor) convertLastFromLiteralToRun() { + // Зменшую кількість елементів в literal блоці + s.h-- + s.pos -= 1 + s.lastDeltaSize + s.buf.SetByte(s.pos, s.h) // закриваю literal блок + s.pos++ + s.lastDeltaSize = s.buf.ReversePutVarUint64(s.pos, s.lastDelta) + s.pos += s.lastDeltaSize + s.h = 0 // run блок, довжини 2 + s.buf.SetByte(s.pos, s.h) + s.pos++ } -func (s *ReverseCumulativeDeltaCompressor) encodeNewDelta(delta uint64, s8 byte, s8q byte) { +func (s *ReverseCumulativeDeltaCompressor) convertLiteralToRun() { + // Знімаю flagLiteral, а лічильник 0 дорівнює 2 елементам в серії. + s.h = 0 + s.buf.SetByte(s.pos-1, s.h) +} + +func (s *ReverseCumulativeDeltaCompressor) appendDeltaToLiteral(delta uint64) { + s.h++ // збільшую к-сть дельт s.lastDelta = delta - s.numIdx = s.pos - n := s.buf.ReversePutVarUint64(s.pos, s.lastDelta) - s.pos += n - s.buf.SetByte(s.pos, s8) - s.pos++ - s.buf.SetByte(s.pos, s8q) + s.pos-- + s.lastDeltaSize = s.buf.ReversePutVarUint64(s.pos, delta) + s.pos += s.lastDeltaSize + s.buf.SetByte(s.pos, s.h) s.pos++ } -func (s *ReverseCumulativeDeltaCompressor) shiftOnePosToRight() { - s.buf.ShiftOnePosToRight(s.numIdx, s.pos) +func (s *ReverseCumulativeDeltaCompressor) appendNewLiteral(delta uint64) { + s.h = flagLiteral + s.lastDelta = delta + s.lastDeltaSize = s.buf.ReversePutVarUint64(s.pos, delta) + s.pos += s.lastDeltaSize + s.buf.SetByte(s.pos, flagLiteral) // literal, length = 1 s.pos++ - s.numIdx++ -} - -func (s *ReverseCumulativeDeltaCompressor) encode2bLength() { - num := s.length - 2 - s.buf.SetByte(s.numIdx-1, byte(num&127)|128) - s.buf.SetByte(s.numIdx-2, byte(num>>7)) } func (s *ReverseCumulativeDeltaCompressor) DeleteLast() { - var ( - s8q = s.buf.GetByte(s.pos - 1) - s8 = s.buf.GetByte(s.pos - 2) - flag byte = 1 << uint(8-s8q) - ) - if s.length > 0 { - if s.length == 2 { - s.length = 0 - s.buf.UnsetFlag(s.pos-2, flag) - s.buf.ShiftOnePosToLeft(s.numIdx, s.pos) - s.numIdx-- - s.pos-- - } else if s.length < 130 { - s.length-- - s.buf.SetByte(s.numIdx-1, byte(s.length)-2) - } else if s.length == 130 { - s.length-- - s.buf.ShiftOnePosToLeft(s.numIdx, s.pos) - s.numIdx-- - s.pos-- - s.buf.SetByte(s.numIdx-1, byte(s.length)-2) - } else { - s.length-- - s.encode2bLength() - } - } else { - if s8q > 1 { - s8q-- - flag = 1 << uint(8-s8q) - s.pos = s.numIdx + 2 - s.buf.SetByte(s.pos-2, s8) - s.buf.SetByte(s.pos-1, s8q) - } else { - s.pos = s.numIdx + 1 - s.buf.SetByte(s.pos-1, 8) - s8 = s.buf.GetByte(s.pos - 2) - flag = 1 - } - var ( - pos = s.pos - 3 - n int - err error - ) - s.lastDelta, n, err = s.buf.ReverseGetVarUint64(pos) - if err != nil { - panic(err) - } - s.numIdx = pos - n - if (s8 & flag) == flag { - s.length, _ = s.buf.DecodeRunLength(s.numIdx - 1) - } +} + +type CumulativeDeltaBound struct { + Pos int + H byte + LastDelta uint64 + Chunks [][]byte +} + +// delta h +func (s *ReverseCumulativeDeltaCompressor) Lock() { + s.state = &CumulativeDeltaBound{ + Pos: s.pos - 1 - s.lastDeltaSize, + H: s.h, + LastDelta: s.lastDelta, + Chunks: s.buf.Chunks(), } } +func (s *ReverseCumulativeDeltaCompressor) CreateDecompressor(fracDigits byte) *ReverseCumulativeDeltaDecompressor { + if s.state == nil { + d := NewReverseCumulativeDeltaDecompressor(s.buf, s.Size(), fracDigits) + d.RestoreFromEnd() + return d + } + d := NewReverseCumulativeDeltaDecompressor(s.buf, s.Size(), fracDigits) + d.RestoreFromBound(*s.state) + return d +} + +func (s *ReverseCumulativeDeltaCompressor) Unlock() { + s.state = nil +} + +// DECOMPRESSOR + type ReverseCumulativeDeltaDecompressor struct { - buf *conbuf.ContinuousBuffer - pos int - bound int - firstValue float64 - lastValue float64 - length uint16 - coef float64 - idxOf8 uint - s8 byte - step byte + buf *conbuf.ContinuousBuffer + coef float64 + pos int + bound int + baseValue float64 + lastValue float64 + isRun bool + pending int + done bool } func NewReverseCumulativeDeltaDecompressor(buf *conbuf.ContinuousBuffer, size int, fracDigits byte) *ReverseCumulativeDeltaDecompressor { @@ -264,67 +221,91 @@ func NewReverseCumulativeDeltaDecompressor(buf *conbuf.ContinuousBuffer, size in if fracDigits > 0 { coef = math.Pow(10, float64(fracDigits)) } - return &ReverseCumulativeDeltaDecompressor{ - buf: buf, - coef: coef, - pos: size, + u64, n, err := buf.GetVarUint64(0) + if err != nil { + log.Fatalf("bug: get base value: %s", err) } + //fmt.Printf("baseValue: %.2f, bound: %d, pos: %d\n", float64(u64)/coef, n, size) + return &ReverseCumulativeDeltaDecompressor{ + buf: buf, + coef: coef, + pos: size - 1, // last elem + baseValue: float64(u64) / coef, + bound: n, + } +} + +func (s *ReverseCumulativeDeltaDecompressor) RestoreFromEnd() { + if s.pos > s.bound { + // читаю заголовок наступної серії + s.readHeader() + s.readValue() + } else { + s.done = true + } +} + +func (s *ReverseCumulativeDeltaDecompressor) RestoreFromBound(bound CumulativeDeltaBound) { + s.pos = bound.Pos - 1 + s.lastValue = s.baseValue + float64(bound.LastDelta) + s.decodeHeaderByte(bound.H) + fmt.Printf("restore from bound: isRun=%t, pending=%d\n", s.isRun, s.pending) } func (s *ReverseCumulativeDeltaDecompressor) NextValue() (value float64, done bool) { - if s.step > 0 { - if s.length > 0 { - s.length-- - return s.lastValue, false - } - if s.pos < s.bound { - return 0, true - } - if s.idxOf8 == 0 { - s.s8 = s.buf.GetByte(s.pos) - s.pos-- - } - s.readVar() - if s.length > 0 { - s.length-- - } - return s.lastValue, false + //fmt.Printf("NextValue(): bound: %d, pos: %d, pending: %d\n", s.bound, s.pos, s.pending) + if s.done { + return 0, true } - u64, n, err := s.buf.GetVarUint64(0) - if err != nil { - panic(err) + // повертаю значення, що було прочитано в методі RestoreFromBound/RestoreFromEnd + value = s.lastValue + s.pending-- + if s.pending > 0 { + // якщо в серії залишаються елементи + if !s.isRun { + s.readValue() + } + } else if s.pos > s.bound { + // в серії більше немає елементів, отже перевіряє чи є ще дані в буфері. + // дані є - читаю заголовок наступної серії + s.readHeader() + s.readValue() + } else { + s.done = true } - s.firstValue = float64(u64) / s.coef - s.bound = n - s.pos-- - s.idxOf8 = uint(8 - s.buf.GetByte(s.pos)) - s.pos-- - s.s8 = s.buf.GetByte(s.pos) - s.pos-- - s.readVar() - if s.length > 0 { - s.length-- - } - s.step = 1 - return s.lastValue, false + // серія завершена - перевіряю чи є ще серії + return value, false } -func (s *ReverseCumulativeDeltaDecompressor) readVar() { +func (s *ReverseCumulativeDeltaDecompressor) readHeader() { + h := s.buf.GetByte(s.pos) + s.pos-- + s.decodeHeaderByte(h) + // fmt.Println("h:", h) + // fmt.Println("isRun:", s.isRun) + // fmt.Println("pending:", s.pending) +} +func (s *ReverseCumulativeDeltaDecompressor) readValue() { u64, n, err := s.buf.ReverseGetVarUint64(s.pos) if err != nil { - panic(err) + log.Fatalln(err) } + // fmt.Println() + // fmt.Println("read from pos:", s.pos) + // fmt.Println("read delta:", u64) + // fmt.Println("read delta n:", n) + // fmt.Println() s.pos -= n - s.lastValue = s.firstValue + float64(u64)/s.coef + s.lastValue = s.baseValue + float64(u64)/s.coef +} - var flag byte = 1 << s.idxOf8 - if (s.s8 & flag) == flag { - s.length, n = s.buf.DecodeRunLength(s.pos) - s.pos -= n - } - if s.idxOf8 == 7 { - s.idxOf8 = 0 +func (s *ReverseCumulativeDeltaDecompressor) decodeHeaderByte(h byte) { + s.isRun = h < 128 + if s.isRun { + s.pending = int(h&127) + 2 } else { - s.idxOf8++ + s.pending = int(h&127) + 1 } } + +// diff --git a/chunkenc/insdelta.go b/chunkenc/insdelta.go index f268440..bfa4033 100644 --- a/chunkenc/insdelta.go +++ b/chunkenc/insdelta.go @@ -2,22 +2,20 @@ package chunkenc import ( "fmt" + "log" "math" - "gordenko.dev/dima/qb/bin" "gordenko.dev/dima/qb/conbuf" ) -// REVERSE - type ReverseInstantDeltaCompressor struct { - buf *conbuf.ContinuousBuffer - coef float64 - pos int - firstValue float64 - lastDelta int64 - length uint16 - numIdx int + buf *conbuf.ContinuousBuffer + coef float64 + pos int + baseValue float64 + lastDelta int64 + lastDeltaSize int + h byte } func NewReverseInstantDeltaCompressor(buf *conbuf.ContinuousBuffer, size int, fracDigits byte) *ReverseInstantDeltaCompressor { @@ -31,246 +29,146 @@ func NewReverseInstantDeltaCompressor(buf *conbuf.ContinuousBuffer, size int, fr coef: coef, } if size > 0 { - s.restoreState() + i64, _, err := s.buf.GetVarInt64(0) + if err != nil { + log.Fatalf("bug: get base value: %s", err) + } + s.baseValue = float64(i64) / s.coef + s.h = s.buf.GetByte(s.pos - 1) + s.lastDelta, s.lastDeltaSize, err = s.buf.ReverseGetVarInt64(s.pos - 2) + if err != nil { + log.Fatalf("bug: get last delta: %s", err) + } } return s } -func (s *ReverseInstantDeltaCompressor) restoreState() { - i64, n, err := s.buf.GetVarInt64(0) - if err != nil { - panic(fmt.Sprintf("bug: get first value: %s", err)) - } - s.firstValue = float64(i64) / s.coef - - if s.pos > n { - pos := s.pos - 1 - idxOf8 := uint(8 - s.buf.GetByte(pos)) - pos-- - s8 := s.buf.GetByte(pos) - pos-- - - var n int - s.lastDelta, n, err = s.buf.ReverseGetVarInt64(pos) - if err != nil { - panic(fmt.Sprintf("bug: get last delta: %s", err)) - } - pos -= n - s.numIdx = pos + 1 - var flag byte = 1 << idxOf8 - if (s8 & flag) == flag { - s.length, _ = s.buf.DecodeRunLength(pos) - } - } -} - func (s *ReverseInstantDeltaCompressor) Size() int { return s.pos } -func (s *ReverseInstantDeltaCompressor) CalcRequiredSpace(value float64) int { - if s.pos == 0 { - n := bin.CalcVarInt64Length(int64(value * s.coef)) - return n + 3 - } - - tmp := (value - s.firstValue) * s.coef - if tmp > 0 { - tmp += eps - } else { - tmp -= eps - } - delta := int64(tmp) - - if delta == s.lastDelta { - if s.length == 0 { - return 1 - } else { - newLength := s.length + 1 - if newLength < 130 { - return 0 - } else if newLength == 130 { - return 1 - } else { - if newLength < 32769 { - return 0 - } else { - n := bin.CalcVarInt64Length(delta) - n += 2 - s8q := s.buf.GetByte(s.pos - 1) - if s8q == 8 { - n -= 1 - } else { - n -= 2 - } - return n - } - } - } - } else { - n := bin.CalcVarInt64Length(delta) - n += 2 - s8q := s.buf.GetByte(s.pos - 1) - if s8q == 8 { - n -= 1 - } else { - n -= 2 - } - return n - } -} - -// В начале буфера кодирую базовое значение. func (s *ReverseInstantDeltaCompressor) Append(value float64) { if s.pos == 0 { + // base value n := s.buf.PutVarInt64(s.pos, int64(value*s.coef)) s.pos += n - s.firstValue = value - s.encodeNewDelta(0, 0, 1) + s.baseValue = value + s.appendNewLiteral(0) } else { - tmp := (value - s.firstValue) * s.coef + tmp := (value - s.baseValue) * s.coef if tmp > 0 { tmp += eps } else { tmp -= eps } delta := int64(tmp) - if delta == s.lastDelta { - if s.length == 0 { - s.length = 2 - s.shiftOnePosToRight() - s.buf.SetByte(s.numIdx-1, 0) - s8q := s.buf.GetByte(s.pos - 1) - s.buf.SetFlag(s.pos-2, 1<<(8-s8q)) - } else { - s.length++ - if s.length < 130 { - s.buf.SetByte(s.numIdx-1, byte(s.length-2)) - } else if s.length == 130 { - s.shiftOnePosToRight() - s.encode2bLength() + if s.h < 128 { + // run блок - отже треба збільшити лічильник + if s.h < 127 { + // increase counter + s.h++ + s.buf.SetByte(s.pos-1, s.h) } else { - if s.length < 32769 { - s.encode2bLength() - } else { - s.appendNewDelta(delta) - } + // не можу збільшити - буде переповнення. Додаю новий literal блок + // counter overflow + // fix encode delta + s.appendNewLiteral(delta) + } + } else { + // literal блок. + // Якщо в ньому лише одне значення - перетворюю його на run блок. + // Інакше забираю останнє значення із literal блока і додаю новий run блок. + q := s.h & 127 + if q == 0 { // 1 кодується як 0 + s.convertLiteralToRun() + } else { + s.convertLastFromLiteralToRun() } } } else { - s.appendNewDelta(delta) + if s.h < 127 { + // end of run + s.appendNewLiteral(delta) + } else { + if s.h < 255 { + // encode value from pos - 1, then append h byte + s.appendDeltaToLiteral(delta) + } else { + // overflowed - encode new + s.appendNewLiteral(delta) + } + } } } } -func (s *ReverseInstantDeltaCompressor) appendNewDelta(delta int64) { - s.length = 0 - - s8 := s.buf.GetByte(s.pos - 2) - s8q := s.buf.GetByte(s.pos - 1) - - if s8q == 8 { - s.pos -= 1 - s8 = 0 - s8q = 1 - } else { - s.pos -= 2 - s8q++ - } - s.encodeNewDelta(delta, s8, s8q) +func (s *ReverseInstantDeltaCompressor) convertLastFromLiteralToRun() { + // Зменшую кількість елементів в literal блоці + s.h-- + s.pos -= 1 + s.lastDeltaSize + s.buf.SetByte(s.pos, s.h) // закриваю literal блок + s.pos++ + s.lastDeltaSize = s.buf.ReversePutVarInt64(s.pos, s.lastDelta) + s.pos += s.lastDeltaSize + s.h = 0 // run блок, довжини 2 + s.buf.SetByte(s.pos, s.h) + s.pos++ } -func (s *ReverseInstantDeltaCompressor) encodeNewDelta(delta int64, s8 byte, s8q byte) { +func (s *ReverseInstantDeltaCompressor) convertLiteralToRun() { + // Знімаю flagLiteral, а лічильник 0 дорівнює 2 елементам в серії. + s.h = 0 + s.buf.SetByte(s.pos-1, s.h) +} + +func (s *ReverseInstantDeltaCompressor) appendDeltaToLiteral(delta int64) { + s.h++ // збільшую к-сть дельт s.lastDelta = delta - s.numIdx = s.pos - n := s.buf.ReversePutVarInt64(s.pos, s.lastDelta) - s.pos += n - s.buf.SetByte(s.pos, s8) - s.pos++ - s.buf.SetByte(s.pos, s8q) + s.pos-- + s.lastDeltaSize = s.buf.ReversePutVarInt64(s.pos, delta) + s.pos += s.lastDeltaSize + s.buf.SetByte(s.pos, s.h) s.pos++ } -func (s *ReverseInstantDeltaCompressor) shiftOnePosToRight() { - s.buf.ShiftOnePosToRight(s.numIdx, s.pos) +func (s *ReverseInstantDeltaCompressor) appendNewLiteral(delta int64) { + s.h = flagLiteral + s.lastDelta = delta + s.lastDeltaSize = s.buf.ReversePutVarInt64(s.pos, delta) + s.pos += s.lastDeltaSize + s.buf.SetByte(s.pos, flagLiteral) // literal, length = 1 s.pos++ - s.numIdx++ } -func (s *ReverseInstantDeltaCompressor) encode2bLength() { - num := s.length - 2 - s.buf.SetByte(s.numIdx-1, byte(num&127)|128) - s.buf.SetByte(s.numIdx-2, byte(num>>7)) +func (s *ReverseInstantDeltaCompressor) DeleteLast() {} + +type InstantDeltaBound struct { + Pos int + H byte + LastDelta int64 } -func (s *ReverseInstantDeltaCompressor) DeleteLast() { - var ( - s8q = s.buf.GetByte(s.pos - 1) - s8 = s.buf.GetByte(s.pos - 2) - flag byte = 1 << uint(8-s8q) - ) - - if s.length > 0 { - if s.length == 2 { - s.length = 0 - s.buf.UnsetFlag(s.pos-2, flag) - s.buf.ShiftOnePosToLeft(s.numIdx, s.pos) - s.numIdx-- - s.pos-- - } else if s.length < 130 { - s.length-- - s.buf.SetByte(s.numIdx-1, byte(s.length)-2) - } else if s.length == 130 { - s.length-- - s.buf.ShiftOnePosToLeft(s.numIdx, s.pos) - s.numIdx-- - s.pos-- - s.buf.SetByte(s.numIdx-1, byte(s.length)-2) - } else { - s.length-- - s.encode2bLength() - } - } else { - if s8q > 1 { - s8q-- - flag = 1 << uint(8-s8q) - s.pos = s.numIdx + 2 - s.buf.SetByte(s.pos-2, s8) - s.buf.SetByte(s.pos-1, s8q) - } else { - s.pos = s.numIdx + 1 - s.buf.SetByte(s.pos-1, 8) - s8 = s.buf.GetByte(s.pos - 2) - flag = 1 - } - var ( - pos = s.pos - 3 - n int - err error - ) - s.lastDelta, n, err = s.buf.ReverseGetVarInt64(pos) - if err != nil { - panic(err) - } - s.numIdx = pos - n - if (s8 & flag) == flag { - s.length, _ = s.buf.DecodeRunLength(s.numIdx - 1) - } +func (s *ReverseInstantDeltaCompressor) GetState() InstantDeltaBound { + return InstantDeltaBound{ + Pos: s.pos - 1 - s.lastDeltaSize, + H: s.h, + LastDelta: s.lastDelta, } } +// DECOMPRESSOR + type ReverseInstantDeltaDecompressor struct { - step byte - buf *conbuf.ContinuousBuffer - pos int - bound int - firstValue float64 - lastValue float64 - length uint16 - coef float64 - idxOf8 uint - s8 byte + buf *conbuf.ContinuousBuffer + coef float64 + pos int + bound int + baseValue float64 + lastValue float64 + isRun bool + pending int + done bool } func NewReverseInstantDeltaDecompressor(buf *conbuf.ContinuousBuffer, size int, fracDigits byte) *ReverseInstantDeltaDecompressor { @@ -278,68 +176,169 @@ func NewReverseInstantDeltaDecompressor(buf *conbuf.ContinuousBuffer, size int, if fracDigits > 0 { coef = math.Pow(10, float64(fracDigits)) } - return &ReverseInstantDeltaDecompressor{ - buf: buf, - coef: coef, - pos: size, + i64, n, err := buf.GetVarInt64(0) + if err != nil { + log.Fatalf("bug: get base value: %s", err) } + //fmt.Printf("baseValue: %.2f, bound: %d, pos: %d\n", float64(u64)/coef, n, size) + return &ReverseInstantDeltaDecompressor{ + buf: buf, + coef: coef, + pos: size - 1, // last elem + baseValue: float64(i64) / coef, + bound: n, + } +} + +func (s *ReverseInstantDeltaDecompressor) RestoreFromEnd() { + if s.pos > s.bound { + // читаю заголовок наступної серії + s.readHeader() + s.readValue() + } else { + s.done = true + } +} + +func (s *ReverseInstantDeltaDecompressor) RestoreFromBound(bound InstantDeltaBound) { + s.pos = bound.Pos - 1 + s.lastValue = s.baseValue + float64(bound.LastDelta) + s.decodeHeaderByte(bound.H) + fmt.Printf("restore from bound: isRun=%t, pending=%d\n", s.isRun, s.pending) } func (s *ReverseInstantDeltaDecompressor) NextValue() (value float64, done bool) { - if s.step > 0 { - if s.length > 0 { - s.length-- - return s.lastValue, false - } - if s.pos < s.bound { - return 0, true - } - - if s.idxOf8 == 0 { - s.s8 = s.buf.GetByte(s.pos) - s.pos-- - } - s.readVar() - if s.length > 0 { - s.length-- - } - return s.lastValue, false + //fmt.Printf("NextValue(): bound: %d, pos: %d, pending: %d\n", s.bound, s.pos, s.pending) + if s.done { + return 0, true } - i64, n, err := s.buf.GetVarInt64(0) - if err != nil { - panic(err) + // повертаю значення, що було прочитано в методі RestoreFromBound/RestoreFromEnd + value = s.lastValue + s.pending-- + if s.pending > 0 { + // якщо в серії залишаються елементи + if !s.isRun { + s.readValue() + } + } else if s.pos > s.bound { + // в серії більше немає елементів, отже перевіряє чи є ще дані в буфері. + // дані є - читаю заголовок наступної серії + s.readHeader() + s.readValue() + } else { + s.done = true } - s.firstValue = float64(i64) / s.coef - s.bound = n - s.pos-- - s.idxOf8 = uint(8 - s.buf.GetByte(s.pos)) - s.pos-- - s.s8 = s.buf.GetByte(s.pos) - s.pos-- - s.readVar() - if s.length > 0 { - s.length-- - } - s.step = 1 - return s.lastValue, false + // серія завершена - перевіряю чи є ще серії + return value, false } -func (s *ReverseInstantDeltaDecompressor) readVar() { +func (s *ReverseInstantDeltaDecompressor) readHeader() { + h := s.buf.GetByte(s.pos) + s.pos-- + s.decodeHeaderByte(h) + // fmt.Println("h:", h) + // fmt.Println("isRun:", s.isRun) + // fmt.Println("pending:", s.pending) +} +func (s *ReverseInstantDeltaDecompressor) readValue() { i64, n, err := s.buf.ReverseGetVarInt64(s.pos) if err != nil { - panic(err) + log.Fatalln(err) } + // fmt.Println() + // fmt.Println("read from pos:", s.pos) + // fmt.Println("read delta:", u64) + // fmt.Println("read delta n:", n) + // fmt.Println() s.pos -= n - s.lastValue = s.firstValue + float64(i64)/s.coef + s.lastValue = s.baseValue + float64(i64)/s.coef +} - var flag byte = 1 << s.idxOf8 - if (s.s8 & flag) == flag { - s.length, n = s.buf.DecodeRunLength(s.pos) - s.pos -= n - } - if s.idxOf8 == 7 { - s.idxOf8 = 0 +func (s *ReverseInstantDeltaDecompressor) decodeHeaderByte(h byte) { + s.isRun = h < 128 + if s.isRun { + s.pending = int(h&127) + 2 } else { - s.idxOf8++ + s.pending = int(h&127) + 1 } } + +// DECOMPRESSOR + +// type ReverseInstantDeltaDecompressor struct { +// buf *conbuf.ContinuousBuffer +// pos int +// bound int +// baseValue float64 +// lastValue float64 +// coef float64 +// isRun bool +// pending int +// } + +// func NewReverseInstantDeltaDecompressor(buf *conbuf.ContinuousBuffer, size int, fracDigits byte) *ReverseInstantDeltaDecompressor { +// var coef float64 = 1 +// if fracDigits > 0 { +// coef = math.Pow(10, float64(fracDigits)) +// } +// u64, n, err := buf.GetVarInt64(0) +// if err != nil { +// log.Fatalf("bug: get base value: %s", err) +// } +// //fmt.Printf("baseValue: %.2f, bound: %d, pos: %d\n", float64(u64)/coef, n, size) +// return &ReverseInstantDeltaDecompressor{ +// buf: buf, +// coef: coef, +// pos: size - 1, // last elem +// baseValue: float64(u64) / coef, +// bound: n, +// } +// } + +// func (s *ReverseInstantDeltaDecompressor) NextValue() (value float64, done bool) { +// //fmt.Printf("NextValue(): bound: %d, pos: %d, pending: %d\n", s.bound, s.pos, s.pending) +// if s.pending > 0 { +// // якщо в серії залишаються елементи +// if !s.isRun { +// s.readValue() +// } +// s.pending-- +// return s.lastValue, false +// } +// if s.pos > s.bound { +// // читаю заголовок наступної серії +// s.readHeader() +// s.readValue() +// s.pending-- +// return s.lastValue, false +// } +// // серія завершена - перевіряю чи є ще серії +// return 0, true +// } + +// func (s *ReverseInstantDeltaDecompressor) readHeader() { +// h := s.buf.GetByte(s.pos) +// s.pos-- +// s.isRun = h < 128 +// if s.isRun { +// s.pending = int(h&127) + 2 +// } else { +// s.pending = int(h&127) + 1 +// } +// // fmt.Println("h:", h) +// // fmt.Println("isRun:", s.isRun) +// // fmt.Println("pending:", s.pending) +// } +// func (s *ReverseInstantDeltaDecompressor) readValue() { +// i64, n, err := s.buf.ReverseGetVarInt64(s.pos) +// if err != nil { +// log.Fatalln(err) +// } +// // fmt.Println() +// // fmt.Println("read from pos:", s.pos) +// // fmt.Println("read delta:", u64) +// // fmt.Println("read delta n:", n) +// // fmt.Println() +// s.pos -= n +// s.lastValue = s.baseValue + float64(i64)/s.coef +// } diff --git a/chunkenc/time_delta.go b/chunkenc/time_delta.go new file mode 100644 index 0000000..255e59f --- /dev/null +++ b/chunkenc/time_delta.go @@ -0,0 +1,288 @@ +package chunkenc + +import ( + "fmt" + "log" + + "gordenko.dev/dima/pretty" + "gordenko.dev/dima/qb/conbuf" +) + +type ReverseTimeDeltaCompressor struct { + buf *conbuf.ContinuousBuffer + pos int + lastUnixtime uint32 + lastDelta uint32 + lastDeltaSize int + h byte +} + +func NewReverseTimeDeltaCompressor(buf *conbuf.ContinuousBuffer, size int) *ReverseTimeDeltaCompressor { + s := &ReverseTimeDeltaCompressor{ + buf: buf, + pos: size, // перший вільний байт + } + if size > 0 { + u64, n, err := s.buf.ReverseGetVarUint64(s.pos - 1) + if err != nil { + log.Fatalf("bug: get last unixtime: %s", err) + } + s.lastUnixtime = uint32(u64) + s.pos -= n + if s.pos > 0 { + s.h = s.buf.GetByte(s.pos - 1) + u64, s.lastDeltaSize, err = s.buf.ReverseGetVarUint64(s.pos - 2) + if err != nil { + log.Fatalf("bug: get last delta: %s", err) + } + s.lastDelta = uint32(u64) + } + } + return s +} + +func (s *ReverseTimeDeltaCompressor) Size() int { + return s.pos +} + +//func (s *ReverseTimeDeltaCompressor) CalcRequiredSpace(value float64) int { +//} + +func (s *ReverseTimeDeltaCompressor) Append(unixtime uint32) { + if s.lastUnixtime > 0 { + // base value + //s.pos += s.buf.PutVarUint64(s.pos, uint64(value*s.coef)) + //s.baseValue = value + //s.appendNewLiteral(0) + + //} else { + delta := unixtime - s.lastUnixtime + if delta == s.lastDelta { + if s.h < 128 { + // run блок - отже треба збільшити лічильник + if s.h < 127 { + // increase counter + s.h++ + s.buf.SetByte(s.pos-1, s.h) + } else { + // не можу збільшити - буде переповнення. Додаю новий literal блок + // counter overflow + s.appendNewLiteral(delta) + } + } else { + // literal блок. + // Якщо в ньому лише одне значення - перетворюю його на run блок. + // Інакше забираю останнє значення із literal блока і додаю новий run блок. + q := s.h & 127 + if q == 0 { // 1 кодується як 0 + s.convertLiteralToRun() + } else { + // забираю останнє значення із literal блоку щоб зробити run блок + s.convertLastFromLiteralToRun() + } + } + } else { + if s.h < 127 { // fix 128? + // end of run + s.appendNewLiteral(delta) + } else { + if s.h < 255 { + // encode value from pos - 1, then append h byte + s.appendDeltaToLiteral(delta) + } else { + // overflowed - encode new + s.appendNewLiteral(delta) + } + } + } + } + s.lastUnixtime = unixtime +} + +func (s *ReverseTimeDeltaCompressor) convertLastFromLiteralToRun() { + // Зменшую кількість елементів в literal блоці + s.h-- + s.pos -= 1 + s.lastDeltaSize + s.buf.SetByte(s.pos, s.h) // закриваю literal блок + s.pos++ + s.lastDeltaSize = s.buf.ReversePutVarUint64(s.pos, uint64(s.lastDelta)) + s.pos += s.lastDeltaSize + s.h = 0 // run блок, довжини 2 + s.buf.SetByte(s.pos, s.h) + s.pos++ +} + +func (s *ReverseTimeDeltaCompressor) convertLiteralToRun() { + // Знімаю flagLiteral, а лічильник 0 дорівнює 2 елементам в серії. + s.h = 0 + s.buf.SetByte(s.pos-1, s.h) +} + +func (s *ReverseTimeDeltaCompressor) appendDeltaToLiteral(delta uint32) { + s.h++ // збільшую к-сть дельт + s.lastDelta = delta + s.pos-- + s.lastDeltaSize = s.buf.ReversePutVarUint64(s.pos, uint64(delta)) + s.pos += s.lastDeltaSize + s.buf.SetByte(s.pos, s.h) + s.pos++ +} + +func (s *ReverseTimeDeltaCompressor) appendNewLiteral(delta uint32) { + //fmt.Println("appendNewLiteral", delta) + s.h = flagLiteral + s.lastDelta = delta + s.lastDeltaSize = s.buf.ReversePutVarUint64(s.pos, uint64(delta)) + s.pos += s.lastDeltaSize + s.buf.SetByte(s.pos, flagLiteral) // literal, length = 1 + s.pos++ +} + +func (s *ReverseTimeDeltaCompressor) DeleteLast() { + +} + +func (s *ReverseTimeDeltaCompressor) Sync() { + s.pos += s.buf.ReversePutVarUint64(s.pos, uint64(s.lastUnixtime)) +} + +type TimeDeltaBound struct { + Pos int + H byte + LastUnixtime uint32 + LastDelta uint32 +} + +// delta h +func (s *ReverseTimeDeltaCompressor) GetState() TimeDeltaBound { + bound := TimeDeltaBound{ + LastUnixtime: s.lastUnixtime, + } + if s.pos > 0 { + bound.Pos = s.pos - 1 - s.lastDeltaSize + bound.H = s.h + bound.LastDelta = s.lastDelta + } + return bound +} + +// DECOMPRESSOR + +type ReverseTimeDeltaDecompressor struct { + buf *conbuf.ContinuousBuffer + pos int + lastDelta uint32 + lastUnixtime uint32 + isRegular bool + isRun bool + pending int + done bool +} + +func NewReverseTimeDeltaDecompressor(buf *conbuf.ContinuousBuffer, size int) *ReverseTimeDeltaDecompressor { + return &ReverseTimeDeltaDecompressor{ + buf: buf, + pos: size, + } +} + +func (s *ReverseTimeDeltaDecompressor) RestoreFromEnd() { + fmt.Println("RestoreFromEnd", s.pos) + if s.pos > 0 { + s.pos-- // перший байт даних + u64, n, err := s.buf.ReverseGetVarUint64(s.pos) + if err != nil { + log.Fatalf("bug: get last unixtime: %s", err) + } + s.lastUnixtime = uint32(u64) + s.pos -= n + fmt.Println("restored", s.lastUnixtime, s.pos) + if s.pos > 0 { + s.readHeader() + s.readDelta() + } + } else { + s.done = true + } +} + +func (s *ReverseTimeDeltaDecompressor) RestoreFromBound(bound TimeDeltaBound) { + pretty.Println(bound) + s.lastUnixtime = bound.LastUnixtime + if bound.LastDelta > 0 { + s.pos = bound.Pos - 1 + s.lastDelta = bound.LastDelta + s.decodeHeaderByte(bound.H) + } else { + s.pos = -1 + } + fmt.Printf("restore from bound: pos=%d, isRun=%t, pending=%d\n", s.pos, s.isRun, s.pending) +} + +func (s *ReverseTimeDeltaDecompressor) NextValue() (value uint32, done bool) { + //fmt.Printf("NextValue(): pos: %d, isRegular: %t, pending: %d\n", s.pos, s.isRegular, s.pending) + if s.done { + return 0, true + } + // повертаю значення, що було прочитано в методі RestoreFromBound/RestoreFromEnd + value = s.lastUnixtime + if s.isRegular { + s.pending-- + if s.pending > 0 { + // якщо в серії залишаються елементи + if !s.isRun { + s.readDelta() + } + s.lastUnixtime -= s.lastDelta + } else if s.pos > 0 { + // в серії більше немає елементів, отже перевіряє чи є ще дані в буфері. + // дані є - читаю заголовок наступної серії + s.readHeader() + s.readDelta() + s.lastUnixtime -= s.lastDelta + } else { + s.done = true + } + } else { + if s.lastDelta > 0 { + s.isRegular = true + s.lastUnixtime -= s.lastDelta + } else { + // був закодований лише last unixtime + s.done = true + } + } + return value, false +} + +func (s *ReverseTimeDeltaDecompressor) readHeader() { + h := s.buf.GetByte(s.pos) + s.pos-- + s.decodeHeaderByte(h) + // fmt.Println("h:", h) + // fmt.Println("isRun:", s.isRun) + // fmt.Println("pending:", s.pending) +} + +func (s *ReverseTimeDeltaDecompressor) decodeHeaderByte(h byte) { + s.isRun = h < 128 + if s.isRun { + s.pending = int(h&127) + 2 + } else { + s.pending = int(h&127) + 1 + } +} + +func (s *ReverseTimeDeltaDecompressor) readDelta() { + u64, n, err := s.buf.ReverseGetVarUint64(s.pos) + if err != nil { + log.Fatalln(err) + } + // fmt.Println() + // fmt.Println("read from pos:", s.pos) + // fmt.Println("read delta:", u64) + // fmt.Println("read delta n:", n) + // fmt.Println() + s.pos -= n + s.lastDelta = uint32(u64) +} diff --git a/chunkenc/time_delta_delta.go b/chunkenc/time_delta_delta.go index 46d5f9f..aebf6c4 100644 --- a/chunkenc/time_delta_delta.go +++ b/chunkenc/time_delta_delta.go @@ -1,374 +1,270 @@ package chunkenc -import ( - "fmt" +// // REVERSE - "gordenko.dev/dima/qb/bin" - "gordenko.dev/dima/qb/conbuf" -) +// // const ( +// // lastUnixtimeIdx = 0 +// // baseDeltaIdx = 4 +// // ) -// REVERSE +// type ReverseTimeDeltaOfDeltaCompressor struct { +// buf *conbuf.ContinuousBuffer +// pos int +// lastDelta uint32 +// lastUnixtime uint32 +// lastDeltaOfDelta int64 +// lastDeltaOfDeltaSize int +// h byte +// } -const ( - lastUnixtimeIdx = 0 - baseDeltaIdx = 4 -) +// func NewReverseTimeDeltaOfDeltaCompressor(buf *conbuf.ContinuousBuffer, size int) *ReverseTimeDeltaOfDeltaCompressor { +// s := &ReverseTimeDeltaOfDeltaCompressor{ +// buf: buf, +// pos: size, +// } +// if size > 0 { +// // s.lastUnixtime = s.buf.GetUint32(lastUnixtimeIdx) +// // u64, _, err := s.buf.GetVarUint64(baseDeltaIdx) +// // if err != nil { +// // log.Fatalf("bug: get base delta: %s", err) +// // } +// // s.baseDelta = uint32(u64) +// // s.h = s.buf.GetByte(s.pos - 1) +// // s.lastDeltaOfDelta, s.lastDeltaOfDeltaSize, err = s.buf.ReverseGetVarInt64(s.pos - 2) +// // if err != nil { +// // log.Fatalf("bug: get last delta of delta: %s", err) +// // } +// } else { +// //s.baseDelta = baseDelta +// } +// return s +// } -type ReverseTimeDeltaOfDeltaCompressor struct { - buf *conbuf.ContinuousBuffer - pos int - baseDelta uint32 - lastUnixtime uint32 - lastDeltaOfDelta int64 - length uint16 - numIdx int -} +// func (s *ReverseTimeDeltaOfDeltaCompressor) Size() int { +// return s.pos +// } -func NewReverseTimeDeltaOfDeltaCompressor(buf *conbuf.ContinuousBuffer, size int) *ReverseTimeDeltaOfDeltaCompressor { - s := &ReverseTimeDeltaOfDeltaCompressor{ - buf: buf, - pos: size, - } - if size > 0 { - s.restoreState() - } - return s -} +// func (s *ReverseTimeDeltaOfDeltaCompressor) Append(unixtime uint32) { +// if s.lastUnixtime > 0 { +// if s.pos == 0 { +// // 2-й unixtime +// s.lastDelta = unixtime - s.lastUnixtime +// s.appendNewLiteral(0) +// } else { +// // 3-й unixtime і наступні +// lastDelta := unixtime - s.lastUnixtime +// deltaOfDelta := int64(lastDelta) - int64(s.lastDelta) +// if deltaOfDelta == s.lastDeltaOfDelta { +// if s.h < 128 { +// // run блок - отже треба збільшити лічильник +// if s.h < 127 { +// // increase counter +// s.h++ +// s.buf.SetByte(s.pos-1, s.h) +// } else { +// // не можу збільшити - буде переповнення. Додаю новий literal блок +// // counter overflow +// // fix encode delta +// s.appendNewLiteral(deltaOfDelta) +// } +// } else { +// // literal блок. +// // Якщо в ньому лише одне значення - перетворюю його на run блок. +// // Інакше забираю останнє значення із literal блока і додаю новий run блок. +// q := s.h & 127 +// if q == 0 { // 1 кодується як 0 +// s.convertLiteralToRun() +// } else { +// s.convertLastFromLiteralToRun(deltaOfDelta) +// } +// } +// } else { +// s.lastDelta = lastDelta +// if s.h < 127 { +// // end of run +// s.appendNewLiteral(deltaOfDelta) +// } else { +// if s.h < 255 { +// // encode value from pos - 1, then append h byte +// s.appendDeltaToLiteral(deltaOfDelta) +// } else { +// // overflowed - encode new +// s.appendNewLiteral(deltaOfDelta) +// } +// } +// } +// } +// } +// // для першого unixtime - лише оновлюємо lastUnixtime, а для наступних - +// // теж оновлюємо, але після запису серій в buf. +// s.lastUnixtime = unixtime +// } -func (s *ReverseTimeDeltaOfDeltaCompressor) restoreState() { - s.lastUnixtime = s.buf.GetUint32(lastUnixtimeIdx) - if s.pos > 4 { - u64, _, err := s.buf.GetVarUint64(baseDeltaIdx) - if err != nil { - panic(fmt.Sprintf("bug: get base delta: %s", err)) - } - s.baseDelta = uint32(u64) +// func (s *ReverseTimeDeltaOfDeltaCompressor) convertLastFromLiteralToRun(deltaOfDelta int64) { +// // Зменшую кількість елементів в literal блоці +// s.h-- +// s.pos -= 1 + s.lastDeltaOfDeltaSize +// s.buf.SetByte(s.pos, s.h) // закриваю literal блок +// s.pos++ +// s.lastDeltaOfDeltaSize = s.buf.ReversePutVarInt64(s.pos, deltaOfDelta) +// s.pos += s.lastDeltaOfDeltaSize +// s.h = 0 // run блок, довжини 2 +// s.buf.SetByte(s.pos, s.h) +// s.pos++ +// } - pos := s.pos - 1 - idxOf8 := uint(8 - s.buf.GetByte(pos)) - pos-- - s8 := s.buf.GetByte(pos) - pos-- +// func (s *ReverseTimeDeltaOfDeltaCompressor) convertLiteralToRun() { +// // Знімаю flagLiteral, а лічильник 0 дорівнює 2 елементам в серії. +// s.h = 0 +// s.buf.SetByte(s.pos-1, s.h) +// } - var n int - s.lastDeltaOfDelta, n, err = s.buf.ReverseGetVarInt64(pos) - if err != nil { - panic(err) - } - pos -= n - s.numIdx = pos + 1 - var flag byte = 1 << idxOf8 - if (s8 & flag) == flag { - s.length, _ = s.buf.DecodeRunLength(pos) - } - } -} +// func (s *ReverseTimeDeltaOfDeltaCompressor) appendDeltaToLiteral(deltaOfDelta int64) { +// s.h++ // збільшую к-сть дельт +// s.lastDeltaOfDelta = deltaOfDelta +// s.pos-- +// s.lastDeltaOfDeltaSize = s.buf.ReversePutVarInt64(s.pos, deltaOfDelta) +// s.pos += s.lastDeltaOfDeltaSize +// s.buf.SetByte(s.pos, s.h) +// s.pos++ +// } -func (s *ReverseTimeDeltaOfDeltaCompressor) Size() int { - return s.pos -} +// func (s *ReverseTimeDeltaOfDeltaCompressor) appendNewLiteral(deltaOfDelta int64) { +// s.h = flagLiteral +// s.lastDeltaOfDelta = deltaOfDelta +// s.lastDeltaOfDeltaSize = s.buf.ReversePutVarInt64(s.pos, deltaOfDelta) +// s.pos += s.lastDeltaOfDeltaSize +// s.buf.SetByte(s.pos, flagLiteral) // literal, length = 1 +// s.pos++ +// } -func (s *ReverseTimeDeltaOfDeltaCompressor) CalcRequiredSpace(unixtime uint32) int { - if s.pos == 0 { - return 4 - } +// func (s *ReverseTimeDeltaOfDeltaCompressor) Sync() { +// // дописую в кінець lastDelta та lastUnixtime +// if s.pos > 0 { +// n := s.buf.ReversePutVarUint64(s.pos, uint64(s.lastDelta)) +// s.pos += n +// } +// if s.lastUnixtime > 0 { +// s.buf.PutUint32(s.pos, s.lastUnixtime) +// s.pos += 4 +// } +// } - if s.baseDelta == 0 { - baseDelta := unixtime - s.lastUnixtime - n := bin.CalcVarUint64Length(uint64(baseDelta)) - return n + 3 - } +// func (s *ReverseTimeDeltaOfDeltaCompressor) DeleteLast() { - deltaOfDelta := int64(unixtime-s.lastUnixtime) - int64(s.baseDelta) - if deltaOfDelta == s.lastDeltaOfDelta { - if s.length == 0 { - return 1 - } else { - newLength := s.length + 1 - if newLength < 130 { - return 0 - } else if newLength == 130 { - return 1 - } else { - if newLength < 32769 { - return 0 - } else { - n := bin.CalcVarInt64Length(deltaOfDelta) - n += 2 - s8q := s.buf.GetByte(s.pos - 1) - if s8q == 8 { - n -= 1 - } else { - n -= 2 - } - return n - } - } - } - } else { - n := bin.CalcVarInt64Length(deltaOfDelta) - n += 2 - s8q := s.buf.GetByte(s.pos - 1) - if s8q == 8 { - n -= 1 - } else { - n -= 2 - } - return n - } -} +// } -func (s *ReverseTimeDeltaOfDeltaCompressor) Append(unixtime uint32) { - if s.pos == 0 { - s.lastUnixtime = unixtime - s.buf.PutUint32(lastUnixtimeIdx, unixtime) - s.pos += 4 - return - } +// type TimeDeltaOfDeltaBound struct { +// LastUnixtime uint32 +// LastDelta uint32 +// Pos int +// H byte +// LastDeltaOfDelta int64 +// } - if s.baseDelta == 0 { - s.baseDelta = unixtime - s.lastUnixtime - s.lastDeltaOfDelta = 0 - s.lastUnixtime = unixtime - s.buf.PutUint32(lastUnixtimeIdx, unixtime) +// func (s *ReverseTimeDeltaOfDeltaCompressor) GetState() TimeDeltaOfDeltaBound { +// return TimeDeltaOfDeltaBound{ +// LastUnixtime: s.lastUnixtime, +// LastDelta: s.lastDelta, +// Pos: s.pos, +// H: s.h, +// LastDeltaOfDelta: s.lastDeltaOfDelta, +// } +// } - n := s.buf.PutVarUint64(s.pos, uint64(s.baseDelta)) - s.pos += n - s.encodeNewDeltaOfDelta(0, 0, 1) - return - } +// // DECOMPRESSOR - deltaOfDelta := int64(unixtime-s.lastUnixtime) - int64(s.baseDelta) - s.lastUnixtime = unixtime - s.buf.PutUint32(lastUnixtimeIdx, unixtime) +// type ReverseTimeDeltaOfDeltaDecompressor struct { +// buf *conbuf.ContinuousBuffer +// size int +// pos int +// lastUnixtime uint32 +// lastDelta uint32 +// //lastDeltaOfDelta int64 +// isRun bool +// pending int +// } - if deltaOfDelta == s.lastDeltaOfDelta { - if s.length == 0 { - s.length = 2 - s.shiftOnePosToRight() - s.buf.SetByte(s.numIdx-1, 0) - s8q := s.buf.GetByte(s.pos - 1) - s.buf.SetFlag(s.pos-2, 1<<(8-s8q)) - } else { - s.length++ - if s.length < 130 { - s.buf.SetByte(s.numIdx-1, byte(s.length-2)) - } else if s.length == 130 { - s.shiftOnePosToRight() - s.encode2bLength() - } else { - if s.length < 32769 { - s.encode2bLength() - } else { - s.appendNewDeltaOfDelta(deltaOfDelta) - } - } - } - } else { - s.appendNewDeltaOfDelta(deltaOfDelta) - } -} +// func NewReverseTimeDeltaOfDeltaDecompressor(buf *conbuf.ContinuousBuffer, size int) *ReverseTimeDeltaOfDeltaDecompressor { +// return &ReverseTimeDeltaOfDeltaDecompressor{ +// buf: buf, +// size: size, +// pos: size, // last elem +// } +// } -func (s *ReverseTimeDeltaOfDeltaCompressor) appendNewDeltaOfDelta(deltaOfDelta int64) { - s.length = 0 - s8 := s.buf.GetByte(s.pos - 2) - s8q := s.buf.GetByte(s.pos - 1) +// func (s *ReverseTimeDeltaOfDeltaDecompressor) NextValue() (value uint32, done bool) { +// fmt.Printf("\nNextValue(): pos: %d, size: %d, pending: %d\n", s.pos, s.size, s.pending) +// if s.pos < s.size { +// if s.pending > 0 { +// //fmt.Println("pending", ) +// // якщо в серії залишаються елементи +// fmt.Println("s.lastUnixtime -= s.lastDelta", s.lastUnixtime, s.lastDelta) +// s.lastUnixtime -= s.lastDelta +// if !s.isRun { +// s.readValue() +// } +// s.pending-- +// return s.lastUnixtime, false +// } +// // FIX - читаю lastUnixtime +// if s.pos > 0 { +// s.lastUnixtime -= s.lastDelta +// // читаю заголовок наступної серії +// s.readHeader() +// s.readValue() +// s.pending-- +// return s.lastUnixtime, false +// } +// // серія завершена - перевіряю чи є ще серії +// return 0, true +// } else { +// s.pos -= 4 +// s.lastUnixtime = s.buf.GetUint32(s.pos) +// fmt.Println("lastUnix", s.lastUnixtime) +// if s.pos > 0 { +// s.pos-- +// u64, n, err := s.buf.ReverseGetVarUint64(s.pos) +// if err != nil { +// log.Fatalln(err) +// } +// s.pos -= n +// s.lastDelta = uint32(u64) +// fmt.Println("lastDelta", s.lastDelta) +// } +// return s.lastUnixtime, false +// } +// } - if s8q == 8 { - s.pos -= 1 - s8 = 0 - s8q = 1 - } else { - s.pos -= 2 - s8q++ - } - - s.encodeNewDeltaOfDelta(deltaOfDelta, s8, s8q) -} - -func (s *ReverseTimeDeltaOfDeltaCompressor) encodeNewDeltaOfDelta(deltaOfDelta int64, s8 byte, s8q byte) { - s.lastDeltaOfDelta = deltaOfDelta - s.numIdx = s.pos - n := s.buf.ReversePutVarInt64(s.pos, deltaOfDelta) - s.pos += n - s.buf.SetByte(s.pos, s8) - s.pos++ - s.buf.SetByte(s.pos, s8q) - s.pos++ -} - -func (s *ReverseTimeDeltaOfDeltaCompressor) shiftOnePosToRight() { - s.buf.ShiftOnePosToRight(s.numIdx, s.pos) - s.pos++ - s.numIdx++ -} - -func (s *ReverseTimeDeltaOfDeltaCompressor) encode2bLength() { - num := s.length - 2 - s.buf.SetByte(s.numIdx-1, byte(num&127)|128) - s.buf.SetByte(s.numIdx-2, byte(num>>7)) -} - -func (s *ReverseTimeDeltaOfDeltaCompressor) DeleteLast() { - var ( - s8q = s.buf.GetByte(s.pos - 1) - s8 = s.buf.GetByte(s.pos - 2) - flag byte = 1 << uint(8-s8q) - ) - - if s.length > 0 { - if s.length == 2 { - s.length = 0 - s.buf.UnsetFlag(s.pos-2, flag) - s.buf.ShiftOnePosToLeft(s.numIdx, s.pos) - s.numIdx-- - s.pos-- - } else if s.length < 130 { - s.length-- - s.buf.SetByte(s.numIdx-1, byte(s.length)-2) - } else if s.length == 130 { - s.length-- - s.buf.ShiftOnePosToLeft(s.numIdx, s.pos) - s.numIdx-- - s.pos-- - s.buf.SetByte(s.numIdx-1, byte(s.length)-2) - } else { - s.length-- - s.encode2bLength() - } - } else { - if s8q > 1 { - s8q-- - flag = 1 << uint(8-s8q) - s.pos = s.numIdx + 2 - s.buf.SetByte(s.pos-2, s8) - s.buf.SetByte(s.pos-1, s8q) - } else { - s.pos = s.numIdx + 1 - s.buf.SetByte(s.pos-1, 8) - s8 = s.buf.GetByte(s.pos - 2) - flag = 1 - } - var ( - pos = s.pos - 3 - n int - err error - ) - s.lastDeltaOfDelta, n, err = s.buf.ReverseGetVarInt64(pos) - if err != nil { - panic(err) - } - s.numIdx = pos - n - if (s8 & flag) == flag { - s.length, _ = s.buf.DecodeRunLength(s.numIdx - 1) - } - } - - delta := int64(s.baseDelta) + s.lastDeltaOfDelta - s.lastUnixtime = uint32(int64(s.lastUnixtime) - delta) - s.buf.PutUint32(lastUnixtimeIdx, s.lastUnixtime) -} - -type ReverseTimeDeltaOfDeltaDecompressor struct { - step byte - buf *conbuf.ContinuousBuffer - pos int - bound int - lastUnixtime uint32 - baseDelta uint32 - lastDeltaOfDelta int64 - length uint16 - idxOf8 uint - s8 byte -} - -func NewReverseTimeDeltaOfDeltaDecompressor(buf *conbuf.ContinuousBuffer, size int) *ReverseTimeDeltaOfDeltaDecompressor { - return &ReverseTimeDeltaOfDeltaDecompressor{ - buf: buf, - pos: size, - } -} - -func (s *ReverseTimeDeltaOfDeltaDecompressor) NextValue() (value uint32, done bool) { - if s.step == 0 { - if s.pos == 0 { - return 0, true - } - s.lastUnixtime = s.buf.GetUint32(lastUnixtimeIdx) - s.step = 1 - return s.lastUnixtime, false - } - - if s.step == 1 { - if s.pos == baseDeltaIdx { - return 0, true - } - u64, n, err := s.buf.GetVarUint64(baseDeltaIdx) - if err != nil { - panic("EOF") - } - s.bound = baseDeltaIdx + n - s.baseDelta = uint32(u64) - - s.pos-- - s.idxOf8 = uint(8 - s.buf.GetByte(s.pos)) - s.pos-- - s.s8 = s.buf.GetByte(s.pos) - s.pos-- - - s.readVar() - if s.length > 0 { - s.length-- - } - s.step = 2 - return s.lastUnixtime, false - } - - if s.length > 0 { - s.length-- - delta := int64(s.baseDelta) + s.lastDeltaOfDelta - s.lastUnixtime = uint32(int64(s.lastUnixtime) - delta) - return s.lastUnixtime, false - } - - if s.pos < s.bound { - return 0, true - } - - if s.idxOf8 == 0 { - s.s8 = s.buf.GetByte(s.pos) - s.pos-- - } - s.readVar() - if s.length > 0 { - s.length-- - } - return s.lastUnixtime, false -} - -func (s *ReverseTimeDeltaOfDeltaDecompressor) readVar() { - var ( - n int - err error - ) - - s.lastDeltaOfDelta, n, err = s.buf.ReverseGetVarInt64(s.pos) - if err != nil { - panic(err) - } - s.pos -= n - - delta := int64(s.baseDelta) + s.lastDeltaOfDelta - s.lastUnixtime = uint32(int64(s.lastUnixtime) - delta) - - var flag byte = 1 << s.idxOf8 - if (s.s8 & flag) == flag { - s.length, n = s.buf.DecodeRunLength(s.pos) - s.pos -= n - } - if s.idxOf8 == 7 { - s.idxOf8 = 0 - } else { - s.idxOf8++ - } -} +// func (s *ReverseTimeDeltaOfDeltaDecompressor) readHeader() { +// h := s.buf.GetByte(s.pos) +// s.pos-- +// s.isRun = h < 128 +// if s.isRun { +// s.pending = int(h&127) + 2 +// } else { +// s.pending = int(h&127) + 1 +// } +// fmt.Println("h:", h) +// fmt.Println("isRun:", s.isRun) +// fmt.Println("pending:", s.pending) +// } +// func (s *ReverseTimeDeltaOfDeltaDecompressor) readValue() { +// //s.lastUnixtime -= s.lastDelta +// // var ( +// // err error +// // n int +// // ) +// lastDeltaOfDelta, n, err := s.buf.ReverseGetVarInt64(s.pos) +// if err != nil { +// log.Fatalln(err) +// } +// fmt.Println() +// fmt.Println("read Value from pos:", s.pos) +// fmt.Println("delta of delta:", lastDeltaOfDelta) +// //fmt.Println("delta of delta n:", n) +// s.pos -= n +// s.lastDelta = uint32(int64(s.lastDelta) - lastDeltaOfDelta) +// fmt.Println("last delta:", s.lastDelta) +// fmt.Println() +// } diff --git a/chunkenc2/chunckenc.go b/chunkenc2/chunckenc.go new file mode 100644 index 0000000..53e145e --- /dev/null +++ b/chunkenc2/chunckenc.go @@ -0,0 +1,3 @@ +package chunkenc2 + +const eps = 0.000001 diff --git a/chunkenc2/cumdelta.go b/chunkenc2/cumdelta.go new file mode 100644 index 0000000..f865849 --- /dev/null +++ b/chunkenc2/cumdelta.go @@ -0,0 +1,330 @@ +package chunkenc2 + +import ( + "fmt" + "math" + + "gordenko.dev/dima/qb/bin" + "gordenko.dev/dima/qb/conbuf" +) + +// REVERSE + +type ReverseCumulativeDeltaCompressor struct { + buf *conbuf.ContinuousBuffer + coef float64 + pos int + firstValue float64 + lastDelta uint64 + length uint16 + numIdx int +} + +func NewReverseCumulativeDeltaCompressor(buf *conbuf.ContinuousBuffer, size int, fracDigits byte) *ReverseCumulativeDeltaCompressor { + var coef float64 = 1 + if fracDigits > 0 { + coef = math.Pow(10, float64(fracDigits)) + } + s := &ReverseCumulativeDeltaCompressor{ + buf: buf, + pos: size, + coef: coef, + } + if size > 0 { + s.restoreState() + } + return s +} + +func (s *ReverseCumulativeDeltaCompressor) restoreState() { + u64, n, err := s.buf.GetVarUint64(0) + if err != nil { + panic(fmt.Sprintf("bug: get first value: %s", err)) + } + s.firstValue = float64(u64) / s.coef + + if s.pos > n { + pos := s.pos - 1 + idxOf8 := uint(8 - s.buf.GetByte(pos)) + pos-- + s8 := s.buf.GetByte(pos) + pos-- + + var n int + s.lastDelta, n, err = s.buf.ReverseGetVarUint64(pos) + if err != nil { + panic(fmt.Sprintf("bug: get last delta: %s", err)) + } + pos -= n + s.numIdx = pos + 1 + + var flag byte = 1 << idxOf8 + if (s8 & flag) == flag { + s.length, _ = s.buf.DecodeRunLength(pos) + } + } +} + +func (s *ReverseCumulativeDeltaCompressor) Size() int { + return s.pos +} + +func (s *ReverseCumulativeDeltaCompressor) CalcRequiredSpace(value float64) int { + if s.pos == 0 { + n := bin.CalcVarUint64Length(uint64(value * s.coef)) + return n + 3 + } + delta := uint64((value-s.firstValue)*s.coef + eps) + if delta == s.lastDelta { + if s.length == 0 { + return 1 + } else { + newLength := s.length + 1 + if newLength < 130 { + return 0 + } else if newLength == 130 { + return 1 + } else { + if newLength < 32769 { + return 0 + } else { + n := bin.CalcVarUint64Length(delta) + n += 2 + s8q := s.buf.GetByte(s.pos - 1) + if s8q == 8 { + n -= 1 + } else { + n -= 2 + } + return n + } + } + } + } else { + n := bin.CalcVarUint64Length(delta) + n += 2 + s8q := s.buf.GetByte(s.pos - 1) + if s8q == 8 { + n -= 1 + } else { + n -= 2 + } + return n + } +} + +func (s *ReverseCumulativeDeltaCompressor) Append(value float64) { + if s.pos == 0 { + n := s.buf.PutVarUint64(s.pos, uint64(value*s.coef)) + s.pos += n + s.firstValue = value + s.encodeNewDelta(0, 0, 1) + } else { + delta := uint64((value-s.firstValue)*s.coef + eps) + if delta == s.lastDelta { + if s.length == 0 { + s.length = 2 + s.shiftOnePosToRight() + s.buf.SetByte(s.numIdx-1, 0) + s8q := s.buf.GetByte(s.pos - 1) + s.buf.SetFlag(s.pos-2, 1<<(8-s8q)) + } else { + s.length++ + if s.length < 130 { + s.buf.SetByte(s.numIdx-1, byte(s.length-2)) + } else if s.length == 130 { + s.shiftOnePosToRight() + s.encode2bLength() + } else { + if s.length < 32769 { + s.encode2bLength() + } else { + s.appendNewDelta(delta) + } + } + } + } else { + s.appendNewDelta(delta) + } + } +} + +func (s *ReverseCumulativeDeltaCompressor) appendNewDelta(delta uint64) { + s.length = 0 + + s8 := s.buf.GetByte(s.pos - 2) + s8q := s.buf.GetByte(s.pos - 1) + + if s8q == 8 { + s.pos -= 1 + s8 = 0 + s8q = 1 + } else { + s.pos -= 2 + s8q++ + } + + s.encodeNewDelta(delta, s8, s8q) +} + +func (s *ReverseCumulativeDeltaCompressor) encodeNewDelta(delta uint64, s8 byte, s8q byte) { + s.lastDelta = delta + s.numIdx = s.pos + n := s.buf.ReversePutVarUint64(s.pos, s.lastDelta) + s.pos += n + s.buf.SetByte(s.pos, s8) + s.pos++ + s.buf.SetByte(s.pos, s8q) + s.pos++ +} + +func (s *ReverseCumulativeDeltaCompressor) shiftOnePosToRight() { + s.buf.ShiftOnePosToRight(s.numIdx, s.pos) + s.pos++ + s.numIdx++ +} + +func (s *ReverseCumulativeDeltaCompressor) encode2bLength() { + num := s.length - 2 + s.buf.SetByte(s.numIdx-1, byte(num&127)|128) + s.buf.SetByte(s.numIdx-2, byte(num>>7)) +} + +func (s *ReverseCumulativeDeltaCompressor) DeleteLast() { + var ( + s8q = s.buf.GetByte(s.pos - 1) + s8 = s.buf.GetByte(s.pos - 2) + flag byte = 1 << uint(8-s8q) + ) + + if s.length > 0 { + if s.length == 2 { + s.length = 0 + s.buf.UnsetFlag(s.pos-2, flag) + s.buf.ShiftOnePosToLeft(s.numIdx, s.pos) + s.numIdx-- + s.pos-- + } else if s.length < 130 { + s.length-- + s.buf.SetByte(s.numIdx-1, byte(s.length)-2) + } else if s.length == 130 { + s.length-- + s.buf.ShiftOnePosToLeft(s.numIdx, s.pos) + s.numIdx-- + s.pos-- + s.buf.SetByte(s.numIdx-1, byte(s.length)-2) + } else { + s.length-- + s.encode2bLength() + } + } else { + if s8q > 1 { + s8q-- + flag = 1 << uint(8-s8q) + s.pos = s.numIdx + 2 + s.buf.SetByte(s.pos-2, s8) + s.buf.SetByte(s.pos-1, s8q) + } else { + s.pos = s.numIdx + 1 + s.buf.SetByte(s.pos-1, 8) + s8 = s.buf.GetByte(s.pos - 2) + flag = 1 + } + var ( + pos = s.pos - 3 + n int + err error + ) + s.lastDelta, n, err = s.buf.ReverseGetVarUint64(pos) + if err != nil { + panic(err) + } + s.numIdx = pos - n + if (s8 & flag) == flag { + s.length, _ = s.buf.DecodeRunLength(s.numIdx - 1) + } + } +} + +type ReverseCumulativeDeltaDecompressor struct { + buf *conbuf.ContinuousBuffer + pos int + bound int + firstValue float64 + lastValue float64 + length uint16 + coef float64 + idxOf8 uint + s8 byte + step byte +} + +func NewReverseCumulativeDeltaDecompressor(buf *conbuf.ContinuousBuffer, size int, fracDigits byte) *ReverseCumulativeDeltaDecompressor { + var coef float64 = 1 + if fracDigits > 0 { + coef = math.Pow(10, float64(fracDigits)) + } + return &ReverseCumulativeDeltaDecompressor{ + buf: buf, + coef: coef, + pos: size, + } +} + +func (s *ReverseCumulativeDeltaDecompressor) NextValue() (value float64, done bool) { + if s.step > 0 { + if s.length > 0 { + s.length-- + return s.lastValue, false + } + if s.pos < s.bound { + return 0, true + } + if s.idxOf8 == 0 { + s.s8 = s.buf.GetByte(s.pos) + s.pos-- + } + s.readVar() + if s.length > 0 { + s.length-- + } + return s.lastValue, false + } + u64, n, err := s.buf.GetVarUint64(0) + if err != nil { + panic(err) + } + s.firstValue = float64(u64) / s.coef + s.bound = n + s.pos-- + s.idxOf8 = uint(8 - s.buf.GetByte(s.pos)) + s.pos-- + s.s8 = s.buf.GetByte(s.pos) + s.pos-- + s.readVar() + if s.length > 0 { + s.length-- + } + s.step = 1 + return s.lastValue, false +} + +func (s *ReverseCumulativeDeltaDecompressor) readVar() { + u64, n, err := s.buf.ReverseGetVarUint64(s.pos) + if err != nil { + panic(err) + } + s.pos -= n + s.lastValue = s.firstValue + float64(u64)/s.coef + + var flag byte = 1 << s.idxOf8 + if (s.s8 & flag) == flag { + s.length, n = s.buf.DecodeRunLength(s.pos) + s.pos -= n + } + if s.idxOf8 == 7 { + s.idxOf8 = 0 + } else { + s.idxOf8++ + } +} diff --git a/chunkenc2/insdelta.go b/chunkenc2/insdelta.go new file mode 100644 index 0000000..056b595 --- /dev/null +++ b/chunkenc2/insdelta.go @@ -0,0 +1,345 @@ +package chunkenc2 + +import ( + "fmt" + "math" + + "gordenko.dev/dima/qb/bin" + "gordenko.dev/dima/qb/conbuf" +) + +// REVERSE + +type ReverseInstantDeltaCompressor struct { + buf *conbuf.ContinuousBuffer + coef float64 + pos int + firstValue float64 + lastDelta int64 + length uint16 + numIdx int +} + +func NewReverseInstantDeltaCompressor(buf *conbuf.ContinuousBuffer, size int, fracDigits byte) *ReverseInstantDeltaCompressor { + var coef float64 = 1 + if fracDigits > 0 { + coef = math.Pow(10, float64(fracDigits)) + } + s := &ReverseInstantDeltaCompressor{ + buf: buf, + pos: size, + coef: coef, + } + if size > 0 { + s.restoreState() + } + return s +} + +func (s *ReverseInstantDeltaCompressor) restoreState() { + i64, n, err := s.buf.GetVarInt64(0) + if err != nil { + panic(fmt.Sprintf("bug: get first value: %s", err)) + } + s.firstValue = float64(i64) / s.coef + + if s.pos > n { + pos := s.pos - 1 + idxOf8 := uint(8 - s.buf.GetByte(pos)) + pos-- + s8 := s.buf.GetByte(pos) + pos-- + + var n int + s.lastDelta, n, err = s.buf.ReverseGetVarInt64(pos) + if err != nil { + panic(fmt.Sprintf("bug: get last delta: %s", err)) + } + pos -= n + s.numIdx = pos + 1 + var flag byte = 1 << idxOf8 + if (s8 & flag) == flag { + s.length, _ = s.buf.DecodeRunLength(pos) + } + } +} + +func (s *ReverseInstantDeltaCompressor) Size() int { + return s.pos +} + +func (s *ReverseInstantDeltaCompressor) CalcRequiredSpace(value float64) int { + if s.pos == 0 { + n := bin.CalcVarInt64Length(int64(value * s.coef)) + return n + 3 + } + + tmp := (value - s.firstValue) * s.coef + if tmp > 0 { + tmp += eps + } else { + tmp -= eps + } + delta := int64(tmp) + + if delta == s.lastDelta { + if s.length == 0 { + return 1 + } else { + newLength := s.length + 1 + if newLength < 130 { + return 0 + } else if newLength == 130 { + return 1 + } else { + if newLength < 32769 { + return 0 + } else { + n := bin.CalcVarInt64Length(delta) + n += 2 + s8q := s.buf.GetByte(s.pos - 1) + if s8q == 8 { + n -= 1 + } else { + n -= 2 + } + return n + } + } + } + } else { + n := bin.CalcVarInt64Length(delta) + n += 2 + s8q := s.buf.GetByte(s.pos - 1) + if s8q == 8 { + n -= 1 + } else { + n -= 2 + } + return n + } +} + +// В начале буфера кодирую базовое значение. +func (s *ReverseInstantDeltaCompressor) Append(value float64) { + if s.pos == 0 { + n := s.buf.PutVarInt64(s.pos, int64(value*s.coef)) + s.pos += n + s.firstValue = value + s.encodeNewDelta(0, 0, 1) + } else { + tmp := (value - s.firstValue) * s.coef + if tmp > 0 { + tmp += eps + } else { + tmp -= eps + } + delta := int64(tmp) + + if delta == s.lastDelta { + if s.length == 0 { + s.length = 2 + s.shiftOnePosToRight() + s.buf.SetByte(s.numIdx-1, 0) + s8q := s.buf.GetByte(s.pos - 1) + s.buf.SetFlag(s.pos-2, 1<<(8-s8q)) + } else { + s.length++ + if s.length < 130 { + s.buf.SetByte(s.numIdx-1, byte(s.length-2)) + } else if s.length == 130 { + s.shiftOnePosToRight() + s.encode2bLength() + } else { + if s.length < 32769 { + s.encode2bLength() + } else { + s.appendNewDelta(delta) + } + } + } + } else { + s.appendNewDelta(delta) + } + } +} + +func (s *ReverseInstantDeltaCompressor) appendNewDelta(delta int64) { + s.length = 0 + + s8 := s.buf.GetByte(s.pos - 2) + s8q := s.buf.GetByte(s.pos - 1) + + if s8q == 8 { + s.pos -= 1 + s8 = 0 + s8q = 1 + } else { + s.pos -= 2 + s8q++ + } + s.encodeNewDelta(delta, s8, s8q) +} + +func (s *ReverseInstantDeltaCompressor) encodeNewDelta(delta int64, s8 byte, s8q byte) { + s.lastDelta = delta + s.numIdx = s.pos + n := s.buf.ReversePutVarInt64(s.pos, s.lastDelta) + s.pos += n + s.buf.SetByte(s.pos, s8) + s.pos++ + s.buf.SetByte(s.pos, s8q) + s.pos++ +} + +func (s *ReverseInstantDeltaCompressor) shiftOnePosToRight() { + s.buf.ShiftOnePosToRight(s.numIdx, s.pos) + s.pos++ + s.numIdx++ +} + +func (s *ReverseInstantDeltaCompressor) encode2bLength() { + num := s.length - 2 + s.buf.SetByte(s.numIdx-1, byte(num&127)|128) + s.buf.SetByte(s.numIdx-2, byte(num>>7)) +} + +func (s *ReverseInstantDeltaCompressor) DeleteLast() { + var ( + s8q = s.buf.GetByte(s.pos - 1) + s8 = s.buf.GetByte(s.pos - 2) + flag byte = 1 << uint(8-s8q) + ) + + if s.length > 0 { + if s.length == 2 { + s.length = 0 + s.buf.UnsetFlag(s.pos-2, flag) + s.buf.ShiftOnePosToLeft(s.numIdx, s.pos) + s.numIdx-- + s.pos-- + } else if s.length < 130 { + s.length-- + s.buf.SetByte(s.numIdx-1, byte(s.length)-2) + } else if s.length == 130 { + s.length-- + s.buf.ShiftOnePosToLeft(s.numIdx, s.pos) + s.numIdx-- + s.pos-- + s.buf.SetByte(s.numIdx-1, byte(s.length)-2) + } else { + s.length-- + s.encode2bLength() + } + } else { + if s8q > 1 { + s8q-- + flag = 1 << uint(8-s8q) + s.pos = s.numIdx + 2 + s.buf.SetByte(s.pos-2, s8) + s.buf.SetByte(s.pos-1, s8q) + } else { + s.pos = s.numIdx + 1 + s.buf.SetByte(s.pos-1, 8) + s8 = s.buf.GetByte(s.pos - 2) + flag = 1 + } + var ( + pos = s.pos - 3 + n int + err error + ) + s.lastDelta, n, err = s.buf.ReverseGetVarInt64(pos) + if err != nil { + panic(err) + } + s.numIdx = pos - n + if (s8 & flag) == flag { + s.length, _ = s.buf.DecodeRunLength(s.numIdx - 1) + } + } +} + +type ReverseInstantDeltaDecompressor struct { + step byte + buf *conbuf.ContinuousBuffer + pos int + bound int + firstValue float64 + lastValue float64 + length uint16 + coef float64 + idxOf8 uint + s8 byte +} + +func NewReverseInstantDeltaDecompressor(buf *conbuf.ContinuousBuffer, size int, fracDigits byte) *ReverseInstantDeltaDecompressor { + var coef float64 = 1 + if fracDigits > 0 { + coef = math.Pow(10, float64(fracDigits)) + } + return &ReverseInstantDeltaDecompressor{ + buf: buf, + coef: coef, + pos: size, + } +} + +func (s *ReverseInstantDeltaDecompressor) NextValue() (value float64, done bool) { + if s.step > 0 { + if s.length > 0 { + s.length-- + return s.lastValue, false + } + if s.pos < s.bound { + return 0, true + } + + if s.idxOf8 == 0 { + s.s8 = s.buf.GetByte(s.pos) + s.pos-- + } + s.readVar() + if s.length > 0 { + s.length-- + } + return s.lastValue, false + } + i64, n, err := s.buf.GetVarInt64(0) + if err != nil { + panic(err) + } + s.firstValue = float64(i64) / s.coef + s.bound = n + s.pos-- + s.idxOf8 = uint(8 - s.buf.GetByte(s.pos)) + s.pos-- + s.s8 = s.buf.GetByte(s.pos) + s.pos-- + s.readVar() + if s.length > 0 { + s.length-- + } + s.step = 1 + return s.lastValue, false +} + +func (s *ReverseInstantDeltaDecompressor) readVar() { + i64, n, err := s.buf.ReverseGetVarInt64(s.pos) + if err != nil { + panic(err) + } + s.pos -= n + s.lastValue = s.firstValue + float64(i64)/s.coef + + var flag byte = 1 << s.idxOf8 + if (s.s8 & flag) == flag { + s.length, n = s.buf.DecodeRunLength(s.pos) + s.pos -= n + } + if s.idxOf8 == 7 { + s.idxOf8 = 0 + } else { + s.idxOf8++ + } +} diff --git a/chunkenc2/time_delta_delta.go b/chunkenc2/time_delta_delta.go new file mode 100644 index 0000000..76b0c49 --- /dev/null +++ b/chunkenc2/time_delta_delta.go @@ -0,0 +1,374 @@ +package chunkenc2 + +import ( + "fmt" + + "gordenko.dev/dima/qb/bin" + "gordenko.dev/dima/qb/conbuf" +) + +// REVERSE + +const ( + lastUnixtimeIdx = 0 + baseDeltaIdx = 4 +) + +type ReverseTimeDeltaOfDeltaCompressor struct { + buf *conbuf.ContinuousBuffer + pos int + baseDelta uint32 + lastUnixtime uint32 + lastDeltaOfDelta int64 + length uint16 + numIdx int +} + +func NewReverseTimeDeltaOfDeltaCompressor(buf *conbuf.ContinuousBuffer, size int) *ReverseTimeDeltaOfDeltaCompressor { + s := &ReverseTimeDeltaOfDeltaCompressor{ + buf: buf, + pos: size, + } + if size > 0 { + s.restoreState() + } + return s +} + +func (s *ReverseTimeDeltaOfDeltaCompressor) restoreState() { + s.lastUnixtime = s.buf.GetUint32(lastUnixtimeIdx) + if s.pos > 4 { + u64, _, err := s.buf.GetVarUint64(baseDeltaIdx) + if err != nil { + panic(fmt.Sprintf("bug: get base delta: %s", err)) + } + s.baseDelta = uint32(u64) + + pos := s.pos - 1 + idxOf8 := uint(8 - s.buf.GetByte(pos)) + pos-- + s8 := s.buf.GetByte(pos) + pos-- + + var n int + s.lastDeltaOfDelta, n, err = s.buf.ReverseGetVarInt64(pos) + if err != nil { + panic(err) + } + pos -= n + s.numIdx = pos + 1 + var flag byte = 1 << idxOf8 + if (s8 & flag) == flag { + s.length, _ = s.buf.DecodeRunLength(pos) + } + } +} + +func (s *ReverseTimeDeltaOfDeltaCompressor) Size() int { + return s.pos +} + +func (s *ReverseTimeDeltaOfDeltaCompressor) CalcRequiredSpace(unixtime uint32) int { + if s.pos == 0 { + return 4 + } + + if s.baseDelta == 0 { + baseDelta := unixtime - s.lastUnixtime + n := bin.CalcVarUint64Length(uint64(baseDelta)) + return n + 3 + } + + deltaOfDelta := int64(unixtime-s.lastUnixtime) - int64(s.baseDelta) + if deltaOfDelta == s.lastDeltaOfDelta { + if s.length == 0 { + return 1 + } else { + newLength := s.length + 1 + if newLength < 130 { + return 0 + } else if newLength == 130 { + return 1 + } else { + if newLength < 32769 { + return 0 + } else { + n := bin.CalcVarInt64Length(deltaOfDelta) + n += 2 + s8q := s.buf.GetByte(s.pos - 1) + if s8q == 8 { + n -= 1 + } else { + n -= 2 + } + return n + } + } + } + } else { + n := bin.CalcVarInt64Length(deltaOfDelta) + n += 2 + s8q := s.buf.GetByte(s.pos - 1) + if s8q == 8 { + n -= 1 + } else { + n -= 2 + } + return n + } +} + +func (s *ReverseTimeDeltaOfDeltaCompressor) Append(unixtime uint32) { + if s.pos == 0 { + s.lastUnixtime = unixtime + s.buf.PutUint32(lastUnixtimeIdx, unixtime) + s.pos += 4 + return + } + + if s.baseDelta == 0 { + s.baseDelta = unixtime - s.lastUnixtime + s.lastDeltaOfDelta = 0 + s.lastUnixtime = unixtime + s.buf.PutUint32(lastUnixtimeIdx, unixtime) + + n := s.buf.PutVarUint64(s.pos, uint64(s.baseDelta)) + s.pos += n + s.encodeNewDeltaOfDelta(0, 0, 1) + return + } + + deltaOfDelta := int64(unixtime-s.lastUnixtime) - int64(s.baseDelta) + s.lastUnixtime = unixtime + s.buf.PutUint32(lastUnixtimeIdx, unixtime) + + if deltaOfDelta == s.lastDeltaOfDelta { + if s.length == 0 { + s.length = 2 + s.shiftOnePosToRight() + s.buf.SetByte(s.numIdx-1, 0) + s8q := s.buf.GetByte(s.pos - 1) + s.buf.SetFlag(s.pos-2, 1<<(8-s8q)) + } else { + s.length++ + if s.length < 130 { + s.buf.SetByte(s.numIdx-1, byte(s.length-2)) + } else if s.length == 130 { + s.shiftOnePosToRight() + s.encode2bLength() + } else { + if s.length < 32769 { + s.encode2bLength() + } else { + s.appendNewDeltaOfDelta(deltaOfDelta) + } + } + } + } else { + s.appendNewDeltaOfDelta(deltaOfDelta) + } +} + +func (s *ReverseTimeDeltaOfDeltaCompressor) appendNewDeltaOfDelta(deltaOfDelta int64) { + s.length = 0 + s8 := s.buf.GetByte(s.pos - 2) + s8q := s.buf.GetByte(s.pos - 1) + + if s8q == 8 { + s.pos -= 1 + s8 = 0 + s8q = 1 + } else { + s.pos -= 2 + s8q++ + } + + s.encodeNewDeltaOfDelta(deltaOfDelta, s8, s8q) +} + +func (s *ReverseTimeDeltaOfDeltaCompressor) encodeNewDeltaOfDelta(deltaOfDelta int64, s8 byte, s8q byte) { + s.lastDeltaOfDelta = deltaOfDelta + s.numIdx = s.pos + n := s.buf.ReversePutVarInt64(s.pos, deltaOfDelta) + s.pos += n + s.buf.SetByte(s.pos, s8) + s.pos++ + s.buf.SetByte(s.pos, s8q) + s.pos++ +} + +func (s *ReverseTimeDeltaOfDeltaCompressor) shiftOnePosToRight() { + s.buf.ShiftOnePosToRight(s.numIdx, s.pos) + s.pos++ + s.numIdx++ +} + +func (s *ReverseTimeDeltaOfDeltaCompressor) encode2bLength() { + num := s.length - 2 + s.buf.SetByte(s.numIdx-1, byte(num&127)|128) + s.buf.SetByte(s.numIdx-2, byte(num>>7)) +} + +func (s *ReverseTimeDeltaOfDeltaCompressor) DeleteLast() { + var ( + s8q = s.buf.GetByte(s.pos - 1) + s8 = s.buf.GetByte(s.pos - 2) + flag byte = 1 << uint(8-s8q) + ) + + if s.length > 0 { + if s.length == 2 { + s.length = 0 + s.buf.UnsetFlag(s.pos-2, flag) + s.buf.ShiftOnePosToLeft(s.numIdx, s.pos) + s.numIdx-- + s.pos-- + } else if s.length < 130 { + s.length-- + s.buf.SetByte(s.numIdx-1, byte(s.length)-2) + } else if s.length == 130 { + s.length-- + s.buf.ShiftOnePosToLeft(s.numIdx, s.pos) + s.numIdx-- + s.pos-- + s.buf.SetByte(s.numIdx-1, byte(s.length)-2) + } else { + s.length-- + s.encode2bLength() + } + } else { + if s8q > 1 { + s8q-- + flag = 1 << uint(8-s8q) + s.pos = s.numIdx + 2 + s.buf.SetByte(s.pos-2, s8) + s.buf.SetByte(s.pos-1, s8q) + } else { + s.pos = s.numIdx + 1 + s.buf.SetByte(s.pos-1, 8) + s8 = s.buf.GetByte(s.pos - 2) + flag = 1 + } + var ( + pos = s.pos - 3 + n int + err error + ) + s.lastDeltaOfDelta, n, err = s.buf.ReverseGetVarInt64(pos) + if err != nil { + panic(err) + } + s.numIdx = pos - n + if (s8 & flag) == flag { + s.length, _ = s.buf.DecodeRunLength(s.numIdx - 1) + } + } + + delta := int64(s.baseDelta) + s.lastDeltaOfDelta + s.lastUnixtime = uint32(int64(s.lastUnixtime) - delta) + s.buf.PutUint32(lastUnixtimeIdx, s.lastUnixtime) +} + +type ReverseTimeDeltaOfDeltaDecompressor struct { + step byte + buf *conbuf.ContinuousBuffer + pos int + bound int + lastUnixtime uint32 + baseDelta uint32 + lastDeltaOfDelta int64 + length uint16 + idxOf8 uint + s8 byte +} + +func NewReverseTimeDeltaOfDeltaDecompressor(buf *conbuf.ContinuousBuffer, size int) *ReverseTimeDeltaOfDeltaDecompressor { + return &ReverseTimeDeltaOfDeltaDecompressor{ + buf: buf, + pos: size, + } +} + +func (s *ReverseTimeDeltaOfDeltaDecompressor) NextValue() (value uint32, done bool) { + if s.step == 0 { + if s.pos == 0 { + return 0, true + } + s.lastUnixtime = s.buf.GetUint32(lastUnixtimeIdx) + s.step = 1 + return s.lastUnixtime, false + } + + if s.step == 1 { + if s.pos == baseDeltaIdx { + return 0, true + } + u64, n, err := s.buf.GetVarUint64(baseDeltaIdx) + if err != nil { + panic("EOF") + } + s.bound = baseDeltaIdx + n + s.baseDelta = uint32(u64) + + s.pos-- + s.idxOf8 = uint(8 - s.buf.GetByte(s.pos)) + s.pos-- + s.s8 = s.buf.GetByte(s.pos) + s.pos-- + + s.readVar() + if s.length > 0 { + s.length-- + } + s.step = 2 + return s.lastUnixtime, false + } + + if s.length > 0 { + s.length-- + delta := int64(s.baseDelta) + s.lastDeltaOfDelta + s.lastUnixtime = uint32(int64(s.lastUnixtime) - delta) + return s.lastUnixtime, false + } + + if s.pos < s.bound { + return 0, true + } + + if s.idxOf8 == 0 { + s.s8 = s.buf.GetByte(s.pos) + s.pos-- + } + s.readVar() + if s.length > 0 { + s.length-- + } + return s.lastUnixtime, false +} + +func (s *ReverseTimeDeltaOfDeltaDecompressor) readVar() { + var ( + n int + err error + ) + + s.lastDeltaOfDelta, n, err = s.buf.ReverseGetVarInt64(s.pos) + if err != nil { + panic(err) + } + s.pos -= n + + delta := int64(s.baseDelta) + s.lastDeltaOfDelta + s.lastUnixtime = uint32(int64(s.lastUnixtime) - delta) + + var flag byte = 1 << s.idxOf8 + if (s.s8 & flag) == flag { + s.length, n = s.buf.DecodeRunLength(s.pos) + s.pos -= n + } + if s.idxOf8 == 7 { + s.idxOf8 = 0 + } else { + s.idxOf8++ + } +} diff --git a/conbuf/conbuf.go b/conbuf/conbuf.go index a10fb0b..19215d2 100644 --- a/conbuf/conbuf.go +++ b/conbuf/conbuf.go @@ -9,6 +9,7 @@ import ( ) const chunkSize = 512 +const ChunkSize = 512 var ( ErrOutOfRange = errors.New("out of range") diff --git a/database/api.go b/database/api.go index 41e874f..2792f88 100644 --- a/database/api.go +++ b/database/api.go @@ -215,7 +215,7 @@ func (s *Database) AddMetric(req proto.AddMetricReq) uint16 { switch resultCode { case Succeed: - waitCh := s.txlog.WriteAddedMetric(txlog.AddedMetric{ + waitCh := s.txlog.Append(txlog.AddedMetric{ MetricID: req.MetricID, MetricType: req.MetricType, FracDigits: req.FracDigits, @@ -289,21 +289,18 @@ func (s *Database) DeleteMetric(req proto.DeleteMetricReq) uint16 { switch result.ResultCode { case Succeed: var ( - freeDataPages []uint32 - freeIndexPages []uint32 + freePageNumbers []uint32 ) if result.RootPageNo > 0 { - pageLists, err := s.atree.GetAllPages(result.RootPageNo) + var err error + freePageNumbers, err = s.atree.GetAllPages(result.RootPageNo) if err != nil { diploma.Abort(diploma.FailedAtreeRequest, err) } - freeDataPages = pageLists.DataPages - freeIndexPages = pageLists.IndexPages } - waitCh := s.txlog.WriteDeletedMetric(txlog.DeletedMetric{ - MetricID: req.MetricID, - FreeDataPages: freeDataPages, - FreeIndexPages: freeIndexPages, + waitCh := s.txlog.Append(txlog.DeletedMetric{ + MetricID: req.MetricID, + FreePageNumbers: freePageNumbers, }) <-waitCh @@ -348,7 +345,7 @@ func (s *Database) AppendMeasure(req proto.AppendMeasureReq) uint16 { switch result.ResultCode { case CanAppend: - waitCh := s.txlog.WriteAppendMeasure(txlog.AppendedMeasure{ + waitCh := s.txlog.Append(txlog.AppendedMeasure{ MetricID: req.MetricID, Timestamp: req.Timestamp, Value: req.Value, @@ -357,35 +354,58 @@ func (s *Database) AppendMeasure(req proto.AppendMeasureReq) uint16 { case NewPage: filled := result.FilledPage - report, err := s.atree.AppendDataPage(atree.AppendDataPageReq{ - MetricID: req.MetricID, - Timestamp: req.Timestamp, - Value: req.Value, - Since: filled.Since, - RootPageNo: filled.RootPageNo, - PrevPageNo: filled.PrevPageNo, + + var path atree.PathToDataPage + if filled.RootPageNo > 0 { + path, err := s.atree.FindPathToLastPage(filled.RootPageNo) + if err != nil { + // FIX + diploma.Abort(diploma.WriteToAtreeFailed, err) + } + // for _, leg := range path.Legs { + // pagesToRelease = append(pagesToRelease, leg.PageNo) + // } + + if path.LastPageNo != filled.PrevPageNo { + diploma.Abort( + diploma.WrongPrevPageNo, + fmt.Errorf("bug: last pageNo %d in tree != prev pageNo %d in _metric", + path.LastPageNo, filled.PrevPageNo), + ) + } + } + + // report, err := s.atree.AppendDataPages(atree.AppendDataPageReq{ + // MetricID: req.MetricID, + // Timestamp: req.Timestamp, + // Value: req.Value, + // Since: filled.Since, + // RootPageNo: filled.RootPageNo, + // PrevPageNo: filled.PrevPageNo, + // TimestampsChunks: filled.TimestampsChunks, + // TimestampsSize: filled.TimestampsSize, + // ValuesChunks: filled.ValuesChunks, + // ValuesSize: filled.ValuesSize, + // }) + // if err != nil { + // diploma.Abort(diploma.WriteToAtreeFailed, err) + // } + // _ = report + + //waitCh := s.txlog.WriteAppended(path, report) + + waitCh := s.txlog.Append(txlog.AppendedPagesReq{ + Legs: path.Legs, + MetricID: req.MetricID, + Timestamp: req.Timestamp, + Value: req.Value, + LastPageNo: filled.PrevPageNo, + //Pages: , TimestampsChunks: filled.TimestampsChunks, TimestampsSize: filled.TimestampsSize, ValuesChunks: filled.ValuesChunks, ValuesSize: filled.ValuesSize, - }) - if err != nil { - diploma.Abort(diploma.WriteToAtreeFailed, err) - } - _ = report - waitCh := s.txlog.WriteAppendedMeasureWithOverflow( - txlog.AppendedMeasureWithOverflow{ - // FIX - // MetricID: req.MetricID, - // Timestamp: req.Timestamp, - // Value: req.Value, - // IsDataPageReused: report.IsDataPageReused, - // DataPageNo: report.DataPageNo, - // IsRootChanged: report.IsRootChanged, - // RootPageNo: report.NewRootPageNo, - // ReusedIndexPages: report.ReusedIndexPages, - }, - false, + }, ) <-waitCh @@ -442,7 +462,9 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 { until = result.Until untilValue = result.UntilValue // - toAppendMeasures []proto.Measure + //toAppendMeasures []proto.Measure + + dataPages []atree.NotLinkedDataPage ) for idx, measure := range req.Measures { @@ -451,30 +473,32 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 { since = measure.Timestamp } else { if measure.Timestamp <= until { - if len(toAppendMeasures) > 0 { - waitCh := s.txlog.WriteAppendMeasures( - txlog.AppendedMeasures{ - MetricID: req.MetricID, - Measures: toAppendMeasures, - }, - false, - ) - <-waitCh - } + // пишу в БД що вже встиг закодувати FIX + // if len(toAppendMeasures) > 0 { + // waitCh := s.txlog.WriteAppendMeasures( + // txlog.AppendedMeasures{ + // MetricID: req.MetricID, + // Measures: toAppendMeasures, + // }, + // false, + // ) + // <-waitCh + // } return proto.ErrExpiredMeasure } if result.MetricType == diploma.Cumulative && measure.Value < untilValue { - if len(toAppendMeasures) > 0 { - waitCh := s.txlog.WriteAppendMeasures( - txlog.AppendedMeasures{ - MetricID: req.MetricID, - Measures: toAppendMeasures, - }, - false, - ) - <-waitCh - } + // пишу в БД що вже встиг закодувати FIX + // if len(toAppendMeasures) > 0 { + // waitCh := s.txlog.WriteAppendMeasures( + // txlog.AppendedMeasures{ + // MetricID: req.MetricID, + // Measures: toAppendMeasures, + // }, + // false, + // ) + // <-waitCh + // } return proto.ErrNonMonotonicValue } } @@ -485,38 +509,40 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 { totalSpace := timestamps.Size() + values.Size() + extraSpace if totalSpace <= atree.DataPagePayloadSize { + // накопичую timestamps.Append(measure.Timestamp) values.Append(measure.Value) - toAppendMeasures = append(toAppendMeasures, measure) + //toAppendMeasures = append(toAppendMeasures, measure) } else { - if len(toAppendMeasures) > 0 { - waitCh := s.txlog.WriteAppendMeasures( - txlog.AppendedMeasures{ - MetricID: req.MetricID, - Measures: toAppendMeasures, - }, - true, - ) - <-waitCh + // сторінка заповнена + // if len(toAppendMeasures) > 0 { + // waitCh := s.txlog.WriteAppendMeasures( + // txlog.AppendedMeasures{ + // MetricID: req.MetricID, + // Measures: toAppendMeasures, + // }, + // true, + // ) + // <-waitCh - toAppendMeasures = nil - } - report, err := s.atree.AppendDataPage(atree.AppendDataPageReq{ - MetricID: req.MetricID, - Timestamp: until, - Value: untilValue, - Since: since, - RootPageNo: rootPageNo, - PrevPageNo: prevPageNo, + // toAppendMeasures = nil + // } + + pageData := make([]byte, atree.PageSize) + + atree.ChunksToNotLinkedDataPage(pageData, atree.ChunksToNotLinkedDataPageReq{ + //PrevPageNo: path.LastPageNo, TimestampsChunks: timestampsBuf.Chunks(), TimestampsSize: uint16(timestamps.Size()), ValuesChunks: valuesBuf.Chunks(), ValuesSize: uint16(values.Size()), }) - if err != nil { - diploma.Abort(diploma.WriteToAtreeFailed, err) - } - _ = report + + dataPages = append(dataPages, atree.NotLinkedDataPage{ + Since: since, + Data: pageData, + }) + // FIX // prevPageNo = report.DataPageNo // if report.IsRootChanged { @@ -540,7 +566,7 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 { timestampsBuf = conbuf.New(nil) valuesBuf = conbuf.New(nil) - timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor( + timestamps = chunkenc.NewReverseTimeDeltaCompressor( timestampsBuf, 0) if result.MetricType == diploma.Cumulative { values = chunkenc.NewReverseCumulativeDeltaCompressor( @@ -560,16 +586,32 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 { untilValue = measure.Value } - if len(toAppendMeasures) > 0 { - waitCh := s.txlog.WriteAppendMeasures( - txlog.AppendedMeasures{ - MetricID: req.MetricID, - Measures: toAppendMeasures, - }, - false, - ) - <-waitCh - } + waitCh := s.txlog.Append(txlog.AppendedPagesReq{ + MetricID: req.MetricID, + Timestamp: until, + Value: untilValue, + LastPageNo: prevPageNo, + //Measures: toAppendMeasures, + }, + ) + <-waitCh + + // report, err := s.atree.AppendDataPage(atree.AppendDataPageReq{}) + // if err != nil { + // diploma.Abort(diploma.WriteToAtreeFailed, err) + // } + // _ = report + + // if len(toAppendMeasures) > 0 { + // waitCh := s.txlog.WriteAppendMeasures( + // txlog.AppendedMeasures{ + // MetricID: req.MetricID, + // Measures: toAppendMeasures, + // }, + // false, + // ) + // <-waitCh + // } case NoMetric: return proto.ErrNoMetric @@ -602,22 +644,21 @@ func (s *Database) DeleteMeasures(req proto.DeleteMeasuresReq) uint16 { case DeleteFromAtreeNotNeeded: // регистрирую удаление в TransactionLog - waitCh := s.txlog.WriteDeletedMeasures(txlog.DeletedMeasures{ + waitCh := s.txlog.Append(txlog.DeletedMeasures{ MetricID: req.MetricID, }) <-waitCh case DeleteFromAtreeRequired: // собираю номера всех data и index страниц метрики (типа запись REDO лога). - pageLists, err := s.atree.GetAllPages(req.MetricID) + pageNumbers, err := s.atree.GetAllPages(req.MetricID) if err != nil { diploma.Abort(diploma.FailedAtreeRequest, err) } // регистрирую удаление в TransactionLog - waitCh := s.txlog.WriteDeletedMeasures(txlog.DeletedMeasures{ - MetricID: req.MetricID, - FreeDataPages: pageLists.DataPages, - FreeIndexPages: pageLists.IndexPages, + waitCh := s.txlog.Append(txlog.DeletedMeasures{ + MetricID: req.MetricID, + FreePageNumbers: pageNumbers, }) <-waitCh diff --git a/database/database.go b/database/database.go index 890f64d..7d12bcf 100644 --- a/database/database.go +++ b/database/database.go @@ -109,7 +109,6 @@ func (s *Database) ListenAndServe() (err error) { s.atree, err = atree.New(atree.Options{ Dir: s.dir, DatabaseName: s.databaseName, - FreeList: s.freeList, }) if err != nil { return fmt.Errorf("atree.New: %s", err) @@ -169,6 +168,8 @@ func (s *Database) recovery() { Dir: s.dir, LogNumber: logNumber, AppendToWorkerQueue: s.appendJobToWorkerQueue, + FreeList: s.freeList, + Atree: s.atree, ExitCh: s.exitCh, WaitGroup: s.waitGroup, }) @@ -374,11 +375,8 @@ func (s *Database) replayChangesRecord(untyped any) error { case txlog.DeletedMetric: delete(s.metrics, rec.MetricID) - if len(rec.FreeDataPages) > 0 { - s.freeList.AddPages(rec.FreeDataPages) - } - if len(rec.FreeIndexPages) > 0 { - s.freeList.AddPages(rec.FreeIndexPages) + if len(rec.FreePageNumbers) > 0 { + s.freeList.AddPages(rec.FreePageNumbers) } case txlog.AppendedMeasure: @@ -413,34 +411,31 @@ func (s *Database) replayChangesRecord(untyped any) error { } } - case txlog.AppendedMeasureWithOverflow: - metric, ok := s.metrics[rec.MetricID] - if ok { - metric.ReinitBy(rec.Timestamp, rec.Value) - if rec.IsRootChanged { - metric.RootPageNo = rec.RootPageNo - } - metric.LastPageNo = rec.DataPageNo - // delete free pages - if rec.IsDataPageReused { - s.freeList.DeleteReservedPages([]uint32{ - rec.DataPageNo, - }) - } - if len(rec.ReusedIndexPages) > 0 { - s.freeList.DeleteReservedPages(rec.ReusedIndexPages) - } - } + // case txlog.AppendedMeasureWithOverflow: + // metric, ok := s.metrics[rec.MetricID] + // if ok { + // metric.ReinitBy(rec.Timestamp, rec.Value) + // if rec.IsRootChanged { + // metric.RootPageNo = rec.RootPageNo + // } + // metric.LastPageNo = rec.DataPageNo + // // delete free pages + // if rec.IsDataPageReused { + // s.freeList.DeleteReservedPages([]uint32{ + // rec.DataPageNo, + // }) + // } + // if len(rec.ReusedIndexPages) > 0 { + // s.freeList.DeleteReservedPages(rec.ReusedIndexPages) + // } + // } case txlog.DeletedMeasures: metric, ok := s.metrics[rec.MetricID] if ok { metric.DeleteMeasures() - if len(rec.FreeDataPages) > 0 { - s.freeList.AddPages(rec.FreeDataPages) - } - if len(rec.FreeDataPages) > 0 { - s.freeList.AddPages(rec.FreeIndexPages) + if len(rec.FreePageNumbers) > 0 { + s.freeList.AddPages(rec.FreePageNumbers) } } diff --git a/database/metric.go b/database/metric.go index a4b8527..b40ad79 100644 --- a/database/metric.go +++ b/database/metric.go @@ -2,7 +2,6 @@ package database import ( octopus "gordenko.dev/dima/qb" - "gordenko.dev/dima/qb/chunkenc" "gordenko.dev/dima/qb/conbuf" ) @@ -24,43 +23,44 @@ type _metric struct { } func (s *_metric) ReinitBy(timestamp uint32, value float64) { - s.TimestampsBuf = conbuf.New(nil) - s.ValuesBuf = conbuf.New(nil) - // - s.Timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor( - s.TimestampsBuf, 0) + // s.TimestampsBuf = conbuf.New(nil) + // s.ValuesBuf = conbuf.New(nil) + // // + // s.Timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor( + // s.TimestampsBuf, 0) - if s.MetricType == octopus.Cumulative { - s.Values = chunkenc.NewReverseCumulativeDeltaCompressor( - s.ValuesBuf, 0, s.FracDigits) - } else { - s.Values = chunkenc.NewReverseInstantDeltaCompressor( - s.ValuesBuf, 0, s.FracDigits) - } + // if s.MetricType == octopus.Cumulative { + // s.Values = chunkenc.NewReverseCumulativeDeltaCompressor( + // s.ValuesBuf, 0, s.FracDigits) + // } else { + // s.Values = chunkenc.NewReverseInstantDeltaCompressor( + // s.ValuesBuf, 0, s.FracDigits) + // } - s.Timestamps.Append(timestamp) - s.Values.Append(value) + // s.Timestamps.Append(timestamp) + // s.Values.Append(value) - s.Since = timestamp - s.SinceValue = value - s.Until = timestamp - s.UntilValue = value + // s.Since = timestamp + // s.SinceValue = value + // s.Until = timestamp + // s.UntilValue = value } func (s *_metric) DeleteMeasures() { s.TimestampsBuf = conbuf.New(nil) s.ValuesBuf = conbuf.New(nil) - // - s.Timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor( - s.TimestampsBuf, 0) + // fix + // s.Timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor( + // s.TimestampsBuf, 0) - if s.MetricType == octopus.Cumulative { - s.Values = chunkenc.NewReverseCumulativeDeltaCompressor( - s.ValuesBuf, 0, s.FracDigits) - } else { - s.Values = chunkenc.NewReverseInstantDeltaCompressor( - s.ValuesBuf, 0, s.FracDigits) - } + // if s.MetricType == octopus.Cumulative { + // s.Values = chunkenc.NewReverseCumulativeDeltaCompressor( + // s.ValuesBuf, 0, s.FracDigits) + // } else { + // s.Values = chunkenc.NewReverseInstantDeltaCompressor( + // s.ValuesBuf, 0, s.FracDigits) + // } + // END OF FIX s.RootPageNo = 0 s.LastPageNo = 0 diff --git a/database/proc.go b/database/proc.go index 2717554..f6c478e 100644 --- a/database/proc.go +++ b/database/proc.go @@ -352,6 +352,11 @@ func (s *Database) tryAppendMeasure(req tryAppendMeasureReq) { s.startAppendMeasure(metric, req, lockEntry) } +type ReadBound struct { + ValuesPos int + ValuesLastValue float64 +} + func (s *Database) startAppendMeasure(metric *_metric, req tryAppendMeasureReq, lockEntry *metricLockEntry) { if req.Timestamp <= metric.Until { req.ResultCh <- tryAppendMeasureResult{ @@ -466,8 +471,7 @@ func (s *Database) appendMeasure(rec txlog.AppendedMeasure) { } } -func (s *Database) appendMeasures(extended txlog.AppendedMeasuresExtended) { - rec := extended.Record +func (s *Database) appendMeasures(rec txlog.AppendedMeasures) { metric, ok := s.metrics[rec.MetricID] if !ok { diploma.Abort(diploma.NoMetricBug, @@ -500,10 +504,8 @@ func (s *Database) appendMeasures(extended txlog.AppendedMeasuresExtended) { metric.UntilValue = measure.Value } - if !extended.HoldLock { - lockEntry.XLock = false - s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry) - } + lockEntry.XLock = false + s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry) } func (s *Database) appendMeasureAfterOverflow(extended txlog.AppendedMeasureWithOverflowExtended) { @@ -544,10 +546,8 @@ func (s *Database) appendMeasureAfterOverflow(extended txlog.AppendedMeasureWith s.freeList.DeleteReservedPages(rec.ReusedIndexPages) } - if !extended.HoldLock { - lockEntry.XLock = false - s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry) - } + lockEntry.XLock = false + s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry) } type tryAppendMeasuresReq struct { @@ -880,7 +880,7 @@ func (s *Database) applyChanges(req txlog.Changes) { case txlog.AppendedMeasure: s.appendMeasure(rec) - case txlog.AppendedMeasuresExtended: + case txlog.AppendedMeasures: s.appendMeasures(rec) case txlog.AppendedMeasureWithOverflowExtended: @@ -1019,11 +1019,8 @@ func (s *Database) deleteMetric(rec txlog.DeletedMetric) { delete(s.metrics, rec.MetricID) delete(s.metricLockEntries, rec.MetricID) - if len(rec.FreeDataPages) > 0 { - s.freeList.AddPages(rec.FreeDataPages) - } - if len(rec.FreeIndexPages) > 0 { - s.freeList.AddPages(rec.FreeIndexPages) + if len(rec.FreePageNumbers) > 0 { + s.freeList.AddPages(rec.FreePageNumbers) } if len(addMetricReqs) > 0 { @@ -1053,11 +1050,8 @@ func (s *Database) deleteMeasures(rec txlog.DeletedMeasures) { } metric.DeleteMeasures() lockEntry.XLock = false - if len(rec.FreeDataPages) > 0 { - s.freeList.AddPages(rec.FreeDataPages) - } - if len(rec.FreeDataPages) > 0 { - s.freeList.AddPages(rec.FreeIndexPages) + if len(rec.FreePageNumbers) > 0 { + s.freeList.AddPages(rec.FreePageNumbers) } s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry) } diff --git a/database_linux b/database_linux deleted file mode 100755 index 48bc8f7..0000000 Binary files a/database_linux and /dev/null differ diff --git a/database_windows b/database_windows deleted file mode 100755 index 09447d8..0000000 Binary files a/database_windows and /dev/null differ diff --git a/go.mod b/go.mod index c233df2..cbdd8af 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,7 @@ module gordenko.dev/dima/qb go 1.24.2 require ( - github.com/RoaringBitmap/roaring/v2 v2.14.4 gopkg.in/ini.v1 v1.67.1 -) - -require ( - github.com/bits-and-blooms/bitset v1.24.2 // indirect - github.com/mschoch/smat v0.2.0 // indirect + gordenko.dev/dima/bin v0.0.0-20251101161620-7fa2d0f2923a + gordenko.dev/dima/pretty v0.0.0-20221225212746-0c27d8c0ac69 ) diff --git a/go.sum b/go.sum index ce22264..157cb07 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,6 @@ -github.com/RoaringBitmap/roaring/v2 v2.14.4 h1:4aKySrrg9G/5oRtJ3TrZLObVqxgQ9f1znCRBwEwjuVw= -github.com/RoaringBitmap/roaring/v2 v2.14.4/go.mod h1:oMvV6omPWr+2ifRdeZvVJyaz+aoEUopyv5iH0u/+wbY= -github.com/bits-and-blooms/bitset v1.24.2 h1:M7/NzVbsytmtfHbumG+K2bremQPMJuqv1JD3vOaFxp0= -github.com/bits-and-blooms/bitset v1.24.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= -github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -24,3 +18,7 @@ gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gordenko.dev/dima/bin v0.0.0-20251101161620-7fa2d0f2923a h1:IAJvA+4IxPAUVXqXAIGrlfljoAbbzek4jPgrLwWJm9E= +gordenko.dev/dima/bin v0.0.0-20251101161620-7fa2d0f2923a/go.mod h1:/I+9fvRUzXHgXSwGEwOK1mBBM4BJ3AtCmoCZ2upDiwk= +gordenko.dev/dima/pretty v0.0.0-20221225212746-0c27d8c0ac69 h1:nyJ3mzTQ46yUeMZCdLyYcs7B5JCS54c67v84miyhq2E= +gordenko.dev/dima/pretty v0.0.0-20221225212746-0c27d8c0ac69/go.mod h1:AxgKDktpqBVyIOhIcP+nlCpK+EsJyjN5kPdqyd8euVU= diff --git a/loadtest_linux b/loadtest_linux deleted file mode 100755 index 122c54b..0000000 Binary files a/loadtest_linux and /dev/null differ diff --git a/loadtest_windows b/loadtest_windows deleted file mode 100755 index a979d15..0000000 Binary files a/loadtest_windows and /dev/null differ diff --git a/requests_linux b/requests_linux deleted file mode 100755 index 620b723..0000000 Binary files a/requests_linux and /dev/null differ diff --git a/requests_windows b/requests_windows deleted file mode 100755 index d57f6fc..0000000 Binary files a/requests_windows and /dev/null differ diff --git a/txlog/reader.go b/txlog/reader.go index a4fd427..849235c 100644 --- a/txlog/reader.go +++ b/txlog/reader.go @@ -9,9 +9,7 @@ import ( "io" "os" - diploma "gordenko.dev/dima/qb" "gordenko.dev/dima/qb/bin" - "gordenko.dev/dima/qb/proto" ) type Reader struct { @@ -102,49 +100,43 @@ func (s *Reader) parseRecords(body []byte) ([]any, error) { switch recordType { case CodeAddedMetric: - var rec AddedMetric - rec, err = s.readAddedMetric(src) - if err != nil { + rec := new(AddedMetric) + if err = rec.Parse(src); err != nil { return nil, err } records = append(records, rec) case CodeDeletedMetric: - var rec DeletedMetric - rec, err = s.readDeletedMetric(src) - if err != nil { + rec := new(DeletedMetric) + if err = rec.Parse(src); err != nil { return nil, err } records = append(records, rec) case CodeAppendedMeasure: - var rec AppendedMeasure - rec, err = s.readAppendedMeasure(src) - if err != nil { + rec := new(AppendedMeasure) + if err = rec.Parse(src); err != nil { return nil, err } records = append(records, rec) case CodeAppendedMeasures: - var rec AppendedMeasures - rec, err = s.readAppendedMeasures(src) - if err != nil { + rec := new(AppendedMeasures) + if err = rec.Parse(src); err != nil { return nil, err } records = append(records, rec) - case CodeAppendedMeasureWithOverflow: - var rec AppendedMeasureWithOverflow - rec, err = s.readAppendedMeasureWithOverflow(src) - if err != nil { + case CodeAppendedPages: + rec := new(AppendedPages) + if err = rec.Parse(src); err != nil { return nil, err } records = append(records, rec) case CodeDeletedMeasures: - var rec DeletedMeasures - rec, err = s.readDeletedMeasures(src) - if err != nil { + rec := new(DeletedMeasures) + if err = rec.Parse(src); err != nil { return nil, err } records = append(records, rec) @@ -155,183 +147,183 @@ func (s *Reader) parseRecords(body []byte) ([]any, error) { } } -func (s *Reader) readAddedMetric(src *bytes.Buffer) (_ AddedMetric, err error) { - arr, err := bin.ReadN(src, 6) - if err != nil { - return - } - return AddedMetric{ - MetricID: bin.GetUint32(arr), - MetricType: diploma.MetricType(arr[4]), - FracDigits: int(arr[5]), - }, nil -} +// func (s *Reader) readAddedMetric(src *bytes.Buffer) (_ AddedMetric, err error) { +// arr, err := bin.ReadN(src, 6) +// if err != nil { +// return +// } +// return AddedMetric{ +// MetricID: bin.GetUint32(arr), +// MetricType: diploma.MetricType(arr[4]), +// FracDigits: int(arr[5]), +// }, nil +// } -func (s *Reader) readDeletedMetric(src *bytes.Buffer) (_ DeletedMetric, err error) { - var rec DeletedMetric - rec.MetricID, err = bin.ReadUint32(src) - if err != nil { - return - } - // free data pages - dataQty, _, err := bin.ReadVarUint64(src) - if err != nil { - return - } - for range dataQty { - var pageNo uint32 - pageNo, err = bin.ReadUint32(src) - if err != nil { - return - } - rec.FreeDataPages = append(rec.FreeDataPages, pageNo) - } - // free index pages - indexQty, _, err := bin.ReadVarUint64(src) - if err != nil { - return - } - for range indexQty { - var pageNo uint32 - pageNo, err = bin.ReadUint32(src) - if err != nil { - return - } - rec.FreeIndexPages = append(rec.FreeIndexPages, pageNo) - } - return rec, nil -} +// func (s *Reader) readDeletedMetric(src *bytes.Buffer) (_ DeletedMetric, err error) { +// var rec DeletedMetric +// rec.MetricID, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// // free data pages +// dataQty, _, err := bin.ReadVarUint64(src) +// if err != nil { +// return +// } +// for range dataQty { +// var pageNo uint32 +// pageNo, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// rec.FreeDataPages = append(rec.FreeDataPages, pageNo) +// } +// // free index pages +// indexQty, _, err := bin.ReadVarUint64(src) +// if err != nil { +// return +// } +// for range indexQty { +// var pageNo uint32 +// pageNo, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// rec.FreeIndexPages = append(rec.FreeIndexPages, pageNo) +// } +// return rec, nil +// } -func (s *Reader) readAppendedMeasure(src *bytes.Buffer) (_ AppendedMeasure, err error) { - arr, err := bin.ReadN(src, 16) - if err != nil { - return - } - return AppendedMeasure{ - MetricID: bin.GetUint32(arr[0:]), - Timestamp: bin.GetUint32(arr[4:]), - Value: bin.GetFloat64(arr[8:]), - }, nil -} +// func (s *Reader) readAppendedMeasure(src *bytes.Buffer) (_ AppendedMeasure, err error) { +// arr, err := bin.ReadN(src, 16) +// if err != nil { +// return +// } +// return AppendedMeasure{ +// MetricID: bin.GetUint32(arr[0:]), +// Timestamp: bin.GetUint32(arr[4:]), +// Value: bin.GetFloat64(arr[8:]), +// }, nil +// } -func (s *Reader) readAppendedMeasures(src *bytes.Buffer) (_ AppendedMeasures, err error) { - var rec AppendedMeasures - rec.MetricID, err = bin.ReadUint32(src) - if err != nil { - return - } - qty, err := bin.ReadUint16(src) - if err != nil { - return - } - for range qty { - var measure proto.Measure - measure.Timestamp, err = bin.ReadUint32(src) - if err != nil { - return - } - measure.Value, err = bin.ReadFloat64(src) - if err != nil { - return - } - rec.Measures = append(rec.Measures, measure) - } - return rec, nil -} +// func (s *Reader) readAppendedMeasures(src *bytes.Buffer) (_ AppendedMeasures, err error) { +// var rec AppendedMeasures +// rec.MetricID, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// qty, err := bin.ReadUint16(src) +// if err != nil { +// return +// } +// for range qty { +// var measure proto.Measure +// measure.Timestamp, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// measure.Value, err = bin.ReadFloat64(src) +// if err != nil { +// return +// } +// rec.Measures = append(rec.Measures, measure) +// } +// return rec, nil +// } -func (s *Reader) readAppendedMeasureWithOverflow(src *bytes.Buffer) (_ AppendedMeasureWithOverflow, err error) { - var ( - b byte - rec AppendedMeasureWithOverflow - ) - rec.MetricID, err = bin.ReadUint32(src) - if err != nil { - return - } - rec.Timestamp, err = bin.ReadUint32(src) - if err != nil { - return - } - rec.Value, err = bin.ReadFloat64(src) - if err != nil { - return - } - b, err = src.ReadByte() - if err != nil { - return - } - rec.IsDataPageReused = b == 1 +// func (s *Reader) readAppendedMeasureWithOverflow(src *bytes.Buffer) (_ AppendedMeasureWithOverflow, err error) { +// var ( +// b byte +// rec AppendedMeasureWithOverflow +// ) +// rec.MetricID, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// rec.Timestamp, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// rec.Value, err = bin.ReadFloat64(src) +// if err != nil { +// return +// } +// b, err = src.ReadByte() +// if err != nil { +// return +// } +// rec.IsDataPageReused = b == 1 - rec.DataPageNo, err = bin.ReadUint32(src) - if err != nil { - return - } - b, err = src.ReadByte() - if err != nil { - return - } - if b == 1 { - rec.IsRootChanged = true - rec.RootPageNo, err = bin.ReadUint32(src) - if err != nil { - return - } - } - // index pages - indexQty, err := src.ReadByte() - if err != nil { - return - } - for range indexQty { - var pageNo uint32 - pageNo, err = bin.ReadUint32(src) - if err != nil { - return - } - rec.ReusedIndexPages = append(rec.ReusedIndexPages, pageNo) - } - return rec, nil -} +// rec.DataPageNo, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// b, err = src.ReadByte() +// if err != nil { +// return +// } +// if b == 1 { +// rec.IsRootChanged = true +// rec.RootPageNo, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// } +// // index pages +// indexQty, err := src.ReadByte() +// if err != nil { +// return +// } +// for range indexQty { +// var pageNo uint32 +// pageNo, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// rec.ReusedIndexPages = append(rec.ReusedIndexPages, pageNo) +// } +// return rec, nil +// } -func (s *Reader) readDeletedMeasures(src *bytes.Buffer) (_ DeletedMeasures, err error) { - var ( - rec DeletedMeasures - ) - rec.MetricID, err = bin.ReadUint32(src) - if err != nil { - return - } - // free data pages - rec.FreeDataPages, err = s.readFreePageNumbers(src) - if err != nil { - return - } - // free index pages - rec.FreeIndexPages, err = s.readFreePageNumbers(src) - if err != nil { - return - } - return rec, nil -} +// func (s *Reader) readDeletedMeasures(src *bytes.Buffer) (_ DeletedMeasures, err error) { +// var ( +// rec DeletedMeasures +// ) +// rec.MetricID, err = bin.ReadUint32(src) +// if err != nil { +// return +// } +// // free data pages +// rec.FreeDataPages, err = s.readFreePageNumbers(src) +// if err != nil { +// return +// } +// // free index pages +// rec.FreeIndexPages, err = s.readFreePageNumbers(src) +// if err != nil { +// return +// } +// return rec, nil +// } // HELPERS -func (s *Reader) readFreePageNumbers(src *bytes.Buffer) ([]uint32, error) { - var freePages []uint32 - qty, _, err := bin.ReadVarUint64(src) - if err != nil { - return nil, err - } - for range qty { - var pageNo uint32 - pageNo, err = bin.ReadUint32(src) - if err != nil { - return nil, err - } - freePages = append(freePages, pageNo) - } - return freePages, nil -} +// func (s *Reader) readFreePageNumbers(src *bytes.Buffer) ([]uint32, error) { +// var freePages []uint32 +// qty, _, err := bin.ReadVarUint64(src) +// if err != nil { +// return nil, err +// } +// for range qty { +// var pageNo uint32 +// pageNo, err = bin.ReadUint32(src) +// if err != nil { +// return nil, err +// } +// freePages = append(freePages, pageNo) +// } +// return freePages, nil +// } func (s *Reader) Seek(offset int64) error { ret, err := s.file.Seek(offset, 0) diff --git a/txlog/txlog.go b/txlog/txlog.go new file mode 100644 index 0000000..d08ce1b --- /dev/null +++ b/txlog/txlog.go @@ -0,0 +1,359 @@ +package txlog + +import ( + "io" + + bin "gordenko.dev/dima/bin/little" + "gordenko.dev/dima/qb" + "gordenko.dev/dima/qb/atree" + "gordenko.dev/dima/qb/conbuf" + "gordenko.dev/dima/qb/proto" +) + +type AddedMetric struct { + MetricID uint32 + MetricType qb.MetricType + FracDigits int +} + +func (s AddedMetric) Pack(w io.Writer) { + arr := []byte{ + CodeAddedMetric, + 0, 0, 0, 0, // + byte(s.MetricType), + byte(s.FracDigits), + } + bin.PutUint32(arr[1:], s.MetricID) + w.Write(arr) +} + +func (s *AddedMetric) Parse(src io.Reader) (err error) { + arr, err := bin.ReadN(src, 6) + if err != nil { + return + } + s.MetricID, _ = bin.GetUint32(arr) + s.MetricType = qb.MetricType(arr[4]) + s.FracDigits = int(arr[5]) + return nil +} + +type DeletedMetric struct { + MetricID uint32 + FreePageNumbers []uint32 +} + +func (s DeletedMetric) Pack(w io.Writer) { + arr := []byte{ + CodeDeletedMetric, + 0, 0, 0, 0, // metricID + } + bin.PutUint32(arr[1:], s.MetricID) + w.Write(arr) + bin.WriteVarSize(w, len(s.FreePageNumbers)) + for _, pageNo := range s.FreePageNumbers { + bin.WriteUint32(w, pageNo) + } +} + +func (s *DeletedMetric) Parse(src io.Reader) (err error) { + s.MetricID, err = bin.ReadUint32(src) + if err != nil { + return + } + // free pages + qty, err := bin.ReadVarSize(src) + if err != nil { + return + } + for range qty { + var pageNo uint32 + pageNo, err = bin.ReadUint32(src) + if err != nil { + return + } + s.FreePageNumbers = append(s.FreePageNumbers, pageNo) + } + return nil +} + +type AppendedMeasure struct { + MetricID uint32 + Timestamp uint32 + Value float64 +} + +func (s AppendedMeasure) Pack(w io.Writer) { + arr := []byte{ + CodeAppendedMeasure, + 0, 0, 0, 0, // metricID + 0, 0, 0, 0, // timestamp + 0, 0, 0, 0, 0, 0, 0, 0, // value + } + bin.PutUint32(arr[1:], s.MetricID) + bin.PutUint32(arr[5:], s.Timestamp) + bin.PutFloat64(arr[9:], s.Value) + w.Write(arr) +} + +func (s *AppendedMeasure) Parse(src io.Reader) (err error) { + arr, err := bin.ReadN(src, 16) + if err != nil { + return + } + s.MetricID, _ = bin.GetUint32(arr[0:]) + s.Timestamp, _ = bin.GetUint32(arr[4:]) + s.Value, _ = bin.GetFloat64(arr[8:]) + return nil +} + +type AppendedMeasures struct { + MetricID uint32 + Measures []proto.Measure +} + +func (s AppendedMeasures) Pack(w io.Writer) { + arr := []byte{ + CodeAppendedMeasures, + 0, 0, 0, 0, // metricID + 0, 0, // qty + } + bin.PutUint32(arr[1:], s.MetricID) + bin.PutUint16(arr[5:], uint16(len(s.Measures))) + w.Write(arr) + for _, measure := range s.Measures { + bin.WriteUint32(w, measure.Timestamp) + bin.WriteFloat64(w, measure.Value) + } +} + +func (s *AppendedMeasures) Parse(src io.Reader) (err error) { + s.MetricID, err = bin.ReadUint32(src) + if err != nil { + return + } + qty, err := bin.ReadUint16(src) + if err != nil { + return + } + for range qty { + var measure proto.Measure + measure.Timestamp, err = bin.ReadUint32(src) + if err != nil { + return + } + measure.Value, err = bin.ReadFloat64(src) + if err != nil { + return + } + s.Measures = append(s.Measures, measure) + } + return nil +} + +type AppendedPagesReq struct { + Legs []atree.PathLeg + MetricID uint32 + Timestamp uint32 // last measure + Value float64 // last measure + //RootPageNo uint32 + LastPageNo uint32 + Pages []atree.NotLinkedDataPage + TimestampsChunks [][]byte + TimestampsSize uint16 + ValuesChunks [][]byte + ValuesSize uint16 +} + +/* +4b metricID +4b timestamp +8b value +4b LastPageNo (0 if no new root page) +4b newRootPageNo (0 if no new root page) +Nb - varsize (pages qty) +[ + + 4b pageNo + 1b isReused + Xb page (page size) + +] +2b - timestamps size (not filled page) +Nb - timestamps payload +2b - values size (not filled page) +Nb - values payload +*/ +type AppendedPages struct { + MetricID uint32 + Timestamp uint32 // last measure + Value float64 // last measure + NewRootPageNo uint32 + LastPageNo uint32 + Pages []atree.PageToWrite + TimestampsChunks [][]byte + TimestampsSize int + ValuesChunks [][]byte + ValuesSize int +} + +func (s *AppendedPages) Pack(w io.Writer) { + w.Write([]byte{ + CodeAppendedPages, + }) + bin.WriteUint32(w, s.MetricID) + bin.WriteUint32(w, s.Timestamp) + bin.WriteFloat64(w, s.Value) + bin.WriteUint32(w, s.LastPageNo) + bin.WriteUint32(w, s.NewRootPageNo) + // + bin.WriteVarSize(w, len(s.Pages)) + for _, p := range s.Pages { + bin.WriteUint32(w, p.PageNo) + if p.IsReused { + w.Write([]byte{1}) + } else { + w.Write([]byte{0}) + } + w.Write(p.Data) + } + // timestamps + writeChunkedPayloadTo(s.TimestampsChunks, int(s.TimestampsSize), w) + // values + writeChunkedPayloadTo(s.ValuesChunks, int(s.ValuesSize), w) +} + +func (s *AppendedPages) Parse(src io.Reader) (err error) { + s.MetricID, err = bin.ReadUint32(src) + if err != nil { + return + } + s.Timestamp, err = bin.ReadUint32(src) + if err != nil { + return + } + s.Value, err = bin.ReadFloat64(src) + if err != nil { + return + } + s.LastPageNo, err = bin.ReadUint32(src) + if err != nil { + return + } + s.NewRootPageNo, err = bin.ReadUint32(src) + if err != nil { + return + } + pagesQty, err := bin.ReadVarSize(src) + if err != nil { + return + } + for range pagesQty { + var page atree.PageToWrite + page.PageNo, err = bin.ReadUint32(src) + if err != nil { + return + } + page.IsReused, err = bin.ReadBool(src) + if err != nil { + return + } + page.Data, err = bin.ReadN(src, atree.PageSize) + if err != nil { + return + } + s.Pages = append(s.Pages, page) + } + s.TimestampsChunks, s.TimestampsSize, err = readChunkedPayload(src) + if err != nil { + return + } + s.ValuesChunks, s.ValuesSize, err = readChunkedPayload(src) + if err != nil { + return + } + return nil +} + +func writeChunkedPayloadTo(chunks [][]byte, size int, w io.Writer) { + bin.WriteVarSize(w, size) + for _, chunk := range chunks { + if size >= len(chunk) { + w.Write(chunk) + size -= len(chunk) + } else { + w.Write(chunk[:size]) + return + } + } +} + +func readChunkedPayload(src io.Reader) (_ [][]byte, _ int, err error) { + var ( + chunks [][]byte + size int + remainingSize = size + ) + size, err = bin.ReadVarSize(src) + if err != nil { + return + } + for remainingSize > 0 { + var chunk []byte + if remainingSize >= conbuf.ChunkSize { + chunk, err = bin.ReadN(src, conbuf.ChunkSize) + if err != nil { + return + } + } else { + chunk = make([]byte, conbuf.ChunkSize) + err = bin.ReadNInto(src, chunk[:remainingSize]) + if err != nil { + return + } + } + chunks = append(chunks, chunk) + remainingSize -= conbuf.ChunkSize + } + return chunks, size, nil +} + +// fix - add changed pages +type DeletedMeasures struct { + MetricID uint32 + FreePageNumbers []uint32 +} + +func (s DeletedMeasures) Pack(w io.Writer) { + arr := []byte{ + CodeDeletedMeasures, + 0, 0, 0, 0, + } + bin.PutUint32(arr[1:], s.MetricID) + w.Write(arr) + bin.WriteVarSize(w, len(s.FreePageNumbers)) + for _, pageNo := range s.FreePageNumbers { + bin.WriteUint32(w, pageNo) + } +} + +func (s *DeletedMeasures) Parse(src io.Reader) (err error) { + s.MetricID, err = bin.ReadUint32(src) + if err != nil { + return + } + // free pages + qty, err := bin.ReadVarSize(src) + if err != nil { + return + } + for range qty { + var pageNo uint32 + pageNo, err = bin.ReadUint32(src) + if err != nil { + return + } + s.FreePageNumbers = append(s.FreePageNumbers, pageNo) + } + return nil +} diff --git a/txlog/writer.go b/txlog/writer.go index d5cc646..2f30d3d 100644 --- a/txlog/writer.go +++ b/txlog/writer.go @@ -9,9 +9,9 @@ import ( "path/filepath" "sync" + bin "gordenko.dev/dima/bin/little" octopus "gordenko.dev/dima/qb" - "gordenko.dev/dima/qb/bin" - "gordenko.dev/dima/qb/proto" + "gordenko.dev/dima/qb/atree" ) const ( @@ -28,14 +28,19 @@ const ( ) const ( - CodeAddedMetric byte = 1 - CodeDeletedMetric byte = 2 - CodeAppendedMeasure byte = 4 - CodeAppendedMeasures byte = 5 - CodeAppendedMeasureWithOverflow byte = 6 - CodeDeletedMeasures byte = 7 + CodeAddedMetric byte = 1 + CodeDeletedMetric byte = 2 + CodeAppendedMeasure byte = 4 + CodeAppendedMeasures byte = 5 + CodeAppendedPages byte = 6 + CodeDeletedMeasures byte = 7 ) +type FreeList interface { + // використовується в allocPage + ReservePage() uint32 +} + func JoinChangesFileName(dir string, logNumber int) string { return filepath.Join(dir, fmt.Sprintf("%d.changes", logNumber)) } @@ -49,11 +54,16 @@ type Changes struct { } type Writer struct { - mutex sync.Mutex - logNumber int - dir string - file *os.File - buf *bytes.Buffer + mutex sync.Mutex + freelist FreeList + atree *atree.Atree + logNumber int + dir string + dataFile *os.File + file *os.File + input []any + //buf *bytes.Buffer + pagesToWrite []atree.PageToWrite workerReqs []any waitCh chan struct{} appendToWorkerQueue func(any) @@ -69,6 +79,8 @@ type WriterOptions struct { Dir string LogNumber int // номер журнала AppendToWorkerQueue func(any) + FreeList FreeList + Atree *atree.Atree ExitCh chan struct{} WaitGroup *sync.WaitGroup } @@ -80,6 +92,12 @@ func NewWriter(opt WriterOptions) (*Writer, error) { if opt.AppendToWorkerQueue == nil { return nil, errors.New("AppendToWorkerQueue option is required") } + if opt.FreeList == nil { + return nil, errors.New("FreeList option is required") + } + if opt.Atree == nil { + return nil, errors.New("Atree option is required") + } if opt.ExitCh == nil { return nil, errors.New("ExitCh option is required") } @@ -89,12 +107,14 @@ func NewWriter(opt WriterOptions) (*Writer, error) { s := &Writer{ dir: opt.Dir, - buf: bytes.NewBuffer(nil), appendToWorkerQueue: opt.AppendToWorkerQueue, + freelist: opt.FreeList, + atree: opt.Atree, logNumber: opt.LogNumber, exitCh: opt.ExitCh, waitGroup: opt.WaitGroup, signalCh: make(chan struct{}, 1), + waitCh: make(chan struct{}), } var err error @@ -119,8 +139,6 @@ func NewWriter(opt WriterOptions) (*Writer, error) { return nil, err } } - - s.reset() return s, nil } @@ -128,7 +146,7 @@ func (s *Writer) Run() { for { select { case <-s.signalCh: - if err := s.flush(); err != nil { + if err := s.packAndWrite(); err != nil { octopus.Abort(octopus.FailedWriteToTxLog, err) } @@ -139,21 +157,67 @@ func (s *Writer) Run() { } } -func (s *Writer) reset() { - s.buf.Reset() - s.buf.Write([]byte{ - 0, 0, 0, 0, // packet length - 0, 0, 0, 0, // crc32 - 0, 0, 0, 0, // lsn - - }) - s.workerReqs = nil +func (s *Writer) packAndWrite() error { + s.mutex.Lock() + input := s.input + waitCh := s.waitCh + s.input = nil s.waitCh = make(chan struct{}) + s.mutex.Unlock() + + w := bytes.NewBuffer(nil) + + for _, untyped := range input { + switch x := untyped.(type) { + case AppendedPagesReq: + s.packWriteAppended(w, x) + case AddedMetric: + case DeletedMetric: + if len(x.FreePageNumbers) > 0 { + //s.freeList.AddPages(x.FreePageNumbers) + } + case AppendedMeasure: + case AppendedMeasures: + case DeletedMeasures: + //case DeletedMeasuresSince: + } + } + // flush + close(waitCh) + return s.flush(w) } -func (s *Writer) flush() error { +func (s *Writer) Append(req any) chan struct{} { + s.mutex.Lock() + s.input = append(s.input, req) + s.workerReqs = append(s.workerReqs, req) + waitCh := s.waitCh + s.mutex.Unlock() + + select { + case s.signalCh <- struct{}{}: + default: + } + return waitCh +} + +// func (s *Writer) reset() { +// s.buf.Reset() +// s.buf.Write([]byte{ +// 0, 0, 0, 0, // packet length +// 0, 0, 0, 0, // crc32 +// 0, 0, 0, 0, // lsn + +// }) +// s.pagesToWrite = nil +// s.workerReqs = nil +// s.waitCh = make(chan struct{}) +// } + +func (s *Writer) flush(w *bytes.Buffer) error { s.mutex.Lock() + pagesToWrite := s.pagesToWrite workerReqs := s.workerReqs waitCh := s.waitCh isExited := s.isExited @@ -163,12 +227,12 @@ func (s *Writer) flush() error { exitWaitGroup = s.waitGroup } - if s.buf.Len() > packetPrefixSize { + if w.Len() > packetPrefixSize { s.lsn++ lsn := s.lsn - packet := make([]byte, s.buf.Len()) - copy(packet, s.buf.Bytes()) - s.reset() + packet := make([]byte, w.Len()) + copy(packet, w.Bytes()) + //s.reset() fix s.written += int64(len(packet)) + 12 s.mutex.Unlock() @@ -181,14 +245,17 @@ func (s *Writer) flush() error { if err != nil { return fmt.Errorf("TxLog write: %s", err) } - if n != len(packet) { return fmt.Errorf("TxLog written %d != packet size %d", n, len(packet)) } - if err := s.file.Sync(); err != nil { return fmt.Errorf("TxLog sync: %s", err) } + + err = s.writePagesToAtree(pagesToWrite) + if err != nil { + return fmt.Errorf("TxLog writePagesToAtree: %s", err) + } } else { s.waitCh = make(chan struct{}) s.mutex.Unlock() @@ -232,263 +299,48 @@ func (s *Writer) flush() error { return nil } +func (s *Writer) writePagesToAtree(pages []atree.PageToWrite) error { + for _, p := range pages { + if len(p.Data) != atree.PageSize { + return fmt.Errorf("wrong page %d size: %d", + p.PageNo, len(p.Data)) + } + + off := (p.PageNo - 1) * atree.PageSize + n, err := s.dataFile.WriteAt(p.Data, int64(off)) + if err != nil { + return err + } + if n != len(p.Data) { + return fmt.Errorf("write %d instead of %d", n, len(p.Data)) + } + } + return nil +} + func (s *Writer) exit() { s.mutex.Lock() s.isExited = true s.mutex.Unlock() - if err := s.flush(); err != nil { + if err := s.packAndWrite(); err != nil { octopus.Abort(octopus.FailedWriteToTxLog, err) } } // API -type AddedMetric struct { - MetricID uint32 - MetricType octopus.MetricType - FracDigits int -} - -func (s *Writer) WriteAddedMetric(req AddedMetric) chan struct{} { - arr := []byte{ - CodeAddedMetric, - 0, 0, 0, 0, // - byte(req.MetricType), - byte(req.FracDigits), - } - bin.PutUint32(arr[1:], req.MetricID) - // пишу в буфер - s.mutex.Lock() - s.buf.Write(arr) - s.workerReqs = append(s.workerReqs, req) - s.mutex.Unlock() - - s.sendSignal() - return s.waitCh -} - -type DeletedMetric struct { - MetricID uint32 - FreeDataPages []uint32 - FreeIndexPages []uint32 -} - -func (s *Writer) WriteDeletedMetric(req DeletedMetric) chan struct{} { - arr := []byte{ - CodeDeletedMetric, - 0, 0, 0, 0, // metricID - } - bin.PutUint32(arr[1:], req.MetricID) - - // пишу в буфер - s.mutex.Lock() - defer s.mutex.Unlock() - - s.buf.Write(arr) - s.packFreeDataAndIndexPages(req.FreeDataPages, req.FreeIndexPages) - s.workerReqs = append(s.workerReqs, req) - - s.sendSignal() - return s.waitCh -} - -type AppendedMeasure struct { - MetricID uint32 - Timestamp uint32 - Value float64 -} - -func (s *Writer) WriteAppendMeasure(req AppendedMeasure) chan struct{} { - arr := []byte{ - CodeAppendedMeasure, - 0, 0, 0, 0, // metricID - 0, 0, 0, 0, // timestamp - 0, 0, 0, 0, 0, 0, 0, 0, // value - } - bin.PutUint32(arr[1:], req.MetricID) - bin.PutUint32(arr[5:], req.Timestamp) - bin.PutFloat64(arr[9:], req.Value) - // - s.mutex.Lock() - s.buf.Write(arr) - s.workerReqs = append(s.workerReqs, req) - s.mutex.Unlock() - - s.sendSignal() - return s.waitCh -} - -type AppendedMeasures struct { - MetricID uint32 - Measures []proto.Measure -} - -type AppendedMeasuresExtended struct { - Record AppendedMeasures - HoldLock bool -} - -func (s *Writer) WriteAppendMeasures(req AppendedMeasures, holdLock bool) chan struct{} { - arr := []byte{ - CodeAppendedMeasures, - 0, 0, 0, 0, // metricID - 0, 0, // qty - } - bin.PutUint32(arr[1:], req.MetricID) - bin.PutUint16(arr[5:], uint16(len(req.Measures))) - // - s.mutex.Lock() - s.buf.Write(arr) - for _, measure := range req.Measures { - bin.WriteUint32(s.buf, measure.Timestamp) - bin.WriteFloat64(s.buf, measure.Value) - } - s.workerReqs = append(s.workerReqs, AppendedMeasuresExtended{ - Record: req, - HoldLock: holdLock, - }) - s.mutex.Unlock() - - s.sendSignal() - return s.waitCh -} - -type AppendedMeasureWithOverflow struct { - MetricID uint32 - Timestamp uint32 - Value float64 - IsDataPageReused bool - DataPageNo uint32 - IsRootChanged bool - RootPageNo uint32 - ReusedIndexPages []uint32 -} - -type AppendedMeasureWithOverflowExtended struct { - Record AppendedMeasureWithOverflow - HoldLock bool -} - -/* -Формат: -1b code -4b metricID -4b timestamp -8b value -1b isReusedDataPage -4b dataPageNo -1b isRootChanged -[4b] newRootPageNo -1b reusedIndexPages length -[N * 4b] reusedIndexPages ------------------- - -4b metricID -4b timestamp -8b value -4b newRootPageNo (0 if no new root page) - -Nb - varsize (N pages) -[ - - 4b pageNo - 1b isReused - Xb page (page size) - -] -Nb - varsize (last data page size) -Nb - payload -*/ -func (s *Writer) WriteAppendedMeasureWithOverflow(req AppendedMeasureWithOverflow, holdLock bool) chan struct{} { - size := 24 + len(req.ReusedIndexPages)*4 - if req.IsRootChanged { - size += 4 - } - - tmp := make([]byte, size) - - tmp[0] = CodeAppendedMeasureWithOverflow - bin.PutUint32(tmp[1:], req.MetricID) - bin.PutUint32(tmp[5:], req.Timestamp) - bin.PutFloat64(tmp[9:], req.Value) - if req.IsDataPageReused { - tmp[17] = 1 - } - bin.PutUint32(tmp[18:], req.DataPageNo) - - pos := 22 - if req.IsRootChanged { - tmp[pos] = 1 - bin.PutUint32(tmp[pos+1:], req.RootPageNo) - pos += 5 - } else { - tmp[pos] = 0 - pos += 1 - } - - tmp[pos] = byte(len(req.ReusedIndexPages)) - pos += 1 - for _, indexPageNo := range req.ReusedIndexPages { - bin.PutUint32(tmp[pos:], indexPageNo) - pos += 4 - } - - s.mutex.Lock() - s.buf.Write(tmp) - s.workerReqs = append(s.workerReqs, AppendedMeasureWithOverflowExtended{ - Record: req, - HoldLock: holdLock, - }) - s.mutex.Unlock() - - s.sendSignal() - return s.waitCh -} - -type DeletedMeasures struct { - MetricID uint32 - FreeDataPages []uint32 - FreeIndexPages []uint32 -} - -/* -Формат: -1b code -4b metricID -1b freeDataPages length -[N * 4b] freeDataPages -1b freeIndexPages length -[N * 4b] freeIndexPages -*/ -func (s *Writer) WriteDeletedMeasures(op DeletedMeasures) chan struct{} { - tmp := []byte{ - CodeDeletedMeasures, - 0, 0, 0, 0, - } - bin.PutUint32(tmp[1:], op.MetricID) - - // записываю часть фиксированного размера - s.mutex.Lock() - s.buf.Write(tmp) - s.packFreeDataAndIndexPages(op.FreeDataPages, op.FreeIndexPages) - s.workerReqs = append(s.workerReqs, op) - s.mutex.Unlock() - - s.sendSignal() - return s.waitCh -} - -type DeletedMeasuresSince struct { - MetricID uint32 - LastPageNo uint32 - IsRootChanged bool - RootPageNo uint32 - FreeDataPages []uint32 - FreeIndexPages []uint32 - TimestampsBuf []byte - ValuesBuf []byte -} +// FIX - add +// type DeletedMeasuresSince struct { +// MetricID uint32 +// LastPageNo uint32 +// IsRootChanged bool +// RootPageNo uint32 +// FreeDataPages []uint32 +// FreeIndexPages []uint32 +// TimestampsBuf []byte +// ValuesBuf []byte +// } func (s *Writer) sendSignal() { select { @@ -497,17 +349,30 @@ func (s *Writer) sendSignal() { } } -// helper +// Якщо (Pages) > 0 - незаповнену сторінку кодую в стисненому вигляді. +// Якщо (Pages) = 0 - кодую просто пари timestamp / value +func (s *Writer) packWriteAppended(w *bytes.Buffer, req AppendedPagesReq) { + // завантажений path. Отже просто додаємо в дерево data сторінки, створюємо нові індексні + // без звернення до диску. + report := s.atree.AppendDataPages(atree.AppendDataPagesReq{ + LastPageNo: req.LastPageNo, + Legs: req.Legs, + DataPages: req.Pages, + }) -func (s *Writer) packFreeDataAndIndexPages(freeDataPages, freeIndexPages []uint32) { - // записываю data pages - bin.WriteVarUint64(s.buf, uint64(len(freeDataPages))) - for _, dataPageNo := range freeDataPages { - bin.WriteUint32(s.buf, dataPageNo) - } - // записываю index pages - bin.WriteVarUint64(s.buf, uint64(len(freeIndexPages))) - for _, indexPageNo := range freeIndexPages { - bin.WriteUint32(s.buf, indexPageNo) + m := AppendedPages{ + MetricID: req.MetricID, + Timestamp: req.Timestamp, + Value: req.Value, + NewRootPageNo: report.NewRootPageNo, + LastPageNo: report.LastPageNo, + Pages: report.Pages, + TimestampsChunks: req.TimestampsChunks, + TimestampsSize: int(req.TimestampsSize), + ValuesChunks: req.ValuesChunks, + ValuesSize: int(req.ValuesSize), } + m.Pack(w) } + +// helpers