diff --git a/README.md b/README.md index e69de29..fb142fc 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,132 @@ +1. Zip-архів із базою даних на 841 млн показань (791 МБ) можна скачати за посиланням https://drive.google.com/file/d/18oks6STkVpg4-TT2WyCIBBpCgRyF29L2 + +В архіві 5 файлів: +2.changes +2.snapshot +test.data +test.index +metrics.info + +Після розпаковки їх необхідно перемістити в директорію testdir. + + +2. Якщо є бажання експериментувати з пустою базою даних, краще створити нову. Наприклад, назвемо її "x": створюємо директорію xdir, створюємо файл x.ini із такими налаштуваннями: +tcpPort = 12345 +dir = xdir +redoDir = xdir +databaseName = x + +Запускаємо СУБД із термінала: +./database_linux -c x.ini + +Всі операції виконуємо в кореневій директорії проєкту. + +3. Файли *.ini: +database.ini - налаштування СУБД (database); +loadtest.ini - налаштування навантажувального тесту (loadtest); +requests.ini - налаштування прикладів запитів (requests). + + +4. Скомпільоване ПО для Linux (64-бітна архітектура): +database_linux - СУБД; +loadtest_linux - навантажувального тест; +requests_linux - приклади запитів. + + +5. Скомпільоване ПО для Windows (64-бітна архітектура): +database_windows - СУБД; +loadtest_windows - навантажувального тест; +requests_windows - приклади запитів. + + +6. Директорія examples має три вкладені директорії із вихідними кодами: +database - запуску СУБД; +loadtest - навантажувального тесту; +requests - різних типів запитів, що підтримуються СУБД. + + +7. Якщо на комп'ютері встановлено компілятор Go, можна скомпілювати згадані вище програми за допомогою bash-скриптів: + +./linux_build.sh +./windows_build.sh + +Скомпільовані версії програм опиняться в кореневій директорії проєкту. + + +8. Файли з даними розміщуються в директорії datadir. В базі даних вже записано 841 млн. показань. Сумарний розмір трохи більше 1.3 GB. + + +9. Налаштування навантажувального тесту можна змінити, відредагувавши файл loadtest.ini. +Опція connections - це кількість одночасно відкритих підключень до СУБД. +Опція requestsPerConn - це кількість запитів, які відправляє потік через одне відкрите підключення. + +Звіт виглядає наступним чином: + +TEST RESULTS: +Time: 2 seconds +Connections: 100 +Requests per conn: 500 +Total requests: 50000 +AVG request time: 3.121022ms +RPS: 26891 + +listCumulativeMeasures: 3099 (6.2%), AVG request time: 3.785916ms +listCumulativePeriods: 12055 (24.1%), AVG request time: 2.726391ms +listInstantMeasures: 1974 (3.9%), AVG request time: 6.726605ms +listInstantPeriods: 7710 (15.4%), AVG request time: 2.9808ms +listCurrentValues: 25162 (50.3%), AVG request time: 2.988301ms + +last day: 20382 (82.1%), AVG request time: 2.954718ms +last week: 2993 (12.1%), AVG request time: 4.050662ms +last month: 708 (2.9%), AVG request time: 8.248486ms +random time range: 755 (3.0%), AVG request time: 3.540239ms + +Навантажувальний тест відправляє випадкові запити до випадкових метрик. Оскільки на реальному проєкті запити та часові діапазони мають різну ймовірність - я реалізував це в навантажувальному тесті. + +Приклад 1: +listCurrentValues: 25162 (50.3%) , AVG request time: 2.988301ms +означає що відправлено 25162 запитів listCurrentValues, що склало 50.3% від загальної кількості запитів. Середній час виконання (Latency) запитів listCurrentValues склав 2.99 міллісекунди. + +Приклад 2: +last month: 708 (2.9%), AVG request time: 8.248486ms +означає що відправлено 708 запитів на отримання даних за останній місяць, що склало 2.9% від загальної кількості. Середній час виконання (Latency) таких запитів склав 8.25 міллісекунди. Часовий діапазон задається для всіх запитів окрім listCurrentValues. + +10. Запуск навантажувального тесту: +Виконуємо пункт 1. + +Запускаємо СУБД із термінала: +./database_linux + +Запускаємо тест із іншого термінала: +./loadtest_linux + +Чекаємо завершення. Звіт буде надруковано у терміналі після завершення тесту. + +Команди для Windows: +./database_windows +./loadtest_windows + +11. Запуск прикладів запитів: +Можна запускати на пустій базі даних (без скачування Zip-архіва із показаннями). + +Запускаємо СУБД із термінала: +./database_linux + +Запускаємо тест із іншого термінала: +./requests_linux + +Результат виконання запитів друкується у терміналі. + +Команди для Windows: +./database_windows +./requests_windows + + +12. Подивитись HELP: +./database_linux -h +./loadtest_linux -h +./requests_linux -h + + +13. Зупинити СУБД, перервати навантажувальний тест: +Ctrl + C diff --git a/atree/aggregate.go b/atree/aggregate.go new file mode 100644 index 0000000..c4081ab --- /dev/null +++ b/atree/aggregate.go @@ -0,0 +1,325 @@ +package atree + +import ( + "fmt" + "time" + + "gordenko.dev/dima/diploma" + "gordenko.dev/dima/diploma/timeutil" +) + +// AGGREGATE + +type InstantAggregator struct { + firstHourOfDay int + lastDayOfMonth int + time2period func(uint32) uint32 + currentPeriod uint32 + since uint32 + until uint32 + min float64 + max float64 + total float64 + entries int +} + +type InstantAggregatorOptions struct { + GroupBy diploma.GroupBy + FirstHourOfDay int + LastDayOfMonth int +} + +func NewInstantAggregator(opt InstantAggregatorOptions) (*InstantAggregator, error) { + s := &InstantAggregator{ + firstHourOfDay: opt.FirstHourOfDay, + lastDayOfMonth: opt.LastDayOfMonth, + } + + switch opt.GroupBy { + case diploma.GroupByHour: + s.time2period = groupByHour + + case diploma.GroupByDay: + if s.firstHourOfDay > 0 { + s.time2period = s.groupByDayUsingFHD + } else { + s.time2period = groupByDay + } + + case diploma.GroupByMonth: + if s.firstHourOfDay > 0 { + if s.lastDayOfMonth > 0 { + s.time2period = s.groupByMonthUsingFHDAndLDM + } else { + s.time2period = s.groupByMonthUsingFHD + } + } else { + if s.lastDayOfMonth > 0 { + s.time2period = s.groupByMonthUsingLDM + } else { + s.time2period = groupByMonth + } + } + + default: + return nil, fmt.Errorf("unknown groupBy %d option", opt.GroupBy) + } + + return s, nil +} + +// Приходят данные от свежих к старым, тоесть сперва получаю Until. +// return period complete flag +func (s *InstantAggregator) Feed(timestamp uint32, value float64, p *InstantPeriod) bool { + period := s.time2period(timestamp) + //fmt.Printf("feed: %s %v, period: %s\n", time.Unix(int64(timestamp), 0), value, time.Unix(int64(period), 0)) + if s.entries == 0 { + s.currentPeriod = period + s.since = timestamp + s.until = timestamp + s.min = value + s.max = value + s.total = value + s.entries = 1 + return false + } + + if period != s.currentPeriod { + // готовый период + s.FillPeriod(timestamp, p) + s.currentPeriod = period + s.since = timestamp + s.until = timestamp + s.min = value + s.max = value + s.total = value + s.entries = 1 + return true + } + + if value < s.min { + s.min = value + } else if value > s.max { + s.max = value + } + // для подсчета AVG + s.total += value + s.entries++ + // начало периода + s.since = timestamp + return false +} + +func (s *InstantAggregator) FillPeriod(prevTimestamp uint32, p *InstantPeriod) bool { + if s.entries == 0 { + return false + } + + //fmt.Printf("FillPeriod: %s, prevTimestamp: %s\n", time.Unix(int64(s.currentPeriod), 0), time.Unix(int64(prevTimestamp), 0)) + p.Period = s.currentPeriod + if prevTimestamp > 0 { + p.Since = prevTimestamp + } else { + p.Since = s.since + } + p.Until = s.until + p.Min = s.min + p.Max = s.max + p.Avg = s.total / float64(s.entries) + return true +} + +func (s *InstantAggregator) groupByDayUsingFHD(timestamp uint32) uint32 { + tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "d") + if tm.Hour() < s.firstHourOfDay { + tm = tm.AddDate(0, 0, -1) + } + return uint32(tm.Unix()) +} + +func (s *InstantAggregator) groupByMonthUsingFHD(timestamp uint32) uint32 { + tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "m") + if tm.Hour() < s.firstHourOfDay { + tm = tm.AddDate(0, 0, -1) + } + return uint32(tm.Unix()) +} + +func (s *InstantAggregator) groupByMonthUsingLDM(timestamp uint32) uint32 { + tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "m") + if tm.Day() > s.lastDayOfMonth { + tm = tm.AddDate(0, 1, 0) + } + return uint32(tm.Unix()) +} + +func (s *InstantAggregator) groupByMonthUsingFHDAndLDM(timestamp uint32) uint32 { + // ВАЖНО! + // Сперва проверяю время. + tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "m") + if tm.Hour() < s.firstHourOfDay { + tm = tm.AddDate(0, 0, -1) + } + if tm.Day() > s.lastDayOfMonth { + tm = tm.AddDate(0, 1, 0) + } + return uint32(tm.Unix()) +} + +// CUMULATIVE + +type CumulativeAggregator struct { + firstHourOfDay int + lastDayOfMonth int + time2period func(uint32) uint32 + currentPeriod uint32 + since uint32 + until uint32 + sinceValue float64 + untilValue float64 + entries int +} + +type CumulativeAggregatorOptions struct { + GroupBy diploma.GroupBy + FirstHourOfDay int + LastDayOfMonth int +} + +func NewCumulativeAggregator(opt CumulativeAggregatorOptions) (*CumulativeAggregator, error) { + s := &CumulativeAggregator{ + firstHourOfDay: opt.FirstHourOfDay, + lastDayOfMonth: opt.LastDayOfMonth, + } + + switch opt.GroupBy { + case diploma.GroupByHour: + s.time2period = groupByHour + + case diploma.GroupByDay: + if s.firstHourOfDay > 0 { + s.time2period = s.groupByDayUsingFHD + } else { + s.time2period = groupByDay + } + + case diploma.GroupByMonth: + if s.firstHourOfDay > 0 { + if s.lastDayOfMonth > 0 { + s.time2period = s.groupByMonthUsingFHDAndLDM + } else { + s.time2period = s.groupByMonthUsingFHD + } + } else { + if s.lastDayOfMonth > 0 { + s.time2period = s.groupByMonthUsingLDM + } else { + s.time2period = groupByMonth + } + } + + default: + return nil, fmt.Errorf("unknown groupBy %d option", opt.GroupBy) + } + + return s, nil +} + +// return period complete flag +func (s *CumulativeAggregator) Feed(timestamp uint32, value float64, p *CumulativePeriod) bool { + period := s.time2period(timestamp) + if s.entries == 0 { + s.currentPeriod = period + s.since = timestamp + s.until = timestamp + s.sinceValue = value + s.untilValue = value + s.entries = 1 + return false + } + + if period != s.currentPeriod { + // готовый период + s.FillPeriod(timestamp, value, p) + s.currentPeriod = period + s.since = timestamp + s.until = timestamp + s.sinceValue = value + s.untilValue = value + s.entries = 1 + return true + } + + // начало периода + s.since = timestamp + s.sinceValue = value + s.entries++ + return false +} + +func (s *CumulativeAggregator) FillPeriod(prevTimestamp uint32, value float64, p *CumulativePeriod) bool { + if s.entries == 0 { + return false + } + p.Period = s.currentPeriod + if prevTimestamp > 0 { + p.Since = prevTimestamp + p.Total = s.untilValue - value + } else { + p.Since = s.since + p.Total = s.untilValue - s.sinceValue + } + p.Until = s.until + p.EndValue = s.untilValue + + return true +} + +func (s *CumulativeAggregator) groupByDayUsingFHD(timestamp uint32) uint32 { + tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "d") + if tm.Hour() < s.firstHourOfDay { + tm = tm.AddDate(0, 0, -1) + } + return uint32(tm.Unix()) +} + +func (s *CumulativeAggregator) groupByMonthUsingFHD(timestamp uint32) uint32 { + tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "m") + if tm.Hour() < s.firstHourOfDay { + tm = tm.AddDate(0, 0, -1) + } + return uint32(tm.Unix()) +} + +func (s *CumulativeAggregator) groupByMonthUsingLDM(timestamp uint32) uint32 { + tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "m") + if tm.Day() > s.lastDayOfMonth { + tm = tm.AddDate(0, 1, 0) + } + return uint32(tm.Unix()) +} + +func (s *CumulativeAggregator) groupByMonthUsingFHDAndLDM(timestamp uint32) uint32 { + // ВАЖНО! + // Сперва проверяю время. + tm := timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "m") + if tm.Hour() < s.firstHourOfDay { + tm = tm.AddDate(0, 0, -1) + } + if tm.Day() > s.lastDayOfMonth { + tm = tm.AddDate(0, 1, 0) + } + return uint32(tm.Unix()) +} + +func groupByHour(timestamp uint32) uint32 { + return uint32(timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "h").Unix()) +} + +func groupByDay(timestamp uint32) uint32 { + return uint32(timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "d").Unix()) +} + +func groupByMonth(timestamp uint32) uint32 { + return uint32(timeutil.FirstSecondInPeriod(time.Unix(int64(timestamp), 0), "m").Unix()) +} diff --git a/atree/atree.go b/atree/atree.go new file mode 100644 index 0000000..d554f91 --- /dev/null +++ b/atree/atree.go @@ -0,0 +1,497 @@ +package atree + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sync" + + "gordenko.dev/dima/diploma" + "gordenko.dev/dima/diploma/atree/redo" + "gordenko.dev/dima/diploma/bin" +) + +const ( + filePerm = 0770 + + // index page + indexRecordsQtyIdx = IndexPageSize - 7 + isDataPageNumbersIdx = IndexPageSize - 5 + indexCRC32Idx = IndexPageSize - 4 + + // data page + timestampsSizeIdx = DataPageSize - 12 + valuesSizeIdx = DataPageSize - 10 + prevPageIdx = DataPageSize - 8 + dataCRC32Idx = DataPageSize - 4 + + timestampSize = 4 + pairSize = timestampSize + PageNoSize + indexFooterIdx = indexRecordsQtyIdx + dataFooterIdx = timestampsSizeIdx + + DataPageSize = 8192 + IndexPageSize = 1024 + PageNoSize = 4 + // + DataPagePayloadSize int = dataFooterIdx +) + +type FreeList interface { + ReservePage() uint32 +} + +type _page struct { + PageNo uint32 + Buf []byte + ReferenceCount int +} + +type Atree struct { + redoDir string + indexFreelist FreeList + dataFreelist FreeList + dataFile *os.File + indexFile *os.File + mutex sync.Mutex + allocatedIndexPagesQty uint32 + allocatedDataPagesQty uint32 + indexPages map[uint32]*_page + dataPages map[uint32]*_page + indexWaits map[uint32][]chan readResult + dataWaits map[uint32][]chan readResult + indexPagesToRead []uint32 + dataPagesToRead []uint32 + readSignalCh chan struct{} + writeSignalCh chan struct{} + writeTasksQueue []WriteTask +} + +type Options struct { + Dir string + RedoDir string + DatabaseName string + DataFreeList FreeList + IndexFreeList FreeList +} + +func New(opt Options) (*Atree, error) { + if opt.Dir == "" { + return nil, errors.New("Dir option is required") + } + if opt.RedoDir == "" { + return nil, errors.New("RedoDir option is required") + } + if opt.DatabaseName == "" { + return nil, errors.New("DatabaseName option is required") + } + if opt.DataFreeList == nil { + return nil, errors.New("DataFreeList option is required") + } + if opt.IndexFreeList == nil { + return nil, errors.New("IndexFreeList option is required") + } + // открываю или создаю dbName.data и dbName.index файлы + var ( + indexFileName = filepath.Join(opt.Dir, opt.DatabaseName+".index") + dataFileName = filepath.Join(opt.Dir, opt.DatabaseName+".data") + + indexFile *os.File + dataFile *os.File + allocatedIndexPagesQty uint32 + allocatedDataPagesQty uint32 + ) + + // При создании data файла сразу создается индекс, поэтому корректное + // состояние БД: либо оба файла есть, либо ни одного файла нет. + isIndexExist, err := isFileExist(indexFileName) + if err != nil { + return nil, fmt.Errorf("check index file is exist: %s", err) + } + + isDataExist, err := isFileExist(dataFileName) + if err != nil { + return nil, fmt.Errorf("check data file is exist: %s", err) + } + + if isIndexExist { + if isDataExist { + // открываю оба файла + indexFile, allocatedIndexPagesQty, err = openFile(indexFileName, IndexPageSize) + if err != nil { + return nil, fmt.Errorf("open index file: %s", err) + } + + dataFile, allocatedDataPagesQty, err = openFile(dataFileName, DataPageSize) + if err != nil { + return nil, fmt.Errorf("open data file: %s", err) + } + } else { + // нет data файла + return nil, errors.New("not found data file") + } + } else { + if isDataExist { + // index файла нет + return nil, errors.New("not found index file") + } else { + // нет обоих файлов + indexFile, err = os.OpenFile(indexFileName, os.O_CREATE|os.O_RDWR, filePerm) + if err != nil { + return nil, err + } + + dataFile, err = os.OpenFile(dataFileName, os.O_CREATE|os.O_RDWR, filePerm) + if err != nil { + return nil, err + } + } + } + + tree := &Atree{ + redoDir: opt.RedoDir, + indexFreelist: opt.IndexFreeList, + dataFreelist: opt.DataFreeList, + indexFile: indexFile, + dataFile: dataFile, + allocatedIndexPagesQty: allocatedIndexPagesQty, + allocatedDataPagesQty: allocatedDataPagesQty, + indexPages: make(map[uint32]*_page), + dataPages: make(map[uint32]*_page), + indexWaits: make(map[uint32][]chan readResult), + dataWaits: make(map[uint32][]chan readResult), + readSignalCh: make(chan struct{}, 1), + writeSignalCh: make(chan struct{}, 1), + } + + return tree, nil +} + +func (s *Atree) Run() { + go s.pageWriter() + go s.pageReader() +} + +// FIND + +func (s *Atree) findDataPage(rootPageNo uint32, timestamp uint32) (uint32, []byte, error) { + indexPageNo := rootPageNo + for { + buf, err := s.fetchIndexPage(indexPageNo) + if err != nil { + return 0, nil, fmt.Errorf("fetchIndexPage(%d): %s", indexPageNo, err) + } + + foundPageNo := findPageNo(buf, timestamp) + s.releaseIndexPage(indexPageNo) + + if buf[isDataPageNumbersIdx] == 1 { + buf, err := s.fetchDataPage(foundPageNo) + if err != nil { + return 0, nil, fmt.Errorf("fetchDataPage(%d): %s", foundPageNo, err) + } + return foundPageNo, buf, nil + } + // вглубь + indexPageNo = foundPageNo + } +} + +type pathLeg struct { + PageNo uint32 + Data []byte +} + +type pathToDataPage struct { + Legs []pathLeg + LastPageNo uint32 +} + +func (s *Atree) findPathToLastPage(rootPageNo uint32) (_ pathToDataPage, err error) { + var ( + pageNo = rootPageNo + legs []pathLeg + ) + + for { + var buf []byte + buf, err = s.fetchIndexPage(pageNo) + if err != nil { + err = fmt.Errorf("FetchIndexPage(%d): %s", pageNo, err) + return + } + + legs = append(legs, pathLeg{ + PageNo: pageNo, + Data: buf, + // childIdx не нужен + }) + + foundPageNo := getLastPageNo(buf) + + if buf[isDataPageNumbersIdx] == 1 { + return pathToDataPage{ + Legs: legs, + LastPageNo: foundPageNo, + }, nil + } + // вглубь + pageNo = foundPageNo + } +} + +// APPEND DATA PAGE + +type AppendDataPageReq struct { + MetricID uint32 + Timestamp uint32 + Value float64 + Since uint32 + RootPageNo uint32 + PrevPageNo uint32 + TimestampsChunks [][]byte + TimestampsSize uint16 + ValuesChunks [][]byte + ValuesSize uint16 +} + +func (s *Atree) AppendDataPage(req AppendDataPageReq) (_ redo.Report, err error) { + var ( + flags byte + dataPagesToRelease []uint32 + indexPagesToRelease []uint32 + ) + + newDataPage := s.allocDataPage() + dataPagesToRelease = append(dataPagesToRelease, newDataPage.PageNo) + + chunksToDataPage(newDataPage.Data, chunksToDataPageReq{ + PrevPageNo: req.PrevPageNo, + TimestampsChunks: req.TimestampsChunks, + TimestampsSize: req.TimestampsSize, + ValuesChunks: req.ValuesChunks, + ValuesSize: req.ValuesSize, + }) + + redoWriter, err := redo.NewWriter(redo.WriterOptions{ + Dir: s.redoDir, + MetricID: req.MetricID, + Timestamp: req.Timestamp, + Value: req.Value, + IsDataPageReused: newDataPage.IsReused, + DataPageNo: newDataPage.PageNo, + Page: newDataPage.Data, + }) + if err != nil { + return + } + + if req.RootPageNo > 0 { + var path pathToDataPage + path, err = s.findPathToLastPage(req.RootPageNo) + if err != nil { + return + } + for _, leg := range path.Legs { + indexPagesToRelease = append(indexPagesToRelease, leg.PageNo) + } + + if path.LastPageNo != req.PrevPageNo { + diploma.Abort( + diploma.WrongPrevPageNo, + fmt.Errorf("bug: last pageNo %d in tree != prev pageNo %d in _metric", + path.LastPageNo, req.PrevPageNo), + ) + } + + newPageNo := newDataPage.PageNo + lastIdx := len(path.Legs) - 1 + + for legIdx := lastIdx; legIdx >= 0; legIdx-- { + leg := path.Legs[legIdx] + + ok := appendPair(leg.Data, req.Since, newPageNo) + if ok { + err = redoWriter.AppendIndexPage(leg.PageNo, leg.Data, 0) + if err != nil { + return + } + break + } + + newIndexPage := s.allocIndexPage() + indexPagesToRelease = append(indexPagesToRelease, newIndexPage.PageNo) + appendPair(newIndexPage.Data, req.Since, newPageNo) + // ставлю мітку що всі pageNo на сторінці - це data pageNo + if legIdx == lastIdx { + newIndexPage.Data[isDataPageNumbersIdx] = 1 + } + + flags = 0 + if newIndexPage.IsReused { + flags |= redo.FlagReused + } + err = redoWriter.AppendIndexPage(newIndexPage.PageNo, newIndexPage.Data, flags) + if err != nil { + return + } + // + newPageNo = newIndexPage.PageNo + + if legIdx == 0 { + newRoot := s.allocIndexPage() + indexPagesToRelease = append(indexPagesToRelease, newRoot.PageNo) + appendPair(newRoot.Data, getSince(leg.Data), leg.PageNo) // old rootPageNo + appendPair(newRoot.Data, req.Since, newIndexPage.PageNo) + + // Фиксирую новый root в REDO логе + flags = redo.FlagNewRoot + if newRoot.IsReused { + flags |= redo.FlagReused + } + err = redoWriter.AppendIndexPage(newRoot.PageNo, newRoot.Data, flags) + if err != nil { + return + } + break + } + } + } else { + newRoot := s.allocIndexPage() + indexPagesToRelease = append(indexPagesToRelease, newRoot.PageNo) + newRoot.Data[isDataPageNumbersIdx] = 1 + appendPair(newRoot.Data, req.Since, newDataPage.PageNo) + + flags = redo.FlagNewRoot + if newRoot.IsReused { + flags |= redo.FlagReused + } + err = redoWriter.AppendIndexPage(newRoot.PageNo, newRoot.Data, flags) + if err != nil { + return + } + } + + err = redoWriter.Close() + if err != nil { + return + } + + // На данний момен схема - наступна. Всі сторінки - data та index - зафіксовані в кеші. + // Отже запис на диск пройде максимально швидко. Після цього ReferenceCount кожної + // сторінки зменшиться на 1. Оскільки на метрику утримується XLock, сторінки мають + // ReferenceCount = 1 (немає інших читачів). + waitCh := make(chan struct{}) + + task := WriteTask{ + WaitCh: waitCh, + DataPage: redo.PageToWrite{ + PageNo: newDataPage.PageNo, + Data: newDataPage.Data, + }, + IndexPages: redoWriter.IndexPagesToWrite(), + } + + s.appendWriteTaskToQueue(task) + + <-waitCh + + for _, pageNo := range dataPagesToRelease { + s.releaseDataPage(pageNo) + } + for _, pageNo := range indexPagesToRelease { + s.releaseIndexPage(pageNo) + } + return redoWriter.GetReport(), nil +} + +// DELETE + +type PageLists struct { + DataPages []uint32 + IndexPages []uint32 +} + +type Level struct { + PageNo uint32 + PageData []byte + Idx int + ChildQty int +} + +func (s *Atree) GetAllPages(rootPageNo uint32) (_ PageLists, err error) { + var ( + dataPages []uint32 + indexPages []uint32 + levels []*Level + ) + + buf, err := s.fetchIndexPage(rootPageNo) + if err != nil { + err = fmt.Errorf("fetchIndexPage(%d): %s", rootPageNo, err) + return + } + indexPages = append(indexPages, rootPageNo) + + if buf[isDataPageNumbersIdx] == 1 { + pageNumbers := listPageNumbers(buf) + dataPages = append(dataPages, pageNumbers...) + + s.releaseIndexPage(rootPageNo) + + return PageLists{ + DataPages: dataPages, + IndexPages: indexPages, + }, nil + } + + levels = append(levels, &Level{ + PageNo: rootPageNo, + PageData: buf, + Idx: 0, + ChildQty: bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]), + }) + + for { + if len(levels) == 0 { + return PageLists{ + DataPages: dataPages, + IndexPages: indexPages, + }, nil + } + + lastIdx := len(levels) - 1 + level := levels[lastIdx] + + if level.Idx < level.ChildQty { + pageNo := getPageNo(level.PageData, level.Idx) + level.Idx++ + + var buf []byte + buf, err = s.fetchIndexPage(pageNo) + if err != nil { + err = fmt.Errorf("fetchIndexPage(%d): %s", pageNo, err) + return + } + indexPages = append(indexPages, pageNo) + + if buf[isDataPageNumbersIdx] == 1 { + pageNumbers := listPageNumbers(buf) + dataPages = append(dataPages, pageNumbers...) + + s.releaseIndexPage(pageNo) + } else { + levels = append(levels, &Level{ + PageNo: pageNo, + PageData: buf, + Idx: 0, + ChildQty: bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]), + }) + } + } else { + s.releaseIndexPage(level.PageNo) + levels = levels[:lastIdx] + } + } +} diff --git a/atree/cursor.go b/atree/cursor.go new file mode 100644 index 0000000..fe69a63 --- /dev/null +++ b/atree/cursor.go @@ -0,0 +1,187 @@ +package atree + +import ( + "errors" + "fmt" + + octopus "gordenko.dev/dima/diploma" + "gordenko.dev/dima/diploma/bin" + "gordenko.dev/dima/diploma/enc" +) + +type BackwardCursor struct { + metricType octopus.MetricType + fracDigits byte + atree *Atree + pageNo uint32 + pageData []byte + timestampDecompressor octopus.TimestampDecompressor + valueDecompressor octopus.ValueDecompressor +} + +type BackwardCursorOptions struct { + MetricType octopus.MetricType + FracDigits byte + PageNo uint32 + PageData []byte + Atree *Atree +} + +func NewBackwardCursor(opt BackwardCursorOptions) (*BackwardCursor, error) { + switch opt.MetricType { + case octopus.Instant, octopus.Cumulative: + // ok + default: + return nil, fmt.Errorf("MetricType option has wrong value: %d", opt.MetricType) + } + if opt.FracDigits > octopus.MaxFracDigits { + return nil, errors.New("FracDigits option is required") + } + if opt.Atree == nil { + return nil, errors.New("Atree option is required") + } + if opt.PageNo == 0 { + return nil, errors.New("PageNo option is required") + } + if len(opt.PageData) == 0 { + return nil, errors.New("PageData option is required") + } + + s := &BackwardCursor{ + metricType: opt.MetricType, + fracDigits: opt.FracDigits, + atree: opt.Atree, + pageNo: opt.PageNo, + pageData: opt.PageData, + } + err := s.makeDecompressors() + if err != nil { + return nil, err + } + return s, nil +} + +// timestamp, value, done, error +func (s *BackwardCursor) Prev() (uint32, float64, bool, error) { + var ( + timestamp uint32 + value float64 + done bool + err error + ) + + timestamp, done = s.timestampDecompressor.NextValue() + if !done { + value, done = s.valueDecompressor.NextValue() + if done { + return 0, 0, false, + fmt.Errorf("corrupted data page %d: has timestamp, no value", + s.pageNo) + } + return timestamp, value, false, nil + } + + prevPageNo := bin.GetUint32(s.pageData[prevPageIdx:]) + if prevPageNo == 0 { + return 0, 0, true, nil + } + s.atree.releaseDataPage(s.pageNo) + + s.pageNo = prevPageNo + s.pageData, err = s.atree.fetchDataPage(s.pageNo) + if err != nil { + return 0, 0, false, fmt.Errorf("atree.fetchDataPage(%d): %s", s.pageNo, err) + } + + err = s.makeDecompressors() + if err != nil { + return 0, 0, false, err + } + + timestamp, done = s.timestampDecompressor.NextValue() + if done { + return 0, 0, false, + fmt.Errorf("corrupted data page %d: no timestamps", + s.pageNo) + } + value, done = s.valueDecompressor.NextValue() + if done { + return 0, 0, false, + fmt.Errorf("corrupted data page %d: no values", + s.pageNo) + } + return timestamp, value, false, nil +} + +func (s *BackwardCursor) Close() { + s.atree.releaseDataPage(s.pageNo) +} + +// HELPER + +func (s *BackwardCursor) makeDecompressors() error { + timestampsPayloadSize := bin.GetUint16(s.pageData[timestampsSizeIdx:]) + valuesPayloadSize := bin.GetUint16(s.pageData[valuesSizeIdx:]) + + payloadSize := timestampsPayloadSize + valuesPayloadSize + + if payloadSize > dataFooterIdx { + return fmt.Errorf("corrupted data page %d: timestamps + values size %d gt payload size", + s.pageNo, payloadSize) + } + + s.timestampDecompressor = enc.NewReverseTimeDeltaOfDeltaDecompressor( + s.pageData[:timestampsPayloadSize], + ) + + vbuf := s.pageData[timestampsPayloadSize : timestampsPayloadSize+valuesPayloadSize] + + switch s.metricType { + case octopus.Instant: + s.valueDecompressor = enc.NewReverseInstantDeltaDecompressor( + vbuf, s.fracDigits) + + case octopus.Cumulative: + s.valueDecompressor = enc.NewReverseCumulativeDeltaDecompressor( + vbuf, s.fracDigits) + + default: + return fmt.Errorf("bug: wrong metricType %d", s.metricType) + } + return nil +} + +func makeDecompressors(pageData []byte, metricType octopus.MetricType, fracDigits byte) ( + octopus.TimestampDecompressor, octopus.ValueDecompressor, error, +) { + timestampsPayloadSize := bin.GetUint16(pageData[timestampsSizeIdx:]) + valuesPayloadSize := bin.GetUint16(pageData[valuesSizeIdx:]) + + payloadSize := timestampsPayloadSize + valuesPayloadSize + + if payloadSize > dataFooterIdx { + return nil, nil, fmt.Errorf("corrupted: timestamps + values size %d > payload size", + payloadSize) + } + + timestampDecompressor := enc.NewReverseTimeDeltaOfDeltaDecompressor( + pageData[:timestampsPayloadSize], + ) + + vbuf := pageData[timestampsPayloadSize : timestampsPayloadSize+valuesPayloadSize] + + var valueDecompressor octopus.ValueDecompressor + switch metricType { + case octopus.Instant: + valueDecompressor = enc.NewReverseInstantDeltaDecompressor( + vbuf, fracDigits) + + case octopus.Cumulative: + valueDecompressor = enc.NewReverseCumulativeDeltaDecompressor( + vbuf, fracDigits) + + default: + return nil, nil, fmt.Errorf("bug: wrong metricType %d", metricType) + } + return timestampDecompressor, valueDecompressor, nil +} diff --git a/atree/io.go b/atree/io.go new file mode 100644 index 0000000..5bb4492 --- /dev/null +++ b/atree/io.go @@ -0,0 +1,430 @@ +package atree + +import ( + "errors" + "fmt" + "hash/crc32" + "io/fs" + "math" + "os" + + octopus "gordenko.dev/dima/diploma" + "gordenko.dev/dima/diploma/atree/redo" + "gordenko.dev/dima/diploma/bin" +) + +type AllocatedPage struct { + PageNo uint32 + Data []byte + IsReused bool +} + +type readResult struct { + Data []byte + Err error +} + +// INDEX PAGES + +func (s *Atree) DeleteIndexPages(pageNumbers []uint32) { + s.mutex.Lock() + for _, pageNo := range pageNumbers { + delete(s.indexPages, pageNo) + } + s.mutex.Unlock() +} + +func (s *Atree) fetchIndexPage(pageNo uint32) ([]byte, error) { + s.mutex.Lock() + p, ok := s.indexPages[pageNo] + if ok { + p.ReferenceCount++ + s.mutex.Unlock() + return p.Buf, nil + } + + resultCh := make(chan readResult, 1) + s.indexWaits[pageNo] = append(s.indexWaits[pageNo], resultCh) + if len(s.indexWaits[pageNo]) == 1 { + s.indexPagesToRead = append(s.indexPagesToRead, pageNo) + s.mutex.Unlock() + + select { + case s.readSignalCh <- struct{}{}: + default: + } + } else { + s.mutex.Unlock() + } + + result := <-resultCh + if result.Err == nil { + result.Err = s.verifyCRC(result.Data, IndexPageSize) + } + return result.Data, result.Err +} + +func (s *Atree) releaseIndexPage(pageNo uint32) { + s.mutex.Lock() + defer s.mutex.Unlock() + + p, ok := s.indexPages[pageNo] + if ok { + if p.ReferenceCount > 0 { + p.ReferenceCount-- + return + } else { + octopus.Abort( + octopus.ReferenceCountBug, + fmt.Errorf("call releaseIndexPage on page %d with reference count = %d", + pageNo, p.ReferenceCount), + ) + } + } +} + +func (s *Atree) allocIndexPage() AllocatedPage { + var ( + allocated = AllocatedPage{ + Data: make([]byte, IndexPageSize), + } + ) + + allocated.PageNo = s.indexFreelist.ReservePage() + if allocated.PageNo > 0 { + allocated.IsReused = true + + s.mutex.Lock() + } else { + s.mutex.Lock() + if s.allocatedIndexPagesQty == math.MaxUint32 { + octopus.Abort(octopus.MaxAtreeSizeExceeded, + errors.New("no space in Atree index")) + } + s.allocatedIndexPagesQty++ + allocated.PageNo = s.allocatedIndexPagesQty + } + + s.indexPages[allocated.PageNo] = &_page{ + PageNo: allocated.PageNo, + Buf: allocated.Data, + ReferenceCount: 1, + } + s.mutex.Unlock() + return allocated +} + +// DATA PAGES + +func (s *Atree) DeleteDataPages(pageNumbers []uint32) { + s.mutex.Lock() + for _, pageNo := range pageNumbers { + delete(s.dataPages, pageNo) + } + s.mutex.Unlock() +} + +func (s *Atree) fetchDataPage(pageNo uint32) ([]byte, error) { + s.mutex.Lock() + p, ok := s.dataPages[pageNo] + if ok { + p.ReferenceCount++ + s.mutex.Unlock() + return p.Buf, nil + } + + resultCh := make(chan readResult, 1) + s.dataWaits[pageNo] = append(s.dataWaits[pageNo], resultCh) + if len(s.dataWaits[pageNo]) == 1 { + s.dataPagesToRead = append(s.dataPagesToRead, pageNo) + s.mutex.Unlock() + + select { + case s.readSignalCh <- struct{}{}: + default: + } + } else { + s.mutex.Unlock() + } + result := <-resultCh + if result.Err == nil { + result.Err = s.verifyCRC(result.Data, DataPageSize) + } + return result.Data, result.Err +} + +func (s *Atree) releaseDataPage(pageNo uint32) { + s.mutex.Lock() + defer s.mutex.Unlock() + + p, ok := s.dataPages[pageNo] + if ok { + if p.ReferenceCount > 0 { + p.ReferenceCount-- + return + } else { + octopus.Abort( + octopus.ReferenceCountBug, + fmt.Errorf("call releaseDataPage on page %d with reference count = %d", + pageNo, p.ReferenceCount), + ) + } + } +} + +func (s *Atree) allocDataPage() AllocatedPage { + var ( + allocated = AllocatedPage{ + Data: make([]byte, DataPageSize), + } + ) + + allocated.PageNo = s.dataFreelist.ReservePage() + if allocated.PageNo > 0 { + allocated.IsReused = true + s.mutex.Lock() + } else { + s.mutex.Lock() + if s.allocatedDataPagesQty == math.MaxUint32 { + octopus.Abort(octopus.MaxAtreeSizeExceeded, + errors.New("no space in Atree index")) + } + s.allocatedDataPagesQty++ + allocated.PageNo = s.allocatedDataPagesQty + } + + s.dataPages[allocated.PageNo] = &_page{ + PageNo: allocated.PageNo, + Buf: allocated.Data, + ReferenceCount: 1, + } + s.mutex.Unlock() + return allocated +} + +// READ + +func (s *Atree) pageReader() { + for { + select { + case <-s.readSignalCh: + s.readPages() + } + } +} + +func (s *Atree) readPages() { + s.mutex.Lock() + if len(s.indexPagesToRead) == 0 && len(s.dataPagesToRead) == 0 { + s.mutex.Unlock() + return + } + indexPagesToRead := s.indexPagesToRead + s.indexPagesToRead = nil + dataPagesToRead := s.dataPagesToRead + s.dataPagesToRead = nil + s.mutex.Unlock() + + for _, pageNo := range dataPagesToRead { + buf := make([]byte, DataPageSize) + off := (pageNo - 1) * DataPageSize + n, err := s.dataFile.ReadAt(buf, int64(off)) + if n != DataPageSize { + err = fmt.Errorf("read %d instead of %d", n, DataPageSize) + } + + s.mutex.Lock() + resultChannels := s.dataWaits[pageNo] + delete(s.dataWaits, pageNo) + + if err != nil { + s.mutex.Unlock() + for _, resultCh := range resultChannels { + resultCh <- readResult{ + Err: err, + } + } + } else { + s.dataPages[pageNo] = &_page{ + PageNo: pageNo, + Buf: buf, + ReferenceCount: len(resultChannels), + } + s.mutex.Unlock() + for _, resultCh := range resultChannels { + resultCh <- readResult{ + Data: buf, + } + } + } + } + + for _, pageNo := range indexPagesToRead { + buf := make([]byte, IndexPageSize) + off := (pageNo - 1) * IndexPageSize + n, err := s.indexFile.ReadAt(buf, int64(off)) + if n != IndexPageSize { + err = fmt.Errorf("read %d instead of %d", n, IndexPageSize) + } + + s.mutex.Lock() + resultChannels := s.indexWaits[pageNo] + delete(s.indexWaits, pageNo) + + if err != nil { + s.mutex.Unlock() + for _, resultCh := range resultChannels { + resultCh <- readResult{ + Err: err, + } + } + } else { + s.indexPages[pageNo] = &_page{ + PageNo: pageNo, + Buf: buf, + ReferenceCount: len(resultChannels), + } + s.mutex.Unlock() + for _, resultCh := range resultChannels { + resultCh <- readResult{ + Data: buf, + } + } + } + + } +} + +// WRITE + +func (s *Atree) pageWriter() { + for { + select { + case <-s.writeSignalCh: + err := s.writeTasks() + if err != nil { + octopus.Abort(octopus.WriteToAtreeFailed, err) + } + } + } +} + +type WriteTask struct { + WaitCh chan struct{} + DataPage redo.PageToWrite + IndexPages []redo.PageToWrite +} + +func (s *Atree) appendWriteTaskToQueue(task WriteTask) { + s.mutex.Lock() + s.writeTasksQueue = append(s.writeTasksQueue, task) + s.mutex.Unlock() + + select { + case s.writeSignalCh <- struct{}{}: + default: + } +} + +func (s *Atree) writeTasks() error { + s.mutex.Lock() + tasks := s.writeTasksQueue + s.writeTasksQueue = nil + s.mutex.Unlock() + + for _, task := range tasks { + // data page + p := task.DataPage + if len(p.Data) != DataPageSize { + return fmt.Errorf("wrong data page %d size: %d", + p.PageNo, len(p.Data)) + } + off := (p.PageNo - 1) * DataPageSize + n, err := s.dataFile.WriteAt(p.Data, int64(off)) + if err != nil { + return err + } + if n != len(p.Data) { + return fmt.Errorf("write %d instead of %d", n, len(p.Data)) + } + + // index pages + for _, p := range task.IndexPages { + if len(p.Data) != IndexPageSize { + return fmt.Errorf("wrong index page %d size: %d", + p.PageNo, len(p.Data)) + } + bin.PutUint32(p.Data[indexCRC32Idx:], crc32.ChecksumIEEE(p.Data[:indexCRC32Idx])) + + off := (p.PageNo - 1) * IndexPageSize + n, err := s.indexFile.WriteAt(p.Data, int64(off)) + if err != nil { + return err + } + if n != len(p.Data) { + return fmt.Errorf("write %d instead of %d", n, len(p.Data)) + } + } + close(task.WaitCh) + } + return nil +} + +// IO + +func isFileExist(fileName string) (bool, error) { + _, err := os.Stat(fileName) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } else { + return false, err + } + } else { + return true, nil + } +} + +func openFile(fileName string, pageSize int) (_ *os.File, _ uint32, err error) { + file, err := os.OpenFile(fileName, os.O_RDWR, filePerm) + if err != nil { + return + } + fi, err := file.Stat() + if err != nil { + return + } + fileSize := fi.Size() + + if (fileSize % int64(pageSize)) > 0 { + err = fmt.Errorf("the file size %d is not a multiple of the page size %d", + fileSize, pageSize) + return + } + + allocatedPagesQty := fileSize / int64(pageSize) + if allocatedPagesQty > math.MaxUint32 { + err = fmt.Errorf("allocated pages %d is > max pages %d", + allocatedPagesQty, math.MaxUint32) + return + } + + return file, uint32(allocatedPagesQty), nil +} + +func (s *Atree) ApplyREDO(task WriteTask) { + s.appendWriteTaskToQueue(task) +} + +func (s *Atree) verifyCRC(data []byte, pageSize int) error { + var ( + pos = pageSize - 4 + calculatedCRC = crc32.ChecksumIEEE(data[:pos]) + storedCRC = bin.GetUint32(data[pos:]) + ) + if calculatedCRC != storedCRC { + return fmt.Errorf("calculatedCRC %d not equal storedCRC %d", + calculatedCRC, storedCRC) + } + return nil +} diff --git a/atree/misc.go b/atree/misc.go new file mode 100644 index 0000000..4ccb434 --- /dev/null +++ b/atree/misc.go @@ -0,0 +1,214 @@ +package atree + +import ( + "hash/crc32" + + "gordenko.dev/dima/diploma/bin" +) + +type ValueAtComparator struct { + buf []byte + timestamp uint32 +} + +func (s ValueAtComparator) CompareTo(elemIdx int) int { + var ( + pos = elemIdx * timestampSize + elem = bin.GetUint32(s.buf[pos:]) + ) + + if s.timestamp < elem { + return -1 + } else if s.timestamp > elem { + return 1 + } else { + return 0 + } +} + +func BinarySearch(qty int, keyComparator bin.KeyComparator) (elemIdx int, isFound bool) { + if qty == 0 { + return + } + a := 0 + b := qty - 1 + for { + var ( + elemIdx = (b-a)/2 + a + code = keyComparator.CompareTo(elemIdx) + ) + if code == 1 { + a = elemIdx + 1 + if a > b { + return elemIdx, false // +1 + } + } else if code == -1 { + b = elemIdx - 1 + if b < a { + if elemIdx == 0 { + return 0, false + } else { + return elemIdx - 1, false + } + } + } else { + return elemIdx, true + } + } +} + +type chunksToDataPageReq struct { + PrevPageNo uint32 + TimestampsChunks [][]byte + TimestampsSize uint16 + ValuesChunks [][]byte + ValuesSize uint16 +} + +func chunksToDataPage(buf []byte, req chunksToDataPageReq) { + bin.PutUint32(buf[prevPageIdx:], req.PrevPageNo) + bin.PutUint16(buf[timestampsSizeIdx:], req.TimestampsSize) + bin.PutUint16(buf[valuesSizeIdx:], req.ValuesSize) + + var ( + remainingSize = int(req.TimestampsSize) + pos = 0 + ) + for _, chunk := range req.TimestampsChunks { + if remainingSize >= len(chunk) { + copy(buf[pos:], chunk) + remainingSize -= len(chunk) + pos += len(chunk) + } else { + copy(buf[pos:], chunk[:remainingSize]) + break + } + } + + remainingSize = int(req.ValuesSize) + pos = int(req.TimestampsSize) + + for _, chunk := range req.ValuesChunks { + if remainingSize >= len(chunk) { + copy(buf[pos:], chunk) + remainingSize -= len(chunk) + pos += len(chunk) + } else { + copy(buf[pos:], chunk[:remainingSize]) + break + } + } + bin.PutUint32(buf[dataCRC32Idx:], crc32.ChecksumIEEE(buf[:dataCRC32Idx])) +} + +func setPrevPageNo(buf []byte, pageNo uint32) { + bin.PutUint32(buf[prevPageIdx:], pageNo) +} + +func getPrevPageNo(buf []byte) uint32 { + return bin.GetUint32(buf[prevPageIdx:]) +} + +func findPageNo(buf []byte, timestamp uint32) (pageNo uint32) { + var ( + qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]) + comparator = ValueAtComparator{ + buf: buf, + timestamp: timestamp, + } + ) + elemIdx, _ := BinarySearch(qty, comparator) + pos := indexFooterIdx - (elemIdx+1)*PageNoSize + return bin.GetUint32(buf[pos:]) +} + +func findPageNoIdx(buf []byte, timestamp uint32) (idx int) { + var ( + qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]) + comparator = ValueAtComparator{ + buf: buf, + timestamp: timestamp, + } + ) + elemIdx, _ := BinarySearch(qty, comparator) + return elemIdx +} + +func findPageNoForDeleteSince(buf []byte, since uint32) (uint32, int, int) { + var ( + qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]) + comparator = ValueAtComparator{ + buf: buf, + timestamp: since, + } + ) + elemIdx, _ := BinarySearch(qty, comparator) + pos := elemIdx * timestampSize + timestamp := bin.GetUint32(buf[pos:]) + + if timestamp == since { + if elemIdx == 0 { + return 0, qty, -1 + } + elemIdx-- + } + pos = indexFooterIdx - (elemIdx+1)*PageNoSize + return bin.GetUint32(buf[pos:]), qty, elemIdx +} + +func getLastPageNo(buf []byte) (pageNo uint32) { + qty := bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]) + pos := indexFooterIdx - qty*PageNoSize + return bin.GetUint32(buf[pos:]) +} + +func getPageNo(buf []byte, idx int) (pageNo uint32) { + pos := indexFooterIdx - (idx+1)*PageNoSize + return bin.GetUint32(buf[pos:]) +} + +func listPageNumbers(buf []byte) (pageNumbers []uint32) { + qty := bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]) + pos := indexFooterIdx - PageNoSize + for range qty { + pageNumbers = append(pageNumbers, bin.GetUint32(buf[pos:])) + pos -= PageNoSize + } + return +} + +// include since timestamp +func listPageNumbersSince(buf []byte, timestamp uint32) (pageNumbers []uint32) { + var ( + qty = bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]) + comparator = ValueAtComparator{ + buf: buf, + timestamp: timestamp, + } + ) + elemIdx, _ := BinarySearch(qty, comparator) + pos := indexFooterIdx - (elemIdx+1)*PageNoSize + for range qty { + pageNumbers = append(pageNumbers, bin.GetUint32(buf[pos:])) + pos -= PageNoSize + } + return +} + +func getSince(buf []byte) uint32 { + return bin.GetUint32(buf[0:]) +} + +func appendPair(buf []byte, timestamp uint32, pageNo uint32) bool { + qty := bin.GetUint16AsInt(buf[indexRecordsQtyIdx:]) + free := indexFooterIdx - qty*pairSize + if free < pairSize { + return false + } + pos := qty * timestampSize + bin.PutUint32(buf[pos:], timestamp) + pos = indexFooterIdx - (qty+1)*PageNoSize + bin.PutUint32(buf[pos:], pageNo) + bin.PutIntAsUint16(buf[indexRecordsQtyIdx:], qty+1) + return true +} diff --git a/atree/redo/reader.go b/atree/redo/reader.go new file mode 100644 index 0000000..c114937 --- /dev/null +++ b/atree/redo/reader.go @@ -0,0 +1,96 @@ +package redo + +import ( + "fmt" + "hash/crc32" + "io" + "os" + + "gordenko.dev/dima/diploma/bin" +) + +type REDOFile struct { + MetricID uint32 + Timestamp uint32 + Value float64 + IsDataPageReused bool + DataPage PageToWrite + IsRootChanged bool + RootPageNo uint32 + ReusedIndexPages []uint32 + IndexPages []PageToWrite +} + +type ReadREDOFileReq struct { + FileName string + DataPageSize int + IndexPageSize int +} + +func ReadREDOFile(req ReadREDOFileReq) (*REDOFile, error) { + buf, err := os.ReadFile(req.FileName) + if err != nil { + return nil, err + } + + if len(buf) < 25 { + return nil, io.EOF + } + + var ( + end = len(buf) - 4 + payload = buf[:end] + checksum = bin.GetUint32(buf[end:]) + calculatedChecksum = crc32.ChecksumIEEE(payload) + ) + + // Помилка чексуми означає що файл або недописаний, або пошкодженний + if checksum != calculatedChecksum { + return nil, fmt.Errorf("written checksum %d not equal calculated checksum %d", + checksum, calculatedChecksum) + } + + var ( + redoLog = REDOFile{ + MetricID: bin.GetUint32(buf[0:]), + Timestamp: bin.GetUint32(buf[4:]), + Value: bin.GetFloat64(buf[8:]), + IsDataPageReused: buf[16] == 1, + DataPage: PageToWrite{ + PageNo: bin.GetUint32(buf[17:]), + Data: buf[21 : 21+req.DataPageSize], + }, + } + pos = 21 + req.DataPageSize + ) + + for { + if pos == len(payload) { + return &redoLog, nil + } + + if pos > len(payload) { + return nil, io.EOF + } + + flags := buf[pos] + + item := PageToWrite{ + PageNo: bin.GetUint32(buf[pos+1:]), + } + pos += 5 // flags + pageNo + item.Data = buf[pos : pos+req.IndexPageSize] + pos += req.IndexPageSize + + redoLog.IndexPages = append(redoLog.IndexPages, item) + + if (flags & FlagReused) == FlagReused { + redoLog.ReusedIndexPages = append(redoLog.ReusedIndexPages, item.PageNo) + } + + if (flags & FlagNewRoot) == FlagNewRoot { + redoLog.IsRootChanged = true + redoLog.RootPageNo = item.PageNo + } + } +} diff --git a/atree/redo/writer.go b/atree/redo/writer.go new file mode 100644 index 0000000..1515bc9 --- /dev/null +++ b/atree/redo/writer.go @@ -0,0 +1,207 @@ +package redo + +import ( + "errors" + "fmt" + "hash" + "hash/crc32" + "os" + "path/filepath" + + "gordenko.dev/dima/diploma/bin" +) + +const ( + FlagReused byte = 1 // сторінка із FreeList + FlagNewRoot byte = 2 // новая страница +) + +type PageToWrite struct { + PageNo uint32 + Data []byte +} + +type Writer struct { + metricID uint32 + timestamp uint32 + value float64 + tmp []byte + fileName string + file *os.File + hasher hash.Hash32 + isDataPageReused bool + dataPageNo uint32 + isRootChanged bool + newRootPageNo uint32 + indexPages []uint32 + reusedIndexPages []uint32 + indexPagesToWrite []PageToWrite +} + +type WriterOptions struct { + Dir string + MetricID uint32 + Value float64 + Timestamp uint32 + IsDataPageReused bool + DataPageNo uint32 + Page []byte +} + +// dataPage можно записати 1 раз. Щоб не заплутувати інтерфейс - передаю data сторінку +// через Options. Index сторінок може бути від 1 до N, тому виділяю окремий метод +func NewWriter(opt WriterOptions) (*Writer, error) { + if opt.Dir == "" { + return nil, errors.New("Dir option is required") + } + if opt.MetricID == 0 { + return nil, errors.New("MetricID option is required") + } + if opt.DataPageNo == 0 { + return nil, errors.New("DataPageNo option is required") + } + // if len(opt.Page) != octopus.DataPageSize { + // return nil, fmt.Errorf("bug: wrong data page size %d", len(opt.Page)) + // } + + s := &Writer{ + fileName: JoinREDOFileName(opt.Dir, opt.MetricID), + metricID: opt.MetricID, + timestamp: opt.Timestamp, + value: opt.Value, + tmp: make([]byte, 21), + isDataPageReused: opt.IsDataPageReused, + dataPageNo: opt.DataPageNo, + hasher: crc32.NewIEEE(), + } + + var err error + s.file, err = os.OpenFile(s.fileName, os.O_CREATE|os.O_WRONLY, 0770) + if err != nil { + return nil, err + } + + err = s.init(opt.Page) + if err != nil { + return nil, err + } + return s, nil +} + +/* +Формат: +4b metricID +8b value +4b timestamp +1b flags (reused) +4b dataPageNo +8KB dataPage +*/ +func (s *Writer) init(dataPage []byte) error { + bin.PutUint32(s.tmp[0:], s.metricID) + bin.PutUint32(s.tmp[4:], s.timestamp) + bin.PutFloat64(s.tmp[8:], s.value) + if s.isDataPageReused { + s.tmp[16] = 1 + } + bin.PutUint32(s.tmp[17:], s.dataPageNo) + + _, err := s.file.Write(s.tmp) + if err != nil { + return err + } + + _, err = s.file.Write(dataPage) + if err != nil { + return err + } + + s.hasher.Write(s.tmp) + s.hasher.Write(dataPage) + return nil +} + +/* +Формат +1b index page flags +4b indexPageNo +Nb indexPage +*/ +func (s *Writer) AppendIndexPage(indexPageNo uint32, indexPage []byte, flags byte) error { + s.tmp[0] = flags + bin.PutUint32(s.tmp[1:], indexPageNo) + _, err := s.file.Write(s.tmp[:5]) + if err != nil { + return err + } + _, err = s.file.Write(indexPage) + if err != nil { + return err + } + + s.hasher.Write(s.tmp[:5]) + s.hasher.Write(indexPage) + + s.indexPages = append(s.indexPages, indexPageNo) + + if (flags & FlagReused) == FlagReused { + s.reusedIndexPages = append(s.reusedIndexPages, indexPageNo) + } + + if (flags & FlagNewRoot) == FlagNewRoot { + s.newRootPageNo = indexPageNo + s.isRootChanged = true + } + + s.indexPagesToWrite = append(s.indexPagesToWrite, + PageToWrite{ + PageNo: indexPageNo, + Data: indexPage, + }) + return nil +} + +func (s *Writer) IndexPagesToWrite() []PageToWrite { + return s.indexPagesToWrite +} + +func (s *Writer) Close() (err error) { + // финализирую запись + bin.PutUint32(s.tmp, s.hasher.Sum32()) + _, err = s.file.Write(s.tmp[:4]) + if err != nil { + return err + } + err = s.file.Sync() + if err != nil { + return + } + return s.file.Close() +} + +type Report struct { + FileName string + IsDataPageReused bool + DataPageNo uint32 + IsRootChanged bool + NewRootPageNo uint32 + ReusedIndexPages []uint32 +} + +func (s *Writer) GetReport() Report { + return Report{ + FileName: s.fileName, + IsDataPageReused: s.isDataPageReused, + DataPageNo: s.dataPageNo, + //IndexPages: s.indexPages, + IsRootChanged: s.isRootChanged, + NewRootPageNo: s.newRootPageNo, + ReusedIndexPages: s.reusedIndexPages, + } +} + +// HELPERS + +func JoinREDOFileName(dir string, metricID uint32) string { + return filepath.Join(dir, fmt.Sprintf("m%d.redo", metricID)) +} diff --git a/atree/select.go b/atree/select.go new file mode 100644 index 0000000..ef002bc --- /dev/null +++ b/atree/select.go @@ -0,0 +1,619 @@ +package atree + +import ( + "fmt" + + octopus "gordenko.dev/dima/diploma" +) + +type IterateAllCumulativeByTreeCursorReq struct { + FracDigits byte + PageNo uint32 + EndTimestamp uint32 + EndValue float64 + ResponseWriter *CumulativeMeasureWriter +} + +func (s *Atree) IterateAllCumulativeByTreeCursor(req IterateAllCumulativeByTreeCursorReq) error { + buf, err := s.fetchDataPage(req.PageNo) + if err != nil { + return err + } + + treeCursor, err := NewBackwardCursor(BackwardCursorOptions{ + PageNo: req.PageNo, + PageData: buf, + Atree: s, + FracDigits: req.FracDigits, + MetricType: octopus.Cumulative, + }) + if err != nil { + return err + } + defer treeCursor.Close() + + var ( + endTimestamp = req.EndTimestamp + endValue = req.EndValue + ) + + for { + timestamp, value, done, err := treeCursor.Prev() + if err != nil { + return err + } + + if done { + err := req.ResponseWriter.WriteMeasure(CumulativeMeasure{ + Timestamp: endTimestamp, + Value: endValue, + Total: endValue, + }) + if err != nil { + return err + } + return nil + } + err = req.ResponseWriter.WriteMeasure(CumulativeMeasure{ + Timestamp: endTimestamp, + Value: endValue, + Total: endValue - value, + }) + if err != nil { + return err + } + endTimestamp = timestamp + endValue = value + } +} + +type ContinueIterateCumulativeByTreeCursorReq struct { + FracDigits byte + Since uint32 + Until uint32 + LastPageNo uint32 + EndTimestamp uint32 + EndValue float64 + ResponseWriter *CumulativeMeasureWriter +} + +func (s *Atree) ContinueIterateCumulativeByTreeCursor(req ContinueIterateCumulativeByTreeCursorReq) error { + buf, err := s.fetchDataPage(req.LastPageNo) + if err != nil { + return fmt.Errorf("fetchDataPage(%d): %s", req.LastPageNo, err) + } + + treeCursor, err := NewBackwardCursor(BackwardCursorOptions{ + PageNo: req.LastPageNo, + PageData: buf, + Atree: s, + FracDigits: req.FracDigits, + MetricType: octopus.Cumulative, + }) + if err != nil { + return err + } + defer treeCursor.Close() + + var ( + endTimestamp = req.EndTimestamp + endValue = req.EndValue + ) + + for { + timestamp, value, done, err := treeCursor.Prev() + if err != nil { + return err + } + + if done { + err := req.ResponseWriter.WriteMeasure(CumulativeMeasure{ + Timestamp: endTimestamp, + Value: endValue, + Total: endValue, + }) + if err != nil { + return err + } + return nil + } + + if timestamp <= req.Until { + err := req.ResponseWriter.WriteMeasure(CumulativeMeasure{ + Timestamp: endTimestamp, + Value: endValue, + Total: endValue - value, + }) + if err != nil { + return err + } + if timestamp < req.Since { + return nil + } + } else { + // bug panic + panic("continue cumulative but timestamp > req.Until") + } + } +} + +type FindAndIterateCumulativeByTreeCursorReq struct { + FracDigits byte + Since uint32 + Until uint32 + RootPageNo uint32 + ResponseWriter *CumulativeMeasureWriter +} + +func (s *Atree) FindAndIterateCumulativeByTreeCursor(req FindAndIterateCumulativeByTreeCursorReq) error { + pageNo, buf, err := s.findDataPage(req.RootPageNo, req.Until) + if err != nil { + return err + } + + treeCursor, err := NewBackwardCursor(BackwardCursorOptions{ + PageNo: pageNo, + PageData: buf, + Atree: s, + FracDigits: req.FracDigits, + MetricType: octopus.Cumulative, + }) + if err != nil { + return err + } + defer treeCursor.Close() + + var ( + endTimestamp uint32 + endValue float64 + ) + + for { + timestamp, value, done, err := treeCursor.Prev() + if err != nil { + return err + } + + if done { + if endTimestamp > 0 { + err := req.ResponseWriter.WriteMeasure(CumulativeMeasure{ + Timestamp: endTimestamp, + Value: endValue, + Total: endValue, + }) + if err != nil { + return err + } + } + return nil + } + + if timestamp > req.Until { + continue + } + + if endTimestamp > 0 { + err := req.ResponseWriter.WriteMeasure(CumulativeMeasure{ + Timestamp: endTimestamp, + Value: endValue, + Total: endValue - value, + }) + if err != nil { + return err + } + } + endTimestamp = timestamp + endValue = value + + if timestamp < req.Since { + return nil + } + } +} + +type IterateAllInstantByTreeCursorReq struct { + FracDigits byte + PageNo uint32 + ResponseWriter *InstantMeasureWriter +} + +func (s *Atree) IterateAllInstantByTreeCursor(req IterateAllInstantByTreeCursorReq) error { + buf, err := s.fetchDataPage(req.PageNo) + if err != nil { + return err + } + + treeCursor, err := NewBackwardCursor(BackwardCursorOptions{ + PageNo: req.PageNo, + PageData: buf, + Atree: s, + FracDigits: req.FracDigits, + MetricType: octopus.Instant, + }) + if err != nil { + return err + } + defer treeCursor.Close() + + for { + timestamp, value, done, err := treeCursor.Prev() + if err != nil { + return err + } + + if done { + return nil + } + + err = req.ResponseWriter.WriteMeasure(InstantMeasure{ + Timestamp: timestamp, + Value: value, + }) + if err != nil { + return err + } + } +} + +type ContinueIterateInstantByTreeCursorReq struct { + FracDigits byte + Since uint32 + Until uint32 + LastPageNo uint32 + ResponseWriter *InstantMeasureWriter +} + +func (s *Atree) ContinueIterateInstantByTreeCursor(req ContinueIterateInstantByTreeCursorReq) error { + buf, err := s.fetchDataPage(req.LastPageNo) + if err != nil { + return fmt.Errorf("fetchDataPage(%d): %s", req.LastPageNo, err) + } + + treeCursor, err := NewBackwardCursor(BackwardCursorOptions{ + PageNo: req.LastPageNo, + PageData: buf, + Atree: s, + FracDigits: req.FracDigits, + MetricType: octopus.Instant, + }) + if err != nil { + return err + } + defer treeCursor.Close() + + for { + timestamp, value, done, err := treeCursor.Prev() + if err != nil { + return err + } + + if done { + // - записи закончились; + return nil + } + + if timestamp > req.Until { + panic("continue instant timestamp > req.Until") + } + + if timestamp < req.Since { + return nil + } + + err = req.ResponseWriter.WriteMeasure(InstantMeasure{ + Timestamp: timestamp, + Value: value, + }) + if err != nil { + return err + } + } +} + +type FindAndIterateInstantByTreeCursorReq struct { + FracDigits byte + Since uint32 + Until uint32 + RootPageNo uint32 + ResponseWriter *InstantMeasureWriter +} + +func (s *Atree) FindAndIterateInstantByTreeCursor(req FindAndIterateInstantByTreeCursorReq) error { + pageNo, buf, err := s.findDataPage(req.RootPageNo, req.Until) + if err != nil { + return err + } + + treeCursor, err := NewBackwardCursor(BackwardCursorOptions{ + PageNo: pageNo, + PageData: buf, + Atree: s, + FracDigits: req.FracDigits, + MetricType: octopus.Instant, + }) + if err != nil { + return err + } + defer treeCursor.Close() + + for { + timestamp, value, done, err := treeCursor.Prev() + if err != nil { + return err + } + + if done { + return nil + } + + if timestamp > req.Until { + continue + } + + if timestamp < req.Since { + return nil + } + + err = req.ResponseWriter.WriteMeasure(InstantMeasure{ + Timestamp: timestamp, + Value: value, + }) + if err != nil { + return err + } + } +} + +type ContinueCollectInstantPeriodsReq struct { + FracDigits byte + Aggregator *InstantAggregator + ResponseWriter *InstantPeriodsWriter + LastPageNo uint32 + Since uint32 + Until uint32 +} + +func (s *Atree) ContinueCollectInstantPeriods(req ContinueCollectInstantPeriodsReq) error { + buf, err := s.fetchDataPage(req.LastPageNo) + if err != nil { + return fmt.Errorf("fetchDataPage(%d): %s", req.LastPageNo, err) + } + + treeCursor, err := NewBackwardCursor(BackwardCursorOptions{ + PageNo: req.LastPageNo, + PageData: buf, + Atree: s, + FracDigits: req.FracDigits, + MetricType: octopus.Instant, + }) + if err != nil { + return err + } + defer treeCursor.Close() + + var period InstantPeriod + + for { + timestamp, value, done, err := treeCursor.Prev() + if err != nil { + return err + } + + if done || timestamp < req.Since { + isCompleted := req.Aggregator.FillPeriod(timestamp, &period) + if isCompleted { + err := req.ResponseWriter.WritePeriod(period) + if err != nil { + return err + } + } + return nil + } + + if timestamp <= req.Until { + isCompleted := req.Aggregator.Feed(timestamp, value, &period) + if isCompleted { + err := req.ResponseWriter.WritePeriod(period) + if err != nil { + return err + } + } + } + } +} + +type FindInstantPeriodsReq struct { + FracDigits byte + ResponseWriter *InstantPeriodsWriter + RootPageNo uint32 + Since uint32 + Until uint32 + GroupBy octopus.GroupBy + FirstHourOfDay int + LastDayOfMonth int +} + +func (s *Atree) FindInstantPeriods(req FindInstantPeriodsReq) error { + pageNo, buf, err := s.findDataPage(req.RootPageNo, req.Until) + if err != nil { + return err + } + + aggregator, err := NewInstantAggregator(InstantAggregatorOptions{ + GroupBy: req.GroupBy, + FirstHourOfDay: req.FirstHourOfDay, + LastDayOfMonth: req.LastDayOfMonth, + }) + if err != nil { + return err + } + + cursor, err := NewBackwardCursor(BackwardCursorOptions{ + PageNo: pageNo, + PageData: buf, + Atree: s, + FracDigits: req.FracDigits, + MetricType: octopus.Instant, + }) + if err != nil { + return err + } + defer cursor.Close() + + var period InstantPeriod + + for { + timestamp, value, done, err := cursor.Prev() + if err != nil { + return err + } + + if done || timestamp < req.Since { + isCompleted := aggregator.FillPeriod(timestamp, &period) + if isCompleted { + err := req.ResponseWriter.WritePeriod(period) + if err != nil { + return err + } + } + return nil + } + + if timestamp <= req.Until { + isCompleted := aggregator.Feed(timestamp, value, &period) + if isCompleted { + err := req.ResponseWriter.WritePeriod(period) + if err != nil { + return err + } + } + } + } +} + +type FindCumulativePeriodsReq struct { + FracDigits byte + ResponseWriter *CumulativePeriodsWriter + RootPageNo uint32 + Since uint32 + Until uint32 + GroupBy octopus.GroupBy + FirstHourOfDay int + LastDayOfMonth int +} + +func (s *Atree) FindCumulativePeriods(req FindCumulativePeriodsReq) error { + pageNo, buf, err := s.findDataPage(req.RootPageNo, req.Until) + if err != nil { + return err + } + + aggregator, err := NewCumulativeAggregator(CumulativeAggregatorOptions{ + GroupBy: req.GroupBy, + FirstHourOfDay: req.FirstHourOfDay, + LastDayOfMonth: req.LastDayOfMonth, + }) + if err != nil { + return err + } + + cursor, err := NewBackwardCursor(BackwardCursorOptions{ + PageNo: pageNo, + PageData: buf, + Atree: s, + FracDigits: req.FracDigits, + MetricType: octopus.Cumulative, + }) + if err != nil { + return err + } + defer cursor.Close() + + var period CumulativePeriod + + for { + timestamp, value, done, err := cursor.Prev() + if err != nil { + return err + } + + if done || timestamp < req.Since { + isCompleted := aggregator.FillPeriod(timestamp, value, &period) + if isCompleted { + err := req.ResponseWriter.WritePeriod(period) + if err != nil { + return err + } + } + return nil + } + + if timestamp <= req.Until { + isCompleted := aggregator.Feed(timestamp, value, &period) + if isCompleted { + err := req.ResponseWriter.WritePeriod(period) + if err != nil { + return err + } + } + } + } +} + +type ContinueCollectCumulativePeriodsReq struct { + FracDigits byte + Aggregator *CumulativeAggregator + ResponseWriter *CumulativePeriodsWriter + LastPageNo uint32 + Since uint32 + Until uint32 +} + +func (s *Atree) ContinueCollectCumulativePeriods(req ContinueCollectCumulativePeriodsReq) error { + buf, err := s.fetchDataPage(req.LastPageNo) + if err != nil { + return fmt.Errorf("fetchDataPage(%d): %s", req.LastPageNo, err) + } + + treeCursor, err := NewBackwardCursor(BackwardCursorOptions{ + PageNo: req.LastPageNo, + PageData: buf, + Atree: s, + FracDigits: req.FracDigits, + MetricType: octopus.Cumulative, + }) + if err != nil { + return err + } + defer treeCursor.Close() + + var period CumulativePeriod + + for { + timestamp, value, done, err := treeCursor.Prev() + if err != nil { + return err + } + + if done || timestamp < req.Since { + isCompleted := req.Aggregator.FillPeriod(timestamp, value, &period) + if isCompleted { + err := req.ResponseWriter.WritePeriod(period) + if err != nil { + return err + } + } + return nil + } + + if timestamp <= req.Until { + isCompleted := req.Aggregator.Feed(timestamp, value, &period) + if isCompleted { + err := req.ResponseWriter.WritePeriod(period) + if err != nil { + return err + } + } + } + } +} diff --git a/atree/writers.go b/atree/writers.go new file mode 100644 index 0000000..06b59ae --- /dev/null +++ b/atree/writers.go @@ -0,0 +1,306 @@ +package atree + +import ( + "bytes" + "fmt" + "io" + + octopus "gordenko.dev/dima/diploma" + "gordenko.dev/dima/diploma/bin" + "gordenko.dev/dima/diploma/proto" +) + +// CURRENT VALUE WRITER + +type CurrentValue struct { + MetricID uint32 + Timestamp uint32 + Value float64 +} + +type CurrentValueWriter struct { + arr []byte + responder *ChunkedResponder +} + +func NewCurrentValueWriter(dst io.Writer) *CurrentValueWriter { + return &CurrentValueWriter{ + arr: make([]byte, 16), + responder: NewChunkedResponder(dst), + } +} + +func (s *CurrentValueWriter) BufferValue(m CurrentValue) { + bin.PutUint32(s.arr[0:], m.MetricID) + bin.PutUint32(s.arr[4:], m.Timestamp) + bin.PutFloat64(s.arr[8:], m.Value) + s.responder.BufferRecord(s.arr) +} + +func (s *CurrentValueWriter) Close() error { + return s.responder.Flush() +} + +// INSTANT MEASURE WRITER + +type InstantMeasure struct { + Timestamp uint32 + Value float64 +} + +type InstantMeasureWriter struct { + arr []byte + responder *ChunkedResponder +} + +func NewInstantMeasureWriter(dst io.Writer) *InstantMeasureWriter { + return &InstantMeasureWriter{ + arr: make([]byte, 12), + responder: NewChunkedResponder(dst), + } +} + +func (s *InstantMeasureWriter) BufferMeasure(m InstantMeasure) { + bin.PutUint32(s.arr[0:], m.Timestamp) + bin.PutFloat64(s.arr[4:], m.Value) + s.responder.BufferRecord(s.arr) +} + +func (s *InstantMeasureWriter) WriteMeasure(m InstantMeasure) error { + bin.PutUint32(s.arr[0:], m.Timestamp) + bin.PutFloat64(s.arr[4:], m.Value) + return s.responder.AppendRecord(s.arr) +} + +func (s *InstantMeasureWriter) Close() error { + return s.responder.Flush() +} + +// CUMULATIVE MEASURE WRITER + +type CumulativeMeasure struct { + Timestamp uint32 + Value float64 + Total float64 +} + +type CumulativeMeasureWriter struct { + arr []byte + responder *ChunkedResponder +} + +func NewCumulativeMeasureWriter(dst io.Writer) *CumulativeMeasureWriter { + return &CumulativeMeasureWriter{ + arr: make([]byte, 20), + responder: NewChunkedResponder(dst), + } +} + +func (s *CumulativeMeasureWriter) BufferMeasure(m CumulativeMeasure) { + bin.PutUint32(s.arr[0:], m.Timestamp) + bin.PutFloat64(s.arr[4:], m.Value) + bin.PutFloat64(s.arr[12:], m.Total) + s.responder.BufferRecord(s.arr) +} + +func (s *CumulativeMeasureWriter) WriteMeasure(m CumulativeMeasure) error { + bin.PutUint32(s.arr[0:], m.Timestamp) + bin.PutFloat64(s.arr[4:], m.Value) + bin.PutFloat64(s.arr[12:], m.Total) + return s.responder.AppendRecord(s.arr) +} + +func (s *CumulativeMeasureWriter) Close() error { + return s.responder.Flush() +} + +// INSTANT AGGREGATE WRITER + +type InstantPeriodsWriter struct { + aggregateFuncs byte + arr []byte + responder *ChunkedResponder +} + +func NewInstantPeriodsWriter(dst io.Writer, aggregateFuncs byte) *InstantPeriodsWriter { + var q int + if (aggregateFuncs & octopus.AggregateMin) == octopus.AggregateMin { + q++ + } + if (aggregateFuncs & octopus.AggregateMax) == octopus.AggregateMax { + q++ + } + if (aggregateFuncs & octopus.AggregateAvg) == octopus.AggregateAvg { + q++ + } + return &InstantPeriodsWriter{ + aggregateFuncs: aggregateFuncs, + arr: make([]byte, 12+q*8), + responder: NewChunkedResponder(dst), + } +} + +type InstantPeriod struct { + Period uint32 + Since uint32 + Until uint32 + Min float64 + Max float64 + Avg float64 +} + +func (s *InstantPeriodsWriter) BufferMeasure(p InstantPeriod) { + s.pack(p) + s.responder.BufferRecord(s.arr) +} + +func (s *InstantPeriodsWriter) WritePeriod(p InstantPeriod) error { + s.pack(p) + return s.responder.AppendRecord(s.arr) +} + +func (s *InstantPeriodsWriter) Close() error { + return s.responder.Flush() +} + +func (s *InstantPeriodsWriter) pack(p InstantPeriod) { + bin.PutUint32(s.arr[0:], p.Period) + bin.PutUint32(s.arr[4:], p.Since) + bin.PutUint32(s.arr[8:], p.Until) + + pos := 12 + if (s.aggregateFuncs & octopus.AggregateMin) == octopus.AggregateMin { + bin.PutFloat64(s.arr[pos:], p.Min) + pos += 8 + } + if (s.aggregateFuncs & octopus.AggregateMax) == octopus.AggregateMax { + bin.PutFloat64(s.arr[pos:], p.Max) + pos += 8 + } + if (s.aggregateFuncs & octopus.AggregateAvg) == octopus.AggregateAvg { + bin.PutFloat64(s.arr[pos:], p.Avg) + } +} + +// CUMULATIVE AGGREGATE WRITER + +type CumulativePeriodsWriter struct { + arr []byte + responder *ChunkedResponder +} + +func NewCumulativePeriodsWriter(dst io.Writer) *CumulativePeriodsWriter { + return &CumulativePeriodsWriter{ + arr: make([]byte, 28), + responder: NewChunkedResponder(dst), + } +} + +type CumulativePeriod struct { + Period uint32 + Since uint32 + Until uint32 + EndValue float64 + Total float64 +} + +func (s *CumulativePeriodsWriter) BufferMeasure(p CumulativePeriod) { + s.pack(p) + s.responder.BufferRecord(s.arr) +} + +func (s *CumulativePeriodsWriter) WritePeriod(p CumulativePeriod) error { + s.pack(p) + return s.responder.AppendRecord(s.arr) +} + +func (s *CumulativePeriodsWriter) Close() error { + return s.responder.Flush() +} + +func (s *CumulativePeriodsWriter) pack(p CumulativePeriod) { + bin.PutUint32(s.arr[0:], p.Period) + bin.PutUint32(s.arr[4:], p.Since) + bin.PutUint32(s.arr[8:], p.Until) + bin.PutFloat64(s.arr[12:], p.EndValue) + bin.PutFloat64(s.arr[20:], p.Total) +} + +// CHUNKED RESPONDER + +//const headerSize = 3 + +var endMsg = []byte{ + proto.RespEndOfValue, // end of stream +} + +type ChunkedResponder struct { + recordsQty int + buf *bytes.Buffer + dst io.Writer +} + +func NewChunkedResponder(dst io.Writer) *ChunkedResponder { + s := &ChunkedResponder{ + recordsQty: 0, + buf: bytes.NewBuffer(nil), + dst: dst, + } + + s.buf.Write([]byte{ + proto.RespPartOfValue, // message type + 0, 0, 0, 0, // records qty + }) + return s +} + +func (s *ChunkedResponder) BufferRecord(rec []byte) { + s.buf.Write(rec) + s.recordsQty++ +} + +func (s *ChunkedResponder) AppendRecord(rec []byte) error { + s.buf.Write(rec) + s.recordsQty++ + + if s.buf.Len() < 1500 { + return nil + } + + if err := s.sendBuffered(); err != nil { + return err + } + + s.buf.Write([]byte{ + proto.RespPartOfValue, // message type + 0, 0, 0, 0, // records qty + }) + s.recordsQty = 0 + return nil +} + +func (s *ChunkedResponder) Flush() error { + if s.recordsQty > 0 { + if err := s.sendBuffered(); err != nil { + return err + } + } + if _, err := s.dst.Write(endMsg); err != nil { + return err + } + return nil +} + +func (s *ChunkedResponder) sendBuffered() (err error) { + msg := s.buf.Bytes() + bin.PutUint32(msg[1:], uint32(s.recordsQty)) + n, err := s.dst.Write(msg) + if err != nil { + return + } + if n != len(msg) { + return fmt.Errorf("incomplete write %d bytes instead of %d", n, len(msg)) + } + s.buf.Reset() + return +} diff --git a/bin/bin.go b/bin/bin.go new file mode 100644 index 0000000..1d14836 --- /dev/null +++ b/bin/bin.go @@ -0,0 +1,621 @@ +package bin + +import ( + "errors" + "fmt" + "io" + "math" +) + +const ( + varInt64SignFlag = 0b01000000 + maxReadAttempts = 5 +) + +var ( + ErrNoSpace = errors.New("no space") + ErrIncompleteWrite = errors.New("incomplete write") + + ErrReadOverflow = errors.New("bin: reader returned 'n' > bufsize") + // ErrNegativeReadCount shows 100% bug in the source reader + ErrNegativeReadCount = errors.New("bin: reader returned negative 'n'") +) + +// READ + +func ReadUint16(src io.Reader) (num uint16, err error) { + var ( + q = 2 + arr = make([]byte, q) + ) + n, err := src.Read(arr) + if err != nil { + return + } + if n != q { + return 0, fmt.Errorf("read %d bytes only", n) + } + return GetUint16(arr), nil +} + +func ReadUint24AsInt(src io.Reader) (num int, err error) { + var ( + q = 3 + arr = make([]byte, q) + ) + n, err := src.Read(arr) + if err != nil { + return + } + if n != q { + return 0, fmt.Errorf("read %d bytes only", n) + } + return GetUint24AsInt(arr), nil +} + +func ReadUint32(src io.Reader) (num uint32, err error) { + var ( + q = 4 + arr = make([]byte, q) + ) + n, err := src.Read(arr) + if err != nil { + return + } + if n != q { + return 0, fmt.Errorf("read %d bytes only", n) + } + return GetUint32(arr), nil +} + +func ReadUint64(src io.Reader) (num uint64, err error) { + var ( + q = 8 + arr = make([]byte, q) + ) + n, err := src.Read(arr) + if err != nil { + return + } + if n != q { + return 0, fmt.Errorf("read %d bytes only", n) + } + return GetUint64(arr), nil +} + +func ReadFloat64(src io.Reader) (num float64, err error) { + var ( + q = 8 + arr = make([]byte, q) + ) + n, err := src.Read(arr) + if err != nil { + return + } + if n != q { + return 0, fmt.Errorf("read %d bytes only", n) + } + return GetFloat64(arr), nil +} + +func ReadUnixtime(src io.Reader) (num int64, err error) { + var ( + q = 8 + arr = make([]byte, q) + ) + n, err := src.Read(arr) + if err != nil { + return + } + if n != q { + return 0, fmt.Errorf("read %d bytes only", n) + } + return int64(GetUint64(arr)), nil +} + +// READ VAR + +func ReadVarUint64(src io.Reader) (num uint64, n int, err error) { + var ( + p = make([]byte, 1) + b byte + ) + for i := range 8 { + _, err = src.Read(p) + if err != nil { + return + } + n++ + b = p[0] + if b >= 128 { + num |= uint64(b&127) << uint(i*7) + return + } + num |= uint64(b) << uint(i*7) + } + _, err = src.Read(p) + if err != nil { + return + } + n++ + num |= uint64(p[0]) << 56 + return +} + +// GET + +func GetUint16(arr []byte) uint16 { + return uint16(arr[0]) | (uint16(arr[1]) << 8) +} + +func GetUint16AsInt(arr []byte) int { + return int(arr[0]) | (int(arr[1]) << 8) +} + +func GetUint24AsInt(arr []byte) int { + return int(arr[0]) | (int(arr[1]) << 8) | (int(arr[2]) << 16) +} + +func GetUint32(arr []byte) uint32 { + return uint32(arr[0]) | (uint32(arr[1]) << 8) | + (uint32(arr[2]) << 16) | (uint32(arr[3]) << 24) +} + +func GetUint32AsInt64(arr []byte) int64 { + u32 := uint32(arr[0]) | (uint32(arr[1]) << 8) | + (uint32(arr[2]) << 16) | (uint32(arr[3]) << 24) + return int64(u32) +} + +func GetUint40(arr []byte) uint64 { + return uint64(arr[0]) | (uint64(arr[1]) << 8) | + (uint64(arr[2]) << 16) | (uint64(arr[3]) << 24) | + (uint64(arr[4]) << 32) +} + +func GetUint48(arr []byte) uint64 { + return uint64(arr[0]) | (uint64(arr[1]) << 8) | + (uint64(arr[2]) << 16) | (uint64(arr[3]) << 24) | + (uint64(arr[4]) << 32) | (uint64(arr[5]) << 40) +} + +func GetUint64(arr []byte) uint64 { + return uint64(arr[0]) | (uint64(arr[1]) << 8) | + (uint64(arr[2]) << 16) | (uint64(arr[3]) << 24) | + (uint64(arr[4]) << 32) | (uint64(arr[5]) << 40) | + (uint64(arr[6]) << 48) | (uint64(arr[7]) << 56) +} + +func GetFloat32(arr []byte) float32 { + return math.Float32frombits(GetUint32(arr)) +} + +func GetFloat64(arr []byte) float64 { + return math.Float64frombits(GetUint64(arr)) +} + +func GetUnixtime(arr []byte) int64 { + u32 := uint32(arr[0]) | (uint32(arr[1]) << 8) | + (uint32(arr[2]) << 16) | (uint32(arr[3]) << 24) + return int64(u32) +} + +func GetVarUint64(arr []byte) (num uint64, n int, err error) { + var b byte + for i := range 8 { + if i >= len(arr) { + return 0, 0, io.EOF + } + b = arr[i] + if b >= 128 { + num |= uint64(b&127) << uint(i*7) + return num, i + 1, nil + } + num |= uint64(b) << uint(i*7) + } + if len(arr) < 9 { + return 0, 0, io.EOF + } + return num | uint64(arr[8])<<56, 9, nil +} + +func ReverseGetVarUint64(arr []byte) (num uint64, n int, err error) { + var ( + b byte + j = len(arr) - 1 + ) + for i := range 8 { + if j < 0 { + return 0, 0, io.EOF + } + b = arr[j] + if b >= 128 { + num |= uint64(b&127) << uint(i*7) + return num, i + 1, nil + } + num |= uint64(b) << uint(i*7) + j-- + } + if j < 0 { + return 0, 0, io.EOF + } + return num | uint64(arr[j])<<56, 9, nil +} + +func GetVarInt64(arr []byte) (num int64, n int, err error) { + u64, n, err := GetVarUint64(arr) + if err != nil { + return + } + return DecodeZigZag(u64), n, nil +} + +func ReverseGetVarInt64(arr []byte) (num int64, n int, err error) { + u64, n, err := ReverseGetVarUint64(arr) + if err != nil { + return + } + return DecodeZigZag(u64), n, nil +} + +// PUT + +func PutUint16(arr []byte, num uint16) { + arr[0] = byte(num) + arr[1] = byte(num >> 8) +} + +func PutIntAsUint16(arr []byte, num int) { + arr[0] = byte(num) + arr[1] = byte(num >> 8) +} + +func PutIntAsUint24(arr []byte, num int) { + arr[0] = byte(num) + arr[1] = byte(num >> 8) + arr[2] = byte(num >> 16) +} + +func PutUint32(arr []byte, num uint32) { + arr[0] = byte(num) + arr[1] = byte(num >> 8) + arr[2] = byte(num >> 16) + arr[3] = byte(num >> 24) +} + +func PutInt64AsUint32(arr []byte, num int64) { + arr[0] = byte(num) + arr[1] = byte(num >> 8) + arr[2] = byte(num >> 16) + arr[3] = byte(num >> 24) +} + +func PutUint40(arr []byte, num uint64) { + arr[0] = byte(num) + arr[1] = byte(num >> 8) + arr[2] = byte(num >> 16) + arr[3] = byte(num >> 24) + arr[4] = byte(num >> 32) +} + +func PutUint48(arr []byte, num uint64) { + arr[0] = byte(num) + arr[1] = byte(num >> 8) + arr[2] = byte(num >> 16) + arr[3] = byte(num >> 24) + arr[4] = byte(num >> 32) + arr[5] = byte(num >> 40) +} + +func PutUint64(arr []byte, num uint64) { + arr[0] = byte(num) + arr[1] = byte(num >> 8) + arr[2] = byte(num >> 16) + arr[3] = byte(num >> 24) + arr[4] = byte(num >> 32) + arr[5] = byte(num >> 40) + arr[6] = byte(num >> 48) + arr[7] = byte(num >> 56) +} + +func PutFloat32(arr []byte, num float32) { + PutUint32(arr, math.Float32bits(num)) +} + +func PutFloat64(arr []byte, num float64) { + PutUint64(arr, math.Float64bits(num)) +} + +// WRITE + +func WriteUint16(dst io.Writer, num uint16) error { + arr := []byte{ + byte(num), + byte(num >> 8), + } + n, err := dst.Write(arr) + if err != nil { + return err + } + if n != 2 { + return ErrIncompleteWrite + } + return err +} + +func WriteUint32(dst io.Writer, num uint32) error { + arr := []byte{ + byte(num), + byte(num >> 8), + byte(num >> 16), + byte(num >> 24), + } + n, err := dst.Write(arr) + if err != nil { + return err + } + if n != 4 { + return ErrIncompleteWrite + } + return err +} + +func WriteFloat64(dst io.Writer, num float64) error { + arr := make([]byte, 8) + PutUint64(arr, math.Float64bits(num)) + n, err := dst.Write(arr) + if err != nil { + return err + } + if n != 2 { + return ErrIncompleteWrite + } + return err +} + +// WRITE VAR + +func WriteVarUint64(dst io.Writer, num uint64) (int, error) { + arr := make([]byte, 9) + for i := range 8 { + arr[i] = byte(num & 127) + num >>= 7 + if num == 0 { + arr[i] |= 128 + size := i + 1 + n, err := dst.Write(arr[:size]) + if err != nil { + return n, err + } + if n != size { + return n, ErrIncompleteWrite + } + return size, nil + } + } + arr[8] = byte(num) + n, err := dst.Write(arr) + if err != nil { + return n, err + } + if n != 9 { + return n, ErrIncompleteWrite + } + return 9, nil +} + +func PutVarUint64(arr []byte, num uint64) (int, error) { + for i := range 8 { + if i >= len(arr) { + return 0, ErrNoSpace + } + arr[i] = byte(num & 127) + num >>= 7 + if num == 0 { + arr[i] |= 128 + return i + 1, nil + } + } + if len(arr) < 9 { + return 0, ErrNoSpace + } + arr[8] = byte(num) + return 9, nil +} + +func ReversePutVarUint64(arr []byte, num uint64) (int, error) { + var tmp [9]byte + for i := range 8 { + tmp[i] = byte(num & 127) + num >>= 7 + if num == 0 { + tmp[i] |= 128 + n := i + 1 + if len(arr) < n { + return 0, ErrNoSpace + } + for j := i; j >= 0; j-- { + arr[i-j] = tmp[j] + } + return n, nil + } + } + tmp[8] = byte(num) + n := 9 + if len(arr) < n { + return 0, ErrNoSpace + } + for j := 8; j >= 0; j-- { + arr[8-j] = tmp[j] + } + return n, nil +} + +func PutVarUint64AtEnd(arr []byte, num uint64) (int, error) { + var ( + tmp [9]byte + n int + ) + for i := range 8 { + tmp[i] = byte(num & 127) + num >>= 7 + if num == 0 { + tmp[i] |= 128 + n = i + 1 + break + } + } + if n == 0 { + tmp[8] = byte(num) + n = 9 + } + if len(arr) < n { + return 0, ErrNoSpace + } + j := len(arr) - n + for i := range n { + arr[j] = tmp[i] + j++ + } + return n, nil +} + +func PutVarInt64(arr []byte, x int64) (int, error) { + return PutVarUint64(arr, EncodeZigZag(x)) +} + +func PutVarInt64AtEnd(arr []byte, x int64) (int, error) { + return PutVarUint64AtEnd(arr, EncodeZigZag(x)) +} + +func ReversePutVarInt64(arr []byte, x int64) (int, error) { + return ReversePutVarUint64(arr, EncodeZigZag(x)) +} + +type KeyComparator interface { + CompareTo(int) int +} + +func BinarySearch(qty int, keyComparator KeyComparator) (elemIdx int, isFound bool) { + if qty == 0 { + return + } + a := 0 + b := qty - 1 + for { + var ( + elemIdx = (b-a)/2 + a + code = keyComparator.CompareTo(elemIdx) + ) + + if code == 1 { + a = elemIdx + 1 + if a > b { + return elemIdx + 1, false + } + } else if code == -1 { + b = elemIdx - 1 + if b < a { + return elemIdx, false + } + } else { + return elemIdx, true + } + } +} + +func DeleteReverseArrElem(arr []byte, qty int, elemSize int, idx int) { + dstIdx := len(arr) - idx*elemSize - 1 + srcIdx := dstIdx - elemSize + + end := len(arr) - qty*elemSize + + for ; srcIdx >= end; srcIdx-- { + arr[dstIdx] = arr[srcIdx] + dstIdx-- + } + + for i := end; i < end+elemSize; i++ { + arr[i] = 0 + } +} + +// ZigZag + +// ZigZag encoding: int64 -> uint64 +func EncodeZigZag(x int64) uint64 { + return uint64(x<<1) ^ uint64(x>>63) +} + +// ZigZag decoding: uint64 -> int64 +func DecodeZigZag(u uint64) int64 { + return int64(u>>1) ^ -(int64(u & 1)) +} + +func ReadN(r io.Reader, n int) (_ []byte, err error) { + if n < 0 { + err = fmt.Errorf("wrong n=%d", n) + return + } + buf := make([]byte, n) + err = ReadNInto(r, buf) + if err != nil { + return + } + return buf, nil +} + +func ReadNInto(r io.Reader, buf []byte) (err error) { + if len(buf) == 0 { + return + } + + var q, total, readAttempts int + + for readAttempts < maxReadAttempts { + bufsize := len(buf) - total + q, err = r.Read(buf[total:]) + if q == bufsize { + return nil + } + if err != nil { + return + } + if q > bufsize { + err = ErrReadOverflow + return + } + if q < 0 { + err = ErrNegativeReadCount + return + } + if q == 0 { + readAttempts++ + } else { + total += q + } + } + err = io.ErrNoProgress + return +} + +func CalcVarUint64Length(num uint64) int { + for i := range 8 { + num >>= 7 + if num == 0 { + return i + 1 + } + } + return 9 +} + +func CalcVarInt64Length(num int64) int { + u64 := EncodeZigZag(num) + for i := range 8 { + u64 >>= 7 + if u64 == 0 { + return i + 1 + } + } + return 9 +} diff --git a/bufreader/bufreader.go b/bufreader/bufreader.go new file mode 100644 index 0000000..684a326 --- /dev/null +++ b/bufreader/bufreader.go @@ -0,0 +1,138 @@ +package bufreader + +import ( + "errors" + "io" +) + +const ( + maxReadAttempts = 5 + defaultBufSize = 1024 +) + +var ( + // ErrReadOverflow shows 100% bug in the source reader + ErrReadOverflow = errors.New("bufreader: reader returned 'n' > bufsize") + // ErrNegativeReadCount shows 100% bug in the source reader + ErrNegativeReadCount = errors.New("bufreader: reader returned negative 'n'") +) + +type BufferedReader struct { + r io.Reader + buf []byte + idx int + end int + totalRead int +} + +func New(r io.Reader, bufsize int) *BufferedReader { + if bufsize == 0 { + bufsize = defaultBufSize + } + return &BufferedReader{ + r: r, + buf: make([]byte, bufsize), + } +} + +func (s *BufferedReader) safeRead(buf []byte) (n int, err error) { + readAttempts := 0 + for readAttempts < maxReadAttempts { + n, err = s.r.Read(buf) + + if n > 0 { + if n > len(buf) { + return 0, ErrReadOverflow + } + if err == io.EOF { + err = nil + } + return + } + + if n < 0 { + return 0, ErrNegativeReadCount + } + + // n == 0 + if err != nil { + return + } + readAttempts++ + } + return 0, io.ErrNoProgress +} + +func (s *BufferedReader) fill() error { + n, err := s.safeRead(s.buf) + s.idx = 0 + s.end = n + return err +} + +func (s *BufferedReader) ReadByte() (b byte, err error) { + if s.idx == s.end { + if err = s.fill(); err != nil { + return + } + } + b = s.buf[s.idx] + s.idx++ + s.totalRead++ + return +} + +func (s *BufferedReader) Read(buf []byte) (int, error) { + size := len(buf) + + buffered := s.end - s.idx + if size <= buffered { + for i, b := range s.buf[s.idx : s.idx+size] { + buf[i] = b + } + s.idx += size + s.totalRead += len(buf) + return size, nil + } + for i, b := range s.buf[s.idx:s.end] { + buf[i] = b + } + s.idx = 0 + s.end = 0 + + n := buffered + rbuf := buf[buffered:] + + var ( + q int + err error + ) + for n < size { + q, err = s.safeRead(rbuf) + n += q + rbuf = rbuf[q:] + + if err != nil { + if err == io.EOF && n == size { + s.totalRead += len(buf) + return n, nil + } + break + } + } + s.totalRead += len(buf[:n]) + return n, err +} + +func (s *BufferedReader) ReadN(size int) ([]byte, error) { + buf := make([]byte, size) + _, err := s.Read(buf) + if err != nil { + return nil, err + } + return buf, nil +} + +func (s *BufferedReader) TotalRead() int { + return s.totalRead +} diff --git a/chunkenc/chunckenc.go b/chunkenc/chunckenc.go new file mode 100644 index 0000000..3973ca9 --- /dev/null +++ b/chunkenc/chunckenc.go @@ -0,0 +1,3 @@ +package chunkenc + +const eps = 0.000001 diff --git a/chunkenc/cumdelta.go b/chunkenc/cumdelta.go new file mode 100644 index 0000000..1afcc46 --- /dev/null +++ b/chunkenc/cumdelta.go @@ -0,0 +1,330 @@ +package chunkenc + +import ( + "fmt" + "math" + + "gordenko.dev/dima/diploma/bin" + "gordenko.dev/dima/diploma/conbuf" +) + +// REVERSE + +type ReverseCumulativeDeltaCompressor struct { + buf *conbuf.ContinuousBuffer + coef float64 + pos int + firstValue float64 + lastDelta uint64 + length uint16 + numIdx int +} + +func NewReverseCumulativeDeltaCompressor(buf *conbuf.ContinuousBuffer, size int, fracDigits byte) *ReverseCumulativeDeltaCompressor { + var coef float64 = 1 + if fracDigits > 0 { + coef = math.Pow(10, float64(fracDigits)) + } + s := &ReverseCumulativeDeltaCompressor{ + buf: buf, + pos: size, + coef: coef, + } + if size > 0 { + s.restoreState() + } + return s +} + +func (s *ReverseCumulativeDeltaCompressor) restoreState() { + u64, n, err := s.buf.GetVarUint64(0) + if err != nil { + panic(fmt.Sprintf("bug: get first value: %s", err)) + } + s.firstValue = float64(u64) / s.coef + + if s.pos > n { + pos := s.pos - 1 + idxOf8 := uint(8 - s.buf.GetByte(pos)) + pos-- + s8 := s.buf.GetByte(pos) + pos-- + + var n int + s.lastDelta, n, err = s.buf.ReverseGetVarUint64(pos) + if err != nil { + panic(fmt.Sprintf("bug: get last delta: %s", err)) + } + pos -= n + s.numIdx = pos + 1 + + var flag byte = 1 << idxOf8 + if (s8 & flag) == flag { + s.length, _ = s.buf.DecodeRunLength(pos) + } + } +} + +func (s *ReverseCumulativeDeltaCompressor) Size() int { + return s.pos +} + +func (s *ReverseCumulativeDeltaCompressor) CalcRequiredSpace(value float64) int { + if s.pos == 0 { + n := bin.CalcVarUint64Length(uint64(value * s.coef)) + return n + 3 + } + delta := uint64((value-s.firstValue)*s.coef + eps) + if delta == s.lastDelta { + if s.length == 0 { + return 1 + } else { + newLength := s.length + 1 + if newLength < 130 { + return 0 + } else if newLength == 130 { + return 1 + } else { + if newLength < 32769 { + return 0 + } else { + n := bin.CalcVarUint64Length(delta) + n += 2 + s8q := s.buf.GetByte(s.pos - 1) + if s8q == 8 { + n -= 1 + } else { + n -= 2 + } + return n + } + } + } + } else { + n := bin.CalcVarUint64Length(delta) + n += 2 + s8q := s.buf.GetByte(s.pos - 1) + if s8q == 8 { + n -= 1 + } else { + n -= 2 + } + return n + } +} + +func (s *ReverseCumulativeDeltaCompressor) Append(value float64) { + if s.pos == 0 { + n := s.buf.PutVarUint64(s.pos, uint64(value*s.coef)) + s.pos += n + s.firstValue = value + s.encodeNewDelta(0, 0, 1) + } else { + delta := uint64((value-s.firstValue)*s.coef + eps) + if delta == s.lastDelta { + if s.length == 0 { + s.length = 2 + s.shiftOnePosToRight() + s.buf.SetByte(s.numIdx-1, 0) + s8q := s.buf.GetByte(s.pos - 1) + s.buf.SetFlag(s.pos-2, 1<<(8-s8q)) + } else { + s.length++ + if s.length < 130 { + s.buf.SetByte(s.numIdx-1, byte(s.length-2)) + } else if s.length == 130 { + s.shiftOnePosToRight() + s.encode2bLength() + } else { + if s.length < 32769 { + s.encode2bLength() + } else { + s.appendNewDelta(delta) + } + } + } + } else { + s.appendNewDelta(delta) + } + } +} + +func (s *ReverseCumulativeDeltaCompressor) appendNewDelta(delta uint64) { + s.length = 0 + + s8 := s.buf.GetByte(s.pos - 2) + s8q := s.buf.GetByte(s.pos - 1) + + if s8q == 8 { + s.pos -= 1 + s8 = 0 + s8q = 1 + } else { + s.pos -= 2 + s8q++ + } + + s.encodeNewDelta(delta, s8, s8q) +} + +func (s *ReverseCumulativeDeltaCompressor) encodeNewDelta(delta uint64, s8 byte, s8q byte) { + s.lastDelta = delta + s.numIdx = s.pos + n := s.buf.ReversePutVarUint64(s.pos, s.lastDelta) + s.pos += n + s.buf.SetByte(s.pos, s8) + s.pos++ + s.buf.SetByte(s.pos, s8q) + s.pos++ +} + +func (s *ReverseCumulativeDeltaCompressor) shiftOnePosToRight() { + s.buf.ShiftOnePosToRight(s.numIdx, s.pos) + s.pos++ + s.numIdx++ +} + +func (s *ReverseCumulativeDeltaCompressor) encode2bLength() { + num := s.length - 2 + s.buf.SetByte(s.numIdx-1, byte(num&127)|128) + s.buf.SetByte(s.numIdx-2, byte(num>>7)) +} + +func (s *ReverseCumulativeDeltaCompressor) DeleteLast() { + var ( + s8q = s.buf.GetByte(s.pos - 1) + s8 = s.buf.GetByte(s.pos - 2) + flag byte = 1 << uint(8-s8q) + ) + + if s.length > 0 { + if s.length == 2 { + s.length = 0 + s.buf.UnsetFlag(s.pos-2, flag) + s.buf.ShiftOnePosToLeft(s.numIdx, s.pos) + s.numIdx-- + s.pos-- + } else if s.length < 130 { + s.length-- + s.buf.SetByte(s.numIdx-1, byte(s.length)-2) + } else if s.length == 130 { + s.length-- + s.buf.ShiftOnePosToLeft(s.numIdx, s.pos) + s.numIdx-- + s.pos-- + s.buf.SetByte(s.numIdx-1, byte(s.length)-2) + } else { + s.length-- + s.encode2bLength() + } + } else { + if s8q > 1 { + s8q-- + flag = 1 << uint(8-s8q) + s.pos = s.numIdx + 2 + s.buf.SetByte(s.pos-2, s8) + s.buf.SetByte(s.pos-1, s8q) + } else { + s.pos = s.numIdx + 1 + s.buf.SetByte(s.pos-1, 8) + s8 = s.buf.GetByte(s.pos - 2) + flag = 1 + } + var ( + pos = s.pos - 3 + n int + err error + ) + s.lastDelta, n, err = s.buf.ReverseGetVarUint64(pos) + if err != nil { + panic(err) + } + s.numIdx = pos - n + if (s8 & flag) == flag { + s.length, _ = s.buf.DecodeRunLength(s.numIdx - 1) + } + } +} + +type ReverseCumulativeDeltaDecompressor struct { + buf *conbuf.ContinuousBuffer + pos int + bound int + firstValue float64 + lastValue float64 + length uint16 + coef float64 + idxOf8 uint + s8 byte + step byte +} + +func NewReverseCumulativeDeltaDecompressor(buf *conbuf.ContinuousBuffer, size int, fracDigits byte) *ReverseCumulativeDeltaDecompressor { + var coef float64 = 1 + if fracDigits > 0 { + coef = math.Pow(10, float64(fracDigits)) + } + return &ReverseCumulativeDeltaDecompressor{ + buf: buf, + coef: coef, + pos: size, + } +} + +func (s *ReverseCumulativeDeltaDecompressor) NextValue() (value float64, done bool) { + if s.step > 0 { + if s.length > 0 { + s.length-- + return s.lastValue, false + } + if s.pos < s.bound { + return 0, true + } + if s.idxOf8 == 0 { + s.s8 = s.buf.GetByte(s.pos) + s.pos-- + } + s.readVar() + if s.length > 0 { + s.length-- + } + return s.lastValue, false + } + u64, n, err := s.buf.GetVarUint64(0) + if err != nil { + panic(err) + } + s.firstValue = float64(u64) / s.coef + s.bound = n + s.pos-- + s.idxOf8 = uint(8 - s.buf.GetByte(s.pos)) + s.pos-- + s.s8 = s.buf.GetByte(s.pos) + s.pos-- + s.readVar() + if s.length > 0 { + s.length-- + } + s.step = 1 + return s.lastValue, false +} + +func (s *ReverseCumulativeDeltaDecompressor) readVar() { + u64, n, err := s.buf.ReverseGetVarUint64(s.pos) + if err != nil { + panic(err) + } + s.pos -= n + s.lastValue = s.firstValue + float64(u64)/s.coef + + var flag byte = 1 << s.idxOf8 + if (s.s8 & flag) == flag { + s.length, n = s.buf.DecodeRunLength(s.pos) + s.pos -= n + } + if s.idxOf8 == 7 { + s.idxOf8 = 0 + } else { + s.idxOf8++ + } +} diff --git a/chunkenc/insdelta.go b/chunkenc/insdelta.go new file mode 100644 index 0000000..30c1903 --- /dev/null +++ b/chunkenc/insdelta.go @@ -0,0 +1,345 @@ +package chunkenc + +import ( + "fmt" + "math" + + "gordenko.dev/dima/diploma/bin" + "gordenko.dev/dima/diploma/conbuf" +) + +// REVERSE + +type ReverseInstantDeltaCompressor struct { + buf *conbuf.ContinuousBuffer + coef float64 + pos int + firstValue float64 + lastDelta int64 + length uint16 + numIdx int +} + +func NewReverseInstantDeltaCompressor(buf *conbuf.ContinuousBuffer, size int, fracDigits byte) *ReverseInstantDeltaCompressor { + var coef float64 = 1 + if fracDigits > 0 { + coef = math.Pow(10, float64(fracDigits)) + } + s := &ReverseInstantDeltaCompressor{ + buf: buf, + pos: size, + coef: coef, + } + if size > 0 { + s.restoreState() + } + return s +} + +func (s *ReverseInstantDeltaCompressor) restoreState() { + i64, n, err := s.buf.GetVarInt64(0) + if err != nil { + panic(fmt.Sprintf("bug: get first value: %s", err)) + } + s.firstValue = float64(i64) / s.coef + + if s.pos > n { + pos := s.pos - 1 + idxOf8 := uint(8 - s.buf.GetByte(pos)) + pos-- + s8 := s.buf.GetByte(pos) + pos-- + + var n int + s.lastDelta, n, err = s.buf.ReverseGetVarInt64(pos) + if err != nil { + panic(fmt.Sprintf("bug: get last delta: %s", err)) + } + pos -= n + s.numIdx = pos + 1 + var flag byte = 1 << idxOf8 + if (s8 & flag) == flag { + s.length, _ = s.buf.DecodeRunLength(pos) + } + } +} + +func (s *ReverseInstantDeltaCompressor) Size() int { + return s.pos +} + +func (s *ReverseInstantDeltaCompressor) CalcRequiredSpace(value float64) int { + if s.pos == 0 { + n := bin.CalcVarInt64Length(int64(value * s.coef)) + return n + 3 + } + + tmp := (value - s.firstValue) * s.coef + if tmp > 0 { + tmp += eps + } else { + tmp -= eps + } + delta := int64(tmp) + + if delta == s.lastDelta { + if s.length == 0 { + return 1 + } else { + newLength := s.length + 1 + if newLength < 130 { + return 0 + } else if newLength == 130 { + return 1 + } else { + if newLength < 32769 { + return 0 + } else { + n := bin.CalcVarInt64Length(delta) + n += 2 + s8q := s.buf.GetByte(s.pos - 1) + if s8q == 8 { + n -= 1 + } else { + n -= 2 + } + return n + } + } + } + } else { + n := bin.CalcVarInt64Length(delta) + n += 2 + s8q := s.buf.GetByte(s.pos - 1) + if s8q == 8 { + n -= 1 + } else { + n -= 2 + } + return n + } +} + +// В начале буфера кодирую базовое значение. +func (s *ReverseInstantDeltaCompressor) Append(value float64) { + if s.pos == 0 { + n := s.buf.PutVarInt64(s.pos, int64(value*s.coef)) + s.pos += n + s.firstValue = value + s.encodeNewDelta(0, 0, 1) + } else { + tmp := (value - s.firstValue) * s.coef + if tmp > 0 { + tmp += eps + } else { + tmp -= eps + } + delta := int64(tmp) + + if delta == s.lastDelta { + if s.length == 0 { + s.length = 2 + s.shiftOnePosToRight() + s.buf.SetByte(s.numIdx-1, 0) + s8q := s.buf.GetByte(s.pos - 1) + s.buf.SetFlag(s.pos-2, 1<<(8-s8q)) + } else { + s.length++ + if s.length < 130 { + s.buf.SetByte(s.numIdx-1, byte(s.length-2)) + } else if s.length == 130 { + s.shiftOnePosToRight() + s.encode2bLength() + } else { + if s.length < 32769 { + s.encode2bLength() + } else { + s.appendNewDelta(delta) + } + } + } + } else { + s.appendNewDelta(delta) + } + } +} + +func (s *ReverseInstantDeltaCompressor) appendNewDelta(delta int64) { + s.length = 0 + + s8 := s.buf.GetByte(s.pos - 2) + s8q := s.buf.GetByte(s.pos - 1) + + if s8q == 8 { + s.pos -= 1 + s8 = 0 + s8q = 1 + } else { + s.pos -= 2 + s8q++ + } + s.encodeNewDelta(delta, s8, s8q) +} + +func (s *ReverseInstantDeltaCompressor) encodeNewDelta(delta int64, s8 byte, s8q byte) { + s.lastDelta = delta + s.numIdx = s.pos + n := s.buf.ReversePutVarInt64(s.pos, s.lastDelta) + s.pos += n + s.buf.SetByte(s.pos, s8) + s.pos++ + s.buf.SetByte(s.pos, s8q) + s.pos++ +} + +func (s *ReverseInstantDeltaCompressor) shiftOnePosToRight() { + s.buf.ShiftOnePosToRight(s.numIdx, s.pos) + s.pos++ + s.numIdx++ +} + +func (s *ReverseInstantDeltaCompressor) encode2bLength() { + num := s.length - 2 + s.buf.SetByte(s.numIdx-1, byte(num&127)|128) + s.buf.SetByte(s.numIdx-2, byte(num>>7)) +} + +func (s *ReverseInstantDeltaCompressor) DeleteLast() { + var ( + s8q = s.buf.GetByte(s.pos - 1) + s8 = s.buf.GetByte(s.pos - 2) + flag byte = 1 << uint(8-s8q) + ) + + if s.length > 0 { + if s.length == 2 { + s.length = 0 + s.buf.UnsetFlag(s.pos-2, flag) + s.buf.ShiftOnePosToLeft(s.numIdx, s.pos) + s.numIdx-- + s.pos-- + } else if s.length < 130 { + s.length-- + s.buf.SetByte(s.numIdx-1, byte(s.length)-2) + } else if s.length == 130 { + s.length-- + s.buf.ShiftOnePosToLeft(s.numIdx, s.pos) + s.numIdx-- + s.pos-- + s.buf.SetByte(s.numIdx-1, byte(s.length)-2) + } else { + s.length-- + s.encode2bLength() + } + } else { + if s8q > 1 { + s8q-- + flag = 1 << uint(8-s8q) + s.pos = s.numIdx + 2 + s.buf.SetByte(s.pos-2, s8) + s.buf.SetByte(s.pos-1, s8q) + } else { + s.pos = s.numIdx + 1 + s.buf.SetByte(s.pos-1, 8) + s8 = s.buf.GetByte(s.pos - 2) + flag = 1 + } + var ( + pos = s.pos - 3 + n int + err error + ) + s.lastDelta, n, err = s.buf.ReverseGetVarInt64(pos) + if err != nil { + panic(err) + } + s.numIdx = pos - n + if (s8 & flag) == flag { + s.length, _ = s.buf.DecodeRunLength(s.numIdx - 1) + } + } +} + +type ReverseInstantDeltaDecompressor struct { + step byte + buf *conbuf.ContinuousBuffer + pos int + bound int + firstValue float64 + lastValue float64 + length uint16 + coef float64 + idxOf8 uint + s8 byte +} + +func NewReverseInstantDeltaDecompressor(buf *conbuf.ContinuousBuffer, size int, fracDigits byte) *ReverseInstantDeltaDecompressor { + var coef float64 = 1 + if fracDigits > 0 { + coef = math.Pow(10, float64(fracDigits)) + } + return &ReverseInstantDeltaDecompressor{ + buf: buf, + coef: coef, + pos: size, + } +} + +func (s *ReverseInstantDeltaDecompressor) NextValue() (value float64, done bool) { + if s.step > 0 { + if s.length > 0 { + s.length-- + return s.lastValue, false + } + if s.pos < s.bound { + return 0, true + } + + if s.idxOf8 == 0 { + s.s8 = s.buf.GetByte(s.pos) + s.pos-- + } + s.readVar() + if s.length > 0 { + s.length-- + } + return s.lastValue, false + } + i64, n, err := s.buf.GetVarInt64(0) + if err != nil { + panic(err) + } + s.firstValue = float64(i64) / s.coef + s.bound = n + s.pos-- + s.idxOf8 = uint(8 - s.buf.GetByte(s.pos)) + s.pos-- + s.s8 = s.buf.GetByte(s.pos) + s.pos-- + s.readVar() + if s.length > 0 { + s.length-- + } + s.step = 1 + return s.lastValue, false +} + +func (s *ReverseInstantDeltaDecompressor) readVar() { + i64, n, err := s.buf.ReverseGetVarInt64(s.pos) + if err != nil { + panic(err) + } + s.pos -= n + s.lastValue = s.firstValue + float64(i64)/s.coef + + var flag byte = 1 << s.idxOf8 + if (s.s8 & flag) == flag { + s.length, n = s.buf.DecodeRunLength(s.pos) + s.pos -= n + } + if s.idxOf8 == 7 { + s.idxOf8 = 0 + } else { + s.idxOf8++ + } +} diff --git a/chunkenc/time_delta_delta.go b/chunkenc/time_delta_delta.go new file mode 100644 index 0000000..c458408 --- /dev/null +++ b/chunkenc/time_delta_delta.go @@ -0,0 +1,374 @@ +package chunkenc + +import ( + "fmt" + + "gordenko.dev/dima/diploma/bin" + "gordenko.dev/dima/diploma/conbuf" +) + +// REVERSE + +const ( + lastUnixtimeIdx = 0 + baseDeltaIdx = 4 +) + +type ReverseTimeDeltaOfDeltaCompressor struct { + buf *conbuf.ContinuousBuffer + pos int + baseDelta uint32 + lastUnixtime uint32 + lastDeltaOfDelta int64 + length uint16 + numIdx int +} + +func NewReverseTimeDeltaOfDeltaCompressor(buf *conbuf.ContinuousBuffer, size int) *ReverseTimeDeltaOfDeltaCompressor { + s := &ReverseTimeDeltaOfDeltaCompressor{ + buf: buf, + pos: size, + } + if size > 0 { + s.restoreState() + } + return s +} + +func (s *ReverseTimeDeltaOfDeltaCompressor) restoreState() { + s.lastUnixtime = s.buf.GetUint32(lastUnixtimeIdx) + if s.pos > 4 { + u64, _, err := s.buf.GetVarUint64(baseDeltaIdx) + if err != nil { + panic(fmt.Sprintf("bug: get base delta: %s", err)) + } + s.baseDelta = uint32(u64) + + pos := s.pos - 1 + idxOf8 := uint(8 - s.buf.GetByte(pos)) + pos-- + s8 := s.buf.GetByte(pos) + pos-- + + var n int + s.lastDeltaOfDelta, n, err = s.buf.ReverseGetVarInt64(pos) + if err != nil { + panic(err) + } + pos -= n + s.numIdx = pos + 1 + var flag byte = 1 << idxOf8 + if (s8 & flag) == flag { + s.length, _ = s.buf.DecodeRunLength(pos) + } + } +} + +func (s *ReverseTimeDeltaOfDeltaCompressor) Size() int { + return s.pos +} + +func (s *ReverseTimeDeltaOfDeltaCompressor) CalcRequiredSpace(unixtime uint32) int { + if s.pos == 0 { + return 4 + } + + if s.baseDelta == 0 { + baseDelta := unixtime - s.lastUnixtime + n := bin.CalcVarUint64Length(uint64(baseDelta)) + return n + 3 + } + + deltaOfDelta := int64(unixtime-s.lastUnixtime) - int64(s.baseDelta) + if deltaOfDelta == s.lastDeltaOfDelta { + if s.length == 0 { + return 1 + } else { + newLength := s.length + 1 + if newLength < 130 { + return 0 + } else if newLength == 130 { + return 1 + } else { + if newLength < 32769 { + return 0 + } else { + n := bin.CalcVarInt64Length(deltaOfDelta) + n += 2 + s8q := s.buf.GetByte(s.pos - 1) + if s8q == 8 { + n -= 1 + } else { + n -= 2 + } + return n + } + } + } + } else { + n := bin.CalcVarInt64Length(deltaOfDelta) + n += 2 + s8q := s.buf.GetByte(s.pos - 1) + if s8q == 8 { + n -= 1 + } else { + n -= 2 + } + return n + } +} + +func (s *ReverseTimeDeltaOfDeltaCompressor) Append(unixtime uint32) { + if s.pos == 0 { + s.lastUnixtime = unixtime + s.buf.PutUint32(lastUnixtimeIdx, unixtime) + s.pos += 4 + return + } + + if s.baseDelta == 0 { + s.baseDelta = unixtime - s.lastUnixtime + s.lastDeltaOfDelta = 0 + s.lastUnixtime = unixtime + s.buf.PutUint32(lastUnixtimeIdx, unixtime) + + n := s.buf.PutVarUint64(s.pos, uint64(s.baseDelta)) + s.pos += n + s.encodeNewDeltaOfDelta(0, 0, 1) + return + } + + deltaOfDelta := int64(unixtime-s.lastUnixtime) - int64(s.baseDelta) + s.lastUnixtime = unixtime + s.buf.PutUint32(lastUnixtimeIdx, unixtime) + + if deltaOfDelta == s.lastDeltaOfDelta { + if s.length == 0 { + s.length = 2 + s.shiftOnePosToRight() + s.buf.SetByte(s.numIdx-1, 0) + s8q := s.buf.GetByte(s.pos - 1) + s.buf.SetFlag(s.pos-2, 1<<(8-s8q)) + } else { + s.length++ + if s.length < 130 { + s.buf.SetByte(s.numIdx-1, byte(s.length-2)) + } else if s.length == 130 { + s.shiftOnePosToRight() + s.encode2bLength() + } else { + if s.length < 32769 { + s.encode2bLength() + } else { + s.appendNewDeltaOfDelta(deltaOfDelta) + } + } + } + } else { + s.appendNewDeltaOfDelta(deltaOfDelta) + } +} + +func (s *ReverseTimeDeltaOfDeltaCompressor) appendNewDeltaOfDelta(deltaOfDelta int64) { + s.length = 0 + s8 := s.buf.GetByte(s.pos - 2) + s8q := s.buf.GetByte(s.pos - 1) + + if s8q == 8 { + s.pos -= 1 + s8 = 0 + s8q = 1 + } else { + s.pos -= 2 + s8q++ + } + + s.encodeNewDeltaOfDelta(deltaOfDelta, s8, s8q) +} + +func (s *ReverseTimeDeltaOfDeltaCompressor) encodeNewDeltaOfDelta(deltaOfDelta int64, s8 byte, s8q byte) { + s.lastDeltaOfDelta = deltaOfDelta + s.numIdx = s.pos + n := s.buf.ReversePutVarInt64(s.pos, deltaOfDelta) + s.pos += n + s.buf.SetByte(s.pos, s8) + s.pos++ + s.buf.SetByte(s.pos, s8q) + s.pos++ +} + +func (s *ReverseTimeDeltaOfDeltaCompressor) shiftOnePosToRight() { + s.buf.ShiftOnePosToRight(s.numIdx, s.pos) + s.pos++ + s.numIdx++ +} + +func (s *ReverseTimeDeltaOfDeltaCompressor) encode2bLength() { + num := s.length - 2 + s.buf.SetByte(s.numIdx-1, byte(num&127)|128) + s.buf.SetByte(s.numIdx-2, byte(num>>7)) +} + +func (s *ReverseTimeDeltaOfDeltaCompressor) DeleteLast() { + var ( + s8q = s.buf.GetByte(s.pos - 1) + s8 = s.buf.GetByte(s.pos - 2) + flag byte = 1 << uint(8-s8q) + ) + + if s.length > 0 { + if s.length == 2 { + s.length = 0 + s.buf.UnsetFlag(s.pos-2, flag) + s.buf.ShiftOnePosToLeft(s.numIdx, s.pos) + s.numIdx-- + s.pos-- + } else if s.length < 130 { + s.length-- + s.buf.SetByte(s.numIdx-1, byte(s.length)-2) + } else if s.length == 130 { + s.length-- + s.buf.ShiftOnePosToLeft(s.numIdx, s.pos) + s.numIdx-- + s.pos-- + s.buf.SetByte(s.numIdx-1, byte(s.length)-2) + } else { + s.length-- + s.encode2bLength() + } + } else { + if s8q > 1 { + s8q-- + flag = 1 << uint(8-s8q) + s.pos = s.numIdx + 2 + s.buf.SetByte(s.pos-2, s8) + s.buf.SetByte(s.pos-1, s8q) + } else { + s.pos = s.numIdx + 1 + s.buf.SetByte(s.pos-1, 8) + s8 = s.buf.GetByte(s.pos - 2) + flag = 1 + } + var ( + pos = s.pos - 3 + n int + err error + ) + s.lastDeltaOfDelta, n, err = s.buf.ReverseGetVarInt64(pos) + if err != nil { + panic(err) + } + s.numIdx = pos - n + if (s8 & flag) == flag { + s.length, _ = s.buf.DecodeRunLength(s.numIdx - 1) + } + } + + delta := int64(s.baseDelta) + s.lastDeltaOfDelta + s.lastUnixtime = uint32(int64(s.lastUnixtime) - delta) + s.buf.PutUint32(lastUnixtimeIdx, s.lastUnixtime) +} + +type ReverseTimeDeltaOfDeltaDecompressor struct { + step byte + buf *conbuf.ContinuousBuffer + pos int + bound int + lastUnixtime uint32 + baseDelta uint32 + lastDeltaOfDelta int64 + length uint16 + idxOf8 uint + s8 byte +} + +func NewReverseTimeDeltaOfDeltaDecompressor(buf *conbuf.ContinuousBuffer, size int) *ReverseTimeDeltaOfDeltaDecompressor { + return &ReverseTimeDeltaOfDeltaDecompressor{ + buf: buf, + pos: size, + } +} + +func (s *ReverseTimeDeltaOfDeltaDecompressor) NextValue() (value uint32, done bool) { + if s.step == 0 { + if s.pos == 0 { + return 0, true + } + s.lastUnixtime = s.buf.GetUint32(lastUnixtimeIdx) + s.step = 1 + return s.lastUnixtime, false + } + + if s.step == 1 { + if s.pos == baseDeltaIdx { + return 0, true + } + u64, n, err := s.buf.GetVarUint64(baseDeltaIdx) + if err != nil { + panic("EOF") + } + s.bound = baseDeltaIdx + n + s.baseDelta = uint32(u64) + + s.pos-- + s.idxOf8 = uint(8 - s.buf.GetByte(s.pos)) + s.pos-- + s.s8 = s.buf.GetByte(s.pos) + s.pos-- + + s.readVar() + if s.length > 0 { + s.length-- + } + s.step = 2 + return s.lastUnixtime, false + } + + if s.length > 0 { + s.length-- + delta := int64(s.baseDelta) + s.lastDeltaOfDelta + s.lastUnixtime = uint32(int64(s.lastUnixtime) - delta) + return s.lastUnixtime, false + } + + if s.pos < s.bound { + return 0, true + } + + if s.idxOf8 == 0 { + s.s8 = s.buf.GetByte(s.pos) + s.pos-- + } + s.readVar() + if s.length > 0 { + s.length-- + } + return s.lastUnixtime, false +} + +func (s *ReverseTimeDeltaOfDeltaDecompressor) readVar() { + var ( + n int + err error + ) + + s.lastDeltaOfDelta, n, err = s.buf.ReverseGetVarInt64(s.pos) + if err != nil { + panic(err) + } + s.pos -= n + + delta := int64(s.baseDelta) + s.lastDeltaOfDelta + s.lastUnixtime = uint32(int64(s.lastUnixtime) - delta) + + var flag byte = 1 << s.idxOf8 + if (s.s8 & flag) == flag { + s.length, n = s.buf.DecodeRunLength(s.pos) + s.pos -= n + } + if s.idxOf8 == 7 { + s.idxOf8 = 0 + } else { + s.idxOf8++ + } +} diff --git a/client/client.go b/client/client.go new file mode 100644 index 0000000..ee6fa1f --- /dev/null +++ b/client/client.go @@ -0,0 +1,755 @@ +package client + +import ( + "fmt" + "net" + + "gordenko.dev/dima/diploma" + "gordenko.dev/dima/diploma/bin" + "gordenko.dev/dima/diploma/bufreader" + "gordenko.dev/dima/diploma/proto" +) + +const ( + metricKeySize = 4 +) + +type Error struct { + Code uint16 + Message string +} + +func (s Error) Error() string { + return fmt.Sprintf("%d: %s", s.Code, s.Message) +} + +type Connection struct { + conn net.Conn + src *bufreader.BufferedReader +} + +func Connect(address string) (*Connection, error) { + conn, err := net.Dial("tcp", address) + if err != nil { + return nil, err + } + + return &Connection{ + conn: conn, + src: bufreader.New(conn, 1500), + }, nil +} + +func (s *Connection) String() string { + return s.conn.LocalAddr().String() +} + +func (s *Connection) Close() { + s.conn.Close() +} + +func (s *Connection) mustSuccess(reader *bufreader.BufferedReader) (err error) { + code, err := reader.ReadByte() + if err != nil { + return fmt.Errorf("read response code: %s", err) + } + + switch code { + case proto.RespSuccess: + return nil // ok + + case proto.RespError: + return s.onError() + + default: + return fmt.Errorf("unknown reponse code %d", code) + } +} + +type Metric struct { + MetricID uint32 + MetricType diploma.MetricType + FracDigits byte +} + +func (s *Connection) AddMetric(req Metric) error { + arr := []byte{ + proto.TypeAddMetric, + 0, 0, 0, 0, // + byte(req.MetricType), + byte(req.FracDigits), + } + bin.PutUint32(arr[1:], req.MetricID) + + if _, err := s.conn.Write(arr); err != nil { + return err + } + return s.mustSuccess(s.src) +} + +func (s *Connection) GetMetric(metricID uint32) (*Metric, error) { + arr := []byte{ + proto.TypeGetMetric, + 0, 0, 0, 0, + } + bin.PutUint32(arr[1:], metricID) + + if _, err := s.conn.Write(arr); err != nil { + return nil, err + } + + code, err := s.src.ReadByte() + if err != nil { + return nil, fmt.Errorf("read response code: %s", err) + } + + switch code { + case proto.RespValue: + arr, err := s.src.ReadN(6) + if err != nil { + return nil, fmt.Errorf("read body: %s", err) + } + + return &Metric{ + MetricID: bin.GetUint32(arr), + MetricType: diploma.MetricType(arr[4]), + FracDigits: arr[5], + }, nil + + case proto.RespError: + return nil, s.onMaybeError() + + default: + return nil, fmt.Errorf("unknown reponse code %d", code) + } +} + +func (s *Connection) DeleteMetric(metricID uint32) error { + arr := []byte{ + proto.TypeDeleteMetric, + 0, 0, 0, 0, // + } + bin.PutUint32(arr[1:], metricID) + + if _, err := s.conn.Write(arr); err != nil { + return err + } + return s.mustSuccess(s.src) +} + +type AppendMeasureReq struct { + MetricID uint32 + Timestamp uint32 + Value float64 +} + +func (s *Connection) AppendMeasure(req AppendMeasureReq) (err error) { + arr := []byte{ + proto.TypeAppendMeasure, + 0, 0, 0, 0, // metricID + 0, 0, 0, 0, // timestamp + 0, 0, 0, 0, 0, 0, 0, 0, // value + } + bin.PutUint32(arr[1:], req.MetricID) + bin.PutUint32(arr[5:], req.Timestamp) + bin.PutFloat64(arr[9:], req.Value) + + if _, err := s.conn.Write(arr); err != nil { + return err + } + return s.mustSuccess(s.src) +} + +type AppendMeasuresReq struct { + MetricID uint32 + Measures []Measure +} + +type Measure struct { + Timestamp uint32 + Value float64 +} + +func (s *Connection) AppendMeasures(req AppendMeasuresReq) (err error) { + if len(req.Measures) > 65535 { + return fmt.Errorf("wrong measures qty: %d", len(req.Measures)) + } + var ( + prefixSize = 7 + recordSize = 12 + arr = make([]byte, prefixSize+len(req.Measures)*recordSize) + ) + arr[0] = proto.TypeAppendMeasures + bin.PutUint32(arr[1:], req.MetricID) + bin.PutUint16(arr[5:], uint16(len(req.Measures))) + pos := prefixSize + for _, measure := range req.Measures { + bin.PutUint32(arr[pos:], measure.Timestamp) + bin.PutFloat64(arr[pos+4:], measure.Value) + pos += recordSize + } + if _, err := s.conn.Write(arr); err != nil { + return err + } + return s.mustSuccess(s.src) +} + +type InstantMeasure struct { + Timestamp uint32 + Value float64 +} + +func (s *Connection) ListAllInstantMeasures(metricID uint32) ([]InstantMeasure, error) { + arr := []byte{ + proto.TypeListAllInstantMeasures, + 0, 0, 0, 0, // metricID + } + bin.PutUint32(arr[1:], metricID) + + if _, err := s.conn.Write(arr); err != nil { + return nil, err + } + + var ( + result []InstantMeasure + tmp = make([]byte, 12) + ) + + for { + code, err := s.src.ReadByte() + if err != nil { + return nil, fmt.Errorf("read response code: %s", err) + } + + switch code { + case proto.RespPartOfValue: + q, err := bin.ReadUint32(s.src) + if err != nil { + return nil, fmt.Errorf("read records qty: %s", err) + } + + for i := range int(q) { + err = bin.ReadNInto(s.src, tmp) + if err != nil { + return nil, fmt.Errorf("read record #%d: %s", i, err) + } + + result = append(result, InstantMeasure{ + Timestamp: bin.GetUint32(tmp), + Value: bin.GetFloat64(tmp[4:]), + }) + } + + case proto.RespEndOfValue: + return result, nil + + case proto.RespError: + return nil, s.onError() + + default: + return nil, fmt.Errorf("unknown reponse code %d", code) + } + } +} + +func (s *Connection) ListInstantMeasures(req proto.ListInstantMeasuresReq) ([]InstantMeasure, error) { + arr := []byte{ + proto.TypeListInstantMeasures, + 0, 0, 0, 0, // metricID + 0, 0, 0, 0, // since + 0, 0, 0, 0, // until + byte(req.FirstHourOfDay), + } + bin.PutUint32(arr[1:], req.MetricID) + bin.PutUint32(arr[5:], req.Since) + bin.PutUint32(arr[9:], req.Until) + + if _, err := s.conn.Write(arr); err != nil { + return nil, err + } + + var ( + result []InstantMeasure + tmp = make([]byte, 12) + ) + + for { + code, err := s.src.ReadByte() + if err != nil { + return nil, fmt.Errorf("read response code: %s", err) + } + + switch code { + case proto.RespPartOfValue: + q, err := bin.ReadUint32(s.src) + if err != nil { + return nil, fmt.Errorf("read records qty: %s", err) + } + + for i := range int(q) { + err = bin.ReadNInto(s.src, tmp) + if err != nil { + return nil, fmt.Errorf("read record #%d: %s", i, err) + } + + result = append(result, InstantMeasure{ + Timestamp: bin.GetUint32(tmp), + Value: bin.GetFloat64(tmp[4:]), + }) + } + + case proto.RespEndOfValue: + return result, nil + + case proto.RespError: + return nil, s.onError() + + default: + return nil, fmt.Errorf("unknown reponse code %d", code) + } + } +} + +type CumulativeMeasure struct { + Timestamp uint32 + Value float64 + Total float64 +} + +func (s *Connection) ListAllCumulativeMeasures(metricID uint32) ([]CumulativeMeasure, error) { + arr := []byte{ + proto.TypeListAllCumulativeMeasures, + 0, 0, 0, 0, // metricID + } + bin.PutUint32(arr[1:], metricID) + + if _, err := s.conn.Write(arr); err != nil { + return nil, err + } + + var ( + result []CumulativeMeasure + tmp = make([]byte, 20) + ) + + for { + code, err := s.src.ReadByte() + if err != nil { + return nil, fmt.Errorf("read response code: %s", err) + } + + switch code { + case proto.RespPartOfValue: + q, err := bin.ReadUint32(s.src) + if err != nil { + return nil, fmt.Errorf("read records qty: %s", err) + } + + for i := range int(q) { + err = bin.ReadNInto(s.src, tmp) + if err != nil { + return nil, fmt.Errorf("read record #%d: %s", i, err) + } + + result = append(result, CumulativeMeasure{ + Timestamp: bin.GetUint32(tmp), + Value: bin.GetFloat64(tmp[4:]), + Total: bin.GetFloat64(tmp[12:]), + }) + } + + case proto.RespEndOfValue: + return result, nil + + case proto.RespError: + return nil, s.onError() + + default: + return nil, fmt.Errorf("unknown reponse code %d", code) + } + } +} + +func (s *Connection) ListCumulativeMeasures(req proto.ListCumulativeMeasuresReq) ([]CumulativeMeasure, error) { + arr := []byte{ + proto.TypeListCumulativeMeasures, + 0, 0, 0, 0, // metricID + 0, 0, 0, 0, // since + 0, 0, 0, 0, // until + byte(req.FirstHourOfDay), + } + bin.PutUint32(arr[1:], req.MetricID) + bin.PutUint32(arr[5:], req.Since) + bin.PutUint32(arr[9:], req.Until) + + if _, err := s.conn.Write(arr); err != nil { + return nil, err + } + + var ( + result []CumulativeMeasure + tmp = make([]byte, 20) + ) + + for { + code, err := s.src.ReadByte() + if err != nil { + return nil, fmt.Errorf("read response code: %s", err) + } + + switch code { + case proto.RespPartOfValue: + q, err := bin.ReadUint32(s.src) + if err != nil { + return nil, fmt.Errorf("read records qty: %s", err) + } + + for i := range int(q) { + err = bin.ReadNInto(s.src, tmp) + if err != nil { + return nil, fmt.Errorf("read record #%d: %s", i, err) + } + + result = append(result, CumulativeMeasure{ + Timestamp: bin.GetUint32(tmp), + Value: bin.GetFloat64(tmp[4:]), + Total: bin.GetFloat64(tmp[12:]), + }) + } + + case proto.RespEndOfValue: + return result, nil + + case proto.RespError: + return nil, s.onError() + + default: + return nil, fmt.Errorf("unknown reponse code %d", code) + } + } +} + +type InstantPeriod struct { + Period uint32 + Since uint32 + Until uint32 + Min float64 + Max float64 + Avg float64 +} + +func (s *Connection) ListInstantPeriods(req proto.ListInstantPeriodsReq) ([]InstantPeriod, error) { + arr := []byte{ + proto.TypeListInstantPeriods, + 0, 0, 0, 0, // metricID + 0, 0, 0, 0, // since + 0, 0, 0, 0, // until + byte(req.GroupBy), + req.AggregateFuncs, + byte(req.FirstHourOfDay), + byte(req.LastDayOfMonth), + } + bin.PutUint32(arr[1:], req.MetricID) + bin.PutUint32(arr[5:], req.Since) + bin.PutUint32(arr[9:], req.Until) + + if _, err := s.conn.Write(arr); err != nil { + return nil, err + } + + var q int + if (req.AggregateFuncs & diploma.AggregateMin) == diploma.AggregateMin { + q++ + } + if (req.AggregateFuncs & diploma.AggregateMax) == diploma.AggregateMax { + q++ + } + if (req.AggregateFuncs & diploma.AggregateAvg) == diploma.AggregateAvg { + q++ + } + + var ( + result []InstantPeriod + // 12 bytes - period, since, until + // q * 8 bytes - min, max, avg + tmp = make([]byte, 12+q*8) + ) + + for { + code, err := s.src.ReadByte() + if err != nil { + return nil, fmt.Errorf("read response code: %s", err) + } + + switch code { + case proto.RespPartOfValue: + q, err := bin.ReadUint32(s.src) + if err != nil { + return nil, fmt.Errorf("read records qty: %s", err) + } + + for i := range int(q) { + err = bin.ReadNInto(s.src, tmp) + if err != nil { + return nil, fmt.Errorf("read record #%d: %s", i, err) + } + + var ( + p = InstantPeriod{ + Period: bin.GetUint32(tmp[0:]), + Since: bin.GetUint32(tmp[4:]), + Until: bin.GetUint32(tmp[8:]), + } + // 12 bytes - period, since, until + pos = 12 + ) + + if (req.AggregateFuncs & diploma.AggregateMin) == diploma.AggregateMin { + p.Min = bin.GetFloat64(tmp[pos:]) + pos += 8 + } + if (req.AggregateFuncs & diploma.AggregateMax) == diploma.AggregateMax { + p.Max = bin.GetFloat64(tmp[pos:]) + pos += 8 + } + if (req.AggregateFuncs & diploma.AggregateAvg) == diploma.AggregateAvg { + p.Avg = bin.GetFloat64(tmp[pos:]) + } + result = append(result, p) + } + + case proto.RespEndOfValue: + return result, nil + + case proto.RespError: + return nil, s.onError() + + default: + return nil, fmt.Errorf("unknown reponse code %d", code) + } + } +} + +type CumulativePeriod struct { + Period uint32 + Since uint32 + Until uint32 + EndValue float64 + Total float64 +} + +func (s *Connection) ListCumulativePeriods(req proto.ListCumulativePeriodsReq) ([]CumulativePeriod, error) { + arr := []byte{ + proto.TypeListCumulativePeriods, + 0, 0, 0, 0, // metricID + 0, 0, 0, 0, // since + 0, 0, 0, 0, // until + byte(req.GroupBy), + byte(req.FirstHourOfDay), + byte(req.LastDayOfMonth), + } + bin.PutUint32(arr[1:], req.MetricID) + bin.PutUint32(arr[5:], req.Since) + bin.PutUint32(arr[9:], req.Until) + + if _, err := s.conn.Write(arr); err != nil { + return nil, err + } + + var ( + result []CumulativePeriod + tmp = make([]byte, 28) + ) + + for { + code, err := s.src.ReadByte() + if err != nil { + return nil, fmt.Errorf("read response code: %s", err) + } + + switch code { + case proto.RespPartOfValue: + q, err := bin.ReadUint32(s.src) + if err != nil { + return nil, fmt.Errorf("read records qty: %s", err) + } + + for i := range int(q) { + err = bin.ReadNInto(s.src, tmp) + if err != nil { + return nil, fmt.Errorf("read record #%d: %s", i, err) + } + result = append(result, CumulativePeriod{ + Period: bin.GetUint32(tmp[0:]), + Since: bin.GetUint32(tmp[4:]), + Until: bin.GetUint32(tmp[8:]), + EndValue: bin.GetFloat64(tmp[12:]), + Total: bin.GetFloat64(tmp[20:]), + }) + } + + case proto.RespEndOfValue: + return result, nil + + case proto.RespError: + return nil, s.onError() + + default: + return nil, fmt.Errorf("unknown reponse code %d", code) + } + } +} + +type CurrentValue struct { + MetricID uint32 + Timestamp uint32 + Value float64 +} + +func (s *Connection) ListCurrentValues(metricIDs []uint32) ([]CurrentValue, error) { + arr := make([]byte, 3+metricKeySize*len(metricIDs)) + arr[0] = proto.TypeListCurrentValues + + bin.PutUint16(arr[1:], uint16(len(metricIDs))) + + off := 3 + for _, metricID := range metricIDs { + bin.PutUint32(arr[off:], metricID) + off += metricKeySize + } + + if _, err := s.conn.Write(arr); err != nil { + return nil, err + } + + var ( + result []CurrentValue + tmp = make([]byte, 16) + ) + + for { + code, err := s.src.ReadByte() + if err != nil { + return nil, fmt.Errorf("read response code: %s", err) + } + + switch code { + case proto.RespPartOfValue: + q, err := bin.ReadUint32(s.src) + if err != nil { + return nil, fmt.Errorf("read records qty: %s", err) + } + + for i := range int(q) { + err = bin.ReadNInto(s.src, tmp) + if err != nil { + return nil, fmt.Errorf("read record #%d: %s", i, err) + } + + result = append(result, CurrentValue{ + MetricID: bin.GetUint32(tmp), + Timestamp: bin.GetUint32(tmp[4:]), + Value: bin.GetFloat64(tmp[8:]), + }) + } + + case proto.RespEndOfValue: + return result, nil + + case proto.RespError: + return nil, s.onError() + + default: + return nil, fmt.Errorf("unknown reponse code %d", code) + } + } +} + +func (s *Connection) DeleteMeasures(req proto.DeleteMeasuresReq) (err error) { + arr := []byte{ + proto.TypeDeleteMeasures, + 0, 0, 0, 0, // metricID + 0, 0, 0, 0, // since + } + bin.PutUint32(arr[1:], req.MetricID) + bin.PutUint32(arr[5:], req.Since) + + if _, err := s.conn.Write(arr); err != nil { + return err + } + return s.mustSuccess(s.src) +} + +type RangeTotalResp struct { + Since uint32 + SinceValue float64 + Until uint32 + UntilValue float64 +} + +func (s *Connection) RangeTotal(req proto.RangeTotalReq) (*RangeTotalResp, error) { + arr := []byte{ + proto.TypeGetMetric, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + } + bin.PutUint32(arr[1:], req.MetricID) + bin.PutUint32(arr[5:], req.Since) + bin.PutUint32(arr[9:], req.MetricID) + + if _, err := s.conn.Write(arr); err != nil { + return nil, err + } + + code, err := s.src.ReadByte() + if err != nil { + return nil, fmt.Errorf("read response code: %s", err) + } + + switch code { + case proto.RespValue: + arr, err := s.src.ReadN(24) + if err != nil { + return nil, fmt.Errorf("read body: %s", err) + } + + return &RangeTotalResp{ + Since: bin.GetUint32(arr), + SinceValue: bin.GetFloat64(arr[4:]), + Until: bin.GetUint32(arr[12:]), + UntilValue: bin.GetFloat64(arr[16:]), + }, nil + + case proto.RespError: + return nil, s.onError() + + default: + return nil, fmt.Errorf("unknown reponse code %d", code) + } +} + +func (s *Connection) onError() error { + errorCode, err := bin.ReadUint16(s.src) + if err != nil { + return fmt.Errorf("read error code: %s", err) + } + return Error{ + Code: errorCode, + Message: proto.ErrorCodeToText(errorCode), + } +} + +func (s *Connection) onMaybeError() error { + errorCode, err := bin.ReadUint16(s.src) + if err != nil { + return fmt.Errorf("read error code: %s", err) + } + if errorCode == proto.ErrNoMetric { + return nil + } + return Error{ + Code: errorCode, + Message: proto.ErrorCodeToText(errorCode), + } +} diff --git a/conbuf/conbuf.go b/conbuf/conbuf.go new file mode 100644 index 0000000..d6f85f2 --- /dev/null +++ b/conbuf/conbuf.go @@ -0,0 +1,459 @@ +package conbuf + +import ( + "errors" + "fmt" + "io" + + "gordenko.dev/dima/diploma/bin" +) + +const chunkSize = 512 + +var ( + ErrOutOfRange = errors.New("out of range") +) + +type ContinuousBuffer struct { + chunks [][]byte +} + +func NewFromBuffer(buf []byte) *ContinuousBuffer { + var ( + chunks [][]byte + copied = 0 + ) + + for copied < len(buf) { + chunk := make([]byte, chunkSize) + end := min(copied+chunkSize, len(buf)) + copy(chunk, buf[copied:end]) + chunks = append(chunks, chunk) + copied += end - copied + } + + return &ContinuousBuffer{ + chunks: chunks, + } +} + +func New(chunks [][]byte) *ContinuousBuffer { + for i, chunk := range chunks { + if len(chunk) != chunkSize { + panic(fmt.Sprintf("wrong chunk #%d size %d", i, len(chunk))) + } + } + return &ContinuousBuffer{ + chunks: chunks, + } +} + +func (s *ContinuousBuffer) Chunks() [][]byte { + return s.chunks +} + +// [0, pos) +func (s *ContinuousBuffer) GetByte(idx int) byte { + chunkIdx := idx / chunkSize + if chunkIdx >= len(s.chunks) { + panic(ErrOutOfRange) + } + byteIdx := idx % chunkSize + return s.chunks[chunkIdx][byteIdx] +} + +func (s *ContinuousBuffer) SetByte(idx int, b byte) { + chunkIdx := idx / chunkSize + if chunkIdx > len(s.chunks) { + panic(ErrOutOfRange) + } + if chunkIdx == len(s.chunks) { + s.chunks = append(s.chunks, make([]byte, chunkSize)) + } + byteIdx := idx % chunkSize + s.chunks[chunkIdx][byteIdx] = b +} + +func (s *ContinuousBuffer) SetFlag(idx int, flag byte) { + chunkIdx := idx / chunkSize + if chunkIdx > len(s.chunks) { + panic(ErrOutOfRange) + } + if chunkIdx == len(s.chunks) { + s.chunks = append(s.chunks, make([]byte, chunkSize)) + } + byteIdx := idx % chunkSize + s.chunks[chunkIdx][byteIdx] |= flag +} + +func (s *ContinuousBuffer) UnsetFlag(idx int, flag byte) { + chunkIdx := idx / chunkSize + if chunkIdx > len(s.chunks) { + panic(ErrOutOfRange) + } + if chunkIdx == len(s.chunks) { + s.chunks = append(s.chunks, make([]byte, chunkSize)) + } + byteIdx := idx % chunkSize + s.chunks[chunkIdx][byteIdx] &^= flag +} + +// [since, until) +func (s *ContinuousBuffer) ShiftOnePosToRight(since int, until int) { + if since < 0 { + panic("since < 0") + } + if since >= until { + panic("since >= until") + } + + chunkIdx := until / chunkSize + byteIdx := until % chunkSize + + if chunkIdx > len(s.chunks) { + panic(ErrOutOfRange) + } + + if chunkIdx == len(s.chunks) { + if byteIdx == 0 { + s.chunks = append(s.chunks, make([]byte, chunkSize)) + } else { + panic(ErrOutOfRange) + } + } + + var ( + qty = until - since + prevChunkIdx int + prevByteIdx int + ) + + for range qty { + prevChunkIdx = chunkIdx + prevByteIdx = byteIdx - 1 + if prevByteIdx < 0 { + prevChunkIdx = chunkIdx - 1 + prevByteIdx = chunkSize - 1 + } + + s.chunks[chunkIdx][byteIdx] = s.chunks[prevChunkIdx][prevByteIdx] + + if byteIdx > 0 { + byteIdx-- + } else { + chunkIdx-- + byteIdx = chunkSize - 1 + } + } +} + +// [since, until) +func (s *ContinuousBuffer) ShiftOnePosToLeft(since int, until int) { + if since <= 0 { + panic("since <= 0") + } + if since >= until { + panic("since >= until") + } + + chunkIdx := since / chunkSize + byteIdx := since % chunkSize + + if until > len(s.chunks)*chunkSize { + panic(ErrOutOfRange) + } + + var ( + qty = until - since + prevChunkIdx int + prevByteIdx int + ) + + for range qty { + prevChunkIdx = chunkIdx + prevByteIdx = byteIdx - 1 + if prevByteIdx < 0 { + prevChunkIdx = chunkIdx - 1 + prevByteIdx = chunkSize - 1 + } + + s.chunks[prevChunkIdx][prevByteIdx] = s.chunks[chunkIdx][byteIdx] + + byteIdx++ + if byteIdx == chunkSize { + chunkIdx++ + byteIdx = 0 + } + } +} + +func (s *ContinuousBuffer) PutUint32(pos int, num uint32) { + s.CopyTo(pos, + []byte{ + byte(num), + byte(num >> 8), + byte(num >> 16), + byte(num >> 24), + }) +} + +func (s *ContinuousBuffer) GetUint32(pos int) uint32 { + arr := s.Slice(pos, pos+4) + return uint32(arr[0]) | (uint32(arr[1]) << 8) | + (uint32(arr[2]) << 16) | (uint32(arr[3]) << 24) +} + +func (s *ContinuousBuffer) ReversePutVarUint64(pos int, num uint64) int { + var tmp [9]byte + for i := range 8 { + tmp[i] = byte(num & 127) + num >>= 7 + if num == 0 { + tmp[i] |= 128 + n := i + 1 + s.ReverseCopyTo(pos, tmp[:n]) + return n + } + } + tmp[8] = byte(num) + n := 9 + s.ReverseCopyTo(pos, tmp[:]) + return n +} + +func (s *ContinuousBuffer) ReverseGetVarUint64(idx int) (num uint64, n int, err error) { + chunkIdx := idx / chunkSize + if chunkIdx >= len(s.chunks) { + panic(ErrOutOfRange) + } + + var ( + byteIdx = idx % chunkSize + chunk = s.chunks[chunkIdx] + b byte + ) + + for i := range 8 { + b = chunk[byteIdx] + if b >= 128 { + num |= uint64(b&127) << uint(i*7) + return num, i + 1, nil + } + num |= uint64(b) << uint(i*7) + + if byteIdx > 0 { + byteIdx-- + } else { + if chunkIdx == 0 { + return 0, 0, io.EOF + } else { + chunkIdx-- + chunk = s.chunks[chunkIdx] + byteIdx = chunkSize - 1 + } + } + } + return num | uint64(chunk[byteIdx])<<56, 9, nil +} + +func (s *ContinuousBuffer) ReversePutVarInt64(pos int, x int64) int { + return s.ReversePutVarUint64(pos, bin.EncodeZigZag(x)) +} + +func (s *ContinuousBuffer) ReverseGetVarInt64(idx int) (int64, int, error) { + u64, n, err := s.ReverseGetVarUint64(idx) + if err != nil { + return 0, 0, err + } + return bin.DecodeZigZag(u64), n, nil +} + +func (s *ContinuousBuffer) PutVarUint64(pos int, num uint64) int { + var tmp [9]byte + for i := range 8 { + tmp[i] = byte(num & 127) + num >>= 7 + if num == 0 { + tmp[i] |= 128 + n := i + 1 + s.CopyTo(pos, tmp[:n]) + return n + } + } + tmp[8] = byte(num) + s.CopyTo(pos, tmp[:]) + return 9 +} + +func (s *ContinuousBuffer) GetVarUint64(idx int) (num uint64, n int, err error) { + chunkIdx := idx / chunkSize + if chunkIdx >= len(s.chunks) { + panic(ErrOutOfRange) + } + + var ( + byteIdx = idx % chunkSize + chunk = s.chunks[chunkIdx] + b byte + ) + + for i := range 8 { + b = chunk[byteIdx] + if b >= 128 { + num |= uint64(b&127) << uint(i*7) + return num, i + 1, nil + } + num |= uint64(b) << uint(i*7) + + byteIdx++ + if byteIdx == chunkSize { + chunkIdx++ + if chunkIdx == len(s.chunks) { + return 0, 0, io.EOF + } + chunk = s.chunks[chunkIdx] + byteIdx = 0 + } + } + return num | uint64(chunk[byteIdx])<<56, 9, nil +} + +func (s *ContinuousBuffer) PutVarInt64(idx int, x int64) int { + return s.PutVarUint64(idx, bin.EncodeZigZag(x)) +} + +func (s *ContinuousBuffer) GetVarInt64(idx int) (int64, int, error) { + u64, n, err := s.GetVarUint64(idx) + if err != nil { + return 0, 0, err + } + return bin.DecodeZigZag(u64), n, nil +} + +func (s *ContinuousBuffer) CopyTo(idx int, data []byte) { + chunkIdx := idx / chunkSize + if chunkIdx > len(s.chunks) { + panic(ErrOutOfRange) + } + if chunkIdx == len(s.chunks) { + s.chunks = append(s.chunks, make([]byte, chunkSize)) + } + byteIdx := idx % chunkSize + chunk := s.chunks[chunkIdx] + copied := 0 + + for _, b := range data { + chunk[byteIdx] = b + copied++ + byteIdx++ + if byteIdx == chunkSize { + byteIdx = 0 + chunkIdx++ + if chunkIdx == len(s.chunks) { + if copied == len(data) { + return + } + s.chunks = append(s.chunks, make([]byte, chunkSize)) + } + chunk = s.chunks[chunkIdx] + } + } +} + +func (s *ContinuousBuffer) ReverseCopyTo(idx int, data []byte) { + chunkIdx := idx / chunkSize + if chunkIdx > len(s.chunks) { + panic(ErrOutOfRange) + } + if chunkIdx == len(s.chunks) { + s.chunks = append(s.chunks, make([]byte, chunkSize)) + } + byteIdx := idx % chunkSize + chunk := s.chunks[chunkIdx] + copied := 0 + + for i := len(data) - 1; i >= 0; i-- { + chunk[byteIdx] = data[i] + copied++ + byteIdx++ + if byteIdx == chunkSize { + byteIdx = 0 + chunkIdx++ + if chunkIdx == len(s.chunks) { + if copied == len(data) { + return + } + s.chunks = append(s.chunks, make([]byte, chunkSize)) + } + chunk = s.chunks[chunkIdx] + } + } +} + +// [since, until) +func (s *ContinuousBuffer) Slice(since int, until int) []byte { + if since >= until { + return nil + } + size := len(s.chunks) * chunkSize + if until >= size { + panic(ErrOutOfRange) + } + + data := make([]byte, until-since) + + chunkIdx := since / chunkSize + byteIdx := since % chunkSize + chunk := s.chunks[chunkIdx] + + for i := range len(data) { + data[i] = chunk[byteIdx] + byteIdx++ + if byteIdx == chunkSize { + byteIdx = 0 + chunkIdx++ + chunk = s.chunks[chunkIdx] + } + } + return data +} + +func (s *ContinuousBuffer) DecodeRunLength(pos int) (length uint16, n int) { + b1 := s.GetByte(pos) + pos-- + if b1 < 128 { + length = uint16(b1) + n = 1 + } else { + b2 := s.GetByte(pos) + length = uint16(b1&127) | (uint16(b2) << 7) + n = 2 + } + length += 2 + return +} + +func (s *ContinuousBuffer) Copy() *ContinuousBuffer { + var copies [][]byte + for _, chunk := range s.chunks { + buf := make([]byte, chunkSize) + copy(buf, chunk) + copies = append(copies, buf) + } + return New(copies) +} + +// size to copy +func (s *ContinuousBuffer) CopyChunksToOneBuffer(dst []byte, size int) { + pos := 0 + for _, chunk := range s.chunks { + if size >= len(chunk) { + copy(dst[pos:], chunk) + size -= len(chunk) + pos += len(chunk) + } else { + copy(dst[pos:], chunk[:size]) + return + } + } +} diff --git a/database.ini b/database.ini new file mode 100644 index 0000000..80e15ab --- /dev/null +++ b/database.ini @@ -0,0 +1,4 @@ +tcpPort = 12345 +dir = testdir +redoDir = testdir +databaseName = test diff --git a/database/api.go b/database/api.go new file mode 100644 index 0000000..70782a5 --- /dev/null +++ b/database/api.go @@ -0,0 +1,1058 @@ +package database + +import ( + "errors" + "fmt" + "io" + "net" + + "gordenko.dev/dima/diploma" + "gordenko.dev/dima/diploma/atree" + "gordenko.dev/dima/diploma/bin" + "gordenko.dev/dima/diploma/bufreader" + "gordenko.dev/dima/diploma/chunkenc" + "gordenko.dev/dima/diploma/conbuf" + "gordenko.dev/dima/diploma/proto" + "gordenko.dev/dima/diploma/txlog" +) + +var ( + ErrNoValueBug = errors.New("has timestamp but no value") + ErrWrongResultCodeBug = errors.New("bug: wrong result code") + + successMsg = []byte{ + proto.RespSuccess, + } +) + +func reply(conn io.Writer, errcode uint16) { + var answer []byte + if errcode == 0 { + answer = successMsg + } else { + answer = []byte{ + proto.RespError, + 0, 0, + } + bin.PutUint16(answer[1:], errcode) + } + + _, err := conn.Write(answer) + if err != nil { + return + } +} + +func (s *Database) handleTCPConn(conn net.Conn) { + defer conn.Close() + + r := bufreader.New(conn, 128) + + for { + err := s.processRequest(conn, r) + if err != nil { + if err != io.EOF { + s.logger.Println(err) + } + return + } + } +} + +func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (err error) { + messageType, err := r.ReadByte() + if err != nil { + if err != io.EOF { + return fmt.Errorf("read messageType: %s", err) + } else { + return err + } + } + + switch messageType { + case proto.TypeGetMetric: + req, err := proto.ReadGetMetricReq(r) + if err != nil { + return fmt.Errorf("proto.ReadGetMetricReq: %s", err) + } + if err = s.GetMetric(conn, req); err != nil { + return fmt.Errorf("GetMetric: %s", err) + } + + case proto.TypeAddMetric: + req, err := proto.ReadAddMetricReq(r) + if err != nil { + return fmt.Errorf("proto.ReadAddMetricReq: %s", err) + } + reply(conn, s.AddMetric(req)) + + case proto.TypeDeleteMetric: + req, err := proto.ReadDeleteMetricReq(r) + if err != nil { + return fmt.Errorf("proto.ReadDeleteMetricReq: %s", err) + } + reply(conn, s.DeleteMetric(req)) + + case proto.TypeAppendMeasure: + req, err := proto.ReadAppendMeasureReq(r) + if err != nil { + return fmt.Errorf("proto.ReadAppendMeasureReq: %s", err) + } + //fmt.Println("append measure", req.MetricID, conn.RemoteAddr().String()) + reply(conn, s.AppendMeasure(req)) + + case proto.TypeAppendMeasures: + req, err := proto.ReadAppendMeasuresReq(r) + if err != nil { + return fmt.Errorf("proto.ReadAppendMeasuresReq: %s", err) + } + //fmt.Println("append measure", req.MetricID, conn.RemoteAddr().String()) + reply(conn, s.AppendMeasures(req)) + + case proto.TypeListInstantMeasures: + req, err := proto.ReadListInstantMeasuresReq(r) + if err != nil { + return fmt.Errorf("proto.ReadListInstantMeasuresReq: %s", err) + } + if err = s.ListInstantMeasures(conn, req); err != nil { + return fmt.Errorf("ListInstantMeasures: %s", err) + } + + case proto.TypeListCumulativeMeasures: + req, err := proto.ReadListCumulativeMeasuresReq(r) + if err != nil { + return fmt.Errorf("proto.ReadListCumulativeMeasuresReq: %s", err) + } + if err = s.ListCumulativeMeasures(conn, req); err != nil { + return fmt.Errorf("ListCumulativeMeasures: %s", err) + } + + case proto.TypeListInstantPeriods: + req, err := proto.ReadListInstantPeriodsReq(r) + if err != nil { + return fmt.Errorf("proto.ReadListInstantPeriodsReq: %s", err) + } + if err = s.ListInstantPeriods(conn, req); err != nil { + return fmt.Errorf("ListInstantPeriods: %s", err) + } + + case proto.TypeListCumulativePeriods: + req, err := proto.ReadListCumulativePeriodsReq(r) + if err != nil { + return fmt.Errorf("proto.ReadListCumulativePeriodsReq: %s", err) + } + if err = s.ListCumulativePeriods(conn, req); err != nil { + return fmt.Errorf("ListCumulativePeriods: %s", err) + } + + case proto.TypeListCurrentValues: + req, err := proto.ReadListCurrentValuesReq(r) + if err != nil { + return fmt.Errorf("proto.ListCurrentValuesReq: %s", err) + } + if err = s.ListCurrentValues(conn, req); err != nil { + return fmt.Errorf("ListCurrentValues: %s", err) + } + + case proto.TypeDeleteMeasures: + req, err := proto.ReadDeleteMeasuresReq(r) + if err != nil { + return fmt.Errorf("proto.ReadDeleteMeasuresReq: %s", err) + } + reply(conn, s.DeleteMeasures(req)) + + case proto.TypeListAllInstantMeasures: + req, err := proto.ReadListAllInstantMeasuresReq(r) + if err != nil { + return fmt.Errorf("proto.ReadListAllInstantMeasuresReq: %s", err) + } + if err = s.ListAllInstantMeasures(conn, req); err != nil { + return fmt.Errorf("ListAllInstantMeasures: %s", err) + } + + case proto.TypeListAllCumulativeMeasures: + req, err := proto.ReadListAllCumulativeMeasuresReq(r) + if err != nil { + return fmt.Errorf("proto.ReadListAllCumulativeMeasuresReq: %s", err) + } + + if err = s.ListAllCumulativeMeasures(conn, req); err != nil { + return fmt.Errorf("ListAllCumulativeMeasures: %s", err) + } + + default: + return fmt.Errorf("unknown messageType: %d", messageType) + } + return +} + +// API + +func (s *Database) AddMetric(req proto.AddMetricReq) uint16 { + // Валидация + if req.MetricID == 0 { + return proto.ErrEmptyMetricID + } + if byte(req.FracDigits) > diploma.MaxFracDigits { + return proto.ErrWrongFracDigits + } + switch req.MetricType { + case diploma.Cumulative, diploma.Instant: + // ok + default: + return proto.ErrWrongMetricType + } + + resultCh := make(chan byte, 1) + + s.appendJobToWorkerQueue(tryAddMetricReq{ + MetricID: req.MetricID, + ResultCh: resultCh, + }) + + resultCode := <-resultCh + + switch resultCode { + case Succeed: + waitCh := s.txlog.WriteAddedMetric(txlog.AddedMetric{ + MetricID: req.MetricID, + MetricType: req.MetricType, + FracDigits: req.FracDigits, + }) + <-waitCh + + case MetricDuplicate: + return proto.ErrDuplicate + + default: + diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug) + } + return 0 +} + +type Metric struct { + MetricType diploma.MetricType + FracDigits byte + ResultCode byte +} + +func (s *Database) GetMetric(conn io.Writer, req proto.GetMetricReq) error { + resultCh := make(chan Metric, 1) + + s.appendJobToWorkerQueue(tryGetMetricReq{ + MetricID: req.MetricID, + ResultCh: resultCh, + }) + + result := <-resultCh + + switch result.ResultCode { + case Succeed: + answer := []byte{ + proto.RespValue, + 0, 0, 0, 0, // metricID + byte(result.MetricType), + result.FracDigits, + } + bin.PutUint32(answer[1:], req.MetricID) + + _, err := conn.Write(answer) + if err != nil { + return err + } + + case NoMetric: + reply(conn, proto.ErrNoMetric) + + default: + diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug) + } + return nil +} + +type tryDeleteMetricResult struct { + ResultCode byte + RootPageNo uint32 +} + +func (s *Database) DeleteMetric(req proto.DeleteMetricReq) uint16 { + resultCh := make(chan tryDeleteMetricResult, 1) + + s.appendJobToWorkerQueue(tryDeleteMetricReq{ + MetricID: req.MetricID, + ResultCh: resultCh, + }) + + result := <-resultCh + + switch result.ResultCode { + case Succeed: + var ( + freeDataPages []uint32 + freeIndexPages []uint32 + ) + if result.RootPageNo > 0 { + pageLists, err := s.atree.GetAllPages(result.RootPageNo) + if err != nil { + diploma.Abort(diploma.FailedAtreeRequest, err) + } + freeDataPages = pageLists.DataPages + freeIndexPages = pageLists.IndexPages + } + waitCh := s.txlog.WriteDeletedMetric(txlog.DeletedMetric{ + MetricID: req.MetricID, + FreeDataPages: freeDataPages, + FreeIndexPages: freeIndexPages, + }) + <-waitCh + + case NoMetric: + return proto.ErrNoMetric + + default: + diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug) + } + return 0 +} + +type FilledPage struct { + Since uint32 + RootPageNo uint32 + PrevPageNo uint32 + TimestampsChunks [][]byte + TimestampsSize uint16 + ValuesChunks [][]byte + ValuesSize uint16 +} + +type tryAppendMeasureResult struct { + FilledPage *FilledPage + ResultCode byte +} + +func (s *Database) AppendMeasure(req proto.AppendMeasureReq) uint16 { + resultCh := make(chan tryAppendMeasureResult, 1) + + s.appendJobToWorkerQueue(tryAppendMeasureReq{ + MetricID: req.MetricID, + Timestamp: req.Timestamp, + Value: req.Value, + ResultCh: resultCh, + }) + + result := <-resultCh + + switch result.ResultCode { + case CanAppend: + waitCh := s.txlog.WriteAppendMeasure(txlog.AppendedMeasure{ + MetricID: req.MetricID, + Timestamp: req.Timestamp, + Value: req.Value, + }) + <-waitCh + + case NewPage: + filled := result.FilledPage + report, err := s.atree.AppendDataPage(atree.AppendDataPageReq{ + MetricID: req.MetricID, + Timestamp: req.Timestamp, + Value: req.Value, + Since: filled.Since, + RootPageNo: filled.RootPageNo, + PrevPageNo: filled.PrevPageNo, + TimestampsChunks: filled.TimestampsChunks, + TimestampsSize: filled.TimestampsSize, + ValuesChunks: filled.ValuesChunks, + ValuesSize: filled.ValuesSize, + }) + if err != nil { + diploma.Abort(diploma.WriteToAtreeFailed, err) + } + waitCh := s.txlog.WriteAppendedMeasureWithOverflow( + txlog.AppendedMeasureWithOverflow{ + MetricID: req.MetricID, + Timestamp: req.Timestamp, + Value: req.Value, + IsDataPageReused: report.IsDataPageReused, + DataPageNo: report.DataPageNo, + IsRootChanged: report.IsRootChanged, + RootPageNo: report.NewRootPageNo, + ReusedIndexPages: report.ReusedIndexPages, + }, + report.FileName, + false, + ) + <-waitCh + + case NoMetric: + return proto.ErrNoMetric + + case ExpiredMeasure: + return proto.ErrExpiredMeasure + + case NonMonotonicValue: + return proto.ErrNonMonotonicValue + + default: + diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug) + } + return 0 +} + +type tryAppendMeasuresResult struct { + ResultCode byte + MetricType diploma.MetricType + FracDigits byte + Since uint32 + Until uint32 + UntilValue float64 + RootPageNo uint32 + PrevPageNo uint32 + TimestampsBuf *conbuf.ContinuousBuffer + ValuesBuf *conbuf.ContinuousBuffer + Timestamps diploma.TimestampCompressor + Values diploma.ValueCompressor +} + +func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 { + resultCh := make(chan tryAppendMeasuresResult, 1) + + s.appendJobToWorkerQueue(tryAppendMeasuresReq{ + MetricID: req.MetricID, + ResultCh: resultCh, + }) + + result := <-resultCh + + switch result.ResultCode { + case CanAppend: + var ( + rootPageNo = result.RootPageNo + prevPageNo = result.PrevPageNo + timestampsBuf = result.TimestampsBuf + valuesBuf = result.ValuesBuf + timestamps = result.Timestamps + values = result.Values + since = result.Since + until = result.Until + untilValue = result.UntilValue + // + toAppendMeasures []proto.Measure + ) + + for idx, measure := range req.Measures { + //fmt.Printf("%d %v\n", measure.Timestamp, measure.Value) + if since == 0 { + since = measure.Timestamp + } else { + if measure.Timestamp <= until { + if len(toAppendMeasures) > 0 { + waitCh := s.txlog.WriteAppendMeasures( + txlog.AppendedMeasures{ + MetricID: req.MetricID, + Measures: toAppendMeasures, + }, + false, + ) + <-waitCh + } + return proto.ErrExpiredMeasure + } + + if result.MetricType == diploma.Cumulative && measure.Value < untilValue { + if len(toAppendMeasures) > 0 { + waitCh := s.txlog.WriteAppendMeasures( + txlog.AppendedMeasures{ + MetricID: req.MetricID, + Measures: toAppendMeasures, + }, + false, + ) + <-waitCh + } + //fmt.Printf("m.Value: %v < untilValue: %v\n", measure.Value, untilValue) + return proto.ErrNonMonotonicValue + } + } + + extraSpace := timestamps.CalcRequiredSpace(measure.Timestamp) + + values.CalcRequiredSpace(measure.Value) + + totalSpace := timestamps.Size() + values.Size() + extraSpace + + if totalSpace <= atree.DataPagePayloadSize { + timestamps.Append(measure.Timestamp) + values.Append(measure.Value) + toAppendMeasures = append(toAppendMeasures, measure) + } else { + if len(toAppendMeasures) > 0 { + waitCh := s.txlog.WriteAppendMeasures( + txlog.AppendedMeasures{ + MetricID: req.MetricID, + Measures: toAppendMeasures, + }, + true, + ) + <-waitCh + + toAppendMeasures = nil + } + report, err := s.atree.AppendDataPage(atree.AppendDataPageReq{ + MetricID: req.MetricID, + Timestamp: until, + Value: untilValue, + Since: since, + RootPageNo: rootPageNo, + PrevPageNo: prevPageNo, + TimestampsChunks: timestampsBuf.Chunks(), + TimestampsSize: uint16(timestamps.Size()), + ValuesChunks: valuesBuf.Chunks(), + ValuesSize: uint16(values.Size()), + }) + if err != nil { + diploma.Abort(diploma.WriteToAtreeFailed, err) + } + + prevPageNo = report.DataPageNo + if report.IsRootChanged { + rootPageNo = report.NewRootPageNo + } + waitCh := s.txlog.WriteAppendedMeasureWithOverflow( + txlog.AppendedMeasureWithOverflow{ + MetricID: req.MetricID, + Timestamp: measure.Timestamp, + Value: measure.Value, + IsDataPageReused: report.IsDataPageReused, + DataPageNo: report.DataPageNo, + IsRootChanged: report.IsRootChanged, + RootPageNo: report.NewRootPageNo, + ReusedIndexPages: report.ReusedIndexPages, + }, + report.FileName, + (idx+1) < len(req.Measures), + ) + <-waitCh + + timestampsBuf = conbuf.New(nil) + valuesBuf = conbuf.New(nil) + + timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor( + timestampsBuf, 0) + if result.MetricType == diploma.Cumulative { + values = chunkenc.NewReverseCumulativeDeltaCompressor( + valuesBuf, 0, result.FracDigits) + } else { + values = chunkenc.NewReverseInstantDeltaCompressor( + valuesBuf, 0, result.FracDigits) + } + + timestamps.Append(measure.Timestamp) + values.Append(measure.Value) + + since = measure.Timestamp + } + + until = measure.Timestamp + untilValue = measure.Value + } + + if len(toAppendMeasures) > 0 { + waitCh := s.txlog.WriteAppendMeasures( + txlog.AppendedMeasures{ + MetricID: req.MetricID, + Measures: toAppendMeasures, + }, + false, + ) + <-waitCh + } + + case NoMetric: + return proto.ErrNoMetric + + default: + diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug) + } + return 0 +} + +type tryDeleteMeasuresResult struct { + ResultCode byte + RootPageNo uint32 +} + +func (s *Database) DeleteMeasures(req proto.DeleteMeasuresReq) uint16 { + resultCh := make(chan tryDeleteMeasuresResult, 1) + + s.appendJobToWorkerQueue(tryDeleteMeasuresReq{ + MetricID: req.MetricID, + Since: req.Since, + ResultCh: resultCh, + }) + + result := <-resultCh + + switch result.ResultCode { + case Succeed: + var ( + freeDataPages []uint32 + freeIndexPages []uint32 + ) + if result.RootPageNo > 0 { + pageLists, err := s.atree.GetAllPages(result.RootPageNo) + if err != nil { + diploma.Abort(diploma.FailedAtreeRequest, err) + } + freeDataPages = pageLists.DataPages + freeIndexPages = pageLists.IndexPages + } + waitCh := s.txlog.WriteDeletedMeasures(txlog.DeletedMeasures{ + MetricID: req.MetricID, + FreeDataPages: freeDataPages, + FreeIndexPages: freeIndexPages, + }) + <-waitCh + + case NoMetric: + return proto.ErrNoMetric + + default: + diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug) + } + return 0 +} + +// SELECT + +type instantMeasuresResult struct { + ResultCode byte + FracDigits byte + PageNo uint32 +} + +func (s *Database) ListAllInstantMeasures(conn net.Conn, req proto.ListAllInstantMetricMeasuresReq) error { + resultCh := make(chan instantMeasuresResult, 1) + + responseWriter := atree.NewInstantMeasureWriter(conn) + + s.appendJobToWorkerQueue(tryListAllInstantMeasuresReq{ + MetricID: req.MetricID, + ResponseWriter: responseWriter, + ResultCh: resultCh, + }) + + result := <-resultCh + + switch result.ResultCode { + case QueryDone: + responseWriter.Close() + + case UntilFound: + err := s.atree.IterateAllInstantByTreeCursor(atree.IterateAllInstantByTreeCursorReq{ + FracDigits: result.FracDigits, + PageNo: result.PageNo, + ResponseWriter: responseWriter, + }) + s.metricRUnlock(req.MetricID) + + if err != nil { + reply(conn, proto.ErrUnexpected) + } else { + responseWriter.Close() + } + + case NoMetric: + reply(conn, proto.ErrNoMetric) + + case WrongMetricType: + reply(conn, proto.ErrWrongMetricType) + + default: + diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug) + } + return nil +} + +func (s *Database) ListAllCumulativeMeasures(conn io.Writer, req proto.ListAllCumulativeMeasuresReq) error { + resultCh := make(chan cumulativeMeasuresResult, 1) + + responseWriter := atree.NewCumulativeMeasureWriter(conn) + + s.appendJobToWorkerQueue(tryListAllCumulativeMeasuresReq{ + MetricID: req.MetricID, + ResponseWriter: responseWriter, + ResultCh: resultCh, + }) + + result := <-resultCh + + switch result.ResultCode { + case QueryDone: + responseWriter.Close() + + case UntilFound: + err := s.atree.IterateAllCumulativeByTreeCursor(atree.IterateAllCumulativeByTreeCursorReq{ + FracDigits: result.FracDigits, + PageNo: result.PageNo, + EndTimestamp: result.EndTimestamp, + EndValue: result.EndValue, + ResponseWriter: responseWriter, + }) + s.metricRUnlock(req.MetricID) + + if err != nil { + reply(conn, proto.ErrUnexpected) + } else { + responseWriter.Close() + } + + case NoMetric: + reply(conn, proto.ErrNoMetric) + + case WrongMetricType: + reply(conn, proto.ErrWrongMetricType) + + default: + diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug) + } + return nil +} + +func (s *Database) ListInstantMeasures(conn net.Conn, req proto.ListInstantMeasuresReq) error { + if req.Since > req.Until { + reply(conn, proto.ErrInvalidRange) + return nil + } + + var since, until uint32 + if req.FirstHourOfDay > 0 { + since, until = correctToFHD(req.Since, req.Until, req.FirstHourOfDay) + } else { + since = req.Since + until = req.Until + } + + resultCh := make(chan instantMeasuresResult, 1) + responseWriter := atree.NewInstantMeasureWriter(conn) + + s.appendJobToWorkerQueue(tryListInstantMeasuresReq{ + MetricID: req.MetricID, + Since: since, + Until: until, + ResponseWriter: responseWriter, + ResultCh: resultCh, + }) + + result := <-resultCh + + switch result.ResultCode { + case QueryDone: + responseWriter.Close() + + case UntilFound: + err := s.atree.ContinueIterateInstantByTreeCursor(atree.ContinueIterateInstantByTreeCursorReq{ + FracDigits: result.FracDigits, + Since: since, + Until: until, + LastPageNo: result.PageNo, + ResponseWriter: responseWriter, + }) + s.metricRUnlock(req.MetricID) + if err != nil { + reply(conn, proto.ErrUnexpected) + } else { + responseWriter.Close() + } + + case UntilNotFound: + err := s.atree.FindAndIterateInstantByTreeCursor(atree.FindAndIterateInstantByTreeCursorReq{ + FracDigits: result.FracDigits, + Since: since, + Until: until, + RootPageNo: result.PageNo, + ResponseWriter: responseWriter, + }) + s.metricRUnlock(req.MetricID) + if err != nil { + reply(conn, proto.ErrUnexpected) + } else { + responseWriter.Close() + } + + case NoMetric: + reply(conn, proto.ErrNoMetric) + + case WrongMetricType: + reply(conn, proto.ErrWrongMetricType) + + default: + diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug) + } + return nil +} + +type cumulativeMeasuresResult struct { + ResultCode byte + FracDigits byte + PageNo uint32 + EndTimestamp uint32 + EndValue float64 +} + +func (s *Database) ListCumulativeMeasures(conn net.Conn, req proto.ListCumulativeMeasuresReq) error { + resultCh := make(chan cumulativeMeasuresResult, 1) + responseWriter := atree.NewCumulativeMeasureWriter(conn) + + s.appendJobToWorkerQueue(tryListCumulativeMeasuresReq{ + MetricID: req.MetricID, + Since: req.Since, + Until: req.Until, + ResponseWriter: responseWriter, + ResultCh: resultCh, + }) + + result := <-resultCh + + switch result.ResultCode { + case QueryDone: + responseWriter.Close() + + case UntilFound: + err := s.atree.ContinueIterateCumulativeByTreeCursor(atree.ContinueIterateCumulativeByTreeCursorReq{ + FracDigits: result.FracDigits, + Since: req.Since, + Until: req.Until, + LastPageNo: result.PageNo, + EndTimestamp: result.EndTimestamp, + EndValue: result.EndValue, + ResponseWriter: responseWriter, + }) + s.metricRUnlock(req.MetricID) + if err != nil { + reply(conn, proto.ErrUnexpected) + } else { + responseWriter.Close() + } + + case UntilNotFound: + err := s.atree.FindAndIterateCumulativeByTreeCursor(atree.FindAndIterateCumulativeByTreeCursorReq{ + FracDigits: result.FracDigits, + Since: req.Since, + Until: req.Until, + RootPageNo: result.PageNo, + ResponseWriter: responseWriter, + }) + s.metricRUnlock(req.MetricID) + if err != nil { + reply(conn, proto.ErrUnexpected) + } else { + responseWriter.Close() + } + + case NoMetric: + reply(conn, proto.ErrNoMetric) + + case WrongMetricType: + reply(conn, proto.ErrWrongMetricType) + + default: + diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug) + } + return nil +} + +type instantPeriodsResult struct { + ResultCode byte + FracDigits byte + PageNo uint32 +} + +func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriodsReq) error { + var ( + since = req.Since + until = req.Until + ) + if req.FirstHourOfDay > 0 { + since, until = correctToFHD(since, until, req.FirstHourOfDay) + } + + if req.LastDayOfMonth > 0 { + // fix + } + + resultCh := make(chan instantPeriodsResult, 1) + + aggregator, err := atree.NewInstantAggregator(atree.InstantAggregatorOptions{ + GroupBy: req.GroupBy, + FirstHourOfDay: req.FirstHourOfDay, + LastDayOfMonth: req.LastDayOfMonth, + }) + if err != nil { + reply(conn, proto.ErrUnexpected) + return nil + } + + responseWriter := atree.NewInstantPeriodsWriter(conn, req.AggregateFuncs) + + s.appendJobToWorkerQueue(tryListInstantPeriodsReq{ + MetricID: req.MetricID, + Since: req.Since, + Until: req.Until, + Aggregator: aggregator, + ResponseWriter: responseWriter, + ResultCh: resultCh, + }) + + result := <-resultCh + + switch result.ResultCode { + case QueryDone: + responseWriter.Close() + + case UntilFound: + err := s.atree.ContinueCollectInstantPeriods(atree.ContinueCollectInstantPeriodsReq{ + FracDigits: result.FracDigits, + Aggregator: aggregator, + ResponseWriter: responseWriter, + LastPageNo: result.PageNo, + Since: req.Since, + Until: req.Until, + }) + s.metricRUnlock(req.MetricID) + + if err != nil { + reply(conn, proto.ErrUnexpected) + } else { + responseWriter.Close() + } + + case UntilNotFound: + err := s.atree.FindInstantPeriods(atree.FindInstantPeriodsReq{ + FracDigits: result.FracDigits, + ResponseWriter: responseWriter, + RootPageNo: result.PageNo, + Since: req.Since, + Until: req.Until, + GroupBy: req.GroupBy, + FirstHourOfDay: req.FirstHourOfDay, + LastDayOfMonth: req.LastDayOfMonth, + }) + s.metricRUnlock(req.MetricID) + + if err != nil { + reply(conn, proto.ErrUnexpected) + } else { + responseWriter.Close() + } + + case NoMetric: + reply(conn, proto.ErrNoMetric) + + case WrongMetricType: + reply(conn, proto.ErrWrongMetricType) + + default: + diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug) + } + return nil +} + +type cumulativePeriodsResult struct { + ResultCode byte + FracDigits byte + PageNo uint32 +} + +func (s *Database) ListCumulativePeriods(conn net.Conn, req proto.ListCumulativePeriodsReq) error { + resultCh := make(chan cumulativePeriodsResult, 1) + + aggregator, err := atree.NewCumulativeAggregator(atree.CumulativeAggregatorOptions{ + GroupBy: req.GroupBy, + FirstHourOfDay: req.FirstHourOfDay, + LastDayOfMonth: req.LastDayOfMonth, + }) + if err != nil { + reply(conn, proto.ErrUnexpected) + return nil + } + + responseWriter := atree.NewCumulativePeriodsWriter(conn) + + s.appendJobToWorkerQueue(tryListCumulativePeriodsReq{ + MetricID: req.MetricID, + Since: req.Since, + Until: req.Until, + Aggregator: aggregator, + ResponseWriter: responseWriter, + ResultCh: resultCh, + }) + + result := <-resultCh + + switch result.ResultCode { + case QueryDone: + responseWriter.Close() + + case UntilFound: + err := s.atree.ContinueCollectCumulativePeriods(atree.ContinueCollectCumulativePeriodsReq{ + FracDigits: result.FracDigits, + Aggregator: aggregator, + ResponseWriter: responseWriter, + LastPageNo: result.PageNo, + Since: req.Since, + Until: req.Until, + }) + s.metricRUnlock(req.MetricID) + + if err != nil { + reply(conn, proto.ErrUnexpected) + } else { + responseWriter.Close() + } + + case UntilNotFound: + err := s.atree.FindCumulativePeriods(atree.FindCumulativePeriodsReq{ + FracDigits: result.FracDigits, + ResponseWriter: responseWriter, + RootPageNo: result.PageNo, + Since: req.Since, + Until: req.Until, + GroupBy: req.GroupBy, + FirstHourOfDay: req.FirstHourOfDay, + LastDayOfMonth: req.LastDayOfMonth, + }) + s.metricRUnlock(req.MetricID) + + if err != nil { + reply(conn, proto.ErrUnexpected) + } else { + responseWriter.Close() + } + + case NoMetric: + reply(conn, proto.ErrNoMetric) + + case WrongMetricType: + reply(conn, proto.ErrWrongMetricType) + + default: + diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug) + } + return nil +} + +func (s *Database) ListCurrentValues(conn net.Conn, req proto.ListCurrentValuesReq) error { + responseWriter := atree.NewCurrentValueWriter(conn) + defer responseWriter.Close() + + resultCh := make(chan struct{}) + + s.appendJobToWorkerQueue(tryListCurrentValuesReq{ + MetricIDs: req.MetricIDs, + ResponseWriter: responseWriter, + ResultCh: resultCh, + }) + + <-resultCh + return nil +} diff --git a/database/database.go b/database/database.go new file mode 100644 index 0000000..fedbf09 --- /dev/null +++ b/database/database.go @@ -0,0 +1,460 @@ +package database + +import ( + "errors" + "fmt" + "hash/crc32" + "io" + "log" + "net" + "os" + "path/filepath" + "regexp" + "sync" + "time" + + "gordenko.dev/dima/diploma" + "gordenko.dev/dima/diploma/atree" + "gordenko.dev/dima/diploma/atree/redo" + "gordenko.dev/dima/diploma/bin" + "gordenko.dev/dima/diploma/chunkenc" + "gordenko.dev/dima/diploma/conbuf" + "gordenko.dev/dima/diploma/freelist" + "gordenko.dev/dima/diploma/recovery" + "gordenko.dev/dima/diploma/txlog" +) + +func JoinSnapshotFileName(dir string, logNumber int) string { + return filepath.Join(dir, fmt.Sprintf("%d.snapshot", logNumber)) +} + +type metricLockEntry struct { + XLock bool + RLocks int + WaitQueue []any +} + +type Database struct { + mutex sync.Mutex + workerSignalCh chan struct{} + workerQueue []any + rLocksToRelease []uint32 + metrics map[uint32]*_metric + metricLockEntries map[uint32]*metricLockEntry + dataFreeList *freelist.FreeList + indexFreeList *freelist.FreeList + dir string + databaseName string + redoDir string + txlog *txlog.Writer + atree *atree.Atree + tcpPort int + logfile *os.File + logger *log.Logger + exitCh chan struct{} + waitGroup *sync.WaitGroup +} + +type Options struct { + TCPPort int + Dir string + DatabaseName string + RedoDir string + Logfile *os.File + ExitCh chan struct{} + WaitGroup *sync.WaitGroup +} + +func New(opt Options) (_ *Database, err error) { + if opt.TCPPort <= 0 { + return nil, errors.New("TCPPort option is required") + } + if opt.Dir == "" { + return nil, errors.New("Dir option is required") + } + if opt.DatabaseName == "" { + return nil, errors.New("DatabaseName option is required") + } + if opt.RedoDir == "" { + return nil, errors.New("RedoDir option is required") + } + if opt.Logfile == nil { + return nil, errors.New("Logfile option is required") + } + if opt.ExitCh == nil { + return nil, errors.New("ExitCh option is required") + } + if opt.WaitGroup == nil { + return nil, errors.New("WaitGroup option is required") + } + + s := &Database{ + workerSignalCh: make(chan struct{}, 1), + dir: opt.Dir, + databaseName: opt.DatabaseName, + redoDir: opt.RedoDir, + metrics: make(map[uint32]*_metric), + metricLockEntries: make(map[uint32]*metricLockEntry), + dataFreeList: freelist.New(), + indexFreeList: freelist.New(), + tcpPort: opt.TCPPort, + logfile: opt.Logfile, + logger: log.New(opt.Logfile, "", log.LstdFlags), + exitCh: opt.ExitCh, + waitGroup: opt.WaitGroup, + } + return s, nil +} + +func (s *Database) ListenAndServe() (err error) { + listener, err := net.Listen("tcp", fmt.Sprintf(":%d", s.tcpPort)) + if err != nil { + return fmt.Errorf("net.Listen: %s; port=%d", err, s.tcpPort) + } + + s.atree, err = atree.New(atree.Options{ + Dir: s.dir, + DatabaseName: s.databaseName, + RedoDir: s.redoDir, + DataFreeList: s.dataFreeList, + IndexFreeList: s.indexFreeList, + }) + if err != nil { + return fmt.Errorf("atree.New: %s", err) + } + s.atree.Run() + + go s.worker() + + s.recovery() + + s.logger.Println("database started") + for { + // Listen for an incoming connection. + conn, err := listener.Accept() + if err != nil { + s.logger.Printf("listener.Accept: %s\n", err) + time.Sleep(time.Second) + } else { + go s.handleTCPConn(conn) + } + } +} + +func (s *Database) recovery() { + advisor, err := recovery.NewRecoveryAdvisor(recovery.RecoveryAdvisorOptions{ + Dir: s.dir, + VerifySnapshot: s.verifySnapshot, + }) + if err != nil { + panic(err) + } + + recipe, err := advisor.GetRecipe() + if err != nil { + diploma.Abort(diploma.GetRecoveryRecipeFailed, err) + } + + var logNumber int + + if recipe != nil { + if recipe.Snapshot != "" { + err = s.loadSnapshot(recipe.Snapshot) + if err != nil { + diploma.Abort(diploma.LoadSnapshotFailed, err) + } + } + for _, changesFileName := range recipe.Changes { + err = s.replayChanges(changesFileName) + if err != nil { + diploma.Abort(diploma.ReplayChangesFailed, err) + } + } + logNumber = recipe.LogNumber + } + + s.txlog, err = txlog.NewWriter(txlog.WriterOptions{ + Dir: s.dir, + LogNumber: logNumber, + AppendToWorkerQueue: s.appendJobToWorkerQueue, + ExitCh: s.exitCh, + WaitGroup: s.waitGroup, + }) + if err != nil { + diploma.Abort(diploma.CreateChangesWriterFailed, err) + + } + go s.txlog.Run() + + fileNames, err := s.searchREDOFiles() + if err != nil { + diploma.Abort(diploma.SearchREDOFilesFailed, err) + } + + if len(fileNames) > 0 { + for _, fileName := range fileNames { + err = s.replayREDOFile(fileName) + if err != nil { + diploma.Abort(diploma.ReplayREDOFileFailed, err) + } + } + + for _, fileName := range fileNames { + err = os.Remove(fileName) + if err != nil { + diploma.Abort(diploma.RemoveREDOFileFailed, err) + } + } + } + + if recipe != nil { + if recipe.CompleteSnapshot { + err = s.dumpSnapshot(logNumber) + if err != nil { + diploma.Abort(diploma.DumpSnapshotFailed, err) + } + } + + for _, fileName := range recipe.ToDelete { + err = os.Remove(fileName) + if err != nil { + diploma.Abort(diploma.RemoveRecipeFileFailed, err) + } + } + } +} + +func (s *Database) searchREDOFiles() ([]string, error) { + var ( + reREDO = regexp.MustCompile(`a\d+\.redo`) + fileNames []string + ) + + entries, err := os.ReadDir(s.redoDir) + if err != nil { + return nil, err + } + + for _, entry := range entries { + if entry.Type().IsRegular() { + baseName := entry.Name() + if reREDO.MatchString(baseName) { + fileNames = append(fileNames, filepath.Join(s.redoDir, baseName)) + } + } + } + return fileNames, nil +} + +func (s *Database) replayREDOFile(fileName string) error { + redoFile, err := redo.ReadREDOFile(redo.ReadREDOFileReq{ + FileName: fileName, + DataPageSize: atree.DataPageSize, + IndexPageSize: atree.IndexPageSize, + }) + if err != nil { + return fmt.Errorf("can't read REDO file %s: %s", fileName, err) + } + + metric, ok := s.metrics[redoFile.MetricID] + if !ok { + return fmt.Errorf("has REDOFile, metric %d not found", redoFile.MetricID) + } + + if metric.Until < redoFile.Timestamp { + waitCh := make(chan struct{}) + s.atree.ApplyREDO(atree.WriteTask{ + DataPage: redoFile.DataPage, + IndexPages: redoFile.IndexPages, + }) + <-waitCh + + waitCh = s.txlog.WriteAppendedMeasureWithOverflow( + txlog.AppendedMeasureWithOverflow{ + MetricID: redoFile.MetricID, + Timestamp: redoFile.Timestamp, + Value: redoFile.Value, + IsDataPageReused: redoFile.IsDataPageReused, + DataPageNo: redoFile.DataPage.PageNo, + IsRootChanged: redoFile.IsRootChanged, + RootPageNo: redoFile.RootPageNo, + ReusedIndexPages: redoFile.ReusedIndexPages, + }, + fileName, + false, + ) + <-waitCh + } + return nil +} + +func (s *Database) verifySnapshot(fileName string) (_ bool, err error) { + file, err := os.Open(fileName) + if err != nil { + return + } + defer file.Close() + + stat, err := file.Stat() + if err != nil { + return + } + + if stat.Size() <= 4 { + return false, nil + } + + var ( + payloadSize = stat.Size() - 4 + hash = crc32.NewIEEE() + ) + + _, err = io.CopyN(hash, file, payloadSize) + if err != nil { + return + } + calculatedCRC := hash.Sum32() + + storedCRC, err := bin.ReadUint32(file) + if err != nil { + return + } + if storedCRC != calculatedCRC { + return false, fmt.Errorf("strored CRC %d not equal calculated CRC %d", + storedCRC, calculatedCRC) + } + return true, nil +} + +func (s *Database) replayChanges(fileName string) error { + walReader, err := txlog.NewReader(txlog.ReaderOptions{ + FileName: fileName, + BufferSize: 1024 * 1024, + }) + if err != nil { + return err + } + + for { + lsn, records, done, err := walReader.ReadPacket() + if err != nil { + return err + } + _ = lsn + + if done { + return nil + } + + for _, record := range records { + if err = s.replayChangesRecord(record); err != nil { + return err + } + } + } +} + +func (s *Database) replayChangesRecord(untyped any) error { + switch rec := untyped.(type) { + case txlog.AddedMetric: + var ( + values diploma.ValueCompressor + timestampsBuf = conbuf.New(nil) + valuesBuf = conbuf.New(nil) + ) + + if rec.MetricType == diploma.Cumulative { + values = chunkenc.NewReverseCumulativeDeltaCompressor( + valuesBuf, 0, byte(rec.FracDigits)) + } else { + values = chunkenc.NewReverseInstantDeltaCompressor( + valuesBuf, 0, byte(rec.FracDigits)) + } + + s.metrics[rec.MetricID] = &_metric{ + MetricType: rec.MetricType, + FracDigits: byte(rec.FracDigits), + TimestampsBuf: timestampsBuf, + ValuesBuf: valuesBuf, + Timestamps: chunkenc.NewReverseTimeDeltaOfDeltaCompressor(timestampsBuf, 0), + Values: values, + } + + case txlog.DeletedMetric: + delete(s.metrics, rec.MetricID) + if len(rec.FreeDataPages) > 0 { + s.dataFreeList.AddPages(rec.FreeDataPages) + } + if len(rec.FreeIndexPages) > 0 { + s.indexFreeList.AddPages(rec.FreeIndexPages) + } + + case txlog.AppendedMeasure: + metric, ok := s.metrics[rec.MetricID] + if ok { + metric.Timestamps.Append(rec.Timestamp) + metric.Values.Append(rec.Value) + + if metric.Since == 0 { + metric.Since = rec.Timestamp + metric.SinceValue = rec.Value + } + + metric.Until = rec.Timestamp + metric.UntilValue = rec.Value + } + + case txlog.AppendedMeasures: + metric, ok := s.metrics[rec.MetricID] + if ok { + for _, measure := range rec.Measures { + metric.Timestamps.Append(measure.Timestamp) + metric.Values.Append(measure.Value) + + if metric.Since == 0 { + metric.Since = measure.Timestamp + metric.SinceValue = measure.Value + } + + metric.Until = measure.Timestamp + metric.UntilValue = measure.Value + } + } + + case txlog.AppendedMeasureWithOverflow: + metric, ok := s.metrics[rec.MetricID] + if ok { + metric.ReinitBy(rec.Timestamp, rec.Value) + if rec.IsRootChanged { + metric.RootPageNo = rec.RootPageNo + } + metric.LastPageNo = rec.DataPageNo + // delete free pages + if rec.IsDataPageReused { + s.dataFreeList.DeletePages([]uint32{ + rec.DataPageNo, + }) + } + if len(rec.ReusedIndexPages) > 0 { + s.indexFreeList.DeletePages(rec.ReusedIndexPages) + } + } + + case txlog.DeletedMeasures: + metric, ok := s.metrics[rec.MetricID] + if ok { + metric.DeleteMeasures() + if len(rec.FreeDataPages) > 0 { + s.dataFreeList.AddPages(rec.FreeDataPages) + } + if len(rec.FreeDataPages) > 0 { + s.indexFreeList.AddPages(rec.FreeIndexPages) + } + } + + default: + diploma.Abort(diploma.UnknownTxLogRecordTypeBug, + fmt.Errorf("bug: unknown record type %T in TransactionLog", rec)) + } + return nil +} diff --git a/database/helpers.go b/database/helpers.go new file mode 100644 index 0000000..8fbcf89 --- /dev/null +++ b/database/helpers.go @@ -0,0 +1,50 @@ +package database + +import ( + "errors" + "io/fs" + "os" + "time" +) + +func isFileExist(fileName string) (bool, error) { + _, err := os.Stat(fileName) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } else { + return false, err + } + } else { + return true, nil + } +} + +func (s *Database) appendJobToWorkerQueue(job any) { + s.mutex.Lock() + s.workerQueue = append(s.workerQueue, job) + s.mutex.Unlock() + + select { + case s.workerSignalCh <- struct{}{}: + default: + } +} + +func (s *Database) metricRUnlock(metricID uint32) { + s.mutex.Lock() + s.rLocksToRelease = append(s.rLocksToRelease, metricID) + s.mutex.Unlock() + + select { + case s.workerSignalCh <- struct{}{}: + default: + } +} + +func correctToFHD(since, until uint32, firstHourOfDay int) (uint32, uint32) { + duration := time.Duration(firstHourOfDay) * time.Hour + since = uint32(time.Unix(int64(since), 0).Add(duration).Unix()) + until = uint32(time.Unix(int64(until), 0).Add(duration).Unix()) + return since, until +} diff --git a/database/metric.go b/database/metric.go new file mode 100644 index 0000000..2b4293a --- /dev/null +++ b/database/metric.go @@ -0,0 +1,71 @@ +package database + +import ( + octopus "gordenko.dev/dima/diploma" + "gordenko.dev/dima/diploma/chunkenc" + "gordenko.dev/dima/diploma/conbuf" +) + +// METRIC + +type _metric struct { + MetricType octopus.MetricType + FracDigits byte + RootPageNo uint32 + LastPageNo uint32 + SinceValue float64 + Since uint32 + UntilValue float64 + Until uint32 + TimestampsBuf *conbuf.ContinuousBuffer + ValuesBuf *conbuf.ContinuousBuffer + Timestamps octopus.TimestampCompressor + Values octopus.ValueCompressor +} + +func (s *_metric) ReinitBy(timestamp uint32, value float64) { + s.TimestampsBuf = conbuf.New(nil) + s.ValuesBuf = conbuf.New(nil) + // + s.Timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor( + s.TimestampsBuf, 0) + + if s.MetricType == octopus.Cumulative { + s.Values = chunkenc.NewReverseCumulativeDeltaCompressor( + s.ValuesBuf, 0, s.FracDigits) + } else { + s.Values = chunkenc.NewReverseInstantDeltaCompressor( + s.ValuesBuf, 0, s.FracDigits) + } + + s.Timestamps.Append(timestamp) + s.Values.Append(value) + + s.Since = timestamp + s.SinceValue = value + s.Until = timestamp + s.UntilValue = value +} + +func (s *_metric) DeleteMeasures() { + s.TimestampsBuf = conbuf.New(nil) + s.ValuesBuf = conbuf.New(nil) + // + s.Timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor( + s.TimestampsBuf, 0) + + if s.MetricType == octopus.Cumulative { + s.Values = chunkenc.NewReverseCumulativeDeltaCompressor( + s.ValuesBuf, 0, s.FracDigits) + } else { + s.Values = chunkenc.NewReverseInstantDeltaCompressor( + s.ValuesBuf, 0, s.FracDigits) + } + + s.RootPageNo = 0 + s.LastPageNo = 0 + s.Since = 0 + s.SinceValue = 0 + s.Until = 0 + s.UntilValue = 0 +} diff --git a/database/proc.go b/database/proc.go new file mode 100644 index 0000000..5d7e330 --- /dev/null +++ b/database/proc.go @@ -0,0 +1,1736 @@ +package database + +import ( + "fmt" + + "gordenko.dev/dima/diploma" + "gordenko.dev/dima/diploma/atree" + "gordenko.dev/dima/diploma/chunkenc" + "gordenko.dev/dima/diploma/conbuf" + "gordenko.dev/dima/diploma/enc" + "gordenko.dev/dima/diploma/txlog" +) + +const ( + QueryDone = 1 + UntilFound = 2 + UntilNotFound = 3 + NoMetric = 4 + MetricDuplicate = 5 + Succeed = 6 + NewPage = 7 + ExpiredMeasure = 8 + NonMonotonicValue = 9 + CanAppend = 10 + WrongMetricType = 11 +) + +func (s *Database) worker() { + for { + select { + case <-s.workerSignalCh: + s.DoWork() + } + } +} + +func (s *Database) DoWork() { + s.mutex.Lock() + rLocksToRelease := s.rLocksToRelease + workerQueue := s.workerQueue + s.rLocksToRelease = nil + s.workerQueue = nil + s.mutex.Unlock() + + for _, metricID := range rLocksToRelease { + lockEntry, ok := s.metricLockEntries[metricID] + if !ok { + diploma.Abort(diploma.NoLockEntryBug, + fmt.Errorf("drainQueues: lockEntry not found for the metric %d", + metricID)) + } + + if lockEntry.XLock { + diploma.Abort(diploma.XLockBug, + fmt.Errorf("drainQueues: xlock is set for the metric %d", + metricID)) + } + + if lockEntry.RLocks <= 0 { + diploma.Abort(diploma.NoRLockBug, + fmt.Errorf("drainQueues: rlock not set for the metric %d", + metricID)) + } + + lockEntry.RLocks-- + + if len(lockEntry.WaitQueue) > 0 { + metric, ok := s.metrics[metricID] + if !ok { + diploma.Abort(diploma.NoMetricBug, + fmt.Errorf("drainQueues: metric %d not found", metricID)) + } + s.processMetricQueue(metricID, metric, lockEntry) + } else { + if lockEntry.RLocks == 0 { + delete(s.metricLockEntries, metricID) + } + } + } + + for _, untyped := range workerQueue { + switch req := untyped.(type) { + case tryAppendMeasureReq: + s.tryAppendMeasure(req) + + case tryAppendMeasuresReq: + s.tryAppendMeasures(req) + + case txlog.Changes: + s.applyChanges(req) // all metrics only + + case tryListCurrentValuesReq: + s.tryListCurrentValues(req) // all metrics only + + case tryListCumulativePeriodsReq: + s.tryListCumulativePeriods(req) + + case tryListInstantPeriodsReq: + s.tryListInstantPeriods(req) + + case tryListCumulativeMeasuresReq: + s.tryListCumulativeMeasures(req) + + case tryListInstantMeasuresReq: + s.tryListInstantMeasures(req) + + case tryListAllInstantMeasuresReq: + s.tryListAllInstantMeasures(req) + + case tryListAllCumulativeMeasuresReq: + s.tryListAllCumulativeMeasures(req) + + case tryAddMetricReq: + s.tryAddMetric(req) + + case tryDeleteMetricReq: + s.tryDeleteMetric(req) + + case tryDeleteMeasuresReq: + s.tryDeleteMeasures(req) + + case tryGetMetricReq: + s.tryGetMetric(req) + + default: + diploma.Abort(diploma.UnknownWorkerQueueItemBug, + fmt.Errorf("bug: unknown worker queue item type %T", req)) + } + } +} + +func (s *Database) processMetricQueue(metricID uint32, metric *_metric, lockEntry *metricLockEntry) { + if len(lockEntry.WaitQueue) == 0 { + return + } + + var modificationReqs []any + + for _, untyped := range lockEntry.WaitQueue { + var rLockRequired bool + switch req := untyped.(type) { + case tryListCumulativePeriodsReq: + rLockRequired = s.startListCumulativePeriods(metric, req) + + case tryListInstantPeriodsReq: + rLockRequired = s.startListInstantPeriods(metric, req) + + case tryListCumulativeMeasuresReq: + rLockRequired = s.startListCumulativeMeasures(metric, req) + + case tryListInstantMeasuresReq: + rLockRequired = s.startListInstantMeasures(metric, req) + + case tryListAllInstantMeasuresReq: + rLockRequired = s.startListAllInstantMeasures(metric, req) + + case tryListAllCumulativeMeasuresReq: + rLockRequired = s.startListAllCumulativeMeasures(metric, req) + + case tryGetMetricReq: + s.tryGetMetric(req) + + default: + modificationReqs = append(modificationReqs, untyped) + } + + if rLockRequired { + lockEntry.RLocks++ + } + } + + lockEntry.WaitQueue = nil + + if lockEntry.RLocks > 0 { + lockEntry.WaitQueue = modificationReqs + } else { + for idx, untyped := range modificationReqs { + switch req := untyped.(type) { + case tryAppendMeasureReq: + s.startAppendMeasure(metric, req, nil) + + case tryAppendMeasuresReq: + s.startAppendMeasures(metric, req, nil) + + case tryDeleteMetricReq: + s.startDeleteMetric(metric, req) + + case tryDeleteMeasuresReq: + s.startDeleteMeasures(metric, req) + + default: + diploma.Abort(diploma.UnknownMetricWaitQueueItemBug, + fmt.Errorf("bug: unknown metric wait queue item type %T", req)) + } + + lockEntry, ok := s.metricLockEntries[metricID] + if ok { + start := idx + 1 + if start < len(modificationReqs) { + lockEntry.WaitQueue = append(lockEntry.WaitQueue, modificationReqs[start:]...) + } + break + } + } + } +} + +type tryAddMetricReq struct { + MetricID uint32 + ResultCh chan byte +} + +func (s *Database) tryAddMetric(req tryAddMetricReq) { + _, ok := s.metrics[req.MetricID] + if ok { + req.ResultCh <- MetricDuplicate + return + } + + lockEntry, ok := s.metricLockEntries[req.MetricID] + if ok { + lockEntry.WaitQueue = append(lockEntry.WaitQueue, req) + } else { + s.metricLockEntries[req.MetricID] = &metricLockEntry{ + XLock: true, + } + req.ResultCh <- Succeed + } +} + +func (s *Database) processTryAddMetricReqsImmediatelyAfterDelete(reqs []tryAddMetricReq) { + if len(reqs) == 0 { + return + } + var ( + req = reqs[0] + waitQueue []any + ) + if len(reqs) > 1 { + for _, req := range reqs[1:] { + waitQueue = append(waitQueue, req) + } + } + s.metricLockEntries[req.MetricID] = &metricLockEntry{ + XLock: true, + WaitQueue: waitQueue, + } + req.ResultCh <- Succeed +} + +type tryGetMetricReq struct { + MetricID uint32 + ResultCh chan Metric +} + +func (s *Database) tryGetMetric(req tryGetMetricReq) { + metric, ok := s.metrics[req.MetricID] + if ok { + req.ResultCh <- Metric{ + ResultCode: Succeed, + MetricType: metric.MetricType, + FracDigits: metric.FracDigits, + } + } else { + req.ResultCh <- Metric{ + ResultCode: NoMetric, + } + } +} + +type tryDeleteMetricReq struct { + MetricID uint32 + ResultCh chan tryDeleteMetricResult +} + +func (s *Database) tryDeleteMetric(req tryDeleteMetricReq) { + metric, ok := s.metrics[req.MetricID] + if !ok { + req.ResultCh <- tryDeleteMetricResult{ + ResultCode: NoMetric, + } + return + } + + lockEntry, ok := s.metricLockEntries[req.MetricID] + if ok { + lockEntry.WaitQueue = append(lockEntry.WaitQueue, req) + } else { + s.startDeleteMetric(metric, req) + } +} + +func (s *Database) startDeleteMetric(metric *_metric, req tryDeleteMetricReq) { + s.metricLockEntries[req.MetricID] = &metricLockEntry{ + XLock: true, + } + req.ResultCh <- tryDeleteMetricResult{ + ResultCode: Succeed, + RootPageNo: metric.RootPageNo, + } +} + +type tryDeleteMeasuresReq struct { + MetricID uint32 + Since uint32 + ResultCh chan tryDeleteMeasuresResult +} + +func (s *Database) tryDeleteMeasures(req tryDeleteMeasuresReq) { + metric, ok := s.metrics[req.MetricID] + if !ok { + req.ResultCh <- tryDeleteMeasuresResult{ + ResultCode: NoMetric, + } + return + } + + lockEntry, ok := s.metricLockEntries[req.MetricID] + if ok { + lockEntry.WaitQueue = append(lockEntry.WaitQueue, req) + } else { + s.startDeleteMeasures(metric, req) + } +} + +func (s *Database) startDeleteMeasures(metric *_metric, req tryDeleteMeasuresReq) { + s.metricLockEntries[req.MetricID] = &metricLockEntry{ + XLock: true, + } + req.ResultCh <- tryDeleteMeasuresResult{ + ResultCode: Succeed, + RootPageNo: metric.RootPageNo, + } +} + +// SELECT + +type tryListAllInstantMeasuresReq struct { + MetricID uint32 + ResponseWriter *atree.InstantMeasureWriter + ResultCh chan instantMeasuresResult +} + +func (s *Database) tryListAllInstantMeasures(req tryListAllInstantMeasuresReq) { + metric, ok := s.metrics[req.MetricID] + if !ok { + req.ResultCh <- instantMeasuresResult{ + ResultCode: NoMetric, + } + return + } + + if metric.MetricType != diploma.Instant { + req.ResultCh <- instantMeasuresResult{ + ResultCode: WrongMetricType, + } + return + } + + lockEntry, ok := s.metricLockEntries[req.MetricID] + if ok { + if lockEntry.XLock { + lockEntry.WaitQueue = append(lockEntry.WaitQueue, req) + return + } + } + + if s.startListAllInstantMeasures(metric, req) { + if lockEntry != nil { + lockEntry.RLocks++ + } else { + s.metricLockEntries[req.MetricID] = &metricLockEntry{ + RLocks: 1, + } + } + } +} + +func (s *Database) startListAllInstantMeasures(metric *_metric, req tryListAllInstantMeasuresReq) bool { + if metric.Since == 0 { + req.ResultCh <- instantMeasuresResult{ + ResultCode: QueryDone, + } + return false + } + + timestampDecompressor := chunkenc.NewReverseTimeDeltaOfDeltaDecompressor( + metric.TimestampsBuf, + metric.Timestamps.Size(), + ) + + valueDecompressor := chunkenc.NewReverseInstantDeltaDecompressor( + metric.ValuesBuf, + metric.Values.Size(), + metric.FracDigits, + ) + + var ( + value float64 + timestamp uint32 + done bool + + buffered int + ) + + for { + timestamp, done = timestampDecompressor.NextValue() + if done { + break + } + + value, done = valueDecompressor.NextValue() + if done { + diploma.Abort(diploma.HasTimestampNoValueBug, ErrNoValueBug) + } + + req.ResponseWriter.BufferMeasure(atree.InstantMeasure{ + Timestamp: timestamp, + Value: value, + }) + + buffered++ + } + + if metric.LastPageNo > 0 { + req.ResultCh <- instantMeasuresResult{ + ResultCode: UntilFound, + PageNo: metric.LastPageNo, + FracDigits: metric.FracDigits, + } + return true + } else { + req.ResultCh <- instantMeasuresResult{ + ResultCode: QueryDone, + } + return false + } +} + +type tryListInstantMeasuresReq struct { + MetricID uint32 + Since uint32 + Until uint32 + ResponseWriter *atree.InstantMeasureWriter + ResultCh chan instantMeasuresResult +} + +func (s *Database) tryListInstantMeasures(req tryListInstantMeasuresReq) { + metric, ok := s.metrics[req.MetricID] + if !ok { + req.ResultCh <- instantMeasuresResult{ + ResultCode: NoMetric, + } + return + } + + if metric.MetricType != diploma.Instant { + req.ResultCh <- instantMeasuresResult{ + ResultCode: WrongMetricType, + } + return + } + + lockEntry, ok := s.metricLockEntries[req.MetricID] + if ok { + if lockEntry.XLock { + lockEntry.WaitQueue = append(lockEntry.WaitQueue, req) + return + } + } + if s.startListInstantMeasures(metric, req) { + if lockEntry != nil { + lockEntry.RLocks++ + } else { + s.metricLockEntries[req.MetricID] = &metricLockEntry{ + RLocks: 1, + } + } + } +} + +func (s *Database) startListInstantMeasures(metric *_metric, req tryListInstantMeasuresReq) bool { + if metric.Since == 0 { + req.ResultCh <- instantMeasuresResult{ + ResultCode: QueryDone, + } + return false + } + + if req.Since > metric.Until { + req.ResultCh <- instantMeasuresResult{ + ResultCode: QueryDone, + } + return false + } + + if req.Until < metric.Since { + if metric.RootPageNo > 0 { + req.ResultCh <- instantMeasuresResult{ + ResultCode: UntilNotFound, + PageNo: metric.RootPageNo, + FracDigits: metric.FracDigits, + } + return true + } else { + req.ResultCh <- instantMeasuresResult{ + ResultCode: QueryDone, + } + return false + } + } + + timestampDecompressor := chunkenc.NewReverseTimeDeltaOfDeltaDecompressor( + metric.TimestampsBuf, + metric.Timestamps.Size(), + ) + + valueDecompressor := chunkenc.NewReverseInstantDeltaDecompressor( + metric.ValuesBuf, + metric.Values.Size(), + metric.FracDigits, + ) + + var ( + value float64 + timestamp uint32 + done bool + ) + + for { + timestamp, done = timestampDecompressor.NextValue() + if done { + break + } + + value, done = valueDecompressor.NextValue() + if done { + diploma.Abort(diploma.HasTimestampNoValueBug, ErrNoValueBug) + } + + if timestamp <= req.Until { + if timestamp < req.Since { + req.ResultCh <- instantMeasuresResult{ + ResultCode: QueryDone, + } + return false + } + + req.ResponseWriter.BufferMeasure(atree.InstantMeasure{ + Timestamp: timestamp, + Value: value, + }) + + if timestamp == req.Since { + req.ResultCh <- instantMeasuresResult{ + ResultCode: QueryDone, + } + return false + } + } + } + + if metric.LastPageNo > 0 { + req.ResultCh <- instantMeasuresResult{ + ResultCode: UntilFound, + PageNo: metric.LastPageNo, + FracDigits: metric.FracDigits, + } + return true + } else { + req.ResultCh <- instantMeasuresResult{ + ResultCode: QueryDone, + } + return false + } +} + +// CUMULATIVE + +type tryListAllCumulativeMeasuresReq struct { + MetricID uint32 + ResponseWriter *atree.CumulativeMeasureWriter + ResultCh chan cumulativeMeasuresResult +} + +func (s *Database) tryListAllCumulativeMeasures(req tryListAllCumulativeMeasuresReq) { + metric, ok := s.metrics[req.MetricID] + if !ok { + req.ResultCh <- cumulativeMeasuresResult{ + ResultCode: NoMetric, + } + return + } + if metric.MetricType != diploma.Cumulative { + req.ResultCh <- cumulativeMeasuresResult{ + ResultCode: WrongMetricType, + } + return + } + + lockEntry, ok := s.metricLockEntries[req.MetricID] + if ok { + if lockEntry.XLock { + lockEntry.WaitQueue = append(lockEntry.WaitQueue, req) + return + } + } + if s.startListAllCumulativeMeasures(metric, req) { + if lockEntry != nil { + lockEntry.RLocks++ + } else { + s.metricLockEntries[req.MetricID] = &metricLockEntry{ + RLocks: 1, + } + } + } +} + +func (s *Database) startListAllCumulativeMeasures(metric *_metric, req tryListAllCumulativeMeasuresReq) bool { + if metric.Since == 0 { + req.ResultCh <- cumulativeMeasuresResult{ + ResultCode: QueryDone, + } + return false + } + + timestampDecompressor := chunkenc.NewReverseTimeDeltaOfDeltaDecompressor( + metric.TimestampsBuf, + metric.Timestamps.Size(), + ) + + valueDecompressor := chunkenc.NewReverseCumulativeDeltaDecompressor( + metric.ValuesBuf, + metric.Values.Size(), + metric.FracDigits, + ) + + endTimestamp, done := timestampDecompressor.NextValue() + if done { + req.ResultCh <- cumulativeMeasuresResult{ + ResultCode: QueryDone, + } + return false + } + + endValue, done := valueDecompressor.NextValue() + if done { + diploma.Abort(diploma.HasTimestampNoValueBug, ErrNoValueBug) + } + + for { + timestamp, done := timestampDecompressor.NextValue() + if done { + break + } + + value, done := valueDecompressor.NextValue() + if done { + diploma.Abort(diploma.HasTimestampNoValueBug, ErrNoValueBug) + } + + req.ResponseWriter.WriteMeasure(atree.CumulativeMeasure{ + Timestamp: endTimestamp, + Value: endValue, + Total: endValue - value, + }) + endTimestamp = timestamp + endValue = value + } + + if metric.LastPageNo > 0 { + req.ResultCh <- cumulativeMeasuresResult{ + ResultCode: UntilFound, + PageNo: metric.LastPageNo, + FracDigits: metric.FracDigits, + EndTimestamp: endTimestamp, + EndValue: endValue, + } + return true + } else { + req.ResponseWriter.WriteMeasure(atree.CumulativeMeasure{ + Timestamp: endTimestamp, + Value: endValue, + Total: endValue, + }) + req.ResultCh <- cumulativeMeasuresResult{ + ResultCode: QueryDone, + } + return false + } +} + +type tryListCumulativeMeasuresReq struct { + MetricID uint32 + Since uint32 + Until uint32 + ResponseWriter *atree.CumulativeMeasureWriter + ResultCh chan cumulativeMeasuresResult +} + +func (s *Database) tryListCumulativeMeasures(req tryListCumulativeMeasuresReq) { + metric, ok := s.metrics[req.MetricID] + if !ok { + req.ResultCh <- cumulativeMeasuresResult{ + ResultCode: NoMetric, + } + return + } + if metric.MetricType != diploma.Cumulative { + req.ResultCh <- cumulativeMeasuresResult{ + ResultCode: WrongMetricType, + } + return + } + + lockEntry, ok := s.metricLockEntries[req.MetricID] + if ok { + if lockEntry.XLock { + lockEntry.WaitQueue = append(lockEntry.WaitQueue, req) + return + } + } + + if s.startListCumulativeMeasures(metric, req) { + if lockEntry != nil { + lockEntry.RLocks++ + } else { + s.metricLockEntries[req.MetricID] = &metricLockEntry{ + RLocks: 1, + } + } + } +} + +func (s *Database) startListCumulativeMeasures(metric *_metric, req tryListCumulativeMeasuresReq) bool { + if metric.Since == 0 { + req.ResultCh <- cumulativeMeasuresResult{ + ResultCode: QueryDone, + } + return false + } + + if req.Since > metric.Until { + req.ResultCh <- cumulativeMeasuresResult{ + ResultCode: QueryDone, + } + return false + } + + if req.Until < metric.Since { + // search in tree + if metric.RootPageNo > 0 { + req.ResultCh <- cumulativeMeasuresResult{ + ResultCode: UntilNotFound, + PageNo: metric.RootPageNo, + FracDigits: metric.FracDigits, + } + return true + } else { + req.ResultCh <- cumulativeMeasuresResult{ + ResultCode: QueryDone, + } + return false + } + } + + timestampDecompressor := chunkenc.NewReverseTimeDeltaOfDeltaDecompressor( + metric.TimestampsBuf, + metric.Timestamps.Size(), + ) + + valueDecompressor := chunkenc.NewReverseCumulativeDeltaDecompressor( + metric.ValuesBuf, + metric.Values.Size(), + metric.FracDigits, + ) + + var ( + endTimestamp uint32 + endValue float64 + ) + + for { + timestamp, done := timestampDecompressor.NextValue() + if done { + break + } + + value, done := valueDecompressor.NextValue() + if done { + diploma.Abort(diploma.HasTimestampNoValueBug, ErrNoValueBug) + } + + if timestamp > req.Until { + continue + } + + if timestamp < req.Since { + req.ResponseWriter.WriteMeasure(atree.CumulativeMeasure{ + Timestamp: endTimestamp, + Value: endValue, + Total: endValue - value, + }) + req.ResultCh <- cumulativeMeasuresResult{ + ResultCode: QueryDone, + } + return false + } + + if endTimestamp > 0 { + req.ResponseWriter.WriteMeasure(atree.CumulativeMeasure{ + Timestamp: endTimestamp, + Value: endValue, + Total: endValue - value, + }) + } + endTimestamp = timestamp + endValue = value + } + + if metric.LastPageNo > 0 { + req.ResultCh <- cumulativeMeasuresResult{ + ResultCode: UntilFound, + PageNo: metric.LastPageNo, + FracDigits: metric.FracDigits, + EndTimestamp: endTimestamp, + EndValue: endValue, + } + return true + } else { + req.ResponseWriter.WriteMeasure(atree.CumulativeMeasure{ + Timestamp: endTimestamp, + Value: endValue, + Total: endValue, + }) + req.ResultCh <- cumulativeMeasuresResult{ + ResultCode: QueryDone, + } + return false + } +} + +type tryAppendMeasureReq struct { + MetricID uint32 + Timestamp uint32 + Value float64 + ResultCh chan tryAppendMeasureResult +} + +func (s *Database) tryAppendMeasure(req tryAppendMeasureReq) { + metric, ok := s.metrics[req.MetricID] + if !ok { + req.ResultCh <- tryAppendMeasureResult{ + ResultCode: NoMetric, + } + return + } + + lockEntry, ok := s.metricLockEntries[req.MetricID] + if ok { + if lockEntry.XLock { + lockEntry.WaitQueue = append(lockEntry.WaitQueue, req) + return + } + } + s.startAppendMeasure(metric, req, lockEntry) +} + +func (s *Database) startAppendMeasure(metric *_metric, req tryAppendMeasureReq, lockEntry *metricLockEntry) { + if req.Timestamp <= metric.Until { + req.ResultCh <- tryAppendMeasureResult{ + ResultCode: ExpiredMeasure, + } + return + } + + if metric.MetricType == diploma.Cumulative && req.Value < metric.UntilValue { + req.ResultCh <- tryAppendMeasureResult{ + ResultCode: NonMonotonicValue, + } + return + } + + extraSpace := metric.Timestamps.CalcRequiredSpace(req.Timestamp) + + metric.Values.CalcRequiredSpace(req.Value) + + totalSpace := metric.Timestamps.Size() + metric.Values.Size() + extraSpace + + if totalSpace <= atree.DataPagePayloadSize { + if lockEntry != nil { + lockEntry.RLocks++ + } else { + s.metricLockEntries[req.MetricID] = &metricLockEntry{ + RLocks: 1, + } + } + req.ResultCh <- tryAppendMeasureResult{ + ResultCode: CanAppend, + } + } else { + if lockEntry != nil { + if lockEntry.RLocks > 0 { + lockEntry.WaitQueue = append(lockEntry.WaitQueue, req) + return + } + lockEntry.XLock = true + } else { + s.metricLockEntries[req.MetricID] = &metricLockEntry{ + XLock: true, + } + } + + req.ResultCh <- tryAppendMeasureResult{ + ResultCode: NewPage, + FilledPage: &FilledPage{ + Since: metric.Since, + RootPageNo: metric.RootPageNo, + PrevPageNo: metric.LastPageNo, + TimestampsChunks: metric.TimestampsBuf.Chunks(), + TimestampsSize: uint16(metric.Timestamps.Size()), + ValuesChunks: metric.ValuesBuf.Chunks(), + ValuesSize: uint16(metric.Values.Size()), + }, + } + } +} + +func (s *Database) appendMeasure(rec txlog.AppendedMeasure) { + metric, ok := s.metrics[rec.MetricID] + if !ok { + diploma.Abort(diploma.NoMetricBug, + fmt.Errorf("appendMeasure: metric %d not found", + rec.MetricID)) + } + + lockEntry, ok := s.metricLockEntries[rec.MetricID] + if !ok { + diploma.Abort(diploma.NoLockEntryBug, + fmt.Errorf("appendMeasure: lockEntry not found for the metric %d", + rec.MetricID)) + } + + if lockEntry.XLock { + diploma.Abort(diploma.XLockBug, + fmt.Errorf("appendMeasure: xlock is set for the metric %d", + rec.MetricID)) + } + + if lockEntry.RLocks <= 0 { + diploma.Abort(diploma.NoRLockBug, + fmt.Errorf("appendMeasure: rlock not set for the metric %d", + rec.MetricID)) + } + + if metric.Since == 0 { + metric.Since = rec.Timestamp + metric.SinceValue = rec.Value + } + + metric.Timestamps.Append(rec.Timestamp) + metric.Values.Append(rec.Value) + + metric.Until = rec.Timestamp + metric.UntilValue = rec.Value + + lockEntry.RLocks-- + if len(lockEntry.WaitQueue) > 0 { + s.processMetricQueue(rec.MetricID, metric, lockEntry) + } else { + if lockEntry.RLocks == 0 { + delete(s.metricLockEntries, rec.MetricID) + } + } +} + +func (s *Database) appendMeasures(extended txlog.AppendedMeasuresExtended) { + rec := extended.Record + metric, ok := s.metrics[rec.MetricID] + if !ok { + diploma.Abort(diploma.NoMetricBug, + fmt.Errorf("appendMeasureAfterOverflow: metric %d not found", + rec.MetricID)) + } + + lockEntry, ok := s.metricLockEntries[rec.MetricID] + if !ok { + diploma.Abort(diploma.NoLockEntryBug, + fmt.Errorf("appendMeasureAfterOverflow: lockEntry not found for the metric %d", + rec.MetricID)) + } + + if !lockEntry.XLock { + diploma.Abort(diploma.NoXLockBug, + fmt.Errorf("appendMeasureAfterOverflow: xlock not set for the metric %d", + rec.MetricID)) + } + + for _, measure := range rec.Measures { + if metric.Since == 0 { + metric.Since = measure.Timestamp + metric.SinceValue = measure.Value + } + + metric.Timestamps.Append(measure.Timestamp) + metric.Values.Append(measure.Value) + + metric.Until = measure.Timestamp + metric.UntilValue = measure.Value + } + + if !extended.HoldLock { + lockEntry.XLock = false + s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry) + } +} + +func (s *Database) appendMeasureAfterOverflow(extended txlog.AppendedMeasureWithOverflowExtended) { + rec := extended.Record + metric, ok := s.metrics[rec.MetricID] + if !ok { + diploma.Abort(diploma.NoMetricBug, + fmt.Errorf("appendMeasureAfterOverflow: metric %d not found", + rec.MetricID)) + } + + lockEntry, ok := s.metricLockEntries[rec.MetricID] + if !ok { + diploma.Abort(diploma.NoLockEntryBug, + fmt.Errorf("appendMeasureAfterOverflow: lockEntry not found for the metric %d", + rec.MetricID)) + } + + if !lockEntry.XLock { + diploma.Abort(diploma.NoXLockBug, + fmt.Errorf("appendMeasureAfterOverflow: xlock not set for the metric %d", + rec.MetricID)) + } + + metric.ReinitBy(rec.Timestamp, rec.Value) + if rec.IsRootChanged { + metric.RootPageNo = rec.RootPageNo + } + metric.LastPageNo = rec.DataPageNo + + // delete free pages + if rec.IsDataPageReused { + s.dataFreeList.DeletePages([]uint32{ + rec.DataPageNo, + }) + } + + if len(rec.ReusedIndexPages) > 0 { + s.indexFreeList.DeletePages(rec.ReusedIndexPages) + } + + if !extended.HoldLock { + lockEntry.XLock = false + s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry) + } +} + +type tryAppendMeasuresReq struct { + MetricID uint32 + ResultCh chan tryAppendMeasuresResult +} + +func (s *Database) tryAppendMeasures(req tryAppendMeasuresReq) { + metric, ok := s.metrics[req.MetricID] + if !ok { + req.ResultCh <- tryAppendMeasuresResult{ + ResultCode: NoMetric, + } + return + } + + lockEntry, ok := s.metricLockEntries[req.MetricID] + if ok { + if lockEntry.XLock { + lockEntry.WaitQueue = append(lockEntry.WaitQueue, req) + return + } + } + s.startAppendMeasures(metric, req, lockEntry) +} + +func (s *Database) startAppendMeasures(metric *_metric, req tryAppendMeasuresReq, lockEntry *metricLockEntry) { + if lockEntry != nil { + if lockEntry.RLocks > 0 { + lockEntry.WaitQueue = append(lockEntry.WaitQueue, req) + return + } + lockEntry.XLock = true + } else { + s.metricLockEntries[req.MetricID] = &metricLockEntry{ + XLock: true, + } + } + + var ( + timestampsBuf *conbuf.ContinuousBuffer + valuesBuf *conbuf.ContinuousBuffer + timestamps diploma.TimestampCompressor + values diploma.ValueCompressor + ) + + if metric.Since > 0 { + timestampsBuf = metric.TimestampsBuf.Copy() + valuesBuf = metric.ValuesBuf.Copy() + + timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor( + timestampsBuf, metric.Timestamps.Size()) + + if metric.MetricType == diploma.Cumulative { + values = chunkenc.NewReverseCumulativeDeltaCompressor( + valuesBuf, metric.Values.Size(), metric.FracDigits) + } else { + values = chunkenc.NewReverseInstantDeltaCompressor( + valuesBuf, metric.Values.Size(), metric.FracDigits) + } + } else { + timestampsBuf = conbuf.New(nil) + valuesBuf = conbuf.New(nil) + timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor( + timestampsBuf, 0) + if metric.MetricType == diploma.Cumulative { + values = chunkenc.NewReverseCumulativeDeltaCompressor( + valuesBuf, 0, metric.FracDigits) + } else { + values = chunkenc.NewReverseInstantDeltaCompressor( + valuesBuf, 0, metric.FracDigits) + } + } + + req.ResultCh <- tryAppendMeasuresResult{ + ResultCode: CanAppend, + MetricType: metric.MetricType, + FracDigits: metric.FracDigits, + Since: metric.Since, + Until: metric.Until, + UntilValue: metric.UntilValue, + RootPageNo: metric.RootPageNo, + PrevPageNo: metric.LastPageNo, + TimestampsBuf: timestampsBuf, + ValuesBuf: valuesBuf, + Timestamps: timestamps, + Values: values, + } +} + +type tryListCumulativePeriodsReq struct { + MetricID uint32 + Since uint32 + Until uint32 + Aggregator *atree.CumulativeAggregator + ResponseWriter *atree.CumulativePeriodsWriter + ResultCh chan cumulativePeriodsResult +} + +func (s *Database) tryListCumulativePeriods(req tryListCumulativePeriodsReq) { + metric, ok := s.metrics[req.MetricID] + if !ok { + req.ResultCh <- cumulativePeriodsResult{ + ResultCode: NoMetric, + } + return + } + if metric.MetricType != diploma.Cumulative { + req.ResultCh <- cumulativePeriodsResult{ + ResultCode: WrongMetricType, + } + return + } + + lockEntry, ok := s.metricLockEntries[req.MetricID] + if ok { + if lockEntry.XLock { + lockEntry.WaitQueue = append(lockEntry.WaitQueue, req) + return + } + } + + if s.startListCumulativePeriods(metric, req) { + if lockEntry != nil { + lockEntry.RLocks++ + } else { + s.metricLockEntries[req.MetricID] = &metricLockEntry{ + RLocks: 1, + } + } + } +} + +func (s *Database) startListCumulativePeriods(metric *_metric, req tryListCumulativePeriodsReq) bool { + if metric.Since == 0 { + req.ResultCh <- cumulativePeriodsResult{ + ResultCode: QueryDone, + } + return false + } + + if req.Since > metric.Until { + req.ResultCh <- cumulativePeriodsResult{ + ResultCode: QueryDone, + } + return false + } + + if req.Until < metric.Since { + if metric.RootPageNo > 0 { + req.ResultCh <- cumulativePeriodsResult{ + ResultCode: UntilNotFound, + PageNo: metric.RootPageNo, + FracDigits: metric.FracDigits, + } + return true + } else { + req.ResultCh <- cumulativePeriodsResult{ + ResultCode: QueryDone, + } + return false + } + } + + timestampDecompressor := chunkenc.NewReverseTimeDeltaOfDeltaDecompressor( + metric.TimestampsBuf, + metric.Timestamps.Size(), + ) + + valueDecompressor := chunkenc.NewReverseCumulativeDeltaDecompressor( + metric.ValuesBuf, + metric.Values.Size(), + metric.FracDigits, + ) + + var period atree.CumulativePeriod + + for { + timestamp, done := timestampDecompressor.NextValue() + if done { + break + } + + value, done := valueDecompressor.NextValue() + if done { + diploma.Abort(diploma.HasTimestampNoValueBug, ErrNoValueBug) + } + + if timestamp > req.Until { + continue + } + + if timestamp < req.Since { + isCompleted := req.Aggregator.FillPeriod(timestamp, value, &period) + if isCompleted { + req.ResponseWriter.WritePeriod(period) + } + req.ResultCh <- cumulativePeriodsResult{ + ResultCode: QueryDone, + } + return false + } + + if timestamp <= req.Until { + isCompleted := req.Aggregator.Feed(timestamp, value, &period) + if isCompleted { + req.ResponseWriter.WritePeriod(period) + } + } + } + + if metric.LastPageNo > 0 { + req.ResultCh <- cumulativePeriodsResult{ + ResultCode: UntilFound, + PageNo: metric.LastPageNo, + FracDigits: metric.FracDigits, + } + return true + } else { + isCompleted := req.Aggregator.FillPeriod(metric.Since, metric.SinceValue, &period) + if isCompleted { + req.ResponseWriter.WritePeriod(period) + } + req.ResultCh <- cumulativePeriodsResult{ + ResultCode: QueryDone, + } + return false + } +} + +type tryListInstantPeriodsReq struct { + MetricID uint32 + Since uint32 + Until uint32 + Aggregator *atree.InstantAggregator + ResponseWriter *atree.InstantPeriodsWriter + ResultCh chan instantPeriodsResult +} + +func (s *Database) tryListInstantPeriods(req tryListInstantPeriodsReq) { + metric, ok := s.metrics[req.MetricID] + if !ok { + req.ResultCh <- instantPeriodsResult{ + ResultCode: NoMetric, + } + return + } + if metric.MetricType != diploma.Instant { + req.ResultCh <- instantPeriodsResult{ + ResultCode: WrongMetricType, + } + return + } + + lockEntry, ok := s.metricLockEntries[req.MetricID] + if ok { + if lockEntry.XLock { + lockEntry.WaitQueue = append(lockEntry.WaitQueue, req) + return + } + } + if s.startListInstantPeriods(metric, req) { + if lockEntry != nil { + lockEntry.RLocks++ + } else { + s.metricLockEntries[req.MetricID] = &metricLockEntry{ + RLocks: 1, + } + } + } +} + +func (*Database) startListInstantPeriods(metric *_metric, req tryListInstantPeriodsReq) bool { + if metric.Since == 0 { + req.ResultCh <- instantPeriodsResult{ + ResultCode: QueryDone, + } + return false + } + + if req.Since > metric.Until { + req.ResultCh <- instantPeriodsResult{ + ResultCode: QueryDone, + } + return false + } + + if req.Until < metric.Since { + // search in tree + if metric.RootPageNo > 0 { + req.ResultCh <- instantPeriodsResult{ + ResultCode: UntilNotFound, + PageNo: metric.RootPageNo, + FracDigits: metric.FracDigits, + } + return true + } else { + req.ResultCh <- instantPeriodsResult{ + ResultCode: QueryDone, + } + return false + } + } + + timestampDecompressor := chunkenc.NewReverseTimeDeltaOfDeltaDecompressor( + metric.TimestampsBuf, + metric.Timestamps.Size(), + ) + + valueDecompressor := chunkenc.NewReverseInstantDeltaDecompressor( + metric.ValuesBuf, + metric.Values.Size(), + metric.FracDigits, + ) + + var period atree.InstantPeriod + + for { + timestamp, done := timestampDecompressor.NextValue() + if done { + break + } + + value, done := valueDecompressor.NextValue() + if done { + diploma.Abort(diploma.HasTimestampNoValueBug, ErrNoValueBug) + } + + if timestamp <= req.Until { + if timestamp < req.Since { + isCompleted := req.Aggregator.FillPeriod(timestamp, &period) + if isCompleted { + req.ResponseWriter.WritePeriod(period) + } + req.ResultCh <- instantPeriodsResult{ + ResultCode: QueryDone, + } + return false + } + + isCompleted := req.Aggregator.Feed(timestamp, value, &period) + if isCompleted { + req.ResponseWriter.WritePeriod(period) + } + } + } + + if metric.LastPageNo > 0 { + req.ResultCh <- instantPeriodsResult{ + ResultCode: UntilFound, + PageNo: metric.LastPageNo, + FracDigits: metric.FracDigits, + } + return true + } else { + isCompleted := req.Aggregator.FillPeriod(metric.Since, &period) + if isCompleted { + req.ResponseWriter.WritePeriod(period) + } + req.ResultCh <- instantPeriodsResult{ + ResultCode: QueryDone, + } + return false + } +} + +type tryListCurrentValuesReq struct { + MetricIDs []uint32 + ResponseWriter *atree.CurrentValueWriter + ResultCh chan struct{} +} + +func (s *Database) tryListCurrentValues(req tryListCurrentValuesReq) { + for _, metricID := range req.MetricIDs { + metric, ok := s.metrics[metricID] + if ok { + req.ResponseWriter.BufferValue(atree.CurrentValue{ + MetricID: metricID, + Timestamp: metric.Until, + Value: metric.UntilValue, + }) + } + } + req.ResultCh <- struct{}{} +} + +func (s *Database) applyChanges(req txlog.Changes) { + for _, untyped := range req.Records { + switch rec := untyped.(type) { + case txlog.AddedMetric: + s.addMetric(rec) + + case txlog.DeletedMetric: + s.deleteMetric(rec) + + case txlog.AppendedMeasure: + s.appendMeasure(rec) + + case txlog.AppendedMeasuresExtended: + s.appendMeasures(rec) + + case txlog.AppendedMeasureWithOverflowExtended: + s.appendMeasureAfterOverflow(rec) + + case txlog.DeletedMeasures: + s.deleteMeasures(rec) + + case txlog.DeletedMeasuresSince: + s.deleteMeasuresSince(rec) + } + } + + if req.ForceSnapshot || req.ExitWaitGroup != nil { + s.dumpSnapshot(req.LogNumber) + } + + close(req.WaitCh) + + if req.ExitWaitGroup != nil { + req.ExitWaitGroup.Done() + } +} + +func (s *Database) addMetric(rec txlog.AddedMetric) { + _, ok := s.metrics[rec.MetricID] + if ok { + diploma.Abort(diploma.MetricAddedBug, + fmt.Errorf("addMetric: metric %d already added", + rec.MetricID)) + } + + lockEntry, ok := s.metricLockEntries[rec.MetricID] + if !ok { + diploma.Abort(diploma.NoLockEntryBug, + fmt.Errorf("addMetric: lockEntry not found for the metric %d", + rec.MetricID)) + } + + if !lockEntry.XLock { + diploma.Abort(diploma.NoXLockBug, + fmt.Errorf("addMetric: xlock not set for the metric %d", + rec.MetricID)) + } + + var ( + values diploma.ValueCompressor + timestampsBuf = conbuf.New(nil) + valuesBuf = conbuf.New(nil) + ) + + if rec.MetricType == diploma.Cumulative { + values = chunkenc.NewReverseCumulativeDeltaCompressor( + valuesBuf, 0, byte(rec.FracDigits)) + } else { + values = chunkenc.NewReverseInstantDeltaCompressor( + valuesBuf, 0, byte(rec.FracDigits)) + } + + s.metrics[rec.MetricID] = &_metric{ + MetricType: rec.MetricType, + FracDigits: byte(rec.FracDigits), + TimestampsBuf: timestampsBuf, + ValuesBuf: valuesBuf, + Timestamps: chunkenc.NewReverseTimeDeltaOfDeltaCompressor(timestampsBuf, 0), + Values: values, + } + + lockEntry.XLock = false + delete(s.metricLockEntries, rec.MetricID) +} + +func (s *Database) deleteMetric(rec txlog.DeletedMetric) { + _, ok := s.metrics[rec.MetricID] + if !ok { + diploma.Abort(diploma.NoMetricBug, + fmt.Errorf("deleteMetric: metric %d not found", + rec.MetricID)) + } + + lockEntry, ok := s.metricLockEntries[rec.MetricID] + if !ok { + diploma.Abort(diploma.NoLockEntryBug, + fmt.Errorf("deleteMetric: lockEntry not found for the metric %d", + rec.MetricID)) + } + + if !lockEntry.XLock { + diploma.Abort(diploma.NoXLockBug, + fmt.Errorf("deleteMetric: xlock not set for the metric %d", + rec.MetricID)) + } + + var addMetricReqs []tryAddMetricReq + + if len(lockEntry.WaitQueue) > 0 { + for _, untyped := range lockEntry.WaitQueue { + switch req := untyped.(type) { + case tryAppendMeasureReq: + req.ResultCh <- tryAppendMeasureResult{ + ResultCode: NoMetric, + } + + case tryListCumulativePeriodsReq: + req.ResultCh <- cumulativePeriodsResult{ + ResultCode: NoMetric, + } + + case tryListInstantPeriodsReq: + req.ResultCh <- instantPeriodsResult{ + ResultCode: NoMetric, + } + + case tryListCumulativeMeasuresReq: + req.ResultCh <- cumulativeMeasuresResult{ + ResultCode: NoMetric, + } + + case tryListInstantMeasuresReq: + req.ResultCh <- instantMeasuresResult{ + ResultCode: NoMetric, + } + + case tryListAllCumulativeMeasuresReq: + req.ResultCh <- cumulativeMeasuresResult{ + ResultCode: NoMetric, + } + + case tryListAllInstantMeasuresReq: + req.ResultCh <- instantMeasuresResult{ + ResultCode: NoMetric, + } + + case tryAddMetricReq: + addMetricReqs = append(addMetricReqs, req) + + case tryDeleteMetricReq: + req.ResultCh <- tryDeleteMetricResult{ + ResultCode: NoMetric, + } + + case tryDeleteMeasuresReq: + req.ResultCh <- tryDeleteMeasuresResult{ + ResultCode: NoMetric, + } + + case tryGetMetricReq: + req.ResultCh <- Metric{ + ResultCode: NoMetric, + } + + default: + diploma.Abort(diploma.UnknownMetricWaitQueueItemBug, + fmt.Errorf("bug: unknown metric wait queue item type %T", req)) + } + } + } + delete(s.metrics, rec.MetricID) + delete(s.metricLockEntries, rec.MetricID) + + if len(rec.FreeDataPages) > 0 { + s.dataFreeList.AddPages(rec.FreeDataPages) + s.atree.DeleteDataPages(rec.FreeDataPages) + } + if len(rec.FreeIndexPages) > 0 { + s.indexFreeList.AddPages(rec.FreeIndexPages) + s.atree.DeleteIndexPages(rec.FreeIndexPages) + } + + if len(addMetricReqs) > 0 { + s.processTryAddMetricReqsImmediatelyAfterDelete(addMetricReqs) + } +} + +func (s *Database) deleteMeasures(rec txlog.DeletedMeasures) { + metric, ok := s.metrics[rec.MetricID] + if !ok { + diploma.Abort(diploma.NoMetricBug, + fmt.Errorf("deleteMeasures: metric %d not found", + rec.MetricID)) + } + + lockEntry, ok := s.metricLockEntries[rec.MetricID] + if !ok { + diploma.Abort(diploma.NoLockEntryBug, + fmt.Errorf("deleteMeasures: lockEntry not found for the metric %d", + rec.MetricID)) + } + + if !lockEntry.XLock { + diploma.Abort(diploma.NoXLockBug, + fmt.Errorf("deleteMeasures: xlock not set for the metric %d", + rec.MetricID)) + } + + metric.DeleteMeasures() + lockEntry.XLock = false + // + if len(rec.FreeDataPages) > 0 { + s.dataFreeList.AddPages(rec.FreeDataPages) + s.atree.DeleteDataPages(rec.FreeDataPages) + } + if len(rec.FreeDataPages) > 0 { + s.indexFreeList.AddPages(rec.FreeIndexPages) + s.atree.DeleteIndexPages(rec.FreeIndexPages) + } + s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry) +} + +func (s *Database) deleteMeasuresSince(rec txlog.DeletedMeasuresSince) { + metric, ok := s.metrics[rec.MetricID] + if !ok { + diploma.Abort(diploma.NoMetricBug, + fmt.Errorf("deleteMeasuresSince: metric %d not found", + rec.MetricID)) + } + + lockEntry, ok := s.metricLockEntries[rec.MetricID] + if !ok { + diploma.Abort(diploma.NoLockEntryBug, + fmt.Errorf("deleteMeasuresSince: lockEntry not found for the metric %d", + rec.MetricID)) + } + + if !lockEntry.XLock { + diploma.Abort(diploma.NoXLockBug, + fmt.Errorf("deleteMeasuresSince: xlock not set for the metric %d", + rec.MetricID)) + } + + if rec.IsRootChanged && rec.RootPageNo == 0 { + metric.DeleteMeasures() + } else { + metric.TimestampsBuf = conbuf.NewFromBuffer(rec.TimestampsBuf) + metric.ValuesBuf = conbuf.NewFromBuffer(rec.ValuesBuf) + + metric.Timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor( + metric.TimestampsBuf, len(rec.TimestampsBuf)) + + if metric.MetricType == diploma.Cumulative { + metric.Values = chunkenc.NewReverseCumulativeDeltaCompressor( + metric.ValuesBuf, len(rec.ValuesBuf), metric.FracDigits) + } else { + metric.Values = chunkenc.NewReverseInstantDeltaCompressor( + metric.ValuesBuf, len(rec.ValuesBuf), metric.FracDigits) + } + + metric.Since, metric.Until = enc.GetTimeRange(rec.TimestampsBuf) + metric.SinceValue, metric.UntilValue = enc.GetValueBounds( + rec.ValuesBuf, metric.MetricType, metric.FracDigits, + ) + + if rec.IsRootChanged { + metric.RootPageNo = rec.RootPageNo + } + metric.LastPageNo = rec.LastPageNo + } + + lockEntry.XLock = false + + if len(rec.FreeDataPages) > 0 { + s.dataFreeList.AddPages(rec.FreeDataPages) + s.atree.DeleteDataPages(rec.FreeDataPages) + } + if len(rec.FreeDataPages) > 0 { + s.indexFreeList.AddPages(rec.FreeIndexPages) + s.atree.DeleteIndexPages(rec.FreeIndexPages) + } + s.doAfterReleaseXLock(rec.MetricID, metric, lockEntry) +} + +func (s *Database) doAfterReleaseXLock(metricID uint32, metric *_metric, lockEntry *metricLockEntry) { + if len(lockEntry.WaitQueue) == 0 { + delete(s.metricLockEntries, metricID) + } else { + s.processMetricQueue(metricID, metric, lockEntry) + } +} diff --git a/database/snapshot.go b/database/snapshot.go new file mode 100644 index 0000000..3911eb9 --- /dev/null +++ b/database/snapshot.go @@ -0,0 +1,287 @@ +package database + +import ( + "fmt" + "hash/crc32" + "io" + "os" + "path/filepath" + + octopus "gordenko.dev/dima/diploma" + "gordenko.dev/dima/diploma/atree" + "gordenko.dev/dima/diploma/bin" + "gordenko.dev/dima/diploma/chunkenc" + "gordenko.dev/dima/diploma/conbuf" + "gordenko.dev/dima/diploma/freelist" +) + +/* +Формат: +//lsn - varuint (останній LSN, що змінив дані у RAM) +metricsQty - varuint +[metric]* +где metric - це: +metricID - 4b +metricType - 1b +fracDigits - 1b +rootPageNo - 4b +lastPageNo - 4b +since - 4b +sinceValue - 8b +until - 4b +untilValue - 8b +timestamps size - 2b +values size - 2b +timestams payload - Nb +values payload - Nb +dataFreeList size - varuint +dataFreeList - Nb +indexFreeList size - varuint +indexFreeList - Nb +CRC32 - 4b +*/ + +const metricHeaderSize = 42 + +func (s *Database) dumpSnapshot(logNumber int) (err error) { + var ( + fileName = filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", logNumber)) + hasher = crc32.NewIEEE() + prefix = make([]byte, metricHeaderSize) + ) + + file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY, 0770) + if err != nil { + return + } + + dst := io.MultiWriter(file, hasher) + + _, err = bin.WriteVarUint64(dst, uint64(len(s.metrics))) + if err != nil { + return + } + + for metricID, metric := range s.metrics { + tSize := metric.Timestamps.Size() + vSize := metric.Values.Size() + + bin.PutUint32(prefix[0:], metricID) + prefix[4] = byte(metric.MetricType) + prefix[5] = metric.FracDigits + bin.PutUint32(prefix[6:], metric.RootPageNo) + bin.PutUint32(prefix[10:], metric.LastPageNo) + bin.PutUint32(prefix[14:], metric.Since) + bin.PutFloat64(prefix[18:], metric.SinceValue) + bin.PutUint32(prefix[26:], metric.Until) + bin.PutFloat64(prefix[30:], metric.UntilValue) + bin.PutUint16(prefix[38:], uint16(tSize)) + bin.PutUint16(prefix[40:], uint16(vSize)) + + _, err = dst.Write(prefix) + if err != nil { + return + } + // copy timestamps + remaining := tSize + for _, buf := range metric.TimestampsBuf.Chunks() { + if remaining < len(buf) { + buf = buf[:remaining] + } + _, err = dst.Write(buf) + if err != nil { + return + } + remaining -= len(buf) + if remaining == 0 { + break + } + } + // copy values + remaining = vSize + for _, buf := range metric.ValuesBuf.Chunks() { + if remaining < len(buf) { + buf = buf[:remaining] + } + _, err = dst.Write(buf) + if err != nil { + return + } + remaining -= len(buf) + if remaining == 0 { + break + } + } + } + // free data pages + err = freeListWriteTo(s.dataFreeList, dst) + if err != nil { + return + } + // free index pages + err = freeListWriteTo(s.indexFreeList, dst) + if err != nil { + return + } + + bin.WriteUint32(file, hasher.Sum32()) + + err = file.Sync() + if err != nil { + return + } + + err = file.Close() + if err != nil { + return + } + + prevLogNumber := logNumber - 1 + prevChanges := filepath.Join(s.dir, fmt.Sprintf("%d.changes", prevLogNumber)) + prevSnapshot := filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", prevLogNumber)) + + isExist, err := isFileExist(prevChanges) + if err != nil { + return + } + + if isExist { + err = os.Remove(prevChanges) + if err != nil { + octopus.Abort(octopus.DeletePrevChangesFileFailed, err) + } + } + + isExist, err = isFileExist(prevSnapshot) + if err != nil { + return + } + + if isExist { + err = os.Remove(prevSnapshot) + if err != nil { + octopus.Abort(octopus.DeletePrevSnapshotFileFailed, err) + } + } + return +} + +func (s *Database) loadSnapshot(fileName string) (err error) { + var ( + hasher = crc32.NewIEEE() + metricsQty int + header = make([]byte, metricHeaderSize) + body = make([]byte, atree.DataPageSize) + ) + + file, err := os.Open(fileName) + if err != nil { + return + } + + src := io.TeeReader(file, hasher) + u64, _, err := bin.ReadVarUint64(src) + if err != nil { + return + } + metricsQty = int(u64) + + for range metricsQty { + var metric _metric + err = bin.ReadNInto(src, header) + if err != nil { + return + } + + metricID := bin.GetUint32(header[0:]) + metric.MetricType = octopus.MetricType(header[4]) + metric.FracDigits = header[5] + metric.RootPageNo = bin.GetUint32(header[6:]) + metric.LastPageNo = bin.GetUint32(header[10:]) + metric.Since = bin.GetUint32(header[14:]) + metric.SinceValue = bin.GetFloat64(header[18:]) + metric.Until = bin.GetUint32(header[26:]) + metric.UntilValue = bin.GetFloat64(header[30:]) + tSize := bin.GetUint16(header[38:]) + vSize := bin.GetUint16(header[40:]) + + buf := body[:tSize] + err = bin.ReadNInto(src, buf) + if err != nil { + return + } + metric.TimestampsBuf = conbuf.NewFromBuffer(buf) + + buf = body[:vSize] + err = bin.ReadNInto(src, buf) + if err != nil { + return + } + metric.ValuesBuf = conbuf.NewFromBuffer(buf) + + metric.Timestamps = chunkenc.NewReverseTimeDeltaOfDeltaCompressor( + metric.TimestampsBuf, int(tSize)) + + if metric.MetricType == octopus.Cumulative { + metric.Values = chunkenc.NewReverseCumulativeDeltaCompressor( + metric.ValuesBuf, int(vSize), metric.FracDigits) + } else { + metric.Values = chunkenc.NewReverseInstantDeltaCompressor( + metric.ValuesBuf, int(vSize), metric.FracDigits) + } + s.metrics[metricID] = &metric + } + + err = restoreFreeList(s.dataFreeList, src) + if err != nil { + return fmt.Errorf("restore dataFreeList: %s", err) + } + + err = restoreFreeList(s.indexFreeList, src) + if err != nil { + return fmt.Errorf("restore indexFreeList: %s", err) + } + + calculatedChecksum := hasher.Sum32() + + writtenChecksum, err := bin.ReadUint32(file) + if err != nil { + return + } + + if calculatedChecksum != writtenChecksum { + return fmt.Errorf("calculated checksum %d not equal written checksum %d", calculatedChecksum, writtenChecksum) + } + return +} + +// HELPERS + +func freeListWriteTo(freeList *freelist.FreeList, dst io.Writer) error { + serialized, err := freeList.Serialize() + if err != nil { + octopus.Abort(octopus.FailedFreeListSerialize, err) + } + _, err = bin.WriteVarUint64(dst, uint64(len(serialized))) + if err != nil { + return err + } + _, err = dst.Write(serialized) + if err != nil { + return err + } + return nil +} + +func restoreFreeList(freeList *freelist.FreeList, src io.Reader) error { + size, _, err := bin.ReadVarUint64(src) + if err != nil { + return err + } + serialized, err := bin.ReadN(src, int(size)) + if err != nil { + return err + } + freeList.Restore(serialized) + return nil +} diff --git a/database_linux b/database_linux new file mode 100755 index 0000000..48bc8f7 Binary files /dev/null and b/database_linux differ diff --git a/database_windows b/database_windows new file mode 100755 index 0000000..09447d8 Binary files /dev/null and b/database_windows differ diff --git a/diploma.go b/diploma.go new file mode 100644 index 0000000..53889c3 --- /dev/null +++ b/diploma.go @@ -0,0 +1,88 @@ +package diploma + +import ( + "fmt" + "os" +) + +type MetricType byte +type GroupBy byte + +const ( + Cumulative MetricType = 1 + Instant MetricType = 2 + MaxFracDigits byte = 7 + + GroupByHour GroupBy = 1 + GroupByDay GroupBy = 2 + GroupByMonth GroupBy = 3 + + AggregateMin byte = 1 + AggregateMax byte = 2 + AggregateAvg byte = 4 +) + +type TimestampCompressor interface { + CalcRequiredSpace(uint32) int + Append(uint32) + Size() int + DeleteLast() + //LastTimestamp() uint32 +} + +type ValueCompressor interface { + CalcRequiredSpace(float64) int + Append(float64) + Size() int + DeleteLast() + //LastValue() float64 +} + +type TimestampDecompressor interface { + NextValue() (uint32, bool) +} + +type ValueDecompressor interface { + NextValue() (float64, bool) +} + +type AbortCode int + +const ( + // Fatal errors + WrongPrevPageNo AbortCode = 1 + WriteToAtreeFailed AbortCode = 2 + MaxAtreeSizeExceeded AbortCode = 3 + FailedWriteToTxLog AbortCode = 4 + ReferenceCountBug AbortCode = 5 + WrongResultCodeBug AbortCode = 6 + RemoveREDOFileFailed AbortCode = 7 + FailedAtreeRequest AbortCode = 8 + UnknownTxLogRecordTypeBug AbortCode = 11 + HasTimestampNoValueBug AbortCode = 12 + NoMetricBug AbortCode = 13 + NoLockEntryBug AbortCode = 14 + NoXLockBug AbortCode = 15 + MetricAddedBug AbortCode = 16 + NoRLockBug AbortCode = 17 + XLockBug AbortCode = 18 + FailedFreeListSerialize AbortCode = 19 + UnknownWorkerQueueItemBug AbortCode = 20 + UnknownMetricWaitQueueItemBug AbortCode = 21 + // + GetRecoveryRecipeFailed AbortCode = 26 + LoadSnapshotFailed AbortCode = 27 + ReplayChangesFailed AbortCode = 28 + CreateChangesWriterFailed AbortCode = 29 + RemoveRecipeFileFailed AbortCode = 30 + DumpSnapshotFailed AbortCode = 31 + SearchREDOFilesFailed AbortCode = 32 + ReplayREDOFileFailed AbortCode = 33 + DeletePrevChangesFileFailed AbortCode = 34 + DeletePrevSnapshotFileFailed AbortCode = 35 +) + +func Abort(code AbortCode, err error) { + fmt.Println(err) + os.Exit(int(code)) +} diff --git a/enc/cumdelta.go b/enc/cumdelta.go new file mode 100644 index 0000000..0c56e84 --- /dev/null +++ b/enc/cumdelta.go @@ -0,0 +1,102 @@ +package enc + +import ( + "math" + + "gordenko.dev/dima/diploma/bin" +) + +type ReverseCumulativeDeltaDecompressor struct { + buf []byte + pos int + bound int + firstValue float64 + lastValue float64 + length uint16 + coef float64 + idxOf8 uint + s8 byte +} + +func NewReverseCumulativeDeltaDecompressor(buf []byte, fracDigits byte) *ReverseCumulativeDeltaDecompressor { + var coef float64 = 1 + if fracDigits > 0 { + coef = math.Pow(10, float64(fracDigits)) + } + return &ReverseCumulativeDeltaDecompressor{ + buf: buf, + coef: coef, + pos: len(buf), + } +} + +func (s *ReverseCumulativeDeltaDecompressor) NextValue() (value float64, done bool) { + if s.length > 0 { + s.length-- + return s.lastValue, false + } + if s.pos < s.bound { + return 0, true + } + + if s.pos == len(s.buf) { + u64, n, err := bin.GetVarUint64(s.buf) + if err != nil { + panic(err) + } + s.firstValue = float64(u64) / s.coef + s.bound = n + s.pos-- + s.idxOf8 = uint(8 - s.buf[s.pos]) + s.pos-- + s.s8 = s.buf[s.pos] + s.pos-- + s.readVar() + if s.length > 0 { + s.length-- + } + return s.lastValue, false + } + + if s.idxOf8 == 0 { + s.s8 = s.buf[s.pos] + s.pos-- + } + s.readVar() + if s.length > 0 { + s.length-- + } + return s.lastValue, false +} + +func (s *ReverseCumulativeDeltaDecompressor) readVar() { + u64, n, err := bin.ReverseGetVarUint64(s.buf[:s.pos+1]) + if err != nil { + panic(err) + } + s.pos -= n + s.lastValue = s.firstValue + float64(u64)/s.coef + + var flag byte = 1 << s.idxOf8 + if (s.s8 & flag) == flag { + s.decodeLength() + } + if s.idxOf8 == 7 { + s.idxOf8 = 0 + } else { + s.idxOf8++ + } +} + +func (s *ReverseCumulativeDeltaDecompressor) decodeLength() { + b1 := s.buf[s.pos] + s.pos-- + if b1 < 128 { + s.length = uint16(b1) + } else { + b2 := s.buf[s.pos] + s.pos-- + s.length = uint16(b1&127) | (uint16(b2) << 7) + } + s.length += 2 +} diff --git a/enc/enc.go b/enc/enc.go new file mode 100644 index 0000000..579d518 --- /dev/null +++ b/enc/enc.go @@ -0,0 +1,3 @@ +package enc + +const eps = 0.000001 diff --git a/enc/insdelta.go b/enc/insdelta.go new file mode 100644 index 0000000..4d11665 --- /dev/null +++ b/enc/insdelta.go @@ -0,0 +1,130 @@ +package enc + +import ( + "fmt" + "math" + + octopus "gordenko.dev/dima/diploma" + "gordenko.dev/dima/diploma/bin" +) + +type ReverseInstantDeltaDecompressor struct { + buf []byte + pos int + bound int + firstValue float64 + lastValue float64 + length uint16 + coef float64 + idxOf8 uint + s8 byte +} + +func NewReverseInstantDeltaDecompressor(buf []byte, fracDigits byte) *ReverseInstantDeltaDecompressor { + var coef float64 = 1 + if fracDigits > 0 { + coef = math.Pow(10, float64(fracDigits)) + } + return &ReverseInstantDeltaDecompressor{ + buf: buf, + coef: coef, + pos: len(buf), + } +} + +func (s *ReverseInstantDeltaDecompressor) NextValue() (value float64, done bool) { + if s.length > 0 { + s.length-- + return s.lastValue, false + } + if s.pos < s.bound { + return 0, true + } + + if s.pos == len(s.buf) { + u64, n, err := bin.GetVarInt64(s.buf) + if err != nil { + panic(err) + } + s.firstValue = float64(u64) / s.coef + s.bound = n + s.pos-- + s.idxOf8 = uint(8 - s.buf[s.pos]) + s.pos-- + s.s8 = s.buf[s.pos] + s.pos-- + s.readVar() + if s.length > 0 { + s.length-- + } + return s.lastValue, false + } + + if s.idxOf8 == 0 { + s.s8 = s.buf[s.pos] + s.pos-- + } + s.readVar() + if s.length > 0 { + s.length-- + } + return s.lastValue, false +} + +func (s *ReverseInstantDeltaDecompressor) readVar() { + i64, n, err := bin.ReverseGetVarInt64(s.buf[:s.pos+1]) + if err != nil { + panic(err) + } + s.pos -= n + s.lastValue = s.firstValue + float64(i64)/s.coef + + var flag byte = 1 << s.idxOf8 + if (s.s8 & flag) == flag { + s.decodeLength() + } + if s.idxOf8 == 7 { + s.idxOf8 = 0 + } else { + s.idxOf8++ + } +} + +func (s *ReverseInstantDeltaDecompressor) decodeLength() { + b1 := s.buf[s.pos] + s.pos-- + if b1 < 128 { + s.length = uint16(b1) + } else { + b2 := s.buf[s.pos] + s.pos-- + s.length = uint16(b1&127) | (uint16(b2) << 7) + } + s.length += 2 +} + +func GetValueBounds(valuesBuf []byte, metricType octopus.MetricType, fracDigits byte) (sinceValue, untilValue float64) { + var decompressor octopus.ValueDecompressor + switch metricType { + case octopus.Instant: + decompressor = NewReverseInstantDeltaDecompressor(valuesBuf, fracDigits) + case octopus.Cumulative: + decompressor = NewReverseCumulativeDeltaDecompressor(valuesBuf, fracDigits) + default: + panic(fmt.Sprintf("unknown metricType %d", metricType)) + } + value, done := decompressor.NextValue() + if done { + return + } + + sinceValue = value + untilValue = value + for { + value, done = decompressor.NextValue() + if done { + return + } + sinceValue = value + } +} diff --git a/enc/time_delta_delta.go b/enc/time_delta_delta.go new file mode 100644 index 0000000..6fa4d26 --- /dev/null +++ b/enc/time_delta_delta.go @@ -0,0 +1,145 @@ +package enc + +import ( + "gordenko.dev/dima/diploma/bin" +) + +// REVERSE + +const ( + lastUnixtimeIdx = 0 + baseDeltaIdx = 4 +) + +type ReverseTimeDeltaOfDeltaDecompressor struct { + step byte + buf []byte + pos int + bound int + lastUnixtime uint32 + baseDelta uint32 + lastDeltaOfDelta int64 + length uint16 + idxOf8 uint + s8 byte +} + +func NewReverseTimeDeltaOfDeltaDecompressor(buf []byte) *ReverseTimeDeltaOfDeltaDecompressor { + return &ReverseTimeDeltaOfDeltaDecompressor{ + buf: buf, + pos: len(buf), + } +} + +func (s *ReverseTimeDeltaOfDeltaDecompressor) NextValue() (value uint32, done bool) { + if s.step == 0 { + if s.pos == 0 { + return 0, true + } + s.lastUnixtime = bin.GetUint32(s.buf[lastUnixtimeIdx:]) + s.step = 1 + return s.lastUnixtime, false + } + + if s.step == 1 { + if s.pos == baseDeltaIdx { + return 0, true + } + u64, n, err := bin.GetVarUint64(s.buf[baseDeltaIdx:]) + if err != nil { + panic("EOF") + } + s.bound = baseDeltaIdx + n + s.baseDelta = uint32(u64) + + s.pos-- + s.idxOf8 = uint(8 - s.buf[s.pos]) + s.pos-- + s.s8 = s.buf[s.pos] + s.pos-- + + s.readVar() + if s.length > 0 { + s.length-- + } + s.step = 2 + return s.lastUnixtime, false + } + + if s.length > 0 { + s.length-- + delta := int64(s.baseDelta) + s.lastDeltaOfDelta + s.lastUnixtime = uint32(int64(s.lastUnixtime) - delta) + return s.lastUnixtime, false + } + + if s.pos < s.bound { + return 0, true + } + if s.idxOf8 == 0 { + s.s8 = s.buf[s.pos] + s.pos-- + } + s.readVar() + if s.length > 0 { + s.length-- + } + return s.lastUnixtime, false +} + +func GetTimeRange(timestampsBuf []byte) (since, until uint32) { + decompressor := NewReverseTimeDeltaOfDeltaDecompressor(timestampsBuf) + value, done := decompressor.NextValue() + if done { + return + } + + since = value + until = value + for { + value, done = decompressor.NextValue() + if done { + return + } + since = value + } +} + +func (s *ReverseTimeDeltaOfDeltaDecompressor) readVar() { + var ( + n int + err error + ) + + s.lastDeltaOfDelta, n, err = bin.ReverseGetVarInt64(s.buf[:s.pos+1]) + if err != nil { + panic(err) + } + s.pos -= n + + delta := int64(s.baseDelta) + s.lastDeltaOfDelta + s.lastUnixtime = uint32(int64(s.lastUnixtime) - delta) + + var flag byte = 1 << s.idxOf8 + if (s.s8 & flag) == flag { + s.decodeLength() + } + if s.idxOf8 == 7 { + s.idxOf8 = 0 + } else { + s.idxOf8++ + } +} + +func (s *ReverseTimeDeltaOfDeltaDecompressor) decodeLength() { + b1 := s.buf[s.pos] + s.pos-- + if b1 < 128 { + s.length = uint16(b1) + } else { + b2 := s.buf[s.pos] + s.pos-- + s.length = uint16(b1&127) | (uint16(b2) << 7) + } + s.length += 2 +} diff --git a/examples/database/main.go b/examples/database/main.go new file mode 100644 index 0000000..6a05bbd --- /dev/null +++ b/examples/database/main.go @@ -0,0 +1,135 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + "os/signal" + "sync" + "syscall" + + "gopkg.in/ini.v1" + "gordenko.dev/dima/diploma/database" +) + +func main() { + var ( + logfile = os.Stdout + iniFileName string + ) + + flag.Usage = func() { + fmt.Fprint(flag.CommandLine.Output(), helpMessage) + fmt.Fprint(flag.CommandLine.Output(), configExample) + fmt.Fprintf(flag.CommandLine.Output(), mainUsage, os.Args[0]) + flag.PrintDefaults() + } + flag.StringVar(&iniFileName, "c", "database.ini", "path to *.ini config file") + flag.Parse() + + config, err := loadConfig(iniFileName) + if err != nil { + log.Fatalln(err) + } + + var ( + exitCh = make(chan struct{}) + wg = new(sync.WaitGroup) + ) + + db, err := database.New(database.Options{ + TCPPort: config.TcpPort, + Dir: config.Dir, + DatabaseName: config.DatabaseName, + RedoDir: config.REDODir, + Logfile: logfile, + ExitCh: exitCh, + WaitGroup: wg, + }) + if err != nil { + log.Fatalf("database.New: %s\n", err) + } + + go func() { + err = db.ListenAndServe() + if err != nil { + log.Fatalln(err) + } + }() + + wg.Add(1) + + fmt.Fprintf(logfile, "database %q started on port %d.\n", + config.DatabaseName, config.TcpPort) + fmt.Fprintln(logfile, config) + + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + + <-sigs + + close(exitCh) + wg.Wait() + + fmt.Fprintln(logfile, "database stopped.") +} + +// config file + +const mainUsage = `Usage: + %s -c path/to/config.ini + +` + +const helpMessage = `Diploma project. Database. Version: 1.0 +created by Dmytro Gordenko, 1.e4.kc6@gmail.com +` + +const configExample = ` +database.ini example: + +tcpPort = 12345 +dir = ../../datadir +redoDir = ../../datadir +databaseName = test + +` + +type Config struct { + TcpPort int + Dir string + REDODir string + DatabaseName string +} + +func (s Config) String() string { + return fmt.Sprintf(`starting options: +tcpPort = %d +dir = %s +redoDir = %s +databaseName = %s +`, + s.TcpPort, s.Dir, s.REDODir, s.DatabaseName) +} + +func loadConfig(iniFileName string) (_ Config, err error) { + file, err := ini.Load(iniFileName) + if err != nil { + return + } + + conf := Config{} + + top := file.Section("") + + conf.TcpPort, err = top.Key("tcpPort").Int() + if err != nil { + err = fmt.Errorf("'tcpPort' option is required in config file") + return + } + conf.Dir = top.Key("dir").String() + conf.REDODir = top.Key("redoDir").String() + conf.DatabaseName = top.Key("databaseName").String() + return conf, nil +} diff --git a/examples/loadtest/loadtest.go b/examples/loadtest/loadtest.go new file mode 100644 index 0000000..7ab57ef --- /dev/null +++ b/examples/loadtest/loadtest.go @@ -0,0 +1,377 @@ +package main + +import ( + "encoding/json" + "fmt" + "math/rand" + "os" + "time" + + "gordenko.dev/dima/diploma" + "gordenko.dev/dima/diploma/client" + "gordenko.dev/dima/diploma/proto" +) + +// METRICS INFO + +type MetricInfo struct { + MetricID uint32 `json:"metricID"` + MetricType diploma.MetricType `json:"metricType"` + FracDigits int `json:"fracDigits"` + Since int64 `json:"since"` + Until int64 `json:"until"` + Qty int `json:"qty"` +} + +func readMetricInfo(fileName string) (list []MetricInfo, err error) { + buf, err := os.ReadFile(fileName) + if err != nil { + return + } + + err = json.Unmarshal(buf, &list) + return +} + +// RANDOM QUERY GENERATOR + +type QueryRecipe struct { + MetricID uint32 + MetricIDs []uint32 + Method int + RangeCode int + Since uint32 + Until uint32 + GroupBy diploma.GroupBy +} + +type RandomQueryGenerator struct { + metrics []MetricInfo + groupByOptions []diploma.GroupBy + listCurrentValuesProbability int + listPeriodsProbability int + timeRangeProbDistribution []int +} + +type RandomQueryGeneratorOptions struct { + Metrics []MetricInfo + ListCurrentValuesProbability int + ListPeriodsProbability int + TimeRangeProbabilities []int +} + +func NewRandomQueryGenerator(opt RandomQueryGeneratorOptions) *RandomQueryGenerator { + if opt.ListCurrentValuesProbability >= 100 { + panic(fmt.Sprintf("wrong ListCurrentValuesProbability: %d", opt.ListCurrentValuesProbability)) + } + + if opt.ListPeriodsProbability >= 100 { + panic(fmt.Sprintf("wrong ListPeriodsProbability: %d", opt.ListPeriodsProbability)) + } + // check total time range propability + var totalTimeRangeProbability int + for _, p := range opt.TimeRangeProbabilities { + totalTimeRangeProbability += p + } + if totalTimeRangeProbability != 100 { + panic(fmt.Sprintf("total time range probabilities != 100: %d", totalTimeRangeProbability)) + } + + // create time range probability distribution + timeRangeProbDistribution := make([]int, len(opt.TimeRangeProbabilities)) + timeRangeProbDistribution[0] = opt.TimeRangeProbabilities[0] + for i := 1; i < len(opt.TimeRangeProbabilities); i++ { + timeRangeProbDistribution[i] = timeRangeProbDistribution[i-1] + opt.TimeRangeProbabilities[i] + } + + return &RandomQueryGenerator{ + metrics: opt.Metrics, + groupByOptions: []diploma.GroupBy{ + diploma.GroupByHour, + diploma.GroupByDay, + diploma.GroupByMonth, + }, + listCurrentValuesProbability: opt.ListCurrentValuesProbability, + listPeriodsProbability: opt.ListPeriodsProbability, + timeRangeProbDistribution: timeRangeProbDistribution, + } +} + +func (s *RandomQueryGenerator) GetQueryRecipe() QueryRecipe { + metric := s.getRandomMetric() + + num := rand.Intn(100) + if num < s.listCurrentValuesProbability { + qty := 5 + rand.Intn(100) // від 5 до 105 + return QueryRecipe{ + MetricIDs: s.listRandomUniqueMetricIDs(qty), + Method: listCurrentValues, + } + } else { + if metric.MetricType == diploma.Cumulative { + num = rand.Intn(100) + if num < s.listPeriodsProbability { + groupBy := s.groupByOptions[rand.Intn(len(s.groupByOptions))] + + var ( + minDays = 1 + maxDays = 7 + ) + + if groupBy == diploma.GroupByDay { + minDays = 1 + maxDays = 30 + } else if groupBy == diploma.GroupByMonth { + minDays = 1 + maxDays = 30 + } + + rangeCode, since, until := s.getRandomTimeRange( + metric.Since, metric.Until, minDays, maxDays) + + return QueryRecipe{ + MetricID: metric.MetricID, + Method: listCumulativePeriods, + RangeCode: rangeCode, + Since: uint32(since), + Until: uint32(until), + GroupBy: groupBy, + } + } else { + var ( + minDays = 1 + maxDays = 3 + ) + + rangeCode, since, until := s.getRandomTimeRange( + metric.Since, metric.Until, minDays, maxDays) + + return QueryRecipe{ + MetricID: metric.MetricID, + Method: listCumulativeMeasures, + RangeCode: rangeCode, + Since: uint32(since), + Until: uint32(until), + } + } + } else { + num = rand.Intn(100) + if num < s.listPeriodsProbability { + groupBy := s.groupByOptions[rand.Intn(len(s.groupByOptions))] + + var ( + minDays = 1 + maxDays = 7 + ) + + if groupBy == diploma.GroupByDay { + minDays = 1 + maxDays = 30 + } else if groupBy == diploma.GroupByMonth { + minDays = 1 + maxDays = 30 + } + + rangeCode, since, until := s.getRandomTimeRange( + metric.Since, metric.Until, minDays, maxDays) + + return QueryRecipe{ + MetricID: metric.MetricID, + Method: listInstantPeriods, + RangeCode: rangeCode, + Since: uint32(since), + Until: uint32(until), + GroupBy: groupBy, + } + } else { + var ( + minDays = 1 + maxDays = 3 + ) + + rangeCode, since, until := s.getRandomTimeRange( + metric.Since, metric.Until, minDays, maxDays) + + return QueryRecipe{ + MetricID: metric.MetricID, + Method: listInstantMeasures, + RangeCode: rangeCode, + Since: uint32(since), + Until: uint32(until), + } + } + } + } + +} + +// Генерує випадковий набір унікальних metricID з [1, 100] +func (s *RandomQueryGenerator) listRandomUniqueMetricIDs(count int) []uint32 { + // переставляю індекси у випадковому порядку + indexes := rand.Perm(len(s.metrics)) + // копіюю metricID із перших випадкових індексів + metricIDs := make([]uint32, count) + for i := range count { + metricIDs[i] = s.metrics[indexes[i]].MetricID + } + return metricIDs +} + +const ( + secondsPerDay = 86400 + dayRange = 0 + weekRange = 1 + monthRange = 2 + randomTimeRange = 3 +) + +// Випадковий часовий діапазон +func (s *RandomQueryGenerator) getRandomTimeRange(start, end int64, minDays, maxDays int) (int, int64, int64) { + var ( + since int64 + until int64 + num = rand.Intn(100) + rangeCode int + threshold int + ) + for rangeCode, threshold = range s.timeRangeProbDistribution { + if num < threshold { + break + } + } + + switch rangeCode { + case dayRange: + since = end - secondsPerDay + until = end + + case weekRange: + since = end - 7*secondsPerDay + until = end + + case monthRange: + since = end - 30*secondsPerDay + until = end + + case randomTimeRange: + if start == end { + return rangeCode, start, end + } + // Випадковий момент часу для since + since = start + rand.Int63n(end-start) + // Випадкова тривалість у днях (але не виходити за межу end) + durationInDays := minDays + rand.Intn(maxDays-minDays) + + until = since + int64(durationInDays)*secondsPerDay + if until > end { + until = end + } + } + return rangeCode, since, until +} + +func (s *RandomQueryGenerator) getRandomMetric() MetricInfo { + return s.metrics[rand.Intn(len(s.metrics))] +} + +// EXECUTE QUERY + +func execQuery(conn *client.Connection, queryGenerator *RandomQueryGenerator, stat *WorkerStat) (err error) { + recipe := queryGenerator.GetQueryRecipe() + + var elapsedTime time.Duration + + switch recipe.Method { + case listCurrentValues: + t1 := time.Now() + _, err := conn.ListCurrentValues(recipe.MetricIDs) + elapsedTime = time.Since(t1) + stat.ElapsedTime += elapsedTime + stat.Queries++ + stat.ElapsedTimeByMethods[recipe.Method] += elapsedTime + stat.MethodCalls[recipe.Method]++ + if err != nil { + return fmt.Errorf("ListCurrentValues: %s", err) + } + + case listInstantMeasures: + t1 := time.Now() + _, err := conn.ListInstantMeasures(proto.ListInstantMeasuresReq{ + MetricID: recipe.MetricID, + Since: recipe.Since, + Until: recipe.Until, + }) + elapsedTime = time.Since(t1) + stat.ElapsedTime += elapsedTime + stat.Queries++ + stat.ElapsedTimeByMethods[recipe.Method] += elapsedTime + stat.MethodCalls[recipe.Method]++ + stat.ElapsedTimeByTimeRanges[recipe.RangeCode] += elapsedTime + stat.TimeRangeCalls[recipe.RangeCode]++ + if err != nil { + return fmt.Errorf("ListInstantMeasures(%d): %s", + recipe.MetricID, err) + } + + case listCumulativeMeasures: + t1 := time.Now() + _, err := conn.ListCumulativeMeasures(proto.ListCumulativeMeasuresReq{ + MetricID: recipe.MetricID, + Since: recipe.Since, + Until: recipe.Until, + }) + elapsedTime = time.Since(t1) + stat.ElapsedTime += elapsedTime + stat.Queries++ + stat.ElapsedTimeByMethods[recipe.Method] += elapsedTime + stat.MethodCalls[recipe.Method]++ + stat.ElapsedTimeByTimeRanges[recipe.RangeCode] += elapsedTime + stat.TimeRangeCalls[recipe.RangeCode]++ + if err != nil { + return fmt.Errorf("ListCumulativeMeasures(%d): %s", + recipe.MetricID, err) + } + + case listInstantPeriods: + t1 := time.Now() + _, err := conn.ListInstantPeriods(proto.ListInstantPeriodsReq{ + MetricID: recipe.MetricID, + Since: recipe.Since, + Until: recipe.Until, + GroupBy: recipe.GroupBy, + AggregateFuncs: diploma.AggregateMin | diploma.AggregateMax | diploma.AggregateAvg, + }) + elapsedTime = time.Since(t1) + stat.ElapsedTime += elapsedTime + stat.Queries++ + stat.ElapsedTimeByMethods[recipe.Method] += elapsedTime + stat.MethodCalls[recipe.Method]++ + stat.ElapsedTimeByTimeRanges[recipe.RangeCode] += elapsedTime + stat.TimeRangeCalls[recipe.RangeCode]++ + if err != nil { + return fmt.Errorf("ListInstantPeriods(%d): %s", + recipe.MetricID, err) + } + + case listCumulativePeriods: + t1 := time.Now() + _, err := conn.ListCumulativePeriods(proto.ListCumulativePeriodsReq{ + MetricID: recipe.MetricID, + Since: recipe.Since, + Until: recipe.Until, + GroupBy: recipe.GroupBy, + }) + elapsedTime = time.Since(t1) + stat.ElapsedTime += elapsedTime + stat.Queries++ + stat.ElapsedTimeByMethods[recipe.Method] += elapsedTime + stat.MethodCalls[recipe.Method]++ + stat.ElapsedTimeByTimeRanges[recipe.RangeCode] += elapsedTime + stat.TimeRangeCalls[recipe.RangeCode]++ + if err != nil { + return fmt.Errorf("ListCumulativePeriods(%d): %s", + recipe.MetricID, err) + } + } + return +} diff --git a/examples/loadtest/main.go b/examples/loadtest/main.go new file mode 100644 index 0000000..2c64739 --- /dev/null +++ b/examples/loadtest/main.go @@ -0,0 +1,261 @@ +package main + +import ( + "flag" + "fmt" + "log" + "math/rand" + "os" + "sync" + "time" + + "gopkg.in/ini.v1" + "gordenko.dev/dima/diploma/client" +) + +const ( + listCumulativeMeasures = 0 + listCumulativePeriods = 1 + listInstantMeasures = 2 + listInstantPeriods = 3 + listCurrentValues = 4 + + methodsQty = 5 + timeRangesQty = 4 +) + +var ( + methodCodeToName = []string{ + "listCumulativeMeasures", + "listCumulativePeriods", + "listInstantMeasures", + "listInstantPeriods", + "listCurrentValues", + } + + rangeCodeToName = []string{ + "last day", + "last week", + "last month", + "random time range", + } +) + +type WorkerStat struct { + Queries int + ElapsedTime time.Duration + MethodCalls []int + ElapsedTimeByMethods []time.Duration + TimeRangeCalls []int + ElapsedTimeByTimeRanges []time.Duration +} + +func main() { + var ( + iniFileName string + ) + + flag.Usage = func() { + fmt.Fprint(flag.CommandLine.Output(), helpMessage) + fmt.Fprint(flag.CommandLine.Output(), configExample) + fmt.Fprintf(flag.CommandLine.Output(), mainUsage, os.Args[0]) + flag.PrintDefaults() + } + flag.StringVar(&iniFileName, "c", "loadtest.ini", "path to *.ini config file") + flag.Parse() + + config, err := loadConfig(iniFileName) + if err != nil { + log.Fatalln(err) + } + + rand.Seed(time.Now().UnixNano()) + + metrics, err := readMetricInfo(config.MetricsInfo) + if err != nil { + log.Fatalln(err) + } + + var ( + wg = new(sync.WaitGroup) + stats = make([]*WorkerStat, config.Connections) + queryGenerator = NewRandomQueryGenerator(RandomQueryGeneratorOptions{ + Metrics: metrics, + // call method probabilitites + ListCurrentValuesProbability: 50, // current values / others + ListPeriodsProbability: 80, // periods / measures + // time range probabilities + TimeRangeProbabilities: []int{ + 82, // last day + 12, // last week + 3, // last month + 3, // any range + }, + }) + ) + + for i := range stats { + stats[i] = &WorkerStat{ + MethodCalls: make([]int, methodsQty), + ElapsedTimeByMethods: make([]time.Duration, methodsQty), + TimeRangeCalls: make([]int, timeRangesQty), + ElapsedTimeByTimeRanges: make([]time.Duration, timeRangesQty), + } + } + + t1 := time.Now() + + for i := range config.Connections { + wg.Add(1) + go func(stat *WorkerStat) { + defer wg.Done() + + conn, err := client.Connect(config.DatabaseAddr) + if err != nil { + log.Fatalln(err) + } + defer conn.Close() + + for range config.RequestsPerConn { + err := execQuery(conn, queryGenerator, stat) + if err != nil { + log.Println(err) + } + } + }(stats[i]) + } + + wg.Wait() + + testingTime := time.Since(t1) + + var ( + methodCalls = make([]int, methodsQty) + elapsedTimeByMethods = make([]time.Duration, methodsQty) + timeRangeCalls = make([]int, timeRangesQty) + elapsedTimeByTimeRanges = make([]time.Duration, timeRangesQty) + totalElapsedTime time.Duration + totalQueries int + avgTimePerQuery time.Duration + rps float64 + ) + + for _, stat := range stats { + totalElapsedTime += stat.ElapsedTime + totalQueries += stat.Queries + + for i, elapsedTime := range stat.ElapsedTimeByMethods { + elapsedTimeByMethods[i] += elapsedTime + } + for i, qty := range stat.MethodCalls { + methodCalls[i] += qty + } + for i, elapsedTime := range stat.ElapsedTimeByTimeRanges { + elapsedTimeByTimeRanges[i] += elapsedTime + } + for i, qty := range stat.TimeRangeCalls { + timeRangeCalls[i] += qty + } + } + + avgTimePerQuery = totalElapsedTime / time.Duration(totalQueries) + rps = float64(config.Connections*config.RequestsPerConn) / testingTime.Seconds() + + fmt.Printf(`TEST RESULTS: +Time: %.0f seconds +Connections: %d +Requests per conn: %d +Total requests: %d +AVG request time: %v +RPS: %d + +`, + testingTime.Seconds(), config.Connections, config.RequestsPerConn, + totalQueries, avgTimePerQuery, int(rps)) + + for i, calls := range methodCalls { + totalElapsedTimeByMethod := elapsedTimeByMethods[i] + + methodPercent := float64(calls*100) / float64(totalQueries) + + fmt.Printf("%s: %d (%.1f%%), AVG request time: %v\n", + methodCodeToName[i], calls, methodPercent, + totalElapsedTimeByMethod/time.Duration(calls)) + } + + fmt.Println() + + for i, calls := range timeRangeCalls { + totalElapsedTimeByTimeRange := elapsedTimeByTimeRanges[i] + + timeRangePercent := float64(calls*100) / float64(totalQueries-methodCalls[listCurrentValues]) + + fmt.Printf("%s: %d (%.1f%%), AVG request time: %v\n", + rangeCodeToName[i], calls, timeRangePercent, + totalElapsedTimeByTimeRange/time.Duration(calls)) + } +} + +// CONFIG FILE + +const mainUsage = `Usage: + %s -c path/to/config.ini + +` + +const helpMessage = `Diploma project. Load test. Version: 1.0 +created by Dmytro Gordenko, 1.e4.kc6@gmail.com +` + +const configExample = ` +loadtest.ini example: + +databaseAddr = :12345 +metricsInfo = ../../datadir/metrics.info +connections = 1000 +requestsPerConn = 500 + +` + +type Config struct { + DatabaseAddr string + MetricsInfo string + Connections int + RequestsPerConn int +} + +func (s Config) String() string { + return fmt.Sprintf(`starting options: +databaseAddr = %s +metricsInfo = %s +connections = %d +requestsPerConn = %d +`, + s.DatabaseAddr, s.MetricsInfo, s.Connections, s.RequestsPerConn) +} + +func loadConfig(iniFileName string) (_ Config, err error) { + file, err := ini.Load(iniFileName) + if err != nil { + return + } + + conf := Config{} + + top := file.Section("") + + conf.DatabaseAddr = top.Key("databaseAddr").String() + conf.MetricsInfo = top.Key("metricsInfo").String() + + conf.Connections, err = top.Key("connections").Int() + if err != nil { + err = fmt.Errorf("'connections' option is required in config file") + return + } + conf.RequestsPerConn, err = top.Key("requestsPerConn").Int() + if err != nil { + err = fmt.Errorf("'requestsPerConn' option is required in config file") + return + } + return conf, nil +} diff --git a/examples/requests/generate.go b/examples/requests/generate.go new file mode 100644 index 0000000..d921e0f --- /dev/null +++ b/examples/requests/generate.go @@ -0,0 +1,81 @@ +package main + +import ( + "math/rand" + "time" + + "gordenko.dev/dima/diploma/client" +) + +func GenerateCumulativeMeasures(days int) []client.Measure { + var ( + measures []client.Measure + minutes = []int{14, 29, 44, 59} + hoursPerDay = 24 + totalHours = days * hoursPerDay + since = time.Now().AddDate(0, 0, -days) + totalValue float64 + ) + + for i := range totalHours { + hourTime := since.Add(time.Duration(i) * time.Hour) + for _, m := range minutes { + measureTime := time.Date( + hourTime.Year(), + hourTime.Month(), + hourTime.Day(), + hourTime.Hour(), + m, // minutes + 0, // seconds + 0, // nanoseconds + time.Local, + ) + + measure := client.Measure{ + Timestamp: uint32(measureTime.Unix()), + Value: totalValue, + } + measures = append(measures, measure) + + totalValue += rand.Float64() + } + } + return measures +} + +func GenerateInstantMeasures(days int, baseValue float64) []client.Measure { + var ( + measures []client.Measure + minutes = []int{14, 29, 44, 59} + hoursPerDay = 24 + totalHours = days * hoursPerDay + since = time.Now().AddDate(0, 0, -days) + ) + + for i := range totalHours { + hourTime := since.Add(time.Duration(i) * time.Hour) + for _, m := range minutes { + measureTime := time.Date( + hourTime.Year(), + hourTime.Month(), + hourTime.Day(), + hourTime.Hour(), + m, // minutes + 0, // seconds + 0, // nanoseconds + time.Local, + ) + + // value = +-10% from base value + fluctuation := baseValue * 0.1 + value := baseValue + (rand.Float64()*2-1)*fluctuation + + measure := client.Measure{ + Timestamp: uint32(measureTime.Unix()), + Value: value, + } + measures = append(measures, measure) + } + } + return measures +} diff --git a/examples/requests/main.go b/examples/requests/main.go new file mode 100644 index 0000000..4209ce7 --- /dev/null +++ b/examples/requests/main.go @@ -0,0 +1,90 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + + "gopkg.in/ini.v1" + "gordenko.dev/dima/diploma/client" +) + +var ( + metricTypeToName = []string{ + "", + "cumulative", + "instant", + } +) + +func main() { + var ( + iniFileName string + ) + + flag.Usage = func() { + fmt.Fprint(flag.CommandLine.Output(), helpMessage) + fmt.Fprint(flag.CommandLine.Output(), configExample) + fmt.Fprintf(flag.CommandLine.Output(), mainUsage, os.Args[0]) + flag.PrintDefaults() + } + flag.StringVar(&iniFileName, "c", "requests.ini", "path to *.ini config file") + flag.Parse() + + config, err := loadConfig(iniFileName) + if err != nil { + log.Fatalln(err) + } + + conn, err := client.Connect(config.DatabaseAddr) + if err != nil { + log.Fatalf("client.Connect(%s): %s\n", config.DatabaseAddr, err) + } else { + fmt.Println("Connected to database") + } + + sendRequests(conn) +} + +// CONFIG FILE + +const mainUsage = `Usage: + %s -c path/to/config.ini + +` + +const helpMessage = `Diploma project. Example requests. Version: 1.0 +created by Dmytro Gordenko, 1.e4.kc6@gmail.com +` + +const configExample = ` +requests.ini example: + +databaseAddr = :12345 + +` + +type Config struct { + DatabaseAddr string +} + +func (s Config) String() string { + return fmt.Sprintf(`starting options: +databaseAddr = %s +`, + s.DatabaseAddr) +} + +func loadConfig(iniFileName string) (_ Config, err error) { + file, err := ini.Load(iniFileName) + if err != nil { + return + } + + conf := Config{} + top := file.Section("") + + conf.DatabaseAddr = top.Key("databaseAddr").String() + return conf, nil +} diff --git a/examples/requests/requests.go b/examples/requests/requests.go new file mode 100644 index 0000000..60213bb --- /dev/null +++ b/examples/requests/requests.go @@ -0,0 +1,361 @@ +package main + +import ( + "fmt" + "log" + "time" + + "gordenko.dev/dima/diploma" + "gordenko.dev/dima/diploma/client" + "gordenko.dev/dima/diploma/proto" +) + +func sendRequests(conn *client.Connection) { + var ( + instantMetricID uint32 = 10000 + cumulativeMetricID uint32 = 10001 + fracDigits byte = 2 + err error + ) + + conn.DeleteMetric(instantMetricID) + conn.DeleteMetric(cumulativeMetricID) + + // ADD INSTANT METRIC + + err = conn.AddMetric(client.Metric{ + MetricID: instantMetricID, + MetricType: diploma.Instant, + FracDigits: fracDigits, + }) + if err != nil { + log.Fatalf("conn.AddMetric: %s\n", err) + } else { + fmt.Printf("\nInstant metric %d added\n", instantMetricID) + } + + // GET INSTANT METRIC + + iMetric, err := conn.GetMetric(instantMetricID) + if err != nil { + log.Fatalf("conn.GetMetric: %s\n", err) + } else { + fmt.Printf(` +GetMetric: + metricID: %d + metricType: %s + fracDigits: %d +`, + iMetric.MetricID, metricTypeToName[iMetric.MetricType], fracDigits) + } + + // APPEND MEASURES + + instantMeasures := GenerateInstantMeasures(62, 220) + + err = conn.AppendMeasures(client.AppendMeasuresReq{ + MetricID: instantMetricID, + Measures: instantMeasures, + }) + if err != nil { + log.Fatalf("conn.AppendMeasures: %s\n", err) + } else { + fmt.Printf("\nAppended %d measures for the metric %d\n", + len(instantMeasures), instantMetricID) + } + + // LIST INSTANT MEASURES + + lastTimestamp := instantMeasures[len(instantMeasures)-1].Timestamp + until := time.Unix(int64(lastTimestamp), 0) + since := until.Add(-5 * time.Hour) + + instantList, err := conn.ListInstantMeasures(proto.ListInstantMeasuresReq{ + MetricID: instantMetricID, + Since: uint32(since.Unix()), + Until: uint32(until.Unix()), + }) + if err != nil { + log.Fatalf("conn.ListInstantMeasures: %s\n", err) + } else { + fmt.Printf("\nListInstantMeasures %s - %s:\n", + formatTime(uint32(since.Unix())), formatTime(uint32(until.Unix()))) + for _, item := range instantList { + fmt.Printf(" %s => %.2f\n", formatTime(item.Timestamp), item.Value) + } + } + + // LIST ALL INSTANT MEASURES + + instantList, err = conn.ListAllInstantMeasures(instantMetricID) + if err != nil { + log.Fatalf("conn.ListAllInstantMeasures: %s\n", err) + } else { + fmt.Printf("\nListAllInstantMeasures (last 15 items):\n") + for _, item := range instantList[:15] { + fmt.Printf(" %s => %.2f\n", formatTime(item.Timestamp), item.Value) + } + } + + // LIST INSTANT PERIODS (group by hour) + + until = time.Unix(int64(lastTimestamp+1), 0) + since = until.Add(-24 * time.Hour) + + instantPeriods, err := conn.ListInstantPeriods(proto.ListInstantPeriodsReq{ + MetricID: instantMetricID, + Since: uint32(since.Unix()), + Until: uint32(until.Unix()), + GroupBy: diploma.GroupByHour, + AggregateFuncs: diploma.AggregateMin | diploma.AggregateMax | diploma.AggregateAvg, + }) + if err != nil { + log.Fatalf("conn.ListInstantPeriods: %s\n", err) + } else { + fmt.Printf("\nListInstantPeriods (1 day, group by hour):\n") + for _, item := range instantPeriods { + fmt.Printf(" %s => min %.2f, max %.2f, avg %.2f\n", formatHourPeriod(item.Period), item.Min, item.Max, item.Avg) + } + } + + // LIST INSTANT PERIODS (group by day) + + until = time.Unix(int64(lastTimestamp+1), 0) + since = until.AddDate(0, 0, -7) + + instantPeriods, err = conn.ListInstantPeriods(proto.ListInstantPeriodsReq{ + MetricID: instantMetricID, + Since: uint32(since.Unix()), + Until: uint32(until.Unix()), + GroupBy: diploma.GroupByDay, + AggregateFuncs: diploma.AggregateMin | diploma.AggregateMax | diploma.AggregateAvg, + }) + if err != nil { + log.Fatalf("conn.ListInstantPeriods: %s\n", err) + } else { + fmt.Printf("\nListInstantPeriods (7 days, group by day):\n") + for _, item := range instantPeriods { + fmt.Printf(" %s => min %.2f, max %.2f, avg %.2f\n", formatDayPeriod(item.Period), item.Min, item.Max, item.Avg) + } + } + + // LIST INSTANT PERIODS (group by month) + + until = time.Unix(int64(lastTimestamp+1), 0) + since = until.AddDate(0, 0, -62) + + instantPeriods, err = conn.ListInstantPeriods(proto.ListInstantPeriodsReq{ + MetricID: instantMetricID, + Since: uint32(since.Unix()), + Until: uint32(until.Unix()), + GroupBy: diploma.GroupByMonth, + AggregateFuncs: diploma.AggregateMin | diploma.AggregateMax | diploma.AggregateAvg, + }) + if err != nil { + log.Fatalf("conn.ListInstantPeriods: %s\n", err) + } else { + fmt.Printf("\nListInstantPeriods (62 days, group by month):\n") + for _, item := range instantPeriods { + fmt.Printf(" %s => min %.2f, max %.2f, avg %.2f\n", formatMonthPeriod(item.Period), item.Min, item.Max, item.Avg) + } + } + + // DELETE INSTANT METRIC MEASURES + + err = conn.DeleteMeasures(proto.DeleteMeasuresReq{ + MetricID: instantMetricID, + }) + if err != nil { + log.Fatalf("conn.DeleteMeasures: %s\n", err) + } else { + fmt.Printf("\nInstant metric %d measures deleted\n", instantMetricID) + } + + // DELETE INSTANT METRIC + + err = conn.DeleteMetric(instantMetricID) + if err != nil { + log.Fatalf("conn.DeleteMetric: %s\n", err) + } else { + fmt.Printf("\nInstant metric %d deleted\n", instantMetricID) + } + + // ADD CUMULATIVE METRIC + + err = conn.AddMetric(client.Metric{ + MetricID: cumulativeMetricID, + MetricType: diploma.Cumulative, + FracDigits: fracDigits, + }) + if err != nil { + log.Fatalf("conn.AddMetric: %s\n", err) + } else { + fmt.Printf("\nCumulative metric %d added\n", cumulativeMetricID) + } + + // GET CUMULATIVE METRIC + + cMetric, err := conn.GetMetric(cumulativeMetricID) + if err != nil { + log.Fatalf("conn.GetMetric: %s\n", err) + } else { + fmt.Printf(` +GetMetric: + metricID: %d + metricType: %s + fracDigits: %d +`, + cMetric.MetricID, metricTypeToName[cMetric.MetricType], fracDigits) + } + + // APPEND MEASURES + + cumulativeMeasures := GenerateCumulativeMeasures(62) + + err = conn.AppendMeasures(client.AppendMeasuresReq{ + MetricID: cumulativeMetricID, + Measures: cumulativeMeasures, + }) + if err != nil { + log.Fatalf("conn.AppendMeasures: %s\n", err) + } else { + fmt.Printf("\nAppended %d measures for the metric %d\n", + len(cumulativeMeasures), cumulativeMetricID) + } + + // LIST CUMULATIVE MEASURES + + lastTimestamp = cumulativeMeasures[len(cumulativeMeasures)-1].Timestamp + until = time.Unix(int64(lastTimestamp), 0) + since = until.Add(-5 * time.Hour) + + cumulativeList, err := conn.ListCumulativeMeasures(proto.ListCumulativeMeasuresReq{ + MetricID: cumulativeMetricID, + Since: uint32(since.Unix()), + Until: uint32(until.Unix()), + }) + if err != nil { + log.Fatalf("conn.ListCumulativeMeasures: %s\n", err) + } else { + fmt.Printf("\nListCumulativeMeasures %s - %s:\n", + formatTime(uint32(since.Unix())), formatTime(uint32(until.Unix()))) + + for _, item := range cumulativeList { + fmt.Printf(" %s => %.2f\n", formatTime(item.Timestamp), item.Value) + } + } + + // LIST ALL CUMULATIVE MEASURES + + cumulativeList, err = conn.ListAllCumulativeMeasures(cumulativeMetricID) + if err != nil { + log.Fatalf("conn.ListAllCumulativeMeasures: %s\n", err) + } else { + fmt.Printf("\nListAllCumulativeMeasures (last 15 items):\n") + for _, item := range cumulativeList[:15] { + fmt.Printf(" %s => %.2f\n", formatTime(item.Timestamp), item.Value) + } + } + + // LIST CUMULATIVE PERIODS (group by hour) + + until = time.Unix(int64(lastTimestamp+1), 0) + since = until.Add(-24 * time.Hour) + + cumulativePeriods, err := conn.ListCumulativePeriods(proto.ListCumulativePeriodsReq{ + MetricID: cumulativeMetricID, + Since: uint32(since.Unix()), + Until: uint32(until.Unix()), + GroupBy: diploma.GroupByHour, + }) + if err != nil { + log.Fatalf("conn.ListCumulativePeriods: %s\n", err) + } else { + fmt.Printf("\nListCumulativePeriods (1 day, group by hour):\n") + for _, item := range cumulativePeriods { + fmt.Printf(" %s => end value %.2f, total %.2f\n", formatHourPeriod(item.Period), item.EndValue, item.Total) + } + } + + // LIST CUMULATIVE PERIODS (group by day) + + until = time.Unix(int64(lastTimestamp+1), 0) + since = until.AddDate(0, 0, -7) + + cumulativePeriods, err = conn.ListCumulativePeriods(proto.ListCumulativePeriodsReq{ + MetricID: cumulativeMetricID, + Since: uint32(since.Unix()), + Until: uint32(until.Unix()), + GroupBy: diploma.GroupByDay, + }) + if err != nil { + log.Fatalf("conn.ListCumulativePeriods: %s\n", err) + } else { + fmt.Printf("\nListCumulativePeriods (7 days, group by day):\n") + for _, item := range cumulativePeriods { + fmt.Printf(" %s => end value %.2f, total %.2f\n", formatDayPeriod(item.Period), item.EndValue, item.Total) + } + } + + // LIST CUMULATIVE PERIODS (group by day) + + until = time.Unix(int64(lastTimestamp+1), 0) + since = until.AddDate(0, 0, -62) + + cumulativePeriods, err = conn.ListCumulativePeriods(proto.ListCumulativePeriodsReq{ + MetricID: cumulativeMetricID, + Since: uint32(since.Unix()), + Until: uint32(until.Unix()), + GroupBy: diploma.GroupByMonth, + }) + if err != nil { + log.Fatalf("conn.ListCumulativePeriods: %s\n", err) + } else { + fmt.Printf("\nListCumulativePeriods (62 days, group by month):\n") + for _, item := range cumulativePeriods { + fmt.Printf(" %s => end value %.2f, total %.2f\n", formatMonthPeriod(item.Period), item.EndValue, item.Total) + } + } + + // DELETE CUMULATIVE METRIC MEASURES + + err = conn.DeleteMeasures(proto.DeleteMeasuresReq{ + MetricID: cumulativeMetricID, + }) + if err != nil { + log.Fatalf("conn.DeleteMeasures: %s\n", err) + } else { + fmt.Printf("\nCumulative metric %d measures deleted\n", cumulativeMetricID) + } + + // DELETE CUMULATIVE METRIC + + err = conn.DeleteMetric(cumulativeMetricID) + if err != nil { + log.Fatalf("conn.DeleteMetric: %s\n", err) + } else { + fmt.Printf("\nCumulative metric %d deleted\n", cumulativeMetricID) + } +} + +const datetimeLayout = "2006-01-02 15:04:05" + +func formatTime(timestamp uint32) string { + tm := time.Unix(int64(timestamp), 0) + return tm.Format(datetimeLayout) +} + +func formatHourPeriod(period uint32) string { + tm := time.Unix(int64(period), 0) + return tm.Format("2006-01-02 15:00 - 15") + ":59" +} + +func formatDayPeriod(period uint32) string { + tm := time.Unix(int64(period), 0) + return tm.Format("2006-01-02") +} + +func formatMonthPeriod(period uint32) string { + tm := time.Unix(int64(period), 0) + return tm.Format("2006-01") +} diff --git a/freelist/freelist.go b/freelist/freelist.go new file mode 100644 index 0000000..e2c8d15 --- /dev/null +++ b/freelist/freelist.go @@ -0,0 +1,72 @@ +package freelist + +import ( + "fmt" + "sync" + + "github.com/RoaringBitmap/roaring/v2" +) + +type FreeList struct { + mutex sync.Mutex + free *roaring.Bitmap + reserved *roaring.Bitmap +} + +func New() *FreeList { + return &FreeList{ + free: roaring.New(), + reserved: roaring.New(), + } +} + +func (s *FreeList) Restore(serialized []byte) error { + err := s.free.UnmarshalBinary(serialized) + if err != nil { + return fmt.Errorf("UnmarshalBinary: %s", err) + } + return nil +} + +func (s *FreeList) AddPages(pageNumbers []uint32) { + if len(pageNumbers) == 0 { + return + } + s.mutex.Lock() + s.free.AddMany(pageNumbers) + s.mutex.Unlock() +} + +// ReserveDataPage - аллокатор резервирует страницу, но не удаляет до визова +// DeleteFromFree, ибо транзакция может не завершится, а между віделением страници +// и падением транзакции - будет создан init файл. +func (s *FreeList) ReservePage() (pageNo uint32) { + s.mutex.Lock() + defer s.mutex.Unlock() + + if s.free.IsEmpty() { + return + } + pageNo = s.free.Minimum() + s.free.Remove(pageNo) + s.reserved.Add(pageNo) + return +} + +// Удаляет ранее зарезервированные страницы +func (s *FreeList) DeletePages(pageNumbers []uint32) { + s.mutex.Lock() + for _, pageNo := range pageNumbers { + s.reserved.Remove(pageNo) + s.free.Remove(pageNo) // прокрута TransactionLog + } + s.mutex.Unlock() +} + +func (s *FreeList) Serialize() ([]byte, error) { + s.mutex.Lock() + defer s.mutex.Unlock() + tmp := roaring.Or(s.free, s.reserved) + tmp.RunOptimize() + return tmp.ToBytes() +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..995e343 --- /dev/null +++ b/go.mod @@ -0,0 +1,13 @@ +module gordenko.dev/dima/diploma + +go 1.24.0 + +require ( + github.com/RoaringBitmap/roaring/v2 v2.5.0 + gopkg.in/ini.v1 v1.67.0 +) + +require ( + github.com/bits-and-blooms/bitset v1.12.0 // indirect + github.com/mschoch/smat v0.2.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..9a5aafa --- /dev/null +++ b/go.sum @@ -0,0 +1,20 @@ +github.com/RoaringBitmap/roaring/v2 v2.5.0 h1:TJ45qCM7D7fIEBwKd9zhoR0/S1egfnSSIzLU1e1eYLY= +github.com/RoaringBitmap/roaring/v2 v2.5.0/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= +github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA= +github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/linux_build.sh b/linux_build.sh new file mode 100755 index 0000000..0747f5d --- /dev/null +++ b/linux_build.sh @@ -0,0 +1,11 @@ +cd examples/database +env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ../../database_linux +cd - + +cd examples/loadtest +env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ../../loadtest_linux +cd - + +cd examples/requests +env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ../../requests_linux +cd - \ No newline at end of file diff --git a/loadtest.ini b/loadtest.ini new file mode 100644 index 0000000..6fd08ad --- /dev/null +++ b/loadtest.ini @@ -0,0 +1,9 @@ + +# host:port +databaseAddr = :12345 +# path to metrics.info file +metricsInfo = testdir/metrics.info +# the number of concurrently open connections. +connections = 1000 +# send requests per one connection +requestsPerConn = 500 diff --git a/loadtest_linux b/loadtest_linux new file mode 100755 index 0000000..122c54b Binary files /dev/null and b/loadtest_linux differ diff --git a/loadtest_windows b/loadtest_windows new file mode 100755 index 0000000..a979d15 Binary files /dev/null and b/loadtest_windows differ diff --git a/proto/proto.go b/proto/proto.go new file mode 100644 index 0000000..1a44cfb --- /dev/null +++ b/proto/proto.go @@ -0,0 +1,473 @@ +package proto + +import ( + "fmt" + + octopus "gordenko.dev/dima/diploma" + "gordenko.dev/dima/diploma/bin" + "gordenko.dev/dima/diploma/bufreader" +) + +const ( + TypeDeleteMeasures byte = 1 + TypeListCurrentValues byte = 2 + TypeListInstantMeasures byte = 3 + TypeListCumulativeMeasures byte = 33 + TypeListInstantPeriods byte = 4 + TypeListCumulativePeriods byte = 44 + TypeGetMetric byte = 5 + TypeAddMetric byte = 6 + TypeListAllInstantMeasures byte = 8 + TypeListAllCumulativeMeasures byte = 88 + TypeRangeTotal byte = 9 + TypeAppendMeasure byte = 10 + TypeAppendMeasures byte = 11 + TypeDeleteMetric byte = 12 + + RespPartOfValue byte = 255 + RespEndOfValue byte = 254 + RespError byte = 253 + RespSuccess byte = 252 + RespValue byte = 251 + + ErrNoMetric = 1 + ErrDuplicate = 2 + ErrWrongMetricType = 3 + ErrWrongFracDigits = 4 + ErrExpiredMeasure = 5 + ErrNonMonotonicValue = 6 + ErrEmptyMetricID = 7 + ErrInvalidRange = 8 + ErrUnexpected = 9 +) + +func ErrorCodeToText(code uint16) string { + switch code { + case ErrNoMetric: + return "NoMetric" + case ErrDuplicate: + return "Duplicate" + case ErrWrongMetricType: + return "WrongMetricType" + case ErrWrongFracDigits: + return "WrongFracDigits" + case ErrExpiredMeasure: + return "ExpiredMeasure" + case ErrNonMonotonicValue: + return "NonMonotonicValue" + case ErrEmptyMetricID: + return "EmptyMetricID" + case ErrInvalidRange: + return "InvalidRange" + case ErrUnexpected: + return "Unexpected" + default: + return "" + } +} + +type GetMetricReq struct { + MetricID uint32 +} + +type ListCurrentValuesReq struct { + MetricIDs []uint32 +} + +type AddMetricReq struct { + MetricID uint32 + MetricType octopus.MetricType + FracDigits int +} + +type UpdateMetricReq struct { + MetricID uint32 + MetricType octopus.MetricType + FracDigits int +} + +type DeleteMetricReq struct { + MetricID uint32 +} + +type DeleteMeasuresReq struct { + MetricID uint32 + Since uint32 // timestamp (optional) +} + +type AppendMeasureReq struct { + MetricID uint32 + Timestamp uint32 + Value float64 +} + +type ListAllInstantMetricMeasuresReq struct { + MetricID uint32 +} + +type ListAllCumulativeMeasuresReq struct { + MetricID uint32 +} + +type ListInstantMeasuresReq struct { + MetricID uint32 + Since uint32 + Until uint32 + FirstHourOfDay int +} + +type ListCumulativeMeasuresReq struct { + MetricID uint32 + Since uint32 + Until uint32 + FirstHourOfDay int +} + +type ListInstantPeriodsReq struct { + MetricID uint32 + Since uint32 + Until uint32 + GroupBy octopus.GroupBy + AggregateFuncs byte + FirstHourOfDay int + LastDayOfMonth int +} + +type ListCumulativePeriodsReq struct { + MetricID uint32 + Since uint32 + Until uint32 + GroupBy octopus.GroupBy + FirstHourOfDay int + LastDayOfMonth int +} + +type Metric struct { + MetricID uint32 + MetricType octopus.MetricType + FracDigits int +} + +type RangeTotalReq struct { + MetricID uint32 + Since uint32 + Until uint32 +} + +func PackAddMetricReq(req AddMetricReq) []byte { + arr := []byte{ + TypeAddMetric, + 0, 0, 0, 0, // + byte(req.MetricType), + byte(req.FracDigits), + } + bin.PutUint32(arr[1:], req.MetricID) + return arr +} + +func PackDeleteMetricReq(req DeleteMetricReq) []byte { + arr := []byte{ + TypeDeleteMetric, + 0, 0, 0, 0, // metricID + } + bin.PutUint32(arr[1:], req.MetricID) + return arr +} + +func PackAppendMeasure(req AppendMeasureReq) []byte { + arr := []byte{ + TypeAppendMeasure, + 0, 0, 0, 0, // metricID + 0, 0, 0, 0, // timestamp + 0, 0, 0, 0, 0, 0, 0, 0, // value + } + bin.PutUint32(arr[1:], req.MetricID) + bin.PutUint32(arr[5:], uint32(req.Timestamp)) + bin.PutFloat64(arr[9:], req.Value) + return arr +} + +func PackDeleteMeasuresReq(req DeleteMeasuresReq) []byte { + arr := []byte{ + TypeDeleteMeasures, + 0, 0, 0, 0, // metricID + 0, 0, 0, 0, // since + } + bin.PutUint32(arr[1:], req.MetricID) + bin.PutUint32(arr[5:], uint32(req.Since)) + return arr +} + +// UNPACK reqs + +func UnpackAddMetricReq(arr []byte) (m AddMetricReq) { + m.MetricID = bin.GetUint32(arr) + m.MetricType = octopus.MetricType(arr[4]) + m.FracDigits = int(arr[5]) + return +} + +func UnpackUpdateMetricReq(arr []byte) (m UpdateMetricReq) { + m.MetricID = bin.GetUint32(arr) + m.MetricType = octopus.MetricType(arr[4]) + m.FracDigits = int(arr[5]) + return +} + +func UnpackDeleteMetricReq(arr []byte) (m DeleteMetricReq) { + m.MetricID = bin.GetUint32(arr) + return +} + +func UnpackAppendMeasureReq(arr []byte) (m AppendMeasureReq) { + m.MetricID = bin.GetUint32(arr) + m.Timestamp = bin.GetUint32(arr[4:]) + m.Value = bin.GetFloat64(arr[8:]) + return +} + +func UnpackDeleteMeasuresReq(arr []byte) (m DeleteMeasuresReq) { + m.MetricID = bin.GetUint32(arr) + m.Since = bin.GetUint32(arr[4:]) + return +} + +func UnpackListInstantMeasuresReq(arr []byte) (m ListInstantMeasuresReq) { + m.MetricID = bin.GetUint32(arr[0:]) + m.Since = bin.GetUint32(arr[4:]) + m.Until = bin.GetUint32(arr[8:]) + m.FirstHourOfDay = int(arr[12]) + return +} + +func UnpackListCumulativeMeasuresReq(arr []byte) (m ListCumulativeMeasuresReq) { + m.MetricID = bin.GetUint32(arr) + m.Since = bin.GetUint32(arr[4:]) + m.Until = bin.GetUint32(arr[8:]) + m.FirstHourOfDay = int(arr[12]) + return +} + +func UnpackListInstantPeriodsReq(arr []byte) (m ListInstantPeriodsReq) { + m.MetricID = bin.GetUint32(arr) + m.Since = bin.GetUint32(arr[4:]) + m.Until = bin.GetUint32(arr[8:]) + m.GroupBy = octopus.GroupBy(arr[12]) + m.AggregateFuncs = arr[13] + m.FirstHourOfDay = int(arr[14]) + m.LastDayOfMonth = int(arr[15]) + return +} + +func UnpackListCumulativePeriodsReq(arr []byte) (m ListCumulativePeriodsReq) { + m.MetricID = bin.GetUint32(arr[0:]) + m.Since = bin.GetUint32(arr[4:]) + m.Until = bin.GetUint32(arr[8:]) + m.GroupBy = octopus.GroupBy(arr[12]) + m.FirstHourOfDay = int(arr[13]) + m.LastDayOfMonth = int(arr[14]) + return +} + +func UnpackRangeTotalReq(arr []byte) (m RangeTotalReq) { + m.MetricID = bin.GetUint32(arr) + m.Since = bin.GetUint32(arr[4:]) + m.Until = bin.GetUint32(arr[8:]) + return +} + +// READ reqs + +func ReadGetMetricReq(r *bufreader.BufferedReader) (m GetMetricReq, err error) { + m.MetricID, err = bin.ReadUint32(r) + if err != nil { + err = fmt.Errorf("read req: %s", err) + return + } + return +} + +func ReadAddMetricReq(r *bufreader.BufferedReader) (m AddMetricReq, err error) { + arr, err := r.ReadN(6) + if err != nil { + err = fmt.Errorf("read req: %s", err) + return + } + return UnpackAddMetricReq(arr), nil +} + +func ReadUpdateMetricReq(r *bufreader.BufferedReader) (m UpdateMetricReq, err error) { + arr, err := r.ReadN(6) + if err != nil { + err = fmt.Errorf("read req: %s", err) + return + } + return UnpackUpdateMetricReq(arr), nil +} + +func ReadDeleteMetricReq(r *bufreader.BufferedReader) (m DeleteMetricReq, err error) { + m.MetricID, err = bin.ReadUint32(r) + if err != nil { + err = fmt.Errorf("read req: %s", err) + return + } + return +} + +func ReadAppendMeasureReq(r *bufreader.BufferedReader) (m AppendMeasureReq, err error) { + arr, err := r.ReadN(16) + if err != nil { + err = fmt.Errorf("read req: %s", err) + return + } + return UnpackAppendMeasureReq(arr), nil +} + +func ReadDeleteMeasuresReq(r *bufreader.BufferedReader) (m DeleteMeasuresReq, err error) { + arr, err := r.ReadN(8) + if err != nil { + err = fmt.Errorf("read req: %s", err) + return + } + return UnpackDeleteMeasuresReq(arr), nil +} + +func ReadListAllInstantMeasuresReq(r *bufreader.BufferedReader) (m ListAllInstantMetricMeasuresReq, err error) { + m.MetricID, err = bin.ReadUint32(r) + if err != nil { + err = fmt.Errorf("read req: %s", err) + return + } + return +} + +func ReadListAllCumulativeMeasuresReq(r *bufreader.BufferedReader) (m ListAllCumulativeMeasuresReq, err error) { + m.MetricID, err = bin.ReadUint32(r) + if err != nil { + err = fmt.Errorf("read req: %s", err) + return + } + return +} + +func ReadListInstantMeasuresReq(r *bufreader.BufferedReader) (m ListInstantMeasuresReq, err error) { + arr, err := r.ReadN(13) + if err != nil { + err = fmt.Errorf("read req: %s", err) + return + } + return UnpackListInstantMeasuresReq(arr), nil +} + +func ReadListCumulativeMeasuresReq(r *bufreader.BufferedReader) (m ListCumulativeMeasuresReq, err error) { + arr, err := r.ReadN(13) + if err != nil { + err = fmt.Errorf("read req: %s", err) + return + } + return UnpackListCumulativeMeasuresReq(arr), nil +} + +func ReadListInstantPeriodsReq(r *bufreader.BufferedReader) (m ListInstantPeriodsReq, err error) { + arr, err := r.ReadN(16) + if err != nil { + err = fmt.Errorf("read req: %s", err) + return + } + return UnpackListInstantPeriodsReq(arr), nil +} + +func ReadListCumulativePeriodsReq(r *bufreader.BufferedReader) (m ListCumulativePeriodsReq, err error) { + arr, err := r.ReadN(15) + if err != nil { + err = fmt.Errorf("read req: %s", err) + return + } + return UnpackListCumulativePeriodsReq(arr), nil +} + +func ReadRangeTotalReq(r *bufreader.BufferedReader) (m RangeTotalReq, err error) { + arr, err := r.ReadN(12) + if err != nil { + err = fmt.Errorf("read req: %s", err) + return + } + return UnpackRangeTotalReq(arr), nil +} + +func ReadListCurrentValuesReq(r *bufreader.BufferedReader) (m ListCurrentValuesReq, err error) { + qty, err := bin.ReadUint16(r) + if err != nil { + err = fmt.Errorf("read req: %s", err) + return + } + + for i := range int(qty) { + var metricID uint32 + metricID, err = bin.ReadUint32(r) + if err != nil { + err = fmt.Errorf("read metricID (#%d): %s", i, err) + return + } + m.MetricIDs = append(m.MetricIDs, metricID) + } + return +} + +type AppendMeasuresReq struct { + MetricID uint32 + Measures []Measure +} + +type Measure struct { + Timestamp uint32 + Value float64 +} + +func PackAppendMeasures(req AppendMeasuresReq) []byte { + if len(req.Measures) > 65535 { + panic(fmt.Errorf("wrong measures qty: %d", len(req.Measures))) + } + var ( + prefixSize = 7 + recordSize = 12 + arr = make([]byte, prefixSize+len(req.Measures)*recordSize) + ) + arr[0] = TypeAppendMeasures + bin.PutUint32(arr[1:], req.MetricID) + bin.PutUint16(arr[5:], uint16(len(req.Measures))) + pos := prefixSize + for _, measure := range req.Measures { + bin.PutUint32(arr[pos:], measure.Timestamp) + bin.PutFloat64(arr[pos+4:], measure.Value) + pos += recordSize + } + return arr +} + +func ReadAppendMeasuresReq(r *bufreader.BufferedReader) (m AppendMeasuresReq, err error) { + prefix, err := bin.ReadN(r, 6) // metricID + measures qty + if err != nil { + err = fmt.Errorf("read prefix: %s", err) + return + } + + m.MetricID = bin.GetUint32(prefix[0:]) + qty := bin.GetUint16(prefix[4:]) + + for i := range int(qty) { + var measure Measure + measure.Timestamp, err = bin.ReadUint32(r) + if err != nil { + err = fmt.Errorf("read timestamp (#%d): %s", i, err) + return + } + measure.Value, err = bin.ReadFloat64(r) + if err != nil { + err = fmt.Errorf("read value (#%d): %s", i, err) + return + } + m.Measures = append(m.Measures, measure) + } + return +} diff --git a/recovery/advisor.go b/recovery/advisor.go new file mode 100644 index 0000000..af08d1e --- /dev/null +++ b/recovery/advisor.go @@ -0,0 +1,255 @@ +package recovery + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" +) + +var ( + reChanges = regexp.MustCompile(`(\d+)\.changes`) + reSnapshot = regexp.MustCompile(`(\d+)\.snapshot`) +) + +func joinChangesFileName(dir string, logNumber int) string { + return filepath.Join(dir, fmt.Sprintf("%d.changes", logNumber)) +} + +type RecoveryRecipe struct { + Snapshot string + Changes []string + LogNumber int + ToDelete []string + CompleteSnapshot bool // флаг - что нужно завершить создание снапшота +} + +type RecoveryAdvisor struct { + dir string + verifySnapshot func(string) (bool, error) // (fileName) isVerified, error +} + +type RecoveryAdvisorOptions struct { + Dir string + VerifySnapshot func(string) (bool, error) +} + +func NewRecoveryAdvisor(opt RecoveryAdvisorOptions) (*RecoveryAdvisor, error) { + if opt.Dir == "" { + return nil, errors.New("Dir option is required") + } + if opt.VerifySnapshot == nil { + return nil, errors.New("VerifySnapshot option is required") + } + return &RecoveryAdvisor{ + dir: opt.Dir, + verifySnapshot: opt.VerifySnapshot, + }, nil +} + +type SnapshotChangesPair struct { + SnapshotFileName string + ChangesFileName string + LogNumber int +} + +func (s *RecoveryAdvisor) getSnapshotChangesPairs() (*SnapshotChangesPair, *SnapshotChangesPair, error) { + var ( + numSet = make(map[int]bool) + changesSet = make(map[int]bool) + snapshotsSet = make(map[int]bool) + pairs []SnapshotChangesPair + ) + + entries, err := os.ReadDir(s.dir) + if err != nil { + return nil, nil, err + } + + for _, entry := range entries { + if entry.Type().IsRegular() { + baseName := entry.Name() + groups := reChanges.FindStringSubmatch(baseName) + if len(groups) == 2 { + num, _ := strconv.Atoi(groups[1]) + + numSet[num] = true + changesSet[num] = true + } + groups = reSnapshot.FindStringSubmatch(baseName) + if len(groups) == 2 { + num, _ := strconv.Atoi(groups[1]) + + numSet[num] = true + snapshotsSet[num] = true + } + } + } + + for logNumber := range numSet { + var ( + snapshotFileName string + changesFileName string + ) + if changesSet[logNumber] { + changesFileName = joinChangesFileName(s.dir, logNumber) + } + if snapshotsSet[logNumber] { + snapshotFileName = filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", logNumber)) + } + pairs = append(pairs, SnapshotChangesPair{ + ChangesFileName: changesFileName, + SnapshotFileName: snapshotFileName, + LogNumber: logNumber, + }) + } + + if len(pairs) == 0 { + return nil, nil, nil + } + + sort.Slice(pairs, func(i, j int) bool { + return pairs[i].LogNumber > pairs[j].LogNumber + }) + + pair := pairs[0] + if pair.ChangesFileName == "" { + return nil, nil, fmt.Errorf("has %d.shapshot file, but %d.changes file not found", + pair.LogNumber, pair.LogNumber) + } + + if len(pairs) > 1 { + prevPair := pairs[1] + if prevPair.SnapshotFileName == "" && prevPair.LogNumber != 1 { + return &pair, nil, nil + } + + if prevPair.ChangesFileName == "" && pair.SnapshotFileName == "" { + return &pair, nil, nil + } + return &pair, &prevPair, nil + } else { + return &pair, nil, nil + } +} + +func (s *RecoveryAdvisor) GetRecipe() (*RecoveryRecipe, error) { + pair, prevPair, err := s.getSnapshotChangesPairs() + if err != nil { + return nil, err + } + + if pair == nil { + return nil, nil + } + + if pair.SnapshotFileName != "" { + isVerified, err := s.verifySnapshot(pair.SnapshotFileName) + if err != nil { + return nil, fmt.Errorf("verifySnapshot %s: %s", + pair.SnapshotFileName, err) + } + + if isVerified { + recipe := &RecoveryRecipe{ + Snapshot: pair.SnapshotFileName, + Changes: []string{ + pair.ChangesFileName, + }, + LogNumber: pair.LogNumber, + } + + if prevPair != nil { + if prevPair.ChangesFileName != "" { + recipe.ToDelete = append(recipe.ToDelete, prevPair.ChangesFileName) + } + if prevPair.SnapshotFileName != "" { + recipe.ToDelete = append(recipe.ToDelete, prevPair.SnapshotFileName) + } + } + return recipe, nil + } + if prevPair != nil { + return s.tryPrevPair(pair, prevPair) + } + return nil, fmt.Errorf("%d.shapshot is corrupted", pair.LogNumber) + } else { + if prevPair != nil { + return s.tryPrevPair(pair, prevPair) + } else { + if pair.LogNumber == 1 { + return &RecoveryRecipe{ + Changes: []string{ + pair.ChangesFileName, + }, + LogNumber: pair.LogNumber, + }, nil + } else { + return nil, fmt.Errorf("%d.snapshot not found", pair.LogNumber) + } + } + } +} + +func (s *RecoveryAdvisor) tryPrevPair(pair, prevPair *SnapshotChangesPair) (*RecoveryRecipe, error) { + if prevPair.ChangesFileName == "" { + if pair.SnapshotFileName != "" { + return nil, fmt.Errorf("%d.shapshot is corrupted and %d.changes not found", + pair.LogNumber, prevPair.LogNumber) + } else { + return nil, fmt.Errorf("%d.changes not found", prevPair.LogNumber) + } + } + + if prevPair.SnapshotFileName == "" { + if prevPair.LogNumber == 1 { + recipe := &RecoveryRecipe{ + Changes: []string{ + prevPair.ChangesFileName, + pair.ChangesFileName, + }, + LogNumber: pair.LogNumber, + CompleteSnapshot: true, + ToDelete: []string{ + prevPair.ChangesFileName, + }, + } + return recipe, nil + } else { + if pair.SnapshotFileName != "" { + return nil, fmt.Errorf("%d.shapshot is corrupted and %d.snapshot not found", + pair.LogNumber, prevPair.LogNumber) + } else { + return nil, fmt.Errorf("%d.snapshot not found", pair.LogNumber) + } + } + } + + isVerified, err := s.verifySnapshot(prevPair.SnapshotFileName) + if err != nil { + return nil, fmt.Errorf("verifySnapshot %s: %s", + prevPair.SnapshotFileName, err) + } + + if !isVerified { + return nil, fmt.Errorf("%d.shapshot is corrupted", prevPair.LogNumber) + } + + recipe := &RecoveryRecipe{ + Snapshot: prevPair.SnapshotFileName, + Changes: []string{ + prevPair.ChangesFileName, + pair.ChangesFileName, + }, + LogNumber: pair.LogNumber, + CompleteSnapshot: true, + ToDelete: []string{ + prevPair.ChangesFileName, + prevPair.SnapshotFileName, + }, + } + return recipe, nil +} diff --git a/requests.ini b/requests.ini new file mode 100644 index 0000000..9c314ac --- /dev/null +++ b/requests.ini @@ -0,0 +1 @@ +databaseAddr = :12345 diff --git a/requests_linux b/requests_linux new file mode 100755 index 0000000..620b723 Binary files /dev/null and b/requests_linux differ diff --git a/requests_windows b/requests_windows new file mode 100755 index 0000000..d57f6fc Binary files /dev/null and b/requests_windows differ diff --git a/timeutil/timeutil.go b/timeutil/timeutil.go new file mode 100644 index 0000000..5cd559d --- /dev/null +++ b/timeutil/timeutil.go @@ -0,0 +1,39 @@ +package timeutil + +import "time" + +func FirstSecondInPeriod(since time.Time, period string) (_ time.Time) { + y, m, d := since.Date() + + switch period { + case "h": + h := since.Hour() + return time.Date(y, m, d, h, 0, 0, 0, time.Local) + case "d": + return time.Date(y, m, d, 0, 0, 0, 0, time.Local) + case "m": + return time.Date(y, m, 1, 0, 0, 0, 0, time.Local) + default: + return since + } +} + +func LastSecondInPeriod(until time.Time, period string) (_ time.Time) { + y, m, d := until.Date() + + switch period { + case "h": + h := until.Hour() + return time.Date(y, m, d, h, 59, 59, 0, time.Local) + case "d": + return time.Date(y, m, d, 23, 59, 59, 0, time.Local) + case "m": + tm := time.Date(y, m, 1, 23, 59, 59, 0, time.Local) + // Додаю місяць + tm = tm.AddDate(0, 1, 0) + // Віднімаю день + return tm.AddDate(0, 0, -1) + default: + return until + } +} diff --git a/txlog/reader.go b/txlog/reader.go new file mode 100644 index 0000000..0f4f55e --- /dev/null +++ b/txlog/reader.go @@ -0,0 +1,346 @@ +package txlog + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "hash/crc32" + "io" + "os" + + "gordenko.dev/dima/diploma" + "gordenko.dev/dima/diploma/bin" + "gordenko.dev/dima/diploma/proto" +) + +type Reader struct { + file *os.File + reader *bufio.Reader +} + +type ReaderOptions struct { + FileName string + BufferSize int +} + +func NewReader(opt ReaderOptions) (*Reader, error) { + if opt.FileName == "" { + return nil, errors.New("FileName option is required") + } + if opt.BufferSize <= 0 { + return nil, errors.New("BufferSize option is required") + } + + file, err := os.Open(opt.FileName) + if err != nil { + return nil, err + } + + return &Reader{ + file: file, + reader: bufio.NewReaderSize(file, 1024*1024), + }, nil +} + +func (s *Reader) Close() { + s.file.Close() +} + +func (s *Reader) ReadPacket() (uint32, []any, bool, error) { + prefix := make([]byte, packetPrefixSize) + n, err := s.reader.Read(prefix) + if err != nil { + if err == io.EOF && n == 0 { + return 0, nil, true, nil + } else { + return 0, nil, false, fmt.Errorf("read packet prefix: %s", err) + } + } + + length := bin.GetUint32(prefix[lengthIdx:]) + storedCRC := bin.GetUint32(prefix[checksumIdx:]) + lsn := bin.GetUint32(prefix[lsnIdx:]) + + body, err := bin.ReadN(s.reader, int(length)) + if err != nil { + return 0, nil, false, fmt.Errorf("read packet body: %s", err) + } + + hasher := crc32.NewIEEE() + hasher.Write(prefix[lsnIdx:]) + hasher.Write(body) + + calculatedCRC := hasher.Sum32() + + if calculatedCRC != storedCRC { + return 0, nil, false, fmt.Errorf("stored CRC %d != calculated CRC %d", + storedCRC, calculatedCRC) + } + + records, err := s.parseRecords(body) + if err != nil { + return 0, nil, false, err + } + return lsn, records, false, nil +} + +func (s *Reader) parseRecords(body []byte) ([]any, error) { + var ( + src = bytes.NewBuffer(body) + records []any + ) + + for { + recordType, err := src.ReadByte() + if err != nil { + if err == io.EOF { + return records, nil + } + return nil, err + } + + switch recordType { + case CodeAddedMetric: + var rec AddedMetric + rec, err = s.readAddedMetric(src) + if err != nil { + return nil, err + } + records = append(records, rec) + + case CodeDeletedMetric: + var rec DeletedMetric + rec, err = s.readDeletedMetric(src) + if err != nil { + return nil, err + } + records = append(records, rec) + + case CodeAppendedMeasure: + var rec AppendedMeasure + rec, err = s.readAppendedMeasure(src) + if err != nil { + return nil, err + } + records = append(records, rec) + + case CodeAppendedMeasures: + var rec AppendedMeasures + rec, err = s.readAppendedMeasures(src) + if err != nil { + return nil, err + } + records = append(records, rec) + + case CodeAppendedMeasureWithOverflow: + var rec AppendedMeasureWithOverflow + rec, err = s.readAppendedMeasureWithOverflow(src) + if err != nil { + return nil, err + } + records = append(records, rec) + + case CodeDeletedMeasures: + var rec DeletedMeasures + rec, err = s.readDeletedMeasures(src) + if err != nil { + return nil, err + } + records = append(records, rec) + + default: + return nil, fmt.Errorf("unknown record type code: %d", recordType) + } + } +} + +func (s *Reader) readAddedMetric(src *bytes.Buffer) (_ AddedMetric, err error) { + arr, err := bin.ReadN(src, 6) + if err != nil { + return + } + return AddedMetric{ + MetricID: bin.GetUint32(arr), + MetricType: diploma.MetricType(arr[4]), + FracDigits: int(arr[5]), + }, nil +} + +func (s *Reader) readDeletedMetric(src *bytes.Buffer) (_ DeletedMetric, err error) { + var rec DeletedMetric + rec.MetricID, err = bin.ReadUint32(src) + if err != nil { + return + } + // free data pages + dataQty, _, err := bin.ReadVarUint64(src) + if err != nil { + return + } + for range dataQty { + var pageNo uint32 + pageNo, err = bin.ReadUint32(src) + if err != nil { + return + } + rec.FreeDataPages = append(rec.FreeDataPages, pageNo) + } + // free index pages + indexQty, _, err := bin.ReadVarUint64(src) + if err != nil { + return + } + for range indexQty { + var pageNo uint32 + pageNo, err = bin.ReadUint32(src) + if err != nil { + return + } + rec.FreeIndexPages = append(rec.FreeIndexPages, pageNo) + } + return rec, nil +} + +func (s *Reader) readAppendedMeasure(src *bytes.Buffer) (_ AppendedMeasure, err error) { + arr, err := bin.ReadN(src, 16) + if err != nil { + return + } + return AppendedMeasure{ + MetricID: bin.GetUint32(arr[0:]), + Timestamp: bin.GetUint32(arr[4:]), + Value: bin.GetFloat64(arr[8:]), + }, nil +} + +func (s *Reader) readAppendedMeasures(src *bytes.Buffer) (_ AppendedMeasures, err error) { + var rec AppendedMeasures + rec.MetricID, err = bin.ReadUint32(src) + if err != nil { + return + } + qty, err := bin.ReadUint16(src) + if err != nil { + return + } + for range qty { + var measure proto.Measure + measure.Timestamp, err = bin.ReadUint32(src) + if err != nil { + return + } + measure.Value, err = bin.ReadFloat64(src) + if err != nil { + return + } + rec.Measures = append(rec.Measures, measure) + } + return rec, nil +} + +func (s *Reader) readAppendedMeasureWithOverflow(src *bytes.Buffer) (_ AppendedMeasureWithOverflow, err error) { + var ( + b byte + rec AppendedMeasureWithOverflow + ) + rec.MetricID, err = bin.ReadUint32(src) + if err != nil { + return + } + rec.Timestamp, err = bin.ReadUint32(src) + if err != nil { + return + } + rec.Value, err = bin.ReadFloat64(src) + if err != nil { + return + } + b, err = src.ReadByte() + if err != nil { + return + } + rec.IsDataPageReused = b == 1 + + rec.DataPageNo, err = bin.ReadUint32(src) + if err != nil { + return + } + b, err = src.ReadByte() + if err != nil { + return + } + if b == 1 { + rec.IsRootChanged = true + rec.RootPageNo, err = bin.ReadUint32(src) + if err != nil { + return + } + } + // index pages + indexQty, err := src.ReadByte() + if err != nil { + return + } + for range indexQty { + var pageNo uint32 + pageNo, err = bin.ReadUint32(src) + if err != nil { + return + } + rec.ReusedIndexPages = append(rec.ReusedIndexPages, pageNo) + } + return rec, nil +} + +func (s *Reader) readDeletedMeasures(src *bytes.Buffer) (_ DeletedMeasures, err error) { + var ( + rec DeletedMeasures + ) + rec.MetricID, err = bin.ReadUint32(src) + if err != nil { + return + } + // free data pages + rec.FreeDataPages, err = s.readFreePageNumbers(src) + if err != nil { + return + } + // free index pages + rec.FreeIndexPages, err = s.readFreePageNumbers(src) + if err != nil { + return + } + return rec, nil +} + +// HELPERS + +func (s *Reader) readFreePageNumbers(src *bytes.Buffer) ([]uint32, error) { + var freePages []uint32 + qty, _, err := bin.ReadVarUint64(src) + if err != nil { + return nil, err + } + for range qty { + var pageNo uint32 + pageNo, err = bin.ReadUint32(src) + if err != nil { + return nil, err + } + freePages = append(freePages, pageNo) + } + return freePages, nil +} + +func (s *Reader) Seek(offset int64) error { + ret, err := s.file.Seek(offset, 0) + if err != nil { + return err + } + + if ret != offset { + return fmt.Errorf("ret %d != offset %d", ret, offset) + } + return nil +} diff --git a/txlog/writer.go b/txlog/writer.go new file mode 100644 index 0000000..14175c7 --- /dev/null +++ b/txlog/writer.go @@ -0,0 +1,507 @@ +package txlog + +import ( + "bytes" + "errors" + "fmt" + "hash/crc32" + "os" + "path/filepath" + "sync" + + octopus "gordenko.dev/dima/diploma" + "gordenko.dev/dima/diploma/bin" + "gordenko.dev/dima/diploma/proto" +) + +const ( + lsnSize = 4 + packetPrefixSize = 12 // 4 lsn + 4 packet length + 4 crc32 + + lengthIdx = 0 + checksumIdx = 4 + lsnIdx = 8 + + filePerm = 0770 + + dumpSnapshotAfterNBytes = 1024 * 1024 * 1024 // 1 GB +) + +const ( + CodeAddedMetric byte = 1 + CodeDeletedMetric byte = 2 + CodeAppendedMeasure byte = 4 + CodeAppendedMeasures byte = 5 + CodeAppendedMeasureWithOverflow byte = 6 + CodeDeletedMeasures byte = 7 +) + +func JoinChangesFileName(dir string, logNumber int) string { + return filepath.Join(dir, fmt.Sprintf("%d.changes", logNumber)) +} + +type Changes struct { + Records []any + LogNumber int + ForceSnapshot bool + ExitWaitGroup *sync.WaitGroup + WaitCh chan struct{} +} + +type Writer struct { + mutex sync.Mutex + logNumber int + dir string + file *os.File + buf *bytes.Buffer + redoFilesToDelete []string + workerReqs []any + waitCh chan struct{} + appendToWorkerQueue func(any) + lsn uint32 + written int64 + isExited bool + exitCh chan struct{} + waitGroup *sync.WaitGroup + signalCh chan struct{} +} + +type WriterOptions struct { + Dir string + LogNumber int // номер журнала + AppendToWorkerQueue func(any) + ExitCh chan struct{} + WaitGroup *sync.WaitGroup +} + +func NewWriter(opt WriterOptions) (*Writer, error) { + if opt.Dir == "" { + return nil, errors.New("Dir option is required") + } + if opt.AppendToWorkerQueue == nil { + return nil, errors.New("AppendToWorkerQueue option is required") + } + if opt.ExitCh == nil { + return nil, errors.New("ExitCh option is required") + } + if opt.WaitGroup == nil { + return nil, errors.New("WaitGroup option is required") + } + + s := &Writer{ + dir: opt.Dir, + buf: bytes.NewBuffer(nil), + appendToWorkerQueue: opt.AppendToWorkerQueue, + logNumber: opt.LogNumber, + exitCh: opt.ExitCh, + waitGroup: opt.WaitGroup, + signalCh: make(chan struct{}, 1), + } + + var err error + + if opt.LogNumber > 0 { + s.file, err = os.OpenFile( + JoinChangesFileName(opt.Dir, s.logNumber), + os.O_APPEND|os.O_WRONLY, + filePerm, + ) + if err != nil { + return nil, err + } + } else { + s.logNumber = 1 + s.file, err = os.OpenFile( + JoinChangesFileName(opt.Dir, s.logNumber), + os.O_CREATE|os.O_WRONLY, + filePerm, + ) + if err != nil { + return nil, err + } + } + + s.reset() + return s, nil +} + +func (s *Writer) Run() { + for { + select { + case <-s.signalCh: + if err := s.flush(); err != nil { + octopus.Abort(octopus.FailedWriteToTxLog, err) + } + + case <-s.exitCh: + s.exit() + return + } + } +} + +func (s *Writer) reset() { + s.buf.Reset() + s.buf.Write([]byte{ + 0, 0, 0, 0, // packet length + 0, 0, 0, 0, // crc32 + 0, 0, 0, 0, // lsn + + }) + s.redoFilesToDelete = nil + s.workerReqs = nil + s.waitCh = make(chan struct{}) +} + +func (s *Writer) flush() error { + s.mutex.Lock() + + workerReqs := s.workerReqs + waitCh := s.waitCh + isExited := s.isExited + + var exitWaitGroup *sync.WaitGroup + if s.isExited { + exitWaitGroup = s.waitGroup + } + + if s.buf.Len() > packetPrefixSize { + redoFilesToDelete := s.redoFilesToDelete + s.lsn++ + lsn := s.lsn + packet := make([]byte, s.buf.Len()) + copy(packet, s.buf.Bytes()) + s.reset() + + s.written += int64(len(packet)) + 12 + s.mutex.Unlock() + + bin.PutUint32(packet[lengthIdx:], uint32(len(packet)-packetPrefixSize)) + bin.PutUint32(packet[lsnIdx:], lsn) + bin.PutUint32(packet[checksumIdx:], crc32.ChecksumIEEE(packet[8:])) + + n, err := s.file.Write(packet) + if err != nil { + return fmt.Errorf("TxLog write: %s", err) + } + + if n != len(packet) { + return fmt.Errorf("TxLog written %d != packet size %d", n, len(packet)) + } + + if err := s.file.Sync(); err != nil { + return fmt.Errorf("TxLog sync: %s", err) + } + + for _, fileName := range redoFilesToDelete { + err = os.Remove(fileName) + if err != nil { + octopus.Abort(octopus.RemoveREDOFileFailed, err) + } + } + } else { + s.waitCh = make(chan struct{}) + s.mutex.Unlock() + } + + var forceSnapshot bool + + if s.written > dumpSnapshotAfterNBytes { + forceSnapshot = true + } + + if isExited && s.written > 0 { + forceSnapshot = true + } + + if forceSnapshot { + if err := s.file.Close(); err != nil { + return fmt.Errorf("close changes file: %s", err) + } + s.logNumber++ + var err error + s.file, err = os.OpenFile( + JoinChangesFileName(s.dir, s.logNumber), + os.O_CREATE|os.O_WRONLY, + filePerm, + ) + if err != nil { + return fmt.Errorf("create new changes file: %s", err) + } + + s.written = 0 + } + + s.appendToWorkerQueue(Changes{ + Records: workerReqs, + ForceSnapshot: forceSnapshot, + LogNumber: s.logNumber, + WaitCh: waitCh, + ExitWaitGroup: exitWaitGroup, + }) + return nil +} + +func (s *Writer) exit() { + s.mutex.Lock() + s.isExited = true + s.mutex.Unlock() + + if err := s.flush(); err != nil { + octopus.Abort(octopus.FailedWriteToTxLog, err) + } +} + +// API + +type AddedMetric struct { + MetricID uint32 + MetricType octopus.MetricType + FracDigits int +} + +func (s *Writer) WriteAddedMetric(req AddedMetric) chan struct{} { + arr := []byte{ + CodeAddedMetric, + 0, 0, 0, 0, // + byte(req.MetricType), + byte(req.FracDigits), + } + bin.PutUint32(arr[1:], req.MetricID) + // пишу в буфер + s.mutex.Lock() + s.buf.Write(arr) + s.workerReqs = append(s.workerReqs, req) + s.mutex.Unlock() + + s.sendSignal() + return s.waitCh +} + +type DeletedMetric struct { + MetricID uint32 + FreeDataPages []uint32 + FreeIndexPages []uint32 +} + +func (s *Writer) WriteDeletedMetric(req DeletedMetric) chan struct{} { + arr := []byte{ + CodeDeletedMetric, + 0, 0, 0, 0, // metricID + } + bin.PutUint32(arr[1:], req.MetricID) + + // пишу в буфер + s.mutex.Lock() + defer s.mutex.Unlock() + + s.buf.Write(arr) + s.packFreeDataAndIndexPages(req.FreeDataPages, req.FreeIndexPages) + s.workerReqs = append(s.workerReqs, req) + + s.sendSignal() + return s.waitCh +} + +type AppendedMeasure struct { + MetricID uint32 + Timestamp uint32 + Value float64 +} + +func (s *Writer) WriteAppendMeasure(req AppendedMeasure) chan struct{} { + arr := []byte{ + CodeAppendedMeasure, + 0, 0, 0, 0, // metricID + 0, 0, 0, 0, // timestamp + 0, 0, 0, 0, 0, 0, 0, 0, // value + } + bin.PutUint32(arr[1:], req.MetricID) + bin.PutUint32(arr[5:], req.Timestamp) + bin.PutFloat64(arr[9:], req.Value) + // + s.mutex.Lock() + s.buf.Write(arr) + s.workerReqs = append(s.workerReqs, req) + s.mutex.Unlock() + + s.sendSignal() + return s.waitCh +} + +type AppendedMeasures struct { + MetricID uint32 + Measures []proto.Measure +} + +type AppendedMeasuresExtended struct { + Record AppendedMeasures + HoldLock bool +} + +func (s *Writer) WriteAppendMeasures(req AppendedMeasures, holdLock bool) chan struct{} { + arr := []byte{ + CodeAppendedMeasures, + 0, 0, 0, 0, // metricID + 0, 0, // qty + } + bin.PutUint32(arr[1:], req.MetricID) + bin.PutUint16(arr[5:], uint16(len(req.Measures))) + // + s.mutex.Lock() + s.buf.Write(arr) + for _, measure := range req.Measures { + bin.WriteUint32(s.buf, measure.Timestamp) + bin.WriteFloat64(s.buf, measure.Value) + } + s.workerReqs = append(s.workerReqs, AppendedMeasuresExtended{ + Record: req, + HoldLock: holdLock, + }) + s.mutex.Unlock() + + s.sendSignal() + return s.waitCh +} + +type AppendedMeasureWithOverflow struct { + MetricID uint32 + Timestamp uint32 + Value float64 + IsDataPageReused bool + DataPageNo uint32 + IsRootChanged bool + RootPageNo uint32 + ReusedIndexPages []uint32 +} + +type AppendedMeasureWithOverflowExtended struct { + Record AppendedMeasureWithOverflow + HoldLock bool +} + +/* +Формат: +1b code +4b metricID +4b timestamp +8b value +1b isReusedDataPage +4b dataPageNo +1b isRootChanged +[4b] newRootPageNo +1b reusedIndexPages length +[N * 4b] reusedIndexPages +*/ +func (s *Writer) WriteAppendedMeasureWithOverflow(req AppendedMeasureWithOverflow, redoFileName string, holdLock bool) chan struct{} { + size := 24 + len(req.ReusedIndexPages)*4 + if req.IsRootChanged { + size += 4 + } + + tmp := make([]byte, size) + + tmp[0] = CodeAppendedMeasureWithOverflow + bin.PutUint32(tmp[1:], req.MetricID) + bin.PutUint32(tmp[5:], req.Timestamp) + bin.PutFloat64(tmp[9:], req.Value) + if req.IsDataPageReused { + tmp[17] = 1 + } + bin.PutUint32(tmp[18:], req.DataPageNo) + + pos := 22 + if req.IsRootChanged { + tmp[pos] = 1 + bin.PutUint32(tmp[pos+1:], req.RootPageNo) + pos += 5 + } else { + tmp[pos] = 0 + pos += 1 + } + + tmp[pos] = byte(len(req.ReusedIndexPages)) + pos += 1 + for _, indexPageNo := range req.ReusedIndexPages { + bin.PutUint32(tmp[pos:], indexPageNo) + pos += 4 + } + + s.mutex.Lock() + s.buf.Write(tmp) + s.workerReqs = append(s.workerReqs, AppendedMeasureWithOverflowExtended{ + Record: req, + HoldLock: holdLock, + }) + s.redoFilesToDelete = append(s.redoFilesToDelete, redoFileName) + s.mutex.Unlock() + + s.sendSignal() + return s.waitCh +} + +type DeletedMeasures struct { + MetricID uint32 + FreeDataPages []uint32 + FreeIndexPages []uint32 +} + +/* +Формат: +1b code +4b metricID +1b freeDataPages length +[N * 4b] freeDataPages +1b freeIndexPages length +[N * 4b] freeIndexPages +*/ +func (s *Writer) WriteDeletedMeasures(op DeletedMeasures) chan struct{} { + tmp := []byte{ + CodeDeletedMeasures, + 0, 0, 0, 0, + } + bin.PutUint32(tmp[1:], op.MetricID) + + // записываю часть фиксированного размера + s.mutex.Lock() + s.buf.Write(tmp) + s.packFreeDataAndIndexPages(op.FreeDataPages, op.FreeIndexPages) + s.workerReqs = append(s.workerReqs, op) + s.mutex.Unlock() + + s.sendSignal() + return s.waitCh +} + +type DeletedMeasuresSince struct { + MetricID uint32 + LastPageNo uint32 + IsRootChanged bool + RootPageNo uint32 + FreeDataPages []uint32 + FreeIndexPages []uint32 + TimestampsBuf []byte + ValuesBuf []byte +} + +func (s *Writer) sendSignal() { + select { + case s.signalCh <- struct{}{}: + default: + } +} + +// helper + +func (s *Writer) packFreeDataAndIndexPages(freeDataPages, freeIndexPages []uint32) { + // записываю data pages + bin.WriteVarUint64(s.buf, uint64(len(freeDataPages))) + for _, dataPageNo := range freeDataPages { + bin.WriteUint32(s.buf, dataPageNo) + } + // записываю index pages + bin.WriteVarUint64(s.buf, uint64(len(freeIndexPages))) + for _, indexPageNo := range freeIndexPages { + bin.WriteUint32(s.buf, indexPageNo) + } +} diff --git a/windows_build.sh b/windows_build.sh new file mode 100755 index 0000000..d25a2f3 --- /dev/null +++ b/windows_build.sh @@ -0,0 +1,11 @@ +cd examples/database +env CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o ../../database_windows +cd - + +cd examples/loadtest +env CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o ../../loadtest_windows +cd - + +cd examples/requests +env CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o ../../requests_windows +cd - \ No newline at end of file