diff --git a/database/api.go b/database/api.go index e2f4610..1089cc6 100644 --- a/database/api.go +++ b/database/api.go @@ -84,7 +84,7 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e if err != nil { return fmt.Errorf("proto.ReadAddMetricReq: %s", err) } - fmt.Println("ReadAddMetricReq:", req) + //fmt.Println("ReadAddMetricReq:", req) reply(conn, s.AddMetric(req)) case proto.TypeDeleteMetric: @@ -534,8 +534,11 @@ func (s *Database) rangeScan(req rangeScanReq) error { LastPageNo: result.LastPageNo, Since: req.Since, }) - //s.metricRUnlock(req.MetricID) fix release unlock + s.workerInbox.Push(worker.ReleaseRLock{ + MetricID: req.MetricID, + }) if err != nil { + s.logger.Printf("ContinueRangeScan: %s\n", err) reply(req.Conn, proto.ErrUnexpected) } else { req.ResponseWriter.Close() @@ -550,8 +553,11 @@ func (s *Database) rangeScan(req rangeScanReq) error { LastPageNo: result.LastPageNo, IsDataPage: result.IsDataPage, }) - //s.metricRUnlock(req.MetricID) // fix release + s.workerInbox.Push(worker.ReleaseRLock{ + MetricID: req.MetricID, + }) if err != nil { + s.logger.Printf("RangeScan: %s\n", err) reply(req.Conn, proto.ErrUnexpected) } else { req.ResponseWriter.Close() @@ -595,8 +601,11 @@ func (s *Database) fullScan(req fullScanReq) error { ResponseWriter: req.ResponseWriter, LastPageNo: result.LastPageNo, }) - //s.worker.AddJobToQueue(req.MetricID) // FIX release rlock + s.workerInbox.Push(worker.ReleaseRLock{ + MetricID: req.MetricID, + }) if err != nil { + s.logger.Printf("ContinueFullScan: %s\n", err) reply(req.Conn, proto.ErrUnexpected) } else { req.ResponseWriter.Close() diff --git a/database/database.go b/database/database.go index 7fa66c1..0ccfa75 100644 --- a/database/database.go +++ b/database/database.go @@ -77,7 +77,7 @@ func New(opt Options) (_ *Database, err error) { return nil, fmt.Errorf("recovery.Recovery: %s", err) } - fmt.Printf("%#v\n", recoveryReport) + //fmt.Printf("%#v\n", recoveryReport) storageInbox := inbox.New() s.workerInbox = inbox.New() @@ -180,6 +180,7 @@ type ContinueFullScanReq struct { } func (s *Database) ContinueFullScan(req ContinueFullScanReq) error { + //fmt.Printf("ContinueFullScan from page %d\n", req.LastPageNo) buf, err := s.dataCache.FetchPage(req.LastPageNo) if err != nil { return fmt.Errorf("dataCache.FetchPage(%d): %s", req.LastPageNo, err) @@ -196,6 +197,7 @@ func (s *Database) ContinueFullScan(req ContinueFullScanReq) error { return err } defer treeCursor.Close() + idx := 0 for { timestamp, value, done, err := treeCursor.Prev() if err != nil { @@ -204,7 +206,9 @@ func (s *Database) ContinueFullScan(req ContinueFullScanReq) error { if done { return nil } + //fmt.Printf(" %d: %s => %.2f\n", idx, formatTime(timestamp), value) req.ResponseWriter.Feed(timestamp, value) + idx++ } } @@ -326,15 +330,15 @@ func (s *Database) findDataPage(foundPageNo uint32, timestamp uint32) (uint32, [ } } -// type PathLeg struct { -// PageNo uint32 -// Data []byte -// } +type PathLeg struct { + PageNo uint32 + Data []byte +} -// type PathToDataPage struct { -// Legs []PathLeg -// LastPageNo uint32 -// } +type PathToDataPage struct { + Legs []PathLeg + LastPageNo uint32 +} // func (s *Database) FindPathToLastPage(rootPageNo uint32) (_ PathToDataPage, err error) { // // var ( @@ -381,82 +385,215 @@ func (s *Database) findDataPage(foundPageNo uint32, timestamp uint32) (uint32, [ // s.mutex.Unlock() // } +type Level struct { + Idx int + PageNumbers []uint32 +} + +func (s *Database) GetAllPages(pageNumbers []uint32) ([]uint32, []uint32, error) { + levels := []*Level{ + { + PageNumbers: pageNumbers, + Idx: 0, + }, + } + return s.collectPages(levels) +} + +func (s *Database) collectPages(levels []*Level) (indexPageNumbers []uint32, dataPageNumbers []uint32, err error) { + var ( + buf []byte + pageNumbers []uint32 + ) + for { + if len(levels) == 0 { + return + } + var ( + lastIdx = len(levels) - 1 + level = levels[lastIdx] + pageNo = level.PageNumbers[level.Idx] + ) + if level.Idx < len(level.PageNumbers) { + buf, err = s.indexCache.FetchPage(pageNo) + if err != nil { + return nil, nil, fmt.Errorf("indexCache.FetchPage(%d): %s", pageNo, err) + } + pageNumbers = storage.ListPageNumbers(buf) + s.indexCache.ReleasePage(pageNo) + + if storage.IsZeroLevelPage(buf) { + dataPageNumbers = append(dataPageNumbers, pageNumbers...) + } else { + levels = append(levels, &Level{ + PageNumbers: pageNumbers, + }) + indexPageNumbers = append(indexPageNumbers, pageNumbers...) + } + level.Idx++ + } else { + levels = levels[:lastIdx] + } + } +} + +type DeleteSinceReport struct { + IndexPageNumbers []uint32 + DataPageNumbers []uint32 + DataPage []byte + IndexLevelTails []storage.IndexLevelTail +} + // type Level struct { -// PageNo uint32 -// PageData []byte -// Idx int -// ChildQty int +// Idx int +// PageNumbers []uint32 // } -// func (s *Atree) GetAllPages(rootPageNo uint32) (_ []uint32, err error) { -// // var ( -// // pageNumbers []uint32 -// // levels []*Level -// // ) +// fix - find by (since - 1) ? -// // buf, err := s.fetchIndexPage(rootPageNo) -// // if err != nil { -// // return nil, fmt.Errorf("fetchIndexPage(%d): %s", rootPageNo, err) -// // } -// // pageNumbers = append(pageNumbers, rootPageNo) +// [since, ...] +func (s *Database) DeleteSince(since uint32, indexLevelTail storage.IndexLevelTail) (_ DeleteSinceReport, err error) { + var ( + indexPageNumbers []uint32 + dataPageNumbers []uint32 + dataPage []byte + buf []byte + indexLevelTails []storage.IndexLevelTail + foundPageNo uint32 + ) -// // // if buf[isDataPageNumbersIdx] == 1 { -// // // pageNumbers := listPageNumbers(buf) -// // // dataPages = append(dataPages, pageNumbers...) + result := storage.DeleteSinceOnIndexTail(indexLevelTail, since) + // + indexLevelTails = append(indexLevelTails, storage.IndexLevelTail{ + Buffer: indexLevelTail.Buffer, + RecordsCount: result.RecordsCount, + }) + levels := []*Level{ + { + PageNumbers: result.PageNumbers, + Idx: 1, + }, + } + foundPageNo = result.PageNumbers[0] + for { + buf, err = s.indexCache.FetchPage(foundPageNo) + if err != nil { + err = fmt.Errorf("indexCache.FetchPage(%d): %s", foundPageNo, err) + return + } + toReleaseIndexPageNo := foundPageNo + result := storage.DeleteSinceOnIndexPage(buf, since) + s.indexCache.ReleasePage(toReleaseIndexPageNo) + // + foundPageNo = result.PageNumbers[0] -// // // s.releasePage(rootPageNo) + levels = append(levels, &Level{ + PageNumbers: result.PageNumbers, + Idx: 1, // 1st - found pageNo + }) -// // // return PageLists{ -// // // DataPages: dataPages, -// // // IndexPages: indexPages, -// // // }, nil -// // // } + indexLevelTails = append(indexLevelTails, storage.IndexLevelTail{ + Buffer: result.Buffer, + RecordsCount: result.RecordsCount, + }) -// // childQty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:]) + if storage.IsZeroLevelPage(buf) { + dataPage, err = s.dataCache.FetchPage(foundPageNo) + if err != nil { + err = fmt.Errorf("dataCache.FetchPage(%d): %s", foundPageNo, err) + return + } + dataPageNumbers = append(dataPageNumbers, result.PageNumbers...) + break + } else { + indexPageNumbers = append(indexPageNumbers, result.PageNumbers...) + } + } + indexPageNumbers, dataPageNumbers, err = s.collectPages(levels) + if err != nil { + return + } + return DeleteSinceReport{ + IndexPageNumbers: indexPageNumbers, + DataPageNumbers: dataPageNumbers, + DataPage: dataPage, + IndexLevelTails: indexLevelTails, + }, nil +} -// // levels = append(levels, &Level{ -// // PageNo: rootPageNo, -// // PageData: buf, -// // Idx: 0, -// // ChildQty: int(childQty), -// // }) +// [..., until] +func (s *Database) DeleteUntil(since uint32, indexLevelTail storage.IndexLevelTail) (_ DeleteSinceReport, err error) { + var ( + indexPageNumbers []uint32 + dataPageNumbers []uint32 + dataPage []byte + buf []byte + indexLevelTails []storage.IndexLevelTail + foundPageNo uint32 + ) -// // for { -// // if len(levels) == 0 { -// // return pageNumbers, nil -// // } + result := storage.DeleteSinceOnIndexTail(indexLevelTail, since) + // + indexLevelTails = append(indexLevelTails, storage.IndexLevelTail{ + Buffer: indexLevelTail.Buffer, + RecordsCount: result.RecordsCount, + }) + levels := []*Level{ + { + PageNumbers: result.PageNumbers, + Idx: 1, + }, + } + foundPageNo = result.PageNumbers[0] + for { + buf, err = s.indexCache.FetchPage(foundPageNo) + if err != nil { + err = fmt.Errorf("indexCache.FetchPage(%d): %s", foundPageNo, err) + return + } + toReleaseIndexPageNo := foundPageNo + result := storage.DeleteSinceOnIndexPage(buf, since) + s.indexCache.ReleasePage(toReleaseIndexPageNo) + // + foundPageNo = result.PageNumbers[0] -// // lastIdx := len(levels) - 1 -// // level := levels[lastIdx] + levels = append(levels, &Level{ + PageNumbers: result.PageNumbers, + Idx: 1, // 1st - found pageNo + }) -// // if level.Idx < level.ChildQty { -// // pageNo := getPageNo(level.PageData, level.Idx) -// // level.Idx++ + indexLevelTails = append(indexLevelTails, storage.IndexLevelTail{ + Buffer: result.Buffer, + RecordsCount: result.RecordsCount, + }) -// // var buf []byte -// // buf, err = s.fetchPage(pageNo) -// // if err != nil { -// // return nil, fmt.Errorf("fetchPage(%d): %s", pageNo, err) -// // } -// // pageNumbers = append(pageNumbers, pageNo) + if storage.IsZeroLevelPage(buf) { + dataPage, err = s.dataCache.FetchPage(foundPageNo) + if err != nil { + err = fmt.Errorf("dataCache.FetchPage(%d): %s", foundPageNo, err) + return + } + dataPageNumbers = append(dataPageNumbers, result.PageNumbers...) + break + } else { + indexPageNumbers = append(indexPageNumbers, result.PageNumbers...) + } + } + indexPageNumbers, dataPageNumbers, err = s.collectPages(levels) + if err != nil { + return + } + return DeleteSinceReport{ + IndexPageNumbers: indexPageNumbers, + DataPageNumbers: dataPageNumbers, + DataPage: dataPage, + IndexLevelTails: indexLevelTails, + }, nil +} -// // if buf[pageTypeIdx] == PageTypeData { -// // //pageNumbers := listPageNumbers(buf) -// // //dataPages = append(dataPages, pageNumbers...) -// // s.releasePage(pageNo) -// // } else { -// // childQty, _ = bin.GetUint16(buf[indexRecordsQtyIdx:]) -// // levels = append(levels, &Level{ -// // PageNo: pageNo, -// // PageData: buf, -// // Idx: 0, -// // ChildQty: int(childQty), -// // }) -// // } -// // } else { -// // s.releasePage(level.PageNo) -// // levels = levels[:lastIdx] -// // } -// // } -// return -// } +const datetimeLayout = "2006-01-02 15:04:05" + +func formatTime(timestamp uint32) string { + tm := time.Unix(int64(timestamp), 0) + return tm.Format(datetimeLayout) +} diff --git a/enc/enc_test.go b/enc/enc_test.go index 052142d..835ab19 100644 --- a/enc/enc_test.go +++ b/enc/enc_test.go @@ -6,9 +6,11 @@ import ( "math" "slices" "testing" + "time" bin "gordenko.dev/dima/bin/little" "gordenko.dev/dima/qb" + "gordenko.dev/dima/textutil" ) func equalFloatSlices(a, b []float64, epsilon float64) bool { @@ -21,23 +23,23 @@ func equalFloatSlices(a, b []float64, epsilon float64) bool { var ( cumulativeTestCases = []struct { - Nums []float64 - Name string - Offset int - ChangeSize int - Buf []byte - BaseValue float64 - LastDelta uint64 + Nums []float64 + Name string + RewindOffset int + ChangeSize int + Buf []byte + BaseValue float64 + LastDelta uint64 }{ { Nums: []float64{ 0, }, - Name: "add 1st value zero", - Offset: 0, - ChangeSize: 3, + Name: "add 1st value zero", + RewindOffset: 0, + ChangeSize: 3, Buf: []byte{ - 0x8f, // base value + 0x80, // base value 0x80, // delta 0 0x80, // literal (len = 1) 0x00, 0x00, 0x00, 0x00, 0x00, @@ -49,9 +51,9 @@ var ( Nums: []float64{ 1.5, }, - Name: "add 1st value non zero", - Offset: 0, - ChangeSize: 3, + Name: "add 1st value non zero", + RewindOffset: 0, + ChangeSize: 3, Buf: []byte{ 0x8f, // base value 0x80, // delta 0 @@ -66,9 +68,9 @@ var ( 1.5, 1.5, }, - Name: "literal switch to run", - Offset: 1, - ChangeSize: 1, + Name: "literal switch to run", + RewindOffset: 1, + ChangeSize: 1, Buf: []byte{ 0x8f, // base value 0x80, // delta 0 @@ -84,9 +86,9 @@ var ( 1.6, 1.6, }, - Name: "literal decrease by 1 and switch to run", - Offset: 2, - ChangeSize: 3, + Name: "literal decrease by 1 and switch to run", + RewindOffset: 2, + ChangeSize: 3, Buf: []byte{ 0x8f, // base value 0x80, // delta 0 @@ -104,9 +106,9 @@ var ( 1.5, 1.5, }, - Name: "increment run", - Offset: 1, - ChangeSize: 1, + Name: "increment run", + RewindOffset: 1, + ChangeSize: 1, Buf: []byte{ 0x8f, // base value 0x80, // delta 0 @@ -123,9 +125,9 @@ var ( 1.5, 1.6, }, - Name: "run switch to literal", - Offset: 0, - ChangeSize: 2, + Name: "run switch to literal", + RewindOffset: 0, + ChangeSize: 2, Buf: []byte{ 0x8f, // base value 0x80, // delta 0 @@ -138,10 +140,10 @@ var ( LastDelta: 1, }, { - Nums: repeatFloat64(1.5, 129), - Name: "increment run to full fill h-byte", - Offset: 1, - ChangeSize: 1, + Nums: repeatFloat64(1.5, 129), + Name: "increment run to full fill h-byte", + RewindOffset: 1, + ChangeSize: 1, Buf: []byte{ 0x8f, // base value 0x80, // delta 0 @@ -152,10 +154,10 @@ var ( LastDelta: 0, }, { - Nums: repeatFloat64(1.5, 130), - Name: "run switch to literal after h-byte overflow", - Offset: 0, - ChangeSize: 2, + Nums: repeatFloat64(1.5, 130), + Name: "run switch to literal after h-byte overflow", + RewindOffset: 0, + ChangeSize: 2, Buf: []byte{ 0x8f, // base value 0x80, // delta 0 @@ -173,9 +175,9 @@ var ( 1.6, 1.7, }, - Name: "increment literal", - Offset: 1, - ChangeSize: 2, + Name: "increment literal", + RewindOffset: 1, + ChangeSize: 2, Buf: []byte{ 0x8f, // base value 0x80, // delta 0 @@ -203,11 +205,11 @@ func TestCumulativeDeltaCompressor(t *testing.T) { c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize) for _, num := range testCase.Nums { report = c.Evaluate(tmp, num) - c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta) + c.Append(report.RewindOffset, tmp[:report.ChangeSize], num, report.Delta) } - if report.Offset != testCase.Offset { + if report.RewindOffset != testCase.RewindOffset { t.Fatalf("%s: got offset %d are not equal expected %d", - testCase.Name, report.Offset, testCase.Offset) + testCase.Name, report.RewindOffset, testCase.RewindOffset) } if report.ChangeSize != testCase.ChangeSize { t.Fatalf("%s: got changeSize %d are not equal expected %d", @@ -241,7 +243,7 @@ func TestRestoreCumulativeDeltaCompressor(t *testing.T) { c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize) for _, num := range testCase.Nums { report = c.Evaluate(tmp, num) - c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta) + c.Append(report.RewindOffset, tmp[:report.ChangeSize], num, report.Delta) } restored := NewValueDeltaCompressor(metricType, fracDigits, buf, c.Size()) @@ -270,7 +272,7 @@ func TestCumulativeDeltaDecompressorFromState(t *testing.T) { c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize) for _, num := range testCase.Nums { report := c.Evaluate(tmp, num) - c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta) + c.Append(report.RewindOffset, tmp[:report.ChangeSize], num, report.Delta) } d := c.CreateDecompressor(metricType, fracDigits) @@ -304,7 +306,7 @@ func TestCumulativeDeltaDecompressorFromEnd(t *testing.T) { c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize) for _, num := range testCase.Nums { report := c.Evaluate(tmp, num) - c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta) + c.Append(report.RewindOffset, tmp[:report.ChangeSize], num, report.Delta) } d := NewValueDeltaDecompressor(metricType, fracDigits) @@ -330,21 +332,21 @@ func TestCumulativeDeltaDecompressorFromEnd(t *testing.T) { var ( instantTestCases = []struct { - Nums []float64 - Name string - Offset int - ChangeSize int - Buf []byte - BaseValue float64 - LastDelta uint64 + Nums []float64 + Name string + RewindOffset int + ChangeSize int + Buf []byte + BaseValue float64 + LastDelta uint64 }{ { Nums: []float64{ 1.5, }, - Name: "add 1st value", - Offset: 0, - ChangeSize: 3, + Name: "add 1st value", + RewindOffset: 0, + ChangeSize: 3, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 @@ -358,9 +360,9 @@ var ( Nums: []float64{ -1.5, }, - Name: "add 1st value (negative)", - Offset: 0, - ChangeSize: 3, + Name: "add 1st value (negative)", + RewindOffset: 0, + ChangeSize: 3, Buf: []byte{ 0x9d, // base value 0x80, // delta 0 @@ -375,9 +377,9 @@ var ( 1.5, 1.5, }, - Name: "literal switch to run", - Offset: 1, - ChangeSize: 1, + Name: "literal switch to run", + RewindOffset: 1, + ChangeSize: 1, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 @@ -392,9 +394,9 @@ var ( -1.5, -1.5, }, - Name: "literal switch to run (negative)", - Offset: 1, - ChangeSize: 1, + Name: "literal switch to run (negative)", + RewindOffset: 1, + ChangeSize: 1, Buf: []byte{ 0x9d, // base value 0x80, // delta 0 @@ -410,9 +412,9 @@ var ( 1.6, 1.6, }, - Name: "literal decrease by 1 and switch to run", - Offset: 2, - ChangeSize: 3, + Name: "literal decrease by 1 and switch to run", + RewindOffset: 2, + ChangeSize: 3, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 @@ -430,9 +432,9 @@ var ( -1.6, -1.6, }, - Name: "literal decrease by 1 and switch to run (negative)", - Offset: 2, - ChangeSize: 3, + Name: "literal decrease by 1 and switch to run (negative)", + RewindOffset: 2, + ChangeSize: 3, Buf: []byte{ 0x9d, // base value 0x80, // delta 0 @@ -450,9 +452,9 @@ var ( 1.4, 1.4, }, - Name: "literal decrease by 1 and switch to run (negative delta)", - Offset: 2, - ChangeSize: 3, + Name: "literal decrease by 1 and switch to run (negative delta)", + RewindOffset: 2, + ChangeSize: 3, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 @@ -470,9 +472,9 @@ var ( -1.4, -1.4, }, - Name: "literal decrease by 1 and switch to run (positive delta, negative)", - Offset: 2, - ChangeSize: 3, + Name: "literal decrease by 1 and switch to run (positive delta, negative)", + RewindOffset: 2, + ChangeSize: 3, Buf: []byte{ 0x9d, // base value 0x80, // delta 0 @@ -490,9 +492,9 @@ var ( 1.5, 1.5, }, - Name: "increment run", - Offset: 1, - ChangeSize: 1, + Name: "increment run", + RewindOffset: 1, + ChangeSize: 1, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 @@ -508,9 +510,9 @@ var ( -1.5, -1.5, }, - Name: "increment run (negative)", - Offset: 1, - ChangeSize: 1, + Name: "increment run (negative)", + RewindOffset: 1, + ChangeSize: 1, Buf: []byte{ 0x9d, // base value 0x80, // delta 0 @@ -527,9 +529,9 @@ var ( 1.5, 1.6, }, - Name: "run switch to literal", - Offset: 0, - ChangeSize: 2, + Name: "run switch to literal", + RewindOffset: 0, + ChangeSize: 2, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 @@ -548,9 +550,9 @@ var ( -1.5, -1.6, }, - Name: "run switch to literal (negative)", - Offset: 0, - ChangeSize: 2, + Name: "run switch to literal (negative)", + RewindOffset: 0, + ChangeSize: 2, Buf: []byte{ 0x9d, // base value 0x80, // delta 0 @@ -569,9 +571,9 @@ var ( 1.5, 1.4, }, - Name: "run switch to literal (negative delta)", - Offset: 0, - ChangeSize: 2, + Name: "run switch to literal (negative delta)", + RewindOffset: 0, + ChangeSize: 2, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 @@ -590,9 +592,9 @@ var ( -1.5, -1.4, }, - Name: "run switch to literal (positive delta, negative version)", - Offset: 0, - ChangeSize: 2, + Name: "run switch to literal (positive delta, negative version)", + RewindOffset: 0, + ChangeSize: 2, Buf: []byte{ 0x9d, // base value 0x80, // delta 0 @@ -605,10 +607,10 @@ var ( LastDelta: bin.EncodeZigZag(1), }, { - Nums: repeatFloat64(1.5, 129), - Name: "increment run to full fill h-byte", - Offset: 1, - ChangeSize: 1, + Nums: repeatFloat64(1.5, 129), + Name: "increment run to full fill h-byte", + RewindOffset: 1, + ChangeSize: 1, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 @@ -619,10 +621,10 @@ var ( LastDelta: 0, }, { - Nums: repeatFloat64(-1.5, 129), - Name: "increment run to full fill h-byte (negative version)", - Offset: 1, - ChangeSize: 1, + Nums: repeatFloat64(-1.5, 129), + Name: "increment run to full fill h-byte (negative version)", + RewindOffset: 1, + ChangeSize: 1, Buf: []byte{ 0x9d, // base value 0x80, // delta 0 @@ -633,10 +635,10 @@ var ( LastDelta: 0, }, { - Nums: repeatFloat64(1.5, 130), - Name: "run switch to literal after h-byte overflow", - Offset: 0, - ChangeSize: 2, + Nums: repeatFloat64(1.5, 130), + Name: "run switch to literal after h-byte overflow", + RewindOffset: 0, + ChangeSize: 2, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 @@ -649,10 +651,10 @@ var ( LastDelta: 0, }, { - Nums: repeatFloat64(-1.5, 130), - Name: "run switch to literal after h-byte overflow (negative version)", - Offset: 0, - ChangeSize: 2, + Nums: repeatFloat64(-1.5, 130), + Name: "run switch to literal after h-byte overflow (negative version)", + RewindOffset: 0, + ChangeSize: 2, Buf: []byte{ 0x9d, // base value 0x80, // delta 0 @@ -670,9 +672,9 @@ var ( 1.6, 1.7, }, - Name: "increment literal", - Offset: 1, - ChangeSize: 2, + Name: "increment literal", + RewindOffset: 1, + ChangeSize: 2, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 @@ -690,9 +692,9 @@ var ( -1.6, -1.7, }, - Name: "increment literal (negative version)", - Offset: 1, - ChangeSize: 2, + Name: "increment literal (negative version)", + RewindOffset: 1, + ChangeSize: 2, Buf: []byte{ 0x9d, // base value 0x80, // delta 0 @@ -710,9 +712,9 @@ var ( 1.4, 1.3, }, - Name: "increment literal (negative delta)", - Offset: 1, - ChangeSize: 2, + Name: "increment literal (negative delta)", + RewindOffset: 1, + ChangeSize: 2, Buf: []byte{ 0x9e, // base value 0x80, // delta 0 @@ -730,9 +732,9 @@ var ( -1.4, -1.3, }, - Name: "increment literal (positive delta, negative version)", - Offset: 1, - ChangeSize: 2, + Name: "increment literal (positive delta, negative version)", + RewindOffset: 1, + ChangeSize: 2, Buf: []byte{ 0x9d, // base value 0x80, // delta 0 @@ -760,11 +762,11 @@ func TestInstantDeltaCompressor(t *testing.T) { c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize) for _, num := range testCase.Nums { report = c.Evaluate(tmp, num) - c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta) + c.Append(report.RewindOffset, tmp[:report.ChangeSize], num, report.Delta) } - if report.Offset != testCase.Offset { - t.Fatalf("%s: got offset %d are not equal expected %d", - testCase.Name, report.Offset, testCase.Offset) + if report.RewindOffset != testCase.RewindOffset { + t.Fatalf("%s: got rewindOffset %d are not equal expected %d", + testCase.Name, report.RewindOffset, testCase.RewindOffset) } if report.ChangeSize != testCase.ChangeSize { t.Fatalf("%s: got changeSize %d are not equal expected %d", @@ -798,7 +800,7 @@ func TestRestoreInstantDeltaCompressor(t *testing.T) { c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize) for _, num := range testCase.Nums { report = c.Evaluate(tmp, num) - c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta) + c.Append(report.RewindOffset, tmp[:report.ChangeSize], num, report.Delta) } restored := NewValueDeltaCompressor(metricType, fracDigits, buf, c.Size()) if restored.baseValue != testCase.BaseValue { @@ -825,7 +827,7 @@ func TestInstantDeltaDecompressorFromState(t *testing.T) { c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize) for _, num := range testCase.Nums { report := c.Evaluate(tmp, num) - c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta) + c.Append(report.RewindOffset, tmp[:report.ChangeSize], num, report.Delta) } d := c.CreateDecompressor(metricType, fracDigits) @@ -859,7 +861,7 @@ func TestInstantDeltaDecompressorFromEnd(t *testing.T) { c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize) for _, num := range testCase.Nums { report := c.Evaluate(tmp, num) - c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta) + c.Append(report.RewindOffset, tmp[:report.ChangeSize], num, report.Delta) } d := NewValueDeltaDecompressor(metricType, fracDigits) @@ -890,16 +892,16 @@ var ( Buf []byte LastUnixtime uint32 LastDelta uint32 - Offset int + RewindOffset int ChangeSize int }{ { Nums: []uint32{ 1780777000, }, - Name: "add 1st value", - Offset: 0, - ChangeSize: 4, + Name: "add 1st value", + RewindOffset: 0, + ChangeSize: 4, Buf: []byte{ 0x00, 0x00, 0x00, 0x00, 0x28, 0x80, 0x24, 0x6a, // since @@ -912,9 +914,9 @@ var ( 1780777000, 1780777060, // +60 }, - Name: "add 1st delta", - Offset: 0, - ChangeSize: 2, + Name: "add 1st delta", + RewindOffset: 0, + ChangeSize: 2, Buf: []byte{ 0x00, 0x00, 0x80, // h-byte (literal, len=1) @@ -930,9 +932,9 @@ var ( 1780777060, // +60 1780777120, // +60 }, - Name: "literal changed to run", - Offset: 1, - ChangeSize: 1, + Name: "literal changed to run", + RewindOffset: 1, + ChangeSize: 1, Buf: []byte{ 0x00, 0x00, 0x00, // h-byte (run, len=2) @@ -949,9 +951,9 @@ var ( 1780777130, // +70 1780777200, // +70 }, - Name: "literal decrease by 1 and switch to run", - Offset: 2, - ChangeSize: 3, + Name: "literal decrease by 1 and switch to run", + RewindOffset: 2, + ChangeSize: 3, Buf: []byte{ 0x00, // h-byte (run, len=2) 0xc6, // delta 70 @@ -969,9 +971,9 @@ var ( 1780777120, // +60 1780777180, // +60 }, - Name: "increment run", - Offset: 1, - ChangeSize: 1, + Name: "increment run", + RewindOffset: 1, + ChangeSize: 1, Buf: []byte{ 0x00, 0x00, 0x01, // h-byte (run, len=2) @@ -989,9 +991,9 @@ var ( 1780777180, // +60 1780777200, // +20 }, - Name: "switch run to literal", - Offset: 0, - ChangeSize: 2, + Name: "switch run to literal", + RewindOffset: 0, + ChangeSize: 2, Buf: []byte{ 0x80, // h-byte (literal, len=1) 0x94, // delta 20 @@ -1003,10 +1005,10 @@ var ( LastDelta: 20, }, { - Nums: generateProgression(1780777000, 60, 130), - Name: "increment run up to full filled h-byte", - Offset: 1, - ChangeSize: 1, + Nums: generateProgression(1780777000, 60, 130), + Name: "increment run up to full filled h-byte", + RewindOffset: 1, + ChangeSize: 1, Buf: []byte{ 0x00, 0x00, 0x7f, // h-byte (run, len=129) @@ -1017,10 +1019,10 @@ var ( LastDelta: 60, }, { - Nums: generateProgression(1780777000, 60, 131), - Name: "run switch to literal after h-byte overflow", - Offset: 0, - ChangeSize: 2, + Nums: generateProgression(1780777000, 60, 131), + Name: "run switch to literal after h-byte overflow", + RewindOffset: 0, + ChangeSize: 2, Buf: []byte{ 0x80, // h-byte (literal, len=1) 0xbc, // delta 60 @@ -1037,9 +1039,9 @@ var ( 1780777060, // +60 1780777130, // +70 }, - Name: "increment literal", - Offset: 1, - ChangeSize: 2, + Name: "increment literal", + RewindOffset: 1, + ChangeSize: 2, Buf: []byte{ 0x00, 0x81, // h-byte (literal, len=2) @@ -1064,11 +1066,11 @@ func TestTimeDeltaCompressor(t *testing.T) { c := NewTimeDeltaCompressor(buf, payloadSize) for _, num := range testCase.Nums { report = c.Evaluate(tmp, num) - c.Append(report.Offset, tmp[:report.ChangeSize], num) + c.Append(report.RewindOffset, tmp[:report.ChangeSize], num) } - if report.Offset != testCase.Offset { - t.Fatalf("%s: got offset %d are not equal expected %d", - testCase.Name, report.Offset, testCase.Offset) + if report.RewindOffset != testCase.RewindOffset { + t.Fatalf("%s: got rewindOffset %d are not equal expected %d", + testCase.Name, report.RewindOffset, testCase.RewindOffset) } if report.ChangeSize != testCase.ChangeSize { t.Fatalf("%s: got changeSize %d are not equal expected %d", @@ -1100,7 +1102,7 @@ func TestRestoreTimeDeltaCompressor(t *testing.T) { c := NewTimeDeltaCompressor(buf, payloadSize) for _, num := range testCase.Nums { report = c.Evaluate(tmp, num) - c.Append(report.Offset, tmp[:report.ChangeSize], num) + c.Append(report.RewindOffset, tmp[:report.ChangeSize], num) } restored := NewTimeDeltaCompressor(buf, c.Size()) if restored.lastUnixtime != testCase.LastUnixtime { @@ -1118,16 +1120,17 @@ func TestTimeDeltaDecompressorFromState(t *testing.T) { for _, testCase := range timeTestCases { var ( tmp = make([]byte, tmpTimeSize) - buf = make([]byte, 8) + buf = make([]byte, 16) payloadSize = 0 decodedNums []uint32 ) c := NewTimeDeltaCompressor(buf, payloadSize) for _, num := range testCase.Nums { report := c.Evaluate(tmp, num) - c.Append(report.Offset, tmp[:report.ChangeSize], num) + c.Append(report.RewindOffset, tmp[:report.ChangeSize], num) } + fmt.Println("-----") d := c.CreateDecompressor() for { num, done := d.NextValue() @@ -1157,7 +1160,7 @@ func TestTimeDeltaDecompressorFromEnd(t *testing.T) { c := NewTimeDeltaCompressor(buf, payloadSize) for _, num := range testCase.Nums { report := c.Evaluate(tmp, num) - c.Append(report.Offset, tmp[:report.ChangeSize], num) + c.Append(report.RewindOffset, tmp[:report.ChangeSize], num) } c.ReplaceSinceWithUntil() @@ -1206,3 +1209,91 @@ func generateProgression(start uint32, delta uint32, n int) []uint32 { } return nums } + +func generateTimestamps(days int) (timestamps []uint32) { + var ( + minutes = []int{14, 29, 44, 59} + hoursPerDay = 24 + totalHours = days * hoursPerDay + since = time.Now().AddDate(0, 0, -days) + ) + + for i := range totalHours { + hourTime := since.Add(time.Duration(i) * time.Hour) + for _, m := range minutes { + measureTime := time.Date( + hourTime.Year(), + hourTime.Month(), + hourTime.Day(), + hourTime.Hour(), + m, // minutes + 0, // seconds + 0, // nanoseconds + time.Local, + ) + timestamps = append(timestamps, uint32(measureTime.Unix())) + } + } + return +} + +func TestXXX(t *testing.T) { + + var ( + timestamps = generateTimestamps(20)[:1231] + tmp = make([]byte, tmpTimeSize) + buf = make([]byte, 64) + payloadSize = 0 + decodedNums []uint32 + ) + c := NewTimeDeltaCompressor(buf, payloadSize) + for _, num := range timestamps { + report := c.Evaluate(tmp, num) + c.Append(report.RewindOffset, tmp[:report.ChangeSize], num) + } + + d := c.CreateDecompressor() + for { + num, done := d.NextValue() + if done { + break + } + decodedNums = append(decodedNums, num) + } + + slices.Reverse(decodedNums) + + if !slices.Equal(timestamps, decodedNums) { + t.Fatalf("got nums %v not equal expected %v", + decodedNums, timestamps) + } +} + +func TestYYY(t *testing.T) { + buf, err := textutil.ParseHex("50 04 87 ac 6b 05 6a") + if err != nil { + t.Fatal(err) + } + + //var decodedNums []uint32 + var prev uint32 + + d := NewTimeDeltaDecompressor() + d.RestoreFromEnd(buf) + + for { + num, done := d.NextValue() + if done { + break + } + fmt.Println(num, prev-num) + prev = num + //decodedNums = append(decodedNums, num) + } + + // //slices.Reverse(decodedNums) + + // for _, num := range decodedNums { + + // } +} diff --git a/enc/time_delta.go b/enc/time_delta.go index 7a00268..b2cacec 100644 --- a/enc/time_delta.go +++ b/enc/time_delta.go @@ -1,7 +1,6 @@ package enc import ( - "fmt" "io" "log" @@ -92,11 +91,11 @@ func (s *TimeDeltaCompressor) restore(payloadSize int) { func (s *TimeDeltaCompressor) Evaluate(tmp []byte, timestamp uint32) qb.TimeEvaluationReport { var ( - delta = timestamp - s.lastUnixtime rewindOffset int i int ) if s.pos < len(s.buf) { + delta := timestamp - s.lastUnixtime if s.lastDelta > 0 { h := s.buf[s.pos] if h < 128 { @@ -108,36 +107,36 @@ func (s *TimeDeltaCompressor) Evaluate(tmp []byte, timestamp uint32) qb.TimeEval rewindOffset = 1 // перезапис h } else { // endSeries - n, _ := bin.PutVarUint64(tmp, uint64(delta)) - i += n tmp[i] = 128 // start new literal (length=1) i++ + n, _ := bin.PutVarUint64(tmp[i:], uint64(delta)) + i += n } } else { // literal if delta != s.lastDelta { if h < 255 { // incrementLiteral - n, _ := bin.PutVarUint64(tmp, uint64(delta)) - i += n tmp[i] = h + 1 i++ + n, _ := bin.PutVarUint64(tmp[i:], uint64(delta)) + i += n rewindOffset = 1 // перезапис h } else { // endSeries - n, _ := bin.PutVarUint64(tmp, uint64(delta)) - i += n tmp[i] = 128 // start new literal (length=1) i++ + n, _ := bin.PutVarUint64(tmp[i:], uint64(delta)) + i += n } } else { // startRun if h > 128 { - tmp[i] = h - 1 // зменшую довжину попередньої серії на 1 + tmp[i] = 0 // start new run (length=2) i++ n, _ := bin.PutVarUint64(tmp[i:], uint64(delta)) i += n - tmp[i] = 0 // start new run (length=2) + tmp[i] = h - 1 // зменшую довжину попередньої серії на 1 i++ rewindOffset = 1 + n // перезапис пари delta/h } else { @@ -148,11 +147,10 @@ func (s *TimeDeltaCompressor) Evaluate(tmp []byte, timestamp uint32) qb.TimeEval } } } else { - // add 1st delta - n, _ := bin.PutVarUint64(tmp, uint64(delta)) - i += n tmp[i] = 128 // start new literal (length=1) i++ + n, _ := bin.PutVarUint64(tmp[i:], uint64(delta)) + i += n } } else { bin.PutUint32(tmp, timestamp) // 1st timestamp (since) @@ -167,16 +165,16 @@ func (s *TimeDeltaCompressor) Evaluate(tmp []byte, timestamp uint32) qb.TimeEval } func (s *TimeDeltaCompressor) Append(rewindOffset int, change []byte, timestamp uint32) { - if s.pos < len(s.buf) { - i := s.pos + rewindOffset - 1 // -1, because s.pos always points to h byte - for _, b := range change { - s.buf[i] = b - i-- - } + // fmt.Printf("----\nchange: % x\n", change) + // fmt.Printf("rewindOffset: %d\n", rewindOffset) + // fmt.Printf("pos: %d\n", s.pos) + // fmt.Printf("buf: %d\n", len(s.buf)) + + if s.lastUnixtime > 0 { s.lastDelta = timestamp - s.lastUnixtime - } else { - copy(s.buf[len(s.buf)-4:], change) // 4b since } + idx := s.pos - len(change) + rewindOffset + copy(s.buf[idx:], change) s.pos -= len(change) - rewindOffset s.lastUnixtime = timestamp } @@ -199,6 +197,10 @@ func (s *TimeDeltaCompressor) getState() qb.TimeDeltaCapturedState { bound = s.pos // only since encoded } state.Payload = s.buf[bound:] + // fmt.Printf("% x\n", s.buf) + // fmt.Printf("% x\n", s.buf[bound:]) + // fmt.Printf("h: % x\n", state.H) + // fmt.Printf("lastDelta: %d\n", s.lastDelta) } return state } @@ -213,7 +215,7 @@ func (s *TimeDeltaCompressor) ForgetCapturedState() { } func (s *TimeDeltaCompressor) Tail(offset int) []byte { - fmt.Println("time tail:", s.pos, len(s.buf), offset) + //fmt.Println("time tail:", s.pos, len(s.buf), offset) return s.buf[s.pos : len(s.buf)-offset] } diff --git a/enc/value_delta.go b/enc/value_delta.go index ba968fd..7d11aef 100644 --- a/enc/value_delta.go +++ b/enc/value_delta.go @@ -157,6 +157,7 @@ func (s *ValueDeltaCompressor) Append(rewindOffset int, change []byte, value flo //fmt.Printf("buf before: % x\n", s.buf[:s.pos]) copy(s.buf[s.pos-rewindOffset:], change) s.pos += len(change) - rewindOffset + //fmt.Printf("v size %d\n", s.pos) //fmt.Printf("buf after: % x\n", s.buf[:s.pos]) //fmt.Printf("buf after: % x\n", s.buf[:s.pos]) } @@ -342,6 +343,7 @@ func (s *ValueDeltaDecompressor) readBaseValue() { } func (s *ValueDeltaDecompressor) NextValue() (value float64, done bool) { + //fmt.Printf("v pos: %d of %d\n", s.pos, len(s.buf)) if s.done { return 0, true } diff --git a/examples/play/main.go b/examples/play/main.go index 8681516..aad82e1 100644 --- a/examples/play/main.go +++ b/examples/play/main.go @@ -302,28 +302,31 @@ func sendRequests(conn *client.Connection) { // fmt.Printf("\nCumulative metric %d added\n", cumulativeMetricID) // } - // // GET CUMULATIVE METRIC + // GET CUMULATIVE METRIC - // cMetric, err := conn.GetMetric(cumulativeMetricID) - // if err != nil { - // log.Fatalf("conn.GetMetric: %s\n", err) - // } else { - // fmt.Printf(` - // GetMetric: - // metricID: %d - // metricType: %s - // fracDigits: %d - // `, - // cMetric.MetricID, metricTypeToName[cMetric.MetricType], cMetric.FracDigits) + cMetric, err := conn.GetMetric(cumulativeMetricID) + if err != nil { + log.Fatalf("conn.GetMetric: %s\n", err) + } else { + fmt.Printf(` + GetMetric: + metricID: %d + metricType: %s + fracDigits: %d + `, + cMetric.MetricID, metricTypeToName[cMetric.MetricType], cMetric.FracDigits) + } + + // APPEND MEASURES + + // cumulativeMeasures := GenerateCumulativeMeasures(200) + // if len(cumulativeMeasures) > 65535 { + // cumulativeMeasures = cumulativeMeasures[:65535] // } - // // APPEND MEASURES - - // cumulativeMeasures := GenerateCumulativeMeasures(62) - // result, err := conn.AppendMeasures(proto.AppendMeasuresReq{ // MetricID: cumulativeMetricID, - // Measures: cumulativeMeasures[:1000], + // Measures: cumulativeMeasures, // }) // if err != nil { // log.Fatalf("conn.AppendMeasures: %s\n", err) @@ -344,7 +347,6 @@ func sendRequests(conn *client.Connection) { // } // LIST CUMULATIVE MEASURES - var cumulativeList []proto.CumulativeMeasure // lastTimestamp = cumulativeMeasures[len(cumulativeMeasures)-1].Timestamp // until = time.Unix(int64(lastTimestamp), 0) @@ -368,14 +370,15 @@ func sendRequests(conn *client.Connection) { // LIST ALL CUMULATIVE MEASURES + var cumulativeList []proto.CumulativeMeasure cumulativeList, err = conn.ListAllCumulativeMeasures(cumulativeMetricID) if err != nil { log.Fatalf("conn.ListAllCumulativeMeasures: %s\n", err) } else { fmt.Printf("\nListAllCumulativeMeasures (last 15 items):\n") //fmt.Printf("%#v\n", cumulativeList) - for _, item := range cumulativeList[:15] { - fmt.Printf(" %s => %.2f\n", formatTime(item.Timestamp), item.Value) + for idx, item := range cumulativeList { + fmt.Printf(" %d: %s => %.2f\n", idx, formatTime(item.Timestamp), item.Value) } } @@ -518,6 +521,7 @@ func GenerateCumulativeMeasures(days int) []proto.Measure { totalValue float64 ) + idx := 0 for i := range totalHours { hourTime := since.Add(time.Duration(i) * time.Hour) for _, m := range minutes { @@ -539,6 +543,16 @@ func GenerateCumulativeMeasures(days int) []proto.Measure { measures = append(measures, measure) totalValue += rand.Float64() + + if idx < 1500 { + fmt.Println(idx, measure.Timestamp) + } + idx++ + } + } + for i := 1; i < len(measures); i++ { + if measures[i].Timestamp <= measures[i-1].Timestamp { + fmt.Println("FUCK", i) } } return measures @@ -580,3 +594,9 @@ func GenerateInstantMeasures(days int, baseValue float64) []proto.Measure { } return measures } + +//02 88 30 3e 88 31 1d 88 31 3e a3 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 28 68 05 6a +//02 88 30 3e 88 31 1d 88 31 3e a3 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 28 68 05 6a + +//02 88 4b 4c 88 4c 06 88 4c 39 ae 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 30 1b 2c 6a +//02 88 4b 4c 88 4c 06 88 4c 39 ae 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 30 1b 2c 6a diff --git a/examples/play/play b/examples/play/play index a7dd256..11df8c4 100755 Binary files a/examples/play/play and b/examples/play/play differ diff --git a/freelist/freelist.go b/freelist/freelist.go index 5b77993..bd12b26 100644 --- a/freelist/freelist.go +++ b/freelist/freelist.go @@ -80,13 +80,13 @@ func (s *FreeList) AddPageNumbers(pageNumbers []uint32) (err error) { // викидаю номери сторінок, які записав на диск copy(s.free, s.free[s.pointersOnPage:]) s.free = s.free[:len(s.free)-s.pointersOnPage] - fmt.Println("free:", s.free) + //fmt.Println("free:", s.free) } return } func (s *FreeList) save() error { - fmt.Println("save:", s.free[:s.pointersOnPage]) + //fmt.Println("save:", s.free[:s.pointersOnPage]) buf := make([]byte, s.pageSize) i := crcSize for _, pageNo := range s.free[:s.pointersOnPage] { @@ -108,7 +108,7 @@ func (s *FreeList) save() error { off = int64(s.basePagesCount * s.pageSize) s.basePagesCount++ } - fmt.Println("write at:", off) + //fmt.Println("write at:", off) n, err := file.WriteAt(buf, off) if err != nil { return err diff --git a/freelist/test.basefree b/freelist/test.basefree deleted file mode 100644 index 923d256..0000000 Binary files a/freelist/test.basefree and /dev/null differ diff --git a/freelist/test.deltafree b/freelist/test.deltafree deleted file mode 100644 index e69de29..0000000 diff --git a/go.mod b/go.mod index 56c6338..d3dfc76 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,13 @@ module gordenko.dev/dima/qb go 1.24.2 require ( + github.com/go-sql-driver/mysql v1.10.0 gopkg.in/ini.v1 v1.67.1 gordenko.dev/dima/bin v0.0.0-20260612161453-4ee9be3474fb + gordenko.dev/dima/fixme v0.0.0-20230701203753-4c2b6c1eef52 + gordenko.dev/dima/pretty v0.0.0-20221225212746-0c27d8c0ac69 + gordenko.dev/dima/qx v0.0.0-20231123051353-23dce7c0ce88 gordenko.dev/dima/textutil v0.0.0-20221225052909-270513dcbbbf ) + +require filippo.io/edwards25519 v1.2.0 // indirect diff --git a/go.sum b/go.sum index 84899b1..b1b2740 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,10 @@ +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw= +github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -20,5 +24,11 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gordenko.dev/dima/bin v0.0.0-20260612161453-4ee9be3474fb h1:XBYZbK5Z+yfpKmSR2wuPLyW14ReCRSNdOlL5MWro8ns= gordenko.dev/dima/bin v0.0.0-20260612161453-4ee9be3474fb/go.mod h1:up64wpJp9xI+HqACtMDBIfPNUH6BYY/b3inKpUfrqDQ= +gordenko.dev/dima/fixme v0.0.0-20230701203753-4c2b6c1eef52 h1:g3f99mV2NtLsN1/O7qkV9CwfEGhcWFS0Y5FX12On/NU= +gordenko.dev/dima/fixme v0.0.0-20230701203753-4c2b6c1eef52/go.mod h1:7tMDvA2ej8e7VwXIlcIZNtUe/Tk5jFAK498WqC+aMYE= +gordenko.dev/dima/pretty v0.0.0-20221225212746-0c27d8c0ac69 h1:nyJ3mzTQ46yUeMZCdLyYcs7B5JCS54c67v84miyhq2E= +gordenko.dev/dima/pretty v0.0.0-20221225212746-0c27d8c0ac69/go.mod h1:AxgKDktpqBVyIOhIcP+nlCpK+EsJyjN5kPdqyd8euVU= +gordenko.dev/dima/qx v0.0.0-20231123051353-23dce7c0ce88 h1:/9KsVGy3tsDR4SmSs9fZQx1hslYY89KqNpIa55kKuX0= +gordenko.dev/dima/qx v0.0.0-20231123051353-23dce7c0ce88/go.mod h1:CR5GISpjHFHZ9HeSIo3fYRyp1+kVx0R9exzBYsIuB3c= gordenko.dev/dima/textutil v0.0.0-20221225052909-270513dcbbbf h1:GCL/m736iGDy3LgOm6iuvkxlGhUEyjfP7zgwZRkTYOg= gordenko.dev/dima/textutil v0.0.0-20221225052909-270513dcbbbf/go.mod h1:M3e8S3N1USCMYmL6Q5uDQJaxfzBlWHnTX7rxCJQ6iJs= diff --git a/mqe/aggregated.go b/mqe/aggregated.go new file mode 100644 index 0000000..0f0a789 --- /dev/null +++ b/mqe/aggregated.go @@ -0,0 +1,340 @@ +package mqe + +import ( + "fmt" + "strings" + "sync" + "time" + + "gordenko.dev/dima/qb/timeutil" +) + +type AggregatedMeasure struct { + Period int64 `json:"p"` + Since int64 `json:"s"` + Until int64 `json:"u"` + // для cumulative метрики 2 значения - value и total + // для instant метрики от 1 до 3 значений - min, max, avg + Values []float64 `json:"v"` +} + +type AggregatedFilter struct { + Since time.Time + Until time.Time + GroupBy GroupBy + LastDayOfMonth int // например, конец месяца 25 число + FirstHourOfDay int // например день начинается в 6:00 + InstantMetrics []InstantMetricAndFuncs + CumulativeMetrics []int64 +} + +type InstantMetricAndFuncs struct { + MetricID int64 + AggregateFuncs byte +} + +func (s *MeasureQueryEngine) SelectAggregatedData(req AggregatedFilter) (resultMap map[int64][]AggregatedMeasure, err error) { + since := req.Since + until := req.Until + // Валидация + switch req.GroupBy { + case ByHour, ByDay, ByMonth: + // pass + default: + err = fix.Field(WrongValue, "groupBy") + return + } + + if req.FirstHourOfDay < 0 || req.FirstHourOfDay > 23 { + err = fix.Field(InvalidFirstHourOfDay, "firstHourOfDay") + return + } + + if req.LastDayOfMonth < 0 || req.LastDayOfMonth > 28 { + err = fix.Field(InvalidLastDayOfMonth, "lastDayOfMonth") + return + } + + // Корректируем, ВСЕГДА на последнюю секунду суток + //until = timeutil.LastSecondInPeriod(until, "d") + + // fmt.Printf(`SelectAggregatedData: { + // Since: %s + // Until: %s + // GroupBy: %s + // FirstHourOfDay: %d + // LastDayOfMonth: %d + // } + // `, since, until, req.GroupBy, req.FirstHourOfDay, req.LastDayOfMonth) + // FIX проверить как добавляются часы в DST часовых поясах + // Для req.FirstHourOfDay=4 since должен быть 4:00:00, until должен быть + // 3:59:59 следующего дня + if req.FirstHourOfDay > 0 { + since = since.Add(time.Duration(req.FirstHourOfDay) * time.Hour) + until = until.Add(time.Duration(req.FirstHourOfDay) * time.Hour) + } + + sinceUnixtime := since.Unix() + untilUnixtime := until.Unix() + + if sinceUnixtime >= untilUnixtime { + err = fix.Error(WrongDateRange) + return + } + + var ( + cumulativeTasks []*AggregatedCumulativeTask + instantTasks []*AggregatedInstantTask + wg = new(sync.WaitGroup) + ) + + // Важно вирутальная метрика зависит от реальных, которые к объекту могут быть + // привязаны или нет. Если нет - не возвращать. + resultMap = make(map[int64][]AggregatedMeasure) + + if len(req.CumulativeMetrics) > 0 { + for _, metricID := range req.CumulativeMetrics { + task := &AggregatedCumulativeTask{ + In: AggregatedMeasuresFilter{ + MetricID: metricID, + Since: sinceUnixtime, + Until: untilUnixtime, + GroupBy: req.GroupBy, + LastDayOfMonth: req.LastDayOfMonth, + FirstHourOfDay: req.FirstHourOfDay, + }, + WaitGroup: wg, + } + + wg.Add(1) + s.aggregatedCumulativeCh <- task + cumulativeTasks = append(cumulativeTasks, task) + } + } + + if len(req.InstantMetrics) > 0 { + for _, metric := range req.InstantMetrics { + task := &AggregatedInstantTask{ + In: AggregatedInstantMeasuresIn{ + MetricID: metric.MetricID, + Since: sinceUnixtime, + Until: untilUnixtime, + GroupBy: req.GroupBy, + LastDayOfMonth: req.LastDayOfMonth, + FirstHourOfDay: req.FirstHourOfDay, + Flags: metric.AggregateFuncs, // ФЛАГИ !!! + }, + WaitGroup: wg, + } + + wg.Add(1) + s.aggregatedInstantCh <- task + instantTasks = append(instantTasks, task) + } + } + + wg.Wait() + + for _, task := range cumulativeTasks { + if task.Err != nil { + err = fmt.Errorf("get data for the metric %d: %s", task.In.MetricID, task.Err) + return + } + resultMap[task.In.MetricID] = task.Result + } + + for _, task := range instantTasks { + if task.Err != nil { + err = fmt.Errorf("get data for the metric %d: %s", task.In.MetricID, task.Err) + return + } + resultMap[task.In.MetricID] = task.Result + } + return +} + +type IndividualAggregatedFilter struct { + Since time.Time + Until time.Time + GroupBy GroupBy + InstantMetrics []IndividualInstantMetric + CumulativeMetrics []IndividualCumulativeMetric +} + +type IndividualInstantMetric struct { + MetricID int64 + AggregateFuncs byte + LastDayOfMonth int // например, конец месяца 25 число + FirstHourOfDay int // например день начинается в 6:00 +} + +type IndividualCumulativeMetric struct { + MetricID int64 + LastDayOfMonth int // например, конец месяца 25 число + FirstHourOfDay int // например день начинается в 6:00 +} + +func (s *MeasureQueryEngine) SelectIndividualAggregatedData(req IndividualAggregatedFilter) (resultMap map[int64][]AggregatedMeasure, err error) { + if req.Since.IsZero() { + err = fmt.Errorf("zero Since") + return + } + + if req.Until.IsZero() { + err = fmt.Errorf("zero Until") + return + } + + if req.Since.After(req.Until) { + err = fmt.Errorf("wrong time range: since %s after until %s", req.Since, req.Until) + return + } + + // Корректируем, ВСЕГДА на первую и последнюю секунду суток + req.Since = timeutil.FirstSecondInPeriod(req.Since, "d") + req.Until = timeutil.LastSecondInPeriod(req.Until, "d") + + // Валидация + switch req.GroupBy { + case ByHour, ByDay, ByMonth: + // pass + default: + err = fix.Field(WrongValue, "groupBy") + return + } + + // fmt.Printf(`SelectIndividualAggregatedData: { + // Since: %s + // Until: %s + // GroupBy: %s + // } + // `, req.Since, req.Until, req.GroupBy) + // Важно вирутальная метрика зависит от реальных, которые к объекту могут быть + // привязаны или нет. Если нет - не возвращать. + + resultMap = make(map[int64][]AggregatedMeasure) + + var ( + unexpectedErrors []error + errorMutex sync.Mutex + + wg sync.WaitGroup + ) + + wg.Add(len(req.CumulativeMetrics) + len(req.InstantMetrics)) + + if len(req.CumulativeMetrics) > 0 { + for _, m := range req.CumulativeMetrics { + go func(metric IndividualCumulativeMetric) { + if metric.FirstHourOfDay < 0 || metric.FirstHourOfDay > 23 { + err = fix.Field(InvalidFirstHourOfDay, "firstHourOfDay") + return + } + + if metric.LastDayOfMonth < 0 || metric.LastDayOfMonth > 28 { + err = fix.Field(InvalidLastDayOfMonth, "lastDayOfMonth") + return + } + + since := req.Since + until := req.Until + // FIX проверить как добавляются часы в DST часовых поясах + // Для req.FirstHourOfDay=4 since должен быть 4:00:00, until должен быть + // 3:59:59 следующего дня + if metric.FirstHourOfDay > 0 { + since = req.Since.Add(time.Duration(metric.FirstHourOfDay) * time.Hour) + until = req.Until.Add(time.Duration(metric.FirstHourOfDay) * time.Hour) + } + + result, err := s.listAggregatedCumulativeMeasures(AggregatedMeasuresFilter{ + MetricID: metric.MetricID, + Since: since.Unix(), + Until: until.Unix(), + GroupBy: req.GroupBy, + LastDayOfMonth: metric.LastDayOfMonth, + FirstHourOfDay: metric.FirstHourOfDay, + }) + if err != nil { + err = fmt.Errorf("ListAggregatedCumulativeMeasures: %s", err) + + errorMutex.Lock() + unexpectedErrors = append(unexpectedErrors, err) + errorMutex.Unlock() + } else { + errorMutex.Lock() + resultMap[metric.MetricID] = result + errorMutex.Unlock() + } + + wg.Done() + }(m) + } + } + + if len(req.InstantMetrics) > 0 { + for _, m := range req.InstantMetrics { + go func(metric IndividualInstantMetric) { + if metric.FirstHourOfDay < 0 || metric.FirstHourOfDay > 23 { + err = fix.Field(InvalidFirstHourOfDay, "firstHourOfDay") + return + } + + if metric.LastDayOfMonth < 0 || metric.LastDayOfMonth > 28 { + err = fix.Field(InvalidLastDayOfMonth, "lastDayOfMonth") + return + } + + since := req.Since + until := req.Until + // FIX проверить как добавляются часы в DST часовых поясах + // Для req.FirstHourOfDay=4 since должен быть 4:00:00, until должен быть + // 3:59:59 следующего дня + if metric.FirstHourOfDay > 0 { + since = req.Since.Add(time.Duration(metric.FirstHourOfDay) * time.Hour) + until = req.Until.Add(time.Duration(metric.FirstHourOfDay) * time.Hour) + } + + result, err := s.listAggregatedInstantMeasures(AggregatedInstantMeasuresIn{ + MetricID: metric.MetricID, + Since: since.Unix(), + Until: until.Unix(), + GroupBy: req.GroupBy, + LastDayOfMonth: metric.LastDayOfMonth, + FirstHourOfDay: metric.FirstHourOfDay, + Flags: metric.AggregateFuncs, // ФЛАГИ !!! + }) + if err != nil { + err = fmt.Errorf("listAggregatedInstantMeasures: %s", err) + + errorMutex.Lock() + unexpectedErrors = append(unexpectedErrors, err) + errorMutex.Unlock() + } else { + errorMutex.Lock() + resultMap[metric.MetricID] = result + errorMutex.Unlock() + } + + wg.Done() + }(m) + } + } + + wg.Wait() + + // Проверяем что все запросы завершились удачно + if len(unexpectedErrors) > 0 { + // Нумеруем сообщения об ошибках и склеиваем в одно большое сообщение + var errorStrings []string + + for idx, err := range unexpectedErrors { + errorStrings = append(errorStrings, fmt.Sprintf("\n#%d %s", idx+1, err)) + } + + err = fmt.Errorf("%d errors occured:%s", len(unexpectedErrors), + strings.Join(errorStrings, "")) + return + } + + return +} diff --git a/mqe/cumulative.go b/mqe/cumulative.go new file mode 100644 index 0000000..8ac9f17 --- /dev/null +++ b/mqe/cumulative.go @@ -0,0 +1,585 @@ +package mqe + +import ( + "database/sql" + "fmt" + "time" +) + +// func applyCorrections(corrections []model.F64Correction, measures []Measure) []Measure { +// for idx, measure := range measures { +// var isCorrected bool +// for _, correction := range corrections { +// if measure.Time >= correction.Time { +// measure.Value += correction.Value +// isCorrected = true +// } +// } +// if isCorrected { +// measures[idx] = measure +// } +// } +// return measures +// } + +func applyCorrectionsToMeasure(corrections []_f64Correction, tm int64, value float64) float64 { + for _, correction := range corrections { + if tm >= correction.Time { + value += correction.Value + } + } + return value +} + +func applyCorrectionsToMeasures(corrections []_f64Correction, measures []_measure) { + for idx, m := range measures { + for _, correction := range corrections { + if m.Time >= correction.Time { + m.Value += correction.Value + } + } + measures[idx] = m + } +} + +func applyCorrectionsToRawMeasures(corrections []_f64Correction, measures []RawMeasure) { + for idx, m := range measures { + for _, correction := range corrections { + if m.Time >= correction.Time { + m.Value += correction.Value + } + } + measures[idx] = m + } +} + +type _measure struct { + Time int64 + Value float64 +} + +// ВАЖНО! +// ВО ВСЕХ ЗАПРОСАХ ОБЯЗАТЕЛЬНА КОРРЕКТНАЯ СОРТИРОВКА! + +type _f64Correction struct { + Time int64 + Value float64 +} + +func listF64CorrectionsTx(tx *sql.Tx, metricID int64) (list []_f64Correction, err error) { + rows, err := tx.Query( + "SELECT tm, value FROM metric_corrections WHERE metricID=? ORDER BY tm ASC", + metricID) + if err != nil { + if err == sql.ErrNoRows { + err = nil + } + return + } + defer rows.Close() + + for rows.Next() { + var correction _f64Correction + err = rows.Scan(&correction.Time, &correction.Value) + if err != nil { + return + } + list = append(list, correction) + } + return +} + +// result = append(result, Measure{ +// Time: tm, +// Values: []float64{ +// value, +// value - prevValue, +// }, +// }) + +// listCumulativeMeasures - возвращает список CumulativeMeasure. +// Метод ничего не знает про FirstHourOfDay, но извне границы Since и Until могут быть +// скорректированы с учетом FirstHourOfDay. Это не поломает выборку. +func (s *MeasureQueryEngine) listCumulativeMeasures(req MetricMeasuresFilter) (_ []Measure, err error) { + tx, err := s.db.Driver().Begin() + if err != nil { + return + } + defer tx.Rollback() + + // corrections, err := listF64CorrectionsTx(tx, req.MetricID) + // if err != nil { + // return + // } + + // ВАЖНО! + // Оптимизация! Запрашиваем больше данных чем нужно, чтобы посчитать Total для + // самого старого измерения в result. + // Отнимаем от since еще 1 час (с запасом) чтобы зацепить extendedMeasure + extendedSince := req.Since - 3600 + + rows, err := tx.Query(` + SELECT tm, value + FROM f64 + WHERE metricID=? AND tm BETWEEN ? AND ? + ORDER BY tm ASC`, + req.MetricID, extendedSince, req.Until) + if err != nil { + if err == sql.ErrNoRows { + err = nil + } + return + } + defer rows.Close() + + var ( + result []Measure + measures []_measure + prev *_measure + ) + + // Вычитываем все найденные показания + for rows.Next() { + var m _measure + err = rows.Scan(&m.Time, &m.Value) + if err != nil { + return + } + + if m.Time < req.Since { + // пока не найдено первое показание внутри диапазона обновлем extendedValue + prev = &_measure{ + Time: m.Time, + Value: m.Value, + } + } else { + measures = append(measures, m) + break + } + } + + // Остальные показания вычитываем без проверок + for rows.Next() { + var m _measure + err = rows.Scan(&m.Time, &m.Value) + if err != nil { + return + } + measures = append(measures, m) + } + + if err = rows.Err(); err != nil { + return + } + + if len(measures) == 0 { + return + } + + if prev == nil { + // extended not found + // extended показание не найдено, а значит для первого показания не сможем показать total. + // Нужен дополнительный запрос. + // Находим максимально свежее показание метрики до extendedSince, от которого и будем + // считать total. + var ( + tm int64 + value float64 + ) + + err = tx.QueryRow(` + SELECT tm, value + FROM f64 + WHERE metricID=? AND tm < ? + ORDER BY tm DESC + LIMIT 1`, + req.MetricID, extendedSince).Scan(&tm, &value) + + if err != nil { + if err != sql.ErrNoRows { + return + } + err = nil + } else { + prev = &_measure{ + Time: tm, + Value: value, + } + } + } + + // ВАЖНО! + // Сперва применяем коррекции к показаниям, а затем рассчитываем сумму за период + + // if len(corrections) > 0 { + // applyCorrectionsToMeasures(corrections, measures) + // if prev != nil { + // prev.Value = applyCorrectionsToMeasure(corrections, prev.Time, prev.Value) + // } + // } + + var prevValue float64 + if prev != nil { + prevValue = prev.Value + } + + for _, m := range measures { + result = append(result, Measure{ + Time: m.Time, + Values: []float64{ + m.Value, + m.Value - prevValue, + }, + }) + prevValue = m.Value + } + + // Корректируем самый первый тотал. Eсли prev показания не было -> total = 0 + if prev == nil { + first := result[0] + first.Values[1] = 0 + result[0] = first + } + + return result, nil +} + +type AggregatedMeasuresFilter struct { + MetricID int64 `json:"metricID"` + Since int64 `json:"since"` + Until int64 `json:"until"` + GroupBy GroupBy `json:"groupBy"` + LastDayOfMonth int `json:"lastDayOfMonth"` // например, конец месяца 25 число + FirstHourOfDay int `json:"firstHourOfDay"` // например день начинается в 6:00 +} + +// listAggregatedCumulativeMeasures - возвращает список AggregatedCumulativeMeasure, +// сгрупированный по какому-то периоду +func (s *MeasureQueryEngine) listAggregatedCumulativeMeasures(req AggregatedMeasuresFilter) (_ []AggregatedMeasure, err error) { + //pretty.PPrintln("listAggregatedCumulativeMeasures", req) + + tx, err := s.db.Driver().Begin() + if err != nil { + return + } + defer tx.Rollback() + + corrections, err := listF64CorrectionsTx(tx, req.MetricID) + if err != nil { + return + } + + var ( + // чтобы не экранировать символы в шаблоне строки + groupByFormat string + parsePeriodLayout string + ) + + switch req.GroupBy { + case ByHour: + groupByFormat = "%Y%m%d%H" + parsePeriodLayout = hourPeriodLayout + + case ByDay: + groupByFormat = "%Y%m%d" + parsePeriodLayout = dayPeriodLayout + + case ByMonth: + groupByFormat = "%Y%m" + parsePeriodLayout = monthPeriodLayout + + default: + // Защита от дурака + err = fmt.Errorf("unknown groupBy: %s", req.GroupBy) + return + } + + // ВАЖНО! + // Оптимизация! Запрашиваем больше данных чем нужно, чтобы посчитать начало + // первого и конец последнего периодов без дополнительных запросов. + // + + // Отнимаем 3 минуты чтобы зацепить последнее показание предыдущего периода + extendedSince := req.Since - 3*60 + // Добавляем 1 час, чтобы зацепить первое показание следующего периода + extendedUntil := req.Until + 60*60 + + //fmt.Printf("extendedSince: %s\n\n", extendedSince) + + // Для каждого периода выбираем первое и последнее (по времени) измерение. + // Если делать выборку одним запросом без UNION, а именно сделать + // INNER JOIN ... ON f64.tm=x.maxtm OR f64.tm=x.mintm - все начинает тормозить. + // Два запроса работают достаточно быстро. + + // ВАЖНО! + // Не забыть про сортировку tm DESC! + + dt, err := getPeriodDateTime(req.GroupBy, req.LastDayOfMonth, req.FirstHourOfDay) + if err != nil { + return + } + + //fmt.Printf("DT: %s\n\n", dt) + + query := fmt.Sprintf(` + WITH + measures AS (SELECT * FROM f64 WHERE metricID=? AND tm BETWEEN ? AND ?), + aggregated AS ( + SELECT MIN(tm) as minTm, MAX(tm) as maxTm, DATE_FORMAT(%s, '%s') as periodStr + FROM measures + GROUP BY periodStr + ) + SELECT aggregated.periodStr, firstMeasures.tm, firstMeasures.value, lastMeasures.tm, lastMeasures.value + FROM aggregated + INNER JOIN measures firstMeasures ON firstMeasures.tm = aggregated.minTm + INNER JOIN measures lastMeasures ON lastMeasures.tm = aggregated.maxTm + ORDER BY aggregated.periodStr ASC`, + dt, groupByFormat) + + rows, err := tx.Query(query, req.MetricID, extendedSince, extendedUntil) + if err != nil { + if err == sql.ErrNoRows { + err = nil + } + return + } + defer rows.Close() + + var result []AggregatedMeasure + + periodCalculator := NewPeriodCalculator(PeriodCalculatorOptions{ + GroupBy: req.GroupBy, + LastDayOfMonth: req.LastDayOfMonth, + FirstHourOfDay: req.FirstHourOfDay, + Since: time.Unix(req.Since, 0), + Until: time.Unix(req.Until, 0), + }) + + periodProducer := NewPeriodProducer(PeriodProducerOptions{ + MetricID: req.MetricID, + ParsePeriodLayout: parsePeriodLayout, + Location: s.location, + Until: req.Until, + Corrections: corrections, + Rows: rows, + Tx: tx, + }) + + result, err = SplitByPeriods(periodProducer, periodCalculator) + if err != nil { + return + } + return result, nil +} + +type RangeTotal struct { + Since int64 `json:"since"` // реальная граница + Until int64 `json:"until"` // реальная граница + StartValue float64 `json:"startValue"` // на конец периода + EndValue float64 `json:"endValue"` // на конец периода +} + +type getCumulativeTotalFilter struct { + MetricID int64 + Since int64 + Until int64 + LastDayOfMonth int + FirstHourOfDay int +} + +func (s *MeasureQueryEngine) getCumulativeTotal(req getCumulativeTotalFilter) (_ *RangeTotal, err error) { + measures, err := s.listAggregatedCumulativeMeasures(AggregatedMeasuresFilter{ + MetricID: req.MetricID, + Since: req.Since, + Until: req.Until, + GroupBy: ByDay, + LastDayOfMonth: req.LastDayOfMonth, + FirstHourOfDay: req.FirstHourOfDay, + }) + if err != nil { + return + } + + if len(measures) == 0 { + return + } + + //pretty.PPrintln("TOTAL", measures) + + first := measures[0] + last := measures[len(measures)-1] + + return &RangeTotal{ + Since: first.Since, + Until: last.Until, + StartValue: first.Values[0] - first.Values[1], // value on period end - period total + EndValue: last.Values[0], + }, nil +} + +/* +// GetCumulativeTotal - фикс, добавить опции FirstHourInDay, IsFindPeriodEndsInFuture +func (s *MeasureQueryEngine) getCumulativeTotal(req getCumulativeTotalFilter) (_ *RangeTotal, err error) { + tx, err := s.db.Driver().Begin() + if err != nil { + return + } + defer tx.Rollback() + + var ( + firstTm, lastTm, futureTm int64 + firstValue, lastValue, futureValue float64 + foundInPast, foundInFuture, foundFirstInRange, foundLastInRange bool + ) + + // Поиск первого показания + + // Заглядываем в прошлое на 3 минуты + firstTm, firstValue, foundInPast, err = s.findLastInRange(tx, req.MetricID, req.Since-3*60, req.Since) + if err != nil { + return + } + + if !foundInPast { + // + firstTm, firstValue, foundFirstInRange, err = s.findFirstInRange(tx, req.MetricID, req.Since, req.Until) + if err != nil { + return + } + + if !foundFirstInRange { + // Начало периода не найдено - значит нет показаний для указанного периода + return + } + } + + // Поиск последнего показания + + lastTm, lastValue, foundLastInRange, err = s.findLastInRange(tx, req.MetricID, req.Since, req.Until) + if err != nil { + return + } + + if req.CanPeriodEndsInFuture { + // Если можем искать конец в периода в будущем + if foundLastInRange { + if lastTm > (req.Until - 3*60) { + // Если lastTm в последние 3 минуты периода - период корректно завершен + return &RangeTotal{ + Since: firstTm, + Until: lastTm, + Value: lastValue, + Total: lastValue - firstValue, + }, nil + } + } + // Период не завершен в последние 3 минуты периода, поэтому ищем в будущем + futureTm, futureValue, foundInFuture, err = s.findFirstAfter(tx, req.MetricID, req.Until) + if err != nil { + return + } + + if foundInFuture { + return &RangeTotal{ + Since: firstTm, + Until: futureTm, + Value: futureValue, + Total: futureValue - firstValue, + }, nil + } else { + // Супер важная проверка, ибо возможна ситуация, когда когда нашли начало + // текущего периода в конце предыдущего (foundInPast). А это показание было + // последним в БД. + if foundLastInRange { + // Последнее показание внутри периода нашли, неважно когда оно пришло - нам + // подходит. + return &RangeTotal{ + Since: firstTm, + Until: lastTm, + Value: lastValue, + Total: lastValue - firstValue, + }, nil + } + // Последнее показание внутри периода не найдено, а значит их не было вообще. + // Просто выходим + return + } + } else { + // В будущем искать нельзя + + // Супер важная проверка, ибо возможна ситуация, когда когда нашли начало + // текущего периода в конце предыдущего (foundInPast). А это показание было + // последним в БД. + if foundLastInRange { + // Последнее показание внутри периода нашли, неважно когда оно пришло - нам + // подходит. + return &RangeTotal{ + Since: firstTm, + Until: lastTm, + Value: lastValue, + Total: lastValue - firstValue, + }, nil + } + // Последнее показание внутри периода не найдено, а значит их не было вообще. + // Просто выходим + return + } +} +*/ + +func (s *MeasureQueryEngine) findLastInRangeTx(tx *sql.Tx, metricID int64, since, until int64) (tm int64, value float64, found bool, err error) { + err = tx.QueryRow(` + SELECT tm, value, true + FROM f64 + WHERE metricID=? AND tm >= ? AND tm < ? + ORDER BY tm DESC + LIMIT 1`, + metricID, since, until).Scan(&tm, &value, &found) + + if err != nil { + if err != sql.ErrNoRows { + return + } + // Нет записей + err = nil + } + return +} + +func (s *MeasureQueryEngine) findFirstInRangeTx(tx *sql.Tx, metricID int64, since, until int64) (tm int64, value float64, found bool, err error) { + err = tx.QueryRow(` + SELECT tm, value, true + FROM f64 + WHERE metricID=? AND tm >= ? AND tm < ? + ORDER BY tm ASC + LIMIT 1`, + metricID, since, until).Scan(&tm, &value, &found) + + if err != nil { + if err != sql.ErrNoRows { + return + } + // Нет записей + err = nil + } + return +} + +func (s *MeasureQueryEngine) findFirstAfterTx(tx *sql.Tx, metricID int64, since int64) (tm int64, value float64, found bool, err error) { + err = tx.QueryRow(` + SELECT tm, value, true + FROM f64 + WHERE metricID=? AND tm >= ? + ORDER BY tm ASC + LIMIT 1`, + metricID, since).Scan(&tm, &value, &found) + + if err != nil { + if err != sql.ErrNoRows { + return + } + // Нет записей + err = nil + } + return +} diff --git a/mqe/fix.go b/mqe/fix.go new file mode 100644 index 0000000..708b138 --- /dev/null +++ b/mqe/fix.go @@ -0,0 +1,29 @@ +package mqe + +import ( + "gordenko.dev/dima/fixme" +) + +const ( + EmptyValue = "EMPTY_VALUE" + BusyName = "BUSY_NAME" + WrongValue = "WRONG_VALUE" + WrongDate = "WRONG_DATE" + WrongDateRange = "WRONG_DATE_RANGE" + InvalidFirstHourOfDay = "WRONG_FIRST_HOUR_OF_DAY" + InvalidLastDayOfMonth = "WRONG_LAST_DAY_OF_MONTH" +) + +var ( + errorMessages = map[fixme.Code]string{ + EmptyValue: "порожнє значення неприпустимо", + BusyName: "имя уже используется", + WrongValue: "неправильне значення", + WrongDate: "некоректна дата", + WrongDateRange: "некоректний діапазон", + InvalidFirstHourOfDay: "корректное значение от 0 до 23", + InvalidLastDayOfMonth: "корректное значение от 0 до 28", + } + + fix = fixme.New(errorMessages) +) diff --git a/mqe/helpers.go b/mqe/helpers.go new file mode 100644 index 0000000..591fe95 --- /dev/null +++ b/mqe/helpers.go @@ -0,0 +1,158 @@ +package mqe + +import ( + "fmt" +) + +func getPeriodDateTime(groupBy GroupBy, lastDayOfMonth int, firstHourOfDay int) (dt string, err error) { + //fmt.Printf("getPeriodDateTime: %s, lastDay: %d, firstHour: %d\n", groupBy, lastDayOfMonth, firstHourOfDay) + // Если FirstHourOfDay и LastDayOfMonth равны 0 - модифицировать дату не нужно + dt = `FROM_UNIXTIME(tm)` + + switch groupBy { + case ByDay: + if firstHourOfDay > 0 { + dt = fmt.Sprintf(` + IF (HOUR(FROM_UNIXTIME(tm)) >= %d, + FROM_UNIXTIME(tm), + DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY) + )`, + firstHourOfDay) + } + + case ByMonth: + if lastDayOfMonth > 0 { + if lastDayOfMonth > 28 { + // На всякий случай добавляем защиту от дурака, ибо модификация даты + // в запросе (+1 месяц) для 29, 30 и 31 чисел сделает запрос неверным. + err = fmt.Errorf("LastDayOfMonth: %d, max allowed value is 28", lastDayOfMonth) + return + } + + if firstHourOfDay > 0 { + // В псевдокоде модификация времени показания по корректный период + // выглядит так: + // + // if hour >= FirstHourOfDay { + // if day <= LastDayOfMonth { + // return tm + // } else { + // return tm + 1 month + // } + // } else { + // if (day - 1) <= LastDayOfMonth { + // return tm - 1 day + // } else { + // return tm - 1 day + 1 month + // } + // } + dt = fmt.Sprintf(` + IF (HOUR(FROM_UNIXTIME(tm)) >= %d, + IF (DAYOFMONTH(FROM_UNIXTIME(tm)) <= %d, + FROM_UNIXTIME(tm), + DATE_ADD(FROM_UNIXTIME(tm), INTERVAL 1 MONTH) + ), + IF (DAYOFMONTH(DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY)) <= %d, + DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY), + DATE_ADD(DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY), INTERVAL 1 MONTH) + ) + )`, + firstHourOfDay, lastDayOfMonth, lastDayOfMonth) + } else { + dt = fmt.Sprintf(` + IF (DAYOFMONTH(FROM_UNIXTIME(tm)) <= %d, + FROM_UNIXTIME(tm), + DATE_ADD(FROM_UNIXTIME(tm), INTERVAL 1 MONTH) + )`, + lastDayOfMonth) + } + } else { + if firstHourOfDay > 0 { + dt = fmt.Sprintf(` + IF (HOUR(FROM_UNIXTIME(tm)) >= %d, + FROM_UNIXTIME(tm), + DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY) + )`, + firstHourOfDay) + } + } + } + return +} + +func getDateTimeFacingFHD(firstHourOfDay int) (dt string, err error) { + if firstHourOfDay > 0 { + dt = fmt.Sprintf(` + IF (HOUR(FROM_UNIXTIME(tm)) >= %d, + FROM_UNIXTIME(tm), + DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY) + )`, + firstHourOfDay) + } else { + // Если FirstHourOfDay = 0 - модифицировать дату не нужно + dt = "FROM_UNIXTIME(tm)" + } + return +} + +func getDateTimeFacingFHDAndLDM(firstHourOfDay int, lastDayOfMonth int) (dt string, err error) { + // Если FirstHourOfDay и LastDayOfMonth равны 0 - модифицировать дату не нужно + dt = "FROM_UNIXTIME(tm)" + + if lastDayOfMonth > 0 { + if lastDayOfMonth > 28 { + // На всякий случай добавляем защиту от дурака, ибо модификация даты + // в запросе (+1 месяц) для 29, 30 и 31 чисел сделает запрос неверным. + err = fmt.Errorf("LastDayOfMonth: %d, max allowed value is 28", lastDayOfMonth) + return + } + + if firstHourOfDay > 0 { + // В псевдокоде модификация времени показания по корректный период + // выглядит так: + // + // if hour >= FirstHourOfDay { + // if day <= LastDayOfMonth { + // return tm + // } else { + // return tm + 1 month + // } + // } else { + // if (day - 1) <= LastDayOfMonth { + // return tm - 1 day + // } else { + // return tm - 1 day + 1 month + // } + // } + dt = fmt.Sprintf(` + IF (HOUR(FROM_UNIXTIME(tm)) >= %d, + IF (DAYOFMONTH(FROM_UNIXTIME(tm)) <= %d, + FROM_UNIXTIME(tm), + DATE_ADD(FROM_UNIXTIME(tm), INTERVAL 1 MONTH) + ), + IF (DAYOFMONTH(DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY)) <= %d, + DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY), + DATE_ADD(DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY), INTERVAL 1 MONTH) + ) + )`, + firstHourOfDay, lastDayOfMonth, lastDayOfMonth) + } else { + dt = fmt.Sprintf(` + IF (DAYOFMONTH(FROM_UNIXTIME(tm)) <= %d, + FROM_UNIXTIME(tm), + DATE_ADD(FROM_UNIXTIME(tm), INTERVAL 1 MONTH) + )`, + lastDayOfMonth) + } + } else { + if firstHourOfDay > 0 { + dt = fmt.Sprintf(` + IF (HOUR(FROM_UNIXTIME(tm)) >= %d, + FROM_UNIXTIME(tm), + DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY) + )`, + firstHourOfDay) + } + } + return +} diff --git a/mqe/index.go b/mqe/index.go new file mode 100644 index 0000000..36c6b4c --- /dev/null +++ b/mqe/index.go @@ -0,0 +1,320 @@ +package mqe + +import ( + "fmt" + "sort" + "strconv" +) + +type GetYearIndexReq struct { + Since int64 `json:"since"` // из фильтр-формы + Until int64 `json:"until"` // из фильтр-формы + LastDayOfMonth int `json:"lastDayOfMonth"` // например, конец месяца 25 число + FirstHourOfDay int `json:"firstHourOfDay"` // например день начинается в 6:00 + MetricIDs []int64 `json:"metricIDs"` +} + +func (s *MeasureQueryEngine) GetYearIndex(req GetYearIndexReq) (years []int, err error) { + if len(req.MetricIDs) == 0 { + return + } + + var ( + lte, gte string + + args = []any{ + req.MetricIDs, + } + ) + + if req.Since > 0 { + lte = " AND tm >= ?" + args = append(args, req.Since) + } + + if req.Until > 0 { + gte = " AND tm <= ?" + args = append(args, req.Until) + } + + q := fmt.Sprintf(` + SELECT YEAR(FROM_UNIXTIME(tm)) as y + FROM f64 + WHERE metricID IN (?)%s%s + GROUP BY y`, + lte, gte) + + // fmt.Println(q) + // fmt.Printf("%v\n", args) + + err = s.db.ListQuery(&years, q, args...) + if err != nil { + return + } + + sort.Ints(years) + return +} + +type GetMonthIndexReq struct { + Since int64 `json:"since"` // из фильтр-формы + Until int64 `json:"until"` // из фильтр-формы + LastDayOfMonth int `json:"lastDayOfMonth"` // например, конец месяца 25 число + FirstHourOfDay int `json:"firstHourOfDay"` // например день начинается в 6:00 + MetricIDs []int64 `json:"metricIDs"` +} + +type YearMonth struct { + Year int `json:"year"` + Months []int `json:"months"` +} + +func (s *MeasureQueryEngine) GetMonthIndex(req GetMonthIndexReq) (years []YearMonth, err error) { + if len(req.MetricIDs) == 0 { + return + } + + dt, err := getDateTimeFacingFHDAndLDM(req.FirstHourOfDay, req.LastDayOfMonth) + if err != nil { + return + } + + var ( + lte, gte string + + args = []any{ + req.MetricIDs, + } + ) + + if req.Since > 0 { + lte = " AND tm >= ?" + args = append(args, req.Since) + } + + if req.Until > 0 { + gte = " AND tm <= ?" + args = append(args, req.Until) + } + + // ВАЖНО! + // Не сортируем средствами MySQL чтобы не создавать временный файл + query := fmt.Sprintf(` + SELECT DATE_FORMAT(%s, '%s') as p + FROM f64 + WHERE metricID IN (?)%s%s + GROUP BY p`, + dt, "%Y%m", lte, gte) + + var ( + temp []string + yms []int + ) + + err = s.db.ListQuery(&temp, query, args...) + if err != nil { + return + } + + for _, str := range temp { + var ym int + ym, err = strconv.Atoi(str) + if err != nil { + return + } + yms = append(yms, ym) + } + return IndexYms(yms), nil +} + +type GetDayIndexReq struct { + Since int64 `json:"since"` // из фильтр-формы + Until int64 `json:"until"` // из фильтр-формы + FirstHourOfDay int `json:"firstHourOfDay"` // например день начинается в 6:00 + MetricIDs []int64 `json:"metricIDs"` +} + +type YearMonthDay struct { + Year int `json:"year"` + Months []MonthDay `json:"months"` +} + +type MonthDay struct { + Month int `json:"month"` + Days []int `json:"days"` +} + +func (s *MeasureQueryEngine) GetDayIndex(req GetDayIndexReq) (years []YearMonthDay, err error) { + if len(req.MetricIDs) == 0 { + return + } + + dt, err := getDateTimeFacingFHD(req.FirstHourOfDay) + if err != nil { + return + } + + var ( + lte, gte string + + args = []any{ + req.MetricIDs, + } + ) + + if req.Since > 0 { + lte = " AND tm >= ?" + args = append(args, req.Since) + } + + if req.Until > 0 { + gte = " AND tm <= ?" + args = append(args, req.Until) + } + // ВАЖНО! + // Не сортируем средствами MySQL чтобы не создавать временный файл + query := fmt.Sprintf(` + SELECT DATE_FORMAT(%s, '%s') as p + FROM f64 + WHERE metricID IN(?)%s%s + GROUP BY p`, + dt, "%Y%m%d", lte, gte) + + //fmt.Printf("%s\n", query) + + var ( + temp []string + ymds []int + ) + + err = s.db.ListQuery(&temp, query, args...) + if err != nil { + return + } + + //pretty.PPrintln("temp", temp) + + for _, str := range temp { + var ymd int + ymd, err = strconv.Atoi(str) + if err != nil { + return + } + ymds = append(ymds, ymd) + } + + return IndexYmds(ymds), nil +} + +func IndexYmds(ymds []int) (years []YearMonthDay) { + if len(ymds) == 0 { + return + } + + sort.Ints(ymds) + + var ( + months []MonthDay + days []int + year, month, day int + ) + + for _, ymd := range ymds { + y := ymd / 10000 + m := (ymd % 10000) / 100 + d := ymd % 100 + + if year == 0 { + // инициализация + year = y + month = m + day = d + } else { + if y != year { + days = append(days, day) + months = append(months, MonthDay{ + Month: month, + Days: days, + }) + years = append(years, YearMonthDay{ + Year: year, + Months: months, + }) + year = y + month = m + day = d + months = nil + days = nil + } else if m != month { + days = append(days, day) + months = append(months, MonthDay{ + Month: month, + Days: days, + }) + month = m + day = d + days = nil + } else if d != day { + days = append(days, day) + day = d + } + } + } + + days = append(days, day) + months = append(months, MonthDay{ + Month: month, + Days: days, + }) + years = append(years, YearMonthDay{ + Year: year, + Months: months, + }) + return +} + +func IndexYms(yms []int) (years []YearMonth) { + if len(yms) == 0 { + return + } + + sort.Ints(yms) + + var ( + months []int + year, month int + ) + + for _, ym := range yms { + y := ym / 100 + m := ym % 100 + + if year == 0 { + // инициализация + year = y + month = m + } else { + if y != year { + months = append(months, month) + years = append(years, YearMonth{ + Year: year, + Months: months, + }) + year = y + month = m + months = nil + } else if m != month { + months = append(months, month) + month = m + } + } + } + months = append(months, month) + years = append(years, YearMonth{ + Year: year, + Months: months, + }) + return +} + +//SELECT DATE_FORMAT(FROM_UNIXTIME(tm), '%Y%m%d') as p FROM f64 WHERE metricID IN (50) AND tm >= 1700863200 AND tm <= 1700949599 GROUP BY p; diff --git a/mqe/index_test.go b/mqe/index_test.go new file mode 100644 index 0000000..d72d056 --- /dev/null +++ b/mqe/index_test.go @@ -0,0 +1,39 @@ +package mqe + +import ( + "testing" + + "gordenko.dev/dima/pretty" +) + +func TestDayIndex(t *testing.T) { + index := IndexYmds([]int{ + 20221201, + 20221202, + 20230110, + 20230210, + 20230330, + 20230331, + 20240101, + 20240102, + 20240103, + }) + + pretty.Println(index) +} + +func TestMonthIndex(t *testing.T) { + index := IndexYms([]int{ + 202211, + 202212, + 202301, + 202302, + 202303, + 202304, + 202404, + 202405, + 202406, + }) + + pretty.Println(index) +} diff --git a/mqe/instant.go b/mqe/instant.go new file mode 100644 index 0000000..bb25ac0 --- /dev/null +++ b/mqe/instant.go @@ -0,0 +1,226 @@ +package mqe + +import ( + "database/sql" + "fmt" + "time" +) + +type CurrentValue struct { + MetricID uint32 + Time int64 `json:"t"` + Value float64 `json:"v"` +} + +func (s *MeasureQueryEngine) ListCurrentValues(metricIDs []uint32) (result []CurrentValue, err error) { + driver := s.db.Driver() + + for _, metricID := range metricIDs { + var rows *sql.Rows + rows, err = driver.Query(` + SELECT tm, value + FROM f64 + WHERE metricID=? + ORDER BY tm DESC + LIMIT 1`, + metricID) + if err != nil { + if err == sql.ErrNoRows { + err = nil + } + return + } + + for rows.Next() { + var ( + tm int64 + value float64 + ) + err = rows.Scan(&tm, &value) + if err != nil { + return + } + result = append(result, CurrentValue{ + MetricID: metricID, + Time: tm, + Value: value, + }) + } + + if err = rows.Err(); err != nil { + return + } + rows.Close() + } + return +} + +// ListInstantMeasures - cписок показаний мгновенных метрик (Температура, Давление, Расход) +// за за интервал без группировки +func (s *MeasureQueryEngine) listInstantMeasures(req MetricMeasuresFilter) (_ []Measure, err error) { + rows, err := s.db.Driver().Query(` + SELECT tm, value + FROM f64 + WHERE metricID=? AND tm BETWEEN ? AND ? + ORDER BY tm ASC`, + req.MetricID, req.Since, req.Until) + if err != nil { + if err == sql.ErrNoRows { + err = nil + } + return + } + defer rows.Close() + + var result []Measure + + for rows.Next() { + var ( + tm int64 + value float64 + ) + err = rows.Scan(&tm, &value) + if err != nil { + return + } + result = append(result, Measure{ + Time: tm, + Values: []float64{ + value, + }, + }) + } + + if err = rows.Err(); err != nil { + return + } + return result, nil +} + +type AggregatedInstantMeasuresIn struct { + MetricID int64 `json:"metricID"` + Since int64 `json:"since"` + Until int64 `json:"until"` + GroupBy GroupBy `json:"groupBy"` + Flags byte `json:"flags"` // из настроек по умолчанию либо из запроса + LastDayOfMonth int `json:"lastDayOfMonth"` // например, конец месяца 25 число + FirstHourOfDay int `json:"firstHourOfDay"` // например день начинается в 6:00 +} + +// Заполняет поля Period и Values (от 1 до 3 значений) в структуре AggregatedMeasure +func (s *MeasureQueryEngine) listAggregatedInstantMeasures(req AggregatedInstantMeasuresIn) (_ []AggregatedMeasure, err error) { + var ( + groupByFormat string + parsePeriodLayout string + + result []AggregatedMeasure + ) + + switch req.GroupBy { + case ByHour: + groupByFormat = "%Y%m%d%H" + parsePeriodLayout = hourPeriodLayout + + case ByDay: + groupByFormat = "%Y%m%d" + parsePeriodLayout = dayPeriodLayout + + case ByMonth: + groupByFormat = "%Y%m" + parsePeriodLayout = monthPeriodLayout + + default: + // Защита от дурака + err = fmt.Errorf("unknown GroupBy: %s", req.GroupBy) + return + } + + valuesQty := 0 + values := "" + + if (req.Flags & AggregateMin) == AggregateMin { + values += " MIN(value)," + valuesQty++ + } + + if (req.Flags & AggregateMax) == AggregateMax { + values += " MAX(value)," + valuesQty++ + } + + if (req.Flags & AggregateAvg) == AggregateAvg { + values += " AVG(value)," + valuesQty++ + } + + dt, err := getPeriodDateTime(req.GroupBy, req.LastDayOfMonth, req.FirstHourOfDay) + if err != nil { + return + } + + query := fmt.Sprintf(` + SELECT%s DATE_FORMAT(%s, '%s') as periodStr + FROM f64 + WHERE metricID=? AND tm BETWEEN ? AND ? + GROUP BY periodStr + ORDER BY periodStr ASC`, + values, dt, groupByFormat) + + rows, err := s.db.Driver().Query(query, req.MetricID, req.Since, req.Until) + if err != nil { + if err == sql.ErrNoRows { + err = nil + } + return + } + + for rows.Next() { + var ( + m = AggregatedMeasure{ + Values: make([]float64, valuesQty), + } + + periodStr string + period time.Time + + valueIdx int + dest []interface{} + ) + + if (req.Flags & AggregateMin) == AggregateMin { + dest = append(dest, &m.Values[valueIdx]) + valueIdx++ + } + + if (req.Flags & AggregateMax) == AggregateMax { + dest = append(dest, &m.Values[valueIdx]) + valueIdx++ + } + + if (req.Flags & AggregateAvg) == AggregateAvg { + dest = append(dest, &m.Values[valueIdx]) + } + + dest = append(dest, &periodStr) + + err = rows.Scan(dest...) + if err != nil { + return + } + + period, err = time.ParseInLocation(parsePeriodLayout, periodStr, s.location) + if err != nil { + err = fmt.Errorf("time.ParseInLocation: %s; layout=%q; str=%q", + err, parsePeriodLayout, periodStr) + return + } + m.Period = period.Unix() + + result = append(result, m) + } + + if err = rows.Err(); err != nil { + return + } + return result, nil +} diff --git a/mqe/mqe.go b/mqe/mqe.go new file mode 100644 index 0000000..b58e215 --- /dev/null +++ b/mqe/mqe.go @@ -0,0 +1,161 @@ +package mqe + +import ( + "sync" + "time" + + "gordenko.dev/dima/qx" +) + +type GroupBy string + +// Для парсинга since и until +const ( + NoAggregateFunc byte = 1 + AggregateMin byte = 2 + AggregateMax byte = 4 + AggregateAvg byte = 8 + + ByHour GroupBy = "h" + ByDay GroupBy = "d" + ByMonth GroupBy = "m" + + hourPeriodLayout = "2006010215" + dayPeriodLayout = "20060102" + monthPeriodLayout = "200601" + + dateLayout = "2006-01-02" +) + +type MetricMeasuresFilter struct { + MetricID int64 `json:"metricID"` + Since int64 `json:"since"` // уже учтен firstHourOfDay + Until int64 `json:"until"` // уже учтен firstHourOfDay +} + +type MeasureQueryEngine struct { + db *qx.Db + location *time.Location + cumulativeCh chan *CumulativeTask + instantCh chan *InstantTask + aggregatedCumulativeCh chan *AggregatedCumulativeTask + aggregatedInstantCh chan *AggregatedInstantTask + cumulativeTotalCh chan *CumulativeTotalTask + metricReadingsCh chan *MetricReadingsTask +} + +type Options struct { + Db *qx.Db + Location *time.Location + MinWorkers int + QueueSize int +} + +func New(opt Options) *MeasureQueryEngine { + if opt.Db == nil { + panic("Db option is required") + } + + if opt.Location == nil { + panic("Location option is required") + } + + if opt.MinWorkers <= 0 { + panic("MinWorkers option is required") + } + + if opt.QueueSize <= 0 { + panic("QueueSize option is required") + } + + s := new(MeasureQueryEngine) + s.db = opt.Db + s.location = opt.Location + // + s.cumulativeCh = make(chan *CumulativeTask, opt.QueueSize) + s.instantCh = make(chan *InstantTask, opt.QueueSize) + s.aggregatedCumulativeCh = make(chan *AggregatedCumulativeTask, opt.QueueSize) + s.aggregatedInstantCh = make(chan *AggregatedInstantTask, opt.QueueSize) + s.cumulativeTotalCh = make(chan *CumulativeTotalTask, opt.QueueSize) + s.metricReadingsCh = make(chan *MetricReadingsTask, opt.QueueSize) + + // + for i := 0; i < opt.MinWorkers; i++ { + go s.worker() + } + return s +} + +func (s *MeasureQueryEngine) worker() { + for { + select { + case task := <-s.cumulativeCh: + task.Result, task.Err = s.listCumulativeMeasures(task.In) + task.WaitGroup.Done() + + case task := <-s.instantCh: + task.Result, task.Err = s.listInstantMeasures(task.In) + task.WaitGroup.Done() + + case task := <-s.aggregatedCumulativeCh: + task.Result, task.Err = s.listAggregatedCumulativeMeasures(task.In) + task.WaitGroup.Done() + + case task := <-s.aggregatedInstantCh: + task.Result, task.Err = s.listAggregatedInstantMeasures(task.In) + task.WaitGroup.Done() + + case task := <-s.cumulativeTotalCh: + task.Result, task.Err = s.getCumulativeTotal(task.In) + task.WaitGroup.Done() + + case task := <-s.metricReadingsCh: + task.Result, task.Err = s.listMetricReadings(task.In) + task.WaitGroup.Done() + } + } +} + +// + +type CumulativeTask struct { + In MetricMeasuresFilter + WaitGroup *sync.WaitGroup + Result []Measure + Err error +} + +type InstantTask struct { + In MetricMeasuresFilter + WaitGroup *sync.WaitGroup + Result []Measure + Err error +} + +type AggregatedCumulativeTask struct { + In AggregatedMeasuresFilter + WaitGroup *sync.WaitGroup + Result []AggregatedMeasure + Err error +} + +type AggregatedInstantTask struct { + In AggregatedInstantMeasuresIn + WaitGroup *sync.WaitGroup + Result []AggregatedMeasure + Err error +} + +type MetricReadingsTask struct { + In MetricMeasuresFilter + WaitGroup *sync.WaitGroup + Result MetricReadings + Err error +} + +type CumulativeTotalTask struct { + In getCumulativeTotalFilter + WaitGroup *sync.WaitGroup + Result *RangeTotal + Err error +} diff --git a/mqe/newreq.go b/mqe/newreq.go new file mode 100644 index 0000000..db83961 --- /dev/null +++ b/mqe/newreq.go @@ -0,0 +1,283 @@ +package mqe + +// НОВАЯ ВЕРСИЯ + +// type MetricsDataFilter2 struct { +// Since time.Time `json:"since"` +// Until time.Time `json:"until"` +// GroupBy string `json:"groupBy"` + +// InstantMetrics []InstantMetricFilter // virtualMetricID + aggregateFuncs +// CumulativeMetrics []CumulativeMetricFilter +// } + +// type InstantMetricFilter struct { +// MetricID int64 +// AggregateFuncs byte +// LastDayOfMonth int // например, конец месяца 25 число +// FirstHourOfDay int // например день начинается в 6:00 +// } + +// type CumulativeMetricFilter struct { +// MetricID int64 +// LastDayOfMonth int // например, конец месяца 25 число +// FirstHourOfDay int // например день начинается в 6:00 +// } + +// func (s *MeasureQueryEngine) SelectMetricsDataInParallel2(req MetricsDataFilter2) (_ MetricsDataResultMaps, err error) { +// pretty.PPrintln("SelectMetricsDataInParallel2", req) +// // Строгая валидация + +// if req.Since.IsZero() { +// err = fmt.Errorf("Since option is required") +// return +// } + +// if req.Until.IsZero() { +// err = fmt.Errorf("Until option is required") +// return +// } + +// if req.Since.After(req.Until) { +// err = fmt.Errorf("Wrong time range: Since %d after Until %d", req.Since.Unix(), req.Until.Unix()) +// return +// } + +// switch req.GroupBy { +// case "h", "d", "m", "": +// // pass +// default: +// err = fmt.Errorf("Wrong groupBy option value: %s", req.GroupBy) +// return +// } + +// for _, mf := range req.InstantMetrics { +// if mf.MetricID == 0 { +// err = fmt.Errorf("Empty instant metric's MetricID: %d", mf.MetricID) +// return +// } + +// if mf.FirstHourOfDay < 0 || mf.FirstHourOfDay > 23 { +// err = fmt.Errorf("Wrong instant metric's FirstHourOfDay option value: %d", mf.FirstHourOfDay) +// return +// } + +// if mf.LastDayOfMonth < 0 || mf.LastDayOfMonth > 28 { +// err = fmt.Errorf("Wrong instant metric's LastDayOfMonth option value: %d", mf.LastDayOfMonth) +// return +// } +// } + +// for _, mf := range req.CumulativeMetrics { +// if mf.MetricID == 0 { +// err = fmt.Errorf("Empty cumulative metric's MetricID: %d", mf.MetricID) +// return +// } + +// if mf.FirstHourOfDay < 0 || mf.FirstHourOfDay > 23 { +// err = fmt.Errorf("Wrong cumulative metric's FirstHourOfDay option value: %d", mf.FirstHourOfDay) +// return +// } + +// if mf.LastDayOfMonth < 0 || mf.LastDayOfMonth > 28 { +// err = fmt.Errorf("Wrong cumulative metric's LastDayOfMonth option value: %d", mf.LastDayOfMonth) +// return +// } +// } + +// // Корректируем, ВСЕГДА на первую и последнюю секунду суток +// since := timeutil.FirstSecondInPeriod(req.Since, "d") +// until := timeutil.LastSecondInPeriod(req.Until, "d") + +// fmt.Printf(`SelectMetricsDataInParallel2: { +// Since: %s +// Until: %s +// GroupBy: %s +// } +// `, since, until, req.GroupBy) + +// // Важно вирутальная метрика зависит от реальных, которые к объекту могут быть +// // привязаны или нет. Если нет - не возвращать. + +// var ( +// unexpectedErrors []error +// errorMutex sync.Mutex + +// wg sync.WaitGroup + +// resultMaps = MetricsDataResultMaps{ +// AggregatedCumulative: make(map[int64][]CalculatedAggregatedMeasure), +// AggregatedByFuncInstant: make(map[int64][]AggregatedByFuncInstantMeasure), +// Cumulative: make(map[int64][]CalculatedMeasure), +// Instant: make(map[int64][]CalculatedMeasure), +// } +// ) + +// wg.Add(len(req.CumulativeMetrics) + len(req.InstantMetrics)) + +// if req.GroupBy != "" { +// // AGGREGATED + +// // CUMULATIVE +// for _, metricFilter := range req.CumulativeMetrics { +// go func(mf CumulativeMetricFilter) { +// // У каждой метрики индивидуальные настройки, +// // ибо метрики принадлежат разным объектам!!! +// metricSince := since +// metricUntil := until + +// if mf.FirstHourOfDay > 0 { +// metricSince = since.Add(time.Duration(mf.FirstHourOfDay) * time.Hour) +// metricUntil = until.Add(time.Duration(mf.FirstHourOfDay) * time.Hour) +// } + +// result, err := s.listAggregatedCumulativeMeasures(AggregatedMeasuresFilter{ +// MetricID: mf.MetricID, +// Since: metricSince.Unix(), +// Until: metricUntil.Unix(), +// GroupBy: req.GroupBy, +// LastDayOfMonth: mf.LastDayOfMonth, +// FirstHourOfDay: mf.FirstHourOfDay, +// }) +// if err != nil { +// err = fmt.Errorf("ListAggregatedCumulativeMeasures: %s", err) + +// errorMutex.Lock() +// unexpectedErrors = append(unexpectedErrors, err) +// errorMutex.Unlock() +// } else { +// errorMutex.Lock() +// resultMaps.AggregatedCumulative[mf.MetricID] = result +// errorMutex.Unlock() +// } + +// wg.Done() +// }(metricFilter) +// } + +// // INSTANT +// for _, metricFilter := range req.InstantMetrics { +// go func(mf InstantMetricFilter) { +// // У каждой метрики индивидуальные настройки, +// // ибо метрики принадлежат разным объектам!!! +// metricSince := since +// metricUntil := until + +// if mf.FirstHourOfDay > 0 { +// metricSince = since.Add(time.Duration(mf.FirstHourOfDay) * time.Hour) +// metricUntil = until.Add(time.Duration(mf.FirstHourOfDay) * time.Hour) +// } + +// result, err := s.listAggregatedByFuncsInstantMeasures(AggregatedByFuncInstantMeasuresFilter{ +// MetricID: mf.MetricID, +// Since: metricSince.Unix(), +// Until: metricUntil.Unix(), +// GroupBy: req.GroupBy, +// LastDayOfMonth: mf.LastDayOfMonth, +// FirstHourOfDay: mf.FirstHourOfDay, +// Flags: mf.AggregateFuncs, // ФЛАГИ !!! +// }) +// if err != nil { +// err = fmt.Errorf("ListAggregatedByFuncInstantMeasures: %s", err) + +// errorMutex.Lock() +// unexpectedErrors = append(unexpectedErrors, err) +// errorMutex.Unlock() +// } else { +// errorMutex.Lock() +// resultMaps.AggregatedByFuncInstant[mf.MetricID] = result +// errorMutex.Unlock() +// } + +// wg.Done() +// }(metricFilter) +// } + +// } else { +// // NON AGGREGATED + +// // CUMULATIVE +// for _, metricFilter := range req.CumulativeMetrics { +// go func(mf CumulativeMetricFilter) { +// // У каждой метрики индивидуальные настройки, +// // ибо метрики принадлежат разным объектам!!! +// metricSince := since +// metricUntil := until + +// if mf.FirstHourOfDay > 0 { +// metricSince = since.Add(time.Duration(mf.FirstHourOfDay) * time.Hour) +// metricUntil = until.Add(time.Duration(mf.FirstHourOfDay) * time.Hour) +// } + +// result, err := s.listCumulativeMeasures(MeasuresFilter{ +// MetricID: mf.MetricID, +// Since: metricSince.Unix(), +// Until: metricUntil.Unix(), +// }) +// if err != nil { +// err = fmt.Errorf("ListCumulativeMeasures: %s", err) + +// errorMutex.Lock() +// unexpectedErrors = append(unexpectedErrors, err) +// errorMutex.Unlock() +// } else { +// errorMutex.Lock() +// resultMaps.Cumulative[mf.MetricID] = result +// errorMutex.Unlock() +// } +// wg.Done() +// }(metricFilter) +// } + +// // INSTANT +// for _, metricFilter := range req.InstantMetrics { +// go func(mf InstantMetricFilter) { +// // У каждой метрики индивидуальные настройки, +// // ибо метрики принадлежат разным объектам!!! +// metricSince := since +// metricUntil := until + +// if mf.FirstHourOfDay > 0 { +// metricSince = since.Add(time.Duration(mf.FirstHourOfDay) * time.Hour) +// metricUntil = until.Add(time.Duration(mf.FirstHourOfDay) * time.Hour) +// } + +// result, err := s.listInstantMeasures(MeasuresFilter{ +// MetricID: mf.MetricID, +// Since: metricSince.Unix(), +// Until: metricUntil.Unix(), +// }) +// if err != nil { +// err = fmt.Errorf("ListInstantMeasures: %s", err) + +// errorMutex.Lock() +// unexpectedErrors = append(unexpectedErrors, err) +// errorMutex.Unlock() +// } else { +// errorMutex.Lock() +// resultMaps.Instant[mf.MetricID] = result +// errorMutex.Unlock() +// } +// wg.Done() +// }(metricFilter) +// } +// } + +// wg.Wait() + +// // Проверяем что все запросы завершились удачно +// if len(unexpectedErrors) > 0 { +// // Нумеруем сообщения об ошибках и склеиваем в одно большое сообщение +// var errorStrings []string + +// for idx, err := range unexpectedErrors { +// errorStrings = append(errorStrings, fmt.Sprintf("\n#%d %s", idx+1, err)) +// } + +// err = fmt.Errorf("%d errors occured:%s", len(unexpectedErrors), +// strings.Join(errorStrings, "")) +// return +// } + +// return resultMaps, nil +// } diff --git a/mqe/period_calculator.go b/mqe/period_calculator.go new file mode 100644 index 0000000..5e91ca7 --- /dev/null +++ b/mqe/period_calculator.go @@ -0,0 +1,172 @@ +package mqe + +import ( + "fmt" + "time" + + "gordenko.dev/dima/qb/timeutil" +) + +type PeriodCalculatorOptions struct { + GroupBy GroupBy + LastDayOfMonth int + FirstHourOfDay int + Since time.Time + Until time.Time +} + +func NewPeriodCalculator(opt PeriodCalculatorOptions) PeriodCalculator { + switch opt.GroupBy { + case ByHour, ByDay, ByMonth: + // pass + + default: + panic(fmt.Sprintf("bug: unknown groupBy %q", opt.GroupBy)) + } + + s := PeriodCalculator{ + groupBy: opt.GroupBy, + lastDayOfMonth: opt.LastDayOfMonth, + firstHourOfDay: opt.FirstHourOfDay, + } + s.sincePeriod = s.TimeToPeriod(opt.Since) + s.untilPeriod = s.TimeToPeriod(opt.Until) + + return s +} + +type PeriodCalculator struct { + groupBy GroupBy + lastDayOfMonth int + firstHourOfDay int + sincePeriod time.Time + untilPeriod time.Time +} + +func (s PeriodCalculator) IsExtendedSincePeriod(period time.Time) bool { + return period.Before(s.sincePeriod) +} + +func (s PeriodCalculator) IsExtendedUntilPeriod(period time.Time) bool { + return period.After(s.untilPeriod) +} + +func (s PeriodCalculator) NextPeriod(period time.Time) time.Time { + switch s.groupBy { + case ByHour: + return period.Add(time.Hour) + + case ByDay: + return period.AddDate(0, 0, 1) + + case ByMonth: + return period.AddDate(0, 1, 0) + + default: + panic(fmt.Sprintf("bug: unknown groupBy %q", s.groupBy)) + } +} + +func (s PeriodCalculator) IsPeriodCorrectEnds(period time.Time, lastMeasureTime int64) bool { + // по умолчанию + tm := timeutil.LastSecondInPeriod(period, string(s.groupBy)) + + switch s.groupBy { + case ByDay: + if s.firstHourOfDay > 0 { + // Было 23:59, стало - 01:59 + tm = tm.Add(time.Duration(s.firstHourOfDay) * time.Hour) + } + + case ByMonth: + if s.lastDayOfMonth == 0 { + if s.firstHourOfDay > 0 { + // Было 23:59, стало - 01:59 + tm = tm.Add(time.Duration(s.firstHourOfDay) * time.Hour) + } + } else { + if s.firstHourOfDay > 0 { + // last day 28, first hour 2 превращается в day 29 01:59:59 + tm = time.Date( + tm.Year(), + tm.Month(), + s.lastDayOfMonth+1, + s.firstHourOfDay-1, + 59, + 59, + 0, + tm.Location(), + ) + } else { + // last day 23:59:59 + tm = time.Date( + tm.Year(), + tm.Month(), + s.lastDayOfMonth, + 23, + 59, + 59, + 0, + tm.Location(), + ) + } + } + } + + timestamp := tm.Add(-179 * time.Second).Unix() + + if lastMeasureTime >= timestamp && lastMeasureTime <= tm.Unix() { + return true + } + //fmt.Printf("Ends: lastMeasureTime=%s, timetsamp=%s\n", time.Unix(lastMeasureTime, 0), time.Unix(timestamp, 0)) + return false +} + +func (s PeriodCalculator) TimeToPeriod(tm time.Time) (period time.Time) { + switch s.groupBy { + case ByDay: + if s.firstHourOfDay > 0 { + if tm.Hour() < s.firstHourOfDay { + tm = tm.AddDate(0, 0, -1) + } + } + + case ByMonth: + if s.lastDayOfMonth > 0 { + if s.lastDayOfMonth > 28 { + // На всякий случай добавляем защиту от дурака, ибо модификация даты + // в запросе (+1 месяц) для 29, 30 и 31 чисел сделает запрос неверным. + panic(fmt.Sprintf("LastDayOfMonth: %d, max allowed value is 28", s.lastDayOfMonth)) + } + + if s.firstHourOfDay > 0 { + if tm.Hour() >= s.firstHourOfDay { + if tm.Day() > s.lastDayOfMonth { + // add 1 month + tm = tm.AddDate(0, 1, 0) + } + } else { + // firstHourOfDay показывает что текущий день еще не наступил, + // поэтому day - 1 + if (tm.Day() - 1) <= s.lastDayOfMonth { + tm = tm.AddDate(0, 0, -1) + } else { + // - 1 day, + 1 month + tm = tm.AddDate(0, 0, -1).AddDate(0, 1, 0) + } + } + } else { + if tm.Day() > s.lastDayOfMonth { + tm = tm.AddDate(0, 1, 0) + } + } + } else { + if s.firstHourOfDay > 0 { + if tm.Hour() < s.firstHourOfDay { + tm = tm.AddDate(0, 0, -1) + } + } + } + } + return timeutil.FirstSecondInPeriod(tm, string(s.groupBy)) +} diff --git a/mqe/period_producers.go b/mqe/period_producers.go new file mode 100644 index 0000000..9bad114 --- /dev/null +++ b/mqe/period_producers.go @@ -0,0 +1,168 @@ +package mqe + +import ( + "database/sql" + "fmt" + "time" +) + +type periodProducer interface { + Next() (time.Time, PeriodBound, PeriodBound, bool, error) + Last() (PeriodBound, bool, error) +} + +type PeriodBound struct { + Time int64 + Value float64 +} + +type PeriodProducer struct { + metricID int64 + parsePeriodLayout string + location *time.Location + until int64 + corrections []_f64Correction + rows *sql.Rows + tx *sql.Tx +} + +type PeriodProducerOptions struct { + MetricID int64 + ParsePeriodLayout string + Location *time.Location + Until int64 + Corrections []_f64Correction + Rows *sql.Rows + Tx *sql.Tx +} + +func NewPeriodProducer(opt PeriodProducerOptions) *PeriodProducer { + if opt.Location == nil { + panic("Location option is required") + } + + if opt.MetricID == 0 { + panic("MetricID option is required") + } + + if opt.ParsePeriodLayout == "" { + panic("ParsePeriodLayout option is required") + } + + if opt.Until == 0 { + panic("Until option is required") + } + + if opt.Rows == nil { + panic("Rows option is required") + } + + if opt.Tx == nil { + panic("Tx option is required") + } + + s := new(PeriodProducer) + s.metricID = opt.MetricID + s.parsePeriodLayout = opt.ParsePeriodLayout + s.location = opt.Location + s.until = opt.Until + s.corrections = opt.Corrections + s.rows = opt.Rows + s.tx = opt.Tx + return s +} + +func (s *PeriodProducer) Next() (period time.Time, first PeriodBound, last PeriodBound, found bool, err error) { + var frameStr string + + hasNext := s.rows.Next() + if hasNext { + err = s.rows.Scan(&frameStr, &first.Time, &first.Value, &last.Time, &last.Value) + if err != nil { + return + } + + first.Value = applyCorrectionsToMeasure(s.corrections, first.Time, first.Value) + last.Value = applyCorrectionsToMeasure(s.corrections, last.Time, last.Value) + + period, err = time.ParseInLocation(s.parsePeriodLayout, frameStr, s.location) + if err != nil { + err = fmt.Errorf("time.ParseInLocation: %s; layout=%q; str=%q", + err, s.parsePeriodLayout, frameStr) + return + } + + found = true + } else { + err = s.rows.Err() + } + return +} + +func (s *PeriodProducer) Last() (next PeriodBound, found bool, err error) { + // Находим следующее показание метрики. + err = s.tx.QueryRow(` + SELECT tm, value, true + FROM f64 + WHERE metricID=? AND tm > ? + ORDER BY tm ASC + LIMIT 1`, + s.metricID, s.until).Scan(&next.Time, &next.Value, &found) + + if err != nil { + if err != sql.ErrNoRows { + return + } + // Нет записей + err = nil + } + + next.Value = applyCorrectionsToMeasure(s.corrections, next.Time, next.Value) + return +} + +// ДЛЯ ТЕСТИРОВАНИЯ + +type stubPeriod struct { + Period time.Time + First PeriodBound + Last PeriodBound +} + +type StubPeriodProducerOptions struct { + Last *PeriodBound + Periods []stubPeriod +} + +func NewStubPeriodProducer(opt StubPeriodProducerOptions) *StubPeriodProducer { + s := new(StubPeriodProducer) + s.last = opt.Last + s.periods = opt.Periods + return s +} + +type StubPeriodProducer struct { + last *PeriodBound // для отдельного запроса + periods []stubPeriod + idx int +} + +func (s *StubPeriodProducer) Next() (period time.Time, first PeriodBound, last PeriodBound, found bool, err error) { + if s.idx < len(s.periods) { + p := s.periods[s.idx] + s.idx++ + period = p.Period + first = p.First + last = p.Last + found = true + } + return +} + +func (s *StubPeriodProducer) Last() (next PeriodBound, found bool, err error) { + if s.last != nil { + next = *s.last + found = true + } + return +} diff --git a/mqe/raw.go b/mqe/raw.go new file mode 100644 index 0000000..d04a37f --- /dev/null +++ b/mqe/raw.go @@ -0,0 +1,81 @@ +package mqe + +import ( + "database/sql" + + "gordenko.dev/dima/qb" +) + +type RawMeasure struct { + Time int64 `json:"t"` + Value float64 `json:"v"` +} + +type RawMeasuresFilter struct { + MetricID int64 `json:"metricID"` + MetricType qb.MetricType `json:"metricType"` + Since int64 `json:"since"` // уже учтен firstHourOfDay + Until int64 `json:"until"` // уже учтен firstHourOfDay +} + +// ListRawMeasures - cписок показаний мгновенных метрик (Температура, Давление, Расход) +// за за интервал без группировки +func (s *MeasureQueryEngine) ListRawMeasures(req RawMeasuresFilter) (_ []RawMeasure, err error) { + tx, err := s.db.Driver().Begin() + if err != nil { + return + } + defer tx.Rollback() + + rows, err := tx.Query(` + SELECT tm, value + FROM f64 + WHERE metricID=? AND tm BETWEEN ? AND ? + ORDER BY tm ASC`, + req.MetricID, req.Since, req.Until) + if err != nil { + if err == sql.ErrNoRows { + err = nil + } + return + } + defer rows.Close() + + var result []RawMeasure + + for rows.Next() { + var ( + tm int64 + value float64 + ) + err = rows.Scan(&tm, &value) + if err != nil { + return + } + result = append(result, RawMeasure{ + Time: tm, + Value: value, + }) + } + + if err = rows.Err(); err != nil { + return + } + + if len(result) == 0 { + return + } + + if req.MetricType == qb.Cumulative { + var corrections []_f64Correction + corrections, err = listF64CorrectionsTx(tx, req.MetricID) + if err != nil { + return + } + + if len(corrections) > 0 { + applyCorrectionsToRawMeasures(corrections, result) + } + } + return result, nil +} diff --git a/mqe/readings.go b/mqe/readings.go new file mode 100644 index 0000000..9fbf85a --- /dev/null +++ b/mqe/readings.go @@ -0,0 +1,179 @@ +package mqe + +import ( + "database/sql" + "fmt" + "sync" + "time" + + "gordenko.dev/dima/pretty" + "gordenko.dev/dima/qb/timeutil" +) + +//////////////////////////// + +type MetricMeasure struct { + Time int64 `json:"t"` + Value float64 `json:"v"` +} + +type MetricError struct { + Time int64 `json:"time"` + Code int64 `json:"code"` +} + +type ReadingsFilter struct { + Since string `json:"since"` + Until string `json:"until"` + MetricIDs []int64 +} + +type MetricReadings struct { + MetricID int64 + Measures []MetricMeasure + Errors []MetricError +} + +// SelectReadingsInParallel - метод достает из БД данные всех метрик, которые связаны +// с объектом. Учитываются настройки объекта (LastDayOfMonth, FirstHourOfDay), а также +// фильтры выбранные пользователем на странице объекта (Since, Until, GroupBy). +// ВАЖНО! +// Запросы по каждой метрике отправляются в БД в отдельной горутине, что позволяет +// в разы ускорить получение данных по объекту. +func (s *MeasureQueryEngine) SelectReadingsInParallel(req ReadingsFilter) (result []MetricReadings, err error) { + pretty.PPrintln("SelectReadingsInParallel", req) + + if len(req.MetricIDs) == 0 { + return + } + + if req.Since == "" { + err = fix.Field(EmptyValue, "since") + return + } + + if req.Until == "" { + err = fix.Field(EmptyValue, "until") + return + } + + // ВАЖНО! + // Если в строке нет таймзоны - Parse функция возвращает время в UTC. + // Поэтому нужно использовать ParseInLocation! + + since, err := time.ParseInLocation("2006-01-02", req.Since, s.location) + if err != nil { + err = fix.Field(WrongValue, "since") + return + } + + until, err := time.ParseInLocation("2006-01-02", req.Until, s.location) + if err != nil { + err = fix.Field(WrongValue, "until") + return + } + + // Корректируем, ВСЕГДА на последнюю секунду суток + until = timeutil.LastSecondInPeriod(until, string(ByDay)) + + fmt.Printf(`SelectReadingsInParallel: { + Since: %s + Until: %s + MetricIDs: %v +} +`, since, until, req.MetricIDs) + // FIX проверить как добавляются часы в DST часовых поясах + // Для req.FirstHourOfDay=4 since должен быть 4:00:00, until должен быть + // 3:59:59 следующего дня + + sinceUnixtime := since.Unix() + untilUnixtime := until.Unix() + + if sinceUnixtime >= untilUnixtime { + err = fix.Error(WrongDateRange) + return + } + + // Важно вирутальная метрика зависит от реальных, которые к объекту могут быть + // привязаны или нет. Если нет - не возвращать. + + var ( + tasks []*MetricReadingsTask + wg = new(sync.WaitGroup) + ) + + for _, metricID := range req.MetricIDs { + task := &MetricReadingsTask{ + In: MetricMeasuresFilter{ + MetricID: metricID, + Since: sinceUnixtime, + Until: untilUnixtime, + }, + WaitGroup: wg, + } + + wg.Add(1) + s.metricReadingsCh <- task + tasks = append(tasks, task) + } + + wg.Wait() + + for _, task := range tasks { + if task.Err != nil { + err = fmt.Errorf("get data for the metric %d: %s", task.In.MetricID, task.Err) + return + } + result = append(result, task.Result) + } + return +} + +// ListInstantMeasures - cписок показаний для юнитов без накопительного итога +// (например, Давление) +func (s *MeasureQueryEngine) listMetricReadings(req MetricMeasuresFilter) (result MetricReadings, err error) { + result.MetricID = req.MetricID + // buf, _ := json.MarshalIndent(req, "", " ") + // fmt.Printf("listInstantMeasures: %s\n\n", buf) + + tx, err := s.db.Driver().Begin() + if err != nil { + return + } + defer tx.Rollback() + + rows, err := tx.Query(` + SELECT tm, value + FROM f64 + WHERE metricID=? AND tm BETWEEN ? AND ? + ORDER BY tm ASC`, + req.MetricID, req.Since, req.Until) + if err != nil { + if err == sql.ErrNoRows { + err = nil + } + return + } + defer rows.Close() + + for rows.Next() { + var measure MetricMeasure + err = rows.Scan(&measure.Time, &measure.Value) + if err != nil { + return + } + // fix - для cumulative метрик учет поправок + result.Measures = append(result.Measures, measure) + } + + if err = rows.Err(); err != nil { + return + } + + //for _, m := range result { + // fmt.Printf("raw: %d\t%f\n", m.Time, m.Value) + //} + //fmt.Printf("listInstantMeasures (qty): %d\n", len(result)) + + return +} diff --git a/mqe/split.go b/mqe/split.go new file mode 100644 index 0000000..87c7df9 --- /dev/null +++ b/mqe/split.go @@ -0,0 +1,570 @@ +package mqe + +import ( + "fmt" + "time" +) + +func SplitByPeriods(producer periodProducer, calc PeriodCalculator) (result []AggregatedMeasure, err error) { + var ( + isOpened bool + openedPeriod time.Time + since int64 + sinceValue float64 + until int64 + untilValue float64 + + // + period time.Time + first, last PeriodBound + found bool + ) + + for { + period, first, last, found, err = producer.Next() + if err != nil { + return + } + + if !found { + break + } + + if isOpened { + if calc.IsExtendedSincePeriod(period) { + err = fmt.Errorf("extended period while already opened") + return + } else if calc.IsExtendedUntilPeriod(period) { + result = append(result, AggregatedMeasure{ + Period: openedPeriod.Unix(), + Since: since, + Until: first.Time, + Values: []float64{ + first.Value, + first.Value - sinceValue, + }, + }) + // Последний период успешно закрыли данными из Extended Until, поэтому выходим + return + } else { + if period.Equal(openedPeriod) { + // + + // Данная ситуация будет когда предыдущий период был корректно + // закрыт за 3 минуты до конца. После этого сразу открылся + // новый период. А запись для нового периода получена только + // сейчас. + if calc.IsPeriodCorrectEnds(period, last.Time) { + // CLOSE + result = append(result, AggregatedMeasure{ + Period: openedPeriod.Unix(), + Since: since, + Until: last.Time, + Values: []float64{ + last.Value, + last.Value - sinceValue, + }, + }) + + // OPEN PERIOD (extended since, last 3 min closed) + openedPeriod = calc.NextPeriod(period) + if calc.IsExtendedUntilPeriod(openedPeriod) { + return + } + since = last.Time + sinceValue = last.Value + until = 0 + untilValue = 0 + } else { + // Период закроем где-то в будущем, на всякий случай запомнили последнее + // показание внутри периода + until = last.Time + untilValue = last.Value + } + } else { + // + + // Периоды не совпадают - смело закрываем + // Возможные ситуации: + // - был открыт мартовский период, а текущий период - апрель + // - + result = append(result, AggregatedMeasure{ + Period: openedPeriod.Unix(), + Since: since, + Until: first.Time, + Values: []float64{ + first.Value, + first.Value - sinceValue, + }, + }) + + if first.Time != last.Time { + // + + openedPeriod = period + since = first.Time + sinceValue = first.Value + until = 0 + untilValue = 0 + + if calc.IsPeriodCorrectEnds(period, last.Time) { + // Закрываем только что открытый период + result = append(result, AggregatedMeasure{ + Period: openedPeriod.Unix(), + Since: since, + Until: last.Time, + Values: []float64{ + last.Value, + last.Value - sinceValue, + }, + }) + + openedPeriod = calc.NextPeriod(period) + if calc.IsExtendedUntilPeriod(openedPeriod) { + return + } + since = last.Time + sinceValue = last.Value + until = 0 + untilValue = 0 + } else { + // Период закроем где-то в будущем, на всякий случай запомнили последнее + // показание внутри периода + until = last.Time + untilValue = last.Value + } + } else { + // + + // first == last + if calc.IsPeriodCorrectEnds(period, last.Time) { + // Допустим закрыли предыдущий период 30 апреля в 23:58, новый стартуем за май. + openedPeriod = calc.NextPeriod(period) + if calc.IsExtendedUntilPeriod(openedPeriod) { + return + } + } else { + // Допустим закрыли предыдущий период 15 апреля, новый стартуем за апрель. + openedPeriod = period + } + since = last.Time + sinceValue = last.Value + until = 0 + untilValue = 0 + } + } + } + } else { + if calc.IsExtendedSincePeriod(period) { + // Если последние 3 минуты + if calc.IsPeriodCorrectEnds(period, last.Time) { + // OPEN PERIOD (extended since, last 3 min closed) + isOpened = true + openedPeriod = calc.NextPeriod(period) + since = last.Time + sinceValue = last.Value + until = 0 + untilValue = 0 + } + } else if calc.IsExtendedUntilPeriod(period) { + // Ничего не нашли, пустой результат + return + } else { + // OPEN PERIOD (regular first) + isOpened = true + openedPeriod = period + since = first.Time + sinceValue = first.Value + until = last.Time + untilValue = last.Value + + if calc.IsPeriodCorrectEnds(openedPeriod, last.Time) { + // Закрываем только что открытый период + result = append(result, AggregatedMeasure{ + Period: openedPeriod.Unix(), + Since: since, + Until: last.Time, + Values: []float64{ + last.Value, + last.Value - sinceValue, + }, + }) + + openedPeriod = calc.NextPeriod(period) + if calc.IsExtendedUntilPeriod(openedPeriod) { + return + } + since = last.Time + sinceValue = last.Value + until = 0 + untilValue = 0 + } + } + } + } + + // Доп запрос. + if openedPeriod.IsZero() { + // Открытого периода не будет если вообще не нашли записей + return + } + + next, found, err := producer.Last() + if err != nil { + return + } + + if found { + result = append(result, AggregatedMeasure{ + Period: openedPeriod.Unix(), + Since: since, + Until: next.Time, + Values: []float64{ + next.Value, + next.Value - sinceValue, + }, + }) + } else { + if until > 0 { + result = append(result, AggregatedMeasure{ + Period: openedPeriod.Unix(), + Since: since, + Until: until, + Values: []float64{ + untilValue, + untilValue - sinceValue, + }, + }) + } + } + return +} + +// Period - это начало периода, например для дней - это 0ч 0м 0с +// lastTm предыдущего периода всегда будет меньше current period +// since - это всегда сдвинутая метка времени (+ FirstHourOfDay * 3600) + +// При FirstHourOfDay = 3 и группировке по дням, будет такая ситуация. +// Period будет указывать на 1 марта 00:00:00, а since на 1 марта 02:00:00, +// а lastTm предыдущего периода на 1 марта 01:59:00. +// Поэтому нельзя сравнивать текущий period и lastTm предыдущего периода. +// Преобразовать lastTm к period непонятно как если нужно учитывать LastDayOfMonth. +// Пример: 28 марта 00ч стало 1 апреля 00ч, а 27 марта 23:59 так и осталось. +// Можно вычислить endPeriodTime для previousPeriodEndTime, но текущий период может +// быть не строго следующим после предыдущего (например, июнь после марта). +// Правильное решение - берем firstTm текущего периода и вычисляем его реальное +// календарное начало. Например, FirstHourOfDay = 3, LastDayOfMonth = 28, +// firstTm = 1 мая. Календарное начало - 28 апреля 03:00:00. И уже с этим значением +// сравниваем previousPeriodEndTime. +// Но можно сравнивать since c lastTm и until c firstTm для определения +// extended периодов. +/* +func periodToCalendarTime(period time.Time, groupBy string, lastDayOfMonth, firstHourOfDay int) time.Time { + switch groupBy { + case "d": + if firstHourOfDay == 0 { + return period + } + // Было 0ч, стало - 2ч + return period.Add(time.Duration(firstHourOfDay) * time.Hour) + + case "m": + if lastDayOfMonth == 0 { + if firstHourOfDay == 0 { + return period + } + // Было 0ч, стало - 2ч + return period.Add(time.Duration(firstHourOfDay) * time.Hour) + } else { + // текущий период начинается в прошлом месяце + tm := period.AddDate(0, -1, 0) + return time.Date( + tm.Year(), + tm.Month(), + lastDayOfMonth+1, + firstHourOfDay, + 0, + 0, + 0, + tm.Location(), + ) + } + default: + return period + } +} +*/ +//var threeMinutes int64 = 3 * 60 + +/* +type readOptions struct { + MetricID int64 // только для readAggregatedMeasuresCanEndsInFuture + LastDayOfMonth int + FirstHourOfDay int + GroupBy string + Since int64 + Until int64 + ParsePeriodLayout string +} +*/ +// Читает из sql.Rows показания и строит периоды +/* +func (s *MeasureQueryEngine) readAggregatedMeasuresCanEndsInFuture(tx *sql.Tx, rows *sql.Rows, opt readOptions) (result []CalculatedAggregatedMeasure, err error) { + var ( + openPeriod time.Time + since int64 + sinceValue float64 + until int64 + untilValue float64 + // + frameStr string + firstTm int64 + firstValue float64 + lastTm int64 + lastValue float64 + period time.Time + ) + + for rows.Next() { + err = rows.Scan(&frameStr, &firstTm, &firstValue, &lastTm, &lastValue) + if err != nil { + return + } + + period, err = time.ParseInLocation(opt.ParsePeriodLayout, frameStr, s.location) + if err != nil { + err = fmt.Errorf("time.ParseInLocation: %s; layout=%q; str=%q", + err, opt.ParsePeriodLayout, frameStr) + return + } + + //fmt.Printf("%s s: %s, %v; u: %s, %v\n", frameStr, time.Unix(firstTm, 0), firstValue, time.Unix(lastTm, 0), lastValue) + + // Читаем записи из БД. Одна запись - один период. Возможны 3 вида периодов: + // 1. extended since - период длиной до 3 минут перед since. Оптимизация, чтобы + // не отправлять дополнительный запрос. + // 2. обычный период. + // 3. extended until - период длиной до 1 часа после until. Оптимизация, чтобы + // не отправлять дополнительный запрос (в большинстве случаев). + // + // ВАЖНО! + // Любой период (1, 2 или 3) может встретится первым. Определить что это + // первая запись модно простой проверкой openPeriod.IsZero() + + if lastTm < opt.Since { + // 1. extended since period + + if isPeriodCorrectEnds(period, opt.GroupBy, opt.LastDayOfMonth, opt.FirstHourOfDay, lastTm) { + openPeriod = nextPeriod(period, opt.GroupBy) + since = lastTm + sinceValue = lastValue + } + continue + } else if firstTm > opt.Until { + // 3. extended until period + + if openPeriod.IsZero() { + // Нет показаний кроме extended until, поэтому выходим + return + } + + // закрываем открытый период + result = append(result, CalculatedAggregatedMeasure{ + Period: openPeriod.Unix(), + Since: since, + Until: firstTm, + Value: firstValue, + Total: firstValue - sinceValue, + }) + return + } + + // 2. обычный период + + if openPeriod.IsZero() { + // это первая запись (extended since period не найден) + openPeriod = period + since = firstTm + sinceValue = firstValue + } else { + if !period.Equal(openPeriod) { + //fmt.Printf("\nperiod %s != openPeriod %s\n", period, openPeriod) + // Пример: открытый период - март. В period - апрель. + // закрываем открытый период + result = append(result, CalculatedAggregatedMeasure{ + Period: openPeriod.Unix(), + Since: since, + Until: firstTm, + Value: firstValue, + Total: firstValue - sinceValue, + }) + + // И сразу открываем новый период + openPeriod = period + since = firstTm + sinceValue = firstValue + } + // Периоды совпадают. Когда такая ситуация возможна? + // Например, на предыдущей итерации был корректно закрыт (57-59 минуты) + // прошлый период и сразу открыт новый (текущий). + } + + if isPeriodCorrectEnds(openPeriod, opt.GroupBy, opt.LastDayOfMonth, opt.FirstHourOfDay, lastTm) { + // fmt.Printf("\nCORRECT END period %s\n", openPeriod) + // закрываем открытый период + result = append(result, CalculatedAggregatedMeasure{ + Period: openPeriod.Unix(), + Since: since, + Until: lastTm, + Value: lastValue, + Total: lastValue - sinceValue, + }) + + // Открываем новый период + openPeriod = nextPeriod(openPeriod, opt.GroupBy) + since = lastTm + sinceValue = lastValue + } else { + // На последних минутах периода (57-59) не было показания, поэтому + // запоминаем конец периода какой есть. Если конец периода в будущем + // не будет найден - закроем период по текущему последнему показанию. + until = lastTm + untilValue = lastValue + } + + // Валидируем новый период. + // Зачем? + // Пример: юзер задал opt.Until - 31 марта 23:59:59. Только что корректно + // закрылся март показанием от 31 марта 23:57:00. И сразу открылся новый + // период за апрель. + // Понять что закрыли последний период из запрошенных юзером просто - + // вычисляем календарное начало нового периода и сравниваем с opt.Until. + mustLessThanUntil := periodToCalendarTime(openPeriod, opt.GroupBy, + opt.LastDayOfMonth, opt.FirstHourOfDay) + + if mustLessThanUntil.Unix() > opt.Until { + // Новый период вышел за opt.Until, а это означает что мы корректно + // закрыли последний период из запрошенных юзером. Bыходим. + return + } + } + + // Цикл завершен, а из метода мы не вышли, а это означает одно - есть незакрытый + // период. Extended until не найден. Значит нужно отправить дополнительный запрос + // к БД. + var ( + isNextValueFound bool + nextTm int64 + nextValue float64 + ) + + // Находим следующее показание метрики. + err = tx.QueryRow(` + SELECT tm, value, true + FROM f64 + WHERE metricID=? AND tm > ? + ORDER BY tm ASC + LIMIT 1`, + opt.MetricID, opt.Until).Scan(&nextTm, &nextValue, &isNextValueFound) + + if err != nil { + if err != sql.ErrNoRows { + return + } + // Нет записей + err = nil + } + + if isNextValueFound { + // Найдено какое-то показание в будущем. + result = append(result, CalculatedAggregatedMeasure{ + Period: openPeriod.Unix(), + Since: since, + Until: nextTm, + Value: nextValue, + Total: nextValue - sinceValue, + }) + } else { + // Закрываем период самым последним показанием внутри периода + // (НЕ 57-59 минуты). + result = append(result, CalculatedAggregatedMeasure{ + Period: openPeriod.Unix(), + Since: since, + Until: until, + Value: untilValue, + Total: untilValue - sinceValue, + }) + } + return +} +*/ + +/* +// Читает из sql.Rows показания и строит периоды +func (s *MeasureQueryEngine) readAggregatedMeasuresEndsInsidePeriod(tx *sql.Tx, rows *sql.Rows, opt readOptions) (result []CalculatedAggregatedMeasure, err error) { + var ( + previousPeriodEndTime int64 + previousPeriodEndValue float64 + // + frameStr string + firstTm int64 + firstValue float64 + lastTm int64 + lastValue float64 + period time.Time + ) + + for rows.Next() { + err = rows.Scan(&frameStr, &firstTm, &firstValue, &lastTm, &lastValue) + if err != nil { + return + } + + period, err = time.ParseInLocation(opt.ParsePeriodLayout, frameStr, s.location) + if err != nil { + err = fmt.Errorf("time.ParseInLocation: %s; layout=%q; str=%q", + err, opt.ParsePeriodLayout, frameStr) + return + } + + //fmt.Printf("%s s: %s, %v; u: %s, %v\n", frameStr, time.Unix(firstTm, 0), firstValue, time.Unix(lastTm, 0), lastValue) + + if lastTm < opt.Since { + // extended since period + previousPeriodEndTime = lastTm + previousPeriodEndValue = lastValue + continue + } else if firstTm > opt.Until { + // extended until period + return + } + + // целевой период. В будущем конец периода не ищем, поэтому закрываем сразу. + + // Вычисляем реальное календарное начало периода. Например, period - 1 мая 00:00:00, + // FirstHourOfDay = 3, LastDayOfMonth = 28. Календарное начало - 29 апреля 03:00:00. + periodCalendarTime := periodToCalendarTime(period, opt.GroupBy, opt.LastDayOfMonth, opt.FirstHourOfDay) + + minAge := periodCalendarTime.Add(-3 * time.Minute).Unix() + + if previousPeriodEndTime > 0 && previousPeriodEndTime >= minAge { + result = append(result, CalculatedAggregatedMeasure{ + Period: period.Unix(), + Since: previousPeriodEndTime, + Until: lastTm, + Value: lastValue, + Total: lastValue - previousPeriodEndValue, + }) + } else { + result = append(result, CalculatedAggregatedMeasure{ + Period: period.Unix(), + Since: firstTm, + Until: lastTm, + Value: lastValue, + Total: lastValue - firstValue, + }) + } + + previousPeriodEndTime = lastTm + previousPeriodEndValue = lastValue + } + + return +} +*/ diff --git a/mqe/total.go b/mqe/total.go new file mode 100644 index 0000000..914c66b --- /dev/null +++ b/mqe/total.go @@ -0,0 +1,141 @@ +package mqe + +import ( + "fmt" + "strings" + "sync" + "time" + + "gordenko.dev/dima/qb/timeutil" +) + +// TOTALS + +type TotalMetricSpec struct { + MetricID int64 + LastDayOfMonth int + FirstHourOfDay int +} + +// LastDayOfMonth и FirstHourOfDay нет, ибо у каждой метрики свои настройки +type RangeTotalsFilter struct { + Since string + Until string + // Если учитывать не нужно - программа уровнем выше может передать нули. + // FIX у каждой метрики свои настройки + // искать ли конец периода в следующих периодах, если в последние 3 минуты периода + // не было показаний + CumulativeMetrics []TotalMetricSpec // metricID +} + +// SelectRangeTotalsInParallel - метод рассчитывает накопленный объем от Since до Until +// по каждой метрике из массива CumulativeMetricIDs. Учитываются настройки объекта +// (LastDayOfMonth, FirstHourOfDay). +// ВАЖНО! +// Запросы по каждой метрике отправляются в БД в отдельной горутине, что позволяет +// в разы ускорить получение данных по объекту. +func (s *MeasureQueryEngine) GetRangeTotalsInParallel(req RangeTotalsFilter) (resultMap map[int64]RangeTotal, err error) { + if req.Since == "" { + err = fix.Field(EmptyValue, "since") + return + } + + if req.Until == "" { + err = fix.Field(EmptyValue, "until") + return + } + + // ВАЖНО! + // Если в строке нет таймзоны - Parse функция возвращает время в UTC. + // Поэтому нужно использовать ParseInLocation! + + since, err := time.ParseInLocation("2006-01-02", req.Since, s.location) + if err != nil { + err = fix.Field(WrongValue, "since") + return + } + + until, err := time.ParseInLocation("2006-01-02", req.Until, s.location) + if err != nil { + err = fix.Field(WrongValue, "until") + return + } + + // Корректируем, чтобы указывало на последнюю секунду суток + until = timeutil.LastSecondInPeriod(until, string(ByDay)) + + var metrics []string + + for _, metric := range req.CumulativeMetrics { + metrics = append(metrics, fmt.Sprintf(` { + MetricID: %d + LastDayOfMonth: %d + FirstHourOfDay: %d + }`, metric.MetricID, metric.LastDayOfMonth, metric.FirstHourOfDay)) + } + + fmt.Printf(`GetRangeTotalsInParallel: { + Since: %s + Until: %s + CumulativeMetrics [ +%s + ] +} +`, since, until, strings.Join(metrics, "\n")) + if since.Unix() >= until.Unix() { + err = fix.Error(WrongDateRange) + return + } + + var ( + tasks []*CumulativeTotalTask + wg = new(sync.WaitGroup) + ) + + resultMap = make(map[int64]RangeTotal) + + // CUMULATIVE METRIC TOTALS + if len(req.CumulativeMetrics) > 0 { + for _, metric := range req.CumulativeMetrics { + // У каждой метрики индивидуальные настройки, + // ибо метрики принадлежат разным объектам!!! + metricSince := since + metricUntil := until + + if metric.FirstHourOfDay > 0 { + metricSince = since.Add(time.Duration(metric.FirstHourOfDay) * time.Hour) + metricUntil = until.Add(time.Duration(metric.FirstHourOfDay) * time.Hour) + } + + task := &CumulativeTotalTask{ + In: getCumulativeTotalFilter{ + MetricID: metric.MetricID, + Since: metricSince.Unix(), + Until: metricUntil.Unix(), + LastDayOfMonth: metric.LastDayOfMonth, + FirstHourOfDay: metric.FirstHourOfDay, + }, + WaitGroup: wg, + } + + wg.Add(1) + s.cumulativeTotalCh <- task + tasks = append(tasks, task) + } + } + + wg.Wait() + + for _, task := range tasks { + if task.Err != nil { + err = fmt.Errorf("get data for the metric %d: %s", task.In.MetricID, task.Err) + return + } + if task.Result != nil { + resultMap[task.In.MetricID] = *task.Result + } else { + resultMap[task.In.MetricID] = RangeTotal{} + } + } + return +} diff --git a/mqe/unaggregated.go b/mqe/unaggregated.go new file mode 100644 index 0000000..efa0bff --- /dev/null +++ b/mqe/unaggregated.go @@ -0,0 +1,138 @@ +package mqe + +import ( + "fmt" + "sync" + "time" +) + +// Measure - рассчитанное показание реальной метрики. Рассчитанное означает +// что учтен Factor. То есть значение уже можно показывать юзеру +// Для instant - одно значение value, для cumulative - 2 значения (value, total) +type Measure struct { + Time int64 `json:"t"` + Values []float64 `json:"v"` +} + +type UnaggregatedFilter struct { + Since time.Time + Until time.Time + LastDayOfMonth int // например, конец месяца 25 число + FirstHourOfDay int // например день начинается в 6:00 + InstantMetrics []int64 + CumulativeMetrics []int64 +} + +// type MetricsDataResultMaps struct { +// AggregatedCumulative map[int64][]CalculatedAggregatedMeasure +// AggregatedByFuncInstant map[int64][]AggregatedByFuncInstantMeasure +// Cumulative map[int64][]CalculatedMeasure +// Instant map[int64][]CalculatedMeasure +// } + +// SelectObjectDataInParallel - метод достает из БД данные всех метрик, которые связаны +// с объектом. Учитываются настройки объекта (LastDayOfMonth, FirstHourOfDay), а также +// фильтры выбранные пользователем на странице объекта (Since, Until, GroupBy). +// ВАЖНО! +// Запросы по каждой метрике отправляются в БД в отдельной горутине, что позволяет +// в разы ускорить получение данных по объекту. +func (s *MeasureQueryEngine) SelectUnaggregatedData(req UnaggregatedFilter) (resultMap map[int64][]Measure, err error) { + since := req.Since + until := req.Until + + if req.FirstHourOfDay < 0 || req.FirstHourOfDay > 23 { + err = fix.Field(InvalidFirstHourOfDay, "firstHourOfDay") + return + } + + if req.LastDayOfMonth < 0 || req.LastDayOfMonth > 28 { + err = fix.Field(InvalidLastDayOfMonth, "lastDayOfMonth") + return + } + + // fmt.Printf(`SelectUnaggregatedData: { + // Since: %s + // Until: %s + // FirstHourOfDay: %d + // LastDayOfMonth: %d + // } + // `, since, until, req.FirstHourOfDay, req.LastDayOfMonth) + // FIX проверить как добавляются часы в DST часовых поясах + // Для req.FirstHourOfDay=4 since должен быть 4:00:00, until должен быть + // 3:59:59 следующего дня + if req.FirstHourOfDay > 0 { + since = since.Add(time.Duration(req.FirstHourOfDay) * time.Hour) + until = until.Add(time.Duration(req.FirstHourOfDay) * time.Hour) + } + + sinceUnixtime := since.Unix() + untilUnixtime := until.Unix() + + if sinceUnixtime >= untilUnixtime { + err = fix.Error(WrongDateRange) + return + } + + var ( + cumulativeTasks []*CumulativeTask + instantTasks []*InstantTask + wg = new(sync.WaitGroup) + ) + + // Важно вирутальная метрика зависит от реальных, которые к объекту могут быть + // привязаны или нет. Если нет - не возвращать. + resultMap = make(map[int64][]Measure) + + if len(req.CumulativeMetrics) > 0 { + for _, metricID := range req.CumulativeMetrics { + task := &CumulativeTask{ + In: MetricMeasuresFilter{ + MetricID: metricID, + Since: sinceUnixtime, + Until: untilUnixtime, + }, + WaitGroup: wg, + } + + wg.Add(1) + s.cumulativeCh <- task + cumulativeTasks = append(cumulativeTasks, task) + } + } + + if len(req.InstantMetrics) > 0 { + for _, metricID := range req.InstantMetrics { + task := &InstantTask{ + In: MetricMeasuresFilter{ + MetricID: metricID, + Since: sinceUnixtime, + Until: untilUnixtime, + }, + WaitGroup: wg, + } + + wg.Add(1) + s.instantCh <- task + instantTasks = append(instantTasks, task) + } + } + + wg.Wait() + + for _, task := range cumulativeTasks { + if task.Err != nil { + err = fmt.Errorf("get data for the metric %d: %s", task.In.MetricID, task.Err) + return + } + resultMap[task.In.MetricID] = task.Result + } + + for _, task := range instantTasks { + if task.Err != nil { + err = fmt.Errorf("get data for the metric %d: %s", task.In.MetricID, task.Err) + return + } + resultMap[task.In.MetricID] = task.Result + } + return +} diff --git a/prepare.sh b/prepare.sh new file mode 100755 index 0000000..59b063e --- /dev/null +++ b/prepare.sh @@ -0,0 +1,3 @@ +cd prepare +env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o prepare +cd - diff --git a/prepare/main.go b/prepare/main.go new file mode 100644 index 0000000..2d9ffab --- /dev/null +++ b/prepare/main.go @@ -0,0 +1,616 @@ +package main + +import ( + "bufio" + "database/sql" + "fmt" + "log" + "os" + "sync" + "time" + + _ "github.com/go-sql-driver/mysql" + bin "gordenko.dev/dima/bin/little" + "gordenko.dev/dima/qx" +) + +type MetricType int +type MetricCategory int + +const ( + MetricTypeCumulative MetricType = 1 + MetricTypeInstant MetricType = 2 + // Категории метрик + CategoryVolume MetricCategory = 1 + CategoryVolumeFlow MetricCategory = 2 + CategoryEnergy MetricCategory = 3 + CategoryMass MetricCategory = 5 + CategoryMassFlow MetricCategory = 6 + CategoryPower MetricCategory = 7 + CategoryTemperature MetricCategory = 8 + CategoryTempDiff MetricCategory = 9 + CategoryLevel MetricCategory = 10 + CategoryPressure MetricCategory = 11 + CategoryFrequency MetricCategory = 12 + CategoryRunningFrequency MetricCategory = 13 + CategoryCurrent MetricCategory = 14 + CategoryVoltage MetricCategory = 15 + CategoryBusVoltage MetricCategory = 16 + CategorySpeed MetricCategory = 17 + CategoryTorque MetricCategory = 18 + CategoryDuration MetricCategory = 19 // час роботи, роботи з помилкою, роботи без помилок + CategoryPressureSetting MetricCategory = 20 // уставка тиску + CategoryBatteryPower MetricCategory = 21 + CategorySignalQuality MetricCategory = 22 + CategoryError MetricCategory = 23 + CategoryPercent MetricCategory = 24 + CategoryResistance MetricCategory = 25 + + DSN = "root:zkx75fcy@/rrc" +) + +var ( + MetricCategoryToMetricType = map[MetricCategory]MetricType{ + CategoryVolume: MetricTypeCumulative, + CategoryVolumeFlow: MetricTypeInstant, + CategoryEnergy: MetricTypeCumulative, + CategoryMass: MetricTypeCumulative, + CategoryMassFlow: MetricTypeInstant, + CategoryPower: MetricTypeInstant, + CategoryTemperature: MetricTypeInstant, + CategoryTempDiff: MetricTypeInstant, + CategoryLevel: MetricTypeInstant, + CategoryPressure: MetricTypeInstant, + CategoryPressureSetting: MetricTypeInstant, + CategoryFrequency: MetricTypeInstant, + CategoryRunningFrequency: MetricTypeInstant, + CategoryCurrent: MetricTypeInstant, + CategoryVoltage: MetricTypeInstant, + CategoryBusVoltage: MetricTypeInstant, + CategorySpeed: MetricTypeInstant, + CategoryTorque: MetricTypeInstant, + CategoryDuration: MetricTypeCumulative, + CategoryBatteryPower: MetricTypeInstant, + CategorySignalQuality: MetricTypeInstant, + CategoryError: MetricTypeInstant, + CategoryPercent: MetricTypeInstant, + CategoryResistance: MetricTypeInstant, + } +) + +func CategoryToMetricType(category MetricCategory) MetricType { + switch category { + case CategoryVolume, CategoryMass, CategoryEnergy, CategoryDuration: + return MetricTypeCumulative + } + return MetricTypeInstant +} + +/* +Список метрик: +ID, тип метрики, fracDigits, начало диапазона, конец диапазона. + +Есть смысл записать показания в файлы. Итерация (запись в субд) пойдет быстрее. + +Получить список метрик. +В цикле получить показания и записатьв файл? +*/ + +type Metric struct { + MetricID int64 `json:"metricID"` + MetricType MetricType `json:"metricType"` + FracDigits int `json:"fracDigits"` +} + +func listMetrics(db *qx.Db) (list []Metric, err error) { + var tmp []struct { + MetricID int64 + FracDigits int + ConfigMetricID int64 + CustomCategory MetricCategory + } + + err = db.ListQuery(&tmp, ` + SELECT m.metricID, mp.fracDigits, mp.configMetricID, mp.customCategory + FROM metrics m + INNER JOIN metric_profiles mp ON mp.profileID = m.metricProfileID + ORDER BY m.metricID ASC`) + + for _, x := range tmp { + var ( + category MetricCategory + metricType MetricType + ) + if x.CustomCategory > 0 { + metricType = CategoryToMetricType(x.CustomCategory) + } else { + if x.ConfigMetricID > 0 { + found, err := db.OneQuery(&category, `SELECT category FROM config_metrics WHERE metricID=?`, x.ConfigMetricID) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("not found configMetric %d", x.ConfigMetricID) + } + + metricType = CategoryToMetricType(category) + } else { + return nil, fmt.Errorf("customCategory not set and configMetricID = 0\n") + } + } + + list = append(list, Metric{ + MetricID: x.MetricID, + FracDigits: x.FracDigits, + MetricType: metricType, + }) + } + return +} + +type Measure struct { + Time int64 `json:"tm" ` + Value float64 `json:"value"` +} + +func listMeasures(db *qx.Db, metricID int64) (list []Measure, err error) { + err = db.ListQuery(&list, "SELECT tm, value FROM f64 WHERE metricID=? ORDER BY tm ASC", metricID) + return +} + +type AppendTask struct { + MetricID uint32 + MetricType MetricType + FracDigits byte +} + +func copyMeasures(db *qx.Db) (err error) { + metrics, err := listMetrics(db) + if err != nil { + return + } + + fmt.Printf("found %d metrics\n", len(metrics)) + + wg := new(sync.WaitGroup) + stopCh := make(chan struct{}) + taskCh := make(chan AppendTask) + + for range 10 { + go writer(db, taskCh, wg, stopCh) + } + + for _, metric := range metrics { + taskCh <- AppendTask{ + MetricID: uint32(metric.MetricID), + MetricType: metric.MetricType, + FracDigits: byte(metric.FracDigits), + } + } + + close(stopCh) + + wg.Wait() + return nil +} + +func writer(db *qx.Db, taskCh chan AppendTask, wg *sync.WaitGroup, stopCh chan struct{}) { + wg.Add(1) + + for { + select { + case task := <-taskCh: + err := writeMetricIntoFile(db, task) + if err != nil { + log.Println(err) + } + + case <-stopCh: + wg.Done() + return + } + } + +} + +// const maxQtyInPack = 65535 + +// func writer(taskCh chan AppendTask, wg *sync.WaitGroup, stopCh chan struct{}) { +// wg.Add(1) + +// c, err := client.Connect(":12345") +// if err != nil { +// log.Fatalln(err) +// } + +// for { +// select { +// case task := <-taskCh: +// err = writeMetricIntoOctopus(c, task) +// if err != nil { +// log.Println(err) +// } + +// case <-stopCh: +// wg.Done() +// return +// } +// } + +// } + +func correctToMonotonic(measures []Measure) []Measure { + if len(measures) == 0 { + return nil + } + + if measures[0].Value < 0 { + var ( + idx int + m Measure + ) + for idx, m = range measures { + if m.Value >= 0 { + break + } + } + measures = measures[idx:] + } + + if len(measures) == 0 { + return nil + } + + var ( + baseValue float64 + prevValue = measures[0].Value // как прислал прибор + ) + for idx := 1; idx < len(measures); idx++ { + measure := measures[idx] + + if measure.Value < 0 { + continue + } + + if measure.Value < prevValue { + // произошел сброс + baseValue += prevValue + } + prevValue = measure.Value + + if baseValue > 0 { + measure.Value += baseValue + measures[idx] = measure + } + } + return measures +} + +func writeMetricIntoFile(db *qx.Db, task AppendTask) (err error) { + measures, err := listMeasures(db, int64(task.MetricID)) + if err != nil { + return fmt.Errorf("listMeasures(%d): %s", task.MetricID, err) + } + //fmt.Printf("metric %d has %d measures\n", metric.MetricID, len(measures)) + if len(measures) < 1000 { + return + } + + filename := fmt.Sprintf("%d.%d", task.MetricID, task.FracDigits) + + if task.MetricType == MetricTypeCumulative { + measures = correctToMonotonic(measures) + + if len(measures) < 1000 { + return + } + + filename = "cumulative/" + filename + } else { + filename = "instant/" + filename + } + + file, err := os.OpenFile(filename, os.O_CREATE|os.O_RDWR, 0666) + if err != nil { + return + } + + dst := bufio.NewWriter(file) + + for _, measure := range measures { + err = bin.WriteUint32(dst, uint32(measure.Time)) + if err != nil { + return + } + err = bin.WriteFloat64(dst, measure.Value) + if err != nil { + return + } + } + + err = dst.Flush() + if err != nil { + return + } + + err = file.Close() + if err != nil { + return + } + fmt.Printf("%d: %d measures written\n", task.MetricID, len(measures)) + return +} + +// func writeMetricIntoOctopus(c *client.Connection, task AppendTask) error { +// err := c.AddMetric(proto.AddMetricReq{ +// MetricID: task.MetricID, +// MetricType: octopus.MetricType(task.MetricType), +// FracDigits: int(task.FracDigits), +// }) +// if err != nil { +// log.Fatal(err) +// } else { +// fmt.Println("metric added") +// } + +// t1 := time.Now() +// var ( +// prevTime int64 +// pack []proto.Measure +// ) + +// for _, measure := range task.Measures { +// if measure.Time <= prevTime { +// continue +// } else { +// prevTime = measure.Time +// } + +// pack = append(pack, proto.Measure{ +// Timestamp: uint32(measure.Time), +// Value: measure.Value, +// }) + +// if len(pack) == maxQtyInPack { +// err = c.AppendMeasures(proto.AppendMeasuresReq{ +// MetricID: task.MetricID, +// Measures: pack, +// }) +// if err != nil { +// return fmt.Errorf("append measures (%d): %s", task.MetricID, err) +// } else { +// //fmt.Printf("written %d measures of %d\n", len(pack), len(task.Measures)) +// pack = nil +// } +// } +// } + +// if len(pack) > 0 { +// err = c.AppendMeasures(proto.AppendMeasuresReq{ +// MetricID: task.MetricID, +// Measures: pack, +// }) +// if err != nil { +// return fmt.Errorf("append measures (%d): %s", task.MetricID, err) +// } else { +// //fmt.Printf("written %d measures of %d\n", len(pack), len(task.Measures)) +// } +// } +//fmt.Printf("written %d in %.2f seconds\n", len(task.Measures), time.Since(t1).Seconds()) +//return nil +//} + +func main() { + db, err := qx.Open("mysql", DSN) + if err != nil { + log.Fatalln(err) + } + + copyAllMetrics(db) + //copyAllMetricsByOne() + //copyOneMetric(db, 134) + // metrics, err := listMetrics(db) + // if err != nil { + // log.Fatalln(err) + // } + + // pretty.Println(metrics) + //checkAllMetrics(db) +} + +func copyAllMetrics(db *qx.Db) { + t1 := time.Now() + err := copyMeasures(db) + if err != nil { + log.Fatalln(err) + } + + fmt.Printf("total time: %.1f seconds\n", time.Since(t1).Seconds()) +} + +// func copyOneMetric(db *qx.Db, metricID int64) { +// measures, err := listMeasures(db, metricID) +// if err != nil { +// log.Fatalln(err) +// } + +// c, err := client.Connect(":12345") +// if err != nil { +// log.Fatalln(err) +// } + +// writeMetricIntoOctopus(c, AppendTask{ +// MetricID: uint32(metricID), +// MetricType: MetricTypeInstant, +// FracDigits: 3, +// Measures: measures, +// }) + +// // measures = measures[:2300] + +// // timestampsBuf := conbuf.New(nil) +// // timestamps := chunkenc.NewReverseTimeDeltaOfDeltaCompressor(timestampsBuf, 0) + +// // valuesBuf := conbuf.New(nil) +// // values := chunkenc.NewReverseInstantDeltaCompressor(valuesBuf, 0, 3) + +// // for _, measure := range measures[:10] { +// // // if measure.Time < 1686671940 { +// // // fmt.Printf("idx: %d\n", idx) +// // // continue +// // // } + +// // //fmt.Printf("ts: %d, v: %.2f\n", measure.Time, measure.Value) + +// // timestamps.Append(uint32(measure.Time)) +// // values.Append(measure.Value) +// // } + +// // timestampDecompressor := chunkenc.NewReverseTimeDeltaOfDeltaDecompressor( +// // timestampsBuf, +// // timestamps.Size(), +// // ) + +// // valueDecompressor := chunkenc.NewReverseInstantDeltaDecompressor( +// // valuesBuf, +// // values.Size(), +// // 3, +// // ) + +// // fmt.Println("----------------------------------------------------") + +// // var ( +// // value float64 +// // timestamp uint32 +// // done bool +// // ) + +// // for { +// // timestamp, done = timestampDecompressor.NextValue() +// // if done { +// // break +// // } + +// // value, done = valueDecompressor.NextValue() +// // if done { +// // fmt.Printf("ts before crash: %d\n", timestamp) +// // panic("FUCK") +// // } + +// // fmt.Printf("ts: %d, v: %.2f\n", timestamp, value) +// // } +// } + +// func checkAllMetrics(db *qx.Db) { +// metrics, err := listMetrics(db) +// if err != nil { +// return +// } + +// //metrics = metrics[:100] + +// c, err := client.Connect(":12345") +// if err != nil { +// log.Fatalln(err) +// } + +// var total int + +// for _, metric := range metrics { +// if metric.MetricType == MetricTypeCumulative { +// list, err := c.ListAllCumulativeMeasures(uint32(metric.MetricID)) +// if err != nil { +// fmt.Printf("ListAllCumulativeMeasures(%d): %s\n", metric.MetricID, err) +// } else { +// total += len(list) +// } +// } else { +// list, err := c.ListAllInstantMeasures(uint32(metric.MetricID)) +// if err != nil { +// fmt.Printf("ListAllInstantMeasures(%d): %s\n", metric.MetricID, err) +// } else { +// total += len(list) +// } +// } +// //fmt.Printf("#%d: %d\n", idx, metric.MetricID) + +// } +// fmt.Printf("total: %d\n", total) +// } + +// копирование равномерное +type Reading struct { + MetricID int + ReadAt time.Time + Value float64 +} + +func copyAllMetricsByOne() { + mydb, err := qx.Open("mysql", DSN) + if err != nil { + log.Fatalln(err) + } + + metrics, err := listMetrics(mydb) + if err != nil { + return + } + mydb.Close() + + //metrics = metrics[:100] + + fmt.Printf("found %d metrics\n", len(metrics)) + + // Підключення до MySQL + db, err := sql.Open("mysql", DSN) + if err != nil { + log.Fatal(err) + } + defer db.Close() + + // Завантажити всі показники, відсортовані по часу + var ( + ts = time.Now().Unix() + total int + ) + + for { + // Рівномірна вставка: поки є хоча б один показник + tx, err := db.Begin() + if err != nil { + log.Fatalln(err) + } + + localTs := ts + + for range 100 { + localTs += 60 + for _, metric := range metrics { + _, err = tx.Exec("INSERT INTO readings (metricID, tm, value) VALUES (?, ?, ?)", + metric.MetricID, localTs, 1232232) + if err != nil { + log.Fatalf("Insert error: %v", err) + } + + total++ + if total > 841000000 { + break + } + } + + } + + err = tx.Commit() + if err != nil { + log.Fatalln(err) + } + + fmt.Printf("inserted: %d\n", total) + + if total > 841000000 { + break + } + + ts += 660 + } +} diff --git a/prepare/main_test.go b/prepare/main_test.go new file mode 100644 index 0000000..06ab7d0 --- /dev/null +++ b/prepare/main_test.go @@ -0,0 +1 @@ +package main diff --git a/prepare/prepare b/prepare/prepare new file mode 100755 index 0000000..2ab8b0d Binary files /dev/null and b/prepare/prepare differ diff --git a/recovery/recovery.go b/recovery/recovery.go index 6fee097..c687899 100644 --- a/recovery/recovery.go +++ b/recovery/recovery.go @@ -175,12 +175,12 @@ func Recovery(dir string, databaseName string) (_ RecoveryReport, err error) { return } - fmt.Printf("before replay\n") + //fmt.Printf("before replay\n") err = walReplayer.Replay() if err != nil { return } - fmt.Printf("after replay\n") + //fmt.Printf("after replay\n") // перезаписую сторінки із останнього комміта var dataFile *os.File dataFile, err = os.OpenFile(qb.GetDataFilePath(dir, databaseName), os.O_CREATE|os.O_WRONLY, 0666) diff --git a/storage/cursor.go b/storage/cursor.go index 8caded6..56e2a12 100644 --- a/storage/cursor.go +++ b/storage/cursor.go @@ -6,6 +6,8 @@ import ( bin "gordenko.dev/dima/bin/little" "gordenko.dev/dima/qb" + "gordenko.dev/dima/qb/enc" + "gordenko.dev/dima/qb/util" ) type BackwardCursor struct { @@ -29,6 +31,7 @@ type BackwardCursorOptions struct { } func NewBackwardCursor(opt BackwardCursorOptions) (*BackwardCursor, error) { + fmt.Printf("footer: % x\n", opt.PageData[DataPageSize-DataPageFooterSize:]) switch opt.MetricType { case qb.Instant, qb.Cumulative: // ok @@ -90,6 +93,7 @@ func (s *BackwardCursor) Prev() (uint32, float64, bool, error) { } prevPageNo, _ := bin.GetUint32(s.pageData[prevPageIdx:]) + fmt.Printf("CURSOR: prev pageNo %d\n", prevPageNo) if prevPageNo == 0 { return 0, 0, true, nil } @@ -129,8 +133,51 @@ func (s *BackwardCursor) Close() { s.releasePage(s.pageNo) } -// HELPER +func CreateTimestampDecompressor(page []byte) (qb.TimestampDecompressor, error) { + size, _ := bin.GetUint16(page[timestampsSizeIdx:]) + if int(size) > DataPagePayloadSize { + return nil, fmt.Errorf("bug: invalid timestamps size %d", size) + } + pos := DataPagePayloadSize - int(size) + d := enc.NewTimeDeltaDecompressor() + d.RestoreFromEnd(page[pos:DataPagePayloadSize]) -//func (s *BackwardCursor) makeDecompressors() error { + payload := page[pos:DataPagePayloadSize] -//} + fmt.Printf("PAGE timestamps (CURSOR) %d:\n% x\n", len(payload), payload) + return d, nil +} + +func CreateValueDecompressor(page []byte, metricType qb.MetricType, fracDigits byte) (qb.ValueDecompressor, error) { + size, _ := bin.GetUint16(page[valuesSizeIdx:]) + if int(size) > DataPagePayloadSize { + return nil, fmt.Errorf("bug: invalid values size %d", size) + } + d := enc.NewValueDeltaDecompressor(metricType, fracDigits) + d.RestoreFromEnd(page[:size]) + return d, nil +} + +func VerifyDataPageCRC32(data []byte) error { + var ( + calculatedCRC = util.CalculateCRC32(data[:dataCRC32Idx]) + writtenCRC, _ = bin.GetUint32(data[dataCRC32Idx:]) + ) + if calculatedCRC != writtenCRC { + return fmt.Errorf("calculated CRC32 %d are not equal written CRC32 %d", + calculatedCRC, writtenCRC) + } + return nil +} + +func VerifyIndexPageCRC32(data []byte) error { + var ( + calculatedCRC = util.CalculateCRC32(data[:indexCRC32Idx]) + writtenCRC, _ = bin.GetUint32(data[indexCRC32Idx:]) + ) + if calculatedCRC != writtenCRC { + return fmt.Errorf("calculated CRC32 %d are not equal written CRC32 %d", + calculatedCRC, writtenCRC) + } + return nil +} diff --git a/storage/misc.go b/storage/misc.go deleted file mode 100644 index 40d4302..0000000 --- a/storage/misc.go +++ /dev/null @@ -1,55 +0,0 @@ -package storage - -import ( - "fmt" - - bin "gordenko.dev/dima/bin/little" - "gordenko.dev/dima/qb" - "gordenko.dev/dima/qb/enc" - "gordenko.dev/dima/qb/util" -) - -func CreateTimestampDecompressor(page []byte) (qb.TimestampDecompressor, error) { - size, _ := bin.GetUint16(page[timestampsSizeIdx:]) - if size > DataPageFooterSize { - return nil, fmt.Errorf("bug: invalid timestamps size %d", size) - } - pos := DataPagePayloadSize - int(size) - d := enc.NewTimeDeltaDecompressor() - d.RestoreFromEnd(page[pos:DataPagePayloadSize]) - return d, nil -} - -func CreateValueDecompressor(page []byte, metricType qb.MetricType, fracDigits byte) (qb.ValueDecompressor, error) { - size, _ := bin.GetUint16(page[valuesSizeIdx:]) - if size > DataPageFooterSize { - return nil, fmt.Errorf("bug: invalid timestamps size %d", size) - } - d := enc.NewValueDeltaDecompressor(metricType, fracDigits) - d.RestoreFromEnd(page[:size]) - return d, nil -} - -func VerifyDataPageCRC32(data []byte) error { - var ( - calculatedCRC = util.CalculateCRC32(data[:dataCRC32Idx]) - writtenCRC, _ = bin.GetUint32(data[dataCRC32Idx:]) - ) - if calculatedCRC != writtenCRC { - return fmt.Errorf("calculated CRC32 %d are not equal written CRC32 %d", - calculatedCRC, writtenCRC) - } - return nil -} - -func VerifyIndexPageCRC32(data []byte) error { - var ( - calculatedCRC = util.CalculateCRC32(data[:indexCRC32Idx]) - writtenCRC, _ = bin.GetUint32(data[indexCRC32Idx:]) - ) - if calculatedCRC != writtenCRC { - return fmt.Errorf("calculated CRC32 %d are not equal written CRC32 %d", - calculatedCRC, writtenCRC) - } - return nil -} diff --git a/storage/navigation.go b/storage/navigation.go index b24a0a8..434c48f 100644 --- a/storage/navigation.go +++ b/storage/navigation.go @@ -1,6 +1,8 @@ package storage -import bin "gordenko.dev/dima/bin/little" +import ( + bin "gordenko.dev/dima/bin/little" +) const ( PageNoSize = 4 @@ -61,16 +63,6 @@ func BinarySearch(qty int, keyComparator KeyComparator) (elemIdx int, isFound bo } } -// func getPrevPageNo(buf []byte) uint32 { -// pageNo, _ := bin.GetUint32(buf[prevPageIdx:]) -// return pageNo -// } - -// func GetIndexRecordsSince(buf []byte) (pageNo uint32) { -// pageNo, _ = bin.GetUint32(buf) -// return -// } - func FindPageOnIndexLevelTail(level IndexLevelTail, timestamp uint32) (pageNo uint32) { comparator := ValueAtComparator{ buf: level.Buffer, @@ -82,17 +74,6 @@ func FindPageOnIndexLevelTail(level IndexLevelTail, timestamp uint32) (pageNo ui return } -// func FindPageOnRecords(records []byte, timestamp uint32) (pageNo uint32) { -// comparator := ValueAtComparator{ -// buf: records, -// timestamp: timestamp, -// } -// count := len(records) / IndexRecordSize -// elemIdx, _ := BinarySearch(count, comparator) -// pageNo, _ = bin.GetUint32(records[elemIdx*IndexRecordSize:]) -// return -// } - func FindPageOnIndexPage(page []byte, timestamp uint32) (pageNo uint32) { comparator := ValueAtComparator{ buf: page, @@ -123,6 +104,108 @@ func FindPageOnIndexLevelTails(levels []IndexLevelTail, timestamp uint32) (pageN return } +type DeleteSinceOnIndexPageResult struct { + Buffer []byte + RecordsCount int + PageNumbers []uint32 + //Idx int +} + +func DeleteSinceOnIndexPage(buf []byte, since uint32) DeleteSinceOnIndexPageResult { + comparator := ValueAtComparator{ + buf: buf, + timestamp: since, + } + count, _ := bin.GetUint16(buf[indexRecordsCountIdx:]) + elemIdx, _ := BinarySearch(int(count), comparator) + newbuf := make([]byte, IndexPageSize) + copy(newbuf, buf[:elemIdx*IndexRecordSize]) // [0..idx) + pos := elemIdx*IndexRecordSize + 4 // timestamp size + deleteCount := int(count) - elemIdx + var pageNumbers []uint32 + for range deleteCount { + pageNo, _ := bin.GetUint32(buf[pos:]) + pageNumbers = append(pageNumbers, pageNo) + pos += IndexRecordSize + } + + return DeleteSinceOnIndexPageResult{ + Buffer: newbuf, + RecordsCount: elemIdx, + PageNumbers: pageNumbers, + //Idx: elemIdx, + } +} + +type DeleteOnIndexTailResult struct { + RecordsCount int + PageNumbers []uint32 +} + +func DeleteSinceOnIndexTail(level IndexLevelTail, since uint32) DeleteOnIndexTailResult { + comparator := ValueAtComparator{ + buf: level.Buffer, + timestamp: since, + } + elemIdx, _ := BinarySearch(level.RecordsCount, comparator) + pos := elemIdx*IndexRecordSize + 4 // timestamp size + deleteCount := level.RecordsCount - elemIdx + var pageNumbers []uint32 + for range deleteCount { + pageNo, _ := bin.GetUint32(level.Buffer[pos:]) + pageNumbers = append(pageNumbers, pageNo) + pos += IndexRecordSize + } + return DeleteOnIndexTailResult{ + RecordsCount: elemIdx, + PageNumbers: pageNumbers, + //Idx: elemIdx, + } +} + +func DeleteUntilOnIndexTail(level IndexLevelTail, until uint32) DeleteOnIndexTailResult { + comparator := ValueAtComparator{ + buf: level.Buffer, + timestamp: until, + } + elemIdx, _ := BinarySearch(level.RecordsCount, comparator) + pos := 4 // timestamp size + deleteCount := elemIdx + 1 + var pageNumbers []uint32 + for range deleteCount { + pageNo, _ := bin.GetUint32(level.Buffer[pos:]) + pageNumbers = append(pageNumbers, pageNo) + pos += IndexRecordSize + } + copy(level.Buffer, level.Buffer[:]) + return DeleteOnIndexTailResult{ + RecordsCount: elemIdx, + PageNumbers: pageNumbers, + //Idx: elemIdx, + } +} + +// func getPrevPageNo(buf []byte) uint32 { +// pageNo, _ := bin.GetUint32(buf[prevPageIdx:]) +// return pageNo +// } + +// func GetIndexRecordsSince(buf []byte) (pageNo uint32) { +// pageNo, _ = bin.GetUint32(buf) +// return +// } + +// func FindPageOnRecords(records []byte, timestamp uint32) (pageNo uint32) { +// comparator := ValueAtComparator{ +// buf: records, +// timestamp: timestamp, +// } +// count := len(records) / IndexRecordSize +// elemIdx, _ := BinarySearch(count, comparator) +// pageNo, _ = bin.GetUint32(records[elemIdx*IndexRecordSize:]) +// return +// } + // func findPageNoIdx(buf []byte, timestamp uint32) (idx int) { // comparator := ValueAtComparator{ // buf: buf, @@ -167,16 +250,17 @@ func FindPageOnIndexLevelTails(levels []IndexLevelTail, timestamp uint32) (pageN // return // } -// func listPageNumbers(buf []byte) (pageNumbers []uint32) { -// qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:]) -// pos := indexFooterIdx - PageNoSize -// for range qty { -// pageNo, _ := bin.GetUint32(buf[pos:]) -// pageNumbers = append(pageNumbers, pageNo) -// pos -= PageNoSize -// } -// return -// } +// from index page +func ListPageNumbers(buf []byte) (pageNumbers []uint32) { + count, _ := bin.GetUint16(buf[indexRecordsCountIdx:]) + pos := timestampSize + for range count { + pageNo, _ := bin.GetUint32(buf[pos:]) + pageNumbers = append(pageNumbers, pageNo) + pos += IndexRecordSize + } + return +} // // include since timestamp // func listPageNumbersSince(buf []byte, timestamp uint32) (pageNumbers []uint32) { diff --git a/storage/replay_metric.go b/storage/replay_metric.go index afadbe0..dd74af8 100644 --- a/storage/replay_metric.go +++ b/storage/replay_metric.go @@ -51,6 +51,7 @@ type ReplayMeasuresAppendWithGrowResult struct { } func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord, isLastPacket bool) (_ ReplayMeasuresAppendWithGrowResult) { + //fmt.Printf("%#v\n", rec) var ( reusedIndexPages int reusedDataPages int @@ -164,17 +165,34 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord, // HELPERS func (s *ReplayMetric) composeHeadIndexPage(levelIdx int, head *IndexPageTail) PageToWrite { - level := s.IndexLevelTails[levelIdx] - // розраховую pos, з якого буду дописувати хвіст - pos := level.RecordsCount * IndexRecordSize - // створюю копію сторінки - page := make([]byte, IndexPageSize) - copy(page, level.Buffer[:pos]) // поточні дані + var ( + page = make([]byte, IndexPageSize) + pos int + prevCount int + ) + //fmt.Printf("composeHeadIndexPage: levels=%d\n", len(s.IndexLevelTails)) + //fmt.Printf("head: % x\n", head.Records) + if levelIdx < len(s.IndexLevelTails) { + level := s.IndexLevelTails[levelIdx] + + //fmt.Printf("level (%d): % x\n", level.RecordsCount, level.Buffer) + // розраховую pos, з якого буду дописувати хвіст + pos = level.RecordsCount * IndexRecordSize + // створюю копію сторінки + copy(page, level.Buffer[:pos]) // поточні дані + prevCount = level.RecordsCount + } else { + // + s.IndexLevelTails = append(s.IndexLevelTails, IndexLevelTail{ + Buffer: page, + RecordsCount: len(head.Records) / IndexRecordSize, + }) + } copy(page[pos:], head.Records) // запечатати сторінку calculatedCRC := SealIndexPage(SealIndexPageIn{ Content: page, - RecordsCount: level.RecordsCount + len(head.Records)/IndexRecordSize, + RecordsCount: prevCount + len(head.Records)/IndexRecordSize, ZeroLevel: levelIdx == 0, }) // перевірка CRC diff --git a/storage/storage.go b/storage/storage.go index 07193ac..8d3fcea 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -17,7 +17,7 @@ var ( prevPageIdx = DataPageSize - 12 // index page - IndexPageSize = 1024 + IndexPageSize = 24 // 1024 IndexPagePayloadSize = IndexPageSize - IndexPageFooterSize //indexPageIncSize = IndexPageIncSize @@ -27,7 +27,7 @@ var ( maxRecordsOnIndexPage = (IndexPageSize - IndexPageFooterSize) / IndexRecordSize - // timestampSize = 4 + timestampSize = 4 // pairSize = timestampSize + PageNoSize // dataFooterIdx = timestampsSizeIdx ) diff --git a/storage/write_preparer.go b/storage/write_preparer.go index 16d2b1e..e392847 100644 --- a/storage/write_preparer.go +++ b/storage/write_preparer.go @@ -2,7 +2,6 @@ package storage import ( "bytes" - "fmt" "io" bin "gordenko.dev/dima/bin/little" @@ -270,10 +269,10 @@ func finalizePacket(w *bytes.Buffer, checksum uint32) []byte { start := 9 - bin.CountVarSize(bodySize) bin.PutVarSize(packet[start:], bodySize) - fmt.Printf("checksum: %d\n", checksum) - fmt.Printf("bodySize: %d\n", bodySize) + //fmt.Printf("checksum: %d\n", checksum) + //fmt.Printf("bodySize: %d\n", bodySize) //fmt.Printf("start: %d\n", start) - fmt.Printf("body: % x\n", packet[9:len(packet)-4]) + //fmt.Printf("body: % x\n", packet[9:len(packet)-4]) return packet[start:] } diff --git a/storage/writer.go b/storage/writer.go index d0412d9..1767024 100644 --- a/storage/writer.go +++ b/storage/writer.go @@ -23,7 +23,7 @@ const ( filePerm = 0770 - dumpSnapshotAfterNBytes = 1024 * 1024 * 1024 // 1 GB + dumpSnapshotAfterNBytes = 20 * 1024 // 1024 * 1024 * 1024 // 1 GB writeBufferSize = 4 * 1024 * 1024 ) @@ -64,19 +64,18 @@ type Writer struct { indexPagesCount uint32 dataFreeList *freelist.FreeList indexFreeList *freelist.FreeList - //atree *atree.Atree - dir string - databaseName string - w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск - wal *os.File - dataFile *os.File - indexFile *os.File - input []any - writePreparer *WritePreparer - written int64 - isExited bool - exitCh chan struct{} - waitGroup *sync.WaitGroup + dir string + databaseName string + w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск + wal *os.File + dataFile *os.File + indexFile *os.File + input []any + writePreparer *WritePreparer + written int64 + isExited bool + exitCh chan struct{} + waitGroup *sync.WaitGroup } type WriterOptions struct { @@ -225,7 +224,7 @@ func (s *Writer) packAndWrite() (err error) { prepared := s.writePreparer.Prepare(input) //fmt.Printf("prepared: %#v\n", prepared) - fmt.Println("prepared") + //fmt.Println("prepared") // 3. Пишу на диск WAL (append) n, err := s.wal.Write(prepared.Packet) @@ -239,7 +238,8 @@ func (s *Writer) packAndWrite() (err error) { if err = s.wal.Sync(); err != nil { return } - fmt.Println("synced to wal") + s.written += int64(len(prepared.Packet)) + //fmt.Println("synced to wal") // 4. Пишу в atree сторінки if len(prepared.WriteToData) > 0 { @@ -250,7 +250,7 @@ func (s *Writer) packAndWrite() (err error) { if err = s.dataFile.Sync(); err != nil { return } - fmt.Println("written to data") + //fmt.Println("written to data") } if len(prepared.WriteToIndex) > 0 { err = WriteIndexPages(s.indexFile, prepared.WriteToIndex) @@ -260,7 +260,7 @@ func (s *Writer) packAndWrite() (err error) { if err = s.indexFile.Sync(); err != nil { return } - fmt.Println("written to index") + //fmt.Println("written to index") } // 6. відправляю input - worker-у @@ -296,6 +296,8 @@ func (s *Writer) packAndWrite() (err error) { snapshotNumber := <-snapshotNumberCh + //fmt.Println("got snapshot number:", snapshotNumber) + // копіюю сторінки із delta файла в base файл і потім роблю Truncate err = s.indexFreeList.Merge() if err != nil { @@ -305,6 +307,7 @@ func (s *Writer) packAndWrite() (err error) { if err != nil { return } + //fmt.Println("free lists merged") var err error s.wal, err = os.OpenFile( @@ -315,11 +318,12 @@ func (s *Writer) packAndWrite() (err error) { if err != nil { return fmt.Errorf("create new changes file: %s", err) } + //fmt.Println("new wal created") } else { s.workerInbox.Push(Changes{ Commits: prepared.Commits, }) - fmt.Println("pushed commits") + //fmt.Println("pushed commits") } return nil } diff --git a/testdir/test.data b/testdir/test.data index e69de29..be83346 100644 Binary files a/testdir/test.data and b/testdir/test.data differ diff --git a/testdir/test.free_data b/testdir/test.free_data deleted file mode 100644 index e69de29..0000000 diff --git a/testdir/test.free_data_delta b/testdir/test.free_data_delta deleted file mode 100644 index e69de29..0000000 diff --git a/testdir/test.free_index b/testdir/test.free_index deleted file mode 100644 index e69de29..0000000 diff --git a/testdir/test.free_index_delta b/testdir/test.free_index_delta deleted file mode 100644 index e69de29..0000000 diff --git a/testdir/test.index b/testdir/test.index index e69de29..d6d5fff 100644 Binary files a/testdir/test.index and b/testdir/test.index differ diff --git a/testdir/test.snapshot_1 b/testdir/test.snapshot_1 new file mode 100644 index 0000000..50d5e89 Binary files /dev/null and b/testdir/test.snapshot_1 differ diff --git a/testdir/test.wal_0 b/testdir/test.wal_0 index dbdb305..54db744 100755 Binary files a/testdir/test.wal_0 and b/testdir/test.wal_0 differ diff --git a/testdir/test.wal_1 b/testdir/test.wal_1 new file mode 100755 index 0000000..ccbf8c9 Binary files /dev/null and b/testdir/test.wal_1 differ diff --git a/worker/metric.go b/worker/metric.go index 10d568b..7e070b0 100644 --- a/worker/metric.go +++ b/worker/metric.go @@ -2,12 +2,13 @@ package worker import ( "errors" - "fmt" "io" + "time" bin "gordenko.dev/dima/bin/little" "gordenko.dev/dima/qb" "gordenko.dev/dima/qb/inbox" + "gordenko.dev/dima/qb/proto" "gordenko.dev/dima/qb/storage" ) @@ -119,37 +120,53 @@ func (s *Metric) ReleaseRLock() { s.rLocks-- if s.rLocks == 0 { if len(s.waitQueue) > 0 { - s.ProcessQueue() + //s.ProcessQueue() } } } -// суть у тому що треба запускати запити, пока не зустріну XLock -func (s *Metric) ProcessQueue(metricID uint32, tmp []byte) { - if len(s.waitQueue) == 0 { - return +type GetMetricResult struct { + MetricType qb.MetricType + FracDigits byte + ResultCode byte +} + +type GetMetricReq struct { + MetricID uint32 + ResultCh chan GetMetricResult +} + +func (s *Metric) GetMetric(req GetMetricReq) { + req.ResultCh <- GetMetricResult{ + ResultCode: Succeed, + MetricType: s.metricType, + FracDigits: s.fracDigits, } - for _, untyped := range s.waitQueue { - switch req := untyped.(type) { - case RangeScanReq: - s.StartRangeScan(req) - case FullScanReq: - s.StartFullScan(req) - case GetMetricReq: - s.GetMetric(req) - case AppendMeasuresReq: - metric.AppendMeasures(req, tmp, s.storageInbox) - case DeleteMetricReq: - s.DeleteMetric(req) - case DeleteMeasuresReq: - metric.DeleteMeasures(req) - default: - qb.Abort(qb.UnknownMetricWaitQueueItemBug, - fmt.Errorf("bug: unknown metric wait queue item type %T", req)) - } +} + +func (s *Metric) DeleteMetric(req DeleteMetricReq) { + if s.xLock || s.rLocks > 0 || s.capturedState != nil { + s.waitQueue = append(s.waitQueue, req) + } else { + s.xLock = true + // fix - has pages -> do query from goroutine + // else push to storage + // collect all pages, than + // s.storageInbox.Push(storage.MetricDelete{ + // MetricID: req.MetricID, + // FreeIndexPages: nil, + // FreeDataPages: nil, + // ResultCh: req.ResultCh, + // }) } } +type AppendMeasuresReq struct { + MetricID uint32 + Measures []proto.Measure + ResultCh chan storage.MeasuresAppendResult +} + func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox *inbox.Inbox) { if s.xLock || s.capturedState != nil { s.waitQueue = append(s.waitQueue, req) @@ -182,7 +199,9 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox s.values.CaptureState() for idx, measure := range req.Measures { + //fmt.Println(idx, measure.Timestamp) if measure.Timestamp <= s.timestamps.Until() { + //fmt.Printf("timestamp %d <= until %d\n", measure.Timestamp, s.timestamps.Until()) resultCode = ExpiredMeasure break } @@ -191,6 +210,8 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox break } + //fmt.Printf(" %d: %s => %.2f\n", idx, formatTime(measure.Timestamp), measure.Value) + tReport := timestamps.Evaluate(tmp[:7], measure.Timestamp) //fmt.Printf("tReport: %#v\n", tReport) vReport := values.Evaluate(tmp[7:], measure.Value) @@ -198,7 +219,7 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox totalRequiredSpace := tReport.TotalSpace + vReport.TotalSpace - if totalRequiredSpace <= len(s.buffer) { + if totalRequiredSpace <= storage.DataPagePayloadSize { // якщо на сторінці є місце timestamps.Append(tReport.RewindOffset, tmp[:tReport.ChangeSize], measure.Timestamp) values.Append(vReport.RewindOffset, tmp[7:7+vReport.ChangeSize], measure.Value, vReport.Delta) @@ -210,7 +231,8 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox valuesRewindOffset = vReport.RewindOffset } } else { - fmt.Println("PAGE FILLED") + //fmt.Println("PAGE FILLED") + //fmt.Println(timestamps.Size() + values.Size()) // сторінка заповнена since := s.timestamps.ReplaceSinceWithUntil() @@ -227,6 +249,9 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox ValuesSize: values.Size(), }) + //xxx := timestamps.Tail(0) + //fmt.Printf("PAGE timestamps %d:\n% x\n", len(xxx), xxx) + buf := make([]byte, storage.DataPageSize) databuf := buf[:storage.DataPagePayloadSize] @@ -257,10 +282,14 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox return } + //xxx := timestamps.Tail(0) + //fmt.Printf("PAGE TAIL timestamps %d:\n% x\n", len(xxx), xxx) + // виділити змінені байти. // скопіювати. Причому можна скопіювати зрізи chunks if len(pages) > 0 { + //fmt.Printf("WITH GROW") // пишу в storage довгим шляхом через redo файл і запис в data файл storageInbox.Push(storage.MeasuresAppendWithGrow{ MetricID: req.MetricID, @@ -278,6 +307,7 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox ResultCh: req.ResultCh, }) } else { + //fmt.Printf("SIMPLE") // короткий шлях - запис лише в storage storageInbox.Push(storage.MeasuresAppend{ MetricID: req.MetricID, @@ -292,6 +322,12 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox } } +type DeleteMeasuresReq struct { + MetricID uint32 + Since uint32 + ResultCh chan byte +} + func (s *Metric) DeleteMeasures(req DeleteMeasuresReq) { // if s.xLock || s.capturedState != nil { // s.waitQueue = append(s.waitQueue, req) @@ -314,7 +350,23 @@ func (s *Metric) DeleteMeasures(req DeleteMeasuresReq) { // } } -func (s *Metric) StartRangeScan(req RangeScanReq) { +type RangeScanResult struct { + ResultCode byte + FracDigits byte + LastPageNo uint32 + IsDataPage bool // for UntilNotFound only +} + +type RangeScanReq struct { + MetricID uint32 + Since uint32 + Until uint32 + MetricType qb.MetricType + ResponseWriter qb.WorkerMeasureConsumer + ResultCh chan RangeScanResult +} + +func (s *Metric) RangeScan(req RangeScanReq) { if s.xLock { s.waitQueue = append(s.waitQueue, req) return @@ -397,7 +449,23 @@ func (s *Metric) StartRangeScan(req RangeScanReq) { } } -func (s *Metric) StartFullScan(req FullScanReq) { +type FullScanResult struct { + ResultCode byte + FracDigits byte + LastPageNo uint32 +} + +type FullScanReq struct { + MetricID uint32 + MetricType qb.MetricType + ResponseWriter qb.WorkerMeasureConsumer + ResultCh chan FullScanResult +} + +func (s *Metric) FullScan(req FullScanReq) { + //fmt.Println("lastPageNo:", s.lastPageNo) + //fmt.Printf("index: %#v\n", s.indexLevelTails) + if s.xLock { s.waitQueue = append(s.waitQueue, req) return @@ -410,18 +478,20 @@ func (s *Metric) StartFullScan(req FullScanReq) { } timestampDecompressor := s.timestamps.CreateDecompressor() valueDecompressor := s.values.CreateDecompressor(s.metricType, s.fracDigits) + idx := 0 for { timestamp, done := timestampDecompressor.NextValue() if done { + //fmt.Println("DONE") break } - //fmt.Println("ts:", timestamp) value, done := valueDecompressor.NextValue() if done { qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug) } - //fmt.Println("value:", value) req.ResponseWriter.FeedNoSend(timestamp, value) + //fmt.Printf(" %d: %s => %.2f\n", idx, formatTime(timestamp), value) + idx++ } if s.lastPageNo > 0 { @@ -441,6 +511,7 @@ func (s *Metric) StartFullScan(req FullScanReq) { // COMMITS func (s *Metric) OnMeasuresAppendCommited(rec storage.MeasuresAppendCommited) { + //fmt.Println("OnMeasuresAppendCommited: written=", rec.WrittenCount) // Видаляю state. Оригінальні Timestamps і Values вже мають останню версію s.capturedState = nil s.timestamps.ForgetCapturedState() @@ -453,6 +524,8 @@ func (s *Metric) OnMeasuresAppendCommited(rec storage.MeasuresAppendCommited) { } func (s *Metric) OnMeasuresAppendWithGrowCommited(rec storage.MeasuresAppendWithGrowCommited) { + //fmt.Println("OnMeasuresAppendWithGrowCommited: written=", rec.WrittenCount, ", lastPageNo=", rec.LastPageNo) + //fmt.Printf("index: %#v\n", rec.Index) // Видаляю state. Оригінальні Timestamps і Values вже мають останню версію s.capturedState = nil s.timestamps.ForgetCapturedState() @@ -480,4 +553,16 @@ func (s *Metric) OnMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) { s.indexLevelTails = nil s.lastPageNo = 0 s.lastValue = 0 + +} + +// const ( +// free +// ) + +const datetimeLayout = "2006-01-02 15:04:05" + +func formatTime(timestamp uint32) string { + tm := time.Unix(int64(timestamp), 0) + return tm.Format(datetimeLayout) } diff --git a/worker/snapshot.go b/worker/snapshot.go index da9f7fb..e27383d 100644 --- a/worker/snapshot.go +++ b/worker/snapshot.go @@ -177,7 +177,7 @@ func ReadSnapshot(filePath string) (out ReadSnapshotOut, err error) { if err != nil { return } - fmt.Println("metricID", metricID) + //fmt.Println("metricID", metricID) metricType, err = bin.ReadByte(src) if err != nil { return @@ -187,13 +187,13 @@ func ReadSnapshot(filePath string) (out ReadSnapshotOut, err error) { if err != nil { return } - fmt.Println("m.MetricType", m.MetricType) - fmt.Println("m.FracDigits", m.FracDigits) + //fmt.Println("m.MetricType", m.MetricType) + //fmt.Println("m.FracDigits", m.FracDigits) m.LastPageNo, err = bin.ReadUint32(src) if err != nil { return } - fmt.Println("m.LastPageNo", m.LastPageNo) + //fmt.Println("m.LastPageNo", m.LastPageNo) m.TimestampsSize, err = bin.ReadUint16AsInt(src) if err != nil { return @@ -202,7 +202,7 @@ func ReadSnapshot(filePath string) (out ReadSnapshotOut, err error) { if err != nil { return } - fmt.Println("m.TimestampsSize", m.TimestampsSize, "m.ValuesSize", m.ValuesSize) + //fmt.Println("m.TimestampsSize", m.TimestampsSize, "m.ValuesSize", m.ValuesSize) err = bin.ReadNInto(src, buf[storage.DataPagePayloadSize-m.TimestampsSize:storage.DataPagePayloadSize]) if err != nil { return @@ -211,13 +211,13 @@ func ReadSnapshot(filePath string) (out ReadSnapshotOut, err error) { if err != nil { return } - fmt.Printf("% x\n", buf) + //fmt.Printf("% x\n", buf) // levelsCount, err = bin.ReadVarSize(src) if err != nil { return } - fmt.Println("levels count:", levelsCount) + //fmt.Println("levels count:", levelsCount) for range levelsCount { var ( buf = make([]byte, storage.IndexPageSize) @@ -227,12 +227,12 @@ func ReadSnapshot(filePath string) (out ReadSnapshotOut, err error) { if err != nil { return } - fmt.Println("records count:", count) + //fmt.Println("records count:", count) err = bin.ReadNInto(src, buf[:count*storage.IndexRecordSize]) if err != nil { return } - fmt.Printf("% x\n", buf) + //fmt.Printf("% x\n", buf) m.IndexLevelTails = append(m.IndexLevelTails, storage.IndexLevelTail{ Buffer: buf, RecordsCount: count, @@ -278,7 +278,7 @@ func writeFreePages(dst io.Writer, frozenPagesCount int, pageNumbers []uint32) ( if err != nil { return } - fmt.Println("write frozenPagesCount:", frozenPagesCount) + //fmt.Println("write frozenPagesCount:", frozenPagesCount) _, err = bin.WriteVarSize(dst, len(pageNumbers)) if err != nil { return @@ -301,7 +301,7 @@ func readFreePages(src io.Reader) (_ int, _ []uint32, err error) { if err != nil { return } - fmt.Println("read frozenPagesCount:", frozenPagesCount) + //fmt.Println("read frozenPagesCount:", frozenPagesCount) pageNumbersCount, err := bin.ReadVarSize(src) if err != nil { return diff --git a/worker/worker.go b/worker/worker.go index c018257..9966677 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -7,7 +7,6 @@ import ( qb "gordenko.dev/dima/qb" "gordenko.dev/dima/qb/enc" "gordenko.dev/dima/qb/inbox" - "gordenko.dev/dima/qb/proto" "gordenko.dev/dima/qb/storage" "gordenko.dev/dima/qb/transform" ) @@ -81,6 +80,8 @@ func New(opt Options) *Worker { values: values, indexLevelTails: x.IndexLevelTails, } + + //fmt.Printf("index: %#v\n", x.IndexLevelTails) } return s } @@ -109,25 +110,25 @@ func (s *Worker) doWork() { for _, untyped := range queue { switch req := untyped.(type) { case AppendMeasuresReq: - s.AppendMeasures(req) + s.appendMeasures(req) case ReleaseRLock: s.releaseRLock(req.MetricID) case storage.Changes: s.applyCommits(req) // all metrics only case ListCurrentValuesReq: - s.ListCurrentValues(req) // all metrics only + s.listCurrentValues(req) // all metrics only case RangeScanReq: - s.RangeScan(req) + s.rangeScan(req) case FullScanReq: - s.FullScan(req) + s.fullScan(req) case AddMetricReq: - s.AddMetric(req) + s.addMetric(req) case DeleteMetricReq: - s.DeleteMetric(req) + s.deleteMetric(req) case DeleteMeasuresReq: - s.DeleteMeasures(req) + s.deleteMeasures(req) case GetMetricReq: - s.GetMetric(req) + s.getMetric(req) default: qb.Abort(qb.UnknownWorkerQueueItemBug, fmt.Errorf("bug: unknown worker queue item type %T", req)) @@ -149,7 +150,7 @@ type AddMetricReq struct { ResultCh chan byte } -func (s *Worker) AddMetric(req AddMetricReq) { +func (s *Worker) addMetric(req AddMetricReq) { _, ok := s.metrics[req.MetricID] if ok { req.ResultCh <- MetricDuplicate @@ -175,30 +176,14 @@ func (s *Worker) AddMetric(req AddMetricReq) { }) } -type GetMetricResult struct { - MetricType qb.MetricType - FracDigits byte - ResultCode byte -} - -type GetMetricReq struct { - MetricID uint32 - ResultCh chan GetMetricResult -} - -func (s *Worker) GetMetric(req GetMetricReq) { +func (s *Worker) getMetric(req GetMetricReq) { metric, ok := s.metrics[req.MetricID] - if ok { - req.ResultCh <- GetMetricResult{ - ResultCode: Succeed, - MetricType: metric.MetricType(), - FracDigits: metric.FracDigits(), - } - } else { + if !ok { req.ResultCh <- GetMetricResult{ ResultCode: NoMetric, } } + metric.GetMetric(req) } type DeleteMetricReq struct { @@ -206,32 +191,16 @@ type DeleteMetricReq struct { ResultCh chan byte } -func (s *Worker) DeleteMetric(req DeleteMetricReq) { +func (s *Worker) deleteMetric(req DeleteMetricReq) { metric, ok := s.metrics[req.MetricID] if !ok { req.ResultCh <- NoMetric return } - if metric.xLock { - metric.waitQueue = append(metric.waitQueue, req) - } else { - // collect all pages, than - s.storageInbox.Push(storage.MetricDelete{ - MetricID: req.MetricID, - FreeIndexPages: nil, - FreeDataPages: nil, - ResultCh: req.ResultCh, - }) - } + metric.DeleteMetric(req) } -type DeleteMeasuresReq struct { - MetricID uint32 - Since uint32 - ResultCh chan byte -} - -func (s *Worker) DeleteMeasures(req DeleteMeasuresReq) { +func (s *Worker) deleteMeasures(req DeleteMeasuresReq) { metric, ok := s.metrics[req.MetricID] if !ok { req.ResultCh <- NoMetric @@ -240,13 +209,7 @@ func (s *Worker) DeleteMeasures(req DeleteMeasuresReq) { metric.DeleteMeasures(req) } -type AppendMeasuresReq struct { - MetricID uint32 - Measures []proto.Measure - ResultCh chan storage.MeasuresAppendResult -} - -func (s *Worker) AppendMeasures(req AppendMeasuresReq) { +func (s *Worker) appendMeasures(req AppendMeasuresReq) { metric, ok := s.metrics[req.MetricID] if !ok { req.ResultCh <- storage.MeasuresAppendResult{ @@ -261,23 +224,7 @@ func (s *Worker) AppendMeasures(req AppendMeasuresReq) { } } -type RangeScanResult struct { - ResultCode byte - FracDigits byte - LastPageNo uint32 - IsDataPage bool // for UntilNotFound only -} - -type RangeScanReq struct { - MetricID uint32 - Since uint32 - Until uint32 - MetricType qb.MetricType - ResponseWriter qb.WorkerMeasureConsumer - ResultCh chan RangeScanResult -} - -func (s *Worker) RangeScan(req RangeScanReq) { +func (s *Worker) rangeScan(req RangeScanReq) { metric, ok := s.metrics[req.MetricID] if !ok { req.ResultCh <- RangeScanResult{ @@ -294,24 +241,11 @@ func (s *Worker) RangeScan(req RangeScanReq) { if metric.xLock { metric.waitQueue = append(metric.waitQueue, req) } else { - metric.StartRangeScan(req) + metric.RangeScan(req) } } -type FullScanResult struct { - ResultCode byte - FracDigits byte - LastPageNo uint32 -} - -type FullScanReq struct { - MetricID uint32 - MetricType qb.MetricType - ResponseWriter qb.WorkerMeasureConsumer - ResultCh chan FullScanResult -} - -func (s *Worker) FullScan(req FullScanReq) { +func (s *Worker) fullScan(req FullScanReq) { metric, ok := s.metrics[req.MetricID] if !ok { req.ResultCh <- FullScanResult{ @@ -328,7 +262,7 @@ func (s *Worker) FullScan(req FullScanReq) { if metric.xLock { metric.waitQueue = append(metric.waitQueue, req) } else { - metric.StartFullScan(req) + metric.FullScan(req) } } @@ -338,7 +272,7 @@ type ListCurrentValuesReq struct { ResultCh chan struct{} } -func (s *Worker) ListCurrentValues(req ListCurrentValuesReq) { +func (s *Worker) listCurrentValues(req ListCurrentValuesReq) { for _, metricID := range req.MetricIDs { metric, ok := s.metrics[metricID] if ok { @@ -478,8 +412,31 @@ func (s *Worker) onMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) { //s.doAfterReleaseXLock(rec.MetricID, metric) } -// func (s *Database) doAfterReleaseXLock(metricID uint32, metric *_metric) { -// if len(metric.WaitQueue) > 0 { -// s.processMetricQueue(metricID, metric) -// } -// } +// rLock сумісний із append measures, +// xLock ні з чим +// після xLock - вся черга +// після rLock, якщо capturedState == nil - вся черга, оскільки rLock блокує лише xLock задачі +// після capturedState = nil, якщо перша задача - appendMeasures - беру, інакше перевірка rLock + +// суть у тому що треба запускати запити, пока не зустріну XLock +func (s *Worker) ProcessQueue(metric *Metric, tmp []byte, storageInbox *inbox.Inbox) { + // for _, untyped := range metric.waitQueue { + // switch req := untyped.(type) { + // case RangeScanReq: + // metric.RangeScan(req) + // case FullScanReq: + // metric.FullScan(req) + // case GetMetricReq: + // metric.GetMetric(req) + // case AppendMeasuresReq: + // metric.AppendMeasures(req, tmp, s.storageInbox) + // case DeleteMetricReq: + // metric.DeleteMetric(req) + // case DeleteMeasuresReq: + // metric.DeleteMeasures(req) + // default: + // qb.Abort(qb.UnknownMetricWaitQueueItemBug, + // fmt.Errorf("bug: unknown metric wait queue item type %T", req)) + // } + // } +}