wp

database/api.go

@@ -84,7 +84,7 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e
 		if err != nil {
 			return fmt.Errorf("proto.ReadAddMetricReq: %s", err)
 		}
-		fmt.Println("ReadAddMetricReq:", req)
+		//fmt.Println("ReadAddMetricReq:", req)
 		reply(conn, s.AddMetric(req))
 
 	case proto.TypeDeleteMetric:
@@ -534,8 +534,11 @@ func (s *Database) rangeScan(req rangeScanReq) error {
 			LastPageNo:     result.LastPageNo,
 			Since:          req.Since,
 		})
-		//s.metricRUnlock(req.MetricID) fix release unlock
+		s.workerInbox.Push(worker.ReleaseRLock{
+			MetricID: req.MetricID,
+		})
 		if err != nil {
+			s.logger.Printf("ContinueRangeScan: %s\n", err)
 			reply(req.Conn, proto.ErrUnexpected)
 		} else {
 			req.ResponseWriter.Close()
@@ -550,8 +553,11 @@ func (s *Database) rangeScan(req rangeScanReq) error {
 			LastPageNo:     result.LastPageNo,
 			IsDataPage:     result.IsDataPage,
 		})
-		//s.metricRUnlock(req.MetricID) // fix release
+		s.workerInbox.Push(worker.ReleaseRLock{
+			MetricID: req.MetricID,
+		})
 		if err != nil {
+			s.logger.Printf("RangeScan: %s\n", err)
 			reply(req.Conn, proto.ErrUnexpected)
 		} else {
 			req.ResponseWriter.Close()
@@ -595,8 +601,11 @@ func (s *Database) fullScan(req fullScanReq) error {
 			ResponseWriter: req.ResponseWriter,
 			LastPageNo:     result.LastPageNo,
 		})
-		//s.worker.AddJobToQueue(req.MetricID) // FIX release rlock
+		s.workerInbox.Push(worker.ReleaseRLock{
+			MetricID: req.MetricID,
+		})
 		if err != nil {
+			s.logger.Printf("ContinueFullScan: %s\n", err)
 			reply(req.Conn, proto.ErrUnexpected)
 		} else {
 			req.ResponseWriter.Close()

database/database.go

@@ -77,7 +77,7 @@ func New(opt Options) (_ *Database, err error) {
 		return nil, fmt.Errorf("recovery.Recovery: %s", err)
 	}
 
-	fmt.Printf("%#v\n", recoveryReport)
+	//fmt.Printf("%#v\n", recoveryReport)
 
 	storageInbox := inbox.New()
 	s.workerInbox = inbox.New()
@@ -180,6 +180,7 @@ type ContinueFullScanReq struct {
 }
 
 func (s *Database) ContinueFullScan(req ContinueFullScanReq) error {
+	//fmt.Printf("ContinueFullScan from page %d\n", req.LastPageNo)
 	buf, err := s.dataCache.FetchPage(req.LastPageNo)
 	if err != nil {
 		return fmt.Errorf("dataCache.FetchPage(%d): %s", req.LastPageNo, err)
@@ -196,6 +197,7 @@ func (s *Database) ContinueFullScan(req ContinueFullScanReq) error {
 		return err
 	}
 	defer treeCursor.Close()
+	idx := 0
 	for {
 		timestamp, value, done, err := treeCursor.Prev()
 		if err != nil {
@@ -204,7 +206,9 @@ func (s *Database) ContinueFullScan(req ContinueFullScanReq) error {
 		if done {
 			return nil
 		}
+		//fmt.Printf("  %d: %s => %.2f\n", idx, formatTime(timestamp), value)
 		req.ResponseWriter.Feed(timestamp, value)
+		idx++
 	}
 }
 
@@ -326,15 +330,15 @@ func (s *Database) findDataPage(foundPageNo uint32, timestamp uint32) (uint32, [
 	}
 }
 
-// type PathLeg struct {
+type PathLeg struct {
-// 	PageNo uint32
+	PageNo uint32
-// 	Data   []byte
+	Data   []byte
-// }
+}
 
-// type PathToDataPage struct {
+type PathToDataPage struct {
-// 	Legs       []PathLeg
+	Legs       []PathLeg
-// 	LastPageNo uint32
+	LastPageNo uint32
-// }
+}
 
 // func (s *Database) FindPathToLastPage(rootPageNo uint32) (_ PathToDataPage, err error) {
 // 	// var (
@@ -381,82 +385,215 @@ func (s *Database) findDataPage(foundPageNo uint32, timestamp uint32) (uint32, [
 // 	s.mutex.Unlock()
 // }
 
+type Level struct {
+	Idx         int
+	PageNumbers []uint32
+}
+
+func (s *Database) GetAllPages(pageNumbers []uint32) ([]uint32, []uint32, error) {
+	levels := []*Level{
+		{
+			PageNumbers: pageNumbers,
+			Idx:         0,
+		},
+	}
+	return s.collectPages(levels)
+}
+
+func (s *Database) collectPages(levels []*Level) (indexPageNumbers []uint32, dataPageNumbers []uint32, err error) {
+	var (
+		buf         []byte
+		pageNumbers []uint32
+	)
+	for {
+		if len(levels) == 0 {
+			return
+		}
+		var (
+			lastIdx = len(levels) - 1
+			level   = levels[lastIdx]
+			pageNo  = level.PageNumbers[level.Idx]
+		)
+		if level.Idx < len(level.PageNumbers) {
+			buf, err = s.indexCache.FetchPage(pageNo)
+			if err != nil {
+				return nil, nil, fmt.Errorf("indexCache.FetchPage(%d): %s", pageNo, err)
+			}
+			pageNumbers = storage.ListPageNumbers(buf)
+			s.indexCache.ReleasePage(pageNo)
+
+			if storage.IsZeroLevelPage(buf) {
+				dataPageNumbers = append(dataPageNumbers, pageNumbers...)
+			} else {
+				levels = append(levels, &Level{
+					PageNumbers: pageNumbers,
+				})
+				indexPageNumbers = append(indexPageNumbers, pageNumbers...)
+			}
+			level.Idx++
+		} else {
+			levels = levels[:lastIdx]
+		}
+	}
+}
+
+type DeleteSinceReport struct {
+	IndexPageNumbers []uint32
+	DataPageNumbers  []uint32
+	DataPage         []byte
+	IndexLevelTails  []storage.IndexLevelTail
+}
+
 // type Level struct {
-// 	PageNo   uint32
+// 	Idx         int
-// 	PageData []byte
+// 	PageNumbers []uint32
-// 	Idx      int
-// 	ChildQty int
 // }
 
-// func (s *Atree) GetAllPages(rootPageNo uint32) (_ []uint32, err error) {
+// fix - find by (since - 1) ?
-// 	// var (
-// 	// 	pageNumbers []uint32
-// 	// 	levels      []*Level
-// 	// )
 
-// 	// buf, err := s.fetchIndexPage(rootPageNo)
+// [since, ...]
-// 	// if err != nil {
+func (s *Database) DeleteSince(since uint32, indexLevelTail storage.IndexLevelTail) (_ DeleteSinceReport, err error) {
-// 	// 	return nil, fmt.Errorf("fetchIndexPage(%d): %s", rootPageNo, err)
+	var (
-// 	// }
+		indexPageNumbers []uint32
-// 	// pageNumbers = append(pageNumbers, rootPageNo)
+		dataPageNumbers  []uint32
+		dataPage         []byte
+		buf              []byte
+		indexLevelTails  []storage.IndexLevelTail
+		foundPageNo      uint32
+	)
 
-// 	// // if buf[isDataPageNumbersIdx] == 1 {
+	result := storage.DeleteSinceOnIndexTail(indexLevelTail, since)
-// 	// // 	pageNumbers := listPageNumbers(buf)
+	//
-// 	// // 	dataPages = append(dataPages, pageNumbers...)
+	indexLevelTails = append(indexLevelTails, storage.IndexLevelTail{
+		Buffer:       indexLevelTail.Buffer,
+		RecordsCount: result.RecordsCount,
+	})
+	levels := []*Level{
+		{
+			PageNumbers: result.PageNumbers,
+			Idx:         1,
+		},
+	}
+	foundPageNo = result.PageNumbers[0]
+	for {
+		buf, err = s.indexCache.FetchPage(foundPageNo)
+		if err != nil {
+			err = fmt.Errorf("indexCache.FetchPage(%d): %s", foundPageNo, err)
+			return
+		}
+		toReleaseIndexPageNo := foundPageNo
+		result := storage.DeleteSinceOnIndexPage(buf, since)
+		s.indexCache.ReleasePage(toReleaseIndexPageNo)
+		//
+		foundPageNo = result.PageNumbers[0]
 
-// 	// // 	s.releasePage(rootPageNo)
+		levels = append(levels, &Level{
+			PageNumbers: result.PageNumbers,
+			Idx:         1, // 1st - found pageNo
+		})
 
-// 	// // 	return PageLists{
+		indexLevelTails = append(indexLevelTails, storage.IndexLevelTail{
-// 	// // 		DataPages:  dataPages,
+			Buffer:       result.Buffer,
-// 	// // 		IndexPages: indexPages,
+			RecordsCount: result.RecordsCount,
-// 	// // 	}, nil
+		})
-// 	// // }
 
-// 	// childQty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
+		if storage.IsZeroLevelPage(buf) {
+			dataPage, err = s.dataCache.FetchPage(foundPageNo)
+			if err != nil {
+				err = fmt.Errorf("dataCache.FetchPage(%d): %s", foundPageNo, err)
+				return
+			}
+			dataPageNumbers = append(dataPageNumbers, result.PageNumbers...)
+			break
+		} else {
+			indexPageNumbers = append(indexPageNumbers, result.PageNumbers...)
+		}
+	}
+	indexPageNumbers, dataPageNumbers, err = s.collectPages(levels)
+	if err != nil {
+		return
+	}
+	return DeleteSinceReport{
+		IndexPageNumbers: indexPageNumbers,
+		DataPageNumbers:  dataPageNumbers,
+		DataPage:         dataPage,
+		IndexLevelTails:  indexLevelTails,
+	}, nil
+}
 
-// 	// levels = append(levels, &Level{
+// [..., until]
-// 	// 	PageNo:   rootPageNo,
+func (s *Database) DeleteUntil(since uint32, indexLevelTail storage.IndexLevelTail) (_ DeleteSinceReport, err error) {
-// 	// 	PageData: buf,
+	var (
-// 	// 	Idx:      0,
+		indexPageNumbers []uint32
-// 	// 	ChildQty: int(childQty),
+		dataPageNumbers  []uint32
-// 	// })
+		dataPage         []byte
+		buf              []byte
+		indexLevelTails  []storage.IndexLevelTail
+		foundPageNo      uint32
+	)
 
-// 	// for {
+	result := storage.DeleteSinceOnIndexTail(indexLevelTail, since)
-// 	// 	if len(levels) == 0 {
+	//
-// 	// 		return pageNumbers, nil
+	indexLevelTails = append(indexLevelTails, storage.IndexLevelTail{
-// 	// 	}
+		Buffer:       indexLevelTail.Buffer,
+		RecordsCount: result.RecordsCount,
+	})
+	levels := []*Level{
+		{
+			PageNumbers: result.PageNumbers,
+			Idx:         1,
+		},
+	}
+	foundPageNo = result.PageNumbers[0]
+	for {
+		buf, err = s.indexCache.FetchPage(foundPageNo)
+		if err != nil {
+			err = fmt.Errorf("indexCache.FetchPage(%d): %s", foundPageNo, err)
+			return
+		}
+		toReleaseIndexPageNo := foundPageNo
+		result := storage.DeleteSinceOnIndexPage(buf, since)
+		s.indexCache.ReleasePage(toReleaseIndexPageNo)
+		//
+		foundPageNo = result.PageNumbers[0]
 
-// 	// 	lastIdx := len(levels) - 1
+		levels = append(levels, &Level{
-// 	// 	level := levels[lastIdx]
+			PageNumbers: result.PageNumbers,
+			Idx:         1, // 1st - found pageNo
+		})
 
-// 	// 	if level.Idx < level.ChildQty {
+		indexLevelTails = append(indexLevelTails, storage.IndexLevelTail{
-// 	// 		pageNo := getPageNo(level.PageData, level.Idx)
+			Buffer:       result.Buffer,
-// 	// 		level.Idx++
+			RecordsCount: result.RecordsCount,
+		})
 
-// 	// 		var buf []byte
+		if storage.IsZeroLevelPage(buf) {
-// 	// 		buf, err = s.fetchPage(pageNo)
+			dataPage, err = s.dataCache.FetchPage(foundPageNo)
-// 	// 		if err != nil {
+			if err != nil {
-// 	// 			return nil, fmt.Errorf("fetchPage(%d): %s", pageNo, err)
+				err = fmt.Errorf("dataCache.FetchPage(%d): %s", foundPageNo, err)
-// 	// 		}
+				return
-// 	// 		pageNumbers = append(pageNumbers, pageNo)
+			}
+			dataPageNumbers = append(dataPageNumbers, result.PageNumbers...)
+			break
+		} else {
+			indexPageNumbers = append(indexPageNumbers, result.PageNumbers...)
+		}
+	}
+	indexPageNumbers, dataPageNumbers, err = s.collectPages(levels)
+	if err != nil {
+		return
+	}
+	return DeleteSinceReport{
+		IndexPageNumbers: indexPageNumbers,
+		DataPageNumbers:  dataPageNumbers,
+		DataPage:         dataPage,
+		IndexLevelTails:  indexLevelTails,
+	}, nil
+}
 
-// 	// 		if buf[pageTypeIdx] == PageTypeData {
+const datetimeLayout = "2006-01-02 15:04:05"
-// 	// 			//pageNumbers := listPageNumbers(buf)
+
-// 	// 			//dataPages = append(dataPages, pageNumbers...)
+func formatTime(timestamp uint32) string {
-// 	// 			s.releasePage(pageNo)
+	tm := time.Unix(int64(timestamp), 0)
-// 	// 		} else {
+	return tm.Format(datetimeLayout)
-// 	// 			childQty, _ = bin.GetUint16(buf[indexRecordsQtyIdx:])
+}
-// 	// 			levels = append(levels, &Level{
-// 	// 				PageNo:   pageNo,
-// 	// 				PageData: buf,
-// 	// 				Idx:      0,
-// 	// 				ChildQty: int(childQty),
-// 	// 			})
-// 	// 		}
-// 	// 	} else {
-// 	// 		s.releasePage(level.PageNo)
-// 	// 		levels = levels[:lastIdx]
-// 	// 	}
-// 	// }
-// 	return
-// }

enc/enc_test.go

@@ -6,9 +6,11 @@ import (
 	"math"
 	"slices"
 	"testing"
+	"time"
 
 	bin "gordenko.dev/dima/bin/little"
 	"gordenko.dev/dima/qb"
+	"gordenko.dev/dima/textutil"
 )
 
 func equalFloatSlices(a, b []float64, epsilon float64) bool {
@@ -21,23 +23,23 @@ func equalFloatSlices(a, b []float64, epsilon float64) bool {
 
 var (
 	cumulativeTestCases = []struct {
-		Nums       []float64
+		Nums         []float64
-		Name       string
+		Name         string
-		Offset     int
+		RewindOffset int
-		ChangeSize int
+		ChangeSize   int
-		Buf        []byte
+		Buf          []byte
-		BaseValue  float64
+		BaseValue    float64
-		LastDelta  uint64
+		LastDelta    uint64
 	}{
 		{
 			Nums: []float64{
 				0,
 			},
-			Name:       "add 1st value zero",
+			Name:         "add 1st value zero",
-			Offset:     0,
+			RewindOffset: 0,
-			ChangeSize: 3,
+			ChangeSize:   3,
 			Buf: []byte{
-				0x8f, // base value
+				0x80, // base value
 				0x80, // delta 0
 				0x80, // literal (len = 1)
 				0x00, 0x00, 0x00, 0x00, 0x00,
@@ -49,9 +51,9 @@ var (
 			Nums: []float64{
 				1.5,
 			},
-			Name:       "add 1st value non zero",
+			Name:         "add 1st value non zero",
-			Offset:     0,
+			RewindOffset: 0,
-			ChangeSize: 3,
+			ChangeSize:   3,
 			Buf: []byte{
 				0x8f, // base value
 				0x80, // delta 0
@@ -66,9 +68,9 @@ var (
 				1.5,
 				1.5,
 			},
-			Name:       "literal switch to run",
+			Name:         "literal switch to run",
-			Offset:     1,
+			RewindOffset: 1,
-			ChangeSize: 1,
+			ChangeSize:   1,
 			Buf: []byte{
 				0x8f, // base value
 				0x80, // delta 0
@@ -84,9 +86,9 @@ var (
 				1.6,
 				1.6,
 			},
-			Name:       "literal decrease by 1 and switch to run",
+			Name:         "literal decrease by 1 and switch to run",
-			Offset:     2,
+			RewindOffset: 2,
-			ChangeSize: 3,
+			ChangeSize:   3,
 			Buf: []byte{
 				0x8f, // base value
 				0x80, // delta 0
@@ -104,9 +106,9 @@ var (
 				1.5,
 				1.5,
 			},
-			Name:       "increment run",
+			Name:         "increment run",
-			Offset:     1,
+			RewindOffset: 1,
-			ChangeSize: 1,
+			ChangeSize:   1,
 			Buf: []byte{
 				0x8f, // base value
 				0x80, // delta 0
@@ -123,9 +125,9 @@ var (
 				1.5,
 				1.6,
 			},
-			Name:       "run switch to literal",
+			Name:         "run switch to literal",
-			Offset:     0,
+			RewindOffset: 0,
-			ChangeSize: 2,
+			ChangeSize:   2,
 			Buf: []byte{
 				0x8f, // base value
 				0x80, // delta 0
@@ -138,10 +140,10 @@ var (
 			LastDelta: 1,
 		},
 		{
-			Nums:       repeatFloat64(1.5, 129),
+			Nums:         repeatFloat64(1.5, 129),
-			Name:       "increment run to full fill h-byte",
+			Name:         "increment run to full fill h-byte",
-			Offset:     1,
+			RewindOffset: 1,
-			ChangeSize: 1,
+			ChangeSize:   1,
 			Buf: []byte{
 				0x8f, // base value
 				0x80, // delta 0
@@ -152,10 +154,10 @@ var (
 			LastDelta: 0,
 		},
 		{
-			Nums:       repeatFloat64(1.5, 130),
+			Nums:         repeatFloat64(1.5, 130),
-			Name:       "run switch to literal after h-byte overflow",
+			Name:         "run switch to literal after h-byte overflow",
-			Offset:     0,
+			RewindOffset: 0,
-			ChangeSize: 2,
+			ChangeSize:   2,
 			Buf: []byte{
 				0x8f, // base value
 				0x80, // delta 0
@@ -173,9 +175,9 @@ var (
 				1.6,
 				1.7,
 			},
-			Name:       "increment literal",
+			Name:         "increment literal",
-			Offset:     1,
+			RewindOffset: 1,
-			ChangeSize: 2,
+			ChangeSize:   2,
 			Buf: []byte{
 				0x8f, // base value
 				0x80, // delta 0
@@ -203,11 +205,11 @@ func TestCumulativeDeltaCompressor(t *testing.T) {
 		c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize)
 		for _, num := range testCase.Nums {
 			report = c.Evaluate(tmp, num)
-			c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
+			c.Append(report.RewindOffset, tmp[:report.ChangeSize], num, report.Delta)
 		}
-		if report.Offset != testCase.Offset {
+		if report.RewindOffset != testCase.RewindOffset {
 			t.Fatalf("%s: got offset %d are not equal expected %d",
-				testCase.Name, report.Offset, testCase.Offset)
+				testCase.Name, report.RewindOffset, testCase.RewindOffset)
 		}
 		if report.ChangeSize != testCase.ChangeSize {
 			t.Fatalf("%s: got changeSize %d are not equal expected %d",
@@ -241,7 +243,7 @@ func TestRestoreCumulativeDeltaCompressor(t *testing.T) {
 		c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize)
 		for _, num := range testCase.Nums {
 			report = c.Evaluate(tmp, num)
-			c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
+			c.Append(report.RewindOffset, tmp[:report.ChangeSize], num, report.Delta)
 		}
 
 		restored := NewValueDeltaCompressor(metricType, fracDigits, buf, c.Size())
@@ -270,7 +272,7 @@ func TestCumulativeDeltaDecompressorFromState(t *testing.T) {
 		c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize)
 		for _, num := range testCase.Nums {
 			report := c.Evaluate(tmp, num)
-			c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
+			c.Append(report.RewindOffset, tmp[:report.ChangeSize], num, report.Delta)
 		}
 
 		d := c.CreateDecompressor(metricType, fracDigits)
@@ -304,7 +306,7 @@ func TestCumulativeDeltaDecompressorFromEnd(t *testing.T) {
 		c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize)
 		for _, num := range testCase.Nums {
 			report := c.Evaluate(tmp, num)
-			c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
+			c.Append(report.RewindOffset, tmp[:report.ChangeSize], num, report.Delta)
 		}
 
 		d := NewValueDeltaDecompressor(metricType, fracDigits)
@@ -330,21 +332,21 @@ func TestCumulativeDeltaDecompressorFromEnd(t *testing.T) {
 
 var (
 	instantTestCases = []struct {
-		Nums       []float64
+		Nums         []float64
-		Name       string
+		Name         string
-		Offset     int
+		RewindOffset int
-		ChangeSize int
+		ChangeSize   int
-		Buf        []byte
+		Buf          []byte
-		BaseValue  float64
+		BaseValue    float64
-		LastDelta  uint64
+		LastDelta    uint64
 	}{
 		{
 			Nums: []float64{
 				1.5,
 			},
-			Name:       "add 1st value",
+			Name:         "add 1st value",
-			Offset:     0,
+			RewindOffset: 0,
-			ChangeSize: 3,
+			ChangeSize:   3,
 			Buf: []byte{
 				0x9e, // base value
 				0x80, // delta 0
@@ -358,9 +360,9 @@ var (
 			Nums: []float64{
 				-1.5,
 			},
-			Name:       "add 1st value (negative)",
+			Name:         "add 1st value (negative)",
-			Offset:     0,
+			RewindOffset: 0,
-			ChangeSize: 3,
+			ChangeSize:   3,
 			Buf: []byte{
 				0x9d, // base value
 				0x80, // delta 0
@@ -375,9 +377,9 @@ var (
 				1.5,
 				1.5,
 			},
-			Name:       "literal switch to run",
+			Name:         "literal switch to run",
-			Offset:     1,
+			RewindOffset: 1,
-			ChangeSize: 1,
+			ChangeSize:   1,
 			Buf: []byte{
 				0x9e, // base value
 				0x80, // delta 0
@@ -392,9 +394,9 @@ var (
 				-1.5,
 				-1.5,
 			},
-			Name:       "literal switch to run (negative)",
+			Name:         "literal switch to run (negative)",
-			Offset:     1,
+			RewindOffset: 1,
-			ChangeSize: 1,
+			ChangeSize:   1,
 			Buf: []byte{
 				0x9d, // base value
 				0x80, // delta 0
@@ -410,9 +412,9 @@ var (
 				1.6,
 				1.6,
 			},
-			Name:       "literal decrease by 1 and switch to run",
+			Name:         "literal decrease by 1 and switch to run",
-			Offset:     2,
+			RewindOffset: 2,
-			ChangeSize: 3,
+			ChangeSize:   3,
 			Buf: []byte{
 				0x9e, // base value
 				0x80, // delta 0
@@ -430,9 +432,9 @@ var (
 				-1.6,
 				-1.6,
 			},
-			Name:       "literal decrease by 1 and switch to run (negative)",
+			Name:         "literal decrease by 1 and switch to run (negative)",
-			Offset:     2,
+			RewindOffset: 2,
-			ChangeSize: 3,
+			ChangeSize:   3,
 			Buf: []byte{
 				0x9d, // base value
 				0x80, // delta 0
@@ -450,9 +452,9 @@ var (
 				1.4,
 				1.4,
 			},
-			Name:       "literal decrease by 1 and switch to run (negative delta)",
+			Name:         "literal decrease by 1 and switch to run (negative delta)",
-			Offset:     2,
+			RewindOffset: 2,
-			ChangeSize: 3,
+			ChangeSize:   3,
 			Buf: []byte{
 				0x9e, // base value
 				0x80, // delta 0
@@ -470,9 +472,9 @@ var (
 				-1.4,
 				-1.4,
 			},
-			Name:       "literal decrease by 1 and switch to run (positive delta, negative)",
+			Name:         "literal decrease by 1 and switch to run (positive delta, negative)",
-			Offset:     2,
+			RewindOffset: 2,
-			ChangeSize: 3,
+			ChangeSize:   3,
 			Buf: []byte{
 				0x9d, // base value
 				0x80, // delta 0
@@ -490,9 +492,9 @@ var (
 				1.5,
 				1.5,
 			},
-			Name:       "increment run",
+			Name:         "increment run",
-			Offset:     1,
+			RewindOffset: 1,
-			ChangeSize: 1,
+			ChangeSize:   1,
 			Buf: []byte{
 				0x9e, // base value
 				0x80, // delta 0
@@ -508,9 +510,9 @@ var (
 				-1.5,
 				-1.5,
 			},
-			Name:       "increment run (negative)",
+			Name:         "increment run (negative)",
-			Offset:     1,
+			RewindOffset: 1,
-			ChangeSize: 1,
+			ChangeSize:   1,
 			Buf: []byte{
 				0x9d, // base value
 				0x80, // delta 0
@@ -527,9 +529,9 @@ var (
 				1.5,
 				1.6,
 			},
-			Name:       "run switch to literal",
+			Name:         "run switch to literal",
-			Offset:     0,
+			RewindOffset: 0,
-			ChangeSize: 2,
+			ChangeSize:   2,
 			Buf: []byte{
 				0x9e, // base value
 				0x80, // delta 0
@@ -548,9 +550,9 @@ var (
 				-1.5,
 				-1.6,
 			},
-			Name:       "run switch to literal (negative)",
+			Name:         "run switch to literal (negative)",
-			Offset:     0,
+			RewindOffset: 0,
-			ChangeSize: 2,
+			ChangeSize:   2,
 			Buf: []byte{
 				0x9d, // base value
 				0x80, // delta 0
@@ -569,9 +571,9 @@ var (
 				1.5,
 				1.4,
 			},
-			Name:       "run switch to literal (negative delta)",
+			Name:         "run switch to literal (negative delta)",
-			Offset:     0,
+			RewindOffset: 0,
-			ChangeSize: 2,
+			ChangeSize:   2,
 			Buf: []byte{
 				0x9e, // base value
 				0x80, // delta 0
@@ -590,9 +592,9 @@ var (
 				-1.5,
 				-1.4,
 			},
-			Name:       "run switch to literal (positive delta, negative version)",
+			Name:         "run switch to literal (positive delta, negative version)",
-			Offset:     0,
+			RewindOffset: 0,
-			ChangeSize: 2,
+			ChangeSize:   2,
 			Buf: []byte{
 				0x9d, // base value
 				0x80, // delta 0
@@ -605,10 +607,10 @@ var (
 			LastDelta: bin.EncodeZigZag(1),
 		},
 		{
-			Nums:       repeatFloat64(1.5, 129),
+			Nums:         repeatFloat64(1.5, 129),
-			Name:       "increment run to full fill h-byte",
+			Name:         "increment run to full fill h-byte",
-			Offset:     1,
+			RewindOffset: 1,
-			ChangeSize: 1,
+			ChangeSize:   1,
 			Buf: []byte{
 				0x9e, // base value
 				0x80, // delta 0
@@ -619,10 +621,10 @@ var (
 			LastDelta: 0,
 		},
 		{
-			Nums:       repeatFloat64(-1.5, 129),
+			Nums:         repeatFloat64(-1.5, 129),
-			Name:       "increment run to full fill h-byte (negative version)",
+			Name:         "increment run to full fill h-byte (negative version)",
-			Offset:     1,
+			RewindOffset: 1,
-			ChangeSize: 1,
+			ChangeSize:   1,
 			Buf: []byte{
 				0x9d, // base value
 				0x80, // delta 0
@@ -633,10 +635,10 @@ var (
 			LastDelta: 0,
 		},
 		{
-			Nums:       repeatFloat64(1.5, 130),
+			Nums:         repeatFloat64(1.5, 130),
-			Name:       "run switch to literal after h-byte overflow",
+			Name:         "run switch to literal after h-byte overflow",
-			Offset:     0,
+			RewindOffset: 0,
-			ChangeSize: 2,
+			ChangeSize:   2,
 			Buf: []byte{
 				0x9e, // base value
 				0x80, // delta 0
@@ -649,10 +651,10 @@ var (
 			LastDelta: 0,
 		},
 		{
-			Nums:       repeatFloat64(-1.5, 130),
+			Nums:         repeatFloat64(-1.5, 130),
-			Name:       "run switch to literal after h-byte overflow (negative version)",
+			Name:         "run switch to literal after h-byte overflow (negative version)",
-			Offset:     0,
+			RewindOffset: 0,
-			ChangeSize: 2,
+			ChangeSize:   2,
 			Buf: []byte{
 				0x9d, // base value
 				0x80, // delta 0
@@ -670,9 +672,9 @@ var (
 				1.6,
 				1.7,
 			},
-			Name:       "increment literal",
+			Name:         "increment literal",
-			Offset:     1,
+			RewindOffset: 1,
-			ChangeSize: 2,
+			ChangeSize:   2,
 			Buf: []byte{
 				0x9e, // base value
 				0x80, // delta 0
@@ -690,9 +692,9 @@ var (
 				-1.6,
 				-1.7,
 			},
-			Name:       "increment literal (negative version)",
+			Name:         "increment literal (negative version)",
-			Offset:     1,
+			RewindOffset: 1,
-			ChangeSize: 2,
+			ChangeSize:   2,
 			Buf: []byte{
 				0x9d, // base value
 				0x80, // delta 0
@@ -710,9 +712,9 @@ var (
 				1.4,
 				1.3,
 			},
-			Name:       "increment literal (negative delta)",
+			Name:         "increment literal (negative delta)",
-			Offset:     1,
+			RewindOffset: 1,
-			ChangeSize: 2,
+			ChangeSize:   2,
 			Buf: []byte{
 				0x9e, // base value
 				0x80, // delta 0
@@ -730,9 +732,9 @@ var (
 				-1.4,
 				-1.3,
 			},
-			Name:       "increment literal (positive delta, negative version)",
+			Name:         "increment literal (positive delta, negative version)",
-			Offset:     1,
+			RewindOffset: 1,
-			ChangeSize: 2,
+			ChangeSize:   2,
 			Buf: []byte{
 				0x9d, // base value
 				0x80, // delta 0
@@ -760,11 +762,11 @@ func TestInstantDeltaCompressor(t *testing.T) {
 		c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize)
 		for _, num := range testCase.Nums {
 			report = c.Evaluate(tmp, num)
-			c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
+			c.Append(report.RewindOffset, tmp[:report.ChangeSize], num, report.Delta)
 		}
-		if report.Offset != testCase.Offset {
+		if report.RewindOffset != testCase.RewindOffset {
-			t.Fatalf("%s: got offset %d are not equal expected %d",
+			t.Fatalf("%s: got rewindOffset %d are not equal expected %d",
-				testCase.Name, report.Offset, testCase.Offset)
+				testCase.Name, report.RewindOffset, testCase.RewindOffset)
 		}
 		if report.ChangeSize != testCase.ChangeSize {
 			t.Fatalf("%s: got changeSize %d are not equal expected %d",
@@ -798,7 +800,7 @@ func TestRestoreInstantDeltaCompressor(t *testing.T) {
 		c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize)
 		for _, num := range testCase.Nums {
 			report = c.Evaluate(tmp, num)
-			c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
+			c.Append(report.RewindOffset, tmp[:report.ChangeSize], num, report.Delta)
 		}
 		restored := NewValueDeltaCompressor(metricType, fracDigits, buf, c.Size())
 		if restored.baseValue != testCase.BaseValue {
@@ -825,7 +827,7 @@ func TestInstantDeltaDecompressorFromState(t *testing.T) {
 		c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize)
 		for _, num := range testCase.Nums {
 			report := c.Evaluate(tmp, num)
-			c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
+			c.Append(report.RewindOffset, tmp[:report.ChangeSize], num, report.Delta)
 		}
 
 		d := c.CreateDecompressor(metricType, fracDigits)
@@ -859,7 +861,7 @@ func TestInstantDeltaDecompressorFromEnd(t *testing.T) {
 		c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize)
 		for _, num := range testCase.Nums {
 			report := c.Evaluate(tmp, num)
-			c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
+			c.Append(report.RewindOffset, tmp[:report.ChangeSize], num, report.Delta)
 		}
 
 		d := NewValueDeltaDecompressor(metricType, fracDigits)
@@ -890,16 +892,16 @@ var (
 		Buf          []byte
 		LastUnixtime uint32
 		LastDelta    uint32
-		Offset       int
+		RewindOffset int
 		ChangeSize   int
 	}{
 		{
 			Nums: []uint32{
 				1780777000,
 			},
-			Name:       "add 1st value",
+			Name:         "add 1st value",
-			Offset:     0,
+			RewindOffset: 0,
-			ChangeSize: 4,
+			ChangeSize:   4,
 			Buf: []byte{
 				0x00, 0x00, 0x00, 0x00,
 				0x28, 0x80, 0x24, 0x6a, // since
@@ -912,9 +914,9 @@ var (
 				1780777000,
 				1780777060, // +60
 			},
-			Name:       "add 1st delta",
+			Name:         "add 1st delta",
-			Offset:     0,
+			RewindOffset: 0,
-			ChangeSize: 2,
+			ChangeSize:   2,
 			Buf: []byte{
 				0x00, 0x00,
 				0x80,                   // h-byte (literal, len=1)
@@ -930,9 +932,9 @@ var (
 				1780777060, // +60
 				1780777120, // +60
 			},
-			Name:       "literal changed to run",
+			Name:         "literal changed to run",
-			Offset:     1,
+			RewindOffset: 1,
-			ChangeSize: 1,
+			ChangeSize:   1,
 			Buf: []byte{
 				0x00, 0x00,
 				0x00,                   // h-byte (run, len=2)
@@ -949,9 +951,9 @@ var (
 				1780777130, // +70
 				1780777200, // +70
 			},
-			Name:       "literal decrease by 1 and switch to run",
+			Name:         "literal decrease by 1 and switch to run",
-			Offset:     2,
+			RewindOffset: 2,
-			ChangeSize: 3,
+			ChangeSize:   3,
 			Buf: []byte{
 				0x00,                   // h-byte (run, len=2)
 				0xc6,                   // delta 70
@@ -969,9 +971,9 @@ var (
 				1780777120, // +60
 				1780777180, // +60
 			},
-			Name:       "increment run",
+			Name:         "increment run",
-			Offset:     1,
+			RewindOffset: 1,
-			ChangeSize: 1,
+			ChangeSize:   1,
 			Buf: []byte{
 				0x00, 0x00,
 				0x01,                   // h-byte (run, len=2)
@@ -989,9 +991,9 @@ var (
 				1780777180, // +60
 				1780777200, // +20
 			},
-			Name:       "switch run to literal",
+			Name:         "switch run to literal",
-			Offset:     0,
+			RewindOffset: 0,
-			ChangeSize: 2,
+			ChangeSize:   2,
 			Buf: []byte{
 				0x80,                   // h-byte (literal, len=1)
 				0x94,                   // delta 20
@@ -1003,10 +1005,10 @@ var (
 			LastDelta:    20,
 		},
 		{
-			Nums:       generateProgression(1780777000, 60, 130),
+			Nums:         generateProgression(1780777000, 60, 130),
-			Name:       "increment run up to full filled h-byte",
+			Name:         "increment run up to full filled h-byte",
-			Offset:     1,
+			RewindOffset: 1,
-			ChangeSize: 1,
+			ChangeSize:   1,
 			Buf: []byte{
 				0x00, 0x00,
 				0x7f,                   // h-byte (run, len=129)
@@ -1017,10 +1019,10 @@ var (
 			LastDelta:    60,
 		},
 		{
-			Nums:       generateProgression(1780777000, 60, 131),
+			Nums:         generateProgression(1780777000, 60, 131),
-			Name:       "run switch to literal after h-byte overflow",
+			Name:         "run switch to literal after h-byte overflow",
-			Offset:     0,
+			RewindOffset: 0,
-			ChangeSize: 2,
+			ChangeSize:   2,
 			Buf: []byte{
 				0x80,                   // h-byte (literal, len=1)
 				0xbc,                   // delta 60
@@ -1037,9 +1039,9 @@ var (
 				1780777060, // +60
 				1780777130, // +70
 			},
-			Name:       "increment literal",
+			Name:         "increment literal",
-			Offset:     1,
+			RewindOffset: 1,
-			ChangeSize: 2,
+			ChangeSize:   2,
 			Buf: []byte{
 				0x00,
 				0x81,                   // h-byte (literal, len=2)
@@ -1064,11 +1066,11 @@ func TestTimeDeltaCompressor(t *testing.T) {
 		c := NewTimeDeltaCompressor(buf, payloadSize)
 		for _, num := range testCase.Nums {
 			report = c.Evaluate(tmp, num)
-			c.Append(report.Offset, tmp[:report.ChangeSize], num)
+			c.Append(report.RewindOffset, tmp[:report.ChangeSize], num)
 		}
-		if report.Offset != testCase.Offset {
+		if report.RewindOffset != testCase.RewindOffset {
-			t.Fatalf("%s: got offset %d are not equal expected %d",
+			t.Fatalf("%s: got rewindOffset %d are not equal expected %d",
-				testCase.Name, report.Offset, testCase.Offset)
+				testCase.Name, report.RewindOffset, testCase.RewindOffset)
 		}
 		if report.ChangeSize != testCase.ChangeSize {
 			t.Fatalf("%s: got changeSize %d are not equal expected %d",
@@ -1100,7 +1102,7 @@ func TestRestoreTimeDeltaCompressor(t *testing.T) {
 		c := NewTimeDeltaCompressor(buf, payloadSize)
 		for _, num := range testCase.Nums {
 			report = c.Evaluate(tmp, num)
-			c.Append(report.Offset, tmp[:report.ChangeSize], num)
+			c.Append(report.RewindOffset, tmp[:report.ChangeSize], num)
 		}
 		restored := NewTimeDeltaCompressor(buf, c.Size())
 		if restored.lastUnixtime != testCase.LastUnixtime {
@@ -1118,16 +1120,17 @@ func TestTimeDeltaDecompressorFromState(t *testing.T) {
 	for _, testCase := range timeTestCases {
 		var (
 			tmp         = make([]byte, tmpTimeSize)
-			buf         = make([]byte, 8)
+			buf         = make([]byte, 16)
 			payloadSize = 0
 			decodedNums []uint32
 		)
 		c := NewTimeDeltaCompressor(buf, payloadSize)
 		for _, num := range testCase.Nums {
 			report := c.Evaluate(tmp, num)
-			c.Append(report.Offset, tmp[:report.ChangeSize], num)
+			c.Append(report.RewindOffset, tmp[:report.ChangeSize], num)
 		}
 
+		fmt.Println("-----")
 		d := c.CreateDecompressor()
 		for {
 			num, done := d.NextValue()
@@ -1157,7 +1160,7 @@ func TestTimeDeltaDecompressorFromEnd(t *testing.T) {
 		c := NewTimeDeltaCompressor(buf, payloadSize)
 		for _, num := range testCase.Nums {
 			report := c.Evaluate(tmp, num)
-			c.Append(report.Offset, tmp[:report.ChangeSize], num)
+			c.Append(report.RewindOffset, tmp[:report.ChangeSize], num)
 		}
 
 		c.ReplaceSinceWithUntil()
@@ -1206,3 +1209,91 @@ func generateProgression(start uint32, delta uint32, n int) []uint32 {
 	}
 	return nums
 }
+
+func generateTimestamps(days int) (timestamps []uint32) {
+	var (
+		minutes     = []int{14, 29, 44, 59}
+		hoursPerDay = 24
+		totalHours  = days * hoursPerDay
+		since       = time.Now().AddDate(0, 0, -days)
+	)
+
+	for i := range totalHours {
+		hourTime := since.Add(time.Duration(i) * time.Hour)
+		for _, m := range minutes {
+			measureTime := time.Date(
+				hourTime.Year(),
+				hourTime.Month(),
+				hourTime.Day(),
+				hourTime.Hour(),
+				m, // minutes
+				0, // seconds
+				0, // nanoseconds
+				time.Local,
+			)
+			timestamps = append(timestamps, uint32(measureTime.Unix()))
+		}
+	}
+	return
+}
+
+func TestXXX(t *testing.T) {
+
+	var (
+		timestamps  = generateTimestamps(20)[:1231]
+		tmp         = make([]byte, tmpTimeSize)
+		buf         = make([]byte, 64)
+		payloadSize = 0
+		decodedNums []uint32
+	)
+	c := NewTimeDeltaCompressor(buf, payloadSize)
+	for _, num := range timestamps {
+		report := c.Evaluate(tmp, num)
+		c.Append(report.RewindOffset, tmp[:report.ChangeSize], num)
+	}
+
+	d := c.CreateDecompressor()
+	for {
+		num, done := d.NextValue()
+		if done {
+			break
+		}
+		decodedNums = append(decodedNums, num)
+	}
+
+	slices.Reverse(decodedNums)
+
+	if !slices.Equal(timestamps, decodedNums) {
+		t.Fatalf("got nums %v not equal expected %v",
+			decodedNums, timestamps)
+	}
+}
+
+func TestYYY(t *testing.T) {
+	buf, err := textutil.ParseHex("50 04 87 ac 6b 05 6a")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//var decodedNums []uint32
+	var prev uint32
+
+	d := NewTimeDeltaDecompressor()
+	d.RestoreFromEnd(buf)
+
+	for {
+		num, done := d.NextValue()
+		if done {
+			break
+		}
+		fmt.Println(num, prev-num)
+		prev = num
+		//decodedNums = append(decodedNums, num)
+	}
+
+	// //slices.Reverse(decodedNums)
+
+	// for _, num := range decodedNums {
+
+	// }
+}

enc/time_delta.go

@@ -1,7 +1,6 @@
 package enc
 
 import (
-	"fmt"
 	"io"
 	"log"
 
@@ -92,11 +91,11 @@ func (s *TimeDeltaCompressor) restore(payloadSize int) {
 
 func (s *TimeDeltaCompressor) Evaluate(tmp []byte, timestamp uint32) qb.TimeEvaluationReport {
 	var (
-		delta        = timestamp - s.lastUnixtime
 		rewindOffset int
 		i            int
 	)
 	if s.pos < len(s.buf) {
+		delta := timestamp - s.lastUnixtime
 		if s.lastDelta > 0 {
 			h := s.buf[s.pos]
 			if h < 128 {
@@ -108,36 +107,36 @@ func (s *TimeDeltaCompressor) Evaluate(tmp []byte, timestamp uint32) qb.TimeEval
 					rewindOffset = 1 // перезапис h
 				} else {
 					// endSeries
-					n, _ := bin.PutVarUint64(tmp, uint64(delta))
-					i += n
 					tmp[i] = 128 // start new literal (length=1)
 					i++
+					n, _ := bin.PutVarUint64(tmp[i:], uint64(delta))
+					i += n
 				}
 			} else {
 				// literal
 				if delta != s.lastDelta {
 					if h < 255 {
 						// incrementLiteral
-						n, _ := bin.PutVarUint64(tmp, uint64(delta))
-						i += n
 						tmp[i] = h + 1
 						i++
+						n, _ := bin.PutVarUint64(tmp[i:], uint64(delta))
+						i += n
 						rewindOffset = 1 // перезапис h
 					} else {
 						// endSeries
-						n, _ := bin.PutVarUint64(tmp, uint64(delta))
-						i += n
 						tmp[i] = 128 // start new literal (length=1)
 						i++
+						n, _ := bin.PutVarUint64(tmp[i:], uint64(delta))
+						i += n
 					}
 				} else {
 					// startRun
 					if h > 128 {
-						tmp[i] = h - 1 // зменшую довжину попередньої серії на 1
+						tmp[i] = 0 // start new run (length=2)
 						i++
 						n, _ := bin.PutVarUint64(tmp[i:], uint64(delta))
 						i += n
-						tmp[i] = 0 // start new run (length=2)
+						tmp[i] = h - 1 // зменшую довжину попередньої серії на 1
 						i++
 						rewindOffset = 1 + n // перезапис пари delta/h
 					} else {
@@ -148,11 +147,10 @@ func (s *TimeDeltaCompressor) Evaluate(tmp []byte, timestamp uint32) qb.TimeEval
 				}
 			}
 		} else {
-			// add 1st delta
-			n, _ := bin.PutVarUint64(tmp, uint64(delta))
-			i += n
 			tmp[i] = 128 // start new literal (length=1)
 			i++
+			n, _ := bin.PutVarUint64(tmp[i:], uint64(delta))
+			i += n
 		}
 	} else {
 		bin.PutUint32(tmp, timestamp) // 1st timestamp (since)
@@ -167,16 +165,16 @@ func (s *TimeDeltaCompressor) Evaluate(tmp []byte, timestamp uint32) qb.TimeEval
 }
 
 func (s *TimeDeltaCompressor) Append(rewindOffset int, change []byte, timestamp uint32) {
-	if s.pos < len(s.buf) {
+	// fmt.Printf("----\nchange: % x\n", change)
-		i := s.pos + rewindOffset - 1 // -1, because s.pos always points to h byte
+	// fmt.Printf("rewindOffset: %d\n", rewindOffset)
-		for _, b := range change {
+	// fmt.Printf("pos: %d\n", s.pos)
-			s.buf[i] = b
+	// fmt.Printf("buf: %d\n", len(s.buf))
-			i--
+
-		}
+	if s.lastUnixtime > 0 {
 		s.lastDelta = timestamp - s.lastUnixtime
-	} else {
-		copy(s.buf[len(s.buf)-4:], change) // 4b since
 	}
+	idx := s.pos - len(change) + rewindOffset
+	copy(s.buf[idx:], change)
 	s.pos -= len(change) - rewindOffset
 	s.lastUnixtime = timestamp
 }
@@ -199,6 +197,10 @@ func (s *TimeDeltaCompressor) getState() qb.TimeDeltaCapturedState {
 			bound = s.pos // only since encoded
 		}
 		state.Payload = s.buf[bound:]
+		// fmt.Printf("% x\n", s.buf)
+		// fmt.Printf("% x\n", s.buf[bound:])
+		// fmt.Printf("h: % x\n", state.H)
+		// fmt.Printf("lastDelta: %d\n", s.lastDelta)
 	}
 	return state
 }
@@ -213,7 +215,7 @@ func (s *TimeDeltaCompressor) ForgetCapturedState() {
 }
 
 func (s *TimeDeltaCompressor) Tail(offset int) []byte {
-	fmt.Println("time tail:", s.pos, len(s.buf), offset)
+	//fmt.Println("time tail:", s.pos, len(s.buf), offset)
 	return s.buf[s.pos : len(s.buf)-offset]
 }
 

enc/value_delta.go

@@ -157,6 +157,7 @@ func (s *ValueDeltaCompressor) Append(rewindOffset int, change []byte, value flo
 	//fmt.Printf("buf before: % x\n", s.buf[:s.pos])
 	copy(s.buf[s.pos-rewindOffset:], change)
 	s.pos += len(change) - rewindOffset
+	//fmt.Printf("v size %d\n", s.pos)
 	//fmt.Printf("buf after: % x\n", s.buf[:s.pos])
 	//fmt.Printf("buf after: % x\n", s.buf[:s.pos])
 }
@@ -342,6 +343,7 @@ func (s *ValueDeltaDecompressor) readBaseValue() {
 }
 
 func (s *ValueDeltaDecompressor) NextValue() (value float64, done bool) {
+	//fmt.Printf("v pos: %d of %d\n", s.pos, len(s.buf))
 	if s.done {
 		return 0, true
 	}

examples/play/main.go

@@ -302,28 +302,31 @@ func sendRequests(conn *client.Connection) {
 	// 	fmt.Printf("\nCumulative metric %d added\n", cumulativeMetricID)
 	// }
 
-	// // GET CUMULATIVE METRIC
+	// GET CUMULATIVE METRIC
 
-	// cMetric, err := conn.GetMetric(cumulativeMetricID)
+	cMetric, err := conn.GetMetric(cumulativeMetricID)
-	// if err != nil {
+	if err != nil {
-	// 	log.Fatalf("conn.GetMetric: %s\n", err)
+		log.Fatalf("conn.GetMetric: %s\n", err)
-	// } else {
+	} else {
-	// 	fmt.Printf(`
+		fmt.Printf(`
-	// GetMetric:
+	GetMetric:
-	//   metricID: %d
+	  metricID: %d
-	//   metricType: %s
+	  metricType: %s
-	//   fracDigits: %d
+	  fracDigits: %d
-	// `,
+	`,
-	// 		cMetric.MetricID, metricTypeToName[cMetric.MetricType], cMetric.FracDigits)
+			cMetric.MetricID, metricTypeToName[cMetric.MetricType], cMetric.FracDigits)
+	}
+
+	// APPEND MEASURES
+
+	// cumulativeMeasures := GenerateCumulativeMeasures(200)
+	// if len(cumulativeMeasures) > 65535 {
+	// 	cumulativeMeasures = cumulativeMeasures[:65535]
 	// }
 
-	// // APPEND MEASURES
-
-	// cumulativeMeasures := GenerateCumulativeMeasures(62)
-
 	// result, err := conn.AppendMeasures(proto.AppendMeasuresReq{
 	// 	MetricID: cumulativeMetricID,
-	// 	Measures: cumulativeMeasures[:1000],
+	// 	Measures: cumulativeMeasures,
 	// })
 	// if err != nil {
 	// 	log.Fatalf("conn.AppendMeasures: %s\n", err)
@@ -344,7 +347,6 @@ func sendRequests(conn *client.Connection) {
 	// }
 
 	// LIST CUMULATIVE MEASURES
-	var cumulativeList []proto.CumulativeMeasure
 
 	// 	lastTimestamp = cumulativeMeasures[len(cumulativeMeasures)-1].Timestamp
 	// 	until = time.Unix(int64(lastTimestamp), 0)
@@ -368,14 +370,15 @@ func sendRequests(conn *client.Connection) {
 
 	// LIST ALL CUMULATIVE MEASURES
 
+	var cumulativeList []proto.CumulativeMeasure
 	cumulativeList, err = conn.ListAllCumulativeMeasures(cumulativeMetricID)
 	if err != nil {
 		log.Fatalf("conn.ListAllCumulativeMeasures: %s\n", err)
 	} else {
 		fmt.Printf("\nListAllCumulativeMeasures (last 15 items):\n")
 		//fmt.Printf("%#v\n", cumulativeList)
-		for _, item := range cumulativeList[:15] {
+		for idx, item := range cumulativeList {
-			fmt.Printf("  %s => %.2f\n", formatTime(item.Timestamp), item.Value)
+			fmt.Printf("  %d: %s => %.2f\n", idx, formatTime(item.Timestamp), item.Value)
 		}
 	}
 
@@ -518,6 +521,7 @@ func GenerateCumulativeMeasures(days int) []proto.Measure {
 		totalValue  float64
 	)
 
+	idx := 0
 	for i := range totalHours {
 		hourTime := since.Add(time.Duration(i) * time.Hour)
 		for _, m := range minutes {
@@ -539,6 +543,16 @@ func GenerateCumulativeMeasures(days int) []proto.Measure {
 			measures = append(measures, measure)
 
 			totalValue += rand.Float64()
+
+			if idx < 1500 {
+				fmt.Println(idx, measure.Timestamp)
+			}
+			idx++
+		}
+	}
+	for i := 1; i < len(measures); i++ {
+		if measures[i].Timestamp <= measures[i-1].Timestamp {
+			fmt.Println("FUCK", i)
 		}
 	}
 	return measures
@@ -580,3 +594,9 @@ func GenerateInstantMeasures(days int, baseValue float64) []proto.Measure {
 	}
 	return measures
 }
+
+//02 88 30 3e 88 31 1d 88 31 3e a3 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 28 68 05 6a
+//02 88 30 3e 88 31 1d 88 31 3e a3 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 28 68 05 6a
+
+//02 88 4b 4c 88 4c 06 88 4c 39 ae 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 30 1b 2c 6a
+//02 88 4b 4c 88 4c 06 88 4c 39 ae 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 7f 04 87 30 1b 2c 6a

freelist/freelist.go

@@ -80,13 +80,13 @@ func (s *FreeList) AddPageNumbers(pageNumbers []uint32) (err error) {
 		// викидаю номери сторінок, які записав на диск
 		copy(s.free, s.free[s.pointersOnPage:])
 		s.free = s.free[:len(s.free)-s.pointersOnPage]
-		fmt.Println("free:", s.free)
+		//fmt.Println("free:", s.free)
 	}
 	return
 }
 
 func (s *FreeList) save() error {
-	fmt.Println("save:", s.free[:s.pointersOnPage])
+	//fmt.Println("save:", s.free[:s.pointersOnPage])
 	buf := make([]byte, s.pageSize)
 	i := crcSize
 	for _, pageNo := range s.free[:s.pointersOnPage] {
@@ -108,7 +108,7 @@ func (s *FreeList) save() error {
 		off = int64(s.basePagesCount * s.pageSize)
 		s.basePagesCount++
 	}
-	fmt.Println("write at:", off)
+	//fmt.Println("write at:", off)
 	n, err := file.WriteAt(buf, off)
 	if err != nil {
 		return err

go.mod

@@ -3,7 +3,13 @@ module gordenko.dev/dima/qb
 go 1.24.2
 
 require (
+	github.com/go-sql-driver/mysql v1.10.0
 	gopkg.in/ini.v1 v1.67.1
 	gordenko.dev/dima/bin v0.0.0-20260612161453-4ee9be3474fb
+	gordenko.dev/dima/fixme v0.0.0-20230701203753-4c2b6c1eef52
+	gordenko.dev/dima/pretty v0.0.0-20221225212746-0c27d8c0ac69
+	gordenko.dev/dima/qx v0.0.0-20231123051353-23dce7c0ce88
 	gordenko.dev/dima/textutil v0.0.0-20221225052909-270513dcbbbf
 )
+
+require filippo.io/edwards25519 v1.2.0 // indirect

go.sum

@@ -1,6 +1,10 @@
+filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
+filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
+github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -20,5 +24,11 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gordenko.dev/dima/bin v0.0.0-20260612161453-4ee9be3474fb h1:XBYZbK5Z+yfpKmSR2wuPLyW14ReCRSNdOlL5MWro8ns=
 gordenko.dev/dima/bin v0.0.0-20260612161453-4ee9be3474fb/go.mod h1:up64wpJp9xI+HqACtMDBIfPNUH6BYY/b3inKpUfrqDQ=
+gordenko.dev/dima/fixme v0.0.0-20230701203753-4c2b6c1eef52 h1:g3f99mV2NtLsN1/O7qkV9CwfEGhcWFS0Y5FX12On/NU=
+gordenko.dev/dima/fixme v0.0.0-20230701203753-4c2b6c1eef52/go.mod h1:7tMDvA2ej8e7VwXIlcIZNtUe/Tk5jFAK498WqC+aMYE=
+gordenko.dev/dima/pretty v0.0.0-20221225212746-0c27d8c0ac69 h1:nyJ3mzTQ46yUeMZCdLyYcs7B5JCS54c67v84miyhq2E=
+gordenko.dev/dima/pretty v0.0.0-20221225212746-0c27d8c0ac69/go.mod h1:AxgKDktpqBVyIOhIcP+nlCpK+EsJyjN5kPdqyd8euVU=
+gordenko.dev/dima/qx v0.0.0-20231123051353-23dce7c0ce88 h1:/9KsVGy3tsDR4SmSs9fZQx1hslYY89KqNpIa55kKuX0=
+gordenko.dev/dima/qx v0.0.0-20231123051353-23dce7c0ce88/go.mod h1:CR5GISpjHFHZ9HeSIo3fYRyp1+kVx0R9exzBYsIuB3c=
 gordenko.dev/dima/textutil v0.0.0-20221225052909-270513dcbbbf h1:GCL/m736iGDy3LgOm6iuvkxlGhUEyjfP7zgwZRkTYOg=
 gordenko.dev/dima/textutil v0.0.0-20221225052909-270513dcbbbf/go.mod h1:M3e8S3N1USCMYmL6Q5uDQJaxfzBlWHnTX7rxCJQ6iJs=

mqe/aggregated.go

@@ -0,0 +1,340 @@
+package mqe
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"gordenko.dev/dima/qb/timeutil"
+)
+
+type AggregatedMeasure struct {
+	Period int64 `json:"p"`
+	Since  int64 `json:"s"`
+	Until  int64 `json:"u"`
+	// для cumulative метрики 2 значения - value и total
+	// для instant метрики от 1 до 3 значений - min, max, avg
+	Values []float64 `json:"v"`
+}
+
+type AggregatedFilter struct {
+	Since             time.Time
+	Until             time.Time
+	GroupBy           GroupBy
+	LastDayOfMonth    int // например, конец месяца 25 число
+	FirstHourOfDay    int // например день начинается в 6:00
+	InstantMetrics    []InstantMetricAndFuncs
+	CumulativeMetrics []int64
+}
+
+type InstantMetricAndFuncs struct {
+	MetricID       int64
+	AggregateFuncs byte
+}
+
+func (s *MeasureQueryEngine) SelectAggregatedData(req AggregatedFilter) (resultMap map[int64][]AggregatedMeasure, err error) {
+	since := req.Since
+	until := req.Until
+	// Валидация
+	switch req.GroupBy {
+	case ByHour, ByDay, ByMonth:
+		// pass
+	default:
+		err = fix.Field(WrongValue, "groupBy")
+		return
+	}
+
+	if req.FirstHourOfDay < 0 || req.FirstHourOfDay > 23 {
+		err = fix.Field(InvalidFirstHourOfDay, "firstHourOfDay")
+		return
+	}
+
+	if req.LastDayOfMonth < 0 || req.LastDayOfMonth > 28 {
+		err = fix.Field(InvalidLastDayOfMonth, "lastDayOfMonth")
+		return
+	}
+
+	// Корректируем, ВСЕГДА на последнюю секунду суток
+	//until = timeutil.LastSecondInPeriod(until, "d")
+
+	// 	fmt.Printf(`SelectAggregatedData: {
+	// 	Since: %s
+	// 	Until: %s
+	// 	GroupBy: %s
+	// 	FirstHourOfDay: %d
+	// 	LastDayOfMonth: %d
+	// }
+	// `, since, until, req.GroupBy, req.FirstHourOfDay, req.LastDayOfMonth)
+	// FIX проверить как добавляются часы в DST часовых поясах
+	// Для req.FirstHourOfDay=4 since должен быть 4:00:00, until должен быть
+	// 3:59:59 следующего дня
+	if req.FirstHourOfDay > 0 {
+		since = since.Add(time.Duration(req.FirstHourOfDay) * time.Hour)
+		until = until.Add(time.Duration(req.FirstHourOfDay) * time.Hour)
+	}
+
+	sinceUnixtime := since.Unix()
+	untilUnixtime := until.Unix()
+
+	if sinceUnixtime >= untilUnixtime {
+		err = fix.Error(WrongDateRange)
+		return
+	}
+
+	var (
+		cumulativeTasks []*AggregatedCumulativeTask
+		instantTasks    []*AggregatedInstantTask
+		wg              = new(sync.WaitGroup)
+	)
+
+	// Важно вирутальная метрика зависит от реальных, которые к объекту могут быть
+	// привязаны или нет. Если нет - не возвращать.
+	resultMap = make(map[int64][]AggregatedMeasure)
+
+	if len(req.CumulativeMetrics) > 0 {
+		for _, metricID := range req.CumulativeMetrics {
+			task := &AggregatedCumulativeTask{
+				In: AggregatedMeasuresFilter{
+					MetricID:       metricID,
+					Since:          sinceUnixtime,
+					Until:          untilUnixtime,
+					GroupBy:        req.GroupBy,
+					LastDayOfMonth: req.LastDayOfMonth,
+					FirstHourOfDay: req.FirstHourOfDay,
+				},
+				WaitGroup: wg,
+			}
+
+			wg.Add(1)
+			s.aggregatedCumulativeCh <- task
+			cumulativeTasks = append(cumulativeTasks, task)
+		}
+	}
+
+	if len(req.InstantMetrics) > 0 {
+		for _, metric := range req.InstantMetrics {
+			task := &AggregatedInstantTask{
+				In: AggregatedInstantMeasuresIn{
+					MetricID:       metric.MetricID,
+					Since:          sinceUnixtime,
+					Until:          untilUnixtime,
+					GroupBy:        req.GroupBy,
+					LastDayOfMonth: req.LastDayOfMonth,
+					FirstHourOfDay: req.FirstHourOfDay,
+					Flags:          metric.AggregateFuncs, // ФЛАГИ !!!
+				},
+				WaitGroup: wg,
+			}
+
+			wg.Add(1)
+			s.aggregatedInstantCh <- task
+			instantTasks = append(instantTasks, task)
+		}
+	}
+
+	wg.Wait()
+
+	for _, task := range cumulativeTasks {
+		if task.Err != nil {
+			err = fmt.Errorf("get data for the metric %d: %s", task.In.MetricID, task.Err)
+			return
+		}
+		resultMap[task.In.MetricID] = task.Result
+	}
+
+	for _, task := range instantTasks {
+		if task.Err != nil {
+			err = fmt.Errorf("get data for the metric %d: %s", task.In.MetricID, task.Err)
+			return
+		}
+		resultMap[task.In.MetricID] = task.Result
+	}
+	return
+}
+
+type IndividualAggregatedFilter struct {
+	Since             time.Time
+	Until             time.Time
+	GroupBy           GroupBy
+	InstantMetrics    []IndividualInstantMetric
+	CumulativeMetrics []IndividualCumulativeMetric
+}
+
+type IndividualInstantMetric struct {
+	MetricID       int64
+	AggregateFuncs byte
+	LastDayOfMonth int // например, конец месяца 25 число
+	FirstHourOfDay int // например день начинается в 6:00
+}
+
+type IndividualCumulativeMetric struct {
+	MetricID       int64
+	LastDayOfMonth int // например, конец месяца 25 число
+	FirstHourOfDay int // например день начинается в 6:00
+}
+
+func (s *MeasureQueryEngine) SelectIndividualAggregatedData(req IndividualAggregatedFilter) (resultMap map[int64][]AggregatedMeasure, err error) {
+	if req.Since.IsZero() {
+		err = fmt.Errorf("zero Since")
+		return
+	}
+
+	if req.Until.IsZero() {
+		err = fmt.Errorf("zero Until")
+		return
+	}
+
+	if req.Since.After(req.Until) {
+		err = fmt.Errorf("wrong time range: since %s after until %s", req.Since, req.Until)
+		return
+	}
+
+	// Корректируем, ВСЕГДА на первую и последнюю секунду суток
+	req.Since = timeutil.FirstSecondInPeriod(req.Since, "d")
+	req.Until = timeutil.LastSecondInPeriod(req.Until, "d")
+
+	// Валидация
+	switch req.GroupBy {
+	case ByHour, ByDay, ByMonth:
+		// pass
+	default:
+		err = fix.Field(WrongValue, "groupBy")
+		return
+	}
+
+	// 	fmt.Printf(`SelectIndividualAggregatedData: {
+	// 	Since: %s
+	// 	Until: %s
+	// 	GroupBy: %s
+	// }
+	// `, req.Since, req.Until, req.GroupBy)
+	// Важно вирутальная метрика зависит от реальных, которые к объекту могут быть
+	// привязаны или нет. Если нет - не возвращать.
+
+	resultMap = make(map[int64][]AggregatedMeasure)
+
+	var (
+		unexpectedErrors []error
+		errorMutex       sync.Mutex
+
+		wg sync.WaitGroup
+	)
+
+	wg.Add(len(req.CumulativeMetrics) + len(req.InstantMetrics))
+
+	if len(req.CumulativeMetrics) > 0 {
+		for _, m := range req.CumulativeMetrics {
+			go func(metric IndividualCumulativeMetric) {
+				if metric.FirstHourOfDay < 0 || metric.FirstHourOfDay > 23 {
+					err = fix.Field(InvalidFirstHourOfDay, "firstHourOfDay")
+					return
+				}
+
+				if metric.LastDayOfMonth < 0 || metric.LastDayOfMonth > 28 {
+					err = fix.Field(InvalidLastDayOfMonth, "lastDayOfMonth")
+					return
+				}
+
+				since := req.Since
+				until := req.Until
+				// FIX проверить как добавляются часы в DST часовых поясах
+				// Для req.FirstHourOfDay=4 since должен быть 4:00:00, until должен быть
+				// 3:59:59 следующего дня
+				if metric.FirstHourOfDay > 0 {
+					since = req.Since.Add(time.Duration(metric.FirstHourOfDay) * time.Hour)
+					until = req.Until.Add(time.Duration(metric.FirstHourOfDay) * time.Hour)
+				}
+
+				result, err := s.listAggregatedCumulativeMeasures(AggregatedMeasuresFilter{
+					MetricID:       metric.MetricID,
+					Since:          since.Unix(),
+					Until:          until.Unix(),
+					GroupBy:        req.GroupBy,
+					LastDayOfMonth: metric.LastDayOfMonth,
+					FirstHourOfDay: metric.FirstHourOfDay,
+				})
+				if err != nil {
+					err = fmt.Errorf("ListAggregatedCumulativeMeasures: %s", err)
+
+					errorMutex.Lock()
+					unexpectedErrors = append(unexpectedErrors, err)
+					errorMutex.Unlock()
+				} else {
+					errorMutex.Lock()
+					resultMap[metric.MetricID] = result
+					errorMutex.Unlock()
+				}
+
+				wg.Done()
+			}(m)
+		}
+	}
+
+	if len(req.InstantMetrics) > 0 {
+		for _, m := range req.InstantMetrics {
+			go func(metric IndividualInstantMetric) {
+				if metric.FirstHourOfDay < 0 || metric.FirstHourOfDay > 23 {
+					err = fix.Field(InvalidFirstHourOfDay, "firstHourOfDay")
+					return
+				}
+
+				if metric.LastDayOfMonth < 0 || metric.LastDayOfMonth > 28 {
+					err = fix.Field(InvalidLastDayOfMonth, "lastDayOfMonth")
+					return
+				}
+
+				since := req.Since
+				until := req.Until
+				// FIX проверить как добавляются часы в DST часовых поясах
+				// Для req.FirstHourOfDay=4 since должен быть 4:00:00, until должен быть
+				// 3:59:59 следующего дня
+				if metric.FirstHourOfDay > 0 {
+					since = req.Since.Add(time.Duration(metric.FirstHourOfDay) * time.Hour)
+					until = req.Until.Add(time.Duration(metric.FirstHourOfDay) * time.Hour)
+				}
+
+				result, err := s.listAggregatedInstantMeasures(AggregatedInstantMeasuresIn{
+					MetricID:       metric.MetricID,
+					Since:          since.Unix(),
+					Until:          until.Unix(),
+					GroupBy:        req.GroupBy,
+					LastDayOfMonth: metric.LastDayOfMonth,
+					FirstHourOfDay: metric.FirstHourOfDay,
+					Flags:          metric.AggregateFuncs, // ФЛАГИ !!!
+				})
+				if err != nil {
+					err = fmt.Errorf("listAggregatedInstantMeasures: %s", err)
+
+					errorMutex.Lock()
+					unexpectedErrors = append(unexpectedErrors, err)
+					errorMutex.Unlock()
+				} else {
+					errorMutex.Lock()
+					resultMap[metric.MetricID] = result
+					errorMutex.Unlock()
+				}
+
+				wg.Done()
+			}(m)
+		}
+	}
+
+	wg.Wait()
+
+	// Проверяем что все запросы завершились удачно
+	if len(unexpectedErrors) > 0 {
+		// Нумеруем сообщения об ошибках и склеиваем в одно большое сообщение
+		var errorStrings []string
+
+		for idx, err := range unexpectedErrors {
+			errorStrings = append(errorStrings, fmt.Sprintf("\n#%d %s", idx+1, err))
+		}
+
+		err = fmt.Errorf("%d errors occured:%s", len(unexpectedErrors),
+			strings.Join(errorStrings, ""))
+		return
+	}
+
+	return
+}

mqe/cumulative.go

@@ -0,0 +1,585 @@
+package mqe
+
+import (
+	"database/sql"
+	"fmt"
+	"time"
+)
+
+// func applyCorrections(corrections []model.F64Correction, measures []Measure) []Measure {
+// 	for idx, measure := range measures {
+// 		var isCorrected bool
+// 		for _, correction := range corrections {
+// 			if measure.Time >= correction.Time {
+// 				measure.Value += correction.Value
+// 				isCorrected = true
+// 			}
+// 		}
+// 		if isCorrected {
+// 			measures[idx] = measure
+// 		}
+// 	}
+// 	return measures
+// }
+
+func applyCorrectionsToMeasure(corrections []_f64Correction, tm int64, value float64) float64 {
+	for _, correction := range corrections {
+		if tm >= correction.Time {
+			value += correction.Value
+		}
+	}
+	return value
+}
+
+func applyCorrectionsToMeasures(corrections []_f64Correction, measures []_measure) {
+	for idx, m := range measures {
+		for _, correction := range corrections {
+			if m.Time >= correction.Time {
+				m.Value += correction.Value
+			}
+		}
+		measures[idx] = m
+	}
+}
+
+func applyCorrectionsToRawMeasures(corrections []_f64Correction, measures []RawMeasure) {
+	for idx, m := range measures {
+		for _, correction := range corrections {
+			if m.Time >= correction.Time {
+				m.Value += correction.Value
+			}
+		}
+		measures[idx] = m
+	}
+}
+
+type _measure struct {
+	Time  int64
+	Value float64
+}
+
+// ВАЖНО!
+// ВО ВСЕХ ЗАПРОСАХ ОБЯЗАТЕЛЬНА КОРРЕКТНАЯ СОРТИРОВКА!
+
+type _f64Correction struct {
+	Time  int64
+	Value float64
+}
+
+func listF64CorrectionsTx(tx *sql.Tx, metricID int64) (list []_f64Correction, err error) {
+	rows, err := tx.Query(
+		"SELECT tm, value FROM metric_corrections WHERE metricID=? ORDER BY tm ASC",
+		metricID)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		}
+		return
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var correction _f64Correction
+		err = rows.Scan(&correction.Time, &correction.Value)
+		if err != nil {
+			return
+		}
+		list = append(list, correction)
+	}
+	return
+}
+
+// result = append(result, Measure{
+// 	Time: tm,
+// 	Values: []float64{
+// 		value,
+// 		value - prevValue,
+// 	},
+// })
+
+// listCumulativeMeasures - возвращает список CumulativeMeasure.
+// Метод ничего не знает про FirstHourOfDay, но извне границы Since и Until могут быть
+// скорректированы с учетом FirstHourOfDay. Это не поломает выборку.
+func (s *MeasureQueryEngine) listCumulativeMeasures(req MetricMeasuresFilter) (_ []Measure, err error) {
+	tx, err := s.db.Driver().Begin()
+	if err != nil {
+		return
+	}
+	defer tx.Rollback()
+
+	// corrections, err := listF64CorrectionsTx(tx, req.MetricID)
+	// if err != nil {
+	// 	return
+	// }
+
+	// ВАЖНО!
+	// Оптимизация! Запрашиваем больше данных чем нужно, чтобы посчитать Total для
+	// самого старого измерения в result.
+	// Отнимаем от since еще 1 час (с запасом) чтобы зацепить extendedMeasure
+	extendedSince := req.Since - 3600
+
+	rows, err := tx.Query(`
+	SELECT tm, value
+	FROM f64
+	WHERE metricID=? AND tm BETWEEN ? AND ?
+	ORDER BY tm ASC`,
+		req.MetricID, extendedSince, req.Until)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		}
+		return
+	}
+	defer rows.Close()
+
+	var (
+		result   []Measure
+		measures []_measure
+		prev     *_measure
+	)
+
+	// Вычитываем все найденные показания
+	for rows.Next() {
+		var m _measure
+		err = rows.Scan(&m.Time, &m.Value)
+		if err != nil {
+			return
+		}
+
+		if m.Time < req.Since {
+			// пока не найдено первое показание внутри диапазона обновлем extendedValue
+			prev = &_measure{
+				Time:  m.Time,
+				Value: m.Value,
+			}
+		} else {
+			measures = append(measures, m)
+			break
+		}
+	}
+
+	// Остальные показания вычитываем без проверок
+	for rows.Next() {
+		var m _measure
+		err = rows.Scan(&m.Time, &m.Value)
+		if err != nil {
+			return
+		}
+		measures = append(measures, m)
+	}
+
+	if err = rows.Err(); err != nil {
+		return
+	}
+
+	if len(measures) == 0 {
+		return
+	}
+
+	if prev == nil {
+		// extended not found
+		// extended показание не найдено, а значит для первого показания не сможем показать total.
+		// Нужен дополнительный запрос.
+		// Находим максимально свежее показание метрики до extendedSince, от которого и будем
+		// считать total.
+		var (
+			tm    int64
+			value float64
+		)
+
+		err = tx.QueryRow(`
+		SELECT tm, value
+		FROM f64
+		WHERE metricID=? AND tm < ?
+		ORDER BY tm DESC
+		LIMIT 1`,
+			req.MetricID, extendedSince).Scan(&tm, &value)
+
+		if err != nil {
+			if err != sql.ErrNoRows {
+				return
+			}
+			err = nil
+		} else {
+			prev = &_measure{
+				Time:  tm,
+				Value: value,
+			}
+		}
+	}
+
+	// ВАЖНО!
+	// Сперва применяем коррекции к показаниям, а затем рассчитываем сумму за период
+
+	// if len(corrections) > 0 {
+	// 	applyCorrectionsToMeasures(corrections, measures)
+	// 	if prev != nil {
+	// 		prev.Value = applyCorrectionsToMeasure(corrections, prev.Time, prev.Value)
+	// 	}
+	// }
+
+	var prevValue float64
+	if prev != nil {
+		prevValue = prev.Value
+	}
+
+	for _, m := range measures {
+		result = append(result, Measure{
+			Time: m.Time,
+			Values: []float64{
+				m.Value,
+				m.Value - prevValue,
+			},
+		})
+		prevValue = m.Value
+	}
+
+	// Корректируем самый первый тотал. Eсли prev показания не было -> total = 0
+	if prev == nil {
+		first := result[0]
+		first.Values[1] = 0
+		result[0] = first
+	}
+
+	return result, nil
+}
+
+type AggregatedMeasuresFilter struct {
+	MetricID       int64   `json:"metricID"`
+	Since          int64   `json:"since"`
+	Until          int64   `json:"until"`
+	GroupBy        GroupBy `json:"groupBy"`
+	LastDayOfMonth int     `json:"lastDayOfMonth"` // например, конец месяца 25 число
+	FirstHourOfDay int     `json:"firstHourOfDay"` // например день начинается в 6:00
+}
+
+// listAggregatedCumulativeMeasures - возвращает список AggregatedCumulativeMeasure,
+// сгрупированный по какому-то периоду
+func (s *MeasureQueryEngine) listAggregatedCumulativeMeasures(req AggregatedMeasuresFilter) (_ []AggregatedMeasure, err error) {
+	//pretty.PPrintln("listAggregatedCumulativeMeasures", req)
+
+	tx, err := s.db.Driver().Begin()
+	if err != nil {
+		return
+	}
+	defer tx.Rollback()
+
+	corrections, err := listF64CorrectionsTx(tx, req.MetricID)
+	if err != nil {
+		return
+	}
+
+	var (
+		// чтобы не экранировать символы в шаблоне строки
+		groupByFormat     string
+		parsePeriodLayout string
+	)
+
+	switch req.GroupBy {
+	case ByHour:
+		groupByFormat = "%Y%m%d%H"
+		parsePeriodLayout = hourPeriodLayout
+
+	case ByDay:
+		groupByFormat = "%Y%m%d"
+		parsePeriodLayout = dayPeriodLayout
+
+	case ByMonth:
+		groupByFormat = "%Y%m"
+		parsePeriodLayout = monthPeriodLayout
+
+	default:
+		// Защита от дурака
+		err = fmt.Errorf("unknown groupBy: %s", req.GroupBy)
+		return
+	}
+
+	// ВАЖНО!
+	// Оптимизация! Запрашиваем больше данных чем нужно, чтобы посчитать начало
+	// первого и конец последнего периодов без дополнительных запросов.
+	//
+
+	// Отнимаем 3 минуты чтобы зацепить последнее показание предыдущего периода
+	extendedSince := req.Since - 3*60
+	// Добавляем 1 час, чтобы зацепить первое показание следующего периода
+	extendedUntil := req.Until + 60*60
+
+	//fmt.Printf("extendedSince: %s\n\n", extendedSince)
+
+	// Для каждого периода выбираем первое и последнее (по времени) измерение.
+	// Если делать выборку одним запросом без UNION, а именно сделать
+	// INNER JOIN ... ON f64.tm=x.maxtm OR f64.tm=x.mintm - все начинает тормозить.
+	// Два запроса работают достаточно быстро.
+
+	// ВАЖНО!
+	// Не забыть про сортировку tm DESC!
+
+	dt, err := getPeriodDateTime(req.GroupBy, req.LastDayOfMonth, req.FirstHourOfDay)
+	if err != nil {
+		return
+	}
+
+	//fmt.Printf("DT: %s\n\n", dt)
+
+	query := fmt.Sprintf(`
+	WITH 
+		measures AS (SELECT * FROM f64 WHERE metricID=? AND tm BETWEEN ? AND ?),
+		aggregated AS (
+			SELECT MIN(tm) as minTm, MAX(tm) as maxTm, DATE_FORMAT(%s, '%s') as periodStr
+			FROM measures
+			GROUP BY periodStr
+		)
+	SELECT aggregated.periodStr, firstMeasures.tm, firstMeasures.value, lastMeasures.tm, lastMeasures.value
+	FROM aggregated
+		INNER JOIN measures firstMeasures ON firstMeasures.tm = aggregated.minTm
+		INNER JOIN measures lastMeasures ON lastMeasures.tm = aggregated.maxTm
+	ORDER BY aggregated.periodStr ASC`,
+		dt, groupByFormat)
+
+	rows, err := tx.Query(query, req.MetricID, extendedSince, extendedUntil)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		}
+		return
+	}
+	defer rows.Close()
+
+	var result []AggregatedMeasure
+
+	periodCalculator := NewPeriodCalculator(PeriodCalculatorOptions{
+		GroupBy:        req.GroupBy,
+		LastDayOfMonth: req.LastDayOfMonth,
+		FirstHourOfDay: req.FirstHourOfDay,
+		Since:          time.Unix(req.Since, 0),
+		Until:          time.Unix(req.Until, 0),
+	})
+
+	periodProducer := NewPeriodProducer(PeriodProducerOptions{
+		MetricID:          req.MetricID,
+		ParsePeriodLayout: parsePeriodLayout,
+		Location:          s.location,
+		Until:             req.Until,
+		Corrections:       corrections,
+		Rows:              rows,
+		Tx:                tx,
+	})
+
+	result, err = SplitByPeriods(periodProducer, periodCalculator)
+	if err != nil {
+		return
+	}
+	return result, nil
+}
+
+type RangeTotal struct {
+	Since      int64   `json:"since"`      // реальная граница
+	Until      int64   `json:"until"`      // реальная граница
+	StartValue float64 `json:"startValue"` // на конец периода
+	EndValue   float64 `json:"endValue"`   // на конец периода
+}
+
+type getCumulativeTotalFilter struct {
+	MetricID       int64
+	Since          int64
+	Until          int64
+	LastDayOfMonth int
+	FirstHourOfDay int
+}
+
+func (s *MeasureQueryEngine) getCumulativeTotal(req getCumulativeTotalFilter) (_ *RangeTotal, err error) {
+	measures, err := s.listAggregatedCumulativeMeasures(AggregatedMeasuresFilter{
+		MetricID:       req.MetricID,
+		Since:          req.Since,
+		Until:          req.Until,
+		GroupBy:        ByDay,
+		LastDayOfMonth: req.LastDayOfMonth,
+		FirstHourOfDay: req.FirstHourOfDay,
+	})
+	if err != nil {
+		return
+	}
+
+	if len(measures) == 0 {
+		return
+	}
+
+	//pretty.PPrintln("TOTAL", measures)
+
+	first := measures[0]
+	last := measures[len(measures)-1]
+
+	return &RangeTotal{
+		Since:      first.Since,
+		Until:      last.Until,
+		StartValue: first.Values[0] - first.Values[1], // value on period end - period total
+		EndValue:   last.Values[0],
+	}, nil
+}
+
+/*
+// GetCumulativeTotal - фикс, добавить опции FirstHourInDay, IsFindPeriodEndsInFuture
+func (s *MeasureQueryEngine) getCumulativeTotal(req getCumulativeTotalFilter) (_ *RangeTotal, err error) {
+	tx, err := s.db.Driver().Begin()
+	if err != nil {
+		return
+	}
+	defer tx.Rollback()
+
+	var (
+		firstTm, lastTm, futureTm                                       int64
+		firstValue, lastValue, futureValue                              float64
+		foundInPast, foundInFuture, foundFirstInRange, foundLastInRange bool
+	)
+
+	// Поиск первого показания
+
+	// Заглядываем в прошлое на 3 минуты
+	firstTm, firstValue, foundInPast, err = s.findLastInRange(tx, req.MetricID, req.Since-3*60, req.Since)
+	if err != nil {
+		return
+	}
+
+	if !foundInPast {
+		//
+		firstTm, firstValue, foundFirstInRange, err = s.findFirstInRange(tx, req.MetricID, req.Since, req.Until)
+		if err != nil {
+			return
+		}
+
+		if !foundFirstInRange {
+			// Начало периода не найдено - значит нет показаний для указанного периода
+			return
+		}
+	}
+
+	// Поиск последнего показания
+
+	lastTm, lastValue, foundLastInRange, err = s.findLastInRange(tx, req.MetricID, req.Since, req.Until)
+	if err != nil {
+		return
+	}
+
+	if req.CanPeriodEndsInFuture {
+		// Если можем искать конец в периода в будущем
+		if foundLastInRange {
+			if lastTm > (req.Until - 3*60) {
+				// Если lastTm в последние 3 минуты периода - период корректно завершен
+				return &RangeTotal{
+					Since: firstTm,
+					Until: lastTm,
+					Value: lastValue,
+					Total: lastValue - firstValue,
+				}, nil
+			}
+		}
+		// Период не завершен в последние 3 минуты периода, поэтому ищем в будущем
+		futureTm, futureValue, foundInFuture, err = s.findFirstAfter(tx, req.MetricID, req.Until)
+		if err != nil {
+			return
+		}
+
+		if foundInFuture {
+			return &RangeTotal{
+				Since: firstTm,
+				Until: futureTm,
+				Value: futureValue,
+				Total: futureValue - firstValue,
+			}, nil
+		} else {
+			// Супер важная проверка, ибо возможна ситуация, когда когда нашли начало
+			// текущего периода в конце предыдущего (foundInPast). А это показание было
+			// последним в БД.
+			if foundLastInRange {
+				// Последнее показание внутри периода нашли, неважно когда оно пришло - нам
+				// подходит.
+				return &RangeTotal{
+					Since: firstTm,
+					Until: lastTm,
+					Value: lastValue,
+					Total: lastValue - firstValue,
+				}, nil
+			}
+			// Последнее показание внутри периода не найдено, а значит их не было вообще.
+			// Просто выходим
+			return
+		}
+	} else {
+		// В будущем искать нельзя
+
+		// Супер важная проверка, ибо возможна ситуация, когда когда нашли начало
+		// текущего периода в конце предыдущего (foundInPast). А это показание было
+		// последним в БД.
+		if foundLastInRange {
+			// Последнее показание внутри периода нашли, неважно когда оно пришло - нам
+			// подходит.
+			return &RangeTotal{
+				Since: firstTm,
+				Until: lastTm,
+				Value: lastValue,
+				Total: lastValue - firstValue,
+			}, nil
+		}
+		// Последнее показание внутри периода не найдено, а значит их не было вообще.
+		// Просто выходим
+		return
+	}
+}
+*/
+
+func (s *MeasureQueryEngine) findLastInRangeTx(tx *sql.Tx, metricID int64, since, until int64) (tm int64, value float64, found bool, err error) {
+	err = tx.QueryRow(`
+	SELECT tm, value, true
+	FROM f64
+	WHERE metricID=? AND tm >= ? AND tm < ?
+	ORDER BY tm DESC
+	LIMIT 1`,
+		metricID, since, until).Scan(&tm, &value, &found)
+
+	if err != nil {
+		if err != sql.ErrNoRows {
+			return
+		}
+		// Нет записей
+		err = nil
+	}
+	return
+}
+
+func (s *MeasureQueryEngine) findFirstInRangeTx(tx *sql.Tx, metricID int64, since, until int64) (tm int64, value float64, found bool, err error) {
+	err = tx.QueryRow(`
+	SELECT tm, value, true
+	FROM f64
+	WHERE metricID=? AND tm >= ? AND tm < ?
+	ORDER BY tm ASC
+	LIMIT 1`,
+		metricID, since, until).Scan(&tm, &value, &found)
+
+	if err != nil {
+		if err != sql.ErrNoRows {
+			return
+		}
+		// Нет записей
+		err = nil
+	}
+	return
+}
+
+func (s *MeasureQueryEngine) findFirstAfterTx(tx *sql.Tx, metricID int64, since int64) (tm int64, value float64, found bool, err error) {
+	err = tx.QueryRow(`
+	SELECT tm, value, true
+	FROM f64
+	WHERE metricID=? AND tm >= ?
+	ORDER BY tm ASC
+	LIMIT 1`,
+		metricID, since).Scan(&tm, &value, &found)
+
+	if err != nil {
+		if err != sql.ErrNoRows {
+			return
+		}
+		// Нет записей
+		err = nil
+	}
+	return
+}

mqe/fix.go

@@ -0,0 +1,29 @@
+package mqe
+
+import (
+	"gordenko.dev/dima/fixme"
+)
+
+const (
+	EmptyValue            = "EMPTY_VALUE"
+	BusyName              = "BUSY_NAME"
+	WrongValue            = "WRONG_VALUE"
+	WrongDate             = "WRONG_DATE"
+	WrongDateRange        = "WRONG_DATE_RANGE"
+	InvalidFirstHourOfDay = "WRONG_FIRST_HOUR_OF_DAY"
+	InvalidLastDayOfMonth = "WRONG_LAST_DAY_OF_MONTH"
+)
+
+var (
+	errorMessages = map[fixme.Code]string{
+		EmptyValue:            "порожнє значення неприпустимо",
+		BusyName:              "имя уже используется",
+		WrongValue:            "неправильне значення",
+		WrongDate:             "некоректна дата",
+		WrongDateRange:        "некоректний діапазон",
+		InvalidFirstHourOfDay: "корректное значение от 0 до 23",
+		InvalidLastDayOfMonth: "корректное значение от 0 до 28",
+	}
+
+	fix = fixme.New(errorMessages)
+)

mqe/helpers.go

@@ -0,0 +1,158 @@
+package mqe
+
+import (
+	"fmt"
+)
+
+func getPeriodDateTime(groupBy GroupBy, lastDayOfMonth int, firstHourOfDay int) (dt string, err error) {
+	//fmt.Printf("getPeriodDateTime: %s, lastDay: %d, firstHour: %d\n", groupBy, lastDayOfMonth, firstHourOfDay)
+	// Если FirstHourOfDay и LastDayOfMonth равны 0 - модифицировать дату не нужно
+	dt = `FROM_UNIXTIME(tm)`
+
+	switch groupBy {
+	case ByDay:
+		if firstHourOfDay > 0 {
+			dt = fmt.Sprintf(`
+			IF (HOUR(FROM_UNIXTIME(tm)) >= %d, 
+				FROM_UNIXTIME(tm), 
+				DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY)
+			)`,
+				firstHourOfDay)
+		}
+
+	case ByMonth:
+		if lastDayOfMonth > 0 {
+			if lastDayOfMonth > 28 {
+				// На всякий случай добавляем защиту от дурака, ибо модификация даты
+				// в запросе (+1 месяц) для 29, 30 и 31 чисел сделает запрос неверным.
+				err = fmt.Errorf("LastDayOfMonth: %d, max allowed value is 28", lastDayOfMonth)
+				return
+			}
+
+			if firstHourOfDay > 0 {
+				// В псевдокоде модификация времени показания по корректный период
+				// выглядит так:
+				//
+				// if hour >= FirstHourOfDay {
+				// 	  if day <= LastDayOfMonth {
+				// 		  return tm
+				// 	  } else {
+				// 		  return tm + 1 month
+				// 	  }
+				// } else {
+				// 	  if (day - 1) <= LastDayOfMonth {
+				// 		  return tm - 1 day
+				// 	  } else {
+				// 		  return tm - 1 day + 1 month
+				// 	  }
+				// }
+				dt = fmt.Sprintf(`
+				IF (HOUR(FROM_UNIXTIME(tm)) >= %d, 
+					IF (DAYOFMONTH(FROM_UNIXTIME(tm)) <= %d, 
+						FROM_UNIXTIME(tm), 
+						DATE_ADD(FROM_UNIXTIME(tm), INTERVAL 1 MONTH)
+					),
+					IF (DAYOFMONTH(DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY)) <= %d, 
+						DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY), 
+						DATE_ADD(DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY), INTERVAL 1 MONTH)
+					)
+				)`,
+					firstHourOfDay, lastDayOfMonth, lastDayOfMonth)
+			} else {
+				dt = fmt.Sprintf(`
+				IF (DAYOFMONTH(FROM_UNIXTIME(tm)) <= %d, 
+					FROM_UNIXTIME(tm), 
+					DATE_ADD(FROM_UNIXTIME(tm), INTERVAL 1 MONTH)
+				)`,
+					lastDayOfMonth)
+			}
+		} else {
+			if firstHourOfDay > 0 {
+				dt = fmt.Sprintf(`
+				IF (HOUR(FROM_UNIXTIME(tm)) >= %d, 
+					FROM_UNIXTIME(tm), 
+					DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY)
+				)`,
+					firstHourOfDay)
+			}
+		}
+	}
+	return
+}
+
+func getDateTimeFacingFHD(firstHourOfDay int) (dt string, err error) {
+	if firstHourOfDay > 0 {
+		dt = fmt.Sprintf(`
+		IF (HOUR(FROM_UNIXTIME(tm)) >= %d, 
+			FROM_UNIXTIME(tm), 
+			DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY)
+		)`,
+			firstHourOfDay)
+	} else {
+		// Если FirstHourOfDay = 0 - модифицировать дату не нужно
+		dt = "FROM_UNIXTIME(tm)"
+	}
+	return
+}
+
+func getDateTimeFacingFHDAndLDM(firstHourOfDay int, lastDayOfMonth int) (dt string, err error) {
+	// Если FirstHourOfDay и LastDayOfMonth равны 0 - модифицировать дату не нужно
+	dt = "FROM_UNIXTIME(tm)"
+
+	if lastDayOfMonth > 0 {
+		if lastDayOfMonth > 28 {
+			// На всякий случай добавляем защиту от дурака, ибо модификация даты
+			// в запросе (+1 месяц) для 29, 30 и 31 чисел сделает запрос неверным.
+			err = fmt.Errorf("LastDayOfMonth: %d, max allowed value is 28", lastDayOfMonth)
+			return
+		}
+
+		if firstHourOfDay > 0 {
+			// В псевдокоде модификация времени показания по корректный период
+			// выглядит так:
+			//
+			// if hour >= FirstHourOfDay {
+			// 	  if day <= LastDayOfMonth {
+			// 		  return tm
+			// 	  } else {
+			// 		  return tm + 1 month
+			// 	  }
+			// } else {
+			// 	  if (day - 1) <= LastDayOfMonth {
+			// 		  return tm - 1 day
+			// 	  } else {
+			// 		  return tm - 1 day + 1 month
+			// 	  }
+			// }
+			dt = fmt.Sprintf(`
+			IF (HOUR(FROM_UNIXTIME(tm)) >= %d, 
+				IF (DAYOFMONTH(FROM_UNIXTIME(tm)) <= %d, 
+					FROM_UNIXTIME(tm), 
+					DATE_ADD(FROM_UNIXTIME(tm), INTERVAL 1 MONTH)
+				),
+				IF (DAYOFMONTH(DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY)) <= %d, 
+					DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY), 
+					DATE_ADD(DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY), INTERVAL 1 MONTH)
+				)
+			)`,
+				firstHourOfDay, lastDayOfMonth, lastDayOfMonth)
+		} else {
+			dt = fmt.Sprintf(`
+			IF (DAYOFMONTH(FROM_UNIXTIME(tm)) <= %d, 
+				FROM_UNIXTIME(tm), 
+				DATE_ADD(FROM_UNIXTIME(tm), INTERVAL 1 MONTH)
+			)`,
+				lastDayOfMonth)
+		}
+	} else {
+		if firstHourOfDay > 0 {
+			dt = fmt.Sprintf(`
+			IF (HOUR(FROM_UNIXTIME(tm)) >= %d, 
+				FROM_UNIXTIME(tm), 
+				DATE_SUB(FROM_UNIXTIME(tm), INTERVAL 1 DAY)
+			)`,
+				firstHourOfDay)
+		}
+	}
+	return
+}

mqe/index.go

@@ -0,0 +1,320 @@
+package mqe
+
+import (
+	"fmt"
+	"sort"
+	"strconv"
+)
+
+type GetYearIndexReq struct {
+	Since          int64   `json:"since"`          // из фильтр-формы
+	Until          int64   `json:"until"`          // из фильтр-формы
+	LastDayOfMonth int     `json:"lastDayOfMonth"` // например, конец месяца 25 число
+	FirstHourOfDay int     `json:"firstHourOfDay"` // например день начинается в 6:00
+	MetricIDs      []int64 `json:"metricIDs"`
+}
+
+func (s *MeasureQueryEngine) GetYearIndex(req GetYearIndexReq) (years []int, err error) {
+	if len(req.MetricIDs) == 0 {
+		return
+	}
+
+	var (
+		lte, gte string
+
+		args = []any{
+			req.MetricIDs,
+		}
+	)
+
+	if req.Since > 0 {
+		lte = " AND tm >= ?"
+		args = append(args, req.Since)
+	}
+
+	if req.Until > 0 {
+		gte = " AND tm <= ?"
+		args = append(args, req.Until)
+	}
+
+	q := fmt.Sprintf(`
+	SELECT YEAR(FROM_UNIXTIME(tm)) as y
+	FROM f64
+	WHERE metricID IN (?)%s%s
+	GROUP BY y`,
+		lte, gte)
+
+	// fmt.Println(q)
+	// fmt.Printf("%v\n", args)
+
+	err = s.db.ListQuery(&years, q, args...)
+	if err != nil {
+		return
+	}
+
+	sort.Ints(years)
+	return
+}
+
+type GetMonthIndexReq struct {
+	Since          int64   `json:"since"`          // из фильтр-формы
+	Until          int64   `json:"until"`          // из фильтр-формы
+	LastDayOfMonth int     `json:"lastDayOfMonth"` // например, конец месяца 25 число
+	FirstHourOfDay int     `json:"firstHourOfDay"` // например день начинается в 6:00
+	MetricIDs      []int64 `json:"metricIDs"`
+}
+
+type YearMonth struct {
+	Year   int   `json:"year"`
+	Months []int `json:"months"`
+}
+
+func (s *MeasureQueryEngine) GetMonthIndex(req GetMonthIndexReq) (years []YearMonth, err error) {
+	if len(req.MetricIDs) == 0 {
+		return
+	}
+
+	dt, err := getDateTimeFacingFHDAndLDM(req.FirstHourOfDay, req.LastDayOfMonth)
+	if err != nil {
+		return
+	}
+
+	var (
+		lte, gte string
+
+		args = []any{
+			req.MetricIDs,
+		}
+	)
+
+	if req.Since > 0 {
+		lte = " AND tm >= ?"
+		args = append(args, req.Since)
+	}
+
+	if req.Until > 0 {
+		gte = " AND tm <= ?"
+		args = append(args, req.Until)
+	}
+
+	// ВАЖНО!
+	// Не сортируем средствами MySQL чтобы не создавать временный файл
+	query := fmt.Sprintf(`
+	SELECT DATE_FORMAT(%s, '%s') as p
+	FROM f64
+	WHERE metricID IN (?)%s%s
+	GROUP BY p`,
+		dt, "%Y%m", lte, gte)
+
+	var (
+		temp []string
+		yms  []int
+	)
+
+	err = s.db.ListQuery(&temp, query, args...)
+	if err != nil {
+		return
+	}
+
+	for _, str := range temp {
+		var ym int
+		ym, err = strconv.Atoi(str)
+		if err != nil {
+			return
+		}
+		yms = append(yms, ym)
+	}
+	return IndexYms(yms), nil
+}
+
+type GetDayIndexReq struct {
+	Since          int64   `json:"since"`          // из фильтр-формы
+	Until          int64   `json:"until"`          // из фильтр-формы
+	FirstHourOfDay int     `json:"firstHourOfDay"` // например день начинается в 6:00
+	MetricIDs      []int64 `json:"metricIDs"`
+}
+
+type YearMonthDay struct {
+	Year   int        `json:"year"`
+	Months []MonthDay `json:"months"`
+}
+
+type MonthDay struct {
+	Month int   `json:"month"`
+	Days  []int `json:"days"`
+}
+
+func (s *MeasureQueryEngine) GetDayIndex(req GetDayIndexReq) (years []YearMonthDay, err error) {
+	if len(req.MetricIDs) == 0 {
+		return
+	}
+
+	dt, err := getDateTimeFacingFHD(req.FirstHourOfDay)
+	if err != nil {
+		return
+	}
+
+	var (
+		lte, gte string
+
+		args = []any{
+			req.MetricIDs,
+		}
+	)
+
+	if req.Since > 0 {
+		lte = " AND tm >= ?"
+		args = append(args, req.Since)
+	}
+
+	if req.Until > 0 {
+		gte = " AND tm <= ?"
+		args = append(args, req.Until)
+	}
+	// ВАЖНО!
+	// Не сортируем средствами MySQL чтобы не создавать временный файл
+	query := fmt.Sprintf(`
+	SELECT DATE_FORMAT(%s, '%s') as p
+	FROM f64
+	WHERE metricID IN(?)%s%s
+	GROUP BY p`,
+		dt, "%Y%m%d", lte, gte)
+
+	//fmt.Printf("%s\n", query)
+
+	var (
+		temp []string
+		ymds []int
+	)
+
+	err = s.db.ListQuery(&temp, query, args...)
+	if err != nil {
+		return
+	}
+
+	//pretty.PPrintln("temp", temp)
+
+	for _, str := range temp {
+		var ymd int
+		ymd, err = strconv.Atoi(str)
+		if err != nil {
+			return
+		}
+		ymds = append(ymds, ymd)
+	}
+
+	return IndexYmds(ymds), nil
+}
+
+func IndexYmds(ymds []int) (years []YearMonthDay) {
+	if len(ymds) == 0 {
+		return
+	}
+
+	sort.Ints(ymds)
+
+	var (
+		months           []MonthDay
+		days             []int
+		year, month, day int
+	)
+
+	for _, ymd := range ymds {
+		y := ymd / 10000
+		m := (ymd % 10000) / 100
+		d := ymd % 100
+
+		if year == 0 {
+			// инициализация
+			year = y
+			month = m
+			day = d
+		} else {
+			if y != year {
+				days = append(days, day)
+				months = append(months, MonthDay{
+					Month: month,
+					Days:  days,
+				})
+				years = append(years, YearMonthDay{
+					Year:   year,
+					Months: months,
+				})
+				year = y
+				month = m
+				day = d
+				months = nil
+				days = nil
+			} else if m != month {
+				days = append(days, day)
+				months = append(months, MonthDay{
+					Month: month,
+					Days:  days,
+				})
+				month = m
+				day = d
+				days = nil
+			} else if d != day {
+				days = append(days, day)
+				day = d
+			}
+		}
+	}
+
+	days = append(days, day)
+	months = append(months, MonthDay{
+		Month: month,
+		Days:  days,
+	})
+	years = append(years, YearMonthDay{
+		Year:   year,
+		Months: months,
+	})
+	return
+}
+
+func IndexYms(yms []int) (years []YearMonth) {
+	if len(yms) == 0 {
+		return
+	}
+
+	sort.Ints(yms)
+
+	var (
+		months      []int
+		year, month int
+	)
+
+	for _, ym := range yms {
+		y := ym / 100
+		m := ym % 100
+
+		if year == 0 {
+			// инициализация
+			year = y
+			month = m
+		} else {
+			if y != year {
+				months = append(months, month)
+				years = append(years, YearMonth{
+					Year:   year,
+					Months: months,
+				})
+				year = y
+				month = m
+				months = nil
+			} else if m != month {
+				months = append(months, month)
+				month = m
+			}
+		}
+	}
+	months = append(months, month)
+	years = append(years, YearMonth{
+		Year:   year,
+		Months: months,
+	})
+	return
+}
+
+//SELECT DATE_FORMAT(FROM_UNIXTIME(tm), '%Y%m%d') as p FROM f64 WHERE metricID IN (50) AND tm >= 1700863200 AND tm <= 1700949599 GROUP BY p;

mqe/index_test.go

@@ -0,0 +1,39 @@
+package mqe
+
+import (
+	"testing"
+
+	"gordenko.dev/dima/pretty"
+)
+
+func TestDayIndex(t *testing.T) {
+	index := IndexYmds([]int{
+		20221201,
+		20221202,
+		20230110,
+		20230210,
+		20230330,
+		20230331,
+		20240101,
+		20240102,
+		20240103,
+	})
+
+	pretty.Println(index)
+}
+
+func TestMonthIndex(t *testing.T) {
+	index := IndexYms([]int{
+		202211,
+		202212,
+		202301,
+		202302,
+		202303,
+		202304,
+		202404,
+		202405,
+		202406,
+	})
+
+	pretty.Println(index)
+}

mqe/instant.go

@@ -0,0 +1,226 @@
+package mqe
+
+import (
+	"database/sql"
+	"fmt"
+	"time"
+)
+
+type CurrentValue struct {
+	MetricID uint32
+	Time     int64   `json:"t"`
+	Value    float64 `json:"v"`
+}
+
+func (s *MeasureQueryEngine) ListCurrentValues(metricIDs []uint32) (result []CurrentValue, err error) {
+	driver := s.db.Driver()
+
+	for _, metricID := range metricIDs {
+		var rows *sql.Rows
+		rows, err = driver.Query(`
+		SELECT tm, value
+		FROM f64
+		WHERE metricID=?
+		ORDER BY tm DESC
+		LIMIT 1`,
+			metricID)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				err = nil
+			}
+			return
+		}
+
+		for rows.Next() {
+			var (
+				tm    int64
+				value float64
+			)
+			err = rows.Scan(&tm, &value)
+			if err != nil {
+				return
+			}
+			result = append(result, CurrentValue{
+				MetricID: metricID,
+				Time:     tm,
+				Value:    value,
+			})
+		}
+
+		if err = rows.Err(); err != nil {
+			return
+		}
+		rows.Close()
+	}
+	return
+}
+
+// ListInstantMeasures - cписок показаний мгновенных метрик (Температура, Давление, Расход)
+// за за интервал без группировки
+func (s *MeasureQueryEngine) listInstantMeasures(req MetricMeasuresFilter) (_ []Measure, err error) {
+	rows, err := s.db.Driver().Query(`
+	SELECT tm, value
+	FROM f64
+	WHERE metricID=? AND tm BETWEEN ? AND ?
+	ORDER BY tm ASC`,
+		req.MetricID, req.Since, req.Until)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		}
+		return
+	}
+	defer rows.Close()
+
+	var result []Measure
+
+	for rows.Next() {
+		var (
+			tm    int64
+			value float64
+		)
+		err = rows.Scan(&tm, &value)
+		if err != nil {
+			return
+		}
+		result = append(result, Measure{
+			Time: tm,
+			Values: []float64{
+				value,
+			},
+		})
+	}
+
+	if err = rows.Err(); err != nil {
+		return
+	}
+	return result, nil
+}
+
+type AggregatedInstantMeasuresIn struct {
+	MetricID       int64   `json:"metricID"`
+	Since          int64   `json:"since"`
+	Until          int64   `json:"until"`
+	GroupBy        GroupBy `json:"groupBy"`
+	Flags          byte    `json:"flags"`          // из настроек по умолчанию либо из запроса
+	LastDayOfMonth int     `json:"lastDayOfMonth"` // например, конец месяца 25 число
+	FirstHourOfDay int     `json:"firstHourOfDay"` // например день начинается в 6:00
+}
+
+// Заполняет поля Period и Values (от 1 до 3 значений) в структуре AggregatedMeasure
+func (s *MeasureQueryEngine) listAggregatedInstantMeasures(req AggregatedInstantMeasuresIn) (_ []AggregatedMeasure, err error) {
+	var (
+		groupByFormat     string
+		parsePeriodLayout string
+
+		result []AggregatedMeasure
+	)
+
+	switch req.GroupBy {
+	case ByHour:
+		groupByFormat = "%Y%m%d%H"
+		parsePeriodLayout = hourPeriodLayout
+
+	case ByDay:
+		groupByFormat = "%Y%m%d"
+		parsePeriodLayout = dayPeriodLayout
+
+	case ByMonth:
+		groupByFormat = "%Y%m"
+		parsePeriodLayout = monthPeriodLayout
+
+	default:
+		// Защита от дурака
+		err = fmt.Errorf("unknown GroupBy: %s", req.GroupBy)
+		return
+	}
+
+	valuesQty := 0
+	values := ""
+
+	if (req.Flags & AggregateMin) == AggregateMin {
+		values += " MIN(value),"
+		valuesQty++
+	}
+
+	if (req.Flags & AggregateMax) == AggregateMax {
+		values += " MAX(value),"
+		valuesQty++
+	}
+
+	if (req.Flags & AggregateAvg) == AggregateAvg {
+		values += " AVG(value),"
+		valuesQty++
+	}
+
+	dt, err := getPeriodDateTime(req.GroupBy, req.LastDayOfMonth, req.FirstHourOfDay)
+	if err != nil {
+		return
+	}
+
+	query := fmt.Sprintf(`
+	SELECT%s DATE_FORMAT(%s, '%s') as periodStr
+	FROM f64
+	WHERE metricID=? AND tm BETWEEN ? AND ?
+	GROUP BY periodStr
+	ORDER BY periodStr ASC`,
+		values, dt, groupByFormat)
+
+	rows, err := s.db.Driver().Query(query, req.MetricID, req.Since, req.Until)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		}
+		return
+	}
+
+	for rows.Next() {
+		var (
+			m = AggregatedMeasure{
+				Values: make([]float64, valuesQty),
+			}
+
+			periodStr string
+			period    time.Time
+
+			valueIdx int
+			dest     []interface{}
+		)
+
+		if (req.Flags & AggregateMin) == AggregateMin {
+			dest = append(dest, &m.Values[valueIdx])
+			valueIdx++
+		}
+
+		if (req.Flags & AggregateMax) == AggregateMax {
+			dest = append(dest, &m.Values[valueIdx])
+			valueIdx++
+		}
+
+		if (req.Flags & AggregateAvg) == AggregateAvg {
+			dest = append(dest, &m.Values[valueIdx])
+		}
+
+		dest = append(dest, &periodStr)
+
+		err = rows.Scan(dest...)
+		if err != nil {
+			return
+		}
+
+		period, err = time.ParseInLocation(parsePeriodLayout, periodStr, s.location)
+		if err != nil {
+			err = fmt.Errorf("time.ParseInLocation: %s; layout=%q; str=%q",
+				err, parsePeriodLayout, periodStr)
+			return
+		}
+		m.Period = period.Unix()
+
+		result = append(result, m)
+	}
+
+	if err = rows.Err(); err != nil {
+		return
+	}
+	return result, nil
+}

mqe/mqe.go

@@ -0,0 +1,161 @@
+package mqe
+
+import (
+	"sync"
+	"time"
+
+	"gordenko.dev/dima/qx"
+)
+
+type GroupBy string
+
+// Для парсинга since и until
+const (
+	NoAggregateFunc byte = 1
+	AggregateMin    byte = 2
+	AggregateMax    byte = 4
+	AggregateAvg    byte = 8
+
+	ByHour  GroupBy = "h"
+	ByDay   GroupBy = "d"
+	ByMonth GroupBy = "m"
+
+	hourPeriodLayout  = "2006010215"
+	dayPeriodLayout   = "20060102"
+	monthPeriodLayout = "200601"
+
+	dateLayout = "2006-01-02"
+)
+
+type MetricMeasuresFilter struct {
+	MetricID int64 `json:"metricID"`
+	Since    int64 `json:"since"` // уже учтен firstHourOfDay
+	Until    int64 `json:"until"` // уже учтен firstHourOfDay
+}
+
+type MeasureQueryEngine struct {
+	db                     *qx.Db
+	location               *time.Location
+	cumulativeCh           chan *CumulativeTask
+	instantCh              chan *InstantTask
+	aggregatedCumulativeCh chan *AggregatedCumulativeTask
+	aggregatedInstantCh    chan *AggregatedInstantTask
+	cumulativeTotalCh      chan *CumulativeTotalTask
+	metricReadingsCh       chan *MetricReadingsTask
+}
+
+type Options struct {
+	Db         *qx.Db
+	Location   *time.Location
+	MinWorkers int
+	QueueSize  int
+}
+
+func New(opt Options) *MeasureQueryEngine {
+	if opt.Db == nil {
+		panic("Db option is required")
+	}
+
+	if opt.Location == nil {
+		panic("Location option is required")
+	}
+
+	if opt.MinWorkers <= 0 {
+		panic("MinWorkers option is required")
+	}
+
+	if opt.QueueSize <= 0 {
+		panic("QueueSize option is required")
+	}
+
+	s := new(MeasureQueryEngine)
+	s.db = opt.Db
+	s.location = opt.Location
+	//
+	s.cumulativeCh = make(chan *CumulativeTask, opt.QueueSize)
+	s.instantCh = make(chan *InstantTask, opt.QueueSize)
+	s.aggregatedCumulativeCh = make(chan *AggregatedCumulativeTask, opt.QueueSize)
+	s.aggregatedInstantCh = make(chan *AggregatedInstantTask, opt.QueueSize)
+	s.cumulativeTotalCh = make(chan *CumulativeTotalTask, opt.QueueSize)
+	s.metricReadingsCh = make(chan *MetricReadingsTask, opt.QueueSize)
+
+	//
+	for i := 0; i < opt.MinWorkers; i++ {
+		go s.worker()
+	}
+	return s
+}
+
+func (s *MeasureQueryEngine) worker() {
+	for {
+		select {
+		case task := <-s.cumulativeCh:
+			task.Result, task.Err = s.listCumulativeMeasures(task.In)
+			task.WaitGroup.Done()
+
+		case task := <-s.instantCh:
+			task.Result, task.Err = s.listInstantMeasures(task.In)
+			task.WaitGroup.Done()
+
+		case task := <-s.aggregatedCumulativeCh:
+			task.Result, task.Err = s.listAggregatedCumulativeMeasures(task.In)
+			task.WaitGroup.Done()
+
+		case task := <-s.aggregatedInstantCh:
+			task.Result, task.Err = s.listAggregatedInstantMeasures(task.In)
+			task.WaitGroup.Done()
+
+		case task := <-s.cumulativeTotalCh:
+			task.Result, task.Err = s.getCumulativeTotal(task.In)
+			task.WaitGroup.Done()
+
+		case task := <-s.metricReadingsCh:
+			task.Result, task.Err = s.listMetricReadings(task.In)
+			task.WaitGroup.Done()
+		}
+	}
+}
+
+//
+
+type CumulativeTask struct {
+	In        MetricMeasuresFilter
+	WaitGroup *sync.WaitGroup
+	Result    []Measure
+	Err       error
+}
+
+type InstantTask struct {
+	In        MetricMeasuresFilter
+	WaitGroup *sync.WaitGroup
+	Result    []Measure
+	Err       error
+}
+
+type AggregatedCumulativeTask struct {
+	In        AggregatedMeasuresFilter
+	WaitGroup *sync.WaitGroup
+	Result    []AggregatedMeasure
+	Err       error
+}
+
+type AggregatedInstantTask struct {
+	In        AggregatedInstantMeasuresIn
+	WaitGroup *sync.WaitGroup
+	Result    []AggregatedMeasure
+	Err       error
+}
+
+type MetricReadingsTask struct {
+	In        MetricMeasuresFilter
+	WaitGroup *sync.WaitGroup
+	Result    MetricReadings
+	Err       error
+}
+
+type CumulativeTotalTask struct {
+	In        getCumulativeTotalFilter
+	WaitGroup *sync.WaitGroup
+	Result    *RangeTotal
+	Err       error
+}

mqe/newreq.go

@@ -0,0 +1,283 @@
+package mqe
+
+// НОВАЯ ВЕРСИЯ
+
+// type MetricsDataFilter2 struct {
+// 	Since   time.Time `json:"since"`
+// 	Until   time.Time `json:"until"`
+// 	GroupBy string    `json:"groupBy"`
+
+// 	InstantMetrics    []InstantMetricFilter // virtualMetricID + aggregateFuncs
+// 	CumulativeMetrics []CumulativeMetricFilter
+// }
+
+// type InstantMetricFilter struct {
+// 	MetricID       int64
+// 	AggregateFuncs byte
+// 	LastDayOfMonth int // например, конец месяца 25 число
+// 	FirstHourOfDay int // например день начинается в 6:00
+// }
+
+// type CumulativeMetricFilter struct {
+// 	MetricID       int64
+// 	LastDayOfMonth int // например, конец месяца 25 число
+// 	FirstHourOfDay int // например день начинается в 6:00
+// }
+
+// func (s *MeasureQueryEngine) SelectMetricsDataInParallel2(req MetricsDataFilter2) (_ MetricsDataResultMaps, err error) {
+// 	pretty.PPrintln("SelectMetricsDataInParallel2", req)
+// 	// Строгая валидация
+
+// 	if req.Since.IsZero() {
+// 		err = fmt.Errorf("Since option is required")
+// 		return
+// 	}
+
+// 	if req.Until.IsZero() {
+// 		err = fmt.Errorf("Until option is required")
+// 		return
+// 	}
+
+// 	if req.Since.After(req.Until) {
+// 		err = fmt.Errorf("Wrong time range: Since %d after Until %d", req.Since.Unix(), req.Until.Unix())
+// 		return
+// 	}
+
+// 	switch req.GroupBy {
+// 	case "h", "d", "m", "":
+// 		// pass
+// 	default:
+// 		err = fmt.Errorf("Wrong groupBy option value: %s", req.GroupBy)
+// 		return
+// 	}
+
+// 	for _, mf := range req.InstantMetrics {
+// 		if mf.MetricID == 0 {
+// 			err = fmt.Errorf("Empty instant metric's MetricID: %d", mf.MetricID)
+// 			return
+// 		}
+
+// 		if mf.FirstHourOfDay < 0 || mf.FirstHourOfDay > 23 {
+// 			err = fmt.Errorf("Wrong instant metric's FirstHourOfDay option value: %d", mf.FirstHourOfDay)
+// 			return
+// 		}
+
+// 		if mf.LastDayOfMonth < 0 || mf.LastDayOfMonth > 28 {
+// 			err = fmt.Errorf("Wrong instant metric's LastDayOfMonth option value: %d", mf.LastDayOfMonth)
+// 			return
+// 		}
+// 	}
+
+// 	for _, mf := range req.CumulativeMetrics {
+// 		if mf.MetricID == 0 {
+// 			err = fmt.Errorf("Empty cumulative metric's MetricID: %d", mf.MetricID)
+// 			return
+// 		}
+
+// 		if mf.FirstHourOfDay < 0 || mf.FirstHourOfDay > 23 {
+// 			err = fmt.Errorf("Wrong cumulative metric's FirstHourOfDay option value: %d", mf.FirstHourOfDay)
+// 			return
+// 		}
+
+// 		if mf.LastDayOfMonth < 0 || mf.LastDayOfMonth > 28 {
+// 			err = fmt.Errorf("Wrong cumulative metric's LastDayOfMonth option value: %d", mf.LastDayOfMonth)
+// 			return
+// 		}
+// 	}
+
+// 	// Корректируем, ВСЕГДА на первую и последнюю секунду суток
+// 	since := timeutil.FirstSecondInPeriod(req.Since, "d")
+// 	until := timeutil.LastSecondInPeriod(req.Until, "d")
+
+// 	fmt.Printf(`SelectMetricsDataInParallel2: {
+// 	Since: %s
+// 	Until: %s
+// 	GroupBy: %s
+// }
+// `, since, until, req.GroupBy)
+
+// 	// Важно вирутальная метрика зависит от реальных, которые к объекту могут быть
+// 	// привязаны или нет. Если нет - не возвращать.
+
+// 	var (
+// 		unexpectedErrors []error
+// 		errorMutex       sync.Mutex
+
+// 		wg sync.WaitGroup
+
+// 		resultMaps = MetricsDataResultMaps{
+// 			AggregatedCumulative:    make(map[int64][]CalculatedAggregatedMeasure),
+// 			AggregatedByFuncInstant: make(map[int64][]AggregatedByFuncInstantMeasure),
+// 			Cumulative:              make(map[int64][]CalculatedMeasure),
+// 			Instant:                 make(map[int64][]CalculatedMeasure),
+// 		}
+// 	)
+
+// 	wg.Add(len(req.CumulativeMetrics) + len(req.InstantMetrics))
+
+// 	if req.GroupBy != "" {
+// 		// AGGREGATED
+
+// 		// CUMULATIVE
+// 		for _, metricFilter := range req.CumulativeMetrics {
+// 			go func(mf CumulativeMetricFilter) {
+// 				// У каждой метрики индивидуальные настройки,
+// 				// ибо метрики принадлежат разным объектам!!!
+// 				metricSince := since
+// 				metricUntil := until
+
+// 				if mf.FirstHourOfDay > 0 {
+// 					metricSince = since.Add(time.Duration(mf.FirstHourOfDay) * time.Hour)
+// 					metricUntil = until.Add(time.Duration(mf.FirstHourOfDay) * time.Hour)
+// 				}
+
+// 				result, err := s.listAggregatedCumulativeMeasures(AggregatedMeasuresFilter{
+// 					MetricID:       mf.MetricID,
+// 					Since:          metricSince.Unix(),
+// 					Until:          metricUntil.Unix(),
+// 					GroupBy:        req.GroupBy,
+// 					LastDayOfMonth: mf.LastDayOfMonth,
+// 					FirstHourOfDay: mf.FirstHourOfDay,
+// 				})
+// 				if err != nil {
+// 					err = fmt.Errorf("ListAggregatedCumulativeMeasures: %s", err)
+
+// 					errorMutex.Lock()
+// 					unexpectedErrors = append(unexpectedErrors, err)
+// 					errorMutex.Unlock()
+// 				} else {
+// 					errorMutex.Lock()
+// 					resultMaps.AggregatedCumulative[mf.MetricID] = result
+// 					errorMutex.Unlock()
+// 				}
+
+// 				wg.Done()
+// 			}(metricFilter)
+// 		}
+
+// 		// INSTANT
+// 		for _, metricFilter := range req.InstantMetrics {
+// 			go func(mf InstantMetricFilter) {
+// 				// У каждой метрики индивидуальные настройки,
+// 				// ибо метрики принадлежат разным объектам!!!
+// 				metricSince := since
+// 				metricUntil := until
+
+// 				if mf.FirstHourOfDay > 0 {
+// 					metricSince = since.Add(time.Duration(mf.FirstHourOfDay) * time.Hour)
+// 					metricUntil = until.Add(time.Duration(mf.FirstHourOfDay) * time.Hour)
+// 				}
+
+// 				result, err := s.listAggregatedByFuncsInstantMeasures(AggregatedByFuncInstantMeasuresFilter{
+// 					MetricID:       mf.MetricID,
+// 					Since:          metricSince.Unix(),
+// 					Until:          metricUntil.Unix(),
+// 					GroupBy:        req.GroupBy,
+// 					LastDayOfMonth: mf.LastDayOfMonth,
+// 					FirstHourOfDay: mf.FirstHourOfDay,
+// 					Flags:          mf.AggregateFuncs, // ФЛАГИ !!!
+// 				})
+// 				if err != nil {
+// 					err = fmt.Errorf("ListAggregatedByFuncInstantMeasures: %s", err)
+
+// 					errorMutex.Lock()
+// 					unexpectedErrors = append(unexpectedErrors, err)
+// 					errorMutex.Unlock()
+// 				} else {
+// 					errorMutex.Lock()
+// 					resultMaps.AggregatedByFuncInstant[mf.MetricID] = result
+// 					errorMutex.Unlock()
+// 				}
+
+// 				wg.Done()
+// 			}(metricFilter)
+// 		}
+
+// 	} else {
+// 		// NON AGGREGATED
+
+// 		// CUMULATIVE
+// 		for _, metricFilter := range req.CumulativeMetrics {
+// 			go func(mf CumulativeMetricFilter) {
+// 				// У каждой метрики индивидуальные настройки,
+// 				// ибо метрики принадлежат разным объектам!!!
+// 				metricSince := since
+// 				metricUntil := until
+
+// 				if mf.FirstHourOfDay > 0 {
+// 					metricSince = since.Add(time.Duration(mf.FirstHourOfDay) * time.Hour)
+// 					metricUntil = until.Add(time.Duration(mf.FirstHourOfDay) * time.Hour)
+// 				}
+
+// 				result, err := s.listCumulativeMeasures(MeasuresFilter{
+// 					MetricID: mf.MetricID,
+// 					Since:    metricSince.Unix(),
+// 					Until:    metricUntil.Unix(),
+// 				})
+// 				if err != nil {
+// 					err = fmt.Errorf("ListCumulativeMeasures: %s", err)
+
+// 					errorMutex.Lock()
+// 					unexpectedErrors = append(unexpectedErrors, err)
+// 					errorMutex.Unlock()
+// 				} else {
+// 					errorMutex.Lock()
+// 					resultMaps.Cumulative[mf.MetricID] = result
+// 					errorMutex.Unlock()
+// 				}
+// 				wg.Done()
+// 			}(metricFilter)
+// 		}
+
+// 		// INSTANT
+// 		for _, metricFilter := range req.InstantMetrics {
+// 			go func(mf InstantMetricFilter) {
+// 				// У каждой метрики индивидуальные настройки,
+// 				// ибо метрики принадлежат разным объектам!!!
+// 				metricSince := since
+// 				metricUntil := until
+
+// 				if mf.FirstHourOfDay > 0 {
+// 					metricSince = since.Add(time.Duration(mf.FirstHourOfDay) * time.Hour)
+// 					metricUntil = until.Add(time.Duration(mf.FirstHourOfDay) * time.Hour)
+// 				}
+
+// 				result, err := s.listInstantMeasures(MeasuresFilter{
+// 					MetricID: mf.MetricID,
+// 					Since:    metricSince.Unix(),
+// 					Until:    metricUntil.Unix(),
+// 				})
+// 				if err != nil {
+// 					err = fmt.Errorf("ListInstantMeasures: %s", err)
+
+// 					errorMutex.Lock()
+// 					unexpectedErrors = append(unexpectedErrors, err)
+// 					errorMutex.Unlock()
+// 				} else {
+// 					errorMutex.Lock()
+// 					resultMaps.Instant[mf.MetricID] = result
+// 					errorMutex.Unlock()
+// 				}
+// 				wg.Done()
+// 			}(metricFilter)
+// 		}
+// 	}
+
+// 	wg.Wait()
+
+// 	// Проверяем что все запросы завершились удачно
+// 	if len(unexpectedErrors) > 0 {
+// 		// Нумеруем сообщения об ошибках и склеиваем в одно большое сообщение
+// 		var errorStrings []string
+
+// 		for idx, err := range unexpectedErrors {
+// 			errorStrings = append(errorStrings, fmt.Sprintf("\n#%d %s", idx+1, err))
+// 		}
+
+// 		err = fmt.Errorf("%d errors occured:%s", len(unexpectedErrors),
+// 			strings.Join(errorStrings, ""))
+// 		return
+// 	}
+
+// 	return resultMaps, nil
+// }

mqe/period_calculator.go

@@ -0,0 +1,172 @@
+package mqe
+
+import (
+	"fmt"
+	"time"
+
+	"gordenko.dev/dima/qb/timeutil"
+)
+
+type PeriodCalculatorOptions struct {
+	GroupBy        GroupBy
+	LastDayOfMonth int
+	FirstHourOfDay int
+	Since          time.Time
+	Until          time.Time
+}
+
+func NewPeriodCalculator(opt PeriodCalculatorOptions) PeriodCalculator {
+	switch opt.GroupBy {
+	case ByHour, ByDay, ByMonth:
+		// pass
+
+	default:
+		panic(fmt.Sprintf("bug: unknown groupBy %q", opt.GroupBy))
+	}
+
+	s := PeriodCalculator{
+		groupBy:        opt.GroupBy,
+		lastDayOfMonth: opt.LastDayOfMonth,
+		firstHourOfDay: opt.FirstHourOfDay,
+	}
+	s.sincePeriod = s.TimeToPeriod(opt.Since)
+	s.untilPeriod = s.TimeToPeriod(opt.Until)
+
+	return s
+}
+
+type PeriodCalculator struct {
+	groupBy        GroupBy
+	lastDayOfMonth int
+	firstHourOfDay int
+	sincePeriod    time.Time
+	untilPeriod    time.Time
+}
+
+func (s PeriodCalculator) IsExtendedSincePeriod(period time.Time) bool {
+	return period.Before(s.sincePeriod)
+}
+
+func (s PeriodCalculator) IsExtendedUntilPeriod(period time.Time) bool {
+	return period.After(s.untilPeriod)
+}
+
+func (s PeriodCalculator) NextPeriod(period time.Time) time.Time {
+	switch s.groupBy {
+	case ByHour:
+		return period.Add(time.Hour)
+
+	case ByDay:
+		return period.AddDate(0, 0, 1)
+
+	case ByMonth:
+		return period.AddDate(0, 1, 0)
+
+	default:
+		panic(fmt.Sprintf("bug: unknown groupBy %q", s.groupBy))
+	}
+}
+
+func (s PeriodCalculator) IsPeriodCorrectEnds(period time.Time, lastMeasureTime int64) bool {
+	// по умолчанию
+	tm := timeutil.LastSecondInPeriod(period, string(s.groupBy))
+
+	switch s.groupBy {
+	case ByDay:
+		if s.firstHourOfDay > 0 {
+			// Было 23:59, стало - 01:59
+			tm = tm.Add(time.Duration(s.firstHourOfDay) * time.Hour)
+		}
+
+	case ByMonth:
+		if s.lastDayOfMonth == 0 {
+			if s.firstHourOfDay > 0 {
+				// Было 23:59, стало - 01:59
+				tm = tm.Add(time.Duration(s.firstHourOfDay) * time.Hour)
+			}
+		} else {
+			if s.firstHourOfDay > 0 {
+				// last day 28, first hour 2 превращается в day 29 01:59:59
+				tm = time.Date(
+					tm.Year(),
+					tm.Month(),
+					s.lastDayOfMonth+1,
+					s.firstHourOfDay-1,
+					59,
+					59,
+					0,
+					tm.Location(),
+				)
+			} else {
+				// last day 23:59:59
+				tm = time.Date(
+					tm.Year(),
+					tm.Month(),
+					s.lastDayOfMonth,
+					23,
+					59,
+					59,
+					0,
+					tm.Location(),
+				)
+			}
+		}
+	}
+
+	timestamp := tm.Add(-179 * time.Second).Unix()
+
+	if lastMeasureTime >= timestamp && lastMeasureTime <= tm.Unix() {
+		return true
+	}
+	//fmt.Printf("Ends: lastMeasureTime=%s, timetsamp=%s\n", time.Unix(lastMeasureTime, 0), time.Unix(timestamp, 0))
+	return false
+}
+
+func (s PeriodCalculator) TimeToPeriod(tm time.Time) (period time.Time) {
+	switch s.groupBy {
+	case ByDay:
+		if s.firstHourOfDay > 0 {
+			if tm.Hour() < s.firstHourOfDay {
+				tm = tm.AddDate(0, 0, -1)
+			}
+		}
+
+	case ByMonth:
+		if s.lastDayOfMonth > 0 {
+			if s.lastDayOfMonth > 28 {
+				// На всякий случай добавляем защиту от дурака, ибо модификация даты
+				// в запросе (+1 месяц) для 29, 30 и 31 чисел сделает запрос неверным.
+				panic(fmt.Sprintf("LastDayOfMonth: %d, max allowed value is 28", s.lastDayOfMonth))
+			}
+
+			if s.firstHourOfDay > 0 {
+				if tm.Hour() >= s.firstHourOfDay {
+					if tm.Day() > s.lastDayOfMonth {
+						// add 1 month
+						tm = tm.AddDate(0, 1, 0)
+					}
+				} else {
+					// firstHourOfDay показывает что текущий день еще не наступил,
+					// поэтому day - 1
+					if (tm.Day() - 1) <= s.lastDayOfMonth {
+						tm = tm.AddDate(0, 0, -1)
+					} else {
+						// - 1 day, + 1 month
+						tm = tm.AddDate(0, 0, -1).AddDate(0, 1, 0)
+					}
+				}
+			} else {
+				if tm.Day() > s.lastDayOfMonth {
+					tm = tm.AddDate(0, 1, 0)
+				}
+			}
+		} else {
+			if s.firstHourOfDay > 0 {
+				if tm.Hour() < s.firstHourOfDay {
+					tm = tm.AddDate(0, 0, -1)
+				}
+			}
+		}
+	}
+	return timeutil.FirstSecondInPeriod(tm, string(s.groupBy))
+}

mqe/period_producers.go

@@ -0,0 +1,168 @@
+package mqe
+
+import (
+	"database/sql"
+	"fmt"
+	"time"
+)
+
+type periodProducer interface {
+	Next() (time.Time, PeriodBound, PeriodBound, bool, error)
+	Last() (PeriodBound, bool, error)
+}
+
+type PeriodBound struct {
+	Time  int64
+	Value float64
+}
+
+type PeriodProducer struct {
+	metricID          int64
+	parsePeriodLayout string
+	location          *time.Location
+	until             int64
+	corrections       []_f64Correction
+	rows              *sql.Rows
+	tx                *sql.Tx
+}
+
+type PeriodProducerOptions struct {
+	MetricID          int64
+	ParsePeriodLayout string
+	Location          *time.Location
+	Until             int64
+	Corrections       []_f64Correction
+	Rows              *sql.Rows
+	Tx                *sql.Tx
+}
+
+func NewPeriodProducer(opt PeriodProducerOptions) *PeriodProducer {
+	if opt.Location == nil {
+		panic("Location option is required")
+	}
+
+	if opt.MetricID == 0 {
+		panic("MetricID option is required")
+	}
+
+	if opt.ParsePeriodLayout == "" {
+		panic("ParsePeriodLayout option is required")
+	}
+
+	if opt.Until == 0 {
+		panic("Until option is required")
+	}
+
+	if opt.Rows == nil {
+		panic("Rows option is required")
+	}
+
+	if opt.Tx == nil {
+		panic("Tx option is required")
+	}
+
+	s := new(PeriodProducer)
+	s.metricID = opt.MetricID
+	s.parsePeriodLayout = opt.ParsePeriodLayout
+	s.location = opt.Location
+	s.until = opt.Until
+	s.corrections = opt.Corrections
+	s.rows = opt.Rows
+	s.tx = opt.Tx
+	return s
+}
+
+func (s *PeriodProducer) Next() (period time.Time, first PeriodBound, last PeriodBound, found bool, err error) {
+	var frameStr string
+
+	hasNext := s.rows.Next()
+	if hasNext {
+		err = s.rows.Scan(&frameStr, &first.Time, &first.Value, &last.Time, &last.Value)
+		if err != nil {
+			return
+		}
+
+		first.Value = applyCorrectionsToMeasure(s.corrections, first.Time, first.Value)
+		last.Value = applyCorrectionsToMeasure(s.corrections, last.Time, last.Value)
+
+		period, err = time.ParseInLocation(s.parsePeriodLayout, frameStr, s.location)
+		if err != nil {
+			err = fmt.Errorf("time.ParseInLocation: %s; layout=%q; str=%q",
+				err, s.parsePeriodLayout, frameStr)
+			return
+		}
+
+		found = true
+	} else {
+		err = s.rows.Err()
+	}
+	return
+}
+
+func (s *PeriodProducer) Last() (next PeriodBound, found bool, err error) {
+	// Находим следующее показание метрики.
+	err = s.tx.QueryRow(`
+	SELECT tm, value, true
+	FROM f64
+	WHERE metricID=? AND tm > ?
+	ORDER BY tm ASC
+	LIMIT 1`,
+		s.metricID, s.until).Scan(&next.Time, &next.Value, &found)
+
+	if err != nil {
+		if err != sql.ErrNoRows {
+			return
+		}
+		// Нет записей
+		err = nil
+	}
+
+	next.Value = applyCorrectionsToMeasure(s.corrections, next.Time, next.Value)
+	return
+}
+
+// ДЛЯ ТЕСТИРОВАНИЯ
+
+type stubPeriod struct {
+	Period time.Time
+	First  PeriodBound
+	Last   PeriodBound
+}
+
+type StubPeriodProducerOptions struct {
+	Last    *PeriodBound
+	Periods []stubPeriod
+}
+
+func NewStubPeriodProducer(opt StubPeriodProducerOptions) *StubPeriodProducer {
+	s := new(StubPeriodProducer)
+	s.last = opt.Last
+	s.periods = opt.Periods
+	return s
+}
+
+type StubPeriodProducer struct {
+	last    *PeriodBound // для отдельного запроса
+	periods []stubPeriod
+	idx     int
+}
+
+func (s *StubPeriodProducer) Next() (period time.Time, first PeriodBound, last PeriodBound, found bool, err error) {
+	if s.idx < len(s.periods) {
+		p := s.periods[s.idx]
+		s.idx++
+		period = p.Period
+		first = p.First
+		last = p.Last
+		found = true
+	}
+	return
+}
+
+func (s *StubPeriodProducer) Last() (next PeriodBound, found bool, err error) {
+	if s.last != nil {
+		next = *s.last
+		found = true
+	}
+	return
+}

mqe/raw.go

@@ -0,0 +1,81 @@
+package mqe
+
+import (
+	"database/sql"
+
+	"gordenko.dev/dima/qb"
+)
+
+type RawMeasure struct {
+	Time  int64   `json:"t"`
+	Value float64 `json:"v"`
+}
+
+type RawMeasuresFilter struct {
+	MetricID   int64         `json:"metricID"`
+	MetricType qb.MetricType `json:"metricType"`
+	Since      int64         `json:"since"` // уже учтен firstHourOfDay
+	Until      int64         `json:"until"` // уже учтен firstHourOfDay
+}
+
+// ListRawMeasures - cписок показаний мгновенных метрик (Температура, Давление, Расход)
+// за за интервал без группировки
+func (s *MeasureQueryEngine) ListRawMeasures(req RawMeasuresFilter) (_ []RawMeasure, err error) {
+	tx, err := s.db.Driver().Begin()
+	if err != nil {
+		return
+	}
+	defer tx.Rollback()
+
+	rows, err := tx.Query(`
+	SELECT tm, value
+	FROM f64
+	WHERE metricID=? AND tm BETWEEN ? AND ?
+	ORDER BY tm ASC`,
+		req.MetricID, req.Since, req.Until)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		}
+		return
+	}
+	defer rows.Close()
+
+	var result []RawMeasure
+
+	for rows.Next() {
+		var (
+			tm    int64
+			value float64
+		)
+		err = rows.Scan(&tm, &value)
+		if err != nil {
+			return
+		}
+		result = append(result, RawMeasure{
+			Time:  tm,
+			Value: value,
+		})
+	}
+
+	if err = rows.Err(); err != nil {
+		return
+	}
+
+	if len(result) == 0 {
+		return
+	}
+
+	if req.MetricType == qb.Cumulative {
+		var corrections []_f64Correction
+		corrections, err = listF64CorrectionsTx(tx, req.MetricID)
+		if err != nil {
+			return
+		}
+
+		if len(corrections) > 0 {
+			applyCorrectionsToRawMeasures(corrections, result)
+		}
+	}
+	return result, nil
+}

mqe/readings.go

@@ -0,0 +1,179 @@
+package mqe
+
+import (
+	"database/sql"
+	"fmt"
+	"sync"
+	"time"
+
+	"gordenko.dev/dima/pretty"
+	"gordenko.dev/dima/qb/timeutil"
+)
+
+////////////////////////////
+
+type MetricMeasure struct {
+	Time  int64   `json:"t"`
+	Value float64 `json:"v"`
+}
+
+type MetricError struct {
+	Time int64 `json:"time"`
+	Code int64 `json:"code"`
+}
+
+type ReadingsFilter struct {
+	Since     string `json:"since"`
+	Until     string `json:"until"`
+	MetricIDs []int64
+}
+
+type MetricReadings struct {
+	MetricID int64
+	Measures []MetricMeasure
+	Errors   []MetricError
+}
+
+// SelectReadingsInParallel - метод достает из БД данные всех метрик, которые связаны
+// с объектом. Учитываются настройки объекта (LastDayOfMonth, FirstHourOfDay), а также
+// фильтры выбранные пользователем на странице объекта (Since, Until, GroupBy).
+// ВАЖНО!
+// Запросы по каждой метрике отправляются в БД в отдельной горутине, что позволяет
+// в разы ускорить получение данных по объекту.
+func (s *MeasureQueryEngine) SelectReadingsInParallel(req ReadingsFilter) (result []MetricReadings, err error) {
+	pretty.PPrintln("SelectReadingsInParallel", req)
+
+	if len(req.MetricIDs) == 0 {
+		return
+	}
+
+	if req.Since == "" {
+		err = fix.Field(EmptyValue, "since")
+		return
+	}
+
+	if req.Until == "" {
+		err = fix.Field(EmptyValue, "until")
+		return
+	}
+
+	// ВАЖНО!
+	// Если в строке нет таймзоны - Parse функция возвращает время в UTC.
+	// Поэтому нужно использовать ParseInLocation!
+
+	since, err := time.ParseInLocation("2006-01-02", req.Since, s.location)
+	if err != nil {
+		err = fix.Field(WrongValue, "since")
+		return
+	}
+
+	until, err := time.ParseInLocation("2006-01-02", req.Until, s.location)
+	if err != nil {
+		err = fix.Field(WrongValue, "until")
+		return
+	}
+
+	// Корректируем, ВСЕГДА на последнюю секунду суток
+	until = timeutil.LastSecondInPeriod(until, string(ByDay))
+
+	fmt.Printf(`SelectReadingsInParallel: {
+	Since: %s
+	Until: %s
+	MetricIDs: %v
+}
+`, since, until, req.MetricIDs)
+	// FIX проверить как добавляются часы в DST часовых поясах
+	// Для req.FirstHourOfDay=4 since должен быть 4:00:00, until должен быть
+	// 3:59:59 следующего дня
+
+	sinceUnixtime := since.Unix()
+	untilUnixtime := until.Unix()
+
+	if sinceUnixtime >= untilUnixtime {
+		err = fix.Error(WrongDateRange)
+		return
+	}
+
+	// Важно вирутальная метрика зависит от реальных, которые к объекту могут быть
+	// привязаны или нет. Если нет - не возвращать.
+
+	var (
+		tasks []*MetricReadingsTask
+		wg    = new(sync.WaitGroup)
+	)
+
+	for _, metricID := range req.MetricIDs {
+		task := &MetricReadingsTask{
+			In: MetricMeasuresFilter{
+				MetricID: metricID,
+				Since:    sinceUnixtime,
+				Until:    untilUnixtime,
+			},
+			WaitGroup: wg,
+		}
+
+		wg.Add(1)
+		s.metricReadingsCh <- task
+		tasks = append(tasks, task)
+	}
+
+	wg.Wait()
+
+	for _, task := range tasks {
+		if task.Err != nil {
+			err = fmt.Errorf("get data for the metric %d: %s", task.In.MetricID, task.Err)
+			return
+		}
+		result = append(result, task.Result)
+	}
+	return
+}
+
+// ListInstantMeasures - cписок показаний для юнитов без накопительного итога
+// (например, Давление)
+func (s *MeasureQueryEngine) listMetricReadings(req MetricMeasuresFilter) (result MetricReadings, err error) {
+	result.MetricID = req.MetricID
+	// buf, _ := json.MarshalIndent(req, "", "    ")
+	// fmt.Printf("listInstantMeasures: %s\n\n", buf)
+
+	tx, err := s.db.Driver().Begin()
+	if err != nil {
+		return
+	}
+	defer tx.Rollback()
+
+	rows, err := tx.Query(`
+	SELECT tm, value
+	FROM f64
+	WHERE metricID=? AND tm BETWEEN ? AND ?
+	ORDER BY tm ASC`,
+		req.MetricID, req.Since, req.Until)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		}
+		return
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var measure MetricMeasure
+		err = rows.Scan(&measure.Time, &measure.Value)
+		if err != nil {
+			return
+		}
+		// fix - для cumulative метрик учет поправок
+		result.Measures = append(result.Measures, measure)
+	}
+
+	if err = rows.Err(); err != nil {
+		return
+	}
+
+	//for _, m := range result {
+	//	fmt.Printf("raw: %d\t%f\n", m.Time, m.Value)
+	//}
+	//fmt.Printf("listInstantMeasures (qty): %d\n", len(result))
+
+	return
+}

mqe/split.go

@@ -0,0 +1,570 @@
+package mqe
+
+import (
+	"fmt"
+	"time"
+)
+
+func SplitByPeriods(producer periodProducer, calc PeriodCalculator) (result []AggregatedMeasure, err error) {
+	var (
+		isOpened     bool
+		openedPeriod time.Time
+		since        int64
+		sinceValue   float64
+		until        int64
+		untilValue   float64
+
+		//
+		period      time.Time
+		first, last PeriodBound
+		found       bool
+	)
+
+	for {
+		period, first, last, found, err = producer.Next()
+		if err != nil {
+			return
+		}
+
+		if !found {
+			break
+		}
+
+		if isOpened {
+			if calc.IsExtendedSincePeriod(period) {
+				err = fmt.Errorf("extended period while already opened")
+				return
+			} else if calc.IsExtendedUntilPeriod(period) {
+				result = append(result, AggregatedMeasure{
+					Period: openedPeriod.Unix(),
+					Since:  since,
+					Until:  first.Time,
+					Values: []float64{
+						first.Value,
+						first.Value - sinceValue,
+					},
+				})
+				// Последний период успешно закрыли данными из Extended Until, поэтому выходим
+				return
+			} else {
+				if period.Equal(openedPeriod) {
+					// +
+					// Данная ситуация будет когда предыдущий период был корректно
+					// закрыт за 3 минуты до конца. После этого сразу открылся
+					// новый период. А запись для нового периода получена только
+					// сейчас.
+					if calc.IsPeriodCorrectEnds(period, last.Time) {
+						// CLOSE
+						result = append(result, AggregatedMeasure{
+							Period: openedPeriod.Unix(),
+							Since:  since,
+							Until:  last.Time,
+							Values: []float64{
+								last.Value,
+								last.Value - sinceValue,
+							},
+						})
+
+						// OPEN PERIOD (extended since, last 3 min closed)
+						openedPeriod = calc.NextPeriod(period)
+						if calc.IsExtendedUntilPeriod(openedPeriod) {
+							return
+						}
+						since = last.Time
+						sinceValue = last.Value
+						until = 0
+						untilValue = 0
+					} else {
+						// Период закроем где-то в будущем, на всякий случай запомнили последнее
+						// показание внутри периода
+						until = last.Time
+						untilValue = last.Value
+					}
+				} else {
+					// +
+					// Периоды не совпадают - смело закрываем
+					// Возможные ситуации:
+					// - был открыт мартовский период, а текущий период - апрель
+					// -
+					result = append(result, AggregatedMeasure{
+						Period: openedPeriod.Unix(),
+						Since:  since,
+						Until:  first.Time,
+						Values: []float64{
+							first.Value,
+							first.Value - sinceValue,
+						},
+					})
+
+					if first.Time != last.Time {
+						// +
+						openedPeriod = period
+						since = first.Time
+						sinceValue = first.Value
+						until = 0
+						untilValue = 0
+
+						if calc.IsPeriodCorrectEnds(period, last.Time) {
+							// Закрываем только что открытый период
+							result = append(result, AggregatedMeasure{
+								Period: openedPeriod.Unix(),
+								Since:  since,
+								Until:  last.Time,
+								Values: []float64{
+									last.Value,
+									last.Value - sinceValue,
+								},
+							})
+
+							openedPeriod = calc.NextPeriod(period)
+							if calc.IsExtendedUntilPeriod(openedPeriod) {
+								return
+							}
+							since = last.Time
+							sinceValue = last.Value
+							until = 0
+							untilValue = 0
+						} else {
+							// Период закроем где-то в будущем, на всякий случай запомнили последнее
+							// показание внутри периода
+							until = last.Time
+							untilValue = last.Value
+						}
+					} else {
+						// +
+						// first == last
+						if calc.IsPeriodCorrectEnds(period, last.Time) {
+							// Допустим закрыли предыдущий период 30 апреля в 23:58, новый стартуем за май.
+							openedPeriod = calc.NextPeriod(period)
+							if calc.IsExtendedUntilPeriod(openedPeriod) {
+								return
+							}
+						} else {
+							// Допустим закрыли предыдущий период 15 апреля, новый стартуем за апрель.
+							openedPeriod = period
+						}
+						since = last.Time
+						sinceValue = last.Value
+						until = 0
+						untilValue = 0
+					}
+				}
+			}
+		} else {
+			if calc.IsExtendedSincePeriod(period) {
+				// Если последние 3 минуты
+				if calc.IsPeriodCorrectEnds(period, last.Time) {
+					// OPEN PERIOD (extended since, last 3 min closed)
+					isOpened = true
+					openedPeriod = calc.NextPeriod(period)
+					since = last.Time
+					sinceValue = last.Value
+					until = 0
+					untilValue = 0
+				}
+			} else if calc.IsExtendedUntilPeriod(period) {
+				// Ничего не нашли, пустой результат
+				return
+			} else {
+				// OPEN PERIOD (regular first)
+				isOpened = true
+				openedPeriod = period
+				since = first.Time
+				sinceValue = first.Value
+				until = last.Time
+				untilValue = last.Value
+
+				if calc.IsPeriodCorrectEnds(openedPeriod, last.Time) {
+					// Закрываем только что открытый период
+					result = append(result, AggregatedMeasure{
+						Period: openedPeriod.Unix(),
+						Since:  since,
+						Until:  last.Time,
+						Values: []float64{
+							last.Value,
+							last.Value - sinceValue,
+						},
+					})
+
+					openedPeriod = calc.NextPeriod(period)
+					if calc.IsExtendedUntilPeriod(openedPeriod) {
+						return
+					}
+					since = last.Time
+					sinceValue = last.Value
+					until = 0
+					untilValue = 0
+				}
+			}
+		}
+	}
+
+	// Доп запрос.
+	if openedPeriod.IsZero() {
+		// Открытого периода не будет если вообще не нашли записей
+		return
+	}
+
+	next, found, err := producer.Last()
+	if err != nil {
+		return
+	}
+
+	if found {
+		result = append(result, AggregatedMeasure{
+			Period: openedPeriod.Unix(),
+			Since:  since,
+			Until:  next.Time,
+			Values: []float64{
+				next.Value,
+				next.Value - sinceValue,
+			},
+		})
+	} else {
+		if until > 0 {
+			result = append(result, AggregatedMeasure{
+				Period: openedPeriod.Unix(),
+				Since:  since,
+				Until:  until,
+				Values: []float64{
+					untilValue,
+					untilValue - sinceValue,
+				},
+			})
+		}
+	}
+	return
+}
+
+// Period - это начало периода, например для дней - это 0ч 0м 0с
+// lastTm предыдущего периода всегда будет меньше current period
+// since - это всегда сдвинутая метка времени (+ FirstHourOfDay * 3600)
+
+// При FirstHourOfDay = 3 и группировке по дням, будет такая ситуация.
+// Period будет указывать на 1 марта 00:00:00, а since на 1 марта 02:00:00,
+// а lastTm предыдущего периода на 1 марта 01:59:00.
+// Поэтому нельзя сравнивать текущий period и lastTm предыдущего периода.
+// Преобразовать lastTm к period непонятно как если нужно учитывать LastDayOfMonth.
+// Пример: 28 марта 00ч стало 1 апреля 00ч, а 27 марта 23:59 так и осталось.
+// Можно вычислить endPeriodTime для previousPeriodEndTime, но текущий период может
+// быть не строго следующим после предыдущего (например, июнь после марта).
+// Правильное решение - берем firstTm текущего периода и вычисляем его реальное
+// календарное начало. Например, FirstHourOfDay = 3, LastDayOfMonth = 28,
+// firstTm = 1 мая. Календарное начало - 28 апреля 03:00:00. И уже с этим значением
+// сравниваем previousPeriodEndTime.
+// Но можно сравнивать since c lastTm и until c firstTm для определения
+// extended периодов.
+/*
+func periodToCalendarTime(period time.Time, groupBy string, lastDayOfMonth, firstHourOfDay int) time.Time {
+	switch groupBy {
+	case "d":
+		if firstHourOfDay == 0 {
+			return period
+		}
+		// Было 0ч, стало - 2ч
+		return period.Add(time.Duration(firstHourOfDay) * time.Hour)
+
+	case "m":
+		if lastDayOfMonth == 0 {
+			if firstHourOfDay == 0 {
+				return period
+			}
+			// Было 0ч, стало - 2ч
+			return period.Add(time.Duration(firstHourOfDay) * time.Hour)
+		} else {
+			// текущий период начинается в прошлом месяце
+			tm := period.AddDate(0, -1, 0)
+			return time.Date(
+				tm.Year(),
+				tm.Month(),
+				lastDayOfMonth+1,
+				firstHourOfDay,
+				0,
+				0,
+				0,
+				tm.Location(),
+			)
+		}
+	default:
+		return period
+	}
+}
+*/
+//var threeMinutes int64 = 3 * 60
+
+/*
+type readOptions struct {
+	MetricID          int64 // только для readAggregatedMeasuresCanEndsInFuture
+	LastDayOfMonth    int
+	FirstHourOfDay    int
+	GroupBy           string
+	Since             int64
+	Until             int64
+	ParsePeriodLayout string
+}
+*/
+// Читает из sql.Rows показания и строит периоды
+/*
+func (s *MeasureQueryEngine) readAggregatedMeasuresCanEndsInFuture(tx *sql.Tx, rows *sql.Rows, opt readOptions) (result []CalculatedAggregatedMeasure, err error) {
+	var (
+		openPeriod time.Time
+		since      int64
+		sinceValue float64
+		until      int64
+		untilValue float64
+		//
+		frameStr   string
+		firstTm    int64
+		firstValue float64
+		lastTm     int64
+		lastValue  float64
+		period     time.Time
+	)
+
+	for rows.Next() {
+		err = rows.Scan(&frameStr, &firstTm, &firstValue, &lastTm, &lastValue)
+		if err != nil {
+			return
+		}
+
+		period, err = time.ParseInLocation(opt.ParsePeriodLayout, frameStr, s.location)
+		if err != nil {
+			err = fmt.Errorf("time.ParseInLocation: %s; layout=%q; str=%q",
+				err, opt.ParsePeriodLayout, frameStr)
+			return
+		}
+
+		//fmt.Printf("%s s: %s, %v; u: %s, %v\n", frameStr, time.Unix(firstTm, 0), firstValue, time.Unix(lastTm, 0), lastValue)
+
+		// Читаем записи из БД. Одна запись - один период. Возможны 3 вида периодов:
+		// 1. extended since - период длиной до 3 минут перед since. Оптимизация, чтобы
+		// не отправлять дополнительный запрос.
+		// 2. обычный период.
+		// 3. extended until - период длиной до 1 часа после until. Оптимизация, чтобы
+		// не отправлять дополнительный запрос (в большинстве случаев).
+		//
+		// ВАЖНО!
+		// Любой период (1, 2 или 3) может встретится первым. Определить что это
+		// первая запись модно простой проверкой openPeriod.IsZero()
+
+		if lastTm < opt.Since {
+			// 1. extended since period
+
+			if isPeriodCorrectEnds(period, opt.GroupBy, opt.LastDayOfMonth, opt.FirstHourOfDay, lastTm) {
+				openPeriod = nextPeriod(period, opt.GroupBy)
+				since = lastTm
+				sinceValue = lastValue
+			}
+			continue
+		} else if firstTm > opt.Until {
+			// 3. extended until period
+
+			if openPeriod.IsZero() {
+				// Нет показаний кроме extended until, поэтому выходим
+				return
+			}
+
+			// закрываем открытый период
+			result = append(result, CalculatedAggregatedMeasure{
+				Period: openPeriod.Unix(),
+				Since:  since,
+				Until:  firstTm,
+				Value:  firstValue,
+				Total:  firstValue - sinceValue,
+			})
+			return
+		}
+
+		// 2. обычный период
+
+		if openPeriod.IsZero() {
+			// это первая запись (extended since period не найден)
+			openPeriod = period
+			since = firstTm
+			sinceValue = firstValue
+		} else {
+			if !period.Equal(openPeriod) {
+				//fmt.Printf("\nperiod %s != openPeriod %s\n", period, openPeriod)
+				// Пример: открытый период - март. В period - апрель.
+				// закрываем открытый период
+				result = append(result, CalculatedAggregatedMeasure{
+					Period: openPeriod.Unix(),
+					Since:  since,
+					Until:  firstTm,
+					Value:  firstValue,
+					Total:  firstValue - sinceValue,
+				})
+
+				// И сразу открываем новый период
+				openPeriod = period
+				since = firstTm
+				sinceValue = firstValue
+			}
+			// Периоды совпадают. Когда такая ситуация возможна?
+			// Например, на предыдущей итерации был корректно закрыт (57-59 минуты)
+			// прошлый период и сразу открыт новый (текущий).
+		}
+
+		if isPeriodCorrectEnds(openPeriod, opt.GroupBy, opt.LastDayOfMonth, opt.FirstHourOfDay, lastTm) {
+			//			fmt.Printf("\nCORRECT END period %s\n", openPeriod)
+			// закрываем открытый период
+			result = append(result, CalculatedAggregatedMeasure{
+				Period: openPeriod.Unix(),
+				Since:  since,
+				Until:  lastTm,
+				Value:  lastValue,
+				Total:  lastValue - sinceValue,
+			})
+
+			// Открываем новый период
+			openPeriod = nextPeriod(openPeriod, opt.GroupBy)
+			since = lastTm
+			sinceValue = lastValue
+		} else {
+			// На последних минутах периода (57-59) не было показания, поэтому
+			// запоминаем конец периода какой есть. Если конец периода в будущем
+			// не будет найден - закроем период по текущему последнему показанию.
+			until = lastTm
+			untilValue = lastValue
+		}
+
+		// Валидируем новый период.
+		// Зачем?
+		// Пример: юзер задал opt.Until - 31 марта 23:59:59. Только что корректно
+		// закрылся март показанием от 31 марта 23:57:00. И сразу открылся новый
+		// период за апрель.
+		// Понять что закрыли последний период из запрошенных юзером просто -
+		// вычисляем календарное начало нового периода и сравниваем с opt.Until.
+		mustLessThanUntil := periodToCalendarTime(openPeriod, opt.GroupBy,
+			opt.LastDayOfMonth, opt.FirstHourOfDay)
+
+		if mustLessThanUntil.Unix() > opt.Until {
+			// Новый период вышел за opt.Until, а это означает что мы корректно
+			// закрыли последний период из запрошенных юзером. Bыходим.
+			return
+		}
+	}
+
+	// Цикл завершен, а из метода мы не вышли, а это означает одно - есть незакрытый
+	// период. Extended until не найден. Значит нужно отправить дополнительный запрос
+	// к БД.
+	var (
+		isNextValueFound bool
+		nextTm           int64
+		nextValue        float64
+	)
+
+	// Находим следующее показание метрики.
+	err = tx.QueryRow(`
+	SELECT tm, value, true
+	FROM f64
+	WHERE metricID=? AND tm > ?
+	ORDER BY tm ASC
+	LIMIT 1`,
+		opt.MetricID, opt.Until).Scan(&nextTm, &nextValue, &isNextValueFound)
+
+	if err != nil {
+		if err != sql.ErrNoRows {
+			return
+		}
+		// Нет записей
+		err = nil
+	}
+
+	if isNextValueFound {
+		// Найдено какое-то показание в будущем.
+		result = append(result, CalculatedAggregatedMeasure{
+			Period: openPeriod.Unix(),
+			Since:  since,
+			Until:  nextTm,
+			Value:  nextValue,
+			Total:  nextValue - sinceValue,
+		})
+	} else {
+		// Закрываем период самым последним показанием внутри периода
+		// (НЕ 57-59 минуты).
+		result = append(result, CalculatedAggregatedMeasure{
+			Period: openPeriod.Unix(),
+			Since:  since,
+			Until:  until,
+			Value:  untilValue,
+			Total:  untilValue - sinceValue,
+		})
+	}
+	return
+}
+*/
+
+/*
+// Читает из sql.Rows показания и строит периоды
+func (s *MeasureQueryEngine) readAggregatedMeasuresEndsInsidePeriod(tx *sql.Tx, rows *sql.Rows, opt readOptions) (result []CalculatedAggregatedMeasure, err error) {
+	var (
+		previousPeriodEndTime  int64
+		previousPeriodEndValue float64
+		//
+		frameStr   string
+		firstTm    int64
+		firstValue float64
+		lastTm     int64
+		lastValue  float64
+		period     time.Time
+	)
+
+	for rows.Next() {
+		err = rows.Scan(&frameStr, &firstTm, &firstValue, &lastTm, &lastValue)
+		if err != nil {
+			return
+		}
+
+		period, err = time.ParseInLocation(opt.ParsePeriodLayout, frameStr, s.location)
+		if err != nil {
+			err = fmt.Errorf("time.ParseInLocation: %s; layout=%q; str=%q",
+				err, opt.ParsePeriodLayout, frameStr)
+			return
+		}
+
+		//fmt.Printf("%s s: %s, %v; u: %s, %v\n", frameStr, time.Unix(firstTm, 0), firstValue, time.Unix(lastTm, 0), lastValue)
+
+		if lastTm < opt.Since {
+			// extended since period
+			previousPeriodEndTime = lastTm
+			previousPeriodEndValue = lastValue
+			continue
+		} else if firstTm > opt.Until {
+			// extended until period
+			return
+		}
+
+		// целевой период. В будущем конец периода не ищем, поэтому закрываем сразу.
+
+		// Вычисляем реальное календарное начало периода. Например, period - 1 мая 00:00:00,
+		// FirstHourOfDay = 3, LastDayOfMonth = 28. Календарное начало - 29 апреля 03:00:00.
+		periodCalendarTime := periodToCalendarTime(period, opt.GroupBy, opt.LastDayOfMonth, opt.FirstHourOfDay)
+
+		minAge := periodCalendarTime.Add(-3 * time.Minute).Unix()
+
+		if previousPeriodEndTime > 0 && previousPeriodEndTime >= minAge {
+			result = append(result, CalculatedAggregatedMeasure{
+				Period: period.Unix(),
+				Since:  previousPeriodEndTime,
+				Until:  lastTm,
+				Value:  lastValue,
+				Total:  lastValue - previousPeriodEndValue,
+			})
+		} else {
+			result = append(result, CalculatedAggregatedMeasure{
+				Period: period.Unix(),
+				Since:  firstTm,
+				Until:  lastTm,
+				Value:  lastValue,
+				Total:  lastValue - firstValue,
+			})
+		}
+
+		previousPeriodEndTime = lastTm
+		previousPeriodEndValue = lastValue
+	}
+
+	return
+}
+*/

mqe/total.go

@@ -0,0 +1,141 @@
+package mqe
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"gordenko.dev/dima/qb/timeutil"
+)
+
+// TOTALS
+
+type TotalMetricSpec struct {
+	MetricID       int64
+	LastDayOfMonth int
+	FirstHourOfDay int
+}
+
+// LastDayOfMonth и FirstHourOfDay нет, ибо у каждой метрики свои настройки
+type RangeTotalsFilter struct {
+	Since string
+	Until string
+	// Если учитывать не нужно - программа уровнем выше может передать нули.
+	// FIX у каждой метрики свои настройки
+	// искать ли конец периода в следующих периодах, если в последние 3 минуты периода
+	// не было показаний
+	CumulativeMetrics []TotalMetricSpec // metricID
+}
+
+// SelectRangeTotalsInParallel - метод рассчитывает накопленный объем от Since до Until
+// по каждой метрике из массива CumulativeMetricIDs. Учитываются настройки объекта
+// (LastDayOfMonth, FirstHourOfDay).
+// ВАЖНО!
+// Запросы по каждой метрике отправляются в БД в отдельной горутине, что позволяет
+// в разы ускорить получение данных по объекту.
+func (s *MeasureQueryEngine) GetRangeTotalsInParallel(req RangeTotalsFilter) (resultMap map[int64]RangeTotal, err error) {
+	if req.Since == "" {
+		err = fix.Field(EmptyValue, "since")
+		return
+	}
+
+	if req.Until == "" {
+		err = fix.Field(EmptyValue, "until")
+		return
+	}
+
+	// ВАЖНО!
+	// Если в строке нет таймзоны - Parse функция возвращает время в UTC.
+	// Поэтому нужно использовать ParseInLocation!
+
+	since, err := time.ParseInLocation("2006-01-02", req.Since, s.location)
+	if err != nil {
+		err = fix.Field(WrongValue, "since")
+		return
+	}
+
+	until, err := time.ParseInLocation("2006-01-02", req.Until, s.location)
+	if err != nil {
+		err = fix.Field(WrongValue, "until")
+		return
+	}
+
+	// Корректируем, чтобы указывало на последнюю секунду суток
+	until = timeutil.LastSecondInPeriod(until, string(ByDay))
+
+	var metrics []string
+
+	for _, metric := range req.CumulativeMetrics {
+		metrics = append(metrics, fmt.Sprintf(`   {
+      MetricID: %d
+      LastDayOfMonth: %d
+      FirstHourOfDay: %d
+   }`, metric.MetricID, metric.LastDayOfMonth, metric.FirstHourOfDay))
+	}
+
+	fmt.Printf(`GetRangeTotalsInParallel: {
+   Since: %s
+   Until: %s
+   CumulativeMetrics [
+%s
+   ]
+}
+`, since, until, strings.Join(metrics, "\n"))
+	if since.Unix() >= until.Unix() {
+		err = fix.Error(WrongDateRange)
+		return
+	}
+
+	var (
+		tasks []*CumulativeTotalTask
+		wg    = new(sync.WaitGroup)
+	)
+
+	resultMap = make(map[int64]RangeTotal)
+
+	// CUMULATIVE METRIC TOTALS
+	if len(req.CumulativeMetrics) > 0 {
+		for _, metric := range req.CumulativeMetrics {
+			// У каждой метрики индивидуальные настройки,
+			// ибо метрики принадлежат разным объектам!!!
+			metricSince := since
+			metricUntil := until
+
+			if metric.FirstHourOfDay > 0 {
+				metricSince = since.Add(time.Duration(metric.FirstHourOfDay) * time.Hour)
+				metricUntil = until.Add(time.Duration(metric.FirstHourOfDay) * time.Hour)
+			}
+
+			task := &CumulativeTotalTask{
+				In: getCumulativeTotalFilter{
+					MetricID:       metric.MetricID,
+					Since:          metricSince.Unix(),
+					Until:          metricUntil.Unix(),
+					LastDayOfMonth: metric.LastDayOfMonth,
+					FirstHourOfDay: metric.FirstHourOfDay,
+				},
+				WaitGroup: wg,
+			}
+
+			wg.Add(1)
+			s.cumulativeTotalCh <- task
+			tasks = append(tasks, task)
+		}
+	}
+
+	wg.Wait()
+
+	for _, task := range tasks {
+		if task.Err != nil {
+			err = fmt.Errorf("get data for the metric %d: %s", task.In.MetricID, task.Err)
+			return
+		}
+		if task.Result != nil {
+			resultMap[task.In.MetricID] = *task.Result
+		} else {
+			resultMap[task.In.MetricID] = RangeTotal{}
+		}
+	}
+	return
+}

mqe/unaggregated.go

@@ -0,0 +1,138 @@
+package mqe
+
+import (
+	"fmt"
+	"sync"
+	"time"
+)
+
+// Measure - рассчитанное показание реальной метрики. Рассчитанное означает
+// что учтен Factor. То есть значение уже можно показывать юзеру
+// Для instant - одно значение value, для cumulative - 2 значения (value, total)
+type Measure struct {
+	Time   int64     `json:"t"`
+	Values []float64 `json:"v"`
+}
+
+type UnaggregatedFilter struct {
+	Since             time.Time
+	Until             time.Time
+	LastDayOfMonth    int // например, конец месяца 25 число
+	FirstHourOfDay    int // например день начинается в 6:00
+	InstantMetrics    []int64
+	CumulativeMetrics []int64
+}
+
+// type MetricsDataResultMaps struct {
+// 	AggregatedCumulative    map[int64][]CalculatedAggregatedMeasure
+// 	AggregatedByFuncInstant map[int64][]AggregatedByFuncInstantMeasure
+// 	Cumulative              map[int64][]CalculatedMeasure
+// 	Instant                 map[int64][]CalculatedMeasure
+// }
+
+// SelectObjectDataInParallel - метод достает из БД данные всех метрик, которые связаны
+// с объектом. Учитываются настройки объекта (LastDayOfMonth, FirstHourOfDay), а также
+// фильтры выбранные пользователем на странице объекта (Since, Until, GroupBy).
+// ВАЖНО!
+// Запросы по каждой метрике отправляются в БД в отдельной горутине, что позволяет
+// в разы ускорить получение данных по объекту.
+func (s *MeasureQueryEngine) SelectUnaggregatedData(req UnaggregatedFilter) (resultMap map[int64][]Measure, err error) {
+	since := req.Since
+	until := req.Until
+
+	if req.FirstHourOfDay < 0 || req.FirstHourOfDay > 23 {
+		err = fix.Field(InvalidFirstHourOfDay, "firstHourOfDay")
+		return
+	}
+
+	if req.LastDayOfMonth < 0 || req.LastDayOfMonth > 28 {
+		err = fix.Field(InvalidLastDayOfMonth, "lastDayOfMonth")
+		return
+	}
+
+	// 	fmt.Printf(`SelectUnaggregatedData: {
+	// 	Since: %s
+	// 	Until: %s
+	// 	FirstHourOfDay: %d
+	// 	LastDayOfMonth: %d
+	// }
+	// `, since, until, req.FirstHourOfDay, req.LastDayOfMonth)
+	// FIX проверить как добавляются часы в DST часовых поясах
+	// Для req.FirstHourOfDay=4 since должен быть 4:00:00, until должен быть
+	// 3:59:59 следующего дня
+	if req.FirstHourOfDay > 0 {
+		since = since.Add(time.Duration(req.FirstHourOfDay) * time.Hour)
+		until = until.Add(time.Duration(req.FirstHourOfDay) * time.Hour)
+	}
+
+	sinceUnixtime := since.Unix()
+	untilUnixtime := until.Unix()
+
+	if sinceUnixtime >= untilUnixtime {
+		err = fix.Error(WrongDateRange)
+		return
+	}
+
+	var (
+		cumulativeTasks []*CumulativeTask
+		instantTasks    []*InstantTask
+		wg              = new(sync.WaitGroup)
+	)
+
+	// Важно вирутальная метрика зависит от реальных, которые к объекту могут быть
+	// привязаны или нет. Если нет - не возвращать.
+	resultMap = make(map[int64][]Measure)
+
+	if len(req.CumulativeMetrics) > 0 {
+		for _, metricID := range req.CumulativeMetrics {
+			task := &CumulativeTask{
+				In: MetricMeasuresFilter{
+					MetricID: metricID,
+					Since:    sinceUnixtime,
+					Until:    untilUnixtime,
+				},
+				WaitGroup: wg,
+			}
+
+			wg.Add(1)
+			s.cumulativeCh <- task
+			cumulativeTasks = append(cumulativeTasks, task)
+		}
+	}
+
+	if len(req.InstantMetrics) > 0 {
+		for _, metricID := range req.InstantMetrics {
+			task := &InstantTask{
+				In: MetricMeasuresFilter{
+					MetricID: metricID,
+					Since:    sinceUnixtime,
+					Until:    untilUnixtime,
+				},
+				WaitGroup: wg,
+			}
+
+			wg.Add(1)
+			s.instantCh <- task
+			instantTasks = append(instantTasks, task)
+		}
+	}
+
+	wg.Wait()
+
+	for _, task := range cumulativeTasks {
+		if task.Err != nil {
+			err = fmt.Errorf("get data for the metric %d: %s", task.In.MetricID, task.Err)
+			return
+		}
+		resultMap[task.In.MetricID] = task.Result
+	}
+
+	for _, task := range instantTasks {
+		if task.Err != nil {
+			err = fmt.Errorf("get data for the metric %d: %s", task.In.MetricID, task.Err)
+			return
+		}
+		resultMap[task.In.MetricID] = task.Result
+	}
+	return
+}

prepare.sh

@@ -0,0 +1,3 @@
+cd prepare
+env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o prepare
+cd -

prepare/main.go

@@ -0,0 +1,616 @@
+package main
+
+import (
+	"bufio"
+	"database/sql"
+	"fmt"
+	"log"
+	"os"
+	"sync"
+	"time"
+
+	_ "github.com/go-sql-driver/mysql"
+	bin "gordenko.dev/dima/bin/little"
+	"gordenko.dev/dima/qx"
+)
+
+type MetricType int
+type MetricCategory int
+
+const (
+	MetricTypeCumulative MetricType = 1
+	MetricTypeInstant    MetricType = 2
+	// Категории метрик
+	CategoryVolume           MetricCategory = 1
+	CategoryVolumeFlow       MetricCategory = 2
+	CategoryEnergy           MetricCategory = 3
+	CategoryMass             MetricCategory = 5
+	CategoryMassFlow         MetricCategory = 6
+	CategoryPower            MetricCategory = 7
+	CategoryTemperature      MetricCategory = 8
+	CategoryTempDiff         MetricCategory = 9
+	CategoryLevel            MetricCategory = 10
+	CategoryPressure         MetricCategory = 11
+	CategoryFrequency        MetricCategory = 12
+	CategoryRunningFrequency MetricCategory = 13
+	CategoryCurrent          MetricCategory = 14
+	CategoryVoltage          MetricCategory = 15
+	CategoryBusVoltage       MetricCategory = 16
+	CategorySpeed            MetricCategory = 17
+	CategoryTorque           MetricCategory = 18
+	CategoryDuration         MetricCategory = 19 // час роботи, роботи з помилкою, роботи без помилок
+	CategoryPressureSetting  MetricCategory = 20 // уставка тиску
+	CategoryBatteryPower     MetricCategory = 21
+	CategorySignalQuality    MetricCategory = 22
+	CategoryError            MetricCategory = 23
+	CategoryPercent          MetricCategory = 24
+	CategoryResistance       MetricCategory = 25
+
+	DSN = "root:zkx75fcy@/rrc"
+)
+
+var (
+	MetricCategoryToMetricType = map[MetricCategory]MetricType{
+		CategoryVolume:           MetricTypeCumulative,
+		CategoryVolumeFlow:       MetricTypeInstant,
+		CategoryEnergy:           MetricTypeCumulative,
+		CategoryMass:             MetricTypeCumulative,
+		CategoryMassFlow:         MetricTypeInstant,
+		CategoryPower:            MetricTypeInstant,
+		CategoryTemperature:      MetricTypeInstant,
+		CategoryTempDiff:         MetricTypeInstant,
+		CategoryLevel:            MetricTypeInstant,
+		CategoryPressure:         MetricTypeInstant,
+		CategoryPressureSetting:  MetricTypeInstant,
+		CategoryFrequency:        MetricTypeInstant,
+		CategoryRunningFrequency: MetricTypeInstant,
+		CategoryCurrent:          MetricTypeInstant,
+		CategoryVoltage:          MetricTypeInstant,
+		CategoryBusVoltage:       MetricTypeInstant,
+		CategorySpeed:            MetricTypeInstant,
+		CategoryTorque:           MetricTypeInstant,
+		CategoryDuration:         MetricTypeCumulative,
+		CategoryBatteryPower:     MetricTypeInstant,
+		CategorySignalQuality:    MetricTypeInstant,
+		CategoryError:            MetricTypeInstant,
+		CategoryPercent:          MetricTypeInstant,
+		CategoryResistance:       MetricTypeInstant,
+	}
+)
+
+func CategoryToMetricType(category MetricCategory) MetricType {
+	switch category {
+	case CategoryVolume, CategoryMass, CategoryEnergy, CategoryDuration:
+		return MetricTypeCumulative
+	}
+	return MetricTypeInstant
+}
+
+/*
+Список метрик:
+ID, тип метрики, fracDigits, начало диапазона, конец диапазона.
+
+Есть смысл записать показания в файлы. Итерация (запись в субд) пойдет быстрее.
+
+Получить список метрик.
+В цикле получить показания и записатьв файл?
+*/
+
+type Metric struct {
+	MetricID   int64      `json:"metricID"`
+	MetricType MetricType `json:"metricType"`
+	FracDigits int        `json:"fracDigits"`
+}
+
+func listMetrics(db *qx.Db) (list []Metric, err error) {
+	var tmp []struct {
+		MetricID       int64
+		FracDigits     int
+		ConfigMetricID int64
+		CustomCategory MetricCategory
+	}
+
+	err = db.ListQuery(&tmp, `
+	SELECT m.metricID, mp.fracDigits, mp.configMetricID, mp.customCategory
+	FROM metrics m
+		INNER JOIN metric_profiles mp ON mp.profileID = m.metricProfileID
+	ORDER BY m.metricID ASC`)
+
+	for _, x := range tmp {
+		var (
+			category   MetricCategory
+			metricType MetricType
+		)
+		if x.CustomCategory > 0 {
+			metricType = CategoryToMetricType(x.CustomCategory)
+		} else {
+			if x.ConfigMetricID > 0 {
+				found, err := db.OneQuery(&category, `SELECT category FROM config_metrics WHERE metricID=?`, x.ConfigMetricID)
+				if err != nil {
+					return nil, err
+				}
+				if !found {
+					return nil, fmt.Errorf("not found configMetric %d", x.ConfigMetricID)
+				}
+
+				metricType = CategoryToMetricType(category)
+			} else {
+				return nil, fmt.Errorf("customCategory not set and configMetricID = 0\n")
+			}
+		}
+
+		list = append(list, Metric{
+			MetricID:   x.MetricID,
+			FracDigits: x.FracDigits,
+			MetricType: metricType,
+		})
+	}
+	return
+}
+
+type Measure struct {
+	Time  int64   `json:"tm" `
+	Value float64 `json:"value"`
+}
+
+func listMeasures(db *qx.Db, metricID int64) (list []Measure, err error) {
+	err = db.ListQuery(&list, "SELECT tm, value FROM f64 WHERE metricID=? ORDER BY tm ASC", metricID)
+	return
+}
+
+type AppendTask struct {
+	MetricID   uint32
+	MetricType MetricType
+	FracDigits byte
+}
+
+func copyMeasures(db *qx.Db) (err error) {
+	metrics, err := listMetrics(db)
+	if err != nil {
+		return
+	}
+
+	fmt.Printf("found %d metrics\n", len(metrics))
+
+	wg := new(sync.WaitGroup)
+	stopCh := make(chan struct{})
+	taskCh := make(chan AppendTask)
+
+	for range 10 {
+		go writer(db, taskCh, wg, stopCh)
+	}
+
+	for _, metric := range metrics {
+		taskCh <- AppendTask{
+			MetricID:   uint32(metric.MetricID),
+			MetricType: metric.MetricType,
+			FracDigits: byte(metric.FracDigits),
+		}
+	}
+
+	close(stopCh)
+
+	wg.Wait()
+	return nil
+}
+
+func writer(db *qx.Db, taskCh chan AppendTask, wg *sync.WaitGroup, stopCh chan struct{}) {
+	wg.Add(1)
+
+	for {
+		select {
+		case task := <-taskCh:
+			err := writeMetricIntoFile(db, task)
+			if err != nil {
+				log.Println(err)
+			}
+
+		case <-stopCh:
+			wg.Done()
+			return
+		}
+	}
+
+}
+
+// const maxQtyInPack = 65535
+
+// func writer(taskCh chan AppendTask, wg *sync.WaitGroup, stopCh chan struct{}) {
+// 	wg.Add(1)
+
+// 	c, err := client.Connect(":12345")
+// 	if err != nil {
+// 		log.Fatalln(err)
+// 	}
+
+// 	for {
+// 		select {
+// 		case task := <-taskCh:
+// 			err = writeMetricIntoOctopus(c, task)
+// 			if err != nil {
+// 				log.Println(err)
+// 			}
+
+// 		case <-stopCh:
+// 			wg.Done()
+// 			return
+// 		}
+// 	}
+
+// }
+
+func correctToMonotonic(measures []Measure) []Measure {
+	if len(measures) == 0 {
+		return nil
+	}
+
+	if measures[0].Value < 0 {
+		var (
+			idx int
+			m   Measure
+		)
+		for idx, m = range measures {
+			if m.Value >= 0 {
+				break
+			}
+		}
+		measures = measures[idx:]
+	}
+
+	if len(measures) == 0 {
+		return nil
+	}
+
+	var (
+		baseValue float64
+		prevValue = measures[0].Value // как прислал прибор
+	)
+	for idx := 1; idx < len(measures); idx++ {
+		measure := measures[idx]
+
+		if measure.Value < 0 {
+			continue
+		}
+
+		if measure.Value < prevValue {
+			// произошел сброс
+			baseValue += prevValue
+		}
+		prevValue = measure.Value
+
+		if baseValue > 0 {
+			measure.Value += baseValue
+			measures[idx] = measure
+		}
+	}
+	return measures
+}
+
+func writeMetricIntoFile(db *qx.Db, task AppendTask) (err error) {
+	measures, err := listMeasures(db, int64(task.MetricID))
+	if err != nil {
+		return fmt.Errorf("listMeasures(%d): %s", task.MetricID, err)
+	}
+	//fmt.Printf("metric %d has %d measures\n", metric.MetricID, len(measures))
+	if len(measures) < 1000 {
+		return
+	}
+
+	filename := fmt.Sprintf("%d.%d", task.MetricID, task.FracDigits)
+
+	if task.MetricType == MetricTypeCumulative {
+		measures = correctToMonotonic(measures)
+
+		if len(measures) < 1000 {
+			return
+		}
+
+		filename = "cumulative/" + filename
+	} else {
+		filename = "instant/" + filename
+	}
+
+	file, err := os.OpenFile(filename, os.O_CREATE|os.O_RDWR, 0666)
+	if err != nil {
+		return
+	}
+
+	dst := bufio.NewWriter(file)
+
+	for _, measure := range measures {
+		err = bin.WriteUint32(dst, uint32(measure.Time))
+		if err != nil {
+			return
+		}
+		err = bin.WriteFloat64(dst, measure.Value)
+		if err != nil {
+			return
+		}
+	}
+
+	err = dst.Flush()
+	if err != nil {
+		return
+	}
+
+	err = file.Close()
+	if err != nil {
+		return
+	}
+	fmt.Printf("%d: %d measures written\n", task.MetricID, len(measures))
+	return
+}
+
+// func writeMetricIntoOctopus(c *client.Connection, task AppendTask) error {
+// err := c.AddMetric(proto.AddMetricReq{
+// 	MetricID:   task.MetricID,
+// 	MetricType: octopus.MetricType(task.MetricType),
+// 	FracDigits: int(task.FracDigits),
+// })
+// if err != nil {
+// 	log.Fatal(err)
+// } else {
+// 	fmt.Println("metric added")
+// }
+
+// t1 := time.Now()
+// var (
+// 	prevTime int64
+// 	pack     []proto.Measure
+// )
+
+// for _, measure := range task.Measures {
+// 	if measure.Time <= prevTime {
+// 		continue
+// 	} else {
+// 		prevTime = measure.Time
+// 	}
+
+// 	pack = append(pack, proto.Measure{
+// 		Timestamp: uint32(measure.Time),
+// 		Value:     measure.Value,
+// 	})
+
+// 	if len(pack) == maxQtyInPack {
+// 		err = c.AppendMeasures(proto.AppendMeasuresReq{
+// 			MetricID: task.MetricID,
+// 			Measures: pack,
+// 		})
+// 		if err != nil {
+// 			return fmt.Errorf("append measures (%d): %s", task.MetricID, err)
+// 		} else {
+// 			//fmt.Printf("written %d measures of %d\n", len(pack), len(task.Measures))
+// 			pack = nil
+// 		}
+// 	}
+// }
+
+// if len(pack) > 0 {
+// 	err = c.AppendMeasures(proto.AppendMeasuresReq{
+// 		MetricID: task.MetricID,
+// 		Measures: pack,
+// 	})
+// 	if err != nil {
+// 		return fmt.Errorf("append measures (%d): %s", task.MetricID, err)
+// 	} else {
+// 		//fmt.Printf("written %d measures of %d\n", len(pack), len(task.Measures))
+// 	}
+// }
+//fmt.Printf("written %d in %.2f seconds\n", len(task.Measures), time.Since(t1).Seconds())
+//return nil
+//}
+
+func main() {
+	db, err := qx.Open("mysql", DSN)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	copyAllMetrics(db)
+	//copyAllMetricsByOne()
+	//copyOneMetric(db, 134)
+	// metrics, err := listMetrics(db)
+	// if err != nil {
+	// 	log.Fatalln(err)
+	// }
+
+	// pretty.Println(metrics)
+	//checkAllMetrics(db)
+}
+
+func copyAllMetrics(db *qx.Db) {
+	t1 := time.Now()
+	err := copyMeasures(db)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	fmt.Printf("total time: %.1f seconds\n", time.Since(t1).Seconds())
+}
+
+// func copyOneMetric(db *qx.Db, metricID int64) {
+// 	measures, err := listMeasures(db, metricID)
+// 	if err != nil {
+// 		log.Fatalln(err)
+// 	}
+
+// 	c, err := client.Connect(":12345")
+// 	if err != nil {
+// 		log.Fatalln(err)
+// 	}
+
+// 	writeMetricIntoOctopus(c, AppendTask{
+// 		MetricID:   uint32(metricID),
+// 		MetricType: MetricTypeInstant,
+// 		FracDigits: 3,
+// 		Measures:   measures,
+// 	})
+
+// 	// measures = measures[:2300]
+
+// 	// timestampsBuf := conbuf.New(nil)
+// 	// timestamps := chunkenc.NewReverseTimeDeltaOfDeltaCompressor(timestampsBuf, 0)
+
+// 	// valuesBuf := conbuf.New(nil)
+// 	// values := chunkenc.NewReverseInstantDeltaCompressor(valuesBuf, 0, 3)
+
+// 	// for _, measure := range measures[:10] {
+// 	// 	// if measure.Time < 1686671940 {
+// 	// 	// 	fmt.Printf("idx: %d\n", idx)
+// 	// 	// 	continue
+// 	// 	// }
+
+// 	// 	//fmt.Printf("ts: %d, v: %.2f\n", measure.Time, measure.Value)
+
+// 	// 	timestamps.Append(uint32(measure.Time))
+// 	// 	values.Append(measure.Value)
+// 	// }
+
+// 	// timestampDecompressor := chunkenc.NewReverseTimeDeltaOfDeltaDecompressor(
+// 	// 	timestampsBuf,
+// 	// 	timestamps.Size(),
+// 	// )
+
+// 	// valueDecompressor := chunkenc.NewReverseInstantDeltaDecompressor(
+// 	// 	valuesBuf,
+// 	// 	values.Size(),
+// 	// 	3,
+// 	// )
+
+// 	// fmt.Println("----------------------------------------------------")
+
+// 	// var (
+// 	// 	value     float64
+// 	// 	timestamp uint32
+// 	// 	done      bool
+// 	// )
+
+// 	// for {
+// 	// 	timestamp, done = timestampDecompressor.NextValue()
+// 	// 	if done {
+// 	// 		break
+// 	// 	}
+
+// 	// 	value, done = valueDecompressor.NextValue()
+// 	// 	if done {
+// 	// 		fmt.Printf("ts before crash: %d\n", timestamp)
+// 	// 		panic("FUCK")
+// 	// 	}
+
+// 	// 	fmt.Printf("ts: %d, v: %.2f\n", timestamp, value)
+// 	// }
+// }
+
+// func checkAllMetrics(db *qx.Db) {
+// 	metrics, err := listMetrics(db)
+// 	if err != nil {
+// 		return
+// 	}
+
+// 	//metrics = metrics[:100]
+
+// 	c, err := client.Connect(":12345")
+// 	if err != nil {
+// 		log.Fatalln(err)
+// 	}
+
+// 	var total int
+
+// 	for _, metric := range metrics {
+// 		if metric.MetricType == MetricTypeCumulative {
+// 			list, err := c.ListAllCumulativeMeasures(uint32(metric.MetricID))
+// 			if err != nil {
+// 				fmt.Printf("ListAllCumulativeMeasures(%d): %s\n", metric.MetricID, err)
+// 			} else {
+// 				total += len(list)
+// 			}
+// 		} else {
+// 			list, err := c.ListAllInstantMeasures(uint32(metric.MetricID))
+// 			if err != nil {
+// 				fmt.Printf("ListAllInstantMeasures(%d): %s\n", metric.MetricID, err)
+// 			} else {
+// 				total += len(list)
+// 			}
+// 		}
+// 		//fmt.Printf("#%d: %d\n", idx, metric.MetricID)
+
+// 	}
+// 	fmt.Printf("total: %d\n", total)
+// }
+
+// копирование равномерное
+type Reading struct {
+	MetricID int
+	ReadAt   time.Time
+	Value    float64
+}
+
+func copyAllMetricsByOne() {
+	mydb, err := qx.Open("mysql", DSN)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	metrics, err := listMetrics(mydb)
+	if err != nil {
+		return
+	}
+	mydb.Close()
+
+	//metrics = metrics[:100]
+
+	fmt.Printf("found %d metrics\n", len(metrics))
+
+	// Підключення до MySQL
+	db, err := sql.Open("mysql", DSN)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer db.Close()
+
+	// Завантажити всі показники, відсортовані по часу
+	var (
+		ts    = time.Now().Unix()
+		total int
+	)
+
+	for {
+		// Рівномірна вставка: поки є хоча б один показник
+		tx, err := db.Begin()
+		if err != nil {
+			log.Fatalln(err)
+		}
+
+		localTs := ts
+
+		for range 100 {
+			localTs += 60
+			for _, metric := range metrics {
+				_, err = tx.Exec("INSERT INTO readings (metricID, tm, value) VALUES (?, ?, ?)",
+					metric.MetricID, localTs, 1232232)
+				if err != nil {
+					log.Fatalf("Insert error: %v", err)
+				}
+
+				total++
+				if total > 841000000 {
+					break
+				}
+			}
+
+		}
+
+		err = tx.Commit()
+		if err != nil {
+			log.Fatalln(err)
+		}
+
+		fmt.Printf("inserted: %d\n", total)
+
+		if total > 841000000 {
+			break
+		}
+
+		ts += 660
+	}
+}

prepare/main_test.go

@@ -0,0 +1 @@
+package main

recovery/recovery.go

@@ -175,12 +175,12 @@ func Recovery(dir string, databaseName string) (_ RecoveryReport, err error) {
 			return
 		}
 
-		fmt.Printf("before replay\n")
+		//fmt.Printf("before replay\n")
 		err = walReplayer.Replay()
 		if err != nil {
 			return
 		}
-		fmt.Printf("after replay\n")
+		//fmt.Printf("after replay\n")
 		// перезаписую сторінки із останнього комміта
 		var dataFile *os.File
 		dataFile, err = os.OpenFile(qb.GetDataFilePath(dir, databaseName), os.O_CREATE|os.O_WRONLY, 0666)

storage/cursor.go

@@ -6,6 +6,8 @@ import (
 
 	bin "gordenko.dev/dima/bin/little"
 	"gordenko.dev/dima/qb"
+	"gordenko.dev/dima/qb/enc"
+	"gordenko.dev/dima/qb/util"
 )
 
 type BackwardCursor struct {
@@ -29,6 +31,7 @@ type BackwardCursorOptions struct {
 }
 
 func NewBackwardCursor(opt BackwardCursorOptions) (*BackwardCursor, error) {
+	fmt.Printf("footer: % x\n", opt.PageData[DataPageSize-DataPageFooterSize:])
 	switch opt.MetricType {
 	case qb.Instant, qb.Cumulative:
 		// ok
@@ -90,6 +93,7 @@ func (s *BackwardCursor) Prev() (uint32, float64, bool, error) {
 	}
 
 	prevPageNo, _ := bin.GetUint32(s.pageData[prevPageIdx:])
+	fmt.Printf("CURSOR: prev pageNo %d\n", prevPageNo)
 	if prevPageNo == 0 {
 		return 0, 0, true, nil
 	}
@@ -129,8 +133,51 @@ func (s *BackwardCursor) Close() {
 	s.releasePage(s.pageNo)
 }
 
-// HELPER
+func CreateTimestampDecompressor(page []byte) (qb.TimestampDecompressor, error) {
+	size, _ := bin.GetUint16(page[timestampsSizeIdx:])
+	if int(size) > DataPagePayloadSize {
+		return nil, fmt.Errorf("bug: invalid timestamps size %d", size)
+	}
+	pos := DataPagePayloadSize - int(size)
+	d := enc.NewTimeDeltaDecompressor()
+	d.RestoreFromEnd(page[pos:DataPagePayloadSize])
 
-//func (s *BackwardCursor) makeDecompressors() error {
+	payload := page[pos:DataPagePayloadSize]
 
-//}
+	fmt.Printf("PAGE timestamps (CURSOR) %d:\n% x\n", len(payload), payload)
+	return d, nil
+}
+
+func CreateValueDecompressor(page []byte, metricType qb.MetricType, fracDigits byte) (qb.ValueDecompressor, error) {
+	size, _ := bin.GetUint16(page[valuesSizeIdx:])
+	if int(size) > DataPagePayloadSize {
+		return nil, fmt.Errorf("bug: invalid values size %d", size)
+	}
+	d := enc.NewValueDeltaDecompressor(metricType, fracDigits)
+	d.RestoreFromEnd(page[:size])
+	return d, nil
+}
+
+func VerifyDataPageCRC32(data []byte) error {
+	var (
+		calculatedCRC = util.CalculateCRC32(data[:dataCRC32Idx])
+		writtenCRC, _ = bin.GetUint32(data[dataCRC32Idx:])
+	)
+	if calculatedCRC != writtenCRC {
+		return fmt.Errorf("calculated CRC32 %d are not equal written CRC32 %d",
+			calculatedCRC, writtenCRC)
+	}
+	return nil
+}
+
+func VerifyIndexPageCRC32(data []byte) error {
+	var (
+		calculatedCRC = util.CalculateCRC32(data[:indexCRC32Idx])
+		writtenCRC, _ = bin.GetUint32(data[indexCRC32Idx:])
+	)
+	if calculatedCRC != writtenCRC {
+		return fmt.Errorf("calculated CRC32 %d are not equal written CRC32 %d",
+			calculatedCRC, writtenCRC)
+	}
+	return nil
+}

storage/misc.go

@@ -1,55 +0,0 @@
-package storage
-
-import (
-	"fmt"
-
-	bin "gordenko.dev/dima/bin/little"
-	"gordenko.dev/dima/qb"
-	"gordenko.dev/dima/qb/enc"
-	"gordenko.dev/dima/qb/util"
-)
-
-func CreateTimestampDecompressor(page []byte) (qb.TimestampDecompressor, error) {
-	size, _ := bin.GetUint16(page[timestampsSizeIdx:])
-	if size > DataPageFooterSize {
-		return nil, fmt.Errorf("bug: invalid timestamps size %d", size)
-	}
-	pos := DataPagePayloadSize - int(size)
-	d := enc.NewTimeDeltaDecompressor()
-	d.RestoreFromEnd(page[pos:DataPagePayloadSize])
-	return d, nil
-}
-
-func CreateValueDecompressor(page []byte, metricType qb.MetricType, fracDigits byte) (qb.ValueDecompressor, error) {
-	size, _ := bin.GetUint16(page[valuesSizeIdx:])
-	if size > DataPageFooterSize {
-		return nil, fmt.Errorf("bug: invalid timestamps size %d", size)
-	}
-	d := enc.NewValueDeltaDecompressor(metricType, fracDigits)
-	d.RestoreFromEnd(page[:size])
-	return d, nil
-}
-
-func VerifyDataPageCRC32(data []byte) error {
-	var (
-		calculatedCRC = util.CalculateCRC32(data[:dataCRC32Idx])
-		writtenCRC, _ = bin.GetUint32(data[dataCRC32Idx:])
-	)
-	if calculatedCRC != writtenCRC {
-		return fmt.Errorf("calculated CRC32 %d are not equal written CRC32 %d",
-			calculatedCRC, writtenCRC)
-	}
-	return nil
-}
-
-func VerifyIndexPageCRC32(data []byte) error {
-	var (
-		calculatedCRC = util.CalculateCRC32(data[:indexCRC32Idx])
-		writtenCRC, _ = bin.GetUint32(data[indexCRC32Idx:])
-	)
-	if calculatedCRC != writtenCRC {
-		return fmt.Errorf("calculated CRC32 %d are not equal written CRC32 %d",
-			calculatedCRC, writtenCRC)
-	}
-	return nil
-}

storage/navigation.go

@@ -1,6 +1,8 @@
 package storage
 
-import bin "gordenko.dev/dima/bin/little"
+import (
+	bin "gordenko.dev/dima/bin/little"
+)
 
 const (
 	PageNoSize = 4
@@ -61,16 +63,6 @@ func BinarySearch(qty int, keyComparator KeyComparator) (elemIdx int, isFound bo
 	}
 }
 
-// func getPrevPageNo(buf []byte) uint32 {
-// 	pageNo, _ := bin.GetUint32(buf[prevPageIdx:])
-// 	return pageNo
-// }
-
-// func GetIndexRecordsSince(buf []byte) (pageNo uint32) {
-// 	pageNo, _ = bin.GetUint32(buf)
-// 	return
-// }
-
 func FindPageOnIndexLevelTail(level IndexLevelTail, timestamp uint32) (pageNo uint32) {
 	comparator := ValueAtComparator{
 		buf:       level.Buffer,
@@ -82,17 +74,6 @@ func FindPageOnIndexLevelTail(level IndexLevelTail, timestamp uint32) (pageNo ui
 	return
 }
 
-// func FindPageOnRecords(records []byte, timestamp uint32) (pageNo uint32) {
-// 	comparator := ValueAtComparator{
-// 		buf:       records,
-// 		timestamp: timestamp,
-// 	}
-// 	count := len(records) / IndexRecordSize
-// 	elemIdx, _ := BinarySearch(count, comparator)
-// 	pageNo, _ = bin.GetUint32(records[elemIdx*IndexRecordSize:])
-// 	return
-// }
-
 func FindPageOnIndexPage(page []byte, timestamp uint32) (pageNo uint32) {
 	comparator := ValueAtComparator{
 		buf:       page,
@@ -123,6 +104,108 @@ func FindPageOnIndexLevelTails(levels []IndexLevelTail, timestamp uint32) (pageN
 	return
 }
 
+type DeleteSinceOnIndexPageResult struct {
+	Buffer       []byte
+	RecordsCount int
+	PageNumbers  []uint32
+	//Idx          int
+}
+
+func DeleteSinceOnIndexPage(buf []byte, since uint32) DeleteSinceOnIndexPageResult {
+	comparator := ValueAtComparator{
+		buf:       buf,
+		timestamp: since,
+	}
+	count, _ := bin.GetUint16(buf[indexRecordsCountIdx:])
+	elemIdx, _ := BinarySearch(int(count), comparator)
+	newbuf := make([]byte, IndexPageSize)
+	copy(newbuf, buf[:elemIdx*IndexRecordSize]) // [0..idx)
+	pos := elemIdx*IndexRecordSize + 4          // timestamp size
+	deleteCount := int(count) - elemIdx
+	var pageNumbers []uint32
+	for range deleteCount {
+		pageNo, _ := bin.GetUint32(buf[pos:])
+		pageNumbers = append(pageNumbers, pageNo)
+		pos += IndexRecordSize
+	}
+
+	return DeleteSinceOnIndexPageResult{
+		Buffer:       newbuf,
+		RecordsCount: elemIdx,
+		PageNumbers:  pageNumbers,
+		//Idx:          elemIdx,
+	}
+}
+
+type DeleteOnIndexTailResult struct {
+	RecordsCount int
+	PageNumbers  []uint32
+}
+
+func DeleteSinceOnIndexTail(level IndexLevelTail, since uint32) DeleteOnIndexTailResult {
+	comparator := ValueAtComparator{
+		buf:       level.Buffer,
+		timestamp: since,
+	}
+	elemIdx, _ := BinarySearch(level.RecordsCount, comparator)
+	pos := elemIdx*IndexRecordSize + 4 // timestamp size
+	deleteCount := level.RecordsCount - elemIdx
+	var pageNumbers []uint32
+	for range deleteCount {
+		pageNo, _ := bin.GetUint32(level.Buffer[pos:])
+		pageNumbers = append(pageNumbers, pageNo)
+		pos += IndexRecordSize
+	}
+	return DeleteOnIndexTailResult{
+		RecordsCount: elemIdx,
+		PageNumbers:  pageNumbers,
+		//Idx:          elemIdx,
+	}
+}
+
+func DeleteUntilOnIndexTail(level IndexLevelTail, until uint32) DeleteOnIndexTailResult {
+	comparator := ValueAtComparator{
+		buf:       level.Buffer,
+		timestamp: until,
+	}
+	elemIdx, _ := BinarySearch(level.RecordsCount, comparator)
+	pos := 4 // timestamp size
+	deleteCount := elemIdx + 1
+	var pageNumbers []uint32
+	for range deleteCount {
+		pageNo, _ := bin.GetUint32(level.Buffer[pos:])
+		pageNumbers = append(pageNumbers, pageNo)
+		pos += IndexRecordSize
+	}
+	copy(level.Buffer, level.Buffer[:])
+	return DeleteOnIndexTailResult{
+		RecordsCount: elemIdx,
+		PageNumbers:  pageNumbers,
+		//Idx:          elemIdx,
+	}
+}
+
+// func getPrevPageNo(buf []byte) uint32 {
+// 	pageNo, _ := bin.GetUint32(buf[prevPageIdx:])
+// 	return pageNo
+// }
+
+// func GetIndexRecordsSince(buf []byte) (pageNo uint32) {
+// 	pageNo, _ = bin.GetUint32(buf)
+// 	return
+// }
+
+// func FindPageOnRecords(records []byte, timestamp uint32) (pageNo uint32) {
+// 	comparator := ValueAtComparator{
+// 		buf:       records,
+// 		timestamp: timestamp,
+// 	}
+// 	count := len(records) / IndexRecordSize
+// 	elemIdx, _ := BinarySearch(count, comparator)
+// 	pageNo, _ = bin.GetUint32(records[elemIdx*IndexRecordSize:])
+// 	return
+// }
+
 // func findPageNoIdx(buf []byte, timestamp uint32) (idx int) {
 // 	comparator := ValueAtComparator{
 // 		buf:       buf,
@@ -167,16 +250,17 @@ func FindPageOnIndexLevelTails(levels []IndexLevelTail, timestamp uint32) (pageN
 // 	return
 // }
 
-// func listPageNumbers(buf []byte) (pageNumbers []uint32) {
+// from index page
-// 	qty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
+func ListPageNumbers(buf []byte) (pageNumbers []uint32) {
-// 	pos := indexFooterIdx - PageNoSize
+	count, _ := bin.GetUint16(buf[indexRecordsCountIdx:])
-// 	for range qty {
+	pos := timestampSize
-// 		pageNo, _ := bin.GetUint32(buf[pos:])
+	for range count {
-// 		pageNumbers = append(pageNumbers, pageNo)
+		pageNo, _ := bin.GetUint32(buf[pos:])
-// 		pos -= PageNoSize
+		pageNumbers = append(pageNumbers, pageNo)
-// 	}
+		pos += IndexRecordSize
-// 	return
+	}
-// }
+	return
+}
 
 // // include since timestamp
 // func listPageNumbersSince(buf []byte, timestamp uint32) (pageNumbers []uint32) {

storage/replay_metric.go

@@ -51,6 +51,7 @@ type ReplayMeasuresAppendWithGrowResult struct {
 }
 
 func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord, isLastPacket bool) (_ ReplayMeasuresAppendWithGrowResult) {
+	//fmt.Printf("%#v\n", rec)
 	var (
 		reusedIndexPages int
 		reusedDataPages  int
@@ -164,17 +165,34 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord,
 // HELPERS
 
 func (s *ReplayMetric) composeHeadIndexPage(levelIdx int, head *IndexPageTail) PageToWrite {
-	level := s.IndexLevelTails[levelIdx]
+	var (
-	// розраховую pos, з якого буду дописувати хвіст
+		page      = make([]byte, IndexPageSize)
-	pos := level.RecordsCount * IndexRecordSize
+		pos       int
-	// створюю копію сторінки
+		prevCount int
-	page := make([]byte, IndexPageSize)
+	)
-	copy(page, level.Buffer[:pos]) // поточні дані
+	//fmt.Printf("composeHeadIndexPage: levels=%d\n", len(s.IndexLevelTails))
+	//fmt.Printf("head: % x\n", head.Records)
+	if levelIdx < len(s.IndexLevelTails) {
+		level := s.IndexLevelTails[levelIdx]
+
+		//fmt.Printf("level (%d): % x\n", level.RecordsCount, level.Buffer)
+		// розраховую pos, з якого буду дописувати хвіст
+		pos = level.RecordsCount * IndexRecordSize
+		// створюю копію сторінки
+		copy(page, level.Buffer[:pos]) // поточні дані
+		prevCount = level.RecordsCount
+	} else {
+		//
+		s.IndexLevelTails = append(s.IndexLevelTails, IndexLevelTail{
+			Buffer:       page,
+			RecordsCount: len(head.Records) / IndexRecordSize,
+		})
+	}
 	copy(page[pos:], head.Records)
 	// запечатати сторінку
 	calculatedCRC := SealIndexPage(SealIndexPageIn{
 		Content:      page,
-		RecordsCount: level.RecordsCount + len(head.Records)/IndexRecordSize,
+		RecordsCount: prevCount + len(head.Records)/IndexRecordSize,
 		ZeroLevel:    levelIdx == 0,
 	})
 	// перевірка CRC

storage/storage.go

@@ -17,7 +17,7 @@ var (
 	prevPageIdx       = DataPageSize - 12
 
 	// index page
-	IndexPageSize = 1024
+	IndexPageSize = 24 // 1024
 
 	IndexPagePayloadSize = IndexPageSize - IndexPageFooterSize
 	//indexPageIncSize     = IndexPageIncSize
@@ -27,7 +27,7 @@ var (
 
 	maxRecordsOnIndexPage = (IndexPageSize - IndexPageFooterSize) / IndexRecordSize
 
-	// timestampSize  = 4
+	timestampSize = 4
 	// pairSize       = timestampSize + PageNoSize
 	// dataFooterIdx  = timestampsSizeIdx
 )

storage/write_preparer.go

@@ -2,7 +2,6 @@ package storage
 
 import (
 	"bytes"
-	"fmt"
 	"io"
 
 	bin "gordenko.dev/dima/bin/little"
@@ -270,10 +269,10 @@ func finalizePacket(w *bytes.Buffer, checksum uint32) []byte {
 	start := 9 - bin.CountVarSize(bodySize)
 	bin.PutVarSize(packet[start:], bodySize)
 
-	fmt.Printf("checksum: %d\n", checksum)
+	//fmt.Printf("checksum: %d\n", checksum)
-	fmt.Printf("bodySize: %d\n", bodySize)
+	//fmt.Printf("bodySize: %d\n", bodySize)
 	//fmt.Printf("start: %d\n", start)
-	fmt.Printf("body: % x\n", packet[9:len(packet)-4])
+	//fmt.Printf("body: % x\n", packet[9:len(packet)-4])
 
 	return packet[start:]
 }

storage/writer.go

@@ -23,7 +23,7 @@ const (
 
 	filePerm = 0770
 
-	dumpSnapshotAfterNBytes = 1024 * 1024 * 1024 // 1 GB
+	dumpSnapshotAfterNBytes = 20 * 1024 // 1024 * 1024 * 1024 // 1 GB
 
 	writeBufferSize = 4 * 1024 * 1024
 )
@@ -64,19 +64,18 @@ type Writer struct {
 	indexPagesCount uint32
 	dataFreeList    *freelist.FreeList
 	indexFreeList   *freelist.FreeList
-	//atree               *atree.Atree
+	dir             string
-	dir           string
+	databaseName    string
-	databaseName  string
+	w               *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
-	w             *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
+	wal             *os.File
-	wal           *os.File
+	dataFile        *os.File
-	dataFile      *os.File
+	indexFile       *os.File
-	indexFile     *os.File
+	input           []any
-	input         []any
+	writePreparer   *WritePreparer
-	writePreparer *WritePreparer
+	written         int64
-	written       int64
+	isExited        bool
-	isExited      bool
+	exitCh          chan struct{}
-	exitCh        chan struct{}
+	waitGroup       *sync.WaitGroup
-	waitGroup     *sync.WaitGroup
 }
 
 type WriterOptions struct {
@@ -225,7 +224,7 @@ func (s *Writer) packAndWrite() (err error) {
 	prepared := s.writePreparer.Prepare(input)
 
 	//fmt.Printf("prepared: %#v\n", prepared)
-	fmt.Println("prepared")
+	//fmt.Println("prepared")
 
 	// 3. Пишу на диск WAL (append)
 	n, err := s.wal.Write(prepared.Packet)
@@ -239,7 +238,8 @@ func (s *Writer) packAndWrite() (err error) {
 	if err = s.wal.Sync(); err != nil {
 		return
 	}
-	fmt.Println("synced to wal")
+	s.written += int64(len(prepared.Packet))
+	//fmt.Println("synced to wal")
 
 	// 4. Пишу в atree сторінки
 	if len(prepared.WriteToData) > 0 {
@@ -250,7 +250,7 @@ func (s *Writer) packAndWrite() (err error) {
 		if err = s.dataFile.Sync(); err != nil {
 			return
 		}
-		fmt.Println("written to data")
+		//fmt.Println("written to data")
 	}
 	if len(prepared.WriteToIndex) > 0 {
 		err = WriteIndexPages(s.indexFile, prepared.WriteToIndex)
@@ -260,7 +260,7 @@ func (s *Writer) packAndWrite() (err error) {
 		if err = s.indexFile.Sync(); err != nil {
 			return
 		}
-		fmt.Println("written to index")
+		//fmt.Println("written to index")
 	}
 	// 6. відправляю input - worker-у
 
@@ -296,6 +296,8 @@ func (s *Writer) packAndWrite() (err error) {
 
 		snapshotNumber := <-snapshotNumberCh
 
+		//fmt.Println("got snapshot number:", snapshotNumber)
+
 		// копіюю сторінки із delta файла в base файл і потім роблю Truncate
 		err = s.indexFreeList.Merge()
 		if err != nil {
@@ -305,6 +307,7 @@ func (s *Writer) packAndWrite() (err error) {
 		if err != nil {
 			return
 		}
+		//fmt.Println("free lists merged")
 
 		var err error
 		s.wal, err = os.OpenFile(
@@ -315,11 +318,12 @@ func (s *Writer) packAndWrite() (err error) {
 		if err != nil {
 			return fmt.Errorf("create new changes file: %s", err)
 		}
+		//fmt.Println("new wal created")
 	} else {
 		s.workerInbox.Push(Changes{
 			Commits: prepared.Commits,
 		})
-		fmt.Println("pushed commits")
+		//fmt.Println("pushed commits")
 	}
 	return nil
 }

worker/metric.go

@@ -2,12 +2,13 @@ package worker
 
 import (
 	"errors"
-	"fmt"
 	"io"
+	"time"
 
 	bin "gordenko.dev/dima/bin/little"
 	"gordenko.dev/dima/qb"
 	"gordenko.dev/dima/qb/inbox"
+	"gordenko.dev/dima/qb/proto"
 	"gordenko.dev/dima/qb/storage"
 )
 
@@ -119,37 +120,53 @@ func (s *Metric) ReleaseRLock() {
 	s.rLocks--
 	if s.rLocks == 0 {
 		if len(s.waitQueue) > 0 {
-			s.ProcessQueue()
+			//s.ProcessQueue()
 		}
 	}
 }
 
-// суть у тому що треба запускати запити, пока не зустріну XLock
+type GetMetricResult struct {
-func (s *Metric) ProcessQueue(metricID uint32, tmp []byte) {
+	MetricType qb.MetricType
-	if len(s.waitQueue) == 0 {
+	FracDigits byte
-		return
+	ResultCode byte
+}
+
+type GetMetricReq struct {
+	MetricID uint32
+	ResultCh chan GetMetricResult
+}
+
+func (s *Metric) GetMetric(req GetMetricReq) {
+	req.ResultCh <- GetMetricResult{
+		ResultCode: Succeed,
+		MetricType: s.metricType,
+		FracDigits: s.fracDigits,
 	}
-	for _, untyped := range s.waitQueue {
+}
-		switch req := untyped.(type) {
+
-		case RangeScanReq:
+func (s *Metric) DeleteMetric(req DeleteMetricReq) {
-			s.StartRangeScan(req)
+	if s.xLock || s.rLocks > 0 || s.capturedState != nil {
-		case FullScanReq:
+		s.waitQueue = append(s.waitQueue, req)
-			s.StartFullScan(req)
+	} else {
-		case GetMetricReq:
+		s.xLock = true
-			s.GetMetric(req)
+		// fix - has pages -> do query from goroutine
-		case AppendMeasuresReq:
+		// else push to storage
-			metric.AppendMeasures(req, tmp, s.storageInbox)
+		// collect all pages, than
-		case DeleteMetricReq:
+		// s.storageInbox.Push(storage.MetricDelete{
-			s.DeleteMetric(req)
+		// 	MetricID:       req.MetricID,
-		case DeleteMeasuresReq:
+		// 	FreeIndexPages: nil,
-			metric.DeleteMeasures(req)
+		// 	FreeDataPages:  nil,
-		default:
+		// 	ResultCh:       req.ResultCh,
-			qb.Abort(qb.UnknownMetricWaitQueueItemBug,
+		// })
-				fmt.Errorf("bug: unknown metric wait queue item type %T", req))
-		}
 	}
 }
 
+type AppendMeasuresReq struct {
+	MetricID uint32
+	Measures []proto.Measure
+	ResultCh chan storage.MeasuresAppendResult
+}
+
 func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox *inbox.Inbox) {
 	if s.xLock || s.capturedState != nil {
 		s.waitQueue = append(s.waitQueue, req)
@@ -182,7 +199,9 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
 	s.values.CaptureState()
 
 	for idx, measure := range req.Measures {
+		//fmt.Println(idx, measure.Timestamp)
 		if measure.Timestamp <= s.timestamps.Until() {
+			//fmt.Printf("timestamp %d <= until %d\n", measure.Timestamp, s.timestamps.Until())
 			resultCode = ExpiredMeasure
 			break
 		}
@@ -191,6 +210,8 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
 			break
 		}
 
+		//fmt.Printf("  %d: %s => %.2f\n", idx, formatTime(measure.Timestamp), measure.Value)
+
 		tReport := timestamps.Evaluate(tmp[:7], measure.Timestamp)
 		//fmt.Printf("tReport: %#v\n", tReport)
 		vReport := values.Evaluate(tmp[7:], measure.Value)
@@ -198,7 +219,7 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
 
 		totalRequiredSpace := tReport.TotalSpace + vReport.TotalSpace
 
-		if totalRequiredSpace <= len(s.buffer) {
+		if totalRequiredSpace <= storage.DataPagePayloadSize {
 			// якщо на сторінці є місце
 			timestamps.Append(tReport.RewindOffset, tmp[:tReport.ChangeSize], measure.Timestamp)
 			values.Append(vReport.RewindOffset, tmp[7:7+vReport.ChangeSize], measure.Value, vReport.Delta)
@@ -210,7 +231,8 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
 				valuesRewindOffset = vReport.RewindOffset
 			}
 		} else {
-			fmt.Println("PAGE FILLED")
+			//fmt.Println("PAGE FILLED")
+			//fmt.Println(timestamps.Size() + values.Size())
 			// сторінка заповнена
 			since := s.timestamps.ReplaceSinceWithUntil()
 
@@ -227,6 +249,9 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
 				ValuesSize:     values.Size(),
 			})
 
+			//xxx := timestamps.Tail(0)
+			//fmt.Printf("PAGE timestamps %d:\n% x\n", len(xxx), xxx)
+
 			buf := make([]byte, storage.DataPageSize)
 			databuf := buf[:storage.DataPagePayloadSize]
 
@@ -257,10 +282,14 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
 		return
 	}
 
+	//xxx := timestamps.Tail(0)
+	//fmt.Printf("PAGE TAIL timestamps %d:\n% x\n", len(xxx), xxx)
+
 	// виділити змінені байти.
 	// скопіювати. Причому можна скопіювати зрізи chunks
 
 	if len(pages) > 0 {
+		//fmt.Printf("WITH GROW")
 		// пишу в storage довгим шляхом через redo файл і запис в data файл
 		storageInbox.Push(storage.MeasuresAppendWithGrow{
 			MetricID:               req.MetricID,
@@ -278,6 +307,7 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
 			ResultCh:               req.ResultCh,
 		})
 	} else {
+		//fmt.Printf("SIMPLE")
 		// короткий шлях - запис лише в storage
 		storageInbox.Push(storage.MeasuresAppend{
 			MetricID:               req.MetricID,
@@ -292,6 +322,12 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
 	}
 }
 
+type DeleteMeasuresReq struct {
+	MetricID uint32
+	Since    uint32
+	ResultCh chan byte
+}
+
 func (s *Metric) DeleteMeasures(req DeleteMeasuresReq) {
 	// if s.xLock || s.capturedState != nil {
 	// 	s.waitQueue = append(s.waitQueue, req)
@@ -314,7 +350,23 @@ func (s *Metric) DeleteMeasures(req DeleteMeasuresReq) {
 	// }
 }
 
-func (s *Metric) StartRangeScan(req RangeScanReq) {
+type RangeScanResult struct {
+	ResultCode byte
+	FracDigits byte
+	LastPageNo uint32
+	IsDataPage bool // for UntilNotFound only
+}
+
+type RangeScanReq struct {
+	MetricID       uint32
+	Since          uint32
+	Until          uint32
+	MetricType     qb.MetricType
+	ResponseWriter qb.WorkerMeasureConsumer
+	ResultCh       chan RangeScanResult
+}
+
+func (s *Metric) RangeScan(req RangeScanReq) {
 	if s.xLock {
 		s.waitQueue = append(s.waitQueue, req)
 		return
@@ -397,7 +449,23 @@ func (s *Metric) StartRangeScan(req RangeScanReq) {
 	}
 }
 
-func (s *Metric) StartFullScan(req FullScanReq) {
+type FullScanResult struct {
+	ResultCode byte
+	FracDigits byte
+	LastPageNo uint32
+}
+
+type FullScanReq struct {
+	MetricID       uint32
+	MetricType     qb.MetricType
+	ResponseWriter qb.WorkerMeasureConsumer
+	ResultCh       chan FullScanResult
+}
+
+func (s *Metric) FullScan(req FullScanReq) {
+	//fmt.Println("lastPageNo:", s.lastPageNo)
+	//fmt.Printf("index: %#v\n", s.indexLevelTails)
+
 	if s.xLock {
 		s.waitQueue = append(s.waitQueue, req)
 		return
@@ -410,18 +478,20 @@ func (s *Metric) StartFullScan(req FullScanReq) {
 	}
 	timestampDecompressor := s.timestamps.CreateDecompressor()
 	valueDecompressor := s.values.CreateDecompressor(s.metricType, s.fracDigits)
+	idx := 0
 	for {
 		timestamp, done := timestampDecompressor.NextValue()
 		if done {
+			//fmt.Println("DONE")
 			break
 		}
-		//fmt.Println("ts:", timestamp)
 		value, done := valueDecompressor.NextValue()
 		if done {
 			qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug)
 		}
-		//fmt.Println("value:", value)
 		req.ResponseWriter.FeedNoSend(timestamp, value)
+		//fmt.Printf("  %d: %s => %.2f\n", idx, formatTime(timestamp), value)
+		idx++
 	}
 
 	if s.lastPageNo > 0 {
@@ -441,6 +511,7 @@ func (s *Metric) StartFullScan(req FullScanReq) {
 // COMMITS
 
 func (s *Metric) OnMeasuresAppendCommited(rec storage.MeasuresAppendCommited) {
+	//fmt.Println("OnMeasuresAppendCommited: written=", rec.WrittenCount)
 	// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
 	s.capturedState = nil
 	s.timestamps.ForgetCapturedState()
@@ -453,6 +524,8 @@ func (s *Metric) OnMeasuresAppendCommited(rec storage.MeasuresAppendCommited) {
 }
 
 func (s *Metric) OnMeasuresAppendWithGrowCommited(rec storage.MeasuresAppendWithGrowCommited) {
+	//fmt.Println("OnMeasuresAppendWithGrowCommited: written=", rec.WrittenCount, ", lastPageNo=", rec.LastPageNo)
+	//fmt.Printf("index: %#v\n", rec.Index)
 	// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
 	s.capturedState = nil
 	s.timestamps.ForgetCapturedState()
@@ -480,4 +553,16 @@ func (s *Metric) OnMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) {
 	s.indexLevelTails = nil
 	s.lastPageNo = 0
 	s.lastValue = 0
+
+}
+
+// const (
+// 	free
+// )
+
+const datetimeLayout = "2006-01-02 15:04:05"
+
+func formatTime(timestamp uint32) string {
+	tm := time.Unix(int64(timestamp), 0)
+	return tm.Format(datetimeLayout)
 }

worker/snapshot.go

@@ -177,7 +177,7 @@ func ReadSnapshot(filePath string) (out ReadSnapshotOut, err error) {
 		if err != nil {
 			return
 		}
-		fmt.Println("metricID", metricID)
+		//fmt.Println("metricID", metricID)
 		metricType, err = bin.ReadByte(src)
 		if err != nil {
 			return
@@ -187,13 +187,13 @@ func ReadSnapshot(filePath string) (out ReadSnapshotOut, err error) {
 		if err != nil {
 			return
 		}
-		fmt.Println("m.MetricType", m.MetricType)
+		//fmt.Println("m.MetricType", m.MetricType)
-		fmt.Println("m.FracDigits", m.FracDigits)
+		//fmt.Println("m.FracDigits", m.FracDigits)
 		m.LastPageNo, err = bin.ReadUint32(src)
 		if err != nil {
 			return
 		}
-		fmt.Println("m.LastPageNo", m.LastPageNo)
+		//fmt.Println("m.LastPageNo", m.LastPageNo)
 		m.TimestampsSize, err = bin.ReadUint16AsInt(src)
 		if err != nil {
 			return
@@ -202,7 +202,7 @@ func ReadSnapshot(filePath string) (out ReadSnapshotOut, err error) {
 		if err != nil {
 			return
 		}
-		fmt.Println("m.TimestampsSize", m.TimestampsSize, "m.ValuesSize", m.ValuesSize)
+		//fmt.Println("m.TimestampsSize", m.TimestampsSize, "m.ValuesSize", m.ValuesSize)
 		err = bin.ReadNInto(src, buf[storage.DataPagePayloadSize-m.TimestampsSize:storage.DataPagePayloadSize])
 		if err != nil {
 			return
@@ -211,13 +211,13 @@ func ReadSnapshot(filePath string) (out ReadSnapshotOut, err error) {
 		if err != nil {
 			return
 		}
-		fmt.Printf("% x\n", buf)
+		//fmt.Printf("% x\n", buf)
 		//
 		levelsCount, err = bin.ReadVarSize(src)
 		if err != nil {
 			return
 		}
-		fmt.Println("levels count:", levelsCount)
+		//fmt.Println("levels count:", levelsCount)
 		for range levelsCount {
 			var (
 				buf   = make([]byte, storage.IndexPageSize)
@@ -227,12 +227,12 @@ func ReadSnapshot(filePath string) (out ReadSnapshotOut, err error) {
 			if err != nil {
 				return
 			}
-			fmt.Println("records count:", count)
+			//fmt.Println("records count:", count)
 			err = bin.ReadNInto(src, buf[:count*storage.IndexRecordSize])
 			if err != nil {
 				return
 			}
-			fmt.Printf("% x\n", buf)
+			//fmt.Printf("% x\n", buf)
 			m.IndexLevelTails = append(m.IndexLevelTails, storage.IndexLevelTail{
 				Buffer:       buf,
 				RecordsCount: count,
@@ -278,7 +278,7 @@ func writeFreePages(dst io.Writer, frozenPagesCount int, pageNumbers []uint32) (
 	if err != nil {
 		return
 	}
-	fmt.Println("write frozenPagesCount:", frozenPagesCount)
+	//fmt.Println("write frozenPagesCount:", frozenPagesCount)
 	_, err = bin.WriteVarSize(dst, len(pageNumbers))
 	if err != nil {
 		return
@@ -301,7 +301,7 @@ func readFreePages(src io.Reader) (_ int, _ []uint32, err error) {
 	if err != nil {
 		return
 	}
-	fmt.Println("read frozenPagesCount:", frozenPagesCount)
+	//fmt.Println("read frozenPagesCount:", frozenPagesCount)
 	pageNumbersCount, err := bin.ReadVarSize(src)
 	if err != nil {
 		return

worker/worker.go

@@ -7,7 +7,6 @@ import (
 	qb "gordenko.dev/dima/qb"
 	"gordenko.dev/dima/qb/enc"
 	"gordenko.dev/dima/qb/inbox"
-	"gordenko.dev/dima/qb/proto"
 	"gordenko.dev/dima/qb/storage"
 	"gordenko.dev/dima/qb/transform"
 )
@@ -81,6 +80,8 @@ func New(opt Options) *Worker {
 			values:          values,
 			indexLevelTails: x.IndexLevelTails,
 		}
+
+		//fmt.Printf("index: %#v\n", x.IndexLevelTails)
 	}
 	return s
 }
@@ -109,25 +110,25 @@ func (s *Worker) doWork() {
 	for _, untyped := range queue {
 		switch req := untyped.(type) {
 		case AppendMeasuresReq:
-			s.AppendMeasures(req)
+			s.appendMeasures(req)
 		case ReleaseRLock:
 			s.releaseRLock(req.MetricID)
 		case storage.Changes:
 			s.applyCommits(req) // all metrics only
 		case ListCurrentValuesReq:
-			s.ListCurrentValues(req) // all metrics only
+			s.listCurrentValues(req) // all metrics only
 		case RangeScanReq:
-			s.RangeScan(req)
+			s.rangeScan(req)
 		case FullScanReq:
-			s.FullScan(req)
+			s.fullScan(req)
 		case AddMetricReq:
-			s.AddMetric(req)
+			s.addMetric(req)
 		case DeleteMetricReq:
-			s.DeleteMetric(req)
+			s.deleteMetric(req)
 		case DeleteMeasuresReq:
-			s.DeleteMeasures(req)
+			s.deleteMeasures(req)
 		case GetMetricReq:
-			s.GetMetric(req)
+			s.getMetric(req)
 		default:
 			qb.Abort(qb.UnknownWorkerQueueItemBug,
 				fmt.Errorf("bug: unknown worker queue item type %T", req))
@@ -149,7 +150,7 @@ type AddMetricReq struct {
 	ResultCh   chan byte
 }
 
-func (s *Worker) AddMetric(req AddMetricReq) {
+func (s *Worker) addMetric(req AddMetricReq) {
 	_, ok := s.metrics[req.MetricID]
 	if ok {
 		req.ResultCh <- MetricDuplicate
@@ -175,30 +176,14 @@ func (s *Worker) AddMetric(req AddMetricReq) {
 	})
 }
 
-type GetMetricResult struct {
+func (s *Worker) getMetric(req GetMetricReq) {
-	MetricType qb.MetricType
-	FracDigits byte
-	ResultCode byte
-}
-
-type GetMetricReq struct {
-	MetricID uint32
-	ResultCh chan GetMetricResult
-}
-
-func (s *Worker) GetMetric(req GetMetricReq) {
 	metric, ok := s.metrics[req.MetricID]
-	if ok {
+	if !ok {
-		req.ResultCh <- GetMetricResult{
-			ResultCode: Succeed,
-			MetricType: metric.MetricType(),
-			FracDigits: metric.FracDigits(),
-		}
-	} else {
 		req.ResultCh <- GetMetricResult{
 			ResultCode: NoMetric,
 		}
 	}
+	metric.GetMetric(req)
 }
 
 type DeleteMetricReq struct {
@@ -206,32 +191,16 @@ type DeleteMetricReq struct {
 	ResultCh chan byte
 }
 
-func (s *Worker) DeleteMetric(req DeleteMetricReq) {
+func (s *Worker) deleteMetric(req DeleteMetricReq) {
 	metric, ok := s.metrics[req.MetricID]
 	if !ok {
 		req.ResultCh <- NoMetric
 		return
 	}
-	if metric.xLock {
+	metric.DeleteMetric(req)
-		metric.waitQueue = append(metric.waitQueue, req)
-	} else {
-		// collect all pages, than
-		s.storageInbox.Push(storage.MetricDelete{
-			MetricID:       req.MetricID,
-			FreeIndexPages: nil,
-			FreeDataPages:  nil,
-			ResultCh:       req.ResultCh,
-		})
-	}
 }
 
-type DeleteMeasuresReq struct {
+func (s *Worker) deleteMeasures(req DeleteMeasuresReq) {
-	MetricID uint32
-	Since    uint32
-	ResultCh chan byte
-}
-
-func (s *Worker) DeleteMeasures(req DeleteMeasuresReq) {
 	metric, ok := s.metrics[req.MetricID]
 	if !ok {
 		req.ResultCh <- NoMetric
@@ -240,13 +209,7 @@ func (s *Worker) DeleteMeasures(req DeleteMeasuresReq) {
 	metric.DeleteMeasures(req)
 }
 
-type AppendMeasuresReq struct {
+func (s *Worker) appendMeasures(req AppendMeasuresReq) {
-	MetricID uint32
-	Measures []proto.Measure
-	ResultCh chan storage.MeasuresAppendResult
-}
-
-func (s *Worker) AppendMeasures(req AppendMeasuresReq) {
 	metric, ok := s.metrics[req.MetricID]
 	if !ok {
 		req.ResultCh <- storage.MeasuresAppendResult{
@@ -261,23 +224,7 @@ func (s *Worker) AppendMeasures(req AppendMeasuresReq) {
 	}
 }
 
-type RangeScanResult struct {
+func (s *Worker) rangeScan(req RangeScanReq) {
-	ResultCode byte
-	FracDigits byte
-	LastPageNo uint32
-	IsDataPage bool // for UntilNotFound only
-}
-
-type RangeScanReq struct {
-	MetricID       uint32
-	Since          uint32
-	Until          uint32
-	MetricType     qb.MetricType
-	ResponseWriter qb.WorkerMeasureConsumer
-	ResultCh       chan RangeScanResult
-}
-
-func (s *Worker) RangeScan(req RangeScanReq) {
 	metric, ok := s.metrics[req.MetricID]
 	if !ok {
 		req.ResultCh <- RangeScanResult{
@@ -294,24 +241,11 @@ func (s *Worker) RangeScan(req RangeScanReq) {
 	if metric.xLock {
 		metric.waitQueue = append(metric.waitQueue, req)
 	} else {
-		metric.StartRangeScan(req)
+		metric.RangeScan(req)
 	}
 }
 
-type FullScanResult struct {
+func (s *Worker) fullScan(req FullScanReq) {
-	ResultCode byte
-	FracDigits byte
-	LastPageNo uint32
-}
-
-type FullScanReq struct {
-	MetricID       uint32
-	MetricType     qb.MetricType
-	ResponseWriter qb.WorkerMeasureConsumer
-	ResultCh       chan FullScanResult
-}
-
-func (s *Worker) FullScan(req FullScanReq) {
 	metric, ok := s.metrics[req.MetricID]
 	if !ok {
 		req.ResultCh <- FullScanResult{
@@ -328,7 +262,7 @@ func (s *Worker) FullScan(req FullScanReq) {
 	if metric.xLock {
 		metric.waitQueue = append(metric.waitQueue, req)
 	} else {
-		metric.StartFullScan(req)
+		metric.FullScan(req)
 	}
 }
 
@@ -338,7 +272,7 @@ type ListCurrentValuesReq struct {
 	ResultCh       chan struct{}
 }
 
-func (s *Worker) ListCurrentValues(req ListCurrentValuesReq) {
+func (s *Worker) listCurrentValues(req ListCurrentValuesReq) {
 	for _, metricID := range req.MetricIDs {
 		metric, ok := s.metrics[metricID]
 		if ok {
@@ -478,8 +412,31 @@ func (s *Worker) onMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) {
 	//s.doAfterReleaseXLock(rec.MetricID, metric)
 }
 
-// func (s *Database) doAfterReleaseXLock(metricID uint32, metric *_metric) {
+// rLock сумісний із append measures,
-// 	if len(metric.WaitQueue) > 0 {
+// xLock ні з чим
-// 		s.processMetricQueue(metricID, metric)
+// після xLock - вся черга
-// 	}
+// після rLock, якщо capturedState == nil - вся черга, оскільки rLock блокує лише xLock задачі
-// }
+// після capturedState = nil, якщо перша задача - appendMeasures - беру, інакше перевірка rLock
+
+// суть у тому що треба запускати запити, пока не зустріну XLock
+func (s *Worker) ProcessQueue(metric *Metric, tmp []byte, storageInbox *inbox.Inbox) {
+	// for _, untyped := range metric.waitQueue {
+	// 	switch req := untyped.(type) {
+	// 	case RangeScanReq:
+	// 		metric.RangeScan(req)
+	// 	case FullScanReq:
+	// 		metric.FullScan(req)
+	// 	case GetMetricReq:
+	// 		metric.GetMetric(req)
+	// 	case AppendMeasuresReq:
+	// 		metric.AppendMeasures(req, tmp, s.storageInbox)
+	// 	case DeleteMetricReq:
+	// 		metric.DeleteMetric(req)
+	// 	case DeleteMeasuresReq:
+	// 		metric.DeleteMeasures(req)
+	// 	default:
+	// 		qb.Abort(qb.UnknownMetricWaitQueueItemBug,
+	// 			fmt.Errorf("bug: unknown metric wait queue item type %T", req))
+	// 	}
+	// }
+}