wp

database/api.go

@@ -8,10 +8,10 @@ import (
 
 	bin "gordenko.dev/dima/bin/little"
 	"gordenko.dev/dima/qb"
-	"gordenko.dev/dima/qb/atree"
 	"gordenko.dev/dima/qb/bufreader"
 	"gordenko.dev/dima/qb/proto"
 	"gordenko.dev/dima/qb/storage"
+	"gordenko.dev/dima/qb/timeutil"
 	"gordenko.dev/dima/qb/transform"
 	"gordenko.dev/dima/qb/worker"
 )
@@ -35,7 +35,6 @@ func reply(conn io.Writer, errcode uint16) {
 		}
 		bin.PutUint16(answer[1:], errcode)
 	}
-
 	_, err := conn.Write(answer)
 	if err != nil {
 		return
@@ -61,7 +60,6 @@ func (s *Database) handleTCPConn(conn net.Conn) {
 }
 
 func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (err error) {
-	//fmt.Println("process request")
 	messageType, err := r.ReadByte()
 	if err != nil {
 		if err != io.EOF {
@@ -71,8 +69,6 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e
 		}
 	}
 
-	//fmt.Println("messageType:", messageType)
-
 	switch messageType {
 	case proto.TypeGetMetric:
 		req, err := proto.ReadGetMetricReq(r)
@@ -111,7 +107,6 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e
 		if err != nil {
 			return fmt.Errorf("proto.ReadAppendMeasuresReq: %s", err)
 		}
-		//fmt.Println("append measure", req.MetricID, conn.RemoteAddr().String())
 		if err = s.AppendMeasures(conn, req); err != nil {
 			return fmt.Errorf("AppendMeasures: %s", err)
 		}
@@ -162,7 +157,6 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e
 		}
 
 	case proto.TypeDeleteMeasures:
-		//fmt.Println("delete metric")
 		req, err := proto.ReadDeleteMeasuresReq(r)
 		if err != nil {
 			return fmt.Errorf("proto.ReadDeleteMeasuresReq: %s", err)
@@ -183,7 +177,6 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e
 		if err != nil {
 			return fmt.Errorf("proto.ReadListAllCumulativeMeasuresReq: %s", err)
 		}
-
 		if err = s.ListAllCumulativeMeasures(conn, req); err != nil {
 			return fmt.Errorf("ListAllCumulativeMeasures: %s", err)
 		}
@@ -198,8 +191,6 @@ func (s *Database) processRequest(conn net.Conn, r *bufreader.BufferedReader) (e
 // API
 
 func (s *Database) AddMetric(req proto.AddMetricReq) uint16 {
-	//fmt.Println("database.AddMetric")
-	// Валидация
 	if req.MetricID == 0 {
 		return proto.ErrEmptyMetricID
 	}
@@ -215,8 +206,6 @@ func (s *Database) AddMetric(req proto.AddMetricReq) uint16 {
 
 	resultCh := make(chan byte, 1)
 
-	fmt.Println("add job")
-
 	s.workerInbox.Push(worker.AddMetricReq{
 		MetricID:   req.MetricID,
 		MetricType: req.MetricType,
@@ -224,20 +213,14 @@ func (s *Database) AddMetric(req proto.AddMetricReq) uint16 {
 		ResultCh:   resultCh,
 	})
 
-	fmt.Println("job added")
-
 	resultCode := <-resultCh
 
 	switch resultCode {
 	case worker.Succeed:
-		fmt.Println("OK")
-
+		//
 	case worker.MetricDuplicate:
-		//fmt.Println("ErrDuplicate")
 		return proto.ErrDuplicate
-
 	default:
-		//fmt.Println("ErrWrongResultCodeBug")
 		qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
 	}
 	return 0
@@ -267,10 +250,8 @@ func (s *Database) GetMetric(conn io.Writer, req proto.GetMetricReq) error {
 		if err != nil {
 			return err
 		}
-
 	case worker.NoMetric:
 		reply(conn, proto.ErrNoMetric)
-
 	default:
 		qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
 	}
@@ -469,12 +450,11 @@ func (s *Database) ListCumulativeMeasures(conn net.Conn, req proto.ListCumulativ
 }
 
 func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriodsReq) error {
-	since, until := timeBoundsOfAggregation(req.Since, req.Until, req.GroupBy, req.FirstHourOfDay)
+	since, until := timeutil.TimeBoundsOfAggregation(req.Since, req.Until, req.GroupBy, req.FirstHourOfDay)
 	if since.After(until) {
 		reply(conn, proto.ErrInvalidRange)
 		return nil
 	}
-
 	responseWriter, err := transform.NewInstantPeriodsWriter(transform.InstantPeriodsWriterOptions{
 		Dst:            conn,
 		GroupBy:        req.GroupBy,
@@ -485,7 +465,6 @@ func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriod
 		reply(conn, proto.ErrUnexpected)
 		return nil
 	}
-
 	return s.rangeScan(rangeScanReq{
 		MetricID:       req.MetricID,
 		MetricType:     qb.Instant,
@@ -497,12 +476,11 @@ func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriod
 }
 
 func (s *Database) ListCumulativePeriods(conn net.Conn, req proto.ListCumulativePeriodsReq) error {
-	since, until := timeBoundsOfAggregation(req.Since, req.Until, req.GroupBy, req.FirstHourOfDay)
+	since, until := timeutil.TimeBoundsOfAggregation(req.Since, req.Until, req.GroupBy, req.FirstHourOfDay)
 	if since.After(until) {
 		reply(conn, proto.ErrInvalidRange)
 		return nil
 	}
-
 	responseWriter, err := transform.NewCumulativePeriodsWriter(transform.CumulativePeriodsWriterOptions{
 		Dst:            conn,
 		GroupBy:        req.GroupBy,
@@ -512,7 +490,6 @@ func (s *Database) ListCumulativePeriods(conn net.Conn, req proto.ListCumulative
 		reply(conn, proto.ErrUnexpected)
 		return nil
 	}
-
 	return s.rangeScan(rangeScanReq{
 		MetricID:       req.MetricID,
 		MetricType:     qb.Cumulative,
@@ -529,7 +506,7 @@ type rangeScanReq struct {
 	Since          uint32
 	Until          uint32
 	Conn           io.Writer
-	ResponseWriter atree.PeriodsWriter
+	ResponseWriter qb.PeriodsWriter
 }
 
 func (s *Database) rangeScan(req rangeScanReq) error {
@@ -549,44 +526,40 @@ func (s *Database) rangeScan(req rangeScanReq) error {
 	switch result.ResultCode {
 	case worker.QueryDone:
 		req.ResponseWriter.Close()
-
 	case worker.UntilFound:
-		// err := s.atree.ContinueRangeScan(atree.ContinueRangeScanReq{
-		// 	FracDigits:     result.FracDigits,
-		// 	ResponseWriter: req.ResponseWriter,
-		// 	LastPageNo:     result.LastPageNo,
-		// 	Since:          req.Since,
-		// })
-		//s.metricRUnlock(req.MetricID)
-
-		// if err != nil {
-		// 	reply(req.Conn, proto.ErrUnexpected)
-		// } else {
-		// 	req.ResponseWriter.Close()
-		// }
-
+		err := s.ContinueRangeScan(ContinueRangeScanReq{
+			MetricType:     req.MetricType,
+			FracDigits:     result.FracDigits,
+			ResponseWriter: req.ResponseWriter,
+			LastPageNo:     result.LastPageNo,
+			Since:          req.Since,
+		})
+		//s.metricRUnlock(req.MetricID) fix release unlock
+		if err != nil {
+			reply(req.Conn, proto.ErrUnexpected)
+		} else {
+			req.ResponseWriter.Close()
+		}
 	case worker.UntilNotFound:
-		// err := s.atree.RangeScan(atree.RangeScanReq{
-		// 	FracDigits:     result.FracDigits,
-		// 	ResponseWriter: req.ResponseWriter,
-		// 	RootPageNo:     result.RootPageNo,
-		// 	Since:          req.Since,
-		// 	Until:          req.Until,
-		// })
-		//s.metricRUnlock(req.MetricID)
-
-		// if err != nil {
-		// 	reply(req.Conn, proto.ErrUnexpected)
-		// } else {
-		// 	req.ResponseWriter.Close()
-		// }
-
+		err := s.RangeScan(RangeScanReq{
+			MetricType:     req.MetricType,
+			FracDigits:     result.FracDigits,
+			ResponseWriter: req.ResponseWriter,
+			Since:          req.Since,
+			Until:          req.Until,
+			LastPageNo:     result.LastPageNo,
+			IsDataPage:     result.IsDataPage,
+		})
+		//s.metricRUnlock(req.MetricID) // fix release
+		if err != nil {
+			reply(req.Conn, proto.ErrUnexpected)
+		} else {
+			req.ResponseWriter.Close()
+		}
 	case worker.NoMetric:
 		reply(req.Conn, proto.ErrNoMetric)
-
 	case worker.WrongMetricType:
 		reply(req.Conn, proto.ErrWrongMetricType)
-
 	default:
 		qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
 	}
@@ -597,7 +570,7 @@ type fullScanReq struct {
 	MetricID       uint32
 	MetricType     qb.MetricType
 	Conn           io.Writer
-	ResponseWriter atree.PeriodsWriter
+	ResponseWriter qb.PeriodsWriter
 }
 
 func (s *Database) fullScan(req fullScanReq) error {
@@ -614,11 +587,10 @@ func (s *Database) fullScan(req fullScanReq) error {
 
 	switch result.ResultCode {
 	case worker.QueryDone:
-		fmt.Printf("query done")
 		req.ResponseWriter.Close()
-
 	case worker.UntilFound:
-		err := s.atree.ContinueFullScan(atree.ContinueFullScanReq{
+		err := s.ContinueFullScan(ContinueFullScanReq{
+			MetricType:     req.MetricType,
 			FracDigits:     result.FracDigits,
 			ResponseWriter: req.ResponseWriter,
 			LastPageNo:     result.LastPageNo,
@@ -629,13 +601,10 @@ func (s *Database) fullScan(req fullScanReq) error {
 		} else {
 			req.ResponseWriter.Close()
 		}
-
 	case worker.NoMetric:
 		reply(req.Conn, proto.ErrNoMetric)
-
 	case worker.WrongMetricType:
 		reply(req.Conn, proto.ErrWrongMetricType)
-
 	default:
 		qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
 	}

database/database.go

@@ -10,8 +10,8 @@ import (
 	"time"
 
 	"gordenko.dev/dima/qb"
-	"gordenko.dev/dima/qb/atree"
 	"gordenko.dev/dima/qb/inbox"
+	"gordenko.dev/dima/qb/pagecache"
 	"gordenko.dev/dima/qb/recovery"
 	"gordenko.dev/dima/qb/storage"
 	"gordenko.dev/dima/qb/worker"
@@ -24,7 +24,8 @@ type Database struct {
 	workerInbox  *inbox.Inbox
 	worker       *worker.Worker
 	storage      *storage.Writer
-	atree        *atree.Atree
+	indexCache   *pagecache.PageCache
+	dataCache    *pagecache.PageCache
 	tcpPort      int
 	logfile      *os.File
 	logger       *log.Logger
@@ -100,6 +101,24 @@ func New(opt Options) (_ *Database, err error) {
 		return nil, err
 	}
 
+	s.indexCache, err = pagecache.New(pagecache.Options{
+		File:          indexFile,
+		PageSize:      storage.IndexPageSize,
+		VerifyPageCRC: storage.VerifyIndexPageCRC32,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("index pagecache.New: %s", err)
+	}
+
+	s.dataCache, err = pagecache.New(pagecache.Options{
+		File:          dataFile,
+		PageSize:      storage.DataPageSize,
+		VerifyPageCRC: storage.VerifyDataPageCRC32,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("data pagecache.New: %s", err)
+	}
+
 	s.storage, err = storage.NewWriter(storage.WriterOptions{
 		Inbox:         storageInbox,
 		WorkerInbox:   s.workerInbox,
@@ -126,14 +145,6 @@ func New(opt Options) (_ *Database, err error) {
 		ExitCh:        opt.ExitCh,
 		WaitGroup:     opt.WaitGroup,
 	})
-
-	s.atree, err = atree.New(atree.Options{
-		DataFile:  dataFile,
-		IndexFile: indexFile,
-	})
-	if err != nil {
-		return nil, fmt.Errorf("atree.New: %s", err)
-	}
 	return s, nil
 }
 
@@ -146,14 +157,11 @@ func (s *Database) ListenAndServe() (err error) {
 	//s.waitGroup.Add(1)
 	go s.storage.Run()
 
-	// s.atree.Run()
-
 	s.waitGroup.Add(1)
 	go s.worker.Run()
 
 	s.logger.Println("database started")
 	for {
-		// Listen for an incoming connection.
 		conn, err := listener.Accept()
 		if err != nil {
 			s.logger.Printf("listener.Accept: %s\n", err)
@@ -164,53 +172,291 @@ func (s *Database) ListenAndServe() (err error) {
 	}
 }
 
-//
+type ContinueFullScanReq struct {
+	MetricType     qb.MetricType
+	FracDigits     byte
+	ResponseWriter qb.AtreeMeasureConsumer
+	LastPageNo     uint32
+}
 
-// зробити object?
+func (s *Database) ContinueFullScan(req ContinueFullScanReq) error {
+	buf, err := s.dataCache.FetchPage(req.LastPageNo)
+	if err != nil {
+		return fmt.Errorf("dataCache.FetchPage(%d): %s", req.LastPageNo, err)
+	}
+	treeCursor, err := storage.NewBackwardCursor(storage.BackwardCursorOptions{
+		MetricType:    req.MetricType,
+		FracDigits:    req.FracDigits,
+		PageNo:        req.LastPageNo,
+		PageData:      buf,
+		FetchDataPage: s.dataCache.FetchPage,
+		ReleasePage:   s.dataCache.ReleasePage,
+	})
+	if err != nil {
+		return err
+	}
+	defer treeCursor.Close()
+	for {
+		timestamp, value, done, err := treeCursor.Prev()
+		if err != nil {
+			return err
+		}
+		if done {
+			return nil
+		}
+		req.ResponseWriter.Feed(timestamp, value)
+	}
+}
 
-// func (s *Database) replayChanges(snapshotNumber int) (err error) {
-// 	snapshot, err := readSnapshot(JoinSnapshotFileName(s.dir, snapshotNumber))
-// 	if err != nil {
-// 		return
-// 	}
+type ContinueRangeScanReq struct {
+	MetricType     qb.MetricType
+	FracDigits     byte
+	ResponseWriter qb.AtreeMeasureConsumer
+	LastPageNo     uint32
+	Since          uint32
+}
 
-// 	return nil
+func (s *Database) ContinueRangeScan(req ContinueRangeScanReq) error {
+	buf, err := s.dataCache.FetchPage(req.LastPageNo)
+	if err != nil {
+		return fmt.Errorf("dataCache.FetchPage(%d): %s", req.LastPageNo, err)
+	}
+	treeCursor, err := storage.NewBackwardCursor(storage.BackwardCursorOptions{
+		MetricType:    req.MetricType,
+		FracDigits:    req.FracDigits,
+		PageNo:        req.LastPageNo,
+		PageData:      buf,
+		FetchDataPage: s.dataCache.FetchPage,
+		ReleasePage:   s.dataCache.ReleasePage,
+	})
+	if err != nil {
+		return err
+	}
+	defer treeCursor.Close()
+	for {
+		timestamp, value, done, err := treeCursor.Prev()
+		if err != nil {
+			return err
+		}
+		if done {
+			return nil
+		}
+		req.ResponseWriter.Feed(timestamp, value)
+		if timestamp < req.Since {
+			return nil
+		}
+	}
+}
+
+type RangeScanReq struct {
+	MetricType     qb.MetricType
+	FracDigits     byte
+	ResponseWriter qb.AtreeMeasureConsumer
+	Since          uint32
+	Until          uint32
+	LastPageNo     uint32
+	IsDataPage     bool
+}
+
+func (s *Database) RangeScan(req RangeScanReq) error {
+	var (
+		pageNo uint32
+		buf    []byte
+		err    error
+	)
+	if req.IsDataPage {
+		pageNo = req.LastPageNo
+		buf, err = s.dataCache.FetchPage(pageNo)
+		if err != nil {
+			return fmt.Errorf("dataCache.FetchPage(%d): %s", pageNo, err)
+		}
+	} else {
+		pageNo, buf, err = s.findDataPage(req.LastPageNo, req.Until)
+		if err != nil {
+			return err
+		}
+	}
+	cursor, err := storage.NewBackwardCursor(storage.BackwardCursorOptions{
+		MetricType:    req.MetricType,
+		FracDigits:    req.FracDigits,
+		PageNo:        pageNo,
+		PageData:      buf,
+		FetchDataPage: s.dataCache.FetchPage,
+		ReleasePage:   s.dataCache.ReleasePage,
+	})
+	if err != nil {
+		return err
+	}
+	defer cursor.Close()
+	for {
+		timestamp, value, done, err := cursor.Prev()
+		if err != nil {
+			return err
+		}
+		if done {
+			return nil
+		}
+		if timestamp <= req.Until {
+			req.ResponseWriter.Feed(timestamp, value)
+			if timestamp < req.Since {
+				// - записи, удовлетворяющие временным рамкам, закончились.
+				return nil
+			}
+		}
+	}
+}
+
+// (records payload only, isZeroLevel, timestamp)
+func (s *Database) findDataPage(foundPageNo uint32, timestamp uint32) (uint32, []byte, error) {
+	for {
+		buf, err := s.indexCache.FetchPage(foundPageNo)
+		if err != nil {
+			return 0, nil, fmt.Errorf("fetchIndexPage(%d): %s", foundPageNo, err)
+		}
+		toReleaseIndexPageNo := foundPageNo
+		foundPageNo = storage.FindPageOnIndexPage(buf, timestamp)
+		s.indexCache.ReleasePage(toReleaseIndexPageNo)
+		if storage.IsZeroLevelPage(buf) {
+			buf, err := s.dataCache.FetchPage(foundPageNo)
+			if err != nil {
+				return 0, nil, fmt.Errorf("fetchDataPage(%d): %s", foundPageNo, err)
+			}
+			return foundPageNo, buf, nil
+		}
+	}
+}
+
+// type PathLeg struct {
+// 	PageNo uint32
+// 	Data   []byte
 // }
 
-//func (s *Database) verifySnapshot(fileName string) (_ bool, err error) {
-// 	file, err := os.Open(fileName)
-// 	if err != nil {
-// 		return
-// 	}
-// 	defer file.Close()
-
-// 	stat, err := file.Stat()
-// 	if err != nil {
-// 		return
-// 	}
-
-// 	if stat.Size() <= 4 {
-// 		return false, nil
-// 	}
-
-// 	var (
-// 		payloadSize = stat.Size() - 4
-// 		hash        = crc32.NewIEEE()
-// 	)
-
-// 	_, err = io.CopyN(hash, file, payloadSize)
-// 	if err != nil {
-// 		return
-// 	}
-// 	calculatedCRC := hash.Sum32()
-
-// 	storedCRC, err := bin.ReadUint32(file)
-// 	if err != nil {
-// 		return
-// 	}
-// 	if storedCRC != calculatedCRC {
-// 		return false, fmt.Errorf("strored CRC %d not equal calculated CRC %d",
-// 			storedCRC, calculatedCRC)
-// 	}
-// 	return true, nil
+// type PathToDataPage struct {
+// 	Legs       []PathLeg
+// 	LastPageNo uint32
+// }
+
+// func (s *Database) FindPathToLastPage(rootPageNo uint32) (_ PathToDataPage, err error) {
+// 	// var (
+// 	// 	pageNo = rootPageNo
+// 	// 	legs   []PathLeg
+// 	// )
+
+// 	// for {
+// 	// 	var buf []byte
+// 	// 	buf, err = s.fetchIndexPage(pageNo)
+// 	// 	if err != nil {
+// 	// 		err = fmt.Errorf("FetchIndexPage(%d): %s", pageNo, err)
+// 	// 		return
+// 	// 	}
+
+// 	// 	legs = append(legs, PathLeg{
+// 	// 		PageNo: pageNo,
+// 	// 		Data:   buf,
+// 	// 		// childIdx не нужен
+// 	// 	})
+
+// 	// 	foundPageNo := getLastPageNo(buf)
+
+// 	// 	// fix
+// 	// 	if buf[isLastLevelIdx] == 1 {
+// 	// 		return PathToDataPage{
+// 	// 			Legs:       legs,
+// 	// 			LastPageNo: foundPageNo,
+// 	// 		}, nil
+// 	// 	}
+// 	// 	// вглубь
+// 	// 	pageNo = foundPageNo
+// 	// }
+// 	return
+// }
+
+// DELETE
+
+// func (s *Atree) DeletePages(pageNumbers []uint32) {
+// 	s.mutex.Lock()
+// 	for _, pageNo := range pageNumbers {
+// 		delete(s.pages, pageNo)
+// 	}
+// 	s.mutex.Unlock()
+// }
+
+// type Level struct {
+// 	PageNo   uint32
+// 	PageData []byte
+// 	Idx      int
+// 	ChildQty int
+// }
+
+// func (s *Atree) GetAllPages(rootPageNo uint32) (_ []uint32, err error) {
+// 	// var (
+// 	// 	pageNumbers []uint32
+// 	// 	levels      []*Level
+// 	// )
+
+// 	// buf, err := s.fetchIndexPage(rootPageNo)
+// 	// if err != nil {
+// 	// 	return nil, fmt.Errorf("fetchIndexPage(%d): %s", rootPageNo, err)
+// 	// }
+// 	// pageNumbers = append(pageNumbers, rootPageNo)
+
+// 	// // if buf[isDataPageNumbersIdx] == 1 {
+// 	// // 	pageNumbers := listPageNumbers(buf)
+// 	// // 	dataPages = append(dataPages, pageNumbers...)
+
+// 	// // 	s.releasePage(rootPageNo)
+
+// 	// // 	return PageLists{
+// 	// // 		DataPages:  dataPages,
+// 	// // 		IndexPages: indexPages,
+// 	// // 	}, nil
+// 	// // }
+
+// 	// childQty, _ := bin.GetUint16(buf[indexRecordsQtyIdx:])
+
+// 	// levels = append(levels, &Level{
+// 	// 	PageNo:   rootPageNo,
+// 	// 	PageData: buf,
+// 	// 	Idx:      0,
+// 	// 	ChildQty: int(childQty),
+// 	// })
+
+// 	// for {
+// 	// 	if len(levels) == 0 {
+// 	// 		return pageNumbers, nil
+// 	// 	}
+
+// 	// 	lastIdx := len(levels) - 1
+// 	// 	level := levels[lastIdx]
+
+// 	// 	if level.Idx < level.ChildQty {
+// 	// 		pageNo := getPageNo(level.PageData, level.Idx)
+// 	// 		level.Idx++
+
+// 	// 		var buf []byte
+// 	// 		buf, err = s.fetchPage(pageNo)
+// 	// 		if err != nil {
+// 	// 			return nil, fmt.Errorf("fetchPage(%d): %s", pageNo, err)
+// 	// 		}
+// 	// 		pageNumbers = append(pageNumbers, pageNo)
+
+// 	// 		if buf[pageTypeIdx] == PageTypeData {
+// 	// 			//pageNumbers := listPageNumbers(buf)
+// 	// 			//dataPages = append(dataPages, pageNumbers...)
+// 	// 			s.releasePage(pageNo)
+// 	// 		} else {
+// 	// 			childQty, _ = bin.GetUint16(buf[indexRecordsQtyIdx:])
+// 	// 			levels = append(levels, &Level{
+// 	// 				PageNo:   pageNo,
+// 	// 				PageData: buf,
+// 	// 				Idx:      0,
+// 	// 				ChildQty: int(childQty),
+// 	// 			})
+// 	// 		}
+// 	// 	} else {
+// 	// 		s.releasePage(level.PageNo)
+// 	// 		levels = levels[:lastIdx]
+// 	// 	}
+// 	// }
+// 	return
 // }

database/database_test.go

@@ -1,24 +0,0 @@
-package database
-
-// func TestComposeHeadIndexPage(t *testing.T) {
-// 	var (
-// 		levelIdx = 0
-// 		level    = storage.IndexLevelTail{
-// 			Buffer: []byte{
-// 				1, 2, 3,
-// 			},
-// 			RecordsCount: 2,
-// 		}
-// 		head = &storage.IndexPageTail{
-// 			PageNo:  100,
-// 			CRC32:   12345,
-// 			Records: []byte{},
-// 		}
-// 	)
-// 	page := composeHeadIndexPage(levelIdx, level, head)
-// 	if page.PageNo != head.PageNo {
-// 		t.Fatalf("PageNo: got %d are not equal expected %v",
-// 			page.PageNo, head.PageNo)
-// 	}
-// 	// fix compare pages
-// }

database/helpers.go

@@ -1,52 +0,0 @@
-package database
-
-import (
-	"errors"
-	"io/fs"
-	"os"
-	"time"
-
-	"gordenko.dev/dima/qb"
-	"gordenko.dev/dima/qb/proto"
-)
-
-func timeBoundsOfAggregation(since, until proto.TimeBound, groupBy qb.GroupBy, firstHourOfDay int) (s time.Time, u time.Time) {
-	switch groupBy {
-	case qb.GroupByHour, qb.GroupByDay:
-		s = time.Date(since.Year, since.Month, since.Day, 0, 0, 0, 0, time.Local)
-		u = time.Date(until.Year, until.Month, until.Day, 0, 0, 0, 0, time.Local)
-
-	case qb.GroupByMonth:
-		s = time.Date(since.Year, since.Month, 1, 0, 0, 0, 0, time.Local)
-		u = time.Date(until.Year, until.Month, 1, 0, 0, 0, 0, time.Local)
-	}
-
-	if firstHourOfDay > 0 {
-		duration := time.Duration(firstHourOfDay) * time.Hour
-		s = s.Add(duration)
-		u = u.Add(duration)
-	}
-
-	u = u.Add(-1 * time.Second)
-	return
-}
-
-func isFileExist(fileName string) (bool, error) {
-	_, err := os.Stat(fileName)
-	if err != nil {
-		if errors.Is(err, fs.ErrNotExist) {
-			return false, nil
-		} else {
-			return false, err
-		}
-	} else {
-		return true, nil
-	}
-}
-
-func correctToFHD(since, until uint32, firstHourOfDay int) (uint32, uint32) {
-	duration := time.Duration(firstHourOfDay) * time.Hour
-	since = uint32(time.Unix(int64(since), 0).Add(duration).Unix())
-	until = uint32(time.Unix(int64(until), 0).Add(duration).Unix())
-	return since, until
-}

database/tmp

@@ -1,93 +0,0 @@
-// IN APPEND MEASURES
-// if len(s.buffer) < storage.DataPageSize {
-// 	// allocate bigger buffer
-// 	buf, databuf := growBuffers(growBuffersIn{
-// 		databuf:       s.buffer,
-// 		tSize:         timestamps.Size(),
-// 		vSize:         values.Size(),
-// 		requiredSpace: totalRequiredSpace,
-// 	})
-
-// 	s.buffer = buf
-// 	// replace buffer in timestamps and values
-// 	timestamps.Rotate(databuf) // fix pos
-// 	values.Rotate(databuf)     // fix pos
-// } else {
-
-// IN DATABASE
-
-// FIX
-// УВАГА!
-// Якщо буфер metric.buffer < розміру сторінки - для timestamps доступний весь буфер.
-// Якщо буфер досяг розміру сторінки - в timestamps я передаю buffer[:DataPagePayloadSize]
-
-// var dataBufferSizes = []int{
-// 	1024,
-// 	2048,
-// 	4096,
-// 	8192,
-// }
-
-// func calculateDataBufferSize(payloadSize int) int {
-// 	for _, bufferSize := range dataBufferSizes {
-// 		if payloadSize <= bufferSize {
-// 			return bufferSize
-// 		}
-// 	}
-// 	return storage.DataPageSize
-// }
-
-// func allocateBuffers(requiredSpace int) (buf, databuf []byte) {
-// 	bufferSize := calculateDataBufferSize(requiredSpace)
-// 	buf = make([]byte, bufferSize)
-// 	if bufferSize == storage.DataPageSize {
-// 		databuf = buf[:storage.DataPagePayloadSize]
-// 	} else {
-// 		databuf = buf
-// 	}
-// 	return
-// }
-
-// // src - databuf
-// type growBuffersIn struct {
-// 	databuf       []byte
-// 	tSize         int
-// 	vSize         int
-// 	requiredSpace int
-// }
-
-// func growBuffers(in growBuffersIn) (buf, databuf []byte) {
-// 	buf, databuf = allocateBuffers(in.requiredSpace)
-// 	copy(databuf[len(databuf)-in.tSize:], in.databuf[len(in.databuf)-in.tSize:])
-// 	copy(databuf, in.databuf[:in.vSize])
-// 	return
-// }
-
-// type shrinkBufferIn struct {
-// 	Buffer         []byte
-// 	TimestampsSize int
-// 	ValuesSize     int
-// 	NewBufferSize  int
-// }
-
-// func shrinkBuffer(in shrinkBufferIn) []byte {
-// 	buf := make([]byte, in.NewBufferSize)
-// 	copy(buf, in.Buffer[:in.ValuesSize])
-// 	copy(buf[:len(buf)-in.TimestampsSize], in.Buffer[len(in.Buffer)-in.TimestampsSize:])
-// 	return buf
-// }
-
-// IN REPLAY_METRIC.TO_METRIC
-
-// var (
-// 	requiredSpace = s.tSize + s.vSize
-// )
-// bufferSize := calculateDataBufferSize(requiredSpace)
-// if len(s.buf) > bufferSize {
-// 	s.buf = shrinkBuffer(shrinkBufferIn{
-// 		Buffer:         s.buf,
-// 		TimestampsSize: s.tSize,
-// 		ValuesSize:     s.vSize,
-// 		NewBufferSize:  bufferSize,
-// 	})
-// }