tmp

txlog/writer.go

@@ -1,11 +1,13 @@
 package txlog
 
 import (
+	"bufio"
 	"bytes"
 	"errors"
 	"fmt"
-	"hash/crc32"
+	"io"
 	"log"
+	"math"
 	"os"
 	"path/filepath"
 	"sync"
@@ -13,6 +15,8 @@ import (
 	bin "gordenko.dev/dima/bin/little"
 	"gordenko.dev/dima/qb"
 	"gordenko.dev/dima/qb/atree"
+	"gordenko.dev/dima/qb/freelist"
+	"gordenko.dev/dima/qb/util"
 )
 
 const (
@@ -26,6 +30,8 @@ const (
 	filePerm = 0770
 
 	dumpSnapshotAfterNBytes = 1024 * 1024 * 1024 // 1 GB
+
+	writeBufferSize = 4 * 1024 * 1024
 )
 
 const (
@@ -46,31 +52,32 @@ func JoinChangesFileName(dir string, logNumber int) string {
 	return filepath.Join(dir, fmt.Sprintf("%d.changes", logNumber))
 }
 
+type MetricsState struct {
+	Metrics []Metric
+	WaitCh  chan struct{}
+}
+
 type Changes struct {
-	Records       []any
-	LogNumber     int
-	ForceSnapshot bool
-	ExitWaitGroup *sync.WaitGroup
-	WaitCh        chan struct{}
+	Records   []any
+	MetricsCh chan MetricsState
 }
 
 type Writer struct {
-	mutex     sync.Mutex
-	freelist  FreeList
-	atree     *atree.Atree
-	logNumber int
-	dir       string
-	w         *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
-	wal       *os.File
-	dataFile  *os.File
-	indexFile *os.File
-	dataPage  []byte
-	indexPage []byte
-	input     []any
-	//buf                 *bytes.Buffer
-	//pagesToWrite []atree.PageToWrite
-	workerReqs []any
-	//waitCh              chan struct{}
+	mutex           sync.Mutex
+	dataPagesCount  uint32
+	indexPagesCount uint32
+	dataFreeList    *freelist.FreeList
+	indexFreeList   *freelist.FreeList
+	atree           *atree.Atree
+	logNumber       int
+	dir             string
+	w               *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
+	wal             *os.File
+	dataFile        *os.File
+	indexFile       *os.File
+	// буфер для складання payload на сторінку, розрахунку CRC32 та інше
+	pageBuffer          []byte
+	input               []any
 	appendToWorkerQueue func(any)
 	//lsn                 uint32
 	written   int64
@@ -84,7 +91,8 @@ type WriterOptions struct {
 	Dir                 string
 	LogNumber           int // номер журнала
 	AppendToWorkerQueue func(any)
-	FreeList            FreeList
+	DataFreeList        *freelist.FreeList
+	IndexFreeList       *freelist.FreeList
 	Atree               *atree.Atree
 	ExitCh              chan struct{}
 	WaitGroup           *sync.WaitGroup
@@ -97,8 +105,11 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
 	if opt.AppendToWorkerQueue == nil {
 		return nil, errors.New("AppendToWorkerQueue option is required")
 	}
-	if opt.FreeList == nil {
-		return nil, errors.New("FreeList option is required")
+	if opt.DataFreeList == nil {
+		return nil, errors.New("DataFreeList option is required")
+	}
+	if opt.IndexFreeList == nil {
+		return nil, errors.New("IndexFreeList option is required")
 	}
 	if opt.Atree == nil {
 		return nil, errors.New("Atree option is required")
@@ -113,7 +124,8 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
 	s := &Writer{
 		dir:                 opt.Dir,
 		appendToWorkerQueue: opt.AppendToWorkerQueue,
-		freelist:            opt.FreeList,
+		dataFreeList:        opt.DataFreeList,
+		indexFreeList:       opt.IndexFreeList,
 		atree:               opt.Atree,
 		logNumber:           opt.LogNumber,
 		exitCh:              opt.ExitCh,
@@ -161,53 +173,140 @@ func (s *Writer) Run() {
 	}
 }
 
+func (s *Writer) getDataPageNumber() (uint32, bool, error) {
+	pageNo, err := s.dataFreeList.GetPageNumber()
+	if err != nil {
+		return 0, false, err
+	}
+	if pageNo > 0 {
+		return pageNo, true, nil
+	}
+	if s.dataPagesCount < math.MaxUint32 {
+		s.dataPagesCount++
+		return s.dataPagesCount, false, nil
+	}
+	return 0, false, errors.New("no space")
+}
+
+func (s *Writer) getIndexPageNumber() (uint32, bool, error) {
+	pageNo, err := s.indexFreeList.GetPageNumber()
+	if err != nil {
+		return 0, false, err
+	}
+	if pageNo > 0 {
+		return pageNo, true, nil
+	}
+	if s.indexPagesCount < math.MaxUint32 {
+		s.indexPagesCount++
+		return s.indexPagesCount, false, nil
+	}
+	return 0, false, errors.New("no space")
+}
+
 func (s *Writer) packAndWrite() (err error) {
 	s.mutex.Lock()
+	//isExited := s.isExited
 	input := s.input
 	s.input = nil
+
+	// var exitWaitGroup *sync.WaitGroup
+	// if s.isExited {
+	// 	exitWaitGroup = s.waitGroup
+	// }
 	s.mutex.Unlock()
 
-	w := bytes.NewBuffer(nil)
-
-	// 1. Пакую всі дані в  WAL буфер запису
-	for _, untyped := range input {
-		switch x := untyped.(type) {
-		case AddedMetric:
-			x.Pack(w)
-		// case DeletedMetric:
-		// 	if len(x.FreePageNumbers) > 0 {
-		// 		s.freeList.AddPageNumbers(x.FreePageNumbers)
-		// 	}
-		case AppendedMeasures:
-			//
-
-			s.packMeasuresIntoWALBuffer(x)
-			//case DeletedMeasures:
-			//case DeletedMeasuresSince:
-		}
-	}
-	// 2. Додаю розмір пакету і чексуму
+	prepared := prepareData(input)
 
 	// 3. Пишу на диск WAL (append)
-
-	// 4. Пишу в atree сторінки
-	for range 1 {
-		err = s.writePagesToAtree(nil, nil)
-		if err != nil {
-			return
-		}
+	n, err := s.wal.Write(prepared.Packet)
+	if err != nil {
+		return
+	}
+	if n != len(prepared.Packet) {
+		return fmt.Errorf("written %d != total size %d", n, len(prepared.Packet))
+	}
+	if err = s.wal.Sync(); err != nil {
+		return
 	}
 
-	// 5. Пишу зміни в free list
+	// 4. Пишу в atree сторінки
+	err = s.writePagesToAtree(prepared.WriteToIndex, prepared.WriteToData)
+	if err != nil {
+		return
+	}
 
 	// 6. відправляю input - worker-у
 
-	// flush
-	err = s.flush(w)
-	if err != nil {
-		return err
+	var (
+		forceSnapshot bool
+	)
+
+	if s.written > dumpSnapshotAfterNBytes {
+		forceSnapshot = true
 	}
 
+	// if isExited && s.written > 0 {
+	// 	forceSnapshot = true
+	// }
+
+	if forceSnapshot {
+		metricsStateCh := make(chan MetricsState)
+
+		s.appendToWorkerQueue(Changes{
+			Records:   prepared.ToWorker,
+			MetricsCh: metricsStateCh,
+		})
+
+		if err := s.wal.Close(); err != nil {
+			return fmt.Errorf("close changes file: %s", err)
+		}
+
+		state := <-metricsStateCh
+
+		s.logNumber++
+
+		// write snapshot FIX
+		err = writeSnapshot(Snapshot{
+			LogNumber:       s.logNumber,
+			Dir:             s.dir,
+			WriteBufferSize: writeBufferSize,
+			Metrics:         state.Metrics,
+			DataFreeList:    s.dataFreeList,
+			IndexFreeList:   s.indexFreeList,
+		})
+		if err != nil {
+			return fmt.Errorf("write snapshot file: %s", err)
+		}
+
+		var err error
+		s.wal, err = os.OpenFile(
+			JoinChangesFileName(s.dir, s.logNumber),
+			os.O_CREATE|os.O_WRONLY,
+			filePerm,
+		)
+		if err != nil {
+			return fmt.Errorf("create new changes file: %s", err)
+		}
+
+		s.written = 0
+		// release Worker
+		close(state.WaitCh)
+	} else {
+		s.appendToWorkerQueue(Changes{
+			Records: prepared.ToWorker,
+		})
+	}
+
+	// Якщо потрібен снапшот - відправляю канал із буфером розміру 1,
+	// в який воркер має покласти снапшот отриманий після застосування змін (records).
+	// Після чого Writer створює нові файли snapshot і changes і працює далі.
+
+	// flush
+	// err = s.flush(w)
+	// if err != nil {
+	// 	return err
+	// }
+
 	// FIX send to worker workerReqs
 	return nil
 }
@@ -215,7 +314,6 @@ func (s *Writer) packAndWrite() (err error) {
 func (s *Writer) Append(req any) {
 	s.mutex.Lock()
 	s.input = append(s.input, req)
-	s.workerReqs = append(s.workerReqs, req)
 	s.mutex.Unlock()
 
 	select {
@@ -237,87 +335,10 @@ func (s *Writer) Append(req any) {
 // 	s.waitCh = make(chan struct{})
 // }
 
-func (s *Writer) flush(w *bytes.Buffer) error {
-	s.mutex.Lock()
+// func (s *Writer) flush(w *bytes.Buffer) error {
 
-	//pagesToWrite := s.pagesToWrite
-	workerReqs := s.workerReqs
-	isExited := s.isExited
-
-	var exitWaitGroup *sync.WaitGroup
-	if s.isExited {
-		exitWaitGroup = s.waitGroup
-	}
-
-	if w.Len() > packetPrefixSize {
-		//s.lsn++
-		//lsn := s.lsn
-		packet := make([]byte, w.Len())
-		copy(packet, w.Bytes())
-		//s.reset() fix
-
-		s.written += int64(len(packet)) + 12
-		s.mutex.Unlock()
-
-		bin.PutUint32(packet[lengthIdx:], uint32(len(packet)-packetPrefixSize))
-		//bin.PutUint32(packet[lsnIdx:], lsn)
-		bin.PutUint32(packet[checksumIdx:], crc32.ChecksumIEEE(packet[8:]))
-
-		n, err := s.wal.Write(packet)
-		if err != nil {
-			return fmt.Errorf("TxLog write: %s", err)
-		}
-		if n != len(packet) {
-			return fmt.Errorf("TxLog written %d != packet size %d", n, len(packet))
-		}
-		if err := s.wal.Sync(); err != nil {
-			return fmt.Errorf("TxLog sync: %s", err)
-		}
-
-		// err = s.writePagesToAtree(pagesToWrite)
-		// if err != nil {
-		// 	return fmt.Errorf("TxLog writePagesToAtree: %s", err)
-		// }
-	} else {
-		s.mutex.Unlock()
-	}
-
-	var forceSnapshot bool
-
-	if s.written > dumpSnapshotAfterNBytes {
-		forceSnapshot = true
-	}
-
-	if isExited && s.written > 0 {
-		forceSnapshot = true
-	}
-
-	if forceSnapshot {
-		if err := s.wal.Close(); err != nil {
-			return fmt.Errorf("close changes file: %s", err)
-		}
-		s.logNumber++
-		var err error
-		s.wal, err = os.OpenFile(
-			JoinChangesFileName(s.dir, s.logNumber),
-			os.O_CREATE|os.O_WRONLY,
-			filePerm,
-		)
-		if err != nil {
-			return fmt.Errorf("create new changes file: %s", err)
-		}
-
-		s.written = 0
-	}
-
-	s.appendToWorkerQueue(Changes{
-		Records:       workerReqs,
-		ForceSnapshot: forceSnapshot,
-		LogNumber:     s.logNumber,
-		ExitWaitGroup: exitWaitGroup,
-	})
-	return nil
-}
+// 	return nil
+// }
 
 func (s *Writer) exit() {
 	s.mutex.Lock()
@@ -329,45 +350,79 @@ func (s *Writer) exit() {
 	}
 }
 
+type IndexPayloadToWrite struct {
+	PageNo    uint32
+	Payload   []byte
+	ZeroLevel bool
+}
+
+type DataPayloadToWrite struct {
+	PrevPageNo     uint32
+	Timestamps     [][]byte // chunks
+	TimestampsSize int
+	Values         [][]byte // chunks
+	ValuesSize     int
+	PageNo         uint32
+}
+
 // writePagesToAtree - записує сторінки в .data та .index файли
-func (s *Writer) writePagesToAtree(levels []*atree.IndexLevel, dataPages []*atree.DataPage) (err error) {
-	for _, p := range dataPages {
-		p.Checksum = atree.ChunksToDataPage(s.dataPage, atree.ChunksToDataPageReq{
+func (s *Writer) writePagesToAtree(indexItems []IndexPayloadToWrite, dataItems []DataPayloadToWrite) (err error) {
+	for _, p := range dataItems {
+		atree.ChunksToDataPage(s.pageBuffer, atree.ChunksToDataPageReq{
 			PrevPageNo:     p.PrevPageNo,
 			Timestamps:     p.Timestamps,
 			TimestampsSize: p.TimestampsSize,
 			Values:         p.Values,
 			ValuesSize:     p.ValuesSize,
 		})
-		off := (p.PageNo - 1) * atree.DataPageSize
-		n, err := s.dataFile.WriteAt(s.dataPage, int64(off))
+		var (
+			off = (p.PageNo - 1) * atree.DataPageSize
+			n   int
+		)
+		n, err = s.dataFile.WriteAt(s.pageBuffer, int64(off))
 		if err != nil {
-			return err
+			return
 		}
 		if n != atree.DataPageSize {
 			return fmt.Errorf("write %d instead of %d", n, atree.DataPageSize)
 		}
 	}
-	for levelIdx, level := range levels {
-		for _, p := range level.Filled {
-			// if len(p.Data) != atree.PageSize {
-			// 	return fmt.Errorf("wrong page %d size: %d",
-			// 		p.PageNo, len(p.Data))
-			// }
-			p.Checksum = atree.DataToIndexPage(s.indexPage, p.Data, levelIdx == 0)
-			off := (p.PageNo - 1) * atree.IndexPageSize
-			n, err := s.indexFile.WriteAt(s.indexPage, int64(off))
-			if err != nil {
-				return err
-			}
-			if n != atree.IndexPageSize {
-				return fmt.Errorf("write %d instead of %d", n, atree.IndexPageSize)
-			}
+	for _, p := range indexItems {
+		// if len(p.Payload) > atree.MaxIndexPayload {
+		// 	return fmt.Errorf("wrong index payload size: %d", len(p.Payload))
+		// }
+		indexPageBuffer := s.pageBuffer[:atree.IndexPageSize]
+		atree.DataToIndexPage(indexPageBuffer, p.Payload, p.ZeroLevel)
+		var (
+			off = (p.PageNo - 1) * atree.IndexPageSize
+			n   int
+		)
+		n, err = s.indexFile.WriteAt(indexPageBuffer, int64(off))
+		if err != nil {
+			return
+		}
+		if n != atree.IndexPageSize {
+			return fmt.Errorf("write %d instead of %d", n, atree.IndexPageSize)
 		}
 	}
 	return nil
 }
 
+type AppendMeasuresSummary struct {
+	MetricID uint32
+	//TimestampsOffset int // (заповнені одразу)
+	//ValuesOffset     int // (заповнені одразу)
+	// Timestamps     [][]byte
+	// TimestampsSize int
+	// Values         [][]byte
+	// ValuesSize     int
+	LastPageNo   uint32
+	Index        [][]byte
+	ResultCode   int
+	WrittenCount int
+	ResultCh     chan struct{}
+}
+
 type AppendedMeasures struct {
 	MetricID         uint32
 	TimestampsOffset int // (заповнені одразу)
@@ -378,6 +433,9 @@ type AppendedMeasures struct {
 	ValuesSize       int
 	IndexLevels      []*atree.IndexLevel
 	DataPages        []*atree.DataPage // fix - prevPageNo for the 1st data page
+	ResultCode       int
+	WrittenCount     int
+	ResultCh         chan struct{}
 }
 
 func (s *Writer) packMeasuresIntoWALBuffer(req AppendedMeasures) (err error) {
@@ -392,7 +450,7 @@ func (s *Writer) packMeasuresIntoWALBuffer(req AppendedMeasures) (err error) {
 		}
 		s.w.WriteByte(reused)
 		bin.WriteUint32(s.w, p.PrevPageNo)
-		bin.WriteUint32(s.w, p.Checksum)
+		//bin.WriteUint32(s.w, p.Checksum)
 		var (
 			timestampsOffset int
 			valuesOffset     int
@@ -430,7 +488,7 @@ func (s *Writer) packMeasuresIntoWALBuffer(req AppendedMeasures) (err error) {
 				reused = 1
 			}
 			s.w.WriteByte(reused)
-			bin.WriteUint32(s.w, p.Checksum)
+			//bin.WriteUint32(s.w, p.Checksum)
 			records := p.Data
 			if idx == 0 {
 				records = p.Data[level.Offset:]
@@ -502,3 +560,204 @@ func (s *Writer) sendSignal() {
 // }
 
 // helpers
+
+/*
+Формат:
+metricsQty - varuint
+[metric]*
+де metric - це:
+metricID - 4b
+metricType - 1b
+fracDigits - 1b
+lastPageNo - 4b
+since - 4b
+sinceValue - 8b
+until - 4b
+untilValue - 8b
+timestamps size - 2b
+values size - 2b
+timestams payload - Nb
+values payload - Nb
+data free list frozen pages - varsize
+dataFreeList size - varsize
+dataFreeList - Nb
+index free list frozen pages - varsize
+indexFreeList size - varsize
+indexFreeList - Nb
+CRC32 - 4b
+*/
+
+const metricHeaderSize = 38
+
+type Metric struct {
+	MetricID       uint32
+	MetricType     qb.MetricType
+	FracDigits     byte
+	LastPageNo     uint32
+	Since          uint32
+	SinceValue     float64
+	Until          uint32
+	UntilValue     float64
+	Timestamps     [][]byte
+	TimestampsSize int
+	Values         [][]byte
+	ValuesSize     int
+}
+
+type Snapshot struct {
+	LogNumber       int
+	Dir             string
+	WriteBufferSize int
+	Metrics         []Metric
+	IndexFreeList   *freelist.FreeList
+	DataFreeList    *freelist.FreeList
+}
+
+func writeSnapshot(snapshot Snapshot) (err error) {
+	var (
+		fileName = filepath.Join(snapshot.Dir, fmt.Sprintf("%d.snapshot", snapshot.LogNumber))
+		hasher   = util.NewHasher()
+		prefix   = make([]byte, metricHeaderSize)
+	)
+
+	file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY, 0770)
+	if err != nil {
+		return
+	}
+
+	dst := io.MultiWriter(bufio.NewWriterSize(file, snapshot.WriteBufferSize), hasher)
+
+	_, err = bin.WriteVarSize(dst, len(snapshot.Metrics))
+	if err != nil {
+		return
+	}
+
+	for _, metric := range snapshot.Metrics {
+		tSize := metric.TimestampsSize
+		vSize := metric.ValuesSize
+
+		bin.PutUint32(prefix[0:], metric.MetricID)
+		prefix[4] = byte(metric.MetricType)
+		prefix[5] = metric.FracDigits
+		bin.PutUint32(prefix[6:], metric.LastPageNo)
+		bin.PutUint32(prefix[10:], metric.Since)
+		bin.PutFloat64(prefix[14:], metric.SinceValue)
+		bin.PutUint32(prefix[22:], metric.Until)
+		bin.PutFloat64(prefix[26:], metric.UntilValue)
+		bin.PutUint16(prefix[34:], uint16(tSize))
+		bin.PutUint16(prefix[36:], uint16(vSize))
+
+		_, err = dst.Write(prefix)
+		if err != nil {
+			return
+		}
+		// copy timestamps
+		writeChunks(dst, metric.Timestamps, tSize)
+		// copy values
+		writeChunks(dst, metric.Values, vSize)
+	}
+	// free data pages
+	_, err = bin.WriteVarSize(dst, snapshot.DataFreeList.Pages())
+	if err != nil {
+		return
+	}
+	err = freeListWriteTo(snapshot.DataFreeList, dst)
+	if err != nil {
+		return
+	}
+	// free index pages
+	_, err = bin.WriteVarSize(dst, snapshot.IndexFreeList.Pages())
+	if err != nil {
+		return
+	}
+	err = freeListWriteTo(snapshot.IndexFreeList, dst)
+	if err != nil {
+		return
+	}
+
+	bin.WriteUint32(file, hasher.Sum32())
+
+	err = file.Sync()
+	if err != nil {
+		return
+	}
+
+	err = file.Close()
+	if err != nil {
+		return
+	}
+	// копіюю сторінки із delta файла в base файл і потім роблю Truncate
+	err = snapshot.DataFreeList.Merge() // fix - get frozen pages count + unfilled
+	if err != nil {
+		return
+	}
+	err = snapshot.IndexFreeList.Merge()
+	if err != nil {
+		return
+	}
+
+	// prevLogNumber := logNumber - 1
+	// prevChanges := filepath.Join(s.dir, fmt.Sprintf("%d.changes", prevLogNumber))
+	// prevSnapshot := filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", prevLogNumber))
+
+	// isExist, err := isFileExist(prevChanges)
+	// if err != nil {
+	// 	return
+	// }
+
+	// if isExist {
+	// 	err = os.Remove(prevChanges)
+	// 	if err != nil {
+	// 		qb.Abort(qb.DeletePrevChangesFileFailed, err)
+	// 	}
+	// }
+
+	// isExist, err = isFileExist(prevSnapshot)
+	// if err != nil {
+	// 	return
+	// }
+
+	// if isExist {
+	// 	err = os.Remove(prevSnapshot)
+	// 	if err != nil {
+	// 		qb.Abort(qb.DeletePrevSnapshotFileFailed, err)
+	// 	}
+	// }
+	return
+}
+
+// HELPERS
+
+func freeListWriteTo(freeList *freelist.FreeList, dst io.Writer) error {
+	serialized, err := freeList.Serialize()
+	if err != nil {
+		qb.Abort(qb.FailedFreeListSerialize, err)
+	}
+	_, err = bin.WriteVarSize(dst, len(serialized))
+	if err != nil {
+		return err
+	}
+	_, err = dst.Write(serialized)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func writeChunks(dst io.Writer, chunks [][]byte, size int) (err error) {
+	remaining := size
+	for _, buf := range chunks {
+		if remaining < len(buf) {
+			buf = buf[:remaining]
+		}
+		_, err = dst.Write(buf)
+		if err != nil {
+			return
+		}
+		remaining -= len(buf)
+		if remaining == 0 {
+			break
+		}
+	}
+	return
+}