qb/storage/wal_records.go

package storage

import (
	"bytes"
	"io"

	bin "gordenko.dev/dima/bin/little"
	"gordenko.dev/dima/qb"
)

type HeadDataPage struct {
	PageNo           uint32
	Reused           bool
	PrevPageNo       uint32
	Checksum         uint32
	TimestampsOffset int // offset on 1st page
	Timestamps       []byte
	ValuesOffset     int // offset on 1st page
	Values           []byte
}

type HeadIndexPage struct {
	PageNo   uint32
	Reused   bool
	Checksum uint32
	Records  []byte // offset не потрібен, оскільки Records додаються в кінець
}

type ChangedIndexLevel struct {
	HeadIndexPage *HeadIndexPage
	IndexPages    []SealedIndexPage
	TailRecords   []byte // payload only
}

type MeasuresAppendWithGrowRecord struct {
	MetricID           uint32
	HeadDataPage       HeadDataPage
	DataPages          []SealedDataPage
	ChangedIndexLevels []ChangedIndexLevel
	TailTimestamps     []byte // data level tail
	TailValues         []byte // data level tail
}

/*
Format appended measures:
1b - tx type
4b - metricID
4b - pageNo
1b - reused
4b - prevPageNo
4b - page crc32
Nb - (varsize) timestamps offset
Nb - (varsize) timestamps tail size on 1st filled data  page
Nb - timestamps tail on 1st filled data page
Nb - (varsize) values offset
Nb - (varsize) values tail size on 1st filled data page
Nb - values tail on 1st filled data page

Nb - (varsize) filled data pages qty
[

	4b - pageNo
	1b - reused
	Nb - (data page size) page content

]
Nb - (varsize) timestamps payload size on tail
Nb - timestamps payload
Nb - (varsize) values payload size on tail
Nb - values payload
Nb - (varsize) qty of index levels
[

	    // NOTE!
		// - idx = 0 - zeroLevel
		// - skipped = maxRecordsOnIndexPage - records count
		4b - pageNo of 1st index page
		1b - reused
		4b - page crc32
		Nb - (varsize) records tail count on 1st filled index page
		Nb - records tail on 1st filled index page


		Nb - (varsize) qty of level filled pages
		[
			4b - pageNo
			1b - reused
			Nb - (index page size) page content
		]
		Nb - (varsize) size of records level tail
		Nb - level tail records

]
*/
func (s MeasuresAppendWithGrowRecord) WriteTo(w *bytes.Buffer) {
	w.WriteByte(CodeMeasuresAppendWithGrow)
	bin.WriteUint32(w, s.MetricID)
	// completed data page
	completed := s.HeadDataPage
	bin.WriteUint32(w, completed.PageNo)
	bin.WriteBool(w, completed.Reused)
	bin.WriteUint32(w, completed.PrevPageNo)
	bin.WriteUint32(w, completed.Checksum)
	bin.WriteVarSize(w, completed.TimestampsOffset)
	bin.WriteVarSize(w, len(completed.Timestamps))
	w.Write(completed.Timestamps)
	bin.WriteVarSize(w, completed.ValuesOffset)
	bin.WriteVarSize(w, len(completed.Values))
	w.Write(completed.Values)
	// data pages
	bin.WriteVarSize(w, len(s.DataPages))
	for _, p := range s.DataPages {
		bin.WriteUint32(w, p.PageNo)
		bin.WriteBool(w, p.Reused)
		bin.WriteVarSize(w, len(p.Content))
		w.Write(p.Content)
	}
	// data tail
	bin.WriteVarSize(w, len(s.TailTimestamps))
	w.Write(s.TailTimestamps)
	bin.WriteVarSize(w, len(s.TailValues))
	w.Write(s.TailValues)
	// changed levels
	bin.WriteVarSize(w, len(s.ChangedIndexLevels))
	for _, level := range s.ChangedIndexLevels {
		// completed index page
		completed := level.HeadIndexPage
		bin.WriteUint32(w, completed.PageNo)
		bin.WriteBool(w, completed.Reused)
		bin.WriteUint32(w, completed.Checksum)
		bin.WriteVarSize(w, len(completed.Records)/indexRecordSize)
		w.Write(completed.Records)
		// full index pages
		for _, p := range level.IndexPages {
			bin.WriteUint32(w, p.PageNo)
			bin.WriteBool(w, p.Reused)
			bin.WriteVarSize(w, len(p.Content))
			w.Write(p.Content)
		}
		// level tail
		bin.WriteVarSize(w, len(level.TailRecords)/indexRecordSize)
		w.Write(level.TailRecords)
	}
}

func (s *MeasuresAppendWithGrowRecord) Parse(r *bytes.Buffer) (err error) {
	var (
		size         int
		recordsCount int
	)
	s.MetricID, err = bin.ReadUint32(r)
	if err != nil {
		return
	}
	s.HeadDataPage.PageNo, err = bin.ReadUint32(r)
	if err != nil {
		return
	}
	s.HeadDataPage.Reused, err = bin.ReadBool(r)
	if err != nil {
		return
	}
	s.HeadDataPage.PrevPageNo, err = bin.ReadUint32(r)
	if err != nil {
		return
	}
	s.HeadDataPage.Checksum, err = bin.ReadUint32(r)
	if err != nil {
		return
	}
	s.HeadDataPage.TimestampsOffset, err = bin.ReadVarSize(r)
	if err != nil {
		return
	}
	size, err = bin.ReadVarSize(r)
	if err != nil {
		return
	}
	s.HeadDataPage.Timestamps, err = bin.ReadN(r, size)
	if err != nil {
		return
	}
	size, err = bin.ReadVarSize(r)
	if err != nil {
		return
	}
	s.HeadDataPage.Values, err = bin.ReadN(r, size)
	if err != nil {
		return
	}
	// data pages
	dataPagesQty, err := bin.ReadVarSize(r)
	if err != nil {
		return
	}
	for range dataPagesQty {
		var (
			p SealedDataPage
		)
		p.PageNo, err = bin.ReadUint32(r)
		if err != nil {
			return
		}
		p.Reused, err = bin.ReadBool(r)
		if err != nil {
			return
		}
		size, err = bin.ReadVarSize(r)
		if err != nil {
			return
		}
		p.Content, err = bin.ReadN(r, size)
		if err != nil {
			return
		}
		s.DataPages = append(s.DataPages, p)
	}
	// data tail
	size, err = bin.ReadVarSize(r)
	if err != nil {
		return
	}
	s.TailTimestamps, err = bin.ReadN(r, size)
	if err != nil {
		return
	}
	size, err = bin.ReadVarSize(r)
	if err != nil {
		return
	}
	s.TailValues, err = bin.ReadN(r, size)
	if err != nil {
		return
	}
	// changed levels
	changedIndexLevelsCount, err := bin.ReadVarSize(r)
	if err != nil {
		return
	}
	for range changedIndexLevelsCount {
		var (
			indexPagesCount int
			level           ChangedIndexLevel
		)
		// completed index page
		level.HeadIndexPage.PageNo, err = bin.ReadUint32(r)
		if err != nil {
			return
		}
		level.HeadIndexPage.Reused, err = bin.ReadBool(r)
		if err != nil {
			return
		}
		level.HeadIndexPage.Checksum, err = bin.ReadUint32(r)
		if err != nil {
			return
		}
		recordsCount, err = bin.ReadVarSize(r)
		if err != nil {
			return
		}
		level.HeadIndexPage.Records, err = bin.ReadN(r, recordsCount*indexRecordSize)
		if err != nil {
			return
		}
		//
		indexPagesCount, err = bin.ReadVarSize(r)
		if err != nil {
			return
		}
		for range indexPagesCount {
			var p SealedIndexPage
			p.PageNo, err = bin.ReadUint32(r)
			if err != nil {
				return
			}
			p.Reused, err = bin.ReadBool(r)
			if err != nil {
				return
			}
			p.Checksum, err = bin.ReadUint32(r)
			if err != nil {
				return
			}
			size, err = bin.ReadVarSize(r)
			if err != nil {
				return
			}
			p.Content, err = bin.ReadN(r, size)
			if err != nil {
				return
			}
			level.IndexPages = append(level.IndexPages, p)
		}
		recordsCount, err = bin.ReadVarSize(r)
		if err != nil {
			return
		}
		level.TailRecords, err = bin.ReadN(r, recordsCount*indexRecordSize)
		if err != nil {
			return
		}
		s.ChangedIndexLevels = append(s.ChangedIndexLevels, level)
	}
	return
}

type MeasuresAppendRecord struct {
	MetricID         uint32
	TimestampsOffset int
	Timestamps       []byte
	ValuesOffset     int
	Values           []byte
}

func (s MeasuresAppendRecord) WriteTo(w *bytes.Buffer) {
	w.WriteByte(CodeMeasuresAppendWithGrow)
	bin.WriteUint32(w, s.MetricID)
	bin.WriteVarSize(w, s.TimestampsOffset)
	bin.WriteVarSize(w, len(s.Timestamps))
	w.Write(s.Timestamps)
	bin.WriteVarSize(w, s.ValuesOffset)
	bin.WriteVarSize(w, len(s.Values))
	w.Write(s.Values)
}

func (s *MeasuresAppendRecord) Parse(r *bytes.Buffer) (err error) {
	var size int
	s.MetricID, err = bin.ReadUint32(r)
	if err != nil {
		return
	}
	s.TimestampsOffset, err = bin.ReadVarSize(r)
	if err != nil {
		return
	}
	size, err = bin.ReadVarSize(r)
	if err != nil {
		return
	}
	s.Timestamps, err = bin.ReadN(r, size)
	if err != nil {
		return
	}
	s.ValuesOffset, err = bin.ReadVarSize(r)
	if err != nil {
		return
	}
	size, err = bin.ReadVarSize(r)
	if err != nil {
		return
	}
	s.Values, err = bin.ReadN(r, size)
	if err != nil {
		return
	}
	return
}

type MetricAddRecord struct {
	MetricID   uint32
	MetricType qb.MetricType
	FracDigits byte
}

func (s MetricAddRecord) Pack(w io.Writer) {
	arr := []byte{
		CodeMetricAdd,
		0, 0, 0, 0, //
		byte(s.MetricType),
		s.FracDigits,
	}
	bin.PutUint32(arr[1:], s.MetricID)
	w.Write(arr)
}

func (s *MetricAddRecord) Parse(src io.Reader) (err error) {
	arr, err := bin.ReadN(src, 6)
	if err != nil {
		return
	}
	s.MetricID, _ = bin.GetUint32(arr)
	s.MetricType = qb.MetricType(arr[4])
	s.FracDigits = arr[5]
	return nil
}

type MetricDeleteRecord struct {
	MetricID       uint32
	FreeIndexPages []uint32
	FreeDataPages  []uint32
}

func (s MetricDeleteRecord) Pack(w io.Writer) {
	arr := []byte{
		CodeMetricDelete,
		0, 0, 0, 0, // metricID
	}
	bin.PutUint32(arr[1:], s.MetricID)
	w.Write(arr)
	// fix
	// bin.WriteVarSize(w, len(s.FreePageNumbers))
	// for _, pageNo := range s.FreePageNumbers {
	// 	bin.WriteUint32(w, pageNo)
	// }
}

func (s *MetricDeleteRecord) Parse(src io.Reader) (err error) {
	s.MetricID, err = bin.ReadUint32(src)
	if err != nil {
		return
	}
	// free pages
	// fix
	// qty, err := bin.ReadVarSize(src)
	// if err != nil {
	// 	return
	// }
	// for range qty {
	// 	var pageNo uint32
	// 	pageNo, err = bin.ReadUint32(src)
	// 	if err != nil {
	// 		return
	// 	}
	// 	//s.FreePageNumbers = append(s.FreePageNumbers, pageNo)
	// }
	return nil
}

// fix - add changed pages
type DeletedMeasures struct {
	MetricID        uint32
	FreePageNumbers []uint32
}

func (s DeletedMeasures) Pack(w io.Writer) {
	arr := []byte{
		CodeMeasuresDelete,
		0, 0, 0, 0,
	}
	bin.PutUint32(arr[1:], s.MetricID)
	w.Write(arr)
	bin.WriteVarSize(w, len(s.FreePageNumbers))
	for _, pageNo := range s.FreePageNumbers {
		bin.WriteUint32(w, pageNo)
	}
}

func (s *DeletedMeasures) Parse(src io.Reader) (err error) {
	s.MetricID, err = bin.ReadUint32(src)
	if err != nil {
		return
	}
	// free pages
	qty, err := bin.ReadVarSize(src)
	if err != nil {
		return
	}
	for range qty {
		var pageNo uint32
		pageNo, err = bin.ReadUint32(src)
		if err != nil {
			return
		}
		s.FreePageNumbers = append(s.FreePageNumbers, pageNo)
	}
	return nil
}

// type AppendedMeasure struct {
// 	MetricID  uint32
// 	Timestamp uint32
// 	Value     float64
// }

// func (s AppendedMeasure) Pack(w io.Writer) {
// 	arr := []byte{
// 		CodeAppendedMeasure,
// 		0, 0, 0, 0, // metricID
// 		0, 0, 0, 0, // timestamp
// 		0, 0, 0, 0, 0, 0, 0, 0, // value
// 	}
// 	bin.PutUint32(arr[1:], s.MetricID)
// 	bin.PutUint32(arr[5:], s.Timestamp)
// 	bin.PutFloat64(arr[9:], s.Value)
// 	w.Write(arr)
// }

// func (s *AppendedMeasure) Parse(src io.Reader) (err error) {
// 	arr, err := bin.ReadN(src, 16)
// 	if err != nil {
// 		return
// 	}
// 	s.MetricID, _ = bin.GetUint32(arr[0:])
// 	s.Timestamp, _ = bin.GetUint32(arr[4:])
// 	s.Value, _ = bin.GetFloat64(arr[8:])
// 	return nil
// }

// type AppendedMeasures struct {
// 	MetricID uint32
// 	Measures []proto.Measure
// }

// func (s AppendedMeasures) Pack(w io.Writer) {
// 	arr := []byte{
// 		CodeAppendedMeasures,
// 		0, 0, 0, 0, // metricID
// 		0, 0, // qty
// 	}
// 	bin.PutUint32(arr[1:], s.MetricID)
// 	bin.PutUint16(arr[5:], uint16(len(s.Measures)))
// 	w.Write(arr)
// 	for _, measure := range s.Measures {
// 		bin.WriteUint32(w, measure.Timestamp)
// 		bin.WriteFloat64(w, measure.Value)
// 	}
// }

// func (s *AppendedMeasures) Parse(src io.Reader) (err error) {
// 	s.MetricID, err = bin.ReadUint32(src)
// 	if err != nil {
// 		return
// 	}
// 	qty, err := bin.ReadUint16(src)
// 	if err != nil {
// 		return
// 	}
// 	for range qty {
// 		var measure proto.Measure
// 		measure.Timestamp, err = bin.ReadUint32(src)
// 		if err != nil {
// 			return
// 		}
// 		measure.Value, err = bin.ReadFloat64(src)
// 		if err != nil {
// 			return
// 		}
// 		s.Measures = append(s.Measures, measure)
// 	}
// 	return nil
// }

// type AppendedPagesReq struct {
// 	Legs      []atree.PathLeg
// 	MetricID  uint32
// 	Timestamp uint32  // last measure
// 	Value     float64 // last  measure
// 	//RootPageNo       uint32
// 	LastPageNo       uint32
// 	Pages            []atree.NotLinkedDataPage
// 	TimestampsChunks [][]byte
// 	TimestampsSize   uint16
// 	ValuesChunks     [][]byte
// 	ValuesSize       uint16
// }

/*
4b   metricID
4b   timestamp
8b   value
4b   LastPageNo (0 if no new root page)
4b   newRootPageNo (0 if no new root page)
Nb - varsize (pages qty)
[

	4b  pageNo
	1b  isReused
	Xb  page (page size)

]
2b - timestamps size (not filled page)
Nb - timestamps payload
2b - values size (not filled page)
Nb - values payload
*/
// type AppendedPages struct {
// 	MetricID         uint32
// 	Timestamp        uint32  // last measure
// 	Value            float64 // last  measure
// 	NewRootPageNo    uint32
// 	LastPageNo       uint32
// 	Pages            []atree.PageToWrite
// 	TimestampsChunks [][]byte
// 	TimestampsSize   int
// 	ValuesChunks     [][]byte
// 	ValuesSize       int
// }

// func (s *AppendedPages) Pack(w io.Writer) {
// 	w.Write([]byte{
// 		CodeAppendedPages,
// 	})
// 	bin.WriteUint32(w, s.MetricID)
// 	bin.WriteUint32(w, s.Timestamp)
// 	bin.WriteFloat64(w, s.Value)
// 	bin.WriteUint32(w, s.LastPageNo)
// 	bin.WriteUint32(w, s.NewRootPageNo)
// 	//
// 	bin.WriteVarSize(w, len(s.Pages))
// 	for _, p := range s.Pages {
// 		bin.WriteUint32(w, p.PageNo)
// 		if p.IsReused {
// 			w.Write([]byte{1})
// 		} else {
// 			w.Write([]byte{0})
// 		}
// 		w.Write(p.Data)
// 	}
// 	// timestamps
// 	writeChunkedPayloadTo(s.TimestampsChunks, int(s.TimestampsSize), w)
// 	// values
// 	writeChunkedPayloadTo(s.ValuesChunks, int(s.ValuesSize), w)
// }

// func (s *AppendedPages) Parse(src io.Reader) (err error) {
// 	s.MetricID, err = bin.ReadUint32(src)
// 	if err != nil {
// 		return
// 	}
// 	s.Timestamp, err = bin.ReadUint32(src)
// 	if err != nil {
// 		return
// 	}
// 	s.Value, err = bin.ReadFloat64(src)
// 	if err != nil {
// 		return
// 	}
// 	s.LastPageNo, err = bin.ReadUint32(src)
// 	if err != nil {
// 		return
// 	}
// 	s.NewRootPageNo, err = bin.ReadUint32(src)
// 	if err != nil {
// 		return
// 	}
// 	pagesQty, err := bin.ReadVarSize(src)
// 	if err != nil {
// 		return
// 	}
// 	for range pagesQty {
// 		var page atree.PageToWrite
// 		page.PageNo, err = bin.ReadUint32(src)
// 		if err != nil {
// 			return
// 		}
// 		page.IsReused, err = bin.ReadBool(src)
// 		if err != nil {
// 			return
// 		}
// 		page.Data, err = bin.ReadN(src, atree.PageSize)
// 		if err != nil {
// 			return
// 		}
// 		s.Pages = append(s.Pages, page)
// 	}
// 	s.TimestampsChunks, s.TimestampsSize, err = readChunkedPayload(src)
// 	if err != nil {
// 		return
// 	}
// 	s.ValuesChunks, s.ValuesSize, err = readChunkedPayload(src)
// 	if err != nil {
// 		return
// 	}
// 	return nil
// }