qb/txlog/helpers.go

package txlog

import (
	"bytes"

	bin "gordenko.dev/dima/bin/little"
)

const recordSize = 8
const chunkSize = 24

/*
Format appended measures:
1b - tx type
4b - metricID
Nb - qty of filled data pages (varsize)
[
	4b - pageNo
	1b - reused
	4b - prevPageNo
	4b - page crc32
	2b - timestamps size
	Nb - timestamps payload
	2b - values size
	Nb - values payload
]
2b - timestamps offset (for the 1st data page only)
2b - values offset (for the 1st data page only)
Nb - timestamps size unfilled (varsize)
Nb - timestamps payload unfilled
Nb - values size unfilled (varsize)
Nb - values payload unfilled
Nb - qty of index levels (varsize)
[
	Nb - qty of level filled pages (varsize)
	[
		4b - pageNo
		1b - reused
		4b - page crc32
		Nb - records qty (varsize)
		Nb - records payload
	]
	Nb - records qty unfilled (varsize)
	Nb - records payload unfilled
]
*/

func copyPayloadFromChunks(w *bytes.Buffer, chunks [][]byte, size int, offset int) {
	bin.WriteUint16(w, uint16(size))
	var (
		chunkIdx = offset / chunkSize
		byteIdx  = offset % chunkSize
	)
	for _, chunk := range chunks[chunkIdx:] {
		available := len(chunk) - byteIdx
		if available <= size {
			w.Write(chunk[byteIdx:])
			size -= available
			if size == 0 {
				return
			}
			byteIdx = 0
		} else {
			w.Write(chunk[byteIdx : byteIdx+size])
			return
		}
	}
}

/////////////////////////////////////////

type DataPage struct {
	PageNo     uint32
	Reused     bool
	PrevPageNo uint32
	Checksum   uint32
	Timestamps []byte
	Values     []byte
}

type IndexPage struct {
	PageNo   uint32
	Reused   bool
	Checksum uint32
	Records  []byte
}

type IndexLevel struct {
	Pages   []IndexPage
	Records []byte // unfilled
}

// Для декодінга
type TxAppendedMeasures struct {
	MetricID  uint32
	DataPages []DataPage
	// offsets потрібні тому що дані не додаються в кінець, а перезаписують кілька
	// останніх байтів unfilled даних. Потрібно для коректного recovery.
	// Якщо є DataPages, то застосовуються для 1-ї Data сторінки, інакше - до timestamps і values (unfilled)
	TimestampsOffset int
	ValuesOffset     int
	Timestamps       []byte // unfilled
	Values           []byte // unfilled
	IndexLevels      []IndexLevel
}

func (s *TxAppendedMeasures) Read(r *bytes.Buffer) (err error) {
	s.MetricID, err = bin.ReadUint32(r)
	if err != nil {
		return
	}
	dataPagesQty, err := bin.ReadVarSize(r)
	if err != nil {
		return
	}
	for range dataPagesQty {
		var (
			p                          DataPage
			timestampsSize, valuesSize int
			//b byte
		)
		p.PageNo, err = bin.ReadUint32(r)
		if err != nil {
			return
		}
		p.Reused, err = bin.ReadBool(r)
		if err != nil {
			return
		}
		p.PrevPageNo, err = bin.ReadUint32(r)
		if err != nil {
			return
		}
		p.Checksum, err = bin.ReadUint32(r)
		if err != nil {
			return
		}
		timestampsSize, err = bin.ReadUint16AsInt(r)
		if err != nil {
			return
		}
		p.Timestamps, err = bin.ReadN(r, timestampsSize)
		if err != nil {
			return
		}
		valuesSize, err = bin.ReadUint16AsInt(r)
		if err != nil {
			return
		}
		p.Values, err = bin.ReadN(r, valuesSize)
		if err != nil {
			return
		}
		s.DataPages = append(s.DataPages, p)
	}
	s.TimestampsOffset, err = bin.ReadUint16AsInt(r)
	if err != nil {
		return
	}
	s.ValuesOffset, err = bin.ReadUint16AsInt(r)
	if err != nil {
		return
	}
	var (
		timestampsSize, valuesSize int
	)
	timestampsSize, err = bin.ReadVarSize(r)
	if err != nil {
		return
	}
	s.Timestamps, err = bin.ReadN(r, timestampsSize)
	if err != nil {
		return
	}
	valuesSize, err = bin.ReadVarSize(r)
	if err != nil {
		return
	}
	s.Values, err = bin.ReadN(r, valuesSize)
	if err != nil {
		return
	}
	if len(s.DataPages) == 0 {
		return
	}
	indexLevelsQty, err := bin.ReadVarSize(r)
	if err != nil {
		return
	}
	for range indexLevelsQty {
		var (
			indexPagesQty int
			level         IndexLevel
			recordsQty    int
		)
		indexPagesQty, err = bin.ReadVarSize(r)
		if err != nil {
			return
		}
		for range indexPagesQty {
			var (
				p          IndexPage
				recordsQty int
			)
			p.PageNo, err = bin.ReadUint32(r)
			if err != nil {
				return
			}
			p.Reused, err = bin.ReadBool(r)
			if err != nil {
				return
			}
			p.Checksum, err = bin.ReadUint32(r)
			if err != nil {
				return
			}
			recordsQty, err = bin.ReadVarSize(r)
			if err != nil {
				return
			}
			p.Records, err = bin.ReadN(r, recordsQty*recordSize)
			if err != nil {
				return
			}
			level.Pages = append(level.Pages, p)
		}
		recordsQty, err = bin.ReadVarSize(r)
		if err != nil {
			return
		}
		level.Records, err = bin.ReadN(r, recordsQty*recordSize)
		if err != nil {
			return
		}
		s.IndexLevels = append(s.IndexLevels, level)
	}
	return
}