wp

txlog/helpers.go

@@ -0,0 +1,238 @@
+package txlog
+
+import (
+	"bytes"
+
+	bin "gordenko.dev/dima/bin/little"
+)
+
+const recordSize = 8
+const chunkSize = 24
+
+/*
+Format appended measures:
+1b - tx type
+4b - metricID
+Nb - qty of filled data pages (varsize)
+[
+	4b - pageNo
+	1b - reused
+	4b - prevPageNo
+	4b - page crc32
+	2b - timestamps size
+	Nb - timestamps payload
+	2b - values size
+	Nb - values payload
+]
+2b - timestamps offset (for the 1st data page only)
+2b - values offset (for the 1st data page only)
+Nb - timestamps size unfilled (varsize)
+Nb - timestamps payload unfilled
+Nb - values size unfilled (varsize)
+Nb - values payload unfilled
+Nb - qty of index levels (varsize)
+[
+	Nb - qty of level filled pages (varsize)
+	[
+		4b - pageNo
+		1b - reused
+		4b - page crc32
+		Nb - records qty (varsize)
+		Nb - records payload
+	]
+	Nb - records qty unfilled (varsize)
+	Nb - records payload unfilled
+]
+*/
+
+func copyPayloadFromChunks(w *bytes.Buffer, chunks [][]byte, size int, offset int) {
+	bin.WriteUint16(w, uint16(size))
+	var (
+		chunkIdx = offset / chunkSize
+		byteIdx  = offset % chunkSize
+	)
+	for _, chunk := range chunks[chunkIdx:] {
+		available := len(chunk) - byteIdx
+		if available <= size {
+			w.Write(chunk[byteIdx:])
+			size -= available
+			if size == 0 {
+				return
+			}
+			byteIdx = 0
+		} else {
+			w.Write(chunk[byteIdx : byteIdx+size])
+			return
+		}
+	}
+}
+
+/////////////////////////////////////////
+
+type DataPage struct {
+	PageNo     uint32
+	Reused     bool
+	PrevPageNo uint32
+	Checksum   uint32
+	Timestamps []byte
+	Values     []byte
+}
+
+type IndexPage struct {
+	PageNo   uint32
+	Reused   bool
+	Checksum uint32
+	Records  []byte
+}
+
+type IndexLevel struct {
+	Pages   []IndexPage
+	Records []byte // unfilled
+}
+
+// Для декодінга
+type TxAppendedMeasures struct {
+	MetricID  uint32
+	DataPages []DataPage
+	// offsets потрібні тому що дані не додаються в кінець, а перезаписують кілька
+	// останніх байтів unfilled даних. Потрібно для коректного recovery.
+	// Якщо є DataPages, то застосовуються для 1-ї Data сторінки, інакше - до timestamps і values (unfilled)
+	TimestampsOffset int
+	ValuesOffset     int
+	Timestamps       []byte // unfilled
+	Values           []byte // unfilled
+	IndexLevels      []IndexLevel
+}
+
+func (s *TxAppendedMeasures) Read(r *bytes.Buffer) (err error) {
+	s.MetricID, err = bin.ReadUint32(r)
+	if err != nil {
+		return
+	}
+	dataPagesQty, err := bin.ReadVarSize(r)
+	if err != nil {
+		return
+	}
+	for range dataPagesQty {
+		var (
+			p                          DataPage
+			timestampsSize, valuesSize int
+			//b byte
+		)
+		p.PageNo, err = bin.ReadUint32(r)
+		if err != nil {
+			return
+		}
+		p.Reused, err = bin.ReadBool(r)
+		if err != nil {
+			return
+		}
+		p.PrevPageNo, err = bin.ReadUint32(r)
+		if err != nil {
+			return
+		}
+		p.Checksum, err = bin.ReadUint32(r)
+		if err != nil {
+			return
+		}
+		timestampsSize, err = bin.ReadUint16AsInt(r)
+		if err != nil {
+			return
+		}
+		p.Timestamps, err = bin.ReadN(r, timestampsSize)
+		if err != nil {
+			return
+		}
+		valuesSize, err = bin.ReadUint16AsInt(r)
+		if err != nil {
+			return
+		}
+		p.Values, err = bin.ReadN(r, valuesSize)
+		if err != nil {
+			return
+		}
+		s.DataPages = append(s.DataPages, p)
+	}
+	s.TimestampsOffset, err = bin.ReadUint16AsInt(r)
+	if err != nil {
+		return
+	}
+	s.ValuesOffset, err = bin.ReadUint16AsInt(r)
+	if err != nil {
+		return
+	}
+	var (
+		timestampsSize, valuesSize int
+	)
+	timestampsSize, err = bin.ReadVarSize(r)
+	if err != nil {
+		return
+	}
+	s.Timestamps, err = bin.ReadN(r, timestampsSize)
+	if err != nil {
+		return
+	}
+	valuesSize, err = bin.ReadVarSize(r)
+	if err != nil {
+		return
+	}
+	s.Values, err = bin.ReadN(r, valuesSize)
+	if err != nil {
+		return
+	}
+	if len(s.DataPages) == 0 {
+		return
+	}
+	indexLevelsQty, err := bin.ReadVarSize(r)
+	if err != nil {
+		return
+	}
+	for range indexLevelsQty {
+		var (
+			indexPagesQty int
+			level         IndexLevel
+			recordsQty    int
+		)
+		indexPagesQty, err = bin.ReadVarSize(r)
+		if err != nil {
+			return
+		}
+		for range indexPagesQty {
+			var (
+				p          IndexPage
+				recordsQty int
+			)
+			p.PageNo, err = bin.ReadUint32(r)
+			if err != nil {
+				return
+			}
+			p.Reused, err = bin.ReadBool(r)
+			if err != nil {
+				return
+			}
+			p.Checksum, err = bin.ReadUint32(r)
+			if err != nil {
+				return
+			}
+			recordsQty, err = bin.ReadVarSize(r)
+			if err != nil {
+				return
+			}
+			p.Records, err = bin.ReadN(r, recordsQty*recordSize)
+			if err != nil {
+				return
+			}
+			level.Pages = append(level.Pages, p)
+		}
+		recordsQty, err = bin.ReadVarSize(r)
+		if err != nil {
+			return
+		}
+		level.Records, err = bin.ReadN(r, recordsQty*recordSize)
+		if err != nil {
+			return
+		}
+		s.IndexLevels = append(s.IndexLevels, level)
+	}
+	return
+}