tmp
This commit is contained in:
205
txlog/txlog.go
205
txlog/txlog.go
@@ -1,11 +1,13 @@
|
||||
package txlog
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
|
||||
bin "gordenko.dev/dima/bin/little"
|
||||
"gordenko.dev/dima/qb"
|
||||
"gordenko.dev/dima/qb/conbuf"
|
||||
"gordenko.dev/dima/qb/util"
|
||||
)
|
||||
|
||||
type AddedMetric struct {
|
||||
@@ -75,6 +77,89 @@ func (s *DeletedMetric) Parse(src io.Reader) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeChunkedPayloadTo(chunks [][]byte, size int, w io.Writer) {
|
||||
bin.WriteVarSize(w, size)
|
||||
for _, chunk := range chunks {
|
||||
if size >= len(chunk) {
|
||||
w.Write(chunk)
|
||||
size -= len(chunk)
|
||||
} else {
|
||||
w.Write(chunk[:size])
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func readChunkedPayload(src io.Reader) (_ [][]byte, _ int, err error) {
|
||||
var (
|
||||
chunks [][]byte
|
||||
size int
|
||||
remainingSize = size
|
||||
)
|
||||
size, err = bin.ReadVarSize(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for remainingSize > 0 {
|
||||
var chunk []byte
|
||||
if remainingSize >= conbuf.ChunkSize {
|
||||
chunk, err = bin.ReadN(src, conbuf.ChunkSize)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
chunk = make([]byte, conbuf.ChunkSize)
|
||||
err = bin.ReadNInto(src, chunk[:remainingSize])
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
chunks = append(chunks, chunk)
|
||||
remainingSize -= conbuf.ChunkSize
|
||||
}
|
||||
return chunks, size, nil
|
||||
}
|
||||
|
||||
// fix - add changed pages
|
||||
type DeletedMeasures struct {
|
||||
MetricID uint32
|
||||
FreePageNumbers []uint32
|
||||
}
|
||||
|
||||
func (s DeletedMeasures) Pack(w io.Writer) {
|
||||
arr := []byte{
|
||||
CodeDeletedMeasures,
|
||||
0, 0, 0, 0,
|
||||
}
|
||||
bin.PutUint32(arr[1:], s.MetricID)
|
||||
w.Write(arr)
|
||||
bin.WriteVarSize(w, len(s.FreePageNumbers))
|
||||
for _, pageNo := range s.FreePageNumbers {
|
||||
bin.WriteUint32(w, pageNo)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DeletedMeasures) Parse(src io.Reader) (err error) {
|
||||
s.MetricID, err = bin.ReadUint32(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// free pages
|
||||
qty, err := bin.ReadVarSize(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for range qty {
|
||||
var pageNo uint32
|
||||
pageNo, err = bin.ReadUint32(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.FreePageNumbers = append(s.FreePageNumbers, pageNo)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// type AppendedMeasure struct {
|
||||
// MetricID uint32
|
||||
// Timestamp uint32
|
||||
@@ -273,85 +358,63 @@ Nb - values payload
|
||||
// return nil
|
||||
// }
|
||||
|
||||
func writeChunkedPayloadTo(chunks [][]byte, size int, w io.Writer) {
|
||||
bin.WriteVarSize(w, size)
|
||||
for _, chunk := range chunks {
|
||||
if size >= len(chunk) {
|
||||
w.Write(chunk)
|
||||
size -= len(chunk)
|
||||
} else {
|
||||
w.Write(chunk[:size])
|
||||
return
|
||||
}
|
||||
}
|
||||
// PACK
|
||||
|
||||
type PreparedData struct {
|
||||
Packet []byte // пакет для запису в WAL
|
||||
WriteToIndex []IndexPayloadToWrite
|
||||
WriteToData []DataPayloadToWrite
|
||||
ToWorker []any
|
||||
}
|
||||
|
||||
func readChunkedPayload(src io.Reader) (_ [][]byte, _ int, err error) {
|
||||
func prepareData(input []any) PreparedData {
|
||||
buffer := bytes.NewBuffer(nil)
|
||||
buffer.Write([]byte{
|
||||
0, 0, 0, 0, 0, 0, 0, 0, // size (max 8 byte)
|
||||
0, 0, 0, 0, // crc32
|
||||
})
|
||||
|
||||
hasher := util.NewHasher()
|
||||
|
||||
w := io.MultiWriter(buffer, hasher)
|
||||
|
||||
var (
|
||||
chunks [][]byte
|
||||
size int
|
||||
remainingSize = size
|
||||
writeToIndex []IndexPayloadToWrite
|
||||
writeToData []DataPayloadToWrite
|
||||
)
|
||||
size, err = bin.ReadVarSize(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for remainingSize > 0 {
|
||||
var chunk []byte
|
||||
if remainingSize >= conbuf.ChunkSize {
|
||||
chunk, err = bin.ReadN(src, conbuf.ChunkSize)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
chunk = make([]byte, conbuf.ChunkSize)
|
||||
err = bin.ReadNInto(src, chunk[:remainingSize])
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// 1. Пакую всі дані в WAL буфер запису
|
||||
for _, untyped := range input {
|
||||
switch x := untyped.(type) {
|
||||
case AddedMetric:
|
||||
x.Pack(w)
|
||||
// case DeletedMetric:
|
||||
// if len(x.FreePageNumbers) > 0 {
|
||||
// s.freeList.AddPageNumbers(x.FreePageNumbers)
|
||||
// }
|
||||
case AppendedMeasures:
|
||||
//s.packMeasuresIntoWALBuffer(x)
|
||||
//case DeletedMeasures:
|
||||
//case DeletedMeasuresSince:
|
||||
}
|
||||
chunks = append(chunks, chunk)
|
||||
remainingSize -= conbuf.ChunkSize
|
||||
}
|
||||
return chunks, size, nil
|
||||
}
|
||||
// 2. Додаю розмір пакету і чексуму
|
||||
|
||||
// fix - add changed pages
|
||||
type DeletedMeasures struct {
|
||||
MetricID uint32
|
||||
FreePageNumbers []uint32
|
||||
}
|
||||
//s.written += int64(len(packet)) + 12
|
||||
|
||||
func (s DeletedMeasures) Pack(w io.Writer) {
|
||||
arr := []byte{
|
||||
CodeDeletedMeasures,
|
||||
0, 0, 0, 0,
|
||||
}
|
||||
bin.PutUint32(arr[1:], s.MetricID)
|
||||
w.Write(arr)
|
||||
bin.WriteVarSize(w, len(s.FreePageNumbers))
|
||||
for _, pageNo := range s.FreePageNumbers {
|
||||
bin.WriteUint32(w, pageNo)
|
||||
}
|
||||
}
|
||||
packet := buffer.Bytes()
|
||||
// write size
|
||||
payloadSize := len(packet) - 12
|
||||
start := 8 - bin.CountVarSize(payloadSize)
|
||||
bin.PutVarSize(packet[start:], payloadSize)
|
||||
// checksum
|
||||
bin.PutUint32(packet[8:], hasher.Sum32())
|
||||
//
|
||||
//totalSize := (12 - start) + payloadSize
|
||||
|
||||
func (s *DeletedMeasures) Parse(src io.Reader) (err error) {
|
||||
s.MetricID, err = bin.ReadUint32(src)
|
||||
if err != nil {
|
||||
return
|
||||
return PreparedData{
|
||||
Packet: packet[start:],
|
||||
WriteToIndex: writeToIndex,
|
||||
WriteToData: writeToData,
|
||||
}
|
||||
// free pages
|
||||
qty, err := bin.ReadVarSize(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for range qty {
|
||||
var pageNo uint32
|
||||
pageNo, err = bin.ReadUint32(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.FreePageNumbers = append(s.FreePageNumbers, pageNo)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user