dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-05-21 23:38:52 +03:00
parent 793df29e11
commit e7ec6083cb
21 changed files with 1234 additions and 1017 deletions
Download Patch File Download Diff File Expand all files Collapse all files

238
txlog/helpers.go Normal file
View File

@@ -0,0 +1,238 @@
package txlog
import (
"bytes"
bin "gordenko.dev/dima/bin/little"
)
const recordSize = 8
const chunkSize = 24
/*
Format appended measures:
1b - tx type
4b - metricID
Nb - qty of filled data pages (varsize)
[
4b - pageNo
1b - reused
4b - prevPageNo
4b - page crc32
2b - timestamps size
Nb - timestamps payload
2b - values size
Nb - values payload
]
2b - timestamps offset (for the 1st data page only)
2b - values offset (for the 1st data page only)
Nb - timestamps size unfilled (varsize)
Nb - timestamps payload unfilled
Nb - values size unfilled (varsize)
Nb - values payload unfilled
Nb - qty of index levels (varsize)
[
Nb - qty of level filled pages (varsize)
[
4b - pageNo
1b - reused
4b - page crc32
Nb - records qty (varsize)
Nb - records payload
]
Nb - records qty unfilled (varsize)
Nb - records payload unfilled
]
*/
func copyPayloadFromChunks(w *bytes.Buffer, chunks [][]byte, size int, offset int) {
bin.WriteUint16(w, uint16(size))
var (
chunkIdx = offset / chunkSize
byteIdx = offset % chunkSize
)
for _, chunk := range chunks[chunkIdx:] {
available := len(chunk) - byteIdx
if available <= size {
w.Write(chunk[byteIdx:])
size -= available
if size == 0 {
return
}
byteIdx = 0
} else {
w.Write(chunk[byteIdx : byteIdx+size])
return
}
}
}
/////////////////////////////////////////
type DataPage struct {
PageNo uint32
Reused bool
PrevPageNo uint32
Checksum uint32
Timestamps []byte
Values []byte
}
type IndexPage struct {
PageNo uint32
Reused bool
Checksum uint32
Records []byte
}
type IndexLevel struct {
Pages []IndexPage
Records []byte // unfilled
}
// Для декодінга
type TxAppendedMeasures struct {
MetricID uint32
DataPages []DataPage
// offsets потрібні тому що дані не додаються в кінець, а перезаписують кілька
// останніх байтів unfilled даних. Потрібно для коректного recovery.
// Якщо є DataPages, то застосовуються для 1-ї Data сторінки, інакше - до timestamps і values (unfilled)
TimestampsOffset int
ValuesOffset int
Timestamps []byte // unfilled
Values []byte // unfilled
IndexLevels []IndexLevel
}
func (s *TxAppendedMeasures) Read(r *bytes.Buffer) (err error) {
s.MetricID, err = bin.ReadUint32(r)
if err != nil {
return
}
dataPagesQty, err := bin.ReadVarSize(r)
if err != nil {
return
}
for range dataPagesQty {
var (
p DataPage
timestampsSize, valuesSize int
//b byte
)
p.PageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
p.Reused, err = bin.ReadBool(r)
if err != nil {
return
}
p.PrevPageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
p.Checksum, err = bin.ReadUint32(r)
if err != nil {
return
}
timestampsSize, err = bin.ReadUint16AsInt(r)
if err != nil {
return
}
p.Timestamps, err = bin.ReadN(r, timestampsSize)
if err != nil {
return
}
valuesSize, err = bin.ReadUint16AsInt(r)
if err != nil {
return
}
p.Values, err = bin.ReadN(r, valuesSize)
if err != nil {
return
}
s.DataPages = append(s.DataPages, p)
}
s.TimestampsOffset, err = bin.ReadUint16AsInt(r)
if err != nil {
return
}
s.ValuesOffset, err = bin.ReadUint16AsInt(r)
if err != nil {
return
}
var (
timestampsSize, valuesSize int
)
timestampsSize, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.Timestamps, err = bin.ReadN(r, timestampsSize)
if err != nil {
return
}
valuesSize, err = bin.ReadVarSize(r)
if err != nil {
return
}
s.Values, err = bin.ReadN(r, valuesSize)
if err != nil {
return
}
if len(s.DataPages) == 0 {
return
}
indexLevelsQty, err := bin.ReadVarSize(r)
if err != nil {
return
}
for range indexLevelsQty {
var (
indexPagesQty int
level IndexLevel
recordsQty int
)
indexPagesQty, err = bin.ReadVarSize(r)
if err != nil {
return
}
for range indexPagesQty {
var (
p IndexPage
recordsQty int
)
p.PageNo, err = bin.ReadUint32(r)
if err != nil {
return
}
p.Reused, err = bin.ReadBool(r)
if err != nil {
return
}
p.Checksum, err = bin.ReadUint32(r)
if err != nil {
return
}
recordsQty, err = bin.ReadVarSize(r)
if err != nil {
return
}
p.Records, err = bin.ReadN(r, recordsQty*recordSize)
if err != nil {
return
}
level.Pages = append(level.Pages, p)
}
recordsQty, err = bin.ReadVarSize(r)
if err != nil {
return
}
level.Records, err = bin.ReadN(r, recordsQty*recordSize)
if err != nil {
return
}
s.IndexLevels = append(s.IndexLevels, level)
}
return
}

36
txlog/reader.go
View File

@@ -113,26 +113,26 @@ func (s *Reader) parseRecords(body []byte) ([]any, error) {
}
records = append(records, rec)
case CodeAppendedMeasure:
rec := new(AppendedMeasure)
if err = rec.Parse(src); err != nil {
return nil, err
}
records = append(records, rec)
// case CodeAppendedMeasure:
// rec := new(AppendedMeasure)
// if err = rec.Parse(src); err != nil {
// return nil, err
// }
// records = append(records, rec)
case CodeAppendedMeasures:
rec := new(AppendedMeasures)
if err = rec.Parse(src); err != nil {
return nil, err
}
records = append(records, rec)
// case CodeAppendedMeasures:
// rec := new(AppendedMeasures)
// if err = rec.Parse(src); err != nil {
// return nil, err
// }
// records = append(records, rec)
case CodeAppendedPages:
rec := new(AppendedPages)
if err = rec.Parse(src); err != nil {
return nil, err
}
records = append(records, rec)
// case CodeAppendedPages:
// rec := new(AppendedPages)
// if err = rec.Parse(src); err != nil {
// return nil, err
// }
// records = append(records, rec)
case CodeDeletedMeasures:
rec := new(DeletedMeasures)

340
txlog/txlog.go
View File

@@ -5,9 +5,7 @@ import (
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/conbuf"
"gordenko.dev/dima/qb/proto"
)
type AddedMetric struct {
@@ -77,93 +75,93 @@ func (s *DeletedMetric) Parse(src io.Reader) (err error) {
return nil
}
type AppendedMeasure struct {
MetricID uint32
Timestamp uint32
Value float64
}
// type AppendedMeasure struct {
// MetricID uint32
// Timestamp uint32
// Value float64
// }
func (s AppendedMeasure) Pack(w io.Writer) {
arr := []byte{
CodeAppendedMeasure,
0, 0, 0, 0, // metricID
0, 0, 0, 0, // timestamp
0, 0, 0, 0, 0, 0, 0, 0, // value
}
bin.PutUint32(arr[1:], s.MetricID)
bin.PutUint32(arr[5:], s.Timestamp)
bin.PutFloat64(arr[9:], s.Value)
w.Write(arr)
}
// func (s AppendedMeasure) Pack(w io.Writer) {
// arr := []byte{
// CodeAppendedMeasure,
// 0, 0, 0, 0, // metricID
// 0, 0, 0, 0, // timestamp
// 0, 0, 0, 0, 0, 0, 0, 0, // value
// }
// bin.PutUint32(arr[1:], s.MetricID)
// bin.PutUint32(arr[5:], s.Timestamp)
// bin.PutFloat64(arr[9:], s.Value)
// w.Write(arr)
// }
func (s *AppendedMeasure) Parse(src io.Reader) (err error) {
arr, err := bin.ReadN(src, 16)
if err != nil {
return
}
s.MetricID, _ = bin.GetUint32(arr[0:])
s.Timestamp, _ = bin.GetUint32(arr[4:])
s.Value, _ = bin.GetFloat64(arr[8:])
return nil
}
// func (s *AppendedMeasure) Parse(src io.Reader) (err error) {
// arr, err := bin.ReadN(src, 16)
// if err != nil {
// return
// }
// s.MetricID, _ = bin.GetUint32(arr[0:])
// s.Timestamp, _ = bin.GetUint32(arr[4:])
// s.Value, _ = bin.GetFloat64(arr[8:])
// return nil
// }
type AppendedMeasures struct {
MetricID uint32
Measures []proto.Measure
}
// type AppendedMeasures struct {
// MetricID uint32
// Measures []proto.Measure
// }
func (s AppendedMeasures) Pack(w io.Writer) {
arr := []byte{
CodeAppendedMeasures,
0, 0, 0, 0, // metricID
0, 0, // qty
}
bin.PutUint32(arr[1:], s.MetricID)
bin.PutUint16(arr[5:], uint16(len(s.Measures)))
w.Write(arr)
for _, measure := range s.Measures {
bin.WriteUint32(w, measure.Timestamp)
bin.WriteFloat64(w, measure.Value)
}
}
// func (s AppendedMeasures) Pack(w io.Writer) {
// arr := []byte{
// CodeAppendedMeasures,
// 0, 0, 0, 0, // metricID
// 0, 0, // qty
// }
// bin.PutUint32(arr[1:], s.MetricID)
// bin.PutUint16(arr[5:], uint16(len(s.Measures)))
// w.Write(arr)
// for _, measure := range s.Measures {
// bin.WriteUint32(w, measure.Timestamp)
// bin.WriteFloat64(w, measure.Value)
// }
// }
func (s *AppendedMeasures) Parse(src io.Reader) (err error) {
s.MetricID, err = bin.ReadUint32(src)
if err != nil {
return
}
qty, err := bin.ReadUint16(src)
if err != nil {
return
}
for range qty {
var measure proto.Measure
measure.Timestamp, err = bin.ReadUint32(src)
if err != nil {
return
}
measure.Value, err = bin.ReadFloat64(src)
if err != nil {
return
}
s.Measures = append(s.Measures, measure)
}
return nil
}
// func (s *AppendedMeasures) Parse(src io.Reader) (err error) {
// s.MetricID, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// qty, err := bin.ReadUint16(src)
// if err != nil {
// return
// }
// for range qty {
// var measure proto.Measure
// measure.Timestamp, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// measure.Value, err = bin.ReadFloat64(src)
// if err != nil {
// return
// }
// s.Measures = append(s.Measures, measure)
// }
// return nil
// }
type AppendedPagesReq struct {
Legs []atree.PathLeg
MetricID uint32
Timestamp uint32 // last measure
Value float64 // last measure
//RootPageNo uint32
LastPageNo uint32
Pages []atree.NotLinkedDataPage
TimestampsChunks [][]byte
TimestampsSize uint16
ValuesChunks [][]byte
ValuesSize uint16
}
// type AppendedPagesReq struct {
// Legs []atree.PathLeg
// MetricID uint32
// Timestamp uint32 // last measure
// Value float64 // last measure
// //RootPageNo uint32
// LastPageNo uint32
// Pages []atree.NotLinkedDataPage
// TimestampsChunks [][]byte
// TimestampsSize uint16
// ValuesChunks [][]byte
// ValuesSize uint16
// }
/*
4b metricID
@@ -184,96 +182,96 @@ Nb - timestamps payload
2b - values size (not filled page)
Nb - values payload
*/
type AppendedPages struct {
MetricID uint32
Timestamp uint32 // last measure
Value float64 // last measure
NewRootPageNo uint32
LastPageNo uint32
Pages []atree.PageToWrite
TimestampsChunks [][]byte
TimestampsSize int
ValuesChunks [][]byte
ValuesSize int
}
// type AppendedPages struct {
// MetricID uint32
// Timestamp uint32 // last measure
// Value float64 // last measure
// NewRootPageNo uint32
// LastPageNo uint32
// Pages []atree.PageToWrite
// TimestampsChunks [][]byte
// TimestampsSize int
// ValuesChunks [][]byte
// ValuesSize int
// }
func (s *AppendedPages) Pack(w io.Writer) {
w.Write([]byte{
CodeAppendedPages,
})
bin.WriteUint32(w, s.MetricID)
bin.WriteUint32(w, s.Timestamp)
bin.WriteFloat64(w, s.Value)
bin.WriteUint32(w, s.LastPageNo)
bin.WriteUint32(w, s.NewRootPageNo)
//
bin.WriteVarSize(w, len(s.Pages))
for _, p := range s.Pages {
bin.WriteUint32(w, p.PageNo)
if p.IsReused {
w.Write([]byte{1})
} else {
w.Write([]byte{0})
}
w.Write(p.Data)
}
// timestamps
writeChunkedPayloadTo(s.TimestampsChunks, int(s.TimestampsSize), w)
// values
writeChunkedPayloadTo(s.ValuesChunks, int(s.ValuesSize), w)
}
// func (s *AppendedPages) Pack(w io.Writer) {
// w.Write([]byte{
// CodeAppendedPages,
// })
// bin.WriteUint32(w, s.MetricID)
// bin.WriteUint32(w, s.Timestamp)
// bin.WriteFloat64(w, s.Value)
// bin.WriteUint32(w, s.LastPageNo)
// bin.WriteUint32(w, s.NewRootPageNo)
// //
// bin.WriteVarSize(w, len(s.Pages))
// for _, p := range s.Pages {
// bin.WriteUint32(w, p.PageNo)
// if p.IsReused {
// w.Write([]byte{1})
// } else {
// w.Write([]byte{0})
// }
// w.Write(p.Data)
// }
// // timestamps
// writeChunkedPayloadTo(s.TimestampsChunks, int(s.TimestampsSize), w)
// // values
// writeChunkedPayloadTo(s.ValuesChunks, int(s.ValuesSize), w)
// }
func (s *AppendedPages) Parse(src io.Reader) (err error) {
s.MetricID, err = bin.ReadUint32(src)
if err != nil {
return
}
s.Timestamp, err = bin.ReadUint32(src)
if err != nil {
return
}
s.Value, err = bin.ReadFloat64(src)
if err != nil {
return
}
s.LastPageNo, err = bin.ReadUint32(src)
if err != nil {
return
}
s.NewRootPageNo, err = bin.ReadUint32(src)
if err != nil {
return
}
pagesQty, err := bin.ReadVarSize(src)
if err != nil {
return
}
for range pagesQty {
var page atree.PageToWrite
page.PageNo, err = bin.ReadUint32(src)
if err != nil {
return
}
page.IsReused, err = bin.ReadBool(src)
if err != nil {
return
}
page.Data, err = bin.ReadN(src, atree.PageSize)
if err != nil {
return
}
s.Pages = append(s.Pages, page)
}
s.TimestampsChunks, s.TimestampsSize, err = readChunkedPayload(src)
if err != nil {
return
}
s.ValuesChunks, s.ValuesSize, err = readChunkedPayload(src)
if err != nil {
return
}
return nil
}
// func (s *AppendedPages) Parse(src io.Reader) (err error) {
// s.MetricID, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// s.Timestamp, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// s.Value, err = bin.ReadFloat64(src)
// if err != nil {
// return
// }
// s.LastPageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// s.NewRootPageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// pagesQty, err := bin.ReadVarSize(src)
// if err != nil {
// return
// }
// for range pagesQty {
// var page atree.PageToWrite
// page.PageNo, err = bin.ReadUint32(src)
// if err != nil {
// return
// }
// page.IsReused, err = bin.ReadBool(src)
// if err != nil {
// return
// }
// page.Data, err = bin.ReadN(src, atree.PageSize)
// if err != nil {
// return
// }
// s.Pages = append(s.Pages, page)
// }
// s.TimestampsChunks, s.TimestampsSize, err = readChunkedPayload(src)
// if err != nil {
// return
// }
// s.ValuesChunks, s.ValuesSize, err = readChunkedPayload(src)
// if err != nil {
// return
// }
// return nil
// }
func writeChunkedPayloadTo(chunks [][]byte, size int, w io.Writer) {
bin.WriteVarSize(w, size)

304
txlog/writer.go
View File

@@ -5,12 +5,13 @@ import (
"errors"
"fmt"
"hash/crc32"
"log"
"os"
"path/filepath"
"sync"
bin "gordenko.dev/dima/bin/little"
octopus "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
)
@@ -38,7 +39,7 @@ const (
type FreeList interface {
// використовується в allocPage
ReservePage() uint32
GetPageNumber() (uint32, error)
}
func JoinChangesFileName(dir string, logNumber int) string {
@@ -59,20 +60,24 @@ type Writer struct {
atree *atree.Atree
logNumber int
dir string
w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
wal *os.File
dataFile *os.File
file *os.File
indexFile *os.File
dataPage []byte
indexPage []byte
input []any
//buf *bytes.Buffer
pagesToWrite []atree.PageToWrite
workerReqs []any
waitCh chan struct{}
//pagesToWrite []atree.PageToWrite
workerReqs []any
//waitCh chan struct{}
appendToWorkerQueue func(any)
lsn uint32
written int64
isExited bool
exitCh chan struct{}
waitGroup *sync.WaitGroup
signalCh chan struct{}
//lsn uint32
written int64
isExited bool
exitCh chan struct{}
waitGroup *sync.WaitGroup
signalCh chan struct{}
}
type WriterOptions struct {
@@ -114,13 +119,12 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
exitCh: opt.ExitCh,
waitGroup: opt.WaitGroup,
signalCh: make(chan struct{}, 1),
waitCh: make(chan struct{}),
}
var err error
if opt.LogNumber > 0 {
s.file, err = os.OpenFile(
s.wal, err = os.OpenFile(
JoinChangesFileName(opt.Dir, s.logNumber),
os.O_APPEND|os.O_WRONLY,
filePerm,
@@ -130,7 +134,7 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
}
} else {
s.logNumber = 1
s.file, err = os.OpenFile(
s.wal, err = os.OpenFile(
JoinChangesFileName(opt.Dir, s.logNumber),
os.O_CREATE|os.O_WRONLY,
filePerm,
@@ -147,7 +151,7 @@ func (s *Writer) Run() {
select {
case <-s.signalCh:
if err := s.packAndWrite(); err != nil {
octopus.Abort(octopus.FailedWriteToTxLog, err)
qb.Abort(qb.FailedWriteToTxLog, err)
}
case <-s.exitCh:
@@ -157,48 +161,67 @@ func (s *Writer) Run() {
}
}
func (s *Writer) packAndWrite() error {
func (s *Writer) packAndWrite() (err error) {
s.mutex.Lock()
input := s.input
waitCh := s.waitCh
s.input = nil
s.waitCh = make(chan struct{})
s.mutex.Unlock()
w := bytes.NewBuffer(nil)
// 1. Пакую всі дані в WAL буфер запису
for _, untyped := range input {
switch x := untyped.(type) {
case AppendedPagesReq:
s.packWriteAppended(w, x)
case AddedMetric:
case DeletedMetric:
if len(x.FreePageNumbers) > 0 {
//s.freeList.AddPages(x.FreePageNumbers)
}
case AppendedMeasure:
x.Pack(w)
// case DeletedMetric:
// if len(x.FreePageNumbers) > 0 {
// s.freeList.AddPageNumbers(x.FreePageNumbers)
// }
case AppendedMeasures:
case DeletedMeasures:
//
s.packMeasuresIntoWALBuffer(x)
//case DeletedMeasures:
//case DeletedMeasuresSince:
}
}
// 2. Додаю розмір пакету і чексуму
// 3. Пишу на диск WAL (append)
// 4. Пишу в atree сторінки
for range 1 {
err = s.writePagesToAtree(nil, nil)
if err != nil {
return
}
}
// 5. Пишу зміни в free list
// 6. відправляю input - worker-у
// flush
close(waitCh)
return s.flush(w)
err = s.flush(w)
if err != nil {
return err
}
// FIX send to worker workerReqs
return nil
}
func (s *Writer) Append(req any) chan struct{} {
func (s *Writer) Append(req any) {
s.mutex.Lock()
s.input = append(s.input, req)
s.workerReqs = append(s.workerReqs, req)
waitCh := s.waitCh
s.mutex.Unlock()
select {
case s.signalCh <- struct{}{}:
default:
}
return waitCh
}
// func (s *Writer) reset() {
@@ -217,9 +240,8 @@ func (s *Writer) Append(req any) chan struct{} {
func (s *Writer) flush(w *bytes.Buffer) error {
s.mutex.Lock()
pagesToWrite := s.pagesToWrite
//pagesToWrite := s.pagesToWrite
workerReqs := s.workerReqs
waitCh := s.waitCh
isExited := s.isExited
var exitWaitGroup *sync.WaitGroup
@@ -228,8 +250,8 @@ func (s *Writer) flush(w *bytes.Buffer) error {
}
if w.Len() > packetPrefixSize {
s.lsn++
lsn := s.lsn
//s.lsn++
//lsn := s.lsn
packet := make([]byte, w.Len())
copy(packet, w.Bytes())
//s.reset() fix
@@ -238,26 +260,25 @@ func (s *Writer) flush(w *bytes.Buffer) error {
s.mutex.Unlock()
bin.PutUint32(packet[lengthIdx:], uint32(len(packet)-packetPrefixSize))
bin.PutUint32(packet[lsnIdx:], lsn)
//bin.PutUint32(packet[lsnIdx:], lsn)
bin.PutUint32(packet[checksumIdx:], crc32.ChecksumIEEE(packet[8:]))
n, err := s.file.Write(packet)
n, err := s.wal.Write(packet)
if err != nil {
return fmt.Errorf("TxLog write: %s", err)
}
if n != len(packet) {
return fmt.Errorf("TxLog written %d != packet size %d", n, len(packet))
}
if err := s.file.Sync(); err != nil {
if err := s.wal.Sync(); err != nil {
return fmt.Errorf("TxLog sync: %s", err)
}
err = s.writePagesToAtree(pagesToWrite)
if err != nil {
return fmt.Errorf("TxLog writePagesToAtree: %s", err)
}
// err = s.writePagesToAtree(pagesToWrite)
// if err != nil {
// return fmt.Errorf("TxLog writePagesToAtree: %s", err)
// }
} else {
s.waitCh = make(chan struct{})
s.mutex.Unlock()
}
@@ -272,12 +293,12 @@ func (s *Writer) flush(w *bytes.Buffer) error {
}
if forceSnapshot {
if err := s.file.Close(); err != nil {
if err := s.wal.Close(); err != nil {
return fmt.Errorf("close changes file: %s", err)
}
s.logNumber++
var err error
s.file, err = os.OpenFile(
s.wal, err = os.OpenFile(
JoinChangesFileName(s.dir, s.logNumber),
os.O_CREATE|os.O_WRONLY,
filePerm,
@@ -293,41 +314,146 @@ func (s *Writer) flush(w *bytes.Buffer) error {
Records: workerReqs,
ForceSnapshot: forceSnapshot,
LogNumber: s.logNumber,
WaitCh: waitCh,
ExitWaitGroup: exitWaitGroup,
})
return nil
}
func (s *Writer) writePagesToAtree(pages []atree.PageToWrite) error {
for _, p := range pages {
if len(p.Data) != atree.PageSize {
return fmt.Errorf("wrong page %d size: %d",
p.PageNo, len(p.Data))
}
off := (p.PageNo - 1) * atree.PageSize
n, err := s.dataFile.WriteAt(p.Data, int64(off))
if err != nil {
return err
}
if n != len(p.Data) {
return fmt.Errorf("write %d instead of %d", n, len(p.Data))
}
}
return nil
}
func (s *Writer) exit() {
s.mutex.Lock()
s.isExited = true
s.mutex.Unlock()
if err := s.packAndWrite(); err != nil {
octopus.Abort(octopus.FailedWriteToTxLog, err)
qb.Abort(qb.FailedWriteToTxLog, err)
}
}
// writePagesToAtree - записує сторінки в .data та .index файли
func (s *Writer) writePagesToAtree(levels []*atree.IndexLevel, dataPages []*atree.DataPage) (err error) {
for _, p := range dataPages {
p.Checksum = atree.ChunksToDataPage(s.dataPage, atree.ChunksToDataPageReq{
PrevPageNo: p.PrevPageNo,
Timestamps: p.Timestamps,
TimestampsSize: p.TimestampsSize,
Values: p.Values,
ValuesSize: p.ValuesSize,
})
off := (p.PageNo - 1) * atree.DataPageSize
n, err := s.dataFile.WriteAt(s.dataPage, int64(off))
if err != nil {
return err
}
if n != atree.DataPageSize {
return fmt.Errorf("write %d instead of %d", n, atree.DataPageSize)
}
}
for levelIdx, level := range levels {
for _, p := range level.Filled {
// if len(p.Data) != atree.PageSize {
// return fmt.Errorf("wrong page %d size: %d",
// p.PageNo, len(p.Data))
// }
p.Checksum = atree.DataToIndexPage(s.indexPage, p.Data, levelIdx == 0)
off := (p.PageNo - 1) * atree.IndexPageSize
n, err := s.indexFile.WriteAt(s.indexPage, int64(off))
if err != nil {
return err
}
if n != atree.IndexPageSize {
return fmt.Errorf("write %d instead of %d", n, atree.IndexPageSize)
}
}
}
return nil
}
type AppendedMeasures struct {
MetricID uint32
TimestampsOffset int // (заповнені одразу)
ValuesOffset int // (заповнені одразу)
Timestamps [][]byte
TimestampsSize int
Values [][]byte
ValuesSize int
IndexLevels []*atree.IndexLevel
DataPages []*atree.DataPage // fix - prevPageNo for the 1st data page
}
func (s *Writer) packMeasuresIntoWALBuffer(req AppendedMeasures) (err error) {
// fix write code
bin.WriteUint32(s.w, req.MetricID)
bin.WriteVarSize(s.w, len(req.DataPages))
for idx, p := range req.DataPages {
bin.WriteUint32(s.w, p.PageNo)
var reused byte
if p.IsReused {
reused = 1
}
s.w.WriteByte(reused)
bin.WriteUint32(s.w, p.PrevPageNo)
bin.WriteUint32(s.w, p.Checksum)
var (
timestampsOffset int
valuesOffset int
)
if idx == 0 {
timestampsOffset = req.TimestampsOffset
valuesOffset = req.ValuesOffset
}
copyPayloadFromChunks(s.w, p.Timestamps, p.TimestampsSize, timestampsOffset)
copyPayloadFromChunks(s.w, p.Values, p.ValuesSize, valuesOffset)
}
bin.WriteUint16(s.w, uint16(req.TimestampsOffset))
bin.WriteUint16(s.w, uint16(req.ValuesOffset))
var (
timestampsOffset int
valuesOffset int
)
if len(req.DataPages) == 0 {
timestampsOffset = req.TimestampsOffset
valuesOffset = req.ValuesOffset
}
copyPayloadFromChunks(s.w, req.Timestamps, req.TimestampsSize, timestampsOffset)
copyPayloadFromChunks(s.w, req.Values, req.ValuesSize, valuesOffset)
if len(req.DataPages) == 0 {
return
}
// fix - levels reduced
bin.WriteVarSize(s.w, len(req.IndexLevels))
for _, level := range req.IndexLevels {
bin.WriteVarSize(s.w, len(level.Filled))
for idx, p := range level.Filled {
bin.WriteUint32(s.w, p.PageNo)
var reused byte
if p.IsReused {
reused = 1
}
s.w.WriteByte(reused)
bin.WriteUint32(s.w, p.Checksum)
records := p.Data
if idx == 0 {
records = p.Data[level.Offset:]
}
if (len(records) % recordSize) != 0 {
log.Fatalf("wrong records length: %d", len(records))
}
bin.WriteVarSize(s.w, len(records)/recordSize)
s.w.Write(records)
}
records := level.Data
if len(level.Filled) == 0 {
records = level.Data[level.Offset:]
}
if (len(records) % recordSize) != 0 {
log.Fatalf("wrong unfilled records length: %d", len(records))
}
bin.WriteVarSize(s.w, len(records)/recordSize)
s.w.Write(records)
}
return
}
// API
// FIX - add
@@ -351,28 +477,28 @@ func (s *Writer) sendSignal() {
// Якщо (Pages) > 0 - незаповнену сторінку кодую в стисненому вигляді.
// Якщо (Pages) = 0 - кодую просто пари timestamp / value
func (s *Writer) packWriteAppended(w *bytes.Buffer, req AppendedPagesReq) {
// завантажений path. Отже просто додаємо в дерево data сторінки, створюємо нові індексні
// без звернення до диску.
report := s.atree.AppendDataPages(atree.AppendDataPagesReq{
LastPageNo: req.LastPageNo,
Legs: req.Legs,
DataPages: req.Pages,
})
// func (s *Writer) packWriteAppended(w *bytes.Buffer, req AppendedPagesReq) {
// // завантажений path. Отже просто додаємо в дерево data сторінки, створюємо нові індексні
// // без звернення до диску.
// report := s.atree.AppendDataPages(atree.AppendDataPagesReq{
// LastPageNo: req.LastPageNo,
// Legs: req.Legs,
// DataPages: req.Pages,
// })
m := AppendedPages{
MetricID: req.MetricID,
Timestamp: req.Timestamp,
Value: req.Value,
NewRootPageNo: report.NewRootPageNo,
LastPageNo: report.LastPageNo,
Pages: report.Pages,
TimestampsChunks: req.TimestampsChunks,
TimestampsSize: int(req.TimestampsSize),
ValuesChunks: req.ValuesChunks,
ValuesSize: int(req.ValuesSize),
}
m.Pack(w)
}
// m := AppendedPages{
// MetricID: req.MetricID,
// Timestamp: req.Timestamp,
// Value: req.Value,
// NewRootPageNo: report.NewRootPageNo,
// LastPageNo: report.LastPageNo,
// Pages: report.Pages,
// TimestampsChunks: req.TimestampsChunks,
// TimestampsSize: int(req.TimestampsSize),
// ValuesChunks: req.ValuesChunks,
// ValuesSize: int(req.ValuesSize),
// }
// m.Pack(w)
// }
// helpers

233
txlog/x.go
View File

@@ -1,233 +0,0 @@
package txlog
import (
"bytes"
"fmt"
"log"
"os"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb/atree"
)
const recordSize = 8
type LogWriter struct {
w *bytes.Buffer
wal *os.File
dataFile *os.File
indexFile *os.File
dataPage []byte
indexPage []byte
}
func (s *LogWriter) writePagesToAtree(levels []*atree.IndexLevel, dataPages []*atree.DataPage) (err error) {
for _, p := range dataPages {
atree.ChunksToDataPage(s.dataPage, atree.ChunksToDataPageReq{
PrevPageNo: p.PrevPageNo,
Timestamps: p.Timestamps,
TimestampsSize: p.TimestampsSize,
Values: p.Values,
ValuesSize: p.ValuesSize,
})
off := (p.PageNo - 1) * atree.DataPageSize
n, err := s.dataFile.WriteAt(s.dataPage, int64(off))
if err != nil {
return err
}
if n != atree.DataPageSize {
return fmt.Errorf("write %d instead of %d", n, atree.DataPageSize)
}
}
for levelIdx, level := range levels {
for _, p := range level.Filled {
// if len(p.Data) != atree.PageSize {
// return fmt.Errorf("wrong page %d size: %d",
// p.PageNo, len(p.Data))
// }
atree.DataToIndexPage(s.indexPage, p.Data, levelIdx == 0)
off := (p.PageNo - 1) * atree.IndexPageSize
n, err := s.indexFile.WriteAt(s.indexPage, int64(off))
if err != nil {
return err
}
if n != atree.IndexPageSize {
return fmt.Errorf("write %d instead of %d", n, atree.IndexPageSize)
}
}
}
return nil
}
type ToWrite struct {
MetricID uint32
TimestampsOffset int // (заповнені одразу)
ValuesOffset int // (заповнені одразу)
Timestamps [][]byte
TimestampsSize int
Values [][]byte
ValuesSize int
IndexLevels []*atree.IndexLevel
DataPages []*atree.DataPage // fix - prevPageNo for the 1st data page
}
type DataPage struct {
PageNo uint32
Reused bool
PrevPageNo uint32
Checksum uint32
Timestamps []byte
Values []byte
}
type IndexPage struct {
PageNo uint32
Reused bool
Checksum uint32
Records []byte
}
type IndexLevel struct {
Pages []IndexPage
// offset потрібен тому що дані не додаються в кінець, а перезаписують кілька
// останніх байтів unfilled даних. Потрібно для коректного recovery.
// Якщо є Pages, то застосовуються для 1-ї Index сторінки, інакше - до records (unfilled)
RecordsOffset int
Records []byte // unfilled
}
type TxAppendedMeasures struct {
MetricID uint32
DataPages []DataPage
// offsets потрібні тому що дані не додаються в кінець, а перезаписують кілька
// останніх байтів unfilled даних. Потрібно для коректного recovery.
// Якщо є DataPages, то застосовуються для 1-ї Data сторінки, інакше - до timestamps і values (unfilled)
TimestampsOffset int
ValuesOffset int
Timestamps []byte // unfilled
Values []byte // unfilled
IndexLevels []DataPage
}
/*
Format appended measures:
1b - tx type
4b - metricID
Nb - qty of filled data pages (varsize)
[
4b - pageNo
1b - reused
4b - prevPageNo
4b - page crc32
Nb - timestamps size (varsize)
Nb - timestamps payload
Nb - values size (varsize)
Nb - values payload
]
2b - timestamps offset (for the 1st data page only)
2b - values offset (for the 1st data page only)
Nb - timestamps size unfilled (varsize)
Nb - timestamps payload unfilled
Nb - values size unfilled (varsize)
Nb - values payload unfilled
Nb - qty of index levels (varsize)
[
Nb - qty of level filled pages (varsize)
[
4b - pageNo
1b - reused
4b - page crc32
Nb - records qty (varsize)
Nb - records payload
]
Nb - records offset (for the 1st index page only)
Nb - records qty unfilled (varsize)
Nb - records payload unfilled
]
*/
func (s *LogWriter) writeToWAL(req ToWrite) (err error) {
// fix write code
bin.WriteUint32(s.w, req.MetricID)
bin.WriteVarSize(s.w, len(req.DataPages))
for idx, p := range req.DataPages {
bin.WriteUint32(s.w, p.PageNo)
var reused byte
if p.IsReused {
reused = 1
}
s.w.WriteByte(reused)
bin.WriteUint32(s.w, p.PrevPageNo)
//bin.WriteUint32(s.w, p.CRC32) fix
var (
timestampsOffset int
valuesOffset int
)
if idx == 0 {
timestampsOffset = req.TimestampsOffset
valuesOffset = req.ValuesOffset
}
copyPayloadFromChunks(s.w, p.Timestamps, p.TimestampsSize, timestampsOffset)
copyPayloadFromChunks(s.w, p.Values, p.ValuesSize, valuesOffset)
}
bin.WriteUint16(s.w, uint16(req.TimestampsOffset))
bin.WriteUint16(s.w, uint16(req.ValuesOffset))
// fix offset
copyPayloadFromChunks(s.w, req.Timestamps, req.TimestampsSize, req.TimestampsOffset)
copyPayloadFromChunks(s.w, req.Values, req.ValuesSize, req.ValuesOffset)
if len(req.DataPages) == 0 {
return
}
bin.WriteVarSize(s.w, len(req.IndexLevels))
for _, level := range req.IndexLevels {
bin.WriteVarSize(s.w, len(level.Filled))
for idx, p := range level.Filled {
bin.WriteUint32(s.w, p.PageNo)
var reused byte
if p.IsReused {
reused = 1
}
s.w.WriteByte(reused)
//bin.WriteUint32(s.w, p.CRC32) fix
records := p.Data
if idx == 0 {
records = p.Data[level.Offset:]
}
if (len(records) % recordSize) != 0 {
log.Fatalf("wrong records length: %d", len(records))
}
bin.WriteVarSize(s.w, len(records)/recordSize)
s.w.Write(records)
}
if len(level.Filled) > 0 {
s.w.Write(level.Data)
} else {
s.w.Write(level.Data[level.Offset:])
}
}
return
}
const chunkSize = 24
func copyPayloadFromChunks(w *bytes.Buffer, chunks [][]byte, size int, offset int) {
bin.WriteVarSize(w, size)
var (
chunkIdx = offset / chunkSize
byteIdx = offset % chunkSize
)
for _, chunk := range chunks[chunkIdx:] {
available := len(chunk) - byteIdx
if available <= size {
w.Write(chunk[byteIdx:])
size -= available
if size == 0 {
return
}
byteIdx = 0
} else {
w.Write(chunk[byteIdx : byteIdx+size])
return
}
}
}