dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
Dima Gordenko
2026-05-21 23:38:52 +03:00
parent 793df29e11
commit e7ec6083cb
21 changed files with 1234 additions and 1017 deletions
Download Patch File Download Diff File Expand all files Collapse all files

304
txlog/writer.go
View File

@@ -5,12 +5,13 @@ import (
"errors"
"fmt"
"hash/crc32"
"log"
"os"
"path/filepath"
"sync"
bin "gordenko.dev/dima/bin/little"
octopus "gordenko.dev/dima/qb"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
)
@@ -38,7 +39,7 @@ const (
type FreeList interface {
// використовується в allocPage
ReservePage() uint32
GetPageNumber() (uint32, error)
}
func JoinChangesFileName(dir string, logNumber int) string {
@@ -59,20 +60,24 @@ type Writer struct {
atree *atree.Atree
logNumber int
dir string
w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
wal *os.File
dataFile *os.File
file *os.File
indexFile *os.File
dataPage []byte
indexPage []byte
input []any
//buf *bytes.Buffer
pagesToWrite []atree.PageToWrite
workerReqs []any
waitCh chan struct{}
//pagesToWrite []atree.PageToWrite
workerReqs []any
//waitCh chan struct{}
appendToWorkerQueue func(any)
lsn uint32
written int64
isExited bool
exitCh chan struct{}
waitGroup *sync.WaitGroup
signalCh chan struct{}
//lsn uint32
written int64
isExited bool
exitCh chan struct{}
waitGroup *sync.WaitGroup
signalCh chan struct{}
}
type WriterOptions struct {
@@ -114,13 +119,12 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
exitCh: opt.ExitCh,
waitGroup: opt.WaitGroup,
signalCh: make(chan struct{}, 1),
waitCh: make(chan struct{}),
}
var err error
if opt.LogNumber > 0 {
s.file, err = os.OpenFile(
s.wal, err = os.OpenFile(
JoinChangesFileName(opt.Dir, s.logNumber),
os.O_APPEND|os.O_WRONLY,
filePerm,
@@ -130,7 +134,7 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
}
} else {
s.logNumber = 1
s.file, err = os.OpenFile(
s.wal, err = os.OpenFile(
JoinChangesFileName(opt.Dir, s.logNumber),
os.O_CREATE|os.O_WRONLY,
filePerm,
@@ -147,7 +151,7 @@ func (s *Writer) Run() {
select {
case <-s.signalCh:
if err := s.packAndWrite(); err != nil {
octopus.Abort(octopus.FailedWriteToTxLog, err)
qb.Abort(qb.FailedWriteToTxLog, err)
}
case <-s.exitCh:
@@ -157,48 +161,67 @@ func (s *Writer) Run() {
}
}
func (s *Writer) packAndWrite() error {
func (s *Writer) packAndWrite() (err error) {
s.mutex.Lock()
input := s.input
waitCh := s.waitCh
s.input = nil
s.waitCh = make(chan struct{})
s.mutex.Unlock()
w := bytes.NewBuffer(nil)
// 1. Пакую всі дані в WAL буфер запису
for _, untyped := range input {
switch x := untyped.(type) {
case AppendedPagesReq:
s.packWriteAppended(w, x)
case AddedMetric:
case DeletedMetric:
if len(x.FreePageNumbers) > 0 {
//s.freeList.AddPages(x.FreePageNumbers)
}
case AppendedMeasure:
x.Pack(w)
// case DeletedMetric:
// if len(x.FreePageNumbers) > 0 {
// s.freeList.AddPageNumbers(x.FreePageNumbers)
// }
case AppendedMeasures:
case DeletedMeasures:
//
s.packMeasuresIntoWALBuffer(x)
//case DeletedMeasures:
//case DeletedMeasuresSince:
}
}
// 2. Додаю розмір пакету і чексуму
// 3. Пишу на диск WAL (append)
// 4. Пишу в atree сторінки
for range 1 {
err = s.writePagesToAtree(nil, nil)
if err != nil {
return
}
}
// 5. Пишу зміни в free list
// 6. відправляю input - worker-у
// flush
close(waitCh)
return s.flush(w)
err = s.flush(w)
if err != nil {
return err
}
// FIX send to worker workerReqs
return nil
}
func (s *Writer) Append(req any) chan struct{} {
func (s *Writer) Append(req any) {
s.mutex.Lock()
s.input = append(s.input, req)
s.workerReqs = append(s.workerReqs, req)
waitCh := s.waitCh
s.mutex.Unlock()
select {
case s.signalCh <- struct{}{}:
default:
}
return waitCh
}
// func (s *Writer) reset() {
@@ -217,9 +240,8 @@ func (s *Writer) Append(req any) chan struct{} {
func (s *Writer) flush(w *bytes.Buffer) error {
s.mutex.Lock()
pagesToWrite := s.pagesToWrite
//pagesToWrite := s.pagesToWrite
workerReqs := s.workerReqs
waitCh := s.waitCh
isExited := s.isExited
var exitWaitGroup *sync.WaitGroup
@@ -228,8 +250,8 @@ func (s *Writer) flush(w *bytes.Buffer) error {
}
if w.Len() > packetPrefixSize {
s.lsn++
lsn := s.lsn
//s.lsn++
//lsn := s.lsn
packet := make([]byte, w.Len())
copy(packet, w.Bytes())
//s.reset() fix
@@ -238,26 +260,25 @@ func (s *Writer) flush(w *bytes.Buffer) error {
s.mutex.Unlock()
bin.PutUint32(packet[lengthIdx:], uint32(len(packet)-packetPrefixSize))
bin.PutUint32(packet[lsnIdx:], lsn)
//bin.PutUint32(packet[lsnIdx:], lsn)
bin.PutUint32(packet[checksumIdx:], crc32.ChecksumIEEE(packet[8:]))
n, err := s.file.Write(packet)
n, err := s.wal.Write(packet)
if err != nil {
return fmt.Errorf("TxLog write: %s", err)
}
if n != len(packet) {
return fmt.Errorf("TxLog written %d != packet size %d", n, len(packet))
}
if err := s.file.Sync(); err != nil {
if err := s.wal.Sync(); err != nil {
return fmt.Errorf("TxLog sync: %s", err)
}
err = s.writePagesToAtree(pagesToWrite)
if err != nil {
return fmt.Errorf("TxLog writePagesToAtree: %s", err)
}
// err = s.writePagesToAtree(pagesToWrite)
// if err != nil {
// return fmt.Errorf("TxLog writePagesToAtree: %s", err)
// }
} else {
s.waitCh = make(chan struct{})
s.mutex.Unlock()
}
@@ -272,12 +293,12 @@ func (s *Writer) flush(w *bytes.Buffer) error {
}
if forceSnapshot {
if err := s.file.Close(); err != nil {
if err := s.wal.Close(); err != nil {
return fmt.Errorf("close changes file: %s", err)
}
s.logNumber++
var err error
s.file, err = os.OpenFile(
s.wal, err = os.OpenFile(
JoinChangesFileName(s.dir, s.logNumber),
os.O_CREATE|os.O_WRONLY,
filePerm,
@@ -293,41 +314,146 @@ func (s *Writer) flush(w *bytes.Buffer) error {
Records: workerReqs,
ForceSnapshot: forceSnapshot,
LogNumber: s.logNumber,
WaitCh: waitCh,
ExitWaitGroup: exitWaitGroup,
})
return nil
}
func (s *Writer) writePagesToAtree(pages []atree.PageToWrite) error {
for _, p := range pages {
if len(p.Data) != atree.PageSize {
return fmt.Errorf("wrong page %d size: %d",
p.PageNo, len(p.Data))
}
off := (p.PageNo - 1) * atree.PageSize
n, err := s.dataFile.WriteAt(p.Data, int64(off))
if err != nil {
return err
}
if n != len(p.Data) {
return fmt.Errorf("write %d instead of %d", n, len(p.Data))
}
}
return nil
}
func (s *Writer) exit() {
s.mutex.Lock()
s.isExited = true
s.mutex.Unlock()
if err := s.packAndWrite(); err != nil {
octopus.Abort(octopus.FailedWriteToTxLog, err)
qb.Abort(qb.FailedWriteToTxLog, err)
}
}
// writePagesToAtree - записує сторінки в .data та .index файли
func (s *Writer) writePagesToAtree(levels []*atree.IndexLevel, dataPages []*atree.DataPage) (err error) {
for _, p := range dataPages {
p.Checksum = atree.ChunksToDataPage(s.dataPage, atree.ChunksToDataPageReq{
PrevPageNo: p.PrevPageNo,
Timestamps: p.Timestamps,
TimestampsSize: p.TimestampsSize,
Values: p.Values,
ValuesSize: p.ValuesSize,
})
off := (p.PageNo - 1) * atree.DataPageSize
n, err := s.dataFile.WriteAt(s.dataPage, int64(off))
if err != nil {
return err
}
if n != atree.DataPageSize {
return fmt.Errorf("write %d instead of %d", n, atree.DataPageSize)
}
}
for levelIdx, level := range levels {
for _, p := range level.Filled {
// if len(p.Data) != atree.PageSize {
// return fmt.Errorf("wrong page %d size: %d",
// p.PageNo, len(p.Data))
// }
p.Checksum = atree.DataToIndexPage(s.indexPage, p.Data, levelIdx == 0)
off := (p.PageNo - 1) * atree.IndexPageSize
n, err := s.indexFile.WriteAt(s.indexPage, int64(off))
if err != nil {
return err
}
if n != atree.IndexPageSize {
return fmt.Errorf("write %d instead of %d", n, atree.IndexPageSize)
}
}
}
return nil
}
type AppendedMeasures struct {
MetricID uint32
TimestampsOffset int // (заповнені одразу)
ValuesOffset int // (заповнені одразу)
Timestamps [][]byte
TimestampsSize int
Values [][]byte
ValuesSize int
IndexLevels []*atree.IndexLevel
DataPages []*atree.DataPage // fix - prevPageNo for the 1st data page
}
func (s *Writer) packMeasuresIntoWALBuffer(req AppendedMeasures) (err error) {
// fix write code
bin.WriteUint32(s.w, req.MetricID)
bin.WriteVarSize(s.w, len(req.DataPages))
for idx, p := range req.DataPages {
bin.WriteUint32(s.w, p.PageNo)
var reused byte
if p.IsReused {
reused = 1
}
s.w.WriteByte(reused)
bin.WriteUint32(s.w, p.PrevPageNo)
bin.WriteUint32(s.w, p.Checksum)
var (
timestampsOffset int
valuesOffset int
)
if idx == 0 {
timestampsOffset = req.TimestampsOffset
valuesOffset = req.ValuesOffset
}
copyPayloadFromChunks(s.w, p.Timestamps, p.TimestampsSize, timestampsOffset)
copyPayloadFromChunks(s.w, p.Values, p.ValuesSize, valuesOffset)
}
bin.WriteUint16(s.w, uint16(req.TimestampsOffset))
bin.WriteUint16(s.w, uint16(req.ValuesOffset))
var (
timestampsOffset int
valuesOffset int
)
if len(req.DataPages) == 0 {
timestampsOffset = req.TimestampsOffset
valuesOffset = req.ValuesOffset
}
copyPayloadFromChunks(s.w, req.Timestamps, req.TimestampsSize, timestampsOffset)
copyPayloadFromChunks(s.w, req.Values, req.ValuesSize, valuesOffset)
if len(req.DataPages) == 0 {
return
}
// fix - levels reduced
bin.WriteVarSize(s.w, len(req.IndexLevels))
for _, level := range req.IndexLevels {
bin.WriteVarSize(s.w, len(level.Filled))
for idx, p := range level.Filled {
bin.WriteUint32(s.w, p.PageNo)
var reused byte
if p.IsReused {
reused = 1
}
s.w.WriteByte(reused)
bin.WriteUint32(s.w, p.Checksum)
records := p.Data
if idx == 0 {
records = p.Data[level.Offset:]
}
if (len(records) % recordSize) != 0 {
log.Fatalf("wrong records length: %d", len(records))
}
bin.WriteVarSize(s.w, len(records)/recordSize)
s.w.Write(records)
}
records := level.Data
if len(level.Filled) == 0 {
records = level.Data[level.Offset:]
}
if (len(records) % recordSize) != 0 {
log.Fatalf("wrong unfilled records length: %d", len(records))
}
bin.WriteVarSize(s.w, len(records)/recordSize)
s.w.Write(records)
}
return
}
// API
// FIX - add
@@ -351,28 +477,28 @@ func (s *Writer) sendSignal() {
// Якщо (Pages) > 0 - незаповнену сторінку кодую в стисненому вигляді.
// Якщо (Pages) = 0 - кодую просто пари timestamp / value
func (s *Writer) packWriteAppended(w *bytes.Buffer, req AppendedPagesReq) {
// завантажений path. Отже просто додаємо в дерево data сторінки, створюємо нові індексні
// без звернення до диску.
report := s.atree.AppendDataPages(atree.AppendDataPagesReq{
LastPageNo: req.LastPageNo,
Legs: req.Legs,
DataPages: req.Pages,
})
// func (s *Writer) packWriteAppended(w *bytes.Buffer, req AppendedPagesReq) {
// // завантажений path. Отже просто додаємо в дерево data сторінки, створюємо нові індексні
// // без звернення до диску.
// report := s.atree.AppendDataPages(atree.AppendDataPagesReq{
// LastPageNo: req.LastPageNo,
// Legs: req.Legs,
// DataPages: req.Pages,
// })
m := AppendedPages{
MetricID: req.MetricID,
Timestamp: req.Timestamp,
Value: req.Value,
NewRootPageNo: report.NewRootPageNo,
LastPageNo: report.LastPageNo,
Pages: report.Pages,
TimestampsChunks: req.TimestampsChunks,
TimestampsSize: int(req.TimestampsSize),
ValuesChunks: req.ValuesChunks,
ValuesSize: int(req.ValuesSize),
}
m.Pack(w)
}
// m := AppendedPages{
// MetricID: req.MetricID,
// Timestamp: req.Timestamp,
// Value: req.Value,
// NewRootPageNo: report.NewRootPageNo,
// LastPageNo: report.LastPageNo,
// Pages: report.Pages,
// TimestampsChunks: req.TimestampsChunks,
// TimestampsSize: int(req.TimestampsSize),
// ValuesChunks: req.ValuesChunks,
// ValuesSize: int(req.ValuesSize),
// }
// m.Pack(w)
// }
// helpers