dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tmp

Browse Source
This commit is contained in:
dima
2026-06-05 19:43:01 +00:00
parent db5ecd3dfd
commit 05eea3b279
16 changed files with 1187 additions and 724 deletions
Download Patch File Download Diff File Expand all files Collapse all files

210
txlog/writer.go
View File

@@ -6,7 +6,6 @@ import (
"errors"
"fmt"
"io"
"log"
"math"
"os"
"path/filepath"
@@ -63,21 +62,22 @@ type Changes struct {
}
type Writer struct {
mutex sync.Mutex
dataPagesCount uint32
indexPagesCount uint32
dataFreeList *freelist.FreeList
indexFreeList *freelist.FreeList
atree *atree.Atree
logNumber int
dir string
w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
wal *os.File
dataFile *os.File
indexFile *os.File
// буфер для складання payload на сторінку, розрахунку CRC32 та інше
pageBuffer []byte
mutex sync.Mutex
dataPagesCount uint32
indexPagesCount uint32
dataFreeList *freelist.FreeList
indexFreeList *freelist.FreeList
atree *atree.Atree
logNumber int
dir string
w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
wal *os.File
dataFile *os.File
indexFile *os.File
indexPageSize int
dataPageSize int
input []any
dataPreparer *DataPreparer
appendToWorkerQueue func(any)
//lsn uint32
written int64
@@ -88,6 +88,8 @@ type Writer struct {
}
type WriterOptions struct {
IndexPageSize int
DataPageSize int
Dir string
LogNumber int // номер журнала
AppendToWorkerQueue func(any)
@@ -99,6 +101,12 @@ type WriterOptions struct {
}
func NewWriter(opt WriterOptions) (*Writer, error) {
if (opt.IndexPageSize % 2) != 0 {
return nil, errors.New("IndexPageSize must be multiple of 2")
}
if (opt.DataPageSize % 2) != 0 {
return nil, errors.New("DataPageSize must be multiple of 2")
}
if opt.Dir == "" {
return nil, errors.New("Dir option is required")
}
@@ -122,6 +130,8 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
}
s := &Writer{
indexPageSize: opt.IndexPageSize,
dataPageSize: opt.DataPageSize,
dir: opt.Dir,
appendToWorkerQueue: opt.AppendToWorkerQueue,
dataFreeList: opt.DataFreeList,
@@ -215,7 +225,7 @@ func (s *Writer) packAndWrite() (err error) {
// }
s.mutex.Unlock()
prepared := prepareData(input)
prepared := s.dataPreparer.Prepare(input)
// 3. Пишу на диск WAL (append)
n, err := s.wal.Write(prepared.Packet)
@@ -253,7 +263,7 @@ func (s *Writer) packAndWrite() (err error) {
metricsStateCh := make(chan MetricsState)
s.appendToWorkerQueue(Changes{
Records: prepared.ToWorker,
Records: prepared.WriteResults,
MetricsCh: metricsStateCh,
})
@@ -293,7 +303,7 @@ func (s *Writer) packAndWrite() (err error) {
close(state.WaitCh)
} else {
s.appendToWorkerQueue(Changes{
Records: prepared.ToWorker,
Records: prepared.WriteResults,
})
}
@@ -350,168 +360,54 @@ func (s *Writer) exit() {
}
}
type IndexPayloadToWrite struct {
PageNo uint32
Payload []byte
ZeroLevel bool
}
// type IndexPayloadToWrite struct {
// PageNo uint32
// Payload []byte
// ZeroLevel bool
// }
type DataPayloadToWrite struct {
PrevPageNo uint32
Timestamps [][]byte // chunks
TimestampsSize int
Values [][]byte // chunks
ValuesSize int
PageNo uint32
type PageToWrite struct {
PageNo uint32
Content []byte
}
// writePagesToAtree - записує сторінки в .data та .index файли
func (s *Writer) writePagesToAtree(indexItems []IndexPayloadToWrite, dataItems []DataPayloadToWrite) (err error) {
for _, p := range dataItems {
atree.ChunksToDataPage(s.pageBuffer, atree.ChunksToDataPageReq{
PrevPageNo: p.PrevPageNo,
Timestamps: p.Timestamps,
TimestampsSize: p.TimestampsSize,
Values: p.Values,
ValuesSize: p.ValuesSize,
})
func (s *Writer) writePagesToAtree(indexPages []PageToWrite, dataPages []PageToWrite) (err error) {
for _, p := range dataPages {
if len(p.Content) != s.dataPageSize {
return fmt.Errorf("wrong data page size: %d", len(p.Content))
}
var (
off = (p.PageNo - 1) * atree.DataPageSize
off = int(p.PageNo-1) * s.dataPageSize
n int
)
n, err = s.dataFile.WriteAt(s.pageBuffer, int64(off))
n, err = s.dataFile.WriteAt(p.Content, int64(off))
if err != nil {
return
}
if n != atree.DataPageSize {
return fmt.Errorf("write %d instead of %d", n, atree.DataPageSize)
if n != s.dataPageSize {
return fmt.Errorf("write %d instead of %d", n, s.dataPageSize)
}
}
for _, p := range indexItems {
// if len(p.Payload) > atree.MaxIndexPayload {
// return fmt.Errorf("wrong index payload size: %d", len(p.Payload))
// }
indexPageBuffer := s.pageBuffer[:atree.IndexPageSize]
atree.DataToIndexPage(indexPageBuffer, p.Payload, p.ZeroLevel)
for _, p := range indexPages {
if len(p.Content) != s.indexPageSize {
return fmt.Errorf("wrong index page size: %d", len(p.Content))
}
var (
off = (p.PageNo - 1) * atree.IndexPageSize
off = int(p.PageNo-1) * s.indexPageSize
n int
)
n, err = s.indexFile.WriteAt(indexPageBuffer, int64(off))
n, err = s.indexFile.WriteAt(p.Content, int64(off))
if err != nil {
return
}
if n != atree.IndexPageSize {
return fmt.Errorf("write %d instead of %d", n, atree.IndexPageSize)
if n != s.indexPageSize {
return fmt.Errorf("write %d instead of %d", n, s.indexPageSize)
}
}
return nil
}
type AppendMeasuresSummary struct {
MetricID uint32
//TimestampsOffset int // (заповнені одразу)
//ValuesOffset int // (заповнені одразу)
// Timestamps [][]byte
// TimestampsSize int
// Values [][]byte
// ValuesSize int
LastPageNo uint32
Index [][]byte
ResultCode int
WrittenCount int
ResultCh chan struct{}
}
type AppendedMeasures struct {
MetricID uint32
TimestampsOffset int // (заповнені одразу)
ValuesOffset int // (заповнені одразу)
Timestamps [][]byte
TimestampsSize int
Values [][]byte
ValuesSize int
IndexLevels []*atree.IndexLevel
DataPages []*atree.DataPage // fix - prevPageNo for the 1st data page
ResultCode int
WrittenCount int
ResultCh chan struct{}
}
func (s *Writer) packMeasuresIntoWALBuffer(req AppendedMeasures) (err error) {
// fix write code
bin.WriteUint32(s.w, req.MetricID)
bin.WriteVarSize(s.w, len(req.DataPages))
for idx, p := range req.DataPages {
bin.WriteUint32(s.w, p.PageNo)
var reused byte
if p.IsReused {
reused = 1
}
s.w.WriteByte(reused)
bin.WriteUint32(s.w, p.PrevPageNo)
//bin.WriteUint32(s.w, p.Checksum)
var (
timestampsOffset int
valuesOffset int
)
if idx == 0 {
timestampsOffset = req.TimestampsOffset
valuesOffset = req.ValuesOffset
}
copyPayloadFromChunks(s.w, p.Timestamps, p.TimestampsSize, timestampsOffset)
copyPayloadFromChunks(s.w, p.Values, p.ValuesSize, valuesOffset)
}
bin.WriteUint16(s.w, uint16(req.TimestampsOffset))
bin.WriteUint16(s.w, uint16(req.ValuesOffset))
var (
timestampsOffset int
valuesOffset int
)
if len(req.DataPages) == 0 {
timestampsOffset = req.TimestampsOffset
valuesOffset = req.ValuesOffset
}
copyPayloadFromChunks(s.w, req.Timestamps, req.TimestampsSize, timestampsOffset)
copyPayloadFromChunks(s.w, req.Values, req.ValuesSize, valuesOffset)
if len(req.DataPages) == 0 {
return
}
// fix - levels reduced
bin.WriteVarSize(s.w, len(req.IndexLevels))
for _, level := range req.IndexLevels {
bin.WriteVarSize(s.w, len(level.Filled))
for idx, p := range level.Filled {
bin.WriteUint32(s.w, p.PageNo)
var reused byte
if p.IsReused {
reused = 1
}
s.w.WriteByte(reused)
//bin.WriteUint32(s.w, p.Checksum)
records := p.Data
if idx == 0 {
records = p.Data[level.Offset:]
}
if (len(records) % recordSize) != 0 {
log.Fatalf("wrong records length: %d", len(records))
}
bin.WriteVarSize(s.w, len(records)/recordSize)
s.w.Write(records)
}
records := level.Data
if len(level.Filled) == 0 {
records = level.Data[level.Offset:]
}
if (len(records) % recordSize) != 0 {
log.Fatalf("wrong unfilled records length: %d", len(records))
}
bin.WriteVarSize(s.w, len(records)/recordSize)
s.w.Write(records)
}
return
}
// API
// FIX - add