dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
dima
2026-06-13 07:13:03 +00:00
parent aca0252cfd
commit 5778782737
10 changed files with 99 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
storage/page_ingester.go
View File

@@ -228,6 +228,8 @@ type appendIndexRecordIn struct {
}
func appendIndexRecord(in appendIndexRecordIn) []byte {
//fmt.Printf("appendIndexRecord: % x\n", in.Records)
//fmt.Printf("timestamp: %d, pageNo: %d\n", in.Timestamp, in.PageNo)
if in.RecordsCount < maxRecordsOnIndexPage {
var (
pos int
@@ -235,11 +237,13 @@ func appendIndexRecord(in appendIndexRecordIn) []byte {
)
if in.RecordsCount > 0 {
pos = in.RecordsCount * indexRecordSize
//fmt.Printf("pos: %d\n", pos)
} else {
buf = make([]byte, IndexPageSize) // IndexPageIncSize
}
bin.PutUint32(buf[pos:], in.Timestamp)
bin.PutUint32(buf[pos+4:], in.PageNo)
//fmt.Printf("appendIndexRecord after: % x\n", buf)
return buf
}
qb.Abort(qb.NoSpaceOnIndexPage, nil)

22
storage/storage.go
View File

@@ -7,8 +7,8 @@ var (
//PageNoSize = 4
// data page
DataPageSize = 8192
DataPageFooterSize int = 12
DataPageSize = 8192
DataPagePayloadSize int = DataPageSize - DataPageFooterSize
dataCRC32Idx = DataPageSize - 4
@@ -17,13 +17,13 @@ var (
prevPageIdx = DataPageSize - 12
// index page
IndexPageSize = 1024
IndexPageFooterSize = 7
IndexPageSize = 1024
//indexPageIncSize = IndexPageIncSize
indexCRC32Idx = IndexPageSize - 4
indexRecordsCountIdx = IndexPageSize - 6
isZeroLevelIdx = IndexPageSize - 7
indexRecordSize = 8
indexCRC32Idx = IndexPageSize - 4
indexRecordsCountIdx = IndexPageSize - 6
isZeroLevelIdx = IndexPageSize - 7
maxRecordsOnIndexPage = (IndexPageSize - IndexPageFooterSize) / indexRecordSize
// timestampSize = 4
@@ -32,6 +32,12 @@ var (
// dataFooterIdx = timestampsSizeIdx
)
const (
DataPageFooterSize = 12
IndexPageFooterSize = 7
indexRecordSize = 8
)
type IndexRecord struct {
Timestamp uint32
PageNo uint32

10
storage/storage_test.go
View File

@@ -623,7 +623,6 @@ func TestMeasuresAppendRecord(t *testing.T) {
func TestMeasuresAppendWithGrowRecord(t *testing.T) {
DataPageSize = 8
IndexPageSize = 4
indexRecordSize = 2
//
rec := MeasuresAppendWithGrowRecord{
MetricID: 12345,
@@ -735,8 +734,7 @@ func TestWritePreparer(t *testing.T) {
prevPageIdx = DataPageSize - 12
// index page
IndexPageSize = 16
indexRecordSize = 4 // 2 records per index page
IndexPageSize = 24
maxRecordsOnIndexPage = 2
indexCRC32Idx = IndexPageSize - 4
indexRecordsCountIdx = IndexPageSize - 6
@@ -835,9 +833,9 @@ func TestWritePreparer(t *testing.T) {
IndexLevelTails: []IndexLevelTail{
{
Buffer: []byte{
0x01, 0x01, // since
0x0a, 0x00, // pageNo 10
0x00, 0x00, 0x00, 0x00, 0x00, // empty space
0x01, 0x01, 0x00, 0x00, // since
0x0a, 0x00, 0x00, 0x00, // pageNo 10
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // empty space
0x00, // zero level
0x00, 0x00, // records count
0x00, 0x00, 0x00, 0x00, // crc32

31
storage/wal_writer.go
View File

@@ -53,7 +53,8 @@ type MetricsState struct {
}
type Changes struct {
Records []any
Commits []any
// for snapshot only
SnapshotNumberCh chan int // log number
FrozenIndexPagesCount int
IndexPageNumbers []uint32
@@ -62,13 +63,12 @@ type Changes struct {
}
type Writer struct {
mutex sync.Mutex
dataPagesCount uint32
indexPagesCount uint32
dataFreeList *freelist.FreeList
indexFreeList *freelist.FreeList
atree *atree.Atree
//logNumber int
mutex sync.Mutex
dataPagesCount uint32
indexPagesCount uint32
dataFreeList *freelist.FreeList
indexFreeList *freelist.FreeList
atree *atree.Atree
dir string
w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
wal *os.File
@@ -79,12 +79,11 @@ type Writer struct {
input []any
dataPreparer *WritePreparer
appendToWorkerQueue func(any)
//lsn uint32
written int64
isExited bool
exitCh chan struct{}
waitGroup *sync.WaitGroup
signalCh chan struct{}
written int64
isExited bool
exitCh chan struct{}
waitGroup *sync.WaitGroup
signalCh chan struct{}
}
type WriterOptions struct {
@@ -259,7 +258,7 @@ func (s *Writer) packAndWrite() (err error) {
snapshotNumberCh := make(chan int, 1)
s.appendToWorkerQueue(Changes{
Records: prepared.Commits,
Commits: prepared.Commits,
SnapshotNumberCh: snapshotNumberCh,
FrozenIndexPagesCount: s.indexFreeList.Pages(),
IndexPageNumbers: s.indexFreeList.Cached(),
@@ -290,7 +289,7 @@ func (s *Writer) packAndWrite() (err error) {
}
} else {
s.appendToWorkerQueue(Changes{
Records: prepared.Commits,
Commits: prepared.Commits,
})
}
return nil

10
storage/write_preparer.go
View File

@@ -115,7 +115,7 @@ func (s *WritePreparer) Prepare(input []any) PreparedData {
}
for _, level := range sealedIndexLevels {
for _, p := range level.IndexPages {
fmt.Printf("records: %v\n", getIndexRecords(p.Content, 2))
//fmt.Printf("records: %v\n", getIndexRecords(p.Content, 2))
s.indexPages = append(s.indexPages, PageToWrite{
PageNo: p.PageNo,
Content: p.Content,
@@ -139,10 +139,12 @@ func (s *WritePreparer) Prepare(input []any) PreparedData {
)
if len(level.IndexPages) > 0 {
first := level.IndexPages[0]
firstCRC32, _ := bin.GetUint32(first.Content[indexCRC32Idx:])
skipSize := level.SkipRecords * indexRecordSize
indexPageTail = &IndexPageTail{
PageNo: first.PageNo,
Reused: first.Reused,
CRC32: firstCRC32,
// в WAL файл попадає лише payload
Records: first.Content[skipSize : maxRecordsOnIndexPage*indexRecordSize],
}
@@ -181,11 +183,13 @@ func (s *WritePreparer) Prepare(input []any) PreparedData {
Values: x.Values,
},
DataPages: sealedDataPages[1:],
TailTimestamps: x.TailTimestamps,
TailValues: x.TailValues,
ChangedIndexLevels: changedIndexPages,
TailTimestamps: x.Timestamps,
TailValues: x.Values,
}
fmt.Printf("%#v\ns", rec.ChangedIndexLevels[0].IndexPageTail)
rec.Pack(w)
// Дані для Worker
s.commits = append(s.commits, MeasuresAppendWithGrowCommited{