dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tmp

Browse Source
This commit is contained in:
dima
2026-05-31 20:01:28 +00:00
parent e7ec6083cb
commit db5ecd3dfd
28 changed files with 2090 additions and 2155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
txlog/snapshot.go Normal file
View File

@@ -0,0 +1,122 @@
package txlog
import (
"fmt"
"io"
"os"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/util"
)
func LoadSnapshot(fileName string) (snapshot Snapshot, err error) {
var (
hasher = util.NewHasher()
header = make([]byte, metricHeaderSize)
body = make([]byte, atree.PageSize)
)
file, err := os.Open(fileName)
if err != nil {
return
}
src := io.TeeReader(file, hasher)
metricsQty, err := bin.ReadVarSize(src)
if err != nil {
return
}
for range metricsQty {
var metric Metric
err = bin.ReadNInto(src, header)
if err != nil {
return
}
metric.MetricID, _ = bin.GetUint32(header[0:])
metric.MetricType = qb.MetricType(header[4])
metric.FracDigits = header[5]
metric.LastPageNo, _ = bin.GetUint32(header[6:])
metric.Since, _ = bin.GetUint32(header[10:])
metric.SinceValue, _ = bin.GetFloat64(header[14:])
metric.Until, _ = bin.GetUint32(header[22:])
metric.UntilValue, _ = bin.GetFloat64(header[26:])
tSize, _ := bin.GetUint16(header[34:])
vSize, _ := bin.GetUint16(header[36:])
buf := body[:tSize]
err = bin.ReadNInto(src, buf)
if err != nil {
return
}
// FIX
// metric.Timestamps = chunkenc.NewReverseTimeDeltaCompressor(
// conbuf.NewFromBuffer(buf),
// int(tSize),
// )
buf = body[:vSize]
err = bin.ReadNInto(src, buf)
if err != nil {
return
}
// FIX
// if metric.MetricType == qb.Cumulative {
// metric.Values = chunkenc.NewReverseCumulativeDeltaCompressor(
// conbuf.NewFromBuffer(buf),
// int(vSize),
// metric.FracDigits,
// )
// } else {
// metric.Values = chunkenc.NewReverseInstantDeltaCompressor(
// conbuf.NewFromBuffer(buf),
// int(vSize),
// metric.FracDigits,
// )
// }
// s.metrics[metricID] = &metric
}
// FIX
// err = restoreFreeList(s.dataFreeList, src)
// if err != nil {
// return fmt.Errorf("restore dataFreeList: %s", err)
// }
// FIX
// err = restoreFreeList(s.indexFreeList, src)
// if err != nil {
// return fmt.Errorf("restore indexFreeList: %s", err)
// }
calculatedChecksum := hasher.Sum32()
writtenChecksum, err := bin.ReadUint32(file)
if err != nil {
return
}
if calculatedChecksum != writtenChecksum {
err = fmt.Errorf("calculated checksum %d not equal written checksum %d",
calculatedChecksum, writtenChecksum)
return
}
return
}
// serialized + frozen pages count
// func restoreFreeList(freeList *freelist.FreeList, src io.Reader) error {
// size, err := bin.ReadVarSize(src)
// if err != nil {
// return err
// }
// serialized, err := bin.ReadN(src, size)
// if err != nil {
// return err
// }
// freeList.Restore(serialized)
// return nil
// }

205
txlog/txlog.go
View File

@@ -1,11 +1,13 @@
package txlog
import (
"bytes"
"io"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/conbuf"
"gordenko.dev/dima/qb/util"
)
type AddedMetric struct {
@@ -75,6 +77,89 @@ func (s *DeletedMetric) Parse(src io.Reader) (err error) {
return nil
}
func writeChunkedPayloadTo(chunks [][]byte, size int, w io.Writer) {
bin.WriteVarSize(w, size)
for _, chunk := range chunks {
if size >= len(chunk) {
w.Write(chunk)
size -= len(chunk)
} else {
w.Write(chunk[:size])
return
}
}
}
func readChunkedPayload(src io.Reader) (_ [][]byte, _ int, err error) {
var (
chunks [][]byte
size int
remainingSize = size
)
size, err = bin.ReadVarSize(src)
if err != nil {
return
}
for remainingSize > 0 {
var chunk []byte
if remainingSize >= conbuf.ChunkSize {
chunk, err = bin.ReadN(src, conbuf.ChunkSize)
if err != nil {
return
}
} else {
chunk = make([]byte, conbuf.ChunkSize)
err = bin.ReadNInto(src, chunk[:remainingSize])
if err != nil {
return
}
}
chunks = append(chunks, chunk)
remainingSize -= conbuf.ChunkSize
}
return chunks, size, nil
}
// fix - add changed pages
type DeletedMeasures struct {
MetricID uint32
FreePageNumbers []uint32
}
func (s DeletedMeasures) Pack(w io.Writer) {
arr := []byte{
CodeDeletedMeasures,
0, 0, 0, 0,
}
bin.PutUint32(arr[1:], s.MetricID)
w.Write(arr)
bin.WriteVarSize(w, len(s.FreePageNumbers))
for _, pageNo := range s.FreePageNumbers {
bin.WriteUint32(w, pageNo)
}
}
func (s *DeletedMeasures) Parse(src io.Reader) (err error) {
s.MetricID, err = bin.ReadUint32(src)
if err != nil {
return
}
// free pages
qty, err := bin.ReadVarSize(src)
if err != nil {
return
}
for range qty {
var pageNo uint32
pageNo, err = bin.ReadUint32(src)
if err != nil {
return
}
s.FreePageNumbers = append(s.FreePageNumbers, pageNo)
}
return nil
}
// type AppendedMeasure struct {
// MetricID uint32
// Timestamp uint32
@@ -273,85 +358,63 @@ Nb - values payload
// return nil
// }
func writeChunkedPayloadTo(chunks [][]byte, size int, w io.Writer) {
bin.WriteVarSize(w, size)
for _, chunk := range chunks {
if size >= len(chunk) {
w.Write(chunk)
size -= len(chunk)
} else {
w.Write(chunk[:size])
return
}
}
// PACK
type PreparedData struct {
Packet []byte // пакет для запису в WAL
WriteToIndex []IndexPayloadToWrite
WriteToData []DataPayloadToWrite
ToWorker []any
}
func readChunkedPayload(src io.Reader) (_ [][]byte, _ int, err error) {
func prepareData(input []any) PreparedData {
buffer := bytes.NewBuffer(nil)
buffer.Write([]byte{
0, 0, 0, 0, 0, 0, 0, 0, // size (max 8 byte)
0, 0, 0, 0, // crc32
})
hasher := util.NewHasher()
w := io.MultiWriter(buffer, hasher)
var (
chunks [][]byte
size int
remainingSize = size
writeToIndex []IndexPayloadToWrite
writeToData []DataPayloadToWrite
)
size, err = bin.ReadVarSize(src)
if err != nil {
return
}
for remainingSize > 0 {
var chunk []byte
if remainingSize >= conbuf.ChunkSize {
chunk, err = bin.ReadN(src, conbuf.ChunkSize)
if err != nil {
return
}
} else {
chunk = make([]byte, conbuf.ChunkSize)
err = bin.ReadNInto(src, chunk[:remainingSize])
if err != nil {
return
}
// 1. Пакую всі дані в WAL буфер запису
for _, untyped := range input {
switch x := untyped.(type) {
case AddedMetric:
x.Pack(w)
// case DeletedMetric:
// if len(x.FreePageNumbers) > 0 {
// s.freeList.AddPageNumbers(x.FreePageNumbers)
// }
case AppendedMeasures:
//s.packMeasuresIntoWALBuffer(x)
//case DeletedMeasures:
//case DeletedMeasuresSince:
}
chunks = append(chunks, chunk)
remainingSize -= conbuf.ChunkSize
}
return chunks, size, nil
}
// 2. Додаю розмір пакету і чексуму
// fix - add changed pages
type DeletedMeasures struct {
MetricID uint32
FreePageNumbers []uint32
}
//s.written += int64(len(packet)) + 12
func (s DeletedMeasures) Pack(w io.Writer) {
arr := []byte{
CodeDeletedMeasures,
0, 0, 0, 0,
}
bin.PutUint32(arr[1:], s.MetricID)
w.Write(arr)
bin.WriteVarSize(w, len(s.FreePageNumbers))
for _, pageNo := range s.FreePageNumbers {
bin.WriteUint32(w, pageNo)
}
}
packet := buffer.Bytes()
// write size
payloadSize := len(packet) - 12
start := 8 - bin.CountVarSize(payloadSize)
bin.PutVarSize(packet[start:], payloadSize)
// checksum
bin.PutUint32(packet[8:], hasher.Sum32())
//
//totalSize := (12 - start) + payloadSize
func (s *DeletedMeasures) Parse(src io.Reader) (err error) {
s.MetricID, err = bin.ReadUint32(src)
if err != nil {
return
return PreparedData{
Packet: packet[start:],
WriteToIndex: writeToIndex,
WriteToData: writeToData,
}
// free pages
qty, err := bin.ReadVarSize(src)
if err != nil {
return
}
for range qty {
var pageNo uint32
pageNo, err = bin.ReadUint32(src)
if err != nil {
return
}
s.FreePageNumbers = append(s.FreePageNumbers, pageNo)
}
return nil
}

583
txlog/writer.go
View File

@@ -1,11 +1,13 @@
package txlog
import (
"bufio"
"bytes"
"errors"
"fmt"
"hash/crc32"
"io"
"log"
"math"
"os"
"path/filepath"
"sync"
@@ -13,6 +15,8 @@ import (
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
"gordenko.dev/dima/qb/atree"
"gordenko.dev/dima/qb/freelist"
"gordenko.dev/dima/qb/util"
)
const (
@@ -26,6 +30,8 @@ const (
filePerm = 0770
dumpSnapshotAfterNBytes = 1024 * 1024 * 1024 // 1 GB
writeBufferSize = 4 * 1024 * 1024
)
const (
@@ -46,31 +52,32 @@ func JoinChangesFileName(dir string, logNumber int) string {
return filepath.Join(dir, fmt.Sprintf("%d.changes", logNumber))
}
type MetricsState struct {
Metrics []Metric
WaitCh chan struct{}
}
type Changes struct {
Records []any
LogNumber int
ForceSnapshot bool
ExitWaitGroup *sync.WaitGroup
WaitCh chan struct{}
Records []any
MetricsCh chan MetricsState
}
type Writer struct {
mutex sync.Mutex
freelist FreeList
atree *atree.Atree
logNumber int
dir string
w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
wal *os.File
dataFile *os.File
indexFile *os.File
dataPage []byte
indexPage []byte
input []any
//buf *bytes.Buffer
//pagesToWrite []atree.PageToWrite
workerReqs []any
//waitCh chan struct{}
mutex sync.Mutex
dataPagesCount uint32
indexPagesCount uint32
dataFreeList *freelist.FreeList
indexFreeList *freelist.FreeList
atree *atree.Atree
logNumber int
dir string
w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
wal *os.File
dataFile *os.File
indexFile *os.File
// буфер для складання payload на сторінку, розрахунку CRC32 та інше
pageBuffer []byte
input []any
appendToWorkerQueue func(any)
//lsn uint32
written int64
@@ -84,7 +91,8 @@ type WriterOptions struct {
Dir string
LogNumber int // номер журнала
AppendToWorkerQueue func(any)
FreeList FreeList
DataFreeList *freelist.FreeList
IndexFreeList *freelist.FreeList
Atree *atree.Atree
ExitCh chan struct{}
WaitGroup *sync.WaitGroup
@@ -97,8 +105,11 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
if opt.AppendToWorkerQueue == nil {
return nil, errors.New("AppendToWorkerQueue option is required")
}
if opt.FreeList == nil {
return nil, errors.New("FreeList option is required")
if opt.DataFreeList == nil {
return nil, errors.New("DataFreeList option is required")
}
if opt.IndexFreeList == nil {
return nil, errors.New("IndexFreeList option is required")
}
if opt.Atree == nil {
return nil, errors.New("Atree option is required")
@@ -113,7 +124,8 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
s := &Writer{
dir: opt.Dir,
appendToWorkerQueue: opt.AppendToWorkerQueue,
freelist: opt.FreeList,
dataFreeList: opt.DataFreeList,
indexFreeList: opt.IndexFreeList,
atree: opt.Atree,
logNumber: opt.LogNumber,
exitCh: opt.ExitCh,
@@ -161,53 +173,140 @@ func (s *Writer) Run() {
}
}
func (s *Writer) getDataPageNumber() (uint32, bool, error) {
pageNo, err := s.dataFreeList.GetPageNumber()
if err != nil {
return 0, false, err
}
if pageNo > 0 {
return pageNo, true, nil
}
if s.dataPagesCount < math.MaxUint32 {
s.dataPagesCount++
return s.dataPagesCount, false, nil
}
return 0, false, errors.New("no space")
}
func (s *Writer) getIndexPageNumber() (uint32, bool, error) {
pageNo, err := s.indexFreeList.GetPageNumber()
if err != nil {
return 0, false, err
}
if pageNo > 0 {
return pageNo, true, nil
}
if s.indexPagesCount < math.MaxUint32 {
s.indexPagesCount++
return s.indexPagesCount, false, nil
}
return 0, false, errors.New("no space")
}
func (s *Writer) packAndWrite() (err error) {
s.mutex.Lock()
//isExited := s.isExited
input := s.input
s.input = nil
// var exitWaitGroup *sync.WaitGroup
// if s.isExited {
// exitWaitGroup = s.waitGroup
// }
s.mutex.Unlock()
w := bytes.NewBuffer(nil)
// 1. Пакую всі дані в WAL буфер запису
for _, untyped := range input {
switch x := untyped.(type) {
case AddedMetric:
x.Pack(w)
// case DeletedMetric:
// if len(x.FreePageNumbers) > 0 {
// s.freeList.AddPageNumbers(x.FreePageNumbers)
// }
case AppendedMeasures:
//
s.packMeasuresIntoWALBuffer(x)
//case DeletedMeasures:
//case DeletedMeasuresSince:
}
}
// 2. Додаю розмір пакету і чексуму
prepared := prepareData(input)
// 3. Пишу на диск WAL (append)
// 4. Пишу в atree сторінки
for range 1 {
err = s.writePagesToAtree(nil, nil)
if err != nil {
return
}
n, err := s.wal.Write(prepared.Packet)
if err != nil {
return
}
if n != len(prepared.Packet) {
return fmt.Errorf("written %d != total size %d", n, len(prepared.Packet))
}
if err = s.wal.Sync(); err != nil {
return
}
// 5. Пишу зміни в free list
// 4. Пишу в atree сторінки
err = s.writePagesToAtree(prepared.WriteToIndex, prepared.WriteToData)
if err != nil {
return
}
// 6. відправляю input - worker-у
// flush
err = s.flush(w)
if err != nil {
return err
var (
forceSnapshot bool
)
if s.written > dumpSnapshotAfterNBytes {
forceSnapshot = true
}
// if isExited && s.written > 0 {
// forceSnapshot = true
// }
if forceSnapshot {
metricsStateCh := make(chan MetricsState)
s.appendToWorkerQueue(Changes{
Records: prepared.ToWorker,
MetricsCh: metricsStateCh,
})
if err := s.wal.Close(); err != nil {
return fmt.Errorf("close changes file: %s", err)
}
state := <-metricsStateCh
s.logNumber++
// write snapshot FIX
err = writeSnapshot(Snapshot{
LogNumber: s.logNumber,
Dir: s.dir,
WriteBufferSize: writeBufferSize,
Metrics: state.Metrics,
DataFreeList: s.dataFreeList,
IndexFreeList: s.indexFreeList,
})
if err != nil {
return fmt.Errorf("write snapshot file: %s", err)
}
var err error
s.wal, err = os.OpenFile(
JoinChangesFileName(s.dir, s.logNumber),
os.O_CREATE|os.O_WRONLY,
filePerm,
)
if err != nil {
return fmt.Errorf("create new changes file: %s", err)
}
s.written = 0
// release Worker
close(state.WaitCh)
} else {
s.appendToWorkerQueue(Changes{
Records: prepared.ToWorker,
})
}
// Якщо потрібен снапшот - відправляю канал із буфером розміру 1,
// в який воркер має покласти снапшот отриманий після застосування змін (records).
// Після чого Writer створює нові файли snapshot і changes і працює далі.
// flush
// err = s.flush(w)
// if err != nil {
// return err
// }
// FIX send to worker workerReqs
return nil
}
@@ -215,7 +314,6 @@ func (s *Writer) packAndWrite() (err error) {
func (s *Writer) Append(req any) {
s.mutex.Lock()
s.input = append(s.input, req)
s.workerReqs = append(s.workerReqs, req)
s.mutex.Unlock()
select {
@@ -237,87 +335,10 @@ func (s *Writer) Append(req any) {
// s.waitCh = make(chan struct{})
// }
func (s *Writer) flush(w *bytes.Buffer) error {
s.mutex.Lock()
// func (s *Writer) flush(w *bytes.Buffer) error {
//pagesToWrite := s.pagesToWrite
workerReqs := s.workerReqs
isExited := s.isExited
var exitWaitGroup *sync.WaitGroup
if s.isExited {
exitWaitGroup = s.waitGroup
}
if w.Len() > packetPrefixSize {
//s.lsn++
//lsn := s.lsn
packet := make([]byte, w.Len())
copy(packet, w.Bytes())
//s.reset() fix
s.written += int64(len(packet)) + 12
s.mutex.Unlock()
bin.PutUint32(packet[lengthIdx:], uint32(len(packet)-packetPrefixSize))
//bin.PutUint32(packet[lsnIdx:], lsn)
bin.PutUint32(packet[checksumIdx:], crc32.ChecksumIEEE(packet[8:]))
n, err := s.wal.Write(packet)
if err != nil {
return fmt.Errorf("TxLog write: %s", err)
}
if n != len(packet) {
return fmt.Errorf("TxLog written %d != packet size %d", n, len(packet))
}
if err := s.wal.Sync(); err != nil {
return fmt.Errorf("TxLog sync: %s", err)
}
// err = s.writePagesToAtree(pagesToWrite)
// if err != nil {
// return fmt.Errorf("TxLog writePagesToAtree: %s", err)
// }
} else {
s.mutex.Unlock()
}
var forceSnapshot bool
if s.written > dumpSnapshotAfterNBytes {
forceSnapshot = true
}
if isExited && s.written > 0 {
forceSnapshot = true
}
if forceSnapshot {
if err := s.wal.Close(); err != nil {
return fmt.Errorf("close changes file: %s", err)
}
s.logNumber++
var err error
s.wal, err = os.OpenFile(
JoinChangesFileName(s.dir, s.logNumber),
os.O_CREATE|os.O_WRONLY,
filePerm,
)
if err != nil {
return fmt.Errorf("create new changes file: %s", err)
}
s.written = 0
}
s.appendToWorkerQueue(Changes{
Records: workerReqs,
ForceSnapshot: forceSnapshot,
LogNumber: s.logNumber,
ExitWaitGroup: exitWaitGroup,
})
return nil
}
// return nil
// }
func (s *Writer) exit() {
s.mutex.Lock()
@@ -329,45 +350,79 @@ func (s *Writer) exit() {
}
}
type IndexPayloadToWrite struct {
PageNo uint32
Payload []byte
ZeroLevel bool
}
type DataPayloadToWrite struct {
PrevPageNo uint32
Timestamps [][]byte // chunks
TimestampsSize int
Values [][]byte // chunks
ValuesSize int
PageNo uint32
}
// writePagesToAtree - записує сторінки в .data та .index файли
func (s *Writer) writePagesToAtree(levels []*atree.IndexLevel, dataPages []*atree.DataPage) (err error) {
for _, p := range dataPages {
p.Checksum = atree.ChunksToDataPage(s.dataPage, atree.ChunksToDataPageReq{
func (s *Writer) writePagesToAtree(indexItems []IndexPayloadToWrite, dataItems []DataPayloadToWrite) (err error) {
for _, p := range dataItems {
atree.ChunksToDataPage(s.pageBuffer, atree.ChunksToDataPageReq{
PrevPageNo: p.PrevPageNo,
Timestamps: p.Timestamps,
TimestampsSize: p.TimestampsSize,
Values: p.Values,
ValuesSize: p.ValuesSize,
})
off := (p.PageNo - 1) * atree.DataPageSize
n, err := s.dataFile.WriteAt(s.dataPage, int64(off))
var (
off = (p.PageNo - 1) * atree.DataPageSize
n int
)
n, err = s.dataFile.WriteAt(s.pageBuffer, int64(off))
if err != nil {
return err
return
}
if n != atree.DataPageSize {
return fmt.Errorf("write %d instead of %d", n, atree.DataPageSize)
}
}
for levelIdx, level := range levels {
for _, p := range level.Filled {
// if len(p.Data) != atree.PageSize {
// return fmt.Errorf("wrong page %d size: %d",
// p.PageNo, len(p.Data))
// }
p.Checksum = atree.DataToIndexPage(s.indexPage, p.Data, levelIdx == 0)
off := (p.PageNo - 1) * atree.IndexPageSize
n, err := s.indexFile.WriteAt(s.indexPage, int64(off))
if err != nil {
return err
}
if n != atree.IndexPageSize {
return fmt.Errorf("write %d instead of %d", n, atree.IndexPageSize)
}
for _, p := range indexItems {
// if len(p.Payload) > atree.MaxIndexPayload {
// return fmt.Errorf("wrong index payload size: %d", len(p.Payload))
// }
indexPageBuffer := s.pageBuffer[:atree.IndexPageSize]
atree.DataToIndexPage(indexPageBuffer, p.Payload, p.ZeroLevel)
var (
off = (p.PageNo - 1) * atree.IndexPageSize
n int
)
n, err = s.indexFile.WriteAt(indexPageBuffer, int64(off))
if err != nil {
return
}
if n != atree.IndexPageSize {
return fmt.Errorf("write %d instead of %d", n, atree.IndexPageSize)
}
}
return nil
}
type AppendMeasuresSummary struct {
MetricID uint32
//TimestampsOffset int // (заповнені одразу)
//ValuesOffset int // (заповнені одразу)
// Timestamps [][]byte
// TimestampsSize int
// Values [][]byte
// ValuesSize int
LastPageNo uint32
Index [][]byte
ResultCode int
WrittenCount int
ResultCh chan struct{}
}
type AppendedMeasures struct {
MetricID uint32
TimestampsOffset int // (заповнені одразу)
@@ -378,6 +433,9 @@ type AppendedMeasures struct {
ValuesSize int
IndexLevels []*atree.IndexLevel
DataPages []*atree.DataPage // fix - prevPageNo for the 1st data page
ResultCode int
WrittenCount int
ResultCh chan struct{}
}
func (s *Writer) packMeasuresIntoWALBuffer(req AppendedMeasures) (err error) {
@@ -392,7 +450,7 @@ func (s *Writer) packMeasuresIntoWALBuffer(req AppendedMeasures) (err error) {
}
s.w.WriteByte(reused)
bin.WriteUint32(s.w, p.PrevPageNo)
bin.WriteUint32(s.w, p.Checksum)
//bin.WriteUint32(s.w, p.Checksum)
var (
timestampsOffset int
valuesOffset int
@@ -430,7 +488,7 @@ func (s *Writer) packMeasuresIntoWALBuffer(req AppendedMeasures) (err error) {
reused = 1
}
s.w.WriteByte(reused)
bin.WriteUint32(s.w, p.Checksum)
//bin.WriteUint32(s.w, p.Checksum)
records := p.Data
if idx == 0 {
records = p.Data[level.Offset:]
@@ -502,3 +560,204 @@ func (s *Writer) sendSignal() {
// }
// helpers
/*
Формат:
metricsQty - varuint
[metric]*
де metric - це:
metricID - 4b
metricType - 1b
fracDigits - 1b
lastPageNo - 4b
since - 4b
sinceValue - 8b
until - 4b
untilValue - 8b
timestamps size - 2b
values size - 2b
timestams payload - Nb
values payload - Nb
data free list frozen pages - varsize
dataFreeList size - varsize
dataFreeList - Nb
index free list frozen pages - varsize
indexFreeList size - varsize
indexFreeList - Nb
CRC32 - 4b
*/
const metricHeaderSize = 38
type Metric struct {
MetricID uint32
MetricType qb.MetricType
FracDigits byte
LastPageNo uint32
Since uint32
SinceValue float64
Until uint32
UntilValue float64
Timestamps [][]byte
TimestampsSize int
Values [][]byte
ValuesSize int
}
type Snapshot struct {
LogNumber int
Dir string
WriteBufferSize int
Metrics []Metric
IndexFreeList *freelist.FreeList
DataFreeList *freelist.FreeList
}
func writeSnapshot(snapshot Snapshot) (err error) {
var (
fileName = filepath.Join(snapshot.Dir, fmt.Sprintf("%d.snapshot", snapshot.LogNumber))
hasher = util.NewHasher()
prefix = make([]byte, metricHeaderSize)
)
file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY, 0770)
if err != nil {
return
}
dst := io.MultiWriter(bufio.NewWriterSize(file, snapshot.WriteBufferSize), hasher)
_, err = bin.WriteVarSize(dst, len(snapshot.Metrics))
if err != nil {
return
}
for _, metric := range snapshot.Metrics {
tSize := metric.TimestampsSize
vSize := metric.ValuesSize
bin.PutUint32(prefix[0:], metric.MetricID)
prefix[4] = byte(metric.MetricType)
prefix[5] = metric.FracDigits
bin.PutUint32(prefix[6:], metric.LastPageNo)
bin.PutUint32(prefix[10:], metric.Since)
bin.PutFloat64(prefix[14:], metric.SinceValue)
bin.PutUint32(prefix[22:], metric.Until)
bin.PutFloat64(prefix[26:], metric.UntilValue)
bin.PutUint16(prefix[34:], uint16(tSize))
bin.PutUint16(prefix[36:], uint16(vSize))
_, err = dst.Write(prefix)
if err != nil {
return
}
// copy timestamps
writeChunks(dst, metric.Timestamps, tSize)
// copy values
writeChunks(dst, metric.Values, vSize)
}
// free data pages
_, err = bin.WriteVarSize(dst, snapshot.DataFreeList.Pages())
if err != nil {
return
}
err = freeListWriteTo(snapshot.DataFreeList, dst)
if err != nil {
return
}
// free index pages
_, err = bin.WriteVarSize(dst, snapshot.IndexFreeList.Pages())
if err != nil {
return
}
err = freeListWriteTo(snapshot.IndexFreeList, dst)
if err != nil {
return
}
bin.WriteUint32(file, hasher.Sum32())
err = file.Sync()
if err != nil {
return
}
err = file.Close()
if err != nil {
return
}
// копіюю сторінки із delta файла в base файл і потім роблю Truncate
err = snapshot.DataFreeList.Merge() // fix - get frozen pages count + unfilled
if err != nil {
return
}
err = snapshot.IndexFreeList.Merge()
if err != nil {
return
}
// prevLogNumber := logNumber - 1
// prevChanges := filepath.Join(s.dir, fmt.Sprintf("%d.changes", prevLogNumber))
// prevSnapshot := filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", prevLogNumber))
// isExist, err := isFileExist(prevChanges)
// if err != nil {
// return
// }
// if isExist {
// err = os.Remove(prevChanges)
// if err != nil {
// qb.Abort(qb.DeletePrevChangesFileFailed, err)
// }
// }
// isExist, err = isFileExist(prevSnapshot)
// if err != nil {
// return
// }
// if isExist {
// err = os.Remove(prevSnapshot)
// if err != nil {
// qb.Abort(qb.DeletePrevSnapshotFileFailed, err)
// }
// }
return
}
// HELPERS
func freeListWriteTo(freeList *freelist.FreeList, dst io.Writer) error {
serialized, err := freeList.Serialize()
if err != nil {
qb.Abort(qb.FailedFreeListSerialize, err)
}
_, err = bin.WriteVarSize(dst, len(serialized))
if err != nil {
return err
}
_, err = dst.Write(serialized)
if err != nil {
return err
}
return nil
}
func writeChunks(dst io.Writer, chunks [][]byte, size int) (err error) {
remaining := size
for _, buf := range chunks {
if remaining < len(buf) {
buf = buf[:remaining]
}
_, err = dst.Write(buf)
if err != nil {
return
}
remaining -= len(buf)
if remaining == 0 {
break
}
}
return
}