dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
dima
2026-06-12 06:11:57 +00:00
parent dc0410c9e4
commit 58db58a0a0
14 changed files with 1305 additions and 1227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
client/client.go
View File

@@ -67,7 +67,7 @@ func (s *Connection) AddMetric(req proto.AddMetricReq) (err error) {
proto.TypeAddMetric, proto.TypeAddMetric,
0, 0, 0, 0, // 0, 0, 0, 0, //
byte(req.MetricType), byte(req.MetricType),
byte(req.FracDigits), req.FracDigits,
} }
bin.PutUint32(arr[1:], req.MetricID) bin.PutUint32(arr[1:], req.MetricID)
// req // req
@@ -82,7 +82,7 @@ func (s *Connection) AddMetric(req proto.AddMetricReq) (err error) {
// arr := []byte{ // arr := []byte{
// proto.TypeUpdateMetric, // proto.TypeUpdateMetric,
// 0, 0, 0, 0, // // 0, 0, 0, 0, //
// byte(req.FracDigits), // req.FracDigits,
// } // }
// bin.PutUint32(arr[1:], req.MetricID) // bin.PutUint32(arr[1:], req.MetricID)

61
database/database.go
View File

@@ -288,67 +288,6 @@ func (s *Database) relayMetricsToMetrics(replayMetrics map[uint32]*ReplayMetric)
} }
} }
// FIX
// УВАГА!
// Якщо буфер metric.buffer < розміру сторінки - для timestamps доступний весь буфер.
// Якщо буфер досяг розміру сторінки - в timestamps я передаю buffer[:DataPagePayloadSize]
var dataBufferSizes = []int{
1024,
2048,
4096,
8192,
}
func calculateDataBufferSize(payloadSize int) int {
for _, bufferSize := range dataBufferSizes {
if payloadSize <= bufferSize {
return bufferSize
}
}
return storage.DataPageSize
}
func allocateBuffers(requiredSpace int) (buf, databuf []byte) {
bufferSize := calculateDataBufferSize(requiredSpace)
buf = make([]byte, bufferSize)
if bufferSize == storage.DataPageSize {
databuf = buf[:storage.DataPagePayloadSize]
} else {
databuf = buf
}
return
}
// src - databuf
type growBuffersIn struct {
databuf []byte
tSize int
vSize int
requiredSpace int
}
func growBuffers(in growBuffersIn) (buf, databuf []byte) {
buf, databuf = allocateBuffers(in.requiredSpace)
copy(databuf[len(databuf)-in.tSize:], in.databuf[len(in.databuf)-in.tSize:])
copy(databuf, in.databuf[:in.vSize])
return
}
type shrinkBufferIn struct {
Buffer []byte
TimestampsSize int
ValuesSize int
NewBufferSize int
}
func shrinkBuffer(in shrinkBufferIn) []byte {
buf := make([]byte, in.NewBufferSize)
copy(buf, in.Buffer[:in.ValuesSize])
copy(buf[:len(buf)-in.TimestampsSize], in.Buffer[len(in.Buffer)-in.TimestampsSize:])
return buf
}
//func (s *Database) verifySnapshot(fileName string) (_ bool, err error) { //func (s *Database) verifySnapshot(fileName string) (_ bool, err error) {
// file, err := os.Open(fileName) // file, err := os.Open(fileName)
// if err != nil { // if err != nil {

69
database/metric.go
View File

@@ -10,7 +10,7 @@ import (
// METRIC // METRIC
const minBufferSize = 1024 //const minBufferSize = 1024
var ( var (
indexRecordSize = 8 indexRecordSize = 8
@@ -79,29 +79,31 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, tmp []byte, send
timestampsOffset int timestampsOffset int
valuesOffset int valuesOffset int
headTimestamps []byte
headValues []byte
written int written int
resultCode byte resultCode byte
) )
s.capturedState = &CapturedState{ s.capturedState = &CapturedState{
// //
TimeState: s.timestamps.CaptureState(),
ValueState: s.values.CaptureState(),
} }
s.timestamps.CaptureState()
s.values.CaptureState()
for idx, measure := range req.Measures { for idx, measure := range req.Measures {
if measure.Timestamp <= s.timestamps.LastTimestamp() { if measure.Timestamp <= s.timestamps.LastTimestamp() {
resultCode = ExpiredMeasure resultCode = ExpiredMeasure
break break
} }
if s.metricType == qb.Cumulative && measure.Value < s.lastValue { if s.metricType == qb.Cumulative && measure.Value < s.lastValue {
resultCode = NonMonotonicValue resultCode = NonMonotonicValue
break break
} }
// fix - 1 + 8 bytes
tReport := timestamps.Evaluate(tmp, measure.Timestamp) tReport := timestamps.Evaluate(tmp, measure.Timestamp)
vReport := values.Evaluate(tmp, measure.Value) vReport := values.Evaluate(tmp, measure.Value)
@@ -114,27 +116,12 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, tmp []byte, send
valuesOffset = vReport.Offset valuesOffset = vReport.Offset
} }
} else { } else {
if len(s.buffer) < storage.DataPageSize {
// allocate bigger buffer
buf, databuf := growBuffers(growBuffersIn{
databuf: s.buffer,
tSize: timestamps.Size(),
vSize: values.Size(),
requiredSpace: totalRequiredSpace,
})
s.buffer = buf
// replace buffer in timestamps and values
timestamps.Rotate(databuf) // fix pos
values.Rotate(databuf) // fix pos
} else {
// сторінка заповнена // сторінка заповнена
since := s.timestamps.ReplaceSinceWithUntil() since := s.timestamps.ReplaceSinceWithUntil()
if len(s.capturedState.Pages) == 0 { if len(s.capturedState.Pages) == 0 {
// head page FIX headTimestamps = timestamps.Tail(timestampsOffset)
// timestampsPayload = headValues = values.Tail(valuesOffset)
// valuesPayload =
} }
s.capturedState.Pages = append(s.capturedState.Pages, storage.DataPayload{ s.capturedState.Pages = append(s.capturedState.Pages, storage.DataPayload{
@@ -144,18 +131,18 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, tmp []byte, send
ValuesSize: values.Size(), ValuesSize: values.Size(),
}) })
buffer := make([]byte, minBufferSize) buf := make([]byte, storage.DataPageSize)
databuf := buf[:storage.DataPagePayloadSize]
timestamps.Rotate(buffer) timestamps.ReplaceBuffer(databuf)
values.Rotate(buffer) values.ReplaceBuffer(databuf)
// renew // renew
s.buffer = buffer s.buffer = buf
}
} }
timestamps.Append(tReport.Offset, tmp[:tReport.ChangeSize], measure.Timestamp) timestamps.Append(tReport.RewindOffset, tmp[:tReport.ChangeSize], measure.Timestamp)
values.Append(vReport.Offset, tmp[7:7+vReport.ChangeSize], measure.Value, vReport.Delta) values.Append(vReport.RewindOffset, tmp[7:7+vReport.ChangeSize], measure.Value, vReport.Delta)
// //
s.lastValue = measure.Value s.lastValue = measure.Value
written++ written++
@@ -173,39 +160,33 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, tmp []byte, send
// скопіювати. Причому можна скопіювати зрізи chunks // скопіювати. Причому можна скопіювати зрізи chunks
if len(s.capturedState.Pages) > 0 { if len(s.capturedState.Pages) > 0 {
head := s.capturedState.Pages[0]
databuf := head.Content[:storage.DataPagePayloadSize]
// пишу в storage довгим шляхом через redo файл і запис в data файл // пишу в storage довгим шляхом через redo файл і запис в data файл
sendToStorage(storage.AppendedMeasures{ sendToStorage(storage.AppendedMeasures{
MetricID: req.MetricID, MetricID: req.MetricID,
LastPageNo: s.lastPageNo, LastPageNo: s.lastPageNo,
TimestampsOffset: timestampsOffset, TimestampsOffset: timestampsOffset,
Timestamps: databuf[len(databuf)-head.TimestampsSize : len(databuf)-timestampsOffset], Timestamps: headTimestamps,
ValuesOffset: valuesOffset, ValuesOffset: valuesOffset,
Values: databuf[valuesOffset:head.ValuesSize], Values: headValues,
IndexLevelTails: s.indexLevelTails, IndexLevelTails: s.indexLevelTails,
DataPages: s.capturedState.Pages, DataPages: s.capturedState.Pages,
TailTimestamps: nil, // payload TailTimestamps: timestamps.Tail(0), // payload
TailValues: s.buffer[:values.Size()], TailValues: values.Tail(0),
ResultCode: resultCode, ResultCode: resultCode,
WrittenCount: written, WrittenCount: written,
ResultCh: nil, //ResultCh: req.ResultCh,
}) })
} else { } else {
databuf := s.buffer
if len(s.buffer) == storage.DataPageSize {
databuf = s.buffer[:storage.DataPagePayloadSize]
}
// короткий шлях - запис лише в storage // короткий шлях - запис лише в storage
sendToStorage(storage.AppendedMeasures{ sendToStorage(storage.AppendedMeasures{
MetricID: req.MetricID, MetricID: req.MetricID,
TimestampsOffset: timestampsOffset, TimestampsOffset: timestampsOffset,
ValuesOffset: valuesOffset, ValuesOffset: valuesOffset,
Timestamps: databuf[len(databuf)-timestamps.Size() : len(databuf)-timestampsOffset], Timestamps: timestamps.Tail(timestampsOffset),
Values: databuf[valuesOffset:values.Size()], Values: values.Tail(valuesOffset),
ResultCode: resultCode, ResultCode: resultCode,
WrittenCount: written, WrittenCount: written,
ResultCh: nil, //ResultCh: req.ResultCh,
}) })
} }

11
database/proc.go
View File

@@ -472,14 +472,15 @@ func (s *Database) applyAddMetric(rec storage.MetricAddRecord) {
func (s *Database) addMetric(rec storage.MetricAddRecord) { func (s *Database) addMetric(rec storage.MetricAddRecord) {
var ( var (
buffer = make([]byte, minBufferSize) buf = make([]byte, storage.DataPageSize)
databuf = buf[:storage.DataPagePayloadSize]
) )
s.metrics[rec.MetricID] = &_metric{ s.metrics[rec.MetricID] = &_metric{
metricType: rec.MetricType, metricType: rec.MetricType,
fracDigits: byte(rec.FracDigits), fracDigits: rec.FracDigits,
buffer: buffer, buffer: buf,
timestamps: enc.NewTimeDeltaCompressor(buffer, 0), timestamps: enc.NewTimeDeltaCompressor(databuf, 0),
values: enc.NewValueDeltaCompressor(rec.MetricType, byte(rec.FracDigits), buffer, 0), values: enc.NewValueDeltaCompressor(rec.MetricType, rec.FracDigits, databuf, 0),
} }
} }

48
database/replay_metric.go
View File

@@ -271,57 +271,17 @@ func composeHeadDataPage(databuf []byte, head storage.HeadDataPage) storage.Page
} }
func (s *ReplayMetric) ToMetric() *_metric { func (s *ReplayMetric) ToMetric() *_metric {
var (
requiredSpace = s.tSize + s.vSize databuf := s.buf[:storage.DataPagePayloadSize]
) values := enc.NewValueDeltaCompressor(s.metricType, s.fracDigits, databuf, s.vSize)
bufferSize := calculateDataBufferSize(requiredSpace)
if len(s.buf) > bufferSize {
s.buf = shrinkBuffer(shrinkBufferIn{
Buffer: s.buf,
TimestampsSize: s.tSize,
ValuesSize: s.vSize,
NewBufferSize: bufferSize,
})
}
values := enc.NewValueDeltaCompressor(s.metricType, s.fracDigits, s.buf, s.vSize)
return &_metric{ return &_metric{
metricType: s.metricType, metricType: s.metricType,
fracDigits: s.fracDigits, fracDigits: s.fracDigits,
lastPageNo: s.lastPageNo, lastPageNo: s.lastPageNo,
lastValue: values.LastValue(), lastValue: values.LastValue(),
buffer: s.buf, buffer: s.buf,
timestamps: enc.NewTimeDeltaCompressor(s.buf, s.tSize), timestamps: enc.NewTimeDeltaCompressor(databuf, s.tSize),
values: values, values: values,
indexLevelTails: s.indexLevelTails, indexLevelTails: s.indexLevelTails,
} }
} }
// func (s *ReplayMetric) ApplyHeadDataPage(p storage.HeadDataPage) {
// // буфер має бути розміру сторінки
// if len(s.buf) != atree.DataPageSize {
// s.buf, s.databuf = growBuffers(growBuffersIn{
// databuf: s.databuf,
// tSize: s.tSize,
// vSize: s.vSize,
// requiredSpace: atree.DataPageSize,
// })
// }
// // дописую дані по зміщенню
// pos := len(s.databuf) - p.TimestampsOffset - len(p.Timestamps)
// copy(s.databuf[pos:], p.Timestamps)
// copy(s.databuf[p.ValuesOffset:], p.Values)
// //
// s.tSize = p.TimestampsOffset + len(p.Timestamps)
// s.tSize = p.ValuesOffset + len(p.Values)
// // fix - заполнить страницу
// // fix - check checksum
// }
// func (s *ReplayMetric) ApplyHeadIndexPages(changedIndexLevels storage.ChangedIndexLevel) {
// for levelIdx, changedLevel := range changedIndexLevels {
//
//
// }
// }

93
database/tmp Normal file
View File

@@ -0,0 +1,93 @@
// IN APPEND MEASURES
// if len(s.buffer) < storage.DataPageSize {
// // allocate bigger buffer
// buf, databuf := growBuffers(growBuffersIn{
// databuf: s.buffer,
// tSize: timestamps.Size(),
// vSize: values.Size(),
// requiredSpace: totalRequiredSpace,
// })
// s.buffer = buf
// // replace buffer in timestamps and values
// timestamps.Rotate(databuf) // fix pos
// values.Rotate(databuf) // fix pos
// } else {
// IN DATABASE
// FIX
// УВАГА!
// Якщо буфер metric.buffer < розміру сторінки - для timestamps доступний весь буфер.
// Якщо буфер досяг розміру сторінки - в timestamps я передаю buffer[:DataPagePayloadSize]
// var dataBufferSizes = []int{
// 1024,
// 2048,
// 4096,
// 8192,
// }
// func calculateDataBufferSize(payloadSize int) int {
// for _, bufferSize := range dataBufferSizes {
// if payloadSize <= bufferSize {
// return bufferSize
// }
// }
// return storage.DataPageSize
// }
// func allocateBuffers(requiredSpace int) (buf, databuf []byte) {
// bufferSize := calculateDataBufferSize(requiredSpace)
// buf = make([]byte, bufferSize)
// if bufferSize == storage.DataPageSize {
// databuf = buf[:storage.DataPagePayloadSize]
// } else {
// databuf = buf
// }
// return
// }
// // src - databuf
// type growBuffersIn struct {
// databuf []byte
// tSize int
// vSize int
// requiredSpace int
// }
// func growBuffers(in growBuffersIn) (buf, databuf []byte) {
// buf, databuf = allocateBuffers(in.requiredSpace)
// copy(databuf[len(databuf)-in.tSize:], in.databuf[len(in.databuf)-in.tSize:])
// copy(databuf, in.databuf[:in.vSize])
// return
// }
// type shrinkBufferIn struct {
// Buffer []byte
// TimestampsSize int
// ValuesSize int
// NewBufferSize int
// }
// func shrinkBuffer(in shrinkBufferIn) []byte {
// buf := make([]byte, in.NewBufferSize)
// copy(buf, in.Buffer[:in.ValuesSize])
// copy(buf[:len(buf)-in.TimestampsSize], in.Buffer[len(in.Buffer)-in.TimestampsSize:])
// return buf
// }
// IN REPLAY_METRIC.TO_METRIC
// var (
// requiredSpace = s.tSize + s.vSize
// )
// bufferSize := calculateDataBufferSize(requiredSpace)
// if len(s.buf) > bufferSize {
// s.buf = shrinkBuffer(shrinkBufferIn{
// Buffer: s.buf,
// TimestampsSize: s.tSize,
// ValuesSize: s.vSize,
// NewBufferSize: bufferSize,
// })
// }

658
enc/enc_test.go
View File

@@ -19,9 +19,8 @@ func equalFloatSlices(a, b []float64, epsilon float64) bool {
// CUMULATIVE // CUMULATIVE
func TestCumulativeDeltaCompressor(t *testing.T) {
var ( var (
testCases = []struct { cumulativeTestCases = []struct {
Nums []float64 Nums []float64
Name string Name string
Offset int Offset int
@@ -175,7 +174,8 @@ func TestCumulativeDeltaCompressor(t *testing.T) {
} }
) )
for _, testCase := range testCases { func TestCumulativeDeltaCompressor(t *testing.T) {
for _, testCase := range cumulativeTestCases {
var ( var (
tmp = make([]byte, tmpValueSize) tmp = make([]byte, tmpValueSize)
metricType = qb.Cumulative metricType = qb.Cumulative
@@ -212,77 +212,37 @@ func TestCumulativeDeltaCompressor(t *testing.T) {
} }
} }
func TestCumulativeDeltaDecompressorFromState(t *testing.T) { func TestRestoreCumulativeDeltaCompressor(t *testing.T) {
for _, testCase := range cumulativeTestCases {
var ( var (
testCases = []struct { tmp = make([]byte, tmpValueSize)
Nums []float64 metricType = qb.Cumulative
//RepeatLastDeltaNTimes int fracDigits byte = 1
}{ buf = make([]byte, 8)
{ payloadSize = 0
// addBaseValue report qb.ValueEvaluationReport
Nums: []float64{
1.5,
},
},
{
// startRun
Nums: []float64{
1.5,
1.5,
},
},
{
// startRun
Nums: []float64{
1.5,
1.6,
1.6,
},
},
{
// incrementRun
Nums: []float64{
1.5,
1.5,
1.5,
},
},
{
// endSeries run
Nums: []float64{
1.5,
1.5,
1.5,
1.6,
},
},
{
// incrementLiteral
Nums: []float64{
1.5,
1.6,
},
},
{
// ...
Nums: []float64{
1.5,
1.5,
1.5, //
1.5, //
1.6, //
1.7, //
1.8, //
1.9, //
2.0, //
2.0, //
2.0, //
2.2, //
},
},
}
) )
for caseIdx, testCase := range testCases { c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize)
for _, num := range testCase.Nums {
report = c.Evaluate(tmp, num)
c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
}
restored := NewValueDeltaCompressor(metricType, fracDigits, buf, c.Size())
if restored.baseValue != testCase.BaseValue {
t.Fatalf("%s: got baseValue %v are not equal expected %v",
testCase.Name, restored.baseValue, testCase.BaseValue)
}
if restored.lastDelta != testCase.LastDelta {
t.Fatalf("%s: got lastDelta %d are not equal expected %d",
testCase.Name, restored.lastDelta, testCase.LastDelta)
}
}
}
func TestCumulativeDeltaDecompressorFromState(t *testing.T) {
for _, testCase := range cumulativeTestCases {
var ( var (
metricType = qb.Cumulative metricType = qb.Cumulative
fracDigits byte = 1 fracDigits byte = 1
@@ -297,18 +257,9 @@ func TestCumulativeDeltaDecompressorFromState(t *testing.T) {
c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta) c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
} }
// d := c.CreateDecompressor(metricType, fracDigits)
c.CaptureState()
fmt.Println("---------------")
d := c.CreateDecompressor()
//fmt.Println(buf)
for { for {
num, done := d.NextValue() num, done := d.NextValue()
//fmt.Println(num, done)
if done { if done {
break break
} }
@@ -317,21 +268,52 @@ func TestCumulativeDeltaDecompressorFromState(t *testing.T) {
slices.Reverse(decodedNums) slices.Reverse(decodedNums)
fmt.Println(testCase.Nums) if !equalFloatSlices(testCase.Nums, decodedNums, 0.0000001) {
fmt.Println(decodedNums) t.Fatalf("%s: got nums %v not equal expected %v",
fmt.Println() testCase.Name, decodedNums, testCase.Nums)
}
}
}
func TestCumulativeDeltaDecompressorFromEnd(t *testing.T) {
for _, testCase := range cumulativeTestCases {
var (
metricType = qb.Cumulative
fracDigits byte = 1
tmp = make([]byte, tmpValueSize)
buf = make([]byte, 16)
payloadSize = 0
decodedNums []float64
)
c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize)
for _, num := range testCase.Nums {
report := c.Evaluate(tmp, num)
c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
}
d := NewValueDeltaDecompressor(metricType, fracDigits)
d.RestoreFromEnd(buf[:c.Size()])
for {
num, done := d.NextValue()
if done {
break
}
decodedNums = append(decodedNums, num)
}
slices.Reverse(decodedNums)
if !equalFloatSlices(testCase.Nums, decodedNums, 0.0000001) { if !equalFloatSlices(testCase.Nums, decodedNums, 0.0000001) {
t.Fatalf("%d: got nums %v not equal expected %v", caseIdx, decodedNums, testCase.Nums) t.Fatalf("%s: got nums %v not equal expected %v",
testCase.Name, decodedNums, testCase.Nums)
} }
} }
} }
// INSTANT // INSTANT
func TestInstantDeltaCompressor(t *testing.T) {
var ( var (
testCases = []struct { instantTestCases = []struct {
Nums []float64 Nums []float64
Name string Name string
Offset int Offset int
@@ -391,6 +373,23 @@ func TestInstantDeltaCompressor(t *testing.T) {
}, },
{ {
Nums: []float64{ Nums: []float64{
-1.5,
-1.5,
},
Name: "literal switch to run (negative)",
Offset: 1,
ChangeSize: 1,
Buf: []byte{
0x9d, // base value
0x80, // delta 0
0x00, // run (len = 2)
0x00, 0x00, 0x00, 0x00, 0x00,
},
BaseValue: -1.5,
LastDelta: 0,
},
{
Nums: []float64{
1.5, 1.5,
1.6, 1.6,
1.6, 1.6,
@@ -411,6 +410,26 @@ func TestInstantDeltaCompressor(t *testing.T) {
}, },
{ {
Nums: []float64{ Nums: []float64{
-1.5,
-1.6,
-1.6,
},
Name: "literal decrease by 1 and switch to run (negative)",
Offset: 2,
ChangeSize: 3,
Buf: []byte{
0x9d, // base value
0x80, // delta 0
0x80, // literal (len = 1)
0x81, // delta -1
0x00, // run (len = 2)
0x00, 0x00, 0x00,
},
BaseValue: -1.5,
LastDelta: bin.EncodeZigZag(-1),
},
{
Nums: []float64{
1.5, 1.5,
1.4, 1.4,
1.4, 1.4,
@@ -431,6 +450,26 @@ func TestInstantDeltaCompressor(t *testing.T) {
}, },
{ {
Nums: []float64{ Nums: []float64{
-1.5,
-1.4,
-1.4,
},
Name: "literal decrease by 1 and switch to run (positive delta, negative)",
Offset: 2,
ChangeSize: 3,
Buf: []byte{
0x9d, // base value
0x80, // delta 0
0x80, // literal (len = 1)
0x82, // delta 1
0x00, // run (len = 2)
0x00, 0x00, 0x00,
},
BaseValue: -1.5,
LastDelta: bin.EncodeZigZag(1),
},
{
Nums: []float64{
1.5, 1.5,
1.5, 1.5,
1.5, 1.5,
@@ -449,6 +488,24 @@ func TestInstantDeltaCompressor(t *testing.T) {
}, },
{ {
Nums: []float64{ Nums: []float64{
-1.5,
-1.5,
-1.5,
},
Name: "increment run (negative)",
Offset: 1,
ChangeSize: 1,
Buf: []byte{
0x9d, // base value
0x80, // delta 0
0x01, // run (len = 3)
0x00, 0x00, 0x00, 0x00, 0x00,
},
BaseValue: -1.5,
LastDelta: 0,
},
{
Nums: []float64{
1.5, 1.5,
1.5, 1.5,
1.5, 1.5,
@@ -470,6 +527,27 @@ func TestInstantDeltaCompressor(t *testing.T) {
}, },
{ {
Nums: []float64{ Nums: []float64{
-1.5,
-1.5,
-1.5,
-1.6,
},
Name: "run switch to literal (negative)",
Offset: 0,
ChangeSize: 2,
Buf: []byte{
0x9d, // base value
0x80, // delta 0
0x01, // run (len = 3)
0x81, // delta -1
0x80, // literal (len = 1)
0x00, 0x00, 0x00,
},
BaseValue: -1.5,
LastDelta: bin.EncodeZigZag(-1),
},
{
Nums: []float64{
1.5, 1.5,
1.5, 1.5,
1.5, 1.5,
@@ -490,6 +568,27 @@ func TestInstantDeltaCompressor(t *testing.T) {
LastDelta: bin.EncodeZigZag(-1), LastDelta: bin.EncodeZigZag(-1),
}, },
{ {
Nums: []float64{
-1.5,
-1.5,
-1.5,
-1.4,
},
Name: "run switch to literal (positive delta, negative version)",
Offset: 0,
ChangeSize: 2,
Buf: []byte{
0x9d, // base value
0x80, // delta 0
0x01, // run (len = 3)
0x82, // delta 1
0x80, // literal (len = 1)
0x00, 0x00, 0x00,
},
BaseValue: -1.5,
LastDelta: bin.EncodeZigZag(1),
},
{
Nums: repeatFloat64(1.5, 129), Nums: repeatFloat64(1.5, 129),
Name: "increment run to full fill h-byte", Name: "increment run to full fill h-byte",
Offset: 1, Offset: 1,
@@ -504,6 +603,20 @@ func TestInstantDeltaCompressor(t *testing.T) {
LastDelta: 0, LastDelta: 0,
}, },
{ {
Nums: repeatFloat64(-1.5, 129),
Name: "increment run to full fill h-byte (negative version)",
Offset: 1,
ChangeSize: 1,
Buf: []byte{
0x9d, // base value
0x80, // delta 0
0x7f, // run (len = 129)
0x00, 0x00, 0x00, 0x00, 0x00,
},
BaseValue: -1.5,
LastDelta: 0,
},
{
Nums: repeatFloat64(1.5, 130), Nums: repeatFloat64(1.5, 130),
Name: "run switch to literal after h-byte overflow", Name: "run switch to literal after h-byte overflow",
Offset: 0, Offset: 0,
@@ -520,6 +633,22 @@ func TestInstantDeltaCompressor(t *testing.T) {
LastDelta: 0, LastDelta: 0,
}, },
{ {
Nums: repeatFloat64(-1.5, 130),
Name: "run switch to literal after h-byte overflow (negative version)",
Offset: 0,
ChangeSize: 2,
Buf: []byte{
0x9d, // base value
0x80, // delta 0
0x7f, // run (len = 129)
0x80, // delta 0
0x80, // literal (len = 1)
0x00, 0x00, 0x00,
},
BaseValue: -1.5,
LastDelta: 0,
},
{
Nums: []float64{ Nums: []float64{
1.5, 1.5,
1.6, 1.6,
@@ -541,6 +670,26 @@ func TestInstantDeltaCompressor(t *testing.T) {
}, },
{ {
Nums: []float64{ Nums: []float64{
-1.5,
-1.6,
-1.7,
},
Name: "increment literal (negative version)",
Offset: 1,
ChangeSize: 2,
Buf: []byte{
0x9d, // base value
0x80, // delta 0
0x81, // delta -1
0x83, // delta -2
0x82, // literal (len = 3)
0x00, 0x00, 0x00,
},
BaseValue: -1.5,
LastDelta: bin.EncodeZigZag(-2),
},
{
Nums: []float64{
1.5, 1.5,
1.4, 1.4,
1.3, 1.3,
@@ -559,12 +708,31 @@ func TestInstantDeltaCompressor(t *testing.T) {
BaseValue: 1.5, BaseValue: 1.5,
LastDelta: bin.EncodeZigZag(-2), LastDelta: bin.EncodeZigZag(-2),
}, },
{
Nums: []float64{
-1.5,
-1.4,
-1.3,
},
Name: "increment literal (positive delta, negative version)",
Offset: 1,
ChangeSize: 2,
Buf: []byte{
0x9d, // base value
0x80, // delta 0
0x82, // delta 1
0x84, // delta 2
0x82, // literal (len = 3)
0x00, 0x00, 0x00,
},
BaseValue: -1.5,
LastDelta: bin.EncodeZigZag(2),
},
} }
) )
for _, testCase := range testCases { func TestInstantDeltaCompressor(t *testing.T) {
// fmt.Println("-----") for _, testCase := range instantTestCases {
// fmt.Println("nums", testCase.Nums)
var ( var (
tmp = make([]byte, tmpValueSize) tmp = make([]byte, tmpValueSize)
metricType = qb.Instant metricType = qb.Instant
@@ -601,77 +769,35 @@ func TestInstantDeltaCompressor(t *testing.T) {
} }
} }
func TestInstantDeltaDecompressorFromState(t *testing.T) { func TestRestoreInstantDeltaCompressor(t *testing.T) {
for _, testCase := range instantTestCases {
var ( var (
testCases = []struct { tmp = make([]byte, tmpValueSize)
Nums []float64 metricType = qb.Instant
//RepeatLastDeltaNTimes int fracDigits byte = 1
}{ buf = make([]byte, 8)
{ payloadSize = 0
// addBaseValue report qb.ValueEvaluationReport
Nums: []float64{
1.5,
},
},
{
// startRun
Nums: []float64{
1.5,
1.5,
},
},
{
// startRun
Nums: []float64{
1.5,
1.6,
1.6,
},
},
{
// incrementRun
Nums: []float64{
1.5,
1.5,
1.5,
},
},
{
// endSeries run
Nums: []float64{
1.5,
1.5,
1.5,
1.6,
},
},
{
// incrementLiteral
Nums: []float64{
1.5,
1.6,
},
},
{
// ...
Nums: []float64{
1.5,
1.5,
1.5, //
1.5, //
1.6, //
1.7, //
1.8, //
1.9, //
2.0, //
2.0, //
2.0, //
2.2, //
},
},
}
) )
for caseIdx, testCase := range testCases { c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize)
for _, num := range testCase.Nums {
report = c.Evaluate(tmp, num)
c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
}
restored := NewValueDeltaCompressor(metricType, fracDigits, buf, c.Size())
if restored.baseValue != testCase.BaseValue {
t.Fatalf("%s: got baseValue %v are not equal expected %v",
testCase.Name, restored.baseValue, testCase.BaseValue)
}
if restored.lastDelta != testCase.LastDelta {
t.Fatalf("%s: got lastDelta %d are not equal expected %d",
testCase.Name, restored.lastDelta, testCase.LastDelta)
}
}
}
func TestInstantDeltaDecompressorFromState(t *testing.T) {
for _, testCase := range instantTestCases {
var ( var (
metricType = qb.Instant metricType = qb.Instant
fracDigits byte = 1 fracDigits byte = 1
@@ -686,18 +812,9 @@ func TestInstantDeltaDecompressorFromState(t *testing.T) {
c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta) c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
} }
// d := c.CreateDecompressor(metricType, fracDigits)
c.CaptureState()
fmt.Println("---------------")
d := c.CreateDecompressor()
//fmt.Println(buf)
for { for {
num, done := d.NextValue() num, done := d.NextValue()
//fmt.Println(num, done)
if done { if done {
break break
} }
@@ -706,21 +823,52 @@ func TestInstantDeltaDecompressorFromState(t *testing.T) {
slices.Reverse(decodedNums) slices.Reverse(decodedNums)
fmt.Println(testCase.Nums) if !equalFloatSlices(testCase.Nums, decodedNums, 0.0000001) {
fmt.Println(decodedNums) t.Fatalf("%s: got nums %v not equal expected %v",
fmt.Println() testCase.Name, decodedNums, testCase.Nums)
}
}
}
func TestInstantDeltaDecompressorFromEnd(t *testing.T) {
for _, testCase := range instantTestCases {
var (
metricType = qb.Instant
fracDigits byte = 1
tmp = make([]byte, tmpValueSize)
buf = make([]byte, 8)
payloadSize = 0
decodedNums []float64
)
c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize)
for _, num := range testCase.Nums {
report := c.Evaluate(tmp, num)
c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
}
d := NewValueDeltaDecompressor(metricType, fracDigits)
d.RestoreFromEnd(buf[:c.Size()])
for {
num, done := d.NextValue()
if done {
break
}
decodedNums = append(decodedNums, num)
}
slices.Reverse(decodedNums)
if !equalFloatSlices(testCase.Nums, decodedNums, 0.0000001) { if !equalFloatSlices(testCase.Nums, decodedNums, 0.0000001) {
t.Fatalf("%d: got nums %v not equal expected %v", caseIdx, decodedNums, testCase.Nums) t.Fatalf("%s: got nums %v not equal expected %v",
testCase.Name, decodedNums, testCase.Nums)
} }
} }
} }
// TIME DELTA // TIME DELTA
func TestTimeDeltaCompressor(t *testing.T) {
var ( var (
testCases = []struct { timeTestCases = []struct {
Name string Name string
Nums []uint32 Nums []uint32
Buf []byte Buf []byte
@@ -889,8 +1037,8 @@ func TestTimeDeltaCompressor(t *testing.T) {
} }
) )
for _, testCase := range testCases { func TestTimeDeltaCompressor(t *testing.T) {
//fmt.Println("----------") for _, testCase := range timeTestCases {
var ( var (
tmp = make([]byte, tmpTimeSize) tmp = make([]byte, tmpTimeSize)
buf = make([]byte, 8) buf = make([]byte, 8)
@@ -900,11 +1048,6 @@ func TestTimeDeltaCompressor(t *testing.T) {
c := NewTimeDeltaCompressor(buf, payloadSize) c := NewTimeDeltaCompressor(buf, payloadSize)
for _, num := range testCase.Nums { for _, num := range testCase.Nums {
report = c.Evaluate(tmp, num) report = c.Evaluate(tmp, num)
// fmt.Println("----------\npos", c.pos)
// fmt.Printf("buf: % x\n", c.buf[c.pos:])
// pretty.Println(report)
// fmt.Printf("change: % x\n", tmp[:report.ChangeSize])
c.Append(report.Offset, tmp[:report.ChangeSize], num) c.Append(report.Offset, tmp[:report.ChangeSize], num)
} }
if report.Offset != testCase.Offset { if report.Offset != testCase.Offset {
@@ -930,82 +1073,36 @@ func TestTimeDeltaCompressor(t *testing.T) {
} }
} }
func TestTimeDeltaDecompressorFromState(t *testing.T) { func TestRestoreTimeDeltaCompressor(t *testing.T) {
var ( for _, testCase := range timeTestCases {
testCases = []struct {
Nums []uint32
//RepeatLastDeltaNTimes int
}{
{
Nums: []uint32{
1780777000,
},
},
{
// add1stDelta
Nums: []uint32{
1780777000,
1780777060, // +60
},
},
{
// startRun
Nums: []uint32{
1780777000,
1780777060, // +60
1780777120, // +60
},
},
{
// incrementRun
Nums: []uint32{
1780777000,
1780777060, // +60
1780777120, // +60
1780777180, // +60
},
},
{
// endRun
Nums: []uint32{
1780777000,
1780777060, // +60
1780777120, // +60
1780777180, // +60
1780777200, // +20
},
},
{
// incrementLiteral
Nums: []uint32{
1780777000,
1780777060, // +60
1780777122, // +62
},
},
{
// ...
Nums: []uint32{
1780777000,
1780777010, // +10
1780777120, // +10
1780777130, // +10
1780777135, // +5
1780777141, // +6
1780777148, // +7
1780777156, // +8
1780777165, // +9
1780777174, // +9
1780777183, // +9
1780777190, // +7
},
},
}
)
for caseIdx, testCase := range testCases {
var ( var (
tmp = make([]byte, tmpTimeSize) tmp = make([]byte, tmpTimeSize)
buf = make([]byte, 16) buf = make([]byte, 8)
payloadSize = 0
report qb.TimeEvaluationReport
)
c := NewTimeDeltaCompressor(buf, payloadSize)
for _, num := range testCase.Nums {
report = c.Evaluate(tmp, num)
c.Append(report.Offset, tmp[:report.ChangeSize], num)
}
restored := NewTimeDeltaCompressor(buf, c.Size())
if restored.lastUnixtime != testCase.LastUnixtime {
t.Fatalf("%s: got lastUnixtime %d are not equal expected %d",
testCase.Name, restored.lastUnixtime, testCase.LastUnixtime)
}
if restored.lastDelta != testCase.LastDelta {
t.Fatalf("%s: got lastDelta %d are not equal expected %d",
testCase.Name, restored.lastDelta, testCase.LastDelta)
}
}
}
func TestTimeDeltaDecompressorFromState(t *testing.T) {
for _, testCase := range timeTestCases {
var (
tmp = make([]byte, tmpTimeSize)
buf = make([]byte, 8)
payloadSize = 0 payloadSize = 0
decodedNums []uint32 decodedNums []uint32
) )
@@ -1015,18 +1112,9 @@ func TestTimeDeltaDecompressorFromState(t *testing.T) {
c.Append(report.Offset, tmp[:report.ChangeSize], num) c.Append(report.Offset, tmp[:report.ChangeSize], num)
} }
//
c.CaptureState()
d := c.CreateDecompressor() d := c.CreateDecompressor()
//fmt.Println(testCase.Nums)
// fmt.Println("---------------")
// fmt.Println(buf)
for { for {
num, done := d.NextValue() num, done := d.NextValue()
//fmt.Println(num, done)
if done { if done {
break break
} }
@@ -1036,7 +1124,43 @@ func TestTimeDeltaDecompressorFromState(t *testing.T) {
slices.Reverse(decodedNums) slices.Reverse(decodedNums)
if !slices.Equal(testCase.Nums, decodedNums) { if !slices.Equal(testCase.Nums, decodedNums) {
t.Fatalf("%d: got nums %v not equal expected %v", caseIdx, decodedNums, testCase.Nums) t.Fatalf("%s: got nums %v not equal expected %v",
testCase.Name, decodedNums, testCase.Nums)
}
}
}
func TestTimeDeltaDecompressorFromEnd(t *testing.T) {
for _, testCase := range timeTestCases {
var (
tmp = make([]byte, tmpTimeSize)
buf = make([]byte, 8)
payloadSize = 0
decodedNums []uint32
)
c := NewTimeDeltaCompressor(buf, payloadSize)
for _, num := range testCase.Nums {
report := c.Evaluate(tmp, num)
c.Append(report.Offset, tmp[:report.ChangeSize], num)
}
c.ReplaceSinceWithUntil()
d := NewTimeDeltaDecompressor()
d.RestoreFromEnd(buf[len(buf)-c.Size():])
for {
num, done := d.NextValue()
if done {
break
}
decodedNums = append(decodedNums, num)
}
slices.Reverse(decodedNums)
if !slices.Equal(testCase.Nums, decodedNums) {
t.Fatalf("%s: got nums %v not equal expected %v",
testCase.Name, decodedNums, testCase.Nums)
} }
} }
} }

176
enc/time_delta.go
View File

@@ -28,6 +28,7 @@ type TimeDeltaCompressor struct {
pos int pos int
lastUnixtime uint32 lastUnixtime uint32
lastDelta uint32 lastDelta uint32
state *qb.TimeDeltaCapturedState
} }
// Початкове створення, коли даних немає // Початкове створення, коли даних немає
@@ -42,28 +43,56 @@ func NewTimeDeltaCompressor(buf []byte, payloadSize int) *TimeDeltaCompressor {
pos: len(buf) - payloadSize, pos: len(buf) - payloadSize,
} }
if payloadSize > 0 { if payloadSize > 0 {
var err error s.restore(payloadSize)
s.lastUnixtime, err = bin.GetUint32(s.buf[len(s.buf)-4:]) }
return s
}
func (s *TimeDeltaCompressor) restore(payloadSize int) {
bound := len(s.buf) - 4
since, err := bin.GetUint32(s.buf[bound:])
if err != nil { if err != nil {
log.Fatalf("bug: get since: %s", err) log.Fatalf("bug: get since: %s", err)
} }
//s.lastUnixtime = lastTimestamp // fix - порахувати if payloadSize == 4 {
s.lastUnixtime = since
if payloadSize > 4 { return
// u64, _, err := bin.GetVarUint64(s.buf[s.pos+1:]) }
// if err != nil { var (
// log.Fatalf("bug: get last delta: %s", err) i = s.pos
// } totalDelta uint64
// s.lastDelta = uint32(u64) )
for i < bound {
h := s.buf[i]
i++
if h < 128 {
// run
count := h + 2
delta, n, _ := bin.GetVarUint64(s.buf[i:])
i += n
totalDelta += delta * uint64(count)
} else {
// literal
count := (h & 127) + 1
for range count {
delta, n, _ := bin.GetVarUint64(s.buf[i:])
i += n
totalDelta += delta
} }
} }
return s }
u64, _, err := bin.GetVarUint64(s.buf[s.pos+1:])
if err != nil {
log.Fatalf("bug: get last delta: %s", err)
}
s.lastDelta = uint32(u64)
s.lastUnixtime = since + uint32(totalDelta)
} }
func (s *TimeDeltaCompressor) Evaluate(tmp []byte, timestamp uint32) qb.TimeEvaluationReport { func (s *TimeDeltaCompressor) Evaluate(tmp []byte, timestamp uint32) qb.TimeEvaluationReport {
var ( var (
delta = timestamp - s.lastUnixtime delta = timestamp - s.lastUnixtime
offset int rewindOffset int
i int i int
) )
if s.pos < len(s.buf) { if s.pos < len(s.buf) {
@@ -75,7 +104,7 @@ func (s *TimeDeltaCompressor) Evaluate(tmp []byte, timestamp uint32) qb.TimeEval
// incrementRun // incrementRun
tmp[i] = h + 1 tmp[i] = h + 1
i++ i++
offset = 1 // перезапис h rewindOffset = 1 // перезапис h
} else { } else {
// endSeries // endSeries
n, _ := bin.PutVarUint64(tmp, uint64(delta)) n, _ := bin.PutVarUint64(tmp, uint64(delta))
@@ -92,7 +121,7 @@ func (s *TimeDeltaCompressor) Evaluate(tmp []byte, timestamp uint32) qb.TimeEval
i += n i += n
tmp[i] = h + 1 tmp[i] = h + 1
i++ i++
offset = 1 // перезапис h rewindOffset = 1 // перезапис h
} else { } else {
// endSeries // endSeries
n, _ := bin.PutVarUint64(tmp, uint64(delta)) n, _ := bin.PutVarUint64(tmp, uint64(delta))
@@ -109,11 +138,11 @@ func (s *TimeDeltaCompressor) Evaluate(tmp []byte, timestamp uint32) qb.TimeEval
i += n i += n
tmp[i] = 0 // start new run (length=2) tmp[i] = 0 // start new run (length=2)
i++ i++
offset = 1 + n // перезапис пари delta/h rewindOffset = 1 + n // перезапис пари delta/h
} else { } else {
tmp[i] = 0 // change literal (length=1) to run (length=2) tmp[i] = 0 // change literal (length=1) to run (length=2)
i++ i++
offset = 1 rewindOffset = 1
} }
} }
} }
@@ -129,15 +158,16 @@ func (s *TimeDeltaCompressor) Evaluate(tmp []byte, timestamp uint32) qb.TimeEval
i += 4 i += 4
} }
return qb.TimeEvaluationReport{ return qb.TimeEvaluationReport{
Offset: offset, RewindOffset: rewindOffset,
Offset: s.pos + rewindOffset,
ChangeSize: i, ChangeSize: i,
TotalSpace: s.pos - offset + i, TotalSpace: s.pos - rewindOffset + i,
} }
} }
func (s *TimeDeltaCompressor) Append(offset int, change []byte, timestamp uint32) { func (s *TimeDeltaCompressor) Append(rewindOffset int, change []byte, timestamp uint32) {
if s.pos < len(s.buf) { if s.pos < len(s.buf) {
i := s.pos + offset - 1 // -1, because s.pos always points to h byte i := s.pos + rewindOffset - 1 // -1, because s.pos always points to h byte
for _, b := range change { for _, b := range change {
s.buf[i] = b s.buf[i] = b
i-- i--
@@ -146,7 +176,7 @@ func (s *TimeDeltaCompressor) Append(offset int, change []byte, timestamp uint32
} else { } else {
copy(s.buf[len(s.buf)-4:], change) // 4b since copy(s.buf[len(s.buf)-4:], change) // 4b since
} }
s.pos -= len(change) - offset s.pos -= len(change) - rewindOffset
s.lastUnixtime = timestamp s.lastUnixtime = timestamp
} }
@@ -154,17 +184,25 @@ func (s *TimeDeltaCompressor) Append(offset int, change []byte, timestamp uint32
// } // }
// delta h func (s *TimeDeltaCompressor) getState() qb.TimeDeltaCapturedState {
func (s *TimeDeltaCompressor) CaptureState() qb.TimeDeltaCapturedState { bound := s.pos + 1 + bin.CountVarUint64(uint64(s.lastDelta))
// позиція посувається вліво, отже може перескочити на попередній chunk
return qb.TimeDeltaCapturedState{ return qb.TimeDeltaCapturedState{
H: s.buf[s.pos], H: s.buf[s.pos],
LastUnixtime: s.lastUnixtime, LastUnixtime: s.lastUnixtime,
LastDelta: s.lastDelta, LastDelta: s.lastDelta,
Payload: s.buf[s.pos:], Payload: s.buf[bound:],
} }
} }
func (s *TimeDeltaCompressor) CaptureState() {
state := s.getState()
s.state = &state
}
func (s *TimeDeltaCompressor) Tail(offset int) []byte {
return s.buf[s.pos : len(s.buf)-offset]
}
// коли сторінка заповнена і відправляється в txlog, since замінюю на until // коли сторінка заповнена і відправляється в txlog, since замінюю на until
func (s *TimeDeltaCompressor) ReplaceSinceWithUntil() uint32 { func (s *TimeDeltaCompressor) ReplaceSinceWithUntil() uint32 {
pos := len(s.buf) - 4 pos := len(s.buf) - 4
@@ -178,12 +216,13 @@ func (s *TimeDeltaCompressor) Size() int {
return len(s.buf) - s.pos return len(s.buf) - s.pos
} }
// if s.state == nil { func (s *TimeDeltaCompressor) StoredSize() int {
// return len(s.buf) - s.pos if s.state == nil {
// } else { return len(s.buf) - s.pos
// return bin.CountVarUint64(uint64(s.state.LastDelta)) + hSize + len(s.state.Payload) } else {
// } return bin.CountVarUint64(uint64(s.state.LastDelta)) + 1 + len(s.state.Payload)
// } }
}
// Snapshot - для створення снапшота. // Snapshot - для створення снапшота.
// FIX - encode firstUnixtime in 1st 4 bytes // FIX - encode firstUnixtime in 1st 4 bytes
@@ -217,9 +256,7 @@ func (s *TimeDeltaCompressor) WritePayloadTo(w io.Writer, state *qb.TimeDeltaCap
} }
} }
func (s *TimeDeltaCompressor) Rotate(newbuf []byte) { func (s *TimeDeltaCompressor) ReplaceBuffer(newbuf []byte) {
// УВАГА!
// state не чіпаємо
s.buf = newbuf s.buf = newbuf
s.pos = len(s.buf) s.pos = len(s.buf)
s.lastUnixtime = 0 s.lastUnixtime = 0
@@ -230,12 +267,18 @@ func (s *TimeDeltaCompressor) CreateDecompressor() qb.TimestampDecompressor {
if s.pos == len(s.buf) { if s.pos == len(s.buf) {
return nil return nil
} }
return NewTimeDeltaDecompressorFromState(TimeDeltaDecompressorFromStateOptions{ d := NewTimeDeltaDecompressor()
H: s.buf[s.pos], if s.state != nil {
LastUnixtime: s.lastUnixtime, d.RestoreFromState(*s.state)
LastDelta: s.lastDelta, } else {
Payload: s.buf[s.pos:], d.RestoreFromState(s.getState())
}) }
// fmt.Printf("h: % x\n", s.buf[s.pos])
// fmt.Printf("lastUnixtime: % x\n", s.lastUnixtime)
// fmt.Printf("lastDelta: % x\n", s.lastDelta)
// fmt.Printf("buf: % x\n", s.buf)
// fmt.Printf("pay: % x\n", s.buf[bound:])
return d
} }
func (s *TimeDeltaCompressor) LastTimestamp() uint32 { func (s *TimeDeltaCompressor) LastTimestamp() uint32 {
@@ -254,66 +297,57 @@ type TimeDeltaDecompressor struct {
done bool done bool
} }
func NewTimeDeltaDecompressor(buf []byte) *TimeDeltaDecompressor { func NewTimeDeltaDecompressor() *TimeDeltaDecompressor {
s := &TimeDeltaDecompressor{ return new(TimeDeltaDecompressor)
buf: buf,
} }
if s.pos < len(s.buf) {
// викликається для повних сторінок
func (s *TimeDeltaDecompressor) RestoreFromEnd(buf []byte) {
s.buf = buf
var err error var err error
s.lastUnixtime, err = bin.GetUint32(s.buf) s.lastUnixtime, err = bin.GetUint32(buf[len(buf)-4:])
if err != nil { if err != nil {
log.Fatalf("bug: get last unixtime: %s", err) log.Fatalf("bug: get last unixtime: %s", err)
} }
s.pos += 4 if len(buf) > 4 {
} else { s.readHeader()
s.done = true s.readDelta()
} }
return s // fmt.Printf("h: % x\n", s.buf[s.pos])
// fmt.Printf("lastUnixtime: %d\n", s.lastUnixtime)
// fmt.Printf("lastDelta: %d\n", s.lastDelta)
// fmt.Printf("buf: % x\n", s.buf)
} }
type TimeDeltaDecompressorFromStateOptions struct { // викликається для data level tail
H byte func (s *TimeDeltaDecompressor) RestoreFromState(state qb.TimeDeltaCapturedState) {
LastUnixtime uint32 s.buf = state.Payload
LastDelta uint32 s.lastUnixtime = state.LastUnixtime
Payload []byte if state.LastDelta > 0 {
s.lastDelta = state.LastDelta
s.decodeHeaderByte(state.H)
} }
func NewTimeDeltaDecompressorFromState(opt TimeDeltaDecompressorFromStateOptions) *TimeDeltaDecompressor {
s := &TimeDeltaDecompressor{
buf: opt.Payload,
lastUnixtime: opt.LastUnixtime,
}
if opt.LastDelta > 0 {
s.lastDelta = opt.LastDelta
s.decodeHeaderByte(opt.H)
}
return s
} }
func (s *TimeDeltaDecompressor) NextValue() (value uint32, done bool) { func (s *TimeDeltaDecompressor) NextValue() (value uint32, done bool) {
if s.done { if s.done {
return 0, true return 0, true
} }
// повертаю значення, що було прочитано в методі RestoreFromBound/RestoreFromEnd
value = s.lastUnixtime value = s.lastUnixtime
if s.lastDelta > 0 { if s.lastDelta > 0 {
s.lastUnixtime -= s.lastDelta s.lastUnixtime -= s.lastDelta
s.pending-- s.pending--
if s.pending > 0 { if s.pending > 0 {
// якщо в серії залишаються елементи
if !s.isRun { if !s.isRun {
s.readDelta() s.readDelta()
} }
} else if s.pos < len(s.buf) { } else if s.pos < len(s.buf)-4 {
// в серії більше немає елементів, отже перевіряє чи є ще дані в буфері.
// дані є - читаю заголовок наступної серії
s.readHeader() s.readHeader()
s.readDelta() s.readDelta()
} else { } else {
s.lastDelta = 0 s.lastDelta = 0
} }
} else { } else {
// був закодований лише last unixtime
s.done = true s.done = true
} }
return value, false return value, false

163
enc/value_delta.go
View File

@@ -25,19 +25,19 @@ type ValueDeltaCompressor struct {
toUint64 func(float64, float64) uint64 toUint64 func(float64, float64) uint64
// (value, coef) => uint64 // (value, coef) => uint64
toFloat64 func(uint64, float64) float64 toFloat64 func(uint64, float64) float64
state *qb.ValueDeltaCapturedState
} }
// Після відновлення із снапшота // Після відновлення із снапшота
func NewValueDeltaCompressor(metricType qb.MetricType, fracDigits byte, buf []byte, payloadSize int) *ValueDeltaCompressor { func NewValueDeltaCompressor(metricType qb.MetricType, fracDigits byte, buf []byte, payloadSize int) *ValueDeltaCompressor {
var coef float64 = 1
if fracDigits > 0 {
coef = math.Pow(10, float64(fracDigits))
}
s := &ValueDeltaCompressor{ s := &ValueDeltaCompressor{
buf: buf, buf: buf,
coef: coef, coef: 1,
pos: payloadSize, pos: payloadSize,
} }
if fracDigits > 0 {
s.coef = math.Pow(10, float64(fracDigits))
}
if metricType == qb.Cumulative { if metricType == qb.Cumulative {
s.calcDelta = calcCumulativeDelta s.calcDelta = calcCumulativeDelta
s.toUint64 = toCumulativeUint64 s.toUint64 = toCumulativeUint64
@@ -48,16 +48,19 @@ func NewValueDeltaCompressor(metricType qb.MetricType, fracDigits byte, buf []by
s.toFloat64 = toInstantFloat64 s.toFloat64 = toInstantFloat64
} }
if payloadSize > 0 { if payloadSize > 0 {
// base value на початку // fmt.Printf("% x\n", buf)
// fmt.Printf("% x\n", buf[:s.pos])
// fmt.Printf("pos: %d\n", s.pos)
u64, _, err := bin.GetVarUint64(s.buf) u64, _, err := bin.GetVarUint64(s.buf)
if err != nil { if err != nil {
log.Fatalf("bug: get base value: %s", err) log.Fatalf("bug: get base value: %s", err)
} }
s.baseValue = float64(u64) / s.coef s.baseValue = s.toFloat64(u64, s.coef)
s.lastDelta, _, err = bin.ReverseGetVarUint64(s.buf[:s.pos-2]) // skip h byte s.lastDelta, _, err = bin.ReverseGetVarUint64(s.buf[:s.pos-1]) // skip h byte
if err != nil { if err != nil {
log.Fatalf("bug: get last delta: %s", err) log.Fatalf("bug: get last delta: %s", err)
} }
//fmt.Printf("lastDelta: %d\n", s.lastDelta)
} }
return s return s
} }
@@ -69,7 +72,7 @@ func NewValueDeltaCompressor(metricType qb.MetricType, fracDigits byte, buf []by
func (s *ValueDeltaCompressor) Evaluate(tmp []byte, value float64) qb.ValueEvaluationReport { func (s *ValueDeltaCompressor) Evaluate(tmp []byte, value float64) qb.ValueEvaluationReport {
var ( var (
delta uint64 delta uint64
offset int rewindOffset int
i int i int
) )
if s.pos > 0 { if s.pos > 0 {
@@ -81,7 +84,7 @@ func (s *ValueDeltaCompressor) Evaluate(tmp []byte, value float64) qb.ValueEvalu
// incrementRun // incrementRun
tmp[i] = h + 1 tmp[i] = h + 1
i++ i++
offset = 1 // перезапис h rewindOffset = 1 // перезапис h
} else { } else {
// endSeries // endSeries
n, _ := bin.ReversePutVarUint64(tmp, delta) n, _ := bin.ReversePutVarUint64(tmp, delta)
@@ -98,7 +101,7 @@ func (s *ValueDeltaCompressor) Evaluate(tmp []byte, value float64) qb.ValueEvalu
i += n i += n
tmp[i] = h + 1 tmp[i] = h + 1
i++ i++
offset = 1 // перезапис h rewindOffset = 1 // перезапис h
} else { } else {
// endSeries // endSeries
n, _ := bin.ReversePutVarUint64(tmp, delta) n, _ := bin.ReversePutVarUint64(tmp, delta)
@@ -115,11 +118,11 @@ func (s *ValueDeltaCompressor) Evaluate(tmp []byte, value float64) qb.ValueEvalu
i += n i += n
tmp[i] = 0 // start new run (length=2) tmp[i] = 0 // start new run (length=2)
i++ i++
offset = 1 + n // перезапис пари delta/h rewindOffset = 1 + n // перезапис пари delta/h
} else { } else {
tmp[i] = 0 // change literal (length=1) to run (length=2) tmp[i] = 0 // change literal (length=1) to run (length=2)
i++ i++
offset = 1 rewindOffset = 1
} }
} }
} }
@@ -132,51 +135,61 @@ func (s *ValueDeltaCompressor) Evaluate(tmp []byte, value float64) qb.ValueEvalu
i++ i++
} }
return qb.ValueEvaluationReport{ return qb.ValueEvaluationReport{
Offset: offset, RewindOffset: rewindOffset,
Offset: s.pos - rewindOffset,
ChangeSize: i, ChangeSize: i,
TotalSpace: s.pos + i - offset, TotalSpace: s.pos + i - rewindOffset,
Delta: delta, Delta: delta,
} }
} }
// pos завжди вказує на h // pos завжди вказує на h
func (s *ValueDeltaCompressor) Append(offset int, change []byte, value float64, delta uint64) { func (s *ValueDeltaCompressor) Append(rewindOffset int, change []byte, value float64, delta uint64) {
if s.pos > 0 { if s.pos > 0 {
s.lastDelta = delta s.lastDelta = delta
} else { } else {
s.baseValue = value s.baseValue = value
} }
copy(s.buf[s.pos-offset:], change) copy(s.buf[s.pos-rewindOffset:], change)
s.pos += len(change) - offset s.pos += len(change) - rewindOffset
} }
func (s *ValueDeltaCompressor) DeleteLast() { func (s *ValueDeltaCompressor) DeleteLast() {
} }
func (s *ValueDeltaCompressor) CaptureState() qb.ValueDeltaCapturedState { func (s *ValueDeltaCompressor) getState() qb.ValueDeltaCapturedState {
// if s.state != nil { bound := s.pos - 1 - bin.CountVarUint64(s.lastDelta)
// qb.Abort(qb.RepeatableLock, nil)
// }
// позиція посувається вліво, отже може перескочити на попередній chunk
pos := s.pos - 1 - bin.CountVarUint64(s.lastDelta)
return qb.ValueDeltaCapturedState{ return qb.ValueDeltaCapturedState{
H: s.buf[s.pos-1], H: s.buf[s.pos-1],
LastDelta: s.lastDelta, LastDelta: s.lastDelta,
Payload: s.buf[:pos], Payload: s.buf[:bound],
} }
} }
func (s *ValueDeltaCompressor) CaptureState() {
state := s.getState()
s.state = &state
}
func (s *ValueDeltaCompressor) Tail(offset int) []byte {
return s.buf[offset:s.pos]
}
// для зростання буфера під час вставки даних. State не цікавить // для зростання буфера під час вставки даних. State не цікавить
func (s *ValueDeltaCompressor) Size() int { func (s *ValueDeltaCompressor) Size() int {
//if s.state == nil {
return s.pos return s.pos
// } else {
// return bin.CountVarUint64(s.state.LastDelta) + hSize + len(s.state.Payload)
// }
} }
// // Snapshot - для створення снапшота. func (s *ValueDeltaCompressor) StoredSize() int {
if s.state == nil {
return len(s.buf) - s.pos
} else {
return bin.CountVarUint64(uint64(s.state.LastDelta)) + 1 + len(s.state.Payload)
}
}
// Snapshot - для створення снапшота.
// func (s *ValueDeltaCompressor) Payload() []byte { // func (s *ValueDeltaCompressor) Payload() []byte {
// if s.state == nil { // if s.state == nil {
// return s.buf[:s.pos] // return s.buf[:s.pos]
@@ -205,9 +218,7 @@ func (s *ValueDeltaCompressor) WritePayloadTo(w io.Writer, state *qb.ValueDeltaC
} }
} }
func (s *ValueDeltaCompressor) Rotate(newbuf []byte) { func (s *ValueDeltaCompressor) ReplaceBuffer(newbuf []byte) {
// УВАГА!
// state не чіпаємо
s.buf = newbuf s.buf = newbuf
s.pos = 0 s.pos = 0
s.baseValue = 0 s.baseValue = 0
@@ -226,17 +237,18 @@ func (s *ValueDeltaCompressor) LastValue() float64 {
return 0 return 0
} }
func (s *ValueDeltaCompressor) CreateDecompressor() qb.ValueDecompressor { // Decompressor читає виключно збережені на диск дані
func (s *ValueDeltaCompressor) CreateDecompressor(metricType qb.MetricType, fracDigits byte) qb.ValueDecompressor {
if s.pos == 0 { if s.pos == 0 {
return nil return nil
} }
return NewValueDeltaDecompressorFromState(ValueDeltaDecompressorFromStateOptions{ d := NewValueDeltaDecompressor(metricType, fracDigits)
Coef: s.coef, if s.state != nil {
ToFloat64: s.toFloat64, d.RestoreFromState(*s.state)
H: s.buf[s.pos-1], } else {
LastDelta: s.lastDelta, d.RestoreFromState(s.getState())
Payload: s.buf[:s.pos], }
}) return d
} }
// DECOMPRESSOR // DECOMPRESSOR
@@ -255,78 +267,62 @@ type ValueDeltaDecompressor struct {
toFloat64 func(uint64, float64) float64 toFloat64 func(uint64, float64) float64
} }
func NewValueDeltaDecompressor(fracDigits byte, buf []byte, toFloat64 func(uint64, float64) float64) *ValueDeltaDecompressor { func NewValueDeltaDecompressor(metricType qb.MetricType, fracDigits byte) *ValueDeltaDecompressor {
var coef float64 = 1
if fracDigits > 0 {
coef = math.Pow(10, float64(fracDigits))
}
u64, n, err := bin.GetVarUint64(buf)
if err != nil {
log.Fatalf("bug: get base value: %s", err)
}
s := &ValueDeltaDecompressor{ s := &ValueDeltaDecompressor{
buf: buf, coef: 1,
coef: coef,
pos: len(buf), // first free
bound: n,
toFloat64: toFloat64,
} }
s.baseValue = s.toFloat64(u64, coef) if fracDigits > 0 {
// читаю заголовок наступної серії s.coef = math.Pow(10, float64(fracDigits))
}
if metricType == qb.Cumulative {
s.toFloat64 = toCumulativeFloat64
} else {
s.toFloat64 = toInstantFloat64
}
return s
}
func (s *ValueDeltaDecompressor) RestoreFromEnd(buf []byte) {
s.buf = buf
s.pos = len(buf) // first free
s.readBaseValue()
s.readHeader() s.readHeader()
s.readValue() s.readValue()
return s
} }
type ValueDeltaDecompressorFromStateOptions struct { func (s *ValueDeltaDecompressor) RestoreFromState(state qb.ValueDeltaCapturedState) {
Coef float64 s.buf = state.Payload
// (value, coef) => uint64 s.pos = len(s.buf) // first free
ToFloat64 func(uint64, float64) float64 s.readBaseValue()
H byte s.lastValue = s.baseValue + s.toFloat64(state.LastDelta, s.coef)
LastDelta uint64 s.decodeHeaderByte(state.H)
Payload []byte
} }
func NewValueDeltaDecompressorFromState(opt ValueDeltaDecompressorFromStateOptions) *ValueDeltaDecompressor { func (s *ValueDeltaDecompressor) readBaseValue() {
u64, n, err := bin.GetVarUint64(opt.Payload) u64, n, err := bin.GetVarUint64(s.buf)
if err != nil { if err != nil {
log.Fatalf("bug: get base value: %s", err) log.Fatalf("bug: get base value: %s", err)
} }
s := &ValueDeltaDecompressor{
buf: opt.Payload,
coef: opt.Coef,
pos: len(opt.Payload), // first free
bound: n,
toFloat64: opt.ToFloat64,
}
s.baseValue = s.toFloat64(u64, s.coef) s.baseValue = s.toFloat64(u64, s.coef)
s.lastValue = s.baseValue + s.toFloat64(opt.LastDelta, s.coef) s.bound = n
s.decodeHeaderByte(opt.H)
return s
} }
func (s *ValueDeltaDecompressor) NextValue() (value float64, done bool) { func (s *ValueDeltaDecompressor) NextValue() (value float64, done bool) {
//fmt.Printf("NextValue(): bound: %d, pos: %d, pending: %d\n", s.bound, s.pos, s.pending)
if s.done { if s.done {
return 0, true return 0, true
} }
// метод працює як do while - спочатку значення, а потім перевірка умови
value = s.lastValue value = s.lastValue
s.pending-- s.pending--
if s.pending > 0 { if s.pending > 0 {
// якщо в серії залишаються елементи
if !s.isRun { if !s.isRun {
s.readValue() s.readValue()
} }
} else if s.pos > s.bound { } else if s.pos > s.bound {
// в серії більше немає елементів, отже перевіряє чи є ще дані в буфері.
// дані є - читаю заголовок наступної серії
s.readHeader() s.readHeader()
s.readValue() s.readValue()
} else { } else {
s.done = true s.done = true
} }
// серія завершена - перевіряю чи є ще серії
return value, false return value, false
} }
@@ -335,6 +331,7 @@ func (s *ValueDeltaDecompressor) readHeader() {
h := s.buf[s.pos] h := s.buf[s.pos]
s.decodeHeaderByte(h) s.decodeHeaderByte(h)
} }
func (s *ValueDeltaDecompressor) readValue() { func (s *ValueDeltaDecompressor) readValue() {
u64, n, err := bin.ReverseGetVarUint64(s.buf[:s.pos]) u64, n, err := bin.ReverseGetVarUint64(s.buf[:s.pos])
if err != nil { if err != nil {

2
examples/requests/requests.go
View File

@@ -14,7 +14,7 @@ func sendRequests(conn *client.Connection) {
var ( var (
instantMetricID uint32 = 10000 instantMetricID uint32 = 10000
cumulativeMetricID uint32 = 10001 cumulativeMetricID uint32 = 10001
fracDigits int = 2 fracDigits byte = 2
err error err error
) )

4
proto/proto.go
View File

@@ -162,7 +162,7 @@ func ReadListCurrentValuesReq(r *bufreader.BufferedReader) (m ListCurrentValuesR
type AddMetricReq struct { type AddMetricReq struct {
MetricID uint32 MetricID uint32
MetricType qb.MetricType MetricType qb.MetricType
FracDigits int FracDigits byte
} }
func ReadAddMetricReq(r *bufreader.BufferedReader) (m AddMetricReq, err error) { func ReadAddMetricReq(r *bufreader.BufferedReader) (m AddMetricReq, err error) {
@@ -177,7 +177,7 @@ func ReadAddMetricReq(r *bufreader.BufferedReader) (m AddMetricReq, err error) {
func UnpackAddMetricReq(arr []byte) (m AddMetricReq) { func UnpackAddMetricReq(arr []byte) (m AddMetricReq) {
m.MetricID, _ = bin.GetUint32(arr) m.MetricID, _ = bin.GetUint32(arr)
m.MetricType = qb.MetricType(arr[4]) m.MetricType = qb.MetricType(arr[4])
m.FracDigits = int(arr[5]) m.FracDigits = arr[5]
return return
} }

24
qb.go
View File

@@ -26,16 +26,18 @@ const (
type TimestampCompressor interface { type TimestampCompressor interface {
// (tmp, timestamp) // (tmp, timestamp)
Evaluate([]byte, uint32) TimeEvaluationReport Evaluate([]byte, uint32) TimeEvaluationReport
// (offset, change, timestamp) // (rewindOffset, change, timestamp)
Append(int, []byte, uint32) Append(int, []byte, uint32)
Size() int Size() int
//Chunks() [][]byte //Chunks() [][]byte
//DeleteLast() //DeleteLast()
CaptureState() TimeDeltaCapturedState CaptureState()
// (offset) => payload
Tail(int) []byte
CreateDecompressor() TimestampDecompressor CreateDecompressor() TimestampDecompressor
//Payload() []byte // для снапшота //Payload() []byte // для снапшота
// Offset() int // Offset() int
Rotate([]byte) ReplaceBuffer([]byte)
WritePayloadTo(io.Writer, *TimeDeltaCapturedState) error WritePayloadTo(io.Writer, *TimeDeltaCapturedState) error
LastTimestamp() uint32 LastTimestamp() uint32
ReplaceSinceWithUntil() uint32 ReplaceSinceWithUntil() uint32
@@ -50,24 +52,27 @@ type TimeDeltaCapturedState struct {
type TimeEvaluationReport struct { type TimeEvaluationReport struct {
TotalSpace int TotalSpace int
Offset int Offset int // from payload start
RewindOffset int // steps to back from pos
ChangeSize int ChangeSize int
} }
type ValueCompressor interface { type ValueCompressor interface {
// (tmp, value) // (tmp, value)
Evaluate([]byte, float64) ValueEvaluationReport Evaluate([]byte, float64) ValueEvaluationReport
// (offset, change, value, delta) // (rewindOffset, change, value, delta)
Append(int, []byte, float64, uint64) Append(int, []byte, float64, uint64)
Size() int Size() int
//Chunks() [][]byte //Chunks() [][]byte
//DeleteLast() //DeleteLast()
CaptureState() ValueDeltaCapturedState CaptureState()
// (offset) => payload
Tail(int) []byte
// fracDigits // fracDigits
CreateDecompressor() ValueDecompressor CreateDecompressor(MetricType, byte) ValueDecompressor
//Payload() []byte // для снапшота //Payload() []byte // для снапшота
//Offset() int //Offset() int
Rotate([]byte) ReplaceBuffer([]byte)
WritePayloadTo(io.Writer, *ValueDeltaCapturedState) error WritePayloadTo(io.Writer, *ValueDeltaCapturedState) error
LastValue() float64 LastValue() float64
} }
@@ -80,7 +85,8 @@ type ValueDeltaCapturedState struct {
type ValueEvaluationReport struct { type ValueEvaluationReport struct {
TotalSpace int TotalSpace int
Offset int Offset int // from payload start
RewindOffset int // steps to back from pos
ChangeSize int ChangeSize int
Delta uint64 Delta uint64
} }

91
storage/page_preparer.go
View File

@@ -15,23 +15,11 @@ import (
// fix - reduce leves // fix - reduce leves
type PagePreparer struct { type PagePreparer struct {
dataChecksumIdx int
maxRecordsOnIndexPage int
indexPageIncSize int // кратно indexPageSize
timestampsSizeIdx int
valuesSizeIdx int
prevPageIdx int
indexRecordsCountIdx int
isZeroLevelIdx int
indexChecksumIdx int
getDataPageNumber func() (uint32, bool, error) getDataPageNumber func() (uint32, bool, error)
getIndexPageNumber func() (uint32, bool, error) getIndexPageNumber func() (uint32, bool, error)
} }
type PagePreparerOptions struct { type PagePreparerOptions struct {
IndexPageSize int
IndexPageIncSize int
DataPageSize int
GetDataPageNumber func() (uint32, bool, error) GetDataPageNumber func() (uint32, bool, error)
GetIndexPageNumber func() (uint32, bool, error) GetIndexPageNumber func() (uint32, bool, error)
} }
@@ -43,22 +31,10 @@ func NewPagePreparer(opt PagePreparerOptions) (*PagePreparer, error) {
if opt.GetDataPageNumber == nil { if opt.GetDataPageNumber == nil {
return nil, errors.New("missing required option: GetDataPageNumber") return nil, errors.New("missing required option: GetDataPageNumber")
} }
if (opt.IndexPageSize % opt.IndexPageIncSize) != 0 {
return nil, errors.New("IndexPageIncSize must multiple of IndexPageSize")
}
s := &PagePreparer{ s := &PagePreparer{
dataChecksumIdx: opt.DataPageSize - 4,
indexPageIncSize: opt.IndexPageIncSize,
timestampsSizeIdx: opt.DataPageSize - 6,
valuesSizeIdx: opt.DataPageSize - 8,
prevPageIdx: opt.DataPageSize - 12,
indexRecordsCountIdx: opt.IndexPageSize - 6,
isZeroLevelIdx: opt.IndexPageSize - 7,
indexChecksumIdx: opt.IndexPageSize - 4,
getIndexPageNumber: opt.GetIndexPageNumber, getIndexPageNumber: opt.GetIndexPageNumber,
getDataPageNumber: opt.GetDataPageNumber, getDataPageNumber: opt.GetDataPageNumber,
} }
s.maxRecordsOnIndexPage = (opt.IndexPageSize - 7) / indexRecordSize
return s, nil return s, nil
} }
@@ -70,7 +46,7 @@ type appendIndexRecordIn struct {
} }
func (s *PagePreparer) appendIndexRecord(in appendIndexRecordIn) []byte { func (s *PagePreparer) appendIndexRecord(in appendIndexRecordIn) []byte {
if in.RecordsCount < s.maxRecordsOnIndexPage { if in.RecordsCount < maxRecordsOnIndexPage {
var ( var (
pos int pos int
buf = in.Records buf = in.Records
@@ -78,7 +54,7 @@ func (s *PagePreparer) appendIndexRecord(in appendIndexRecordIn) []byte {
if in.RecordsCount > 0 { if in.RecordsCount > 0 {
pos = in.RecordsCount * indexRecordSize pos = in.RecordsCount * indexRecordSize
} else { } else {
buf = make([]byte, s.indexPageIncSize) buf = make([]byte, IndexPageSize) // IndexPageIncSize
} }
bin.PutUint32(buf[pos:], in.Timestamp) bin.PutUint32(buf[pos:], in.Timestamp)
bin.PutUint32(buf[pos+4:], in.PageNo) bin.PutUint32(buf[pos+4:], in.PageNo)
@@ -88,39 +64,6 @@ func (s *PagePreparer) appendIndexRecord(in appendIndexRecordIn) []byte {
return nil return nil
} }
type sealDataPageIn struct {
Content []byte
PrevPageNo uint32
TimestampsSize int
ValuesSize int
}
func (s *PagePreparer) sealDataPage(in sealDataPageIn) (checksum uint32) {
bin.PutUint16(in.Content[s.timestampsSizeIdx:], uint16(in.TimestampsSize))
bin.PutUint16(in.Content[s.valuesSizeIdx:], uint16(in.ValuesSize))
bin.PutUint32(in.Content[s.prevPageIdx:], in.PrevPageNo)
checksum = util.CalcChecksum(in.Content[:s.dataChecksumIdx])
bin.PutUint32(in.Content[s.dataChecksumIdx:], checksum)
return
}
type sealIndexPageIn struct {
Content []byte
RecordsCount int
LastLevel bool
}
func (s *PagePreparer) sealIndexPage(in sealIndexPageIn) (checksum uint32) {
bin.PutUint16(in.Content[s.indexRecordsCountIdx:], uint16(in.RecordsCount))
if in.LastLevel {
in.Content[s.isZeroLevelIdx] = 1
}
checksum = util.CalcChecksum(in.Content[:s.indexChecksumIdx])
bin.PutUint32(in.Content[s.indexChecksumIdx:], checksum)
return
}
type IndexLevelTail struct { type IndexLevelTail struct {
Buffer []byte // розмір більший за кількість Buffer []byte // розмір більший за кількість
RecordsCount int RecordsCount int
@@ -210,7 +153,7 @@ func (s *PagePreparer) SealPages(in SealPagesIn) (result SealResult) {
PrevPageNo: prevPageNo, PrevPageNo: prevPageNo,
PageNo: pageNo, PageNo: pageNo,
Reused: reused, Reused: reused,
Checksum: s.sealDataPage(sealDataPageIn{ Checksum: SealDataPage(SealDataPageIn{
Content: d.Content, Content: d.Content,
PrevPageNo: prevPageNo, PrevPageNo: prevPageNo,
TimestampsSize: d.TimestampsSize, TimestampsSize: d.TimestampsSize,
@@ -232,7 +175,7 @@ func (s *PagePreparer) SealPages(in SealPagesIn) (result SealResult) {
break break
} }
level := sealedIndexLevels[levelIdx] level := sealedIndexLevels[levelIdx]
if level.TailRecordsCount < s.maxRecordsOnIndexPage { if level.TailRecordsCount < maxRecordsOnIndexPage {
level.TailRecords = s.appendIndexRecord(appendIndexRecordIn{ level.TailRecords = s.appendIndexRecord(appendIndexRecordIn{
Records: level.TailRecords, Records: level.TailRecords,
Timestamp: upTimestamp, Timestamp: upTimestamp,
@@ -252,10 +195,10 @@ func (s *PagePreparer) SealPages(in SealPagesIn) (result SealResult) {
Content: level.TailRecords, Content: level.TailRecords,
PageNo: pageNo, PageNo: pageNo,
Reused: reused, Reused: reused,
Checksum: s.sealIndexPage(sealIndexPageIn{ Checksum: SealIndexPage(SealIndexPageIn{
Content: level.TailRecords, Content: level.TailRecords,
RecordsCount: level.TailRecordsCount, RecordsCount: level.TailRecordsCount,
LastLevel: levelIdx == 0, ZeroLevel: levelIdx == 0,
}), }),
} }
level.IndexPages = append(level.IndexPages, filled) level.IndexPages = append(level.IndexPages, filled)
@@ -288,12 +231,12 @@ type SealDataPageIn struct {
} }
func SealDataPage(in SealDataPageIn) (checksum uint32) { func SealDataPage(in SealDataPageIn) (checksum uint32) {
// bin.PutUint16(in.Content[s.timestampsSizeIdx:], uint16(in.TimestampsSize)) bin.PutUint16(in.Content[timestampsSizeIdx:], uint16(in.TimestampsSize))
// bin.PutUint16(in.Content[s.valuesSizeIdx:], uint16(in.ValuesSize)) bin.PutUint16(in.Content[valuesSizeIdx:], uint16(in.ValuesSize))
// bin.PutUint32(in.Content[s.prevPageIdx:], in.PrevPageNo) bin.PutUint32(in.Content[prevPageIdx:], in.PrevPageNo)
// checksum = util.CalcChecksum(in.Content[:s.dataChecksumIdx]) checksum = util.CalcChecksum(in.Content[:dataCRC32Idx])
// bin.PutUint32(in.Content[s.dataChecksumIdx:], checksum) bin.PutUint32(in.Content[dataCRC32Idx:], checksum)
return return
} }
@@ -304,11 +247,11 @@ type SealIndexPageIn struct {
} }
func SealIndexPage(in SealIndexPageIn) (checksum uint32) { func SealIndexPage(in SealIndexPageIn) (checksum uint32) {
// bin.PutUint16(in.Content[s.indexRecordsCountIdx:], uint16(in.RecordsCount)) bin.PutUint16(in.Content[indexRecordsCountIdx:], uint16(in.RecordsCount))
// if in.LastLevel { if in.ZeroLevel {
// in.Content[s.isZeroLevelIdx] = 1 in.Content[isZeroLevelIdx] = 1
// } }
// checksum = util.CalcChecksum(in.Content[:s.indexChecksumIdx]) checksum = util.CalcChecksum(in.Content[:indexCRC32Idx])
// bin.PutUint32(in.Content[s.indexChecksumIdx:], checksum) bin.PutUint32(in.Content[indexCRC32Idx:], checksum)
return return
} }

6
storage/wal_records.go
View File

@@ -360,7 +360,7 @@ func (s *MeasuresAppendRecord) Parse(r *bytes.Buffer) (err error) {
type MetricAddRecord struct { type MetricAddRecord struct {
MetricID uint32 MetricID uint32
MetricType qb.MetricType MetricType qb.MetricType
FracDigits int FracDigits byte
} }
func (s MetricAddRecord) Pack(w io.Writer) { func (s MetricAddRecord) Pack(w io.Writer) {
@@ -368,7 +368,7 @@ func (s MetricAddRecord) Pack(w io.Writer) {
CodeMetricAdd, CodeMetricAdd,
0, 0, 0, 0, // 0, 0, 0, 0, //
byte(s.MetricType), byte(s.MetricType),
byte(s.FracDigits), s.FracDigits,
} }
bin.PutUint32(arr[1:], s.MetricID) bin.PutUint32(arr[1:], s.MetricID)
w.Write(arr) w.Write(arr)
@@ -381,7 +381,7 @@ func (s *MetricAddRecord) Parse(src io.Reader) (err error) {
} }
s.MetricID, _ = bin.GetUint32(arr) s.MetricID, _ = bin.GetUint32(arr)
s.MetricType = qb.MetricType(arr[4]) s.MetricType = qb.MetricType(arr[4])
s.FracDigits = int(arr[5]) s.FracDigits = arr[5]
return nil return nil
} }