dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
dima
2026-06-11 14:27:38 +00:00
parent 09f270aa6f
commit ed203195fa
7 changed files with 431 additions and 1295 deletions
Download Patch File Download Diff File Expand all files Collapse all files

447
enc/enc_test.go
View File

@@ -19,7 +19,7 @@ func equalFloatSlices(a, b []float64, epsilon float64) bool {
// CUMULATIVE
func TestValueDeltaCompressor(t *testing.T) {
func TestCumulativeDeltaCompressor(t *testing.T) {
var (
testCases = []struct {
Nums []float64
@@ -76,7 +76,7 @@ func TestValueDeltaCompressor(t *testing.T) {
0x8f, // base value
0x80, // delta 0
0x80, // literal (len = 1)
0x81, // delta 10
0x81, // delta 1
0x00, // run (len = 2)
0x00, 0x00, 0x00,
},
@@ -115,45 +115,43 @@ func TestValueDeltaCompressor(t *testing.T) {
0x8f, // base value
0x80, // delta 0
0x01, // run (len = 3)
0x81, // delta 10
0x81, // delta 1
0x80, // literal (len = 1)
0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: 1,
},
// {
// Nums: []float64{
// 1.5,
// },
// RepeatLastDeltaNTimes: 128,
// Code: incrementRun, // h byte full filled
// RequiredSpace: 2,
// Buf: []byte{
// 143, // base value
// 0, 0, 0, 0, 0, 0, 0,
// },
// BaseValue: 1.5,
// LastDelta: 0,
// H: 127, // run, length = 129
// },
// {
// Nums: []float64{
// 1.5,
// },
// RepeatLastDeltaNTimes: 129,
// Code: endSeries, // run, h byte overflowed
// RequiredSpace: 4, // run delta, h of run, 1st literal delta, h of literal
// Buf: []byte{
// 143, // base value
// 128, // run delta (0)
// 127, // h of run (length = 129)
// 0, 0, 0, 0, 0,
// },
// BaseValue: 1.5,
// LastDelta: 0,
// H: 128, // literal, length = 1
// },
{
Nums: repeatFloat64(1.5, 129),
Name: "increment run to full fill h-byte",
Offset: 1,
ChangeSize: 1,
Buf: []byte{
0x8f, // base value
0x80, // delta 0
0x7f, // run (len = 129)
0x00, 0x00, 0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: 0,
},
{
Nums: repeatFloat64(1.5, 130),
Name: "run switch to literal after h-byte overflow",
Offset: 0,
ChangeSize: 2,
Buf: []byte{
0x8f, // base value
0x80, // delta 0
0x7f, // run (len = 129)
0x80, // delta 0
0x80, // literal (len = 1)
0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: 0,
},
{
Nums: []float64{
1.5,
@@ -166,8 +164,8 @@ func TestValueDeltaCompressor(t *testing.T) {
Buf: []byte{
0x8f, // base value
0x80, // delta 0
0x81, // delta 10
0x82, // delta 20
0x81, // delta 1
0x82, // delta 2
0x82, // literal (len = 3)
0x00, 0x00, 0x00,
},
@@ -180,12 +178,13 @@ func TestValueDeltaCompressor(t *testing.T) {
for _, testCase := range testCases {
var (
tmp = make([]byte, tmpValueSize)
metricType = qb.Cumulative
fracDigits byte = 1
buf = make([]byte, 8)
payloadSize = 0
report qb.ValueEvaluationReport
)
c := NewValueDeltaCompressor(fracDigits, buf, payloadSize)
c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize)
for _, num := range testCase.Nums {
report = c.Evaluate(tmp, num)
c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
@@ -285,15 +284,17 @@ func TestCumulativeDeltaDecompressorFromState(t *testing.T) {
)
for caseIdx, testCase := range testCases {
var (
metricType = qb.Cumulative
fracDigits byte = 1
tmp = make([]byte, tmpValueSize)
buf = make([]byte, 16)
payloadSize = 0
decodedNums []float64
)
c := NewCumulativeDeltaCompressor(fracDigits, buf, payloadSize)
c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize)
for _, num := range testCase.Nums {
compressionWay, _ := c.Evaluate(num)
c.Compress(compressionWay, num)
report := c.Evaluate(tmp, num)
c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
}
//
@@ -331,41 +332,62 @@ func TestCumulativeDeltaDecompressorFromState(t *testing.T) {
func TestInstantDeltaCompressor(t *testing.T) {
var (
testCases = []struct {
Nums []float64
RepeatLastDeltaNTimes int
Code int
RequiredSpace int
Buf []byte
BaseValue float64
LastDelta int64
H byte
Nums []float64
Name string
Offset int
ChangeSize int
Buf []byte
BaseValue float64
LastDelta uint64
}{
{
Nums: []float64{
1.5,
},
Code: addBaseValue,
RequiredSpace: 3, // base value, delta, h
Name: "add 1st value",
Offset: 0,
ChangeSize: 3,
Buf: []byte{
158, 0, 0, 0, 0, 0, 0, 0,
0x9e, // base value
0x80, // delta 0
0x80, // literal (len = 1)
0x00, 0x00, 0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: 0,
H: 128, // literal, length = 1
},
{
Nums: []float64{
-1.5,
},
Name: "add 1st value (negative)",
Offset: 0,
ChangeSize: 3,
Buf: []byte{
0x9d, // base value
0x80, // delta 0
0x80, // literal (len = 1)
0x00, 0x00, 0x00, 0x00, 0x00,
},
BaseValue: -1.5,
LastDelta: 0,
},
{
Nums: []float64{
1.5,
1.5,
},
Code: startRun, // literal changed to run
RequiredSpace: 2, // delta, h
Name: "literal switch to run",
Offset: 1,
ChangeSize: 1,
Buf: []byte{
158, 0, 0, 0, 0, 0, 0, 0,
0x9e, // base value
0x80, // delta 0
0x00, // run (len = 2)
0x00, 0x00, 0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: 0,
H: 0, // run, length = 2
},
{
Nums: []float64{
@@ -373,36 +395,39 @@ func TestInstantDeltaCompressor(t *testing.T) {
1.6,
1.6,
},
Code: startRun, // literal decreased by 1
RequiredSpace: 3, // end of literal, new delta, new h
Name: "literal decrease by 1 and switch to run",
Offset: 2,
ChangeSize: 3,
Buf: []byte{
158, // base value
128, // literal 1st delta (0)
128, // h - end of literal, length = 1
0, 0, 0, 0, 0,
0x9e, // base value
0x80, // delta 0
0x80, // literal (len = 1)
0x82, // delta 1
0x00, // run (len = 2)
0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: 1,
H: 0, // run, length = 2
LastDelta: bin.EncodeZigZag(1),
},
{
// same as prev, but negative delta
Nums: []float64{
1.5,
1.4,
1.4,
},
Code: startRun, // literal decreased by 1
RequiredSpace: 3, // end of literal, new delta, new h
Name: "literal decrease by 1 and switch to run (negative delta)",
Offset: 2,
ChangeSize: 3,
Buf: []byte{
158, // base value
128, // literal 1st delta (0)
128, // h - end of literal, length = 1
0, 0, 0, 0, 0,
0x9e, // base value
0x80, // delta 0
0x80, // literal (len = 1)
0x81, // delta -1
0x00, // run (len = 2)
0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: -1,
H: 0, // run, length = 2
LastDelta: bin.EncodeZigZag(-1),
},
{
Nums: []float64{
@@ -410,15 +435,17 @@ func TestInstantDeltaCompressor(t *testing.T) {
1.5,
1.5,
},
Code: incrementRun,
RequiredSpace: 2,
Name: "increment run",
Offset: 1,
ChangeSize: 1,
Buf: []byte{
158, // base value
0, 0, 0, 0, 0, 0, 0,
0x9e, // base value
0x80, // delta 0
0x01, // run (len = 3)
0x00, 0x00, 0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: 0,
H: 1, // run, length = 3
},
{
Nums: []float64{
@@ -427,149 +454,149 @@ func TestInstantDeltaCompressor(t *testing.T) {
1.5,
1.6,
},
Code: endSeries, // end of run
RequiredSpace: 4,
Name: "run switch to literal",
Offset: 0,
ChangeSize: 2,
Buf: []byte{
158, // base value
128, // run delta (0)
1, // h of run, length = 3
0, 0, 0, 0, 0,
0x9e, // base value
0x80, // delta 0
0x01, // run (len = 3)
0x82, // delta 1
0x80, // literal (len = 1)
0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: 1,
H: 128, // literal, length = 1
LastDelta: bin.EncodeZigZag(1),
},
{
// same as prev, but negative delta
Nums: []float64{
1.5,
1.5,
1.5,
1.4,
},
Code: endSeries, // end of run
RequiredSpace: 4,
Name: "run switch to literal (negative delta)",
Offset: 0,
ChangeSize: 2,
Buf: []byte{
158, // base value
128, // run delta (0)
1, // h of run, length = 3
0, 0, 0, 0, 0,
0x9e, // base value
0x80, // delta 0
0x01, // run (len = 3)
0x81, // delta -1
0x80, // literal (len = 1)
0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: -1,
H: 128, // literal, length = 1
LastDelta: bin.EncodeZigZag(-1),
},
{
Nums: []float64{
1.5,
},
RepeatLastDeltaNTimes: 128,
Code: incrementRun, // h byte full filled
RequiredSpace: 2,
Nums: repeatFloat64(1.5, 129),
Name: "increment run to full fill h-byte",
Offset: 1,
ChangeSize: 1,
Buf: []byte{
158, // base value
0, 0, 0, 0, 0, 0, 0,
0x9e, // base value
0x80, // delta 0
0x7f, // run (len = 129)
0x00, 0x00, 0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: 0,
H: 127, // run, length = 129
},
{
Nums: []float64{
1.5,
},
RepeatLastDeltaNTimes: 129,
Code: endSeries, // run, h byte overflowed
RequiredSpace: 4, // run delta, h of run, 1st literal delta, h of literal
Nums: repeatFloat64(1.5, 130),
Name: "run switch to literal after h-byte overflow",
Offset: 0,
ChangeSize: 2,
Buf: []byte{
158, // base value
128, // run delta (0)
127, // h of run (length = 129)
0, 0, 0, 0, 0,
0x9e, // base value
0x80, // delta 0
0x7f, // run (len = 129)
0x80, // delta 0
0x80, // literal (len = 1)
0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: 0,
H: 128, // literal, length = 1
},
{
Nums: []float64{
1.5,
1.6,
1.7,
},
Code: incrementLiteral,
RequiredSpace: 3, // previous delta, new delta, h
Name: "increment literal",
Offset: 1,
ChangeSize: 2,
Buf: []byte{
158, // base value
128, // literal 1st delta (0)
0, 0, 0, 0, 0, 0,
0x9e, // base value
0x80, // delta 0
0x82, // delta 1
0x84, // delta 2
0x82, // literal (len = 3)
0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: 1,
H: 129, // literal, length = 2
LastDelta: bin.EncodeZigZag(2),
},
{
// same as prev, but negative delta
Nums: []float64{
1.5,
1.4,
1.3,
},
Code: incrementLiteral,
RequiredSpace: 3, // previous delta, new delta, h
Name: "increment literal (negative delta)",
Offset: 1,
ChangeSize: 2,
Buf: []byte{
158, // base value
128, // literal 1st delta (0)
0, 0, 0, 0, 0, 0,
0x9e, // base value
0x80, // delta 0
0x81, // delta -1
0x83, // delta -2
0x82, // literal (len = 3)
0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: -1,
H: 129, // literal, length = 2
LastDelta: bin.EncodeZigZag(-2),
},
}
)
for caseIdx, testCase := range testCases {
for _, testCase := range testCases {
// fmt.Println("-----")
// fmt.Println("nums", testCase.Nums)
var (
fracDigits byte = 1
buf = make([]byte, 8)
payloadSize = 0
compressionWay int
requiredSpace int
tmp = make([]byte, tmpValueSize)
metricType = qb.Instant
fracDigits byte = 1
buf = make([]byte, 8)
payloadSize = 0
report qb.ValueEvaluationReport
)
c := NewInstantDeltaCompressor(fracDigits, buf, payloadSize)
c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize)
for _, num := range testCase.Nums {
compressionWay, requiredSpace = c.Evaluate(num)
c.Compress(compressionWay, num)
report = c.Evaluate(tmp, num)
c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
}
if testCase.RepeatLastDeltaNTimes > 0 {
num := testCase.Nums[len(testCase.Nums)-1]
for range testCase.RepeatLastDeltaNTimes {
compressionWay, requiredSpace = c.Evaluate(num)
c.Compress(compressionWay, num)
}
if report.Offset != testCase.Offset {
t.Fatalf("%s: got offset %d are not equal expected %d",
testCase.Name, report.Offset, testCase.Offset)
}
if compressionWay != testCase.Code {
t.Fatalf("%d: got code %d are not equal expected %d",
caseIdx, compressionWay, testCase.Code)
}
if requiredSpace != testCase.RequiredSpace {
t.Fatalf("%d: got requiredSpace %d are not equal expected %d",
caseIdx, requiredSpace, testCase.RequiredSpace)
if report.ChangeSize != testCase.ChangeSize {
t.Fatalf("%s: got changeSize %d are not equal expected %d",
testCase.Name, report.ChangeSize, testCase.ChangeSize)
}
if !bytes.Equal(buf, testCase.Buf) {
t.Fatalf("%d: got buf %v are not equal expected %v",
caseIdx, buf, testCase.Buf)
t.Fatalf("%s: got buf % x are not equal expected % x",
testCase.Name, buf, testCase.Buf)
}
if c.baseValue != testCase.BaseValue {
t.Fatalf("%d: got lastUnixtime %v are not equal expected %v",
caseIdx, c.baseValue, testCase.BaseValue)
t.Fatalf("%s: got baseValue %v are not equal expected %v",
testCase.Name, c.baseValue, testCase.BaseValue)
}
if c.lastDelta != testCase.LastDelta {
t.Fatalf("%d: got lastDelta %d are not equal expected %d",
caseIdx, c.lastDelta, testCase.LastDelta)
}
if c.h != testCase.H {
t.Fatalf("%d: got h %d are not equal expected %d",
caseIdx, c.h, testCase.H)
t.Fatalf("%s: got lastDelta %d are not equal expected %d",
testCase.Name, c.lastDelta, testCase.LastDelta)
}
}
}
@@ -646,15 +673,17 @@ func TestInstantDeltaDecompressorFromState(t *testing.T) {
)
for caseIdx, testCase := range testCases {
var (
metricType = qb.Instant
fracDigits byte = 1
tmp = make([]byte, tmpValueSize)
buf = make([]byte, 16)
payloadSize = 0
decodedNums []float64
)
c := NewInstantDeltaCompressor(fracDigits, buf, payloadSize)
c := NewValueDeltaCompressor(metricType, fracDigits, buf, payloadSize)
for _, num := range testCase.Nums {
compressionWay, _ := c.Evaluate(num)
c.Compress(compressionWay, num)
report := c.Evaluate(tmp, num)
c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
}
//
@@ -809,42 +838,35 @@ func TestTimeDeltaCompressor(t *testing.T) {
LastUnixtime: 1780777200,
LastDelta: 20,
},
// {
// Nums: []uint32{
// 1780777000,
// 1780777060, // +60
// },
// RepeatLastDeltaNTimes: 128,
// CompressionWay: incrementRun, // h byte full filled
// RequiredSpace: 6,
// Buf: []byte{
// 0x80, // h-byte (literal, len=1)
// 0x94, // delta 20
// 0x01, // h-byte (run, len=3)
// 0xbc, // delta 60
// 0x28, 0x80, 0x24, 0x6a, // since
// },
// LastUnixtime: 1780777060 + 128*60,
// LastDelta: 60,
// H: 127, // run, length = 129
// },
// {
// Nums: []uint32{
// 1780777000,
// 1780777060, // +60
// },
// RepeatLastDeltaNTimes: 129,
// CompressionWay: endSeries, // run, h byte overflowed
// RequiredSpace: 8,
// Buf: []byte{
// 0, 0, 0, 0, 0, 0,
// 127, // h - end of run (length = 129)
// 188, // varUint64(60)
// },
// LastUnixtime: 1780777060 + 129*60,
// LastDelta: 60,
// H: 128, // literal, length = 1
// },
{
Nums: generateProgression(1780777000, 60, 130),
Name: "increment run up to full filled h-byte",
Offset: 1,
ChangeSize: 1,
Buf: []byte{
0x00, 0x00,
0x7f, // h-byte (run, len=129)
0xbc, // delta 60
0x28, 0x80, 0x24, 0x6a, // since
},
LastUnixtime: 1780777000 + 129*60,
LastDelta: 60,
},
{
Nums: generateProgression(1780777000, 60, 131),
Name: "run switch to literal after h-byte overflow",
Offset: 0,
ChangeSize: 2,
Buf: []byte{
0x80, // h-byte (literal, len=1)
0xbc, // delta 60
0x7f, // h-byte (run, len=129)
0xbc, // delta 60
0x28, 0x80, 0x24, 0x6a, // since
},
LastUnixtime: 1780777000 + 130*60,
LastDelta: 60,
},
{
Nums: []uint32{
1780777000,
@@ -1019,11 +1041,28 @@ func TestTimeDeltaDecompressorFromState(t *testing.T) {
}
}
func TestVarUint64(t *testing.T) {
func TestVarInt64(t *testing.T) {
arr := make([]byte, 9)
n, err := bin.PutVarInt64(arr, 15)
n, err := bin.PutVarInt64(arr, -15)
if err != nil {
t.Fatal(err)
}
fmt.Println(arr[:n])
}
func repeatFloat64(num float64, n int) []float64 {
nums := make([]float64, n)
for i := range nums {
nums[i] = num
}
return nums
}
func generateProgression(start uint32, delta uint32, n int) []uint32 {
nums := make([]uint32, n)
nums[0] = start
for i := 1; i < n; i++ {
nums[i] = nums[i-1] + delta
}
return nums
}