dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
dima
2026-06-11 01:46:25 +00:00
parent 9ec86282ad
commit 09f270aa6f
5 changed files with 770 additions and 335 deletions
Download Patch File Download Diff File Expand all files Collapse all files

448
enc/enc_test.go
View File

@@ -8,6 +8,7 @@ import (
"testing"
bin "gordenko.dev/dima/bin/little"
"gordenko.dev/dima/qb"
)
func equalFloatSlices(a, b []float64, epsilon float64) bool {
@@ -18,44 +19,49 @@ func equalFloatSlices(a, b []float64, epsilon float64) bool {
// CUMULATIVE
func TestCumulativeDeltaCompressor(t *testing.T) {
func TestValueDeltaCompressor(t *testing.T) {
var (
testCases = []struct {
Nums []float64
RepeatLastDeltaNTimes int
Code int
RequiredSpace int
Buf []byte
BaseValue float64
LastDelta uint64
H byte
Nums []float64
Name string
Offset int
ChangeSize int
Buf []byte
BaseValue float64
LastDelta uint64
}{
{
Nums: []float64{
1.5,
},
Code: addBaseValue,
RequiredSpace: 3, // base value, delta, h
Name: "add 1st value",
Offset: 0,
ChangeSize: 3,
Buf: []byte{
143, 0, 0, 0, 0, 0, 0, 0,
0x8f, // base value
0x80, // delta 0
0x80, // literal (len = 1)
0x00, 0x00, 0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: 0,
H: 128, // literal, length = 1
},
{
Nums: []float64{
1.5,
1.5,
},
Code: startRun, // literal changed to run
RequiredSpace: 2, // delta, h
Name: "literal switch to run",
Offset: 1,
ChangeSize: 1,
Buf: []byte{
143, 0, 0, 0, 0, 0, 0, 0,
0x8f, // base value
0x80, // delta 0
0x00, // run (len = 2)
0x00, 0x00, 0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: 0,
H: 0, // run, length = 2
},
{
Nums: []float64{
@@ -63,17 +69,19 @@ func TestCumulativeDeltaCompressor(t *testing.T) {
1.6,
1.6,
},
Code: startRun, // literal decreased by 1
RequiredSpace: 3, // end of literal, new delta, new h
Name: "literal decrease by 1 and switch to run",
Offset: 2,
ChangeSize: 3,
Buf: []byte{
143, // base value
128, // literal 1st delta (0)
128, // h - end of literal, length = 1
0, 0, 0, 0, 0,
0x8f, // base value
0x80, // delta 0
0x80, // literal (len = 1)
0x81, // delta 10
0x00, // run (len = 2)
0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: 1,
H: 0, // run, length = 2
},
{
Nums: []float64{
@@ -81,15 +89,17 @@ func TestCumulativeDeltaCompressor(t *testing.T) {
1.5,
1.5,
},
Code: incrementRun,
RequiredSpace: 2,
Name: "increment run",
Offset: 1,
ChangeSize: 1,
Buf: []byte{
143, // base value
0, 0, 0, 0, 0, 0, 0,
0x8f, // base value
0x80, // delta 0
0x01, // run (len = 3)
0x00, 0x00, 0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: 0,
H: 1, // run, length = 3
},
{
Nums: []float64{
@@ -98,112 +108,107 @@ func TestCumulativeDeltaCompressor(t *testing.T) {
1.5,
1.6,
},
Code: endSeries, // end of run
RequiredSpace: 4,
Name: "run switch to literal",
Offset: 0,
ChangeSize: 2,
Buf: []byte{
143, // base value
128, // run delta (0)
1, // h of run, length = 3
0, 0, 0, 0, 0,
0x8f, // base value
0x80, // delta 0
0x01, // run (len = 3)
0x81, // delta 10
0x80, // literal (len = 1)
0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: 1,
H: 128, // literal, length = 1
},
{
Nums: []float64{
1.5,
},
RepeatLastDeltaNTimes: 128,
Code: incrementRun, // h byte full filled
RequiredSpace: 2,
Buf: []byte{
143, // base value
0, 0, 0, 0, 0, 0, 0,
},
BaseValue: 1.5,
LastDelta: 0,
H: 127, // run, length = 129
},
{
Nums: []float64{
1.5,
},
RepeatLastDeltaNTimes: 129,
Code: endSeries, // run, h byte overflowed
RequiredSpace: 4, // run delta, h of run, 1st literal delta, h of literal
Buf: []byte{
143, // base value
128, // run delta (0)
127, // h of run (length = 129)
0, 0, 0, 0, 0,
},
BaseValue: 1.5,
LastDelta: 0,
H: 128, // literal, length = 1
},
// {
// Nums: []float64{
// 1.5,
// },
// RepeatLastDeltaNTimes: 128,
// Code: incrementRun, // h byte full filled
// RequiredSpace: 2,
// Buf: []byte{
// 143, // base value
// 0, 0, 0, 0, 0, 0, 0,
// },
// BaseValue: 1.5,
// LastDelta: 0,
// H: 127, // run, length = 129
// },
// {
// Nums: []float64{
// 1.5,
// },
// RepeatLastDeltaNTimes: 129,
// Code: endSeries, // run, h byte overflowed
// RequiredSpace: 4, // run delta, h of run, 1st literal delta, h of literal
// Buf: []byte{
// 143, // base value
// 128, // run delta (0)
// 127, // h of run (length = 129)
// 0, 0, 0, 0, 0,
// },
// BaseValue: 1.5,
// LastDelta: 0,
// H: 128, // literal, length = 1
// },
{
Nums: []float64{
1.5,
1.6,
1.7,
},
Code: incrementLiteral,
RequiredSpace: 3, // previous delta, new delta, h
Name: "increment literal",
Offset: 1,
ChangeSize: 2,
Buf: []byte{
143, // base value
128, // literal 1st delta (0)
0, 0, 0, 0, 0, 0,
0x8f, // base value
0x80, // delta 0
0x81, // delta 10
0x82, // delta 20
0x82, // literal (len = 3)
0x00, 0x00, 0x00,
},
BaseValue: 1.5,
LastDelta: 1,
H: 129, // literal, length = 2
LastDelta: 2,
},
}
)
for caseIdx, testCase := range testCases {
for _, testCase := range testCases {
var (
fracDigits byte = 1
buf = make([]byte, 8)
payloadSize = 0
compressionWay int
requiredSpace int
tmp = make([]byte, tmpValueSize)
fracDigits byte = 1
buf = make([]byte, 8)
payloadSize = 0
report qb.ValueEvaluationReport
)
c := NewCumulativeDeltaCompressor(fracDigits, buf, payloadSize)
c := NewValueDeltaCompressor(fracDigits, buf, payloadSize)
for _, num := range testCase.Nums {
compressionWay, requiredSpace = c.Evaluate(num)
c.Compress(compressionWay, num)
report = c.Evaluate(tmp, num)
c.Append(report.Offset, tmp[:report.ChangeSize], num, report.Delta)
}
if testCase.RepeatLastDeltaNTimes > 0 {
num := testCase.Nums[len(testCase.Nums)-1]
for range testCase.RepeatLastDeltaNTimes {
compressionWay, requiredSpace = c.Evaluate(num)
c.Compress(compressionWay, num)
}
if report.Offset != testCase.Offset {
t.Fatalf("%s: got offset %d are not equal expected %d",
testCase.Name, report.Offset, testCase.Offset)
}
if compressionWay != testCase.Code {
t.Fatalf("%d: got code %d are not equal expected %d",
caseIdx, compressionWay, testCase.Code)
}
if requiredSpace != testCase.RequiredSpace {
t.Fatalf("%d: got requiredSpace %d are not equal expected %d",
caseIdx, requiredSpace, testCase.RequiredSpace)
if report.ChangeSize != testCase.ChangeSize {
t.Fatalf("%s: got changeSize %d are not equal expected %d",
testCase.Name, report.ChangeSize, testCase.ChangeSize)
}
if !bytes.Equal(buf, testCase.Buf) {
t.Fatalf("%d: got buf %v are not equal expected %v",
caseIdx, buf, testCase.Buf)
t.Fatalf("%s: got buf % x are not equal expected % x",
testCase.Name, buf, testCase.Buf)
}
if c.baseValue != testCase.BaseValue {
t.Fatalf("%d: got lastUnixtime %v are not equal expected %v",
caseIdx, c.baseValue, testCase.BaseValue)
t.Fatalf("%s: got baseValue %v are not equal expected %v",
testCase.Name, c.baseValue, testCase.BaseValue)
}
if c.lastDelta != testCase.LastDelta {
t.Fatalf("%d: got lastDelta %d are not equal expected %d",
caseIdx, c.lastDelta, testCase.LastDelta)
}
if c.h != testCase.H {
t.Fatalf("%d: got h %d are not equal expected %d",
caseIdx, c.h, testCase.H)
t.Fatalf("%s: got lastDelta %d are not equal expected %d",
testCase.Name, c.lastDelta, testCase.LastDelta)
}
}
}
@@ -687,41 +692,44 @@ func TestInstantDeltaDecompressorFromState(t *testing.T) {
func TestTimeDeltaCompressor(t *testing.T) {
var (
testCases = []struct {
Nums []uint32
RepeatLastDeltaNTimes int
CompressionWay int
RequiredSpace int
Buf []byte
LastUnixtime uint32
LastDelta uint32
H byte
Name string
Nums []uint32
Buf []byte
LastUnixtime uint32
LastDelta uint32
Offset int
ChangeSize int
}{
{
Nums: []uint32{
1780777000,
},
CompressionWay: addUnixtime,
RequiredSpace: 4,
Name: "add 1st value",
Offset: 0,
ChangeSize: 4,
Buf: []byte{
0, 0, 0, 0, 0, 0, 0, 0,
0x00, 0x00, 0x00, 0x00,
0x28, 0x80, 0x24, 0x6a, // since
},
LastUnixtime: 1780777000,
LastDelta: 0,
H: 0,
},
{
Nums: []uint32{
1780777000,
1780777060, // +60
},
CompressionWay: add1stDelta,
RequiredSpace: 6,
Name: "add 1st delta",
Offset: 0,
ChangeSize: 2,
Buf: []byte{
0, 0, 0, 0, 0, 0, 0, 0,
0x00, 0x00,
0x80, // h-byte (literal, len=1)
0xbc, // delta (60)
0x28, 0x80, 0x24, 0x6a, // since
},
LastUnixtime: 1780777060,
LastDelta: 60,
H: 128,
},
{
Nums: []uint32{
@@ -729,14 +737,17 @@ func TestTimeDeltaCompressor(t *testing.T) {
1780777060, // +60
1780777120, // +60
},
CompressionWay: startRun, // literal changed by run
RequiredSpace: 6,
Name: "literal changed to run",
Offset: 1,
ChangeSize: 1,
Buf: []byte{
0, 0, 0, 0, 0, 0, 0, 0,
0x00, 0x00,
0x00, // h-byte (run, len=2)
0xbc, // delta
0x28, 0x80, 0x24, 0x6a, // since
},
LastUnixtime: 1780777120,
LastDelta: 60,
H: 0, // run, length = 2
},
{
Nums: []uint32{
@@ -745,14 +756,18 @@ func TestTimeDeltaCompressor(t *testing.T) {
1780777130, // +70
1780777200, // +70
},
CompressionWay: startRun, // literal decreased by 1
RequiredSpace: 7, // unixtime, end of literal, new delta, new h
Name: "literal decrease by 1 and switch to run",
Offset: 2,
ChangeSize: 3,
Buf: []byte{
0, 0, 0, 0, 0, 0, 128, 188,
0x00, // h-byte (run, len=2)
0xc6, // delta 70
0x80, // h-byte (literal, len=1)
0xbc, // delta 60
0x28, 0x80, 0x24, 0x6a, // since
},
LastUnixtime: 1780777200,
LastDelta: 70,
H: 0, // run, length = 2
},
{
Nums: []uint32{
@@ -761,14 +776,17 @@ func TestTimeDeltaCompressor(t *testing.T) {
1780777120, // +60
1780777180, // +60
},
CompressionWay: incrementRun,
RequiredSpace: 6,
Name: "increment run",
Offset: 1,
ChangeSize: 1,
Buf: []byte{
0, 0, 0, 0, 0, 0, 0, 0,
0x00, 0x00,
0x01, // h-byte (run, len=2)
0xbc, // delta 60
0x28, 0x80, 0x24, 0x6a, // since
},
LastUnixtime: 1780777180,
LastDelta: 60,
H: 1, // run, length = 3
},
{
Nums: []uint32{
@@ -778,111 +796,114 @@ func TestTimeDeltaCompressor(t *testing.T) {
1780777180, // +60
1780777200, // +20
},
CompressionWay: endSeries, // run
RequiredSpace: 8,
Name: "switch run to literal",
Offset: 0,
ChangeSize: 2,
Buf: []byte{
0, 0, 0, 0, 0, 0,
1, // h of run, length = 3
188, // varUint64(60)
0x80, // h-byte (literal, len=1)
0x94, // delta 20
0x01, // h-byte (run, len=3)
0xbc, // delta 60
0x28, 0x80, 0x24, 0x6a, // since
},
LastUnixtime: 1780777200,
LastDelta: 20,
H: 128, // literal, length = 1
},
// {
// Nums: []uint32{
// 1780777000,
// 1780777060, // +60
// },
// RepeatLastDeltaNTimes: 128,
// CompressionWay: incrementRun, // h byte full filled
// RequiredSpace: 6,
// Buf: []byte{
// 0x80, // h-byte (literal, len=1)
// 0x94, // delta 20
// 0x01, // h-byte (run, len=3)
// 0xbc, // delta 60
// 0x28, 0x80, 0x24, 0x6a, // since
// },
// LastUnixtime: 1780777060 + 128*60,
// LastDelta: 60,
// H: 127, // run, length = 129
// },
// {
// Nums: []uint32{
// 1780777000,
// 1780777060, // +60
// },
// RepeatLastDeltaNTimes: 129,
// CompressionWay: endSeries, // run, h byte overflowed
// RequiredSpace: 8,
// Buf: []byte{
// 0, 0, 0, 0, 0, 0,
// 127, // h - end of run (length = 129)
// 188, // varUint64(60)
// },
// LastUnixtime: 1780777060 + 129*60,
// LastDelta: 60,
// H: 128, // literal, length = 1
// },
{
Nums: []uint32{
1780777000,
1780777060, // +60
1780777130, // +70
},
RepeatLastDeltaNTimes: 128,
CompressionWay: incrementRun, // h byte full filled
RequiredSpace: 6,
Name: "increment literal",
Offset: 1,
ChangeSize: 2,
Buf: []byte{
0, 0, 0, 0, 0, 0, 0, 0,
0x00,
0x81, // h-byte (literal, len=2)
0xc6, // delta 70
0xbc, // delta 60
0x28, 0x80, 0x24, 0x6a, // since
},
LastUnixtime: 1780777060 + 128*60,
LastDelta: 60,
H: 127, // run, length = 129
},
{
Nums: []uint32{
1780777000,
1780777060, // +60
},
RepeatLastDeltaNTimes: 129,
CompressionWay: endSeries, // run, h byte overflowed
RequiredSpace: 8,
Buf: []byte{
0, 0, 0, 0, 0, 0,
127, // h - end of run (length = 129)
188, // varUint64(60)
},
LastUnixtime: 1780777060 + 129*60,
LastDelta: 60,
H: 128, // literal, length = 1
},
{
Nums: []uint32{
1780777000,
1780777060, // +60
1780777122, // +62
},
CompressionWay: incrementLiteral,
RequiredSpace: 7,
Buf: []byte{
0, 0, 0, 0, 0, 0, 0,
188, // varUint64(60)
},
LastUnixtime: 1780777122,
LastDelta: 62,
H: 129, // literal, length = 2
LastUnixtime: 1780777130,
LastDelta: 70,
},
}
)
for caseIdx, testCase := range testCases {
for _, testCase := range testCases {
//fmt.Println("----------")
var (
buf = make([]byte, 8)
payloadSize = 0
compressionWay int
requiredSpace int
tmp = make([]byte, tmpTimeSize)
buf = make([]byte, 8)
payloadSize = 0
report qb.TimeEvaluationReport
)
c := NewTimeDeltaCompressor(buf, payloadSize)
for _, num := range testCase.Nums {
compressionWay, requiredSpace = c.Evaluate(num)
c.Compress(compressionWay, num)
report = c.Evaluate(tmp, num)
// fmt.Println("----------\npos", c.pos)
// fmt.Printf("buf: % x\n", c.buf[c.pos:])
// pretty.Println(report)
// fmt.Printf("change: % x\n", tmp[:report.ChangeSize])
c.Append(report.Offset, tmp[:report.ChangeSize], num)
}
if testCase.RepeatLastDeltaNTimes > 0 {
for range testCase.RepeatLastDeltaNTimes {
num := c.lastUnixtime + c.lastDelta
compressionWay, requiredSpace = c.Evaluate(num)
c.Compress(compressionWay, num)
}
if report.Offset != testCase.Offset {
t.Fatalf("%s: got offset %d are not equal expected %d",
testCase.Name, report.Offset, testCase.Offset)
}
if compressionWay != testCase.CompressionWay {
t.Fatalf("%d: got code %d are not equal expected %d",
caseIdx, compressionWay, testCase.CompressionWay)
}
if requiredSpace != testCase.RequiredSpace {
t.Fatalf("%d: got requiredSpace %d are not equal expected %d",
caseIdx, requiredSpace, testCase.RequiredSpace)
if report.ChangeSize != testCase.ChangeSize {
t.Fatalf("%s: got changeSize %d are not equal expected %d",
testCase.Name, report.ChangeSize, testCase.ChangeSize)
}
if !bytes.Equal(buf, testCase.Buf) {
t.Fatalf("%d: got buf %v are not equal expected %v",
caseIdx, buf, testCase.Buf)
t.Fatalf("%s: got buf % x are not equal expected % x",
testCase.Name, buf, testCase.Buf)
}
if c.lastUnixtime != testCase.LastUnixtime {
t.Fatalf("%d: got lastUnixtime %d are not equal expected %d",
caseIdx, c.lastUnixtime, testCase.LastUnixtime)
t.Fatalf("%s: got lastUnixtime %d are not equal expected %d",
testCase.Name, c.lastUnixtime, testCase.LastUnixtime)
}
if c.lastDelta != testCase.LastDelta {
t.Fatalf("%d: got lastDelta %d are not equal expected %d",
caseIdx, c.lastDelta, testCase.LastDelta)
}
if c.h != testCase.H {
t.Fatalf("%d: got h %d are not equal expected %d",
caseIdx, c.h, testCase.H)
t.Fatalf("%s: got lastDelta %d are not equal expected %d",
testCase.Name, c.lastDelta, testCase.LastDelta)
}
}
}
@@ -961,14 +982,15 @@ func TestTimeDeltaDecompressorFromState(t *testing.T) {
)
for caseIdx, testCase := range testCases {
var (
tmp = make([]byte, tmpTimeSize)
buf = make([]byte, 16)
payloadSize = 0
decodedNums []uint32
)
c := NewTimeDeltaCompressor(buf, payloadSize)
for _, num := range testCase.Nums {
compressionWay, _ := c.Evaluate(num)
c.Compress(compressionWay, num)
report := c.Evaluate(tmp, num)
c.Append(report.Offset, tmp[:report.ChangeSize], num)
}
//