dima/qb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wp

Browse Source
This commit is contained in:
dima
2026-06-07 01:06:16 +00:00
parent 0e019118c2
commit 776a341c57
9 changed files with 738 additions and 747 deletions
Download Patch File Download Diff File Expand all files Collapse all files

423
chunkenc/chunkenc_test.go
View File

@@ -1,7 +1,9 @@
package chunkenc
import (
"bytes"
"fmt"
"slices"
"testing"
)
@@ -12,7 +14,7 @@ func TestCumdelta(t *testing.T) {
value float64
done bool
)
c := NewReverseCumulativeDeltaCompressor(buf, 0, fracDigits)
c := NewCumulativeDeltaCompressor(buf, 0, fracDigits)
// c.Append(1.55)
// c.Append(23)
// c.Append(23)
@@ -28,7 +30,7 @@ func TestCumdelta(t *testing.T) {
//fmt.Printf("pos: %d\n", c.pos)
//fmt.Printf("%d\n", buf.Chunks()[0])
d := NewReverseCumulativeDeltaDecompressor(buf, c.Size(), fracDigits)
d := NewCumulativeDeltaDecompressor(buf, c.Size(), fracDigits)
for range 8 {
value, done = d.NextValue()
@@ -43,7 +45,7 @@ func TestInsdelta(t *testing.T) {
value float64
done bool
)
c := NewReverseInstantDeltaCompressor(buf, 0, fracDigits)
c := NewInstantDeltaCompressor(buf, 0, fracDigits)
c.Append(-1.55)
c.Append(23)
@@ -58,7 +60,7 @@ func TestInsdelta(t *testing.T) {
//fmt.Printf("pos: %d\n", c.pos)
//fmt.Printf("%d\n", buf.Chunks()[0])
d := NewReverseInstantDeltaDecompressor(buf, c.Size(), fracDigits)
d := NewInstantDeltaDecompressor(buf, c.Size(), fracDigits)
for range 8 {
value, done = d.NextValue()
@@ -66,55 +68,366 @@ func TestInsdelta(t *testing.T) {
}
}
func TestTimeDelta(t *testing.T) {
func TestTimeDeltaCompressor(t *testing.T) {
var (
buf = make([]byte, minBufferSize)
value uint32
done bool
testCases = []struct {
Nums []uint32
RepeatLastDeltaNTimes int
Code int
RequiredSize int
Buf []byte
LastUnixtime uint32
LastDelta uint32
H byte
}{
{
Nums: []uint32{
1780777000,
},
Code: addUnixtime,
RequiredSize: 4,
Buf: []byte{
0, 0, 0, 0, 0, 0, 0, 0,
},
LastUnixtime: 1780777000,
LastDelta: 0,
H: 0,
},
{
Nums: []uint32{
1780777000,
1780777060, // +60
},
Code: add1stDelta,
RequiredSize: 6,
Buf: []byte{
0, 0, 0, 0, 0, 0, 0, 0,
},
LastUnixtime: 1780777060,
LastDelta: 60,
H: 128,
},
{
Nums: []uint32{
1780777000,
1780777060, // +60
1780777120, // +60
},
Code: startRun, // literal changed by run
RequiredSize: 6,
Buf: []byte{
0, 0, 0, 0, 0, 0, 0, 0,
},
LastUnixtime: 1780777120,
LastDelta: 60,
H: 0, // run, length = 2
},
{
Nums: []uint32{
1780777000,
1780777060, // +60
1780777130, // +70
1780777200, // +70
},
Code: startRun, // literal decreased by 1
RequiredSize: 7, // unixtime, end of literal, new delta, new h
Buf: []byte{
0, 0, 0, 0, 0, 0, 128, 188,
},
LastUnixtime: 1780777200,
LastDelta: 70,
H: 0, // run, length = 2
},
{
Nums: []uint32{
1780777000,
1780777060, // +60
1780777120, // +60
1780777180, // +60
},
Code: incrementRun,
RequiredSize: 5,
Buf: []byte{
0, 0, 0, 0, 0, 0, 0, 0,
},
LastUnixtime: 1780777180,
LastDelta: 60,
H: 1, // run, length = 3
},
{
Nums: []uint32{
1780777000,
1780777060, // +60
1780777120, // +60
1780777180, // +60
1780777200, // +20
},
Code: endSeries, // run
RequiredSize: 8,
Buf: []byte{
0, 0, 0, 0, 0, 0,
1, // h of run, length = 3
188, // varUint64(60)
},
LastUnixtime: 1780777200,
LastDelta: 20,
H: 128, // literal, length = 1
},
{
Nums: []uint32{
1780777000,
1780777060, // +60
},
RepeatLastDeltaNTimes: 128,
Code: incrementRun, // h byte full filled
RequiredSize: 5,
Buf: []byte{
0, 0, 0, 0, 0, 0, 0, 0,
},
LastUnixtime: 1780777060 + 128*60,
LastDelta: 60,
H: 127, // run, length = 129
},
{
Nums: []uint32{
1780777000,
1780777060, // +60
},
RepeatLastDeltaNTimes: 129,
Code: endSeries, // run, h byte overflowed
RequiredSize: 8,
Buf: []byte{
0, 0, 0, 0, 0, 0,
127, // h - end of run (length = 129)
188, // varUint64(60)
},
LastUnixtime: 1780777060 + 129*60,
LastDelta: 60,
H: 128, // literal, length = 1
},
{
Nums: []uint32{
1780777000,
1780777060, // +60
1780777122, // +62
},
Code: incrementLiteral,
RequiredSize: 7,
Buf: []byte{
0, 0, 0, 0, 0, 0, 0,
188, // varUint64(60)
},
LastUnixtime: 1780777122,
LastDelta: 62,
H: 129, // literal, length = 2
},
}
)
c := NewReverseTimeDeltaCompressor(buf, 0)
c.Append(10)
//c.Append(131)
//fmt.Printf("pos: %d\n", c.pos)
//fmt.Printf("%d\n", buf.Chunks()[0])
c.Append(70)
c.Append(130)
c.Append(191)
c.Append(248)
c.Append(305)
c.Append(330)
c.Append(390)
c.Append(450)
// fmt.Printf("pos: %d\n", c.pos)
// fmt.Printf("%d\n", buf.Chunks()[0])
//c.Sync()
// bound := c.GetState()
// fmt.Println("AFTER SYNC")
// fmt.Printf("pos: %d\n", c.pos)
// chunks := buf.Chunks()
// if len(chunks) > 0 {
// fmt.Printf("%d\n", chunks[0])
// }
d := NewReverseTimeDeltaDecompressor(buf, c.Size())
// d.RestoreFromBound(bound)
//d.RestoreFromEnd()
for range 12 {
value, done = d.NextValue()
fmt.Println(value, done)
for caseIdx, testCase := range testCases {
//fmt.Println("----------")
var (
buf = make([]byte, 8)
code int
requiredSize int
)
c := NewTimeDeltaCompressor(buf, 0)
for _, num := range testCase.Nums {
code, requiredSize = c.CalcRequiredSpace(num)
c.Append(code, num)
}
if testCase.RepeatLastDeltaNTimes > 0 {
for range testCase.RepeatLastDeltaNTimes {
num := c.lastUnixtime + c.lastDelta
code, requiredSize = c.CalcRequiredSpace(num)
c.Append(code, num)
}
}
if code != testCase.Code {
t.Fatalf("%d: got code %d are not equal expected %d",
caseIdx, code, testCase.Code)
}
if requiredSize != testCase.RequiredSize {
t.Fatalf("%d: got requiredSize %d are not equal expected %d",
caseIdx, requiredSize, testCase.RequiredSize)
}
if !bytes.Equal(buf, testCase.Buf) {
t.Fatalf("%d: got buf %v are not equal expected %v",
caseIdx, buf, testCase.Buf)
}
if c.lastUnixtime != testCase.LastUnixtime {
t.Fatalf("%d: got lastUnixtime %d are not equal expected %d",
caseIdx, c.lastUnixtime, testCase.LastUnixtime)
}
if c.lastDelta != testCase.LastDelta {
t.Fatalf("%d: got lastDelta %d are not equal expected %d",
caseIdx, c.lastDelta, testCase.LastDelta)
}
if c.h != testCase.H {
t.Fatalf("%d: got h %d are not equal expected %d",
caseIdx, c.h, testCase.H)
}
}
}
func TestTimeDeltaDecompressorFromCapturedState(t *testing.T) {
var (
testCases = []struct {
Nums []uint32
//RepeatLastDeltaNTimes int
}{
{
Nums: []uint32{
1780777000,
},
},
{
// add1stDelta
Nums: []uint32{
1780777000,
1780777060, // +60
},
},
{
// startRun
Nums: []uint32{
1780777000,
1780777060, // +60
1780777120, // +60
},
},
{
// incrementRun
Nums: []uint32{
1780777000,
1780777060, // +60
1780777120, // +60
1780777180, // +60
},
},
{
// endRun
Nums: []uint32{
1780777000,
1780777060, // +60
1780777120, // +60
1780777180, // +60
1780777200, // +20
},
},
{
// incrementLiteral
Nums: []uint32{
1780777000,
1780777060, // +60
1780777122, // +62
},
},
{
// ...
Nums: []uint32{
1780777000,
1780777010, // +10
1780777120, // +10
1780777130, // +10
1780777135, // +5
1780777141, // +6
1780777148, // +7
1780777156, // +8
1780777165, // +9
1780777174, // +9
1780777183, // +9
1780777190, // +7
},
},
}
)
for caseIdx, testCase := range testCases {
var (
buf = make([]byte, 16)
decodedNums []uint32
)
c := NewTimeDeltaCompressor(buf, 0)
for _, num := range testCase.Nums {
code, _ := c.CalcRequiredSpace(num)
c.Append(code, num)
}
//
c.CaptureState()
d := c.CreateDecompressor()
//fmt.Println(testCase.Nums)
// fmt.Println("---------------")
// fmt.Println(buf)
for {
num, done := d.NextValue()
//fmt.Println(num, done)
if done {
break
}
decodedNums = append(decodedNums, num)
}
slices.Reverse(decodedNums)
if !slices.Equal(testCase.Nums, decodedNums) {
t.Fatalf("%d: got nums %v not equal expected %v", caseIdx, decodedNums, testCase.Nums)
}
}
}
// func TestTimeDelta(t *testing.T) {
// var (
// buf = make([]byte, minBufferSize)
// value uint32
// done bool
// )
// c := NewReverseTimeDeltaCompressor(buf, 0)
// c.Append(10)
// //c.Append(131)
// //fmt.Printf("pos: %d\n", c.pos)
// //fmt.Printf("%d\n", buf.Chunks()[0])
// c.Append(70)
// c.Append(130)
// c.Append(191)
// c.Append(248)
// c.Append(305)
// c.Append(330)
// c.Append(390)
// c.Append(450)
// // fmt.Printf("pos: %d\n", c.pos)
// // fmt.Printf("%d\n", buf.Chunks()[0])
// //c.Sync()
// // bound := c.GetState()
// // fmt.Println("AFTER SYNC")
// // fmt.Printf("pos: %d\n", c.pos)
// // chunks := buf.Chunks()
// // if len(chunks) > 0 {
// // fmt.Printf("%d\n", chunks[0])
// // }
// d := NewReverseTimeDeltaDecompressor(buf, c.Size())
// // d.RestoreFromBound(bound)
// //d.RestoreFromEnd()
// for range 12 {
// value, done = d.NextValue()
// fmt.Println(value, done)
// }
// }
func TestCumdeltaBound(t *testing.T) {
var (
fracDigits byte = 0
@@ -122,7 +435,7 @@ func TestCumdeltaBound(t *testing.T) {
value float64
done bool
)
c := NewReverseCumulativeDeltaCompressor(buf, 0, fracDigits)
c := NewCumulativeDeltaCompressor(buf, 0, fracDigits)
// c.Append(1.55)
// c.Append(23)
// c.Append(23)
@@ -157,7 +470,7 @@ func TestCumdeltaBound(t *testing.T) {
fmt.Printf("pos: %d\n", c.pos)
fmt.Printf("%d\n", c.Chunks())
d := NewReverseCumulativeDeltaDecompressor(buf, c.Size(), fracDigits)
d := NewCumulativeDeltaDecompressor(buf, c.Size(), fracDigits)
d.RestoreFromEnd()
for range 8 {
@@ -183,7 +496,7 @@ func TestInsdeltaBound(t *testing.T) {
value float64
done bool
)
c := NewReverseInstantDeltaCompressor(buf, 0, fracDigits)
c := NewInstantDeltaCompressor(buf, 0, fracDigits)
// c.Append(1.55)
// c.Append(23)
// c.Append(23)
@@ -218,7 +531,7 @@ func TestInsdeltaBound(t *testing.T) {
fmt.Printf("pos: %d\n", c.pos)
fmt.Printf("%d\n", c.Chunks())
d := NewReverseInstantDeltaDecompressor(buf, c.Size(), fracDigits)
d := NewInstantDeltaDecompressor(buf, c.Size(), fracDigits)
d.RestoreFromEnd()
for range 8 {
@@ -226,7 +539,7 @@ func TestInsdeltaBound(t *testing.T) {
fmt.Println(value, done)
}
boundDecompressor := NewReverseInstantDeltaDecompressor(buf, c.Size(), fracDigits)
boundDecompressor := NewInstantDeltaDecompressor(buf, c.Size(), fracDigits)
//boundDecompressor.RestoreFromBound(bound)
fmt.Println("from bound:")
@@ -243,7 +556,7 @@ func TestInsdeltaBound2(t *testing.T) {
value float64
done bool
)
c := NewReverseInstantDeltaCompressor(buf, 0, fracDigits)
c := NewInstantDeltaCompressor(buf, 0, fracDigits)
// c.Append(1.55)
// c.Append(23)
// c.Append(23)
@@ -278,7 +591,7 @@ func TestInsdeltaBound2(t *testing.T) {
fmt.Printf("pos: %d\n", c.pos)
fmt.Printf("%d\n", c.Chunks())
d := NewReverseInstantDeltaDecompressor(buf, c.Size(), fracDigits)
d := NewInstantDeltaDecompressor(buf, c.Size(), fracDigits)
d.RestoreFromEnd()
for range 8 {
@@ -286,7 +599,7 @@ func TestInsdeltaBound2(t *testing.T) {
fmt.Println(value, done)
}
boundDecompressor := NewReverseInstantDeltaDecompressor(buf, c.Size(), fracDigits)
boundDecompressor := NewInstantDeltaDecompressor(buf, c.Size(), fracDigits)
//sboundDecompressor.RestoreFromBound(bound)
fmt.Println("from bound:")