package enc import ( "bytes" "fmt" "math" "slices" "testing" bin "gordenko.dev/dima/bin/little" ) func equalFloatSlices(a, b []float64, epsilon float64) bool { return slices.EqualFunc(a, b, func(x, y float64) bool { return math.Abs(x-y) <= epsilon }) } // CUMULATIVE func TestCumulativeDeltaCompressor(t *testing.T) { var ( testCases = []struct { Nums []float64 RepeatLastDeltaNTimes int Code int RequiredSpace int Buf []byte BaseValue float64 LastDelta uint64 H byte }{ { Nums: []float64{ 1.5, }, Code: addBaseValue, RequiredSpace: 3, // base value, delta, h Buf: []byte{ 143, 0, 0, 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: 0, H: 128, // literal, length = 1 }, { Nums: []float64{ 1.5, 1.5, }, Code: startRun, // literal changed to run RequiredSpace: 2, // delta, h Buf: []byte{ 143, 0, 0, 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: 0, H: 0, // run, length = 2 }, { Nums: []float64{ 1.5, 1.6, 1.6, }, Code: startRun, // literal decreased by 1 RequiredSpace: 3, // end of literal, new delta, new h Buf: []byte{ 143, // base value 128, // literal 1st delta (0) 128, // h - end of literal, length = 1 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: 1, H: 0, // run, length = 2 }, { Nums: []float64{ 1.5, 1.5, 1.5, }, Code: incrementRun, RequiredSpace: 2, Buf: []byte{ 143, // base value 0, 0, 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: 0, H: 1, // run, length = 3 }, { Nums: []float64{ 1.5, 1.5, 1.5, 1.6, }, Code: endSeries, // end of run RequiredSpace: 4, Buf: []byte{ 143, // base value 128, // run delta (0) 1, // h of run, length = 3 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: 1, H: 128, // literal, length = 1 }, { Nums: []float64{ 1.5, }, RepeatLastDeltaNTimes: 128, Code: incrementRun, // h byte full filled RequiredSpace: 2, Buf: []byte{ 143, // base value 0, 0, 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: 0, H: 127, // run, length = 129 }, { Nums: []float64{ 1.5, }, RepeatLastDeltaNTimes: 129, Code: endSeries, // run, h byte overflowed RequiredSpace: 4, // run delta, h of run, 1st literal delta, h of literal Buf: []byte{ 143, // base value 128, // run delta (0) 127, // h of run (length = 129) 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: 0, H: 128, // literal, length = 1 }, { Nums: []float64{ 1.5, 1.6, }, Code: incrementLiteral, RequiredSpace: 3, // previous delta, new delta, h Buf: []byte{ 143, // base value 128, // literal 1st delta (0) 0, 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: 1, H: 129, // literal, length = 2 }, } ) for caseIdx, testCase := range testCases { var ( fracDigits byte = 1 buf = make([]byte, 8) compressionWay int requiredSpace int ) c := NewCumulativeDeltaCompressor(buf, fracDigits) for _, num := range testCase.Nums { compressionWay, requiredSpace = c.Evaluate(num) c.Compress(compressionWay, num) } if testCase.RepeatLastDeltaNTimes > 0 { num := testCase.Nums[len(testCase.Nums)-1] for range testCase.RepeatLastDeltaNTimes { compressionWay, requiredSpace = c.Evaluate(num) c.Compress(compressionWay, num) } } if compressionWay != testCase.Code { t.Fatalf("%d: got code %d are not equal expected %d", caseIdx, compressionWay, testCase.Code) } if requiredSpace != testCase.RequiredSpace { t.Fatalf("%d: got requiredSpace %d are not equal expected %d", caseIdx, requiredSpace, testCase.RequiredSpace) } if !bytes.Equal(buf, testCase.Buf) { t.Fatalf("%d: got buf %v are not equal expected %v", caseIdx, buf, testCase.Buf) } if c.baseValue != testCase.BaseValue { t.Fatalf("%d: got lastUnixtime %v are not equal expected %v", caseIdx, c.baseValue, testCase.BaseValue) } if c.lastDelta != testCase.LastDelta { t.Fatalf("%d: got lastDelta %d are not equal expected %d", caseIdx, c.lastDelta, testCase.LastDelta) } if c.h != testCase.H { t.Fatalf("%d: got h %d are not equal expected %d", caseIdx, c.h, testCase.H) } } } func TestCumulativeDeltaDecompressorFromState(t *testing.T) { var ( testCases = []struct { Nums []float64 //RepeatLastDeltaNTimes int }{ { // addBaseValue Nums: []float64{ 1.5, }, }, { // startRun Nums: []float64{ 1.5, 1.5, }, }, { // startRun Nums: []float64{ 1.5, 1.6, 1.6, }, }, { // incrementRun Nums: []float64{ 1.5, 1.5, 1.5, }, }, { // endSeries run Nums: []float64{ 1.5, 1.5, 1.5, 1.6, }, }, { // incrementLiteral Nums: []float64{ 1.5, 1.6, }, }, { // ... Nums: []float64{ 1.5, 1.5, 1.5, // 1.5, // 1.6, // 1.7, // 1.8, // 1.9, // 2.0, // 2.0, // 2.0, // 2.2, // }, }, } ) for caseIdx, testCase := range testCases { var ( fracDigits byte = 1 buf = make([]byte, 16) decodedNums []float64 ) c := NewCumulativeDeltaCompressor(buf, fracDigits) for _, num := range testCase.Nums { compressionWay, _ := c.Evaluate(num) c.Compress(compressionWay, num) } // c.CaptureState() fmt.Println("---------------") d := c.CreateDecompressor() //fmt.Println(buf) for { num, done := d.NextValue() //fmt.Println(num, done) if done { break } decodedNums = append(decodedNums, num) } slices.Reverse(decodedNums) fmt.Println(testCase.Nums) fmt.Println(decodedNums) fmt.Println() if !equalFloatSlices(testCase.Nums, decodedNums, 0.0000001) { t.Fatalf("%d: got nums %v not equal expected %v", caseIdx, decodedNums, testCase.Nums) } } } // INSTANT func TestInstantDeltaCompressor(t *testing.T) { var ( testCases = []struct { Nums []float64 RepeatLastDeltaNTimes int Code int RequiredSpace int Buf []byte BaseValue float64 LastDelta int64 H byte }{ { Nums: []float64{ 1.5, }, Code: addBaseValue, RequiredSpace: 3, // base value, delta, h Buf: []byte{ 158, 0, 0, 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: 0, H: 128, // literal, length = 1 }, { Nums: []float64{ 1.5, 1.5, }, Code: startRun, // literal changed to run RequiredSpace: 2, // delta, h Buf: []byte{ 158, 0, 0, 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: 0, H: 0, // run, length = 2 }, { Nums: []float64{ 1.5, 1.6, 1.6, }, Code: startRun, // literal decreased by 1 RequiredSpace: 3, // end of literal, new delta, new h Buf: []byte{ 158, // base value 128, // literal 1st delta (0) 128, // h - end of literal, length = 1 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: 1, H: 0, // run, length = 2 }, { // same as prev, but negative delta Nums: []float64{ 1.5, 1.4, 1.4, }, Code: startRun, // literal decreased by 1 RequiredSpace: 3, // end of literal, new delta, new h Buf: []byte{ 158, // base value 128, // literal 1st delta (0) 128, // h - end of literal, length = 1 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: -1, H: 0, // run, length = 2 }, { Nums: []float64{ 1.5, 1.5, 1.5, }, Code: incrementRun, RequiredSpace: 2, Buf: []byte{ 158, // base value 0, 0, 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: 0, H: 1, // run, length = 3 }, { Nums: []float64{ 1.5, 1.5, 1.5, 1.6, }, Code: endSeries, // end of run RequiredSpace: 4, Buf: []byte{ 158, // base value 128, // run delta (0) 1, // h of run, length = 3 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: 1, H: 128, // literal, length = 1 }, { // same as prev, but negative delta Nums: []float64{ 1.5, 1.5, 1.5, 1.4, }, Code: endSeries, // end of run RequiredSpace: 4, Buf: []byte{ 158, // base value 128, // run delta (0) 1, // h of run, length = 3 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: -1, H: 128, // literal, length = 1 }, { Nums: []float64{ 1.5, }, RepeatLastDeltaNTimes: 128, Code: incrementRun, // h byte full filled RequiredSpace: 2, Buf: []byte{ 158, // base value 0, 0, 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: 0, H: 127, // run, length = 129 }, { Nums: []float64{ 1.5, }, RepeatLastDeltaNTimes: 129, Code: endSeries, // run, h byte overflowed RequiredSpace: 4, // run delta, h of run, 1st literal delta, h of literal Buf: []byte{ 158, // base value 128, // run delta (0) 127, // h of run (length = 129) 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: 0, H: 128, // literal, length = 1 }, { Nums: []float64{ 1.5, 1.6, }, Code: incrementLiteral, RequiredSpace: 3, // previous delta, new delta, h Buf: []byte{ 158, // base value 128, // literal 1st delta (0) 0, 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: 1, H: 129, // literal, length = 2 }, { // same as prev, but negative delta Nums: []float64{ 1.5, 1.4, }, Code: incrementLiteral, RequiredSpace: 3, // previous delta, new delta, h Buf: []byte{ 158, // base value 128, // literal 1st delta (0) 0, 0, 0, 0, 0, 0, }, BaseValue: 1.5, LastDelta: -1, H: 129, // literal, length = 2 }, } ) for caseIdx, testCase := range testCases { var ( fracDigits byte = 1 buf = make([]byte, 8) compressionWay int requiredSpace int ) c := NewInstantDeltaCompressor(buf, fracDigits) for _, num := range testCase.Nums { compressionWay, requiredSpace = c.Evaluate(num) c.Compress(compressionWay, num) } if testCase.RepeatLastDeltaNTimes > 0 { num := testCase.Nums[len(testCase.Nums)-1] for range testCase.RepeatLastDeltaNTimes { compressionWay, requiredSpace = c.Evaluate(num) c.Compress(compressionWay, num) } } if compressionWay != testCase.Code { t.Fatalf("%d: got code %d are not equal expected %d", caseIdx, compressionWay, testCase.Code) } if requiredSpace != testCase.RequiredSpace { t.Fatalf("%d: got requiredSpace %d are not equal expected %d", caseIdx, requiredSpace, testCase.RequiredSpace) } if !bytes.Equal(buf, testCase.Buf) { t.Fatalf("%d: got buf %v are not equal expected %v", caseIdx, buf, testCase.Buf) } if c.baseValue != testCase.BaseValue { t.Fatalf("%d: got lastUnixtime %v are not equal expected %v", caseIdx, c.baseValue, testCase.BaseValue) } if c.lastDelta != testCase.LastDelta { t.Fatalf("%d: got lastDelta %d are not equal expected %d", caseIdx, c.lastDelta, testCase.LastDelta) } if c.h != testCase.H { t.Fatalf("%d: got h %d are not equal expected %d", caseIdx, c.h, testCase.H) } } } func TestInstantDeltaDecompressorFromState(t *testing.T) { var ( testCases = []struct { Nums []float64 //RepeatLastDeltaNTimes int }{ { // addBaseValue Nums: []float64{ 1.5, }, }, { // startRun Nums: []float64{ 1.5, 1.5, }, }, { // startRun Nums: []float64{ 1.5, 1.6, 1.6, }, }, { // incrementRun Nums: []float64{ 1.5, 1.5, 1.5, }, }, { // endSeries run Nums: []float64{ 1.5, 1.5, 1.5, 1.6, }, }, { // incrementLiteral Nums: []float64{ 1.5, 1.6, }, }, { // ... Nums: []float64{ 1.5, 1.5, 1.5, // 1.5, // 1.6, // 1.7, // 1.8, // 1.9, // 2.0, // 2.0, // 2.0, // 2.2, // }, }, } ) for caseIdx, testCase := range testCases { var ( fracDigits byte = 1 buf = make([]byte, 16) decodedNums []float64 ) c := NewInstantDeltaCompressor(buf, fracDigits) for _, num := range testCase.Nums { compressionWay, _ := c.Evaluate(num) c.Compress(compressionWay, num) } // c.CaptureState() fmt.Println("---------------") d := c.CreateDecompressor() //fmt.Println(buf) for { num, done := d.NextValue() //fmt.Println(num, done) if done { break } decodedNums = append(decodedNums, num) } slices.Reverse(decodedNums) fmt.Println(testCase.Nums) fmt.Println(decodedNums) fmt.Println() if !equalFloatSlices(testCase.Nums, decodedNums, 0.0000001) { t.Fatalf("%d: got nums %v not equal expected %v", caseIdx, decodedNums, testCase.Nums) } } } // TIME DELTA func TestTimeDeltaCompressor(t *testing.T) { var ( testCases = []struct { Nums []uint32 RepeatLastDeltaNTimes int CompressionWay int RequiredSpace int Buf []byte LastUnixtime uint32 LastDelta uint32 H byte }{ { Nums: []uint32{ 1780777000, }, CompressionWay: addUnixtime, RequiredSpace: 4, Buf: []byte{ 0, 0, 0, 0, 0, 0, 0, 0, }, LastUnixtime: 1780777000, LastDelta: 0, H: 0, }, { Nums: []uint32{ 1780777000, 1780777060, // +60 }, CompressionWay: add1stDelta, RequiredSpace: 6, Buf: []byte{ 0, 0, 0, 0, 0, 0, 0, 0, }, LastUnixtime: 1780777060, LastDelta: 60, H: 128, }, { Nums: []uint32{ 1780777000, 1780777060, // +60 1780777120, // +60 }, CompressionWay: startRun, // literal changed by run RequiredSpace: 6, Buf: []byte{ 0, 0, 0, 0, 0, 0, 0, 0, }, LastUnixtime: 1780777120, LastDelta: 60, H: 0, // run, length = 2 }, { Nums: []uint32{ 1780777000, 1780777060, // +60 1780777130, // +70 1780777200, // +70 }, CompressionWay: startRun, // literal decreased by 1 RequiredSpace: 7, // unixtime, end of literal, new delta, new h Buf: []byte{ 0, 0, 0, 0, 0, 0, 128, 188, }, LastUnixtime: 1780777200, LastDelta: 70, H: 0, // run, length = 2 }, { Nums: []uint32{ 1780777000, 1780777060, // +60 1780777120, // +60 1780777180, // +60 }, CompressionWay: incrementRun, RequiredSpace: 6, Buf: []byte{ 0, 0, 0, 0, 0, 0, 0, 0, }, LastUnixtime: 1780777180, LastDelta: 60, H: 1, // run, length = 3 }, { Nums: []uint32{ 1780777000, 1780777060, // +60 1780777120, // +60 1780777180, // +60 1780777200, // +20 }, CompressionWay: endSeries, // run RequiredSpace: 8, Buf: []byte{ 0, 0, 0, 0, 0, 0, 1, // h of run, length = 3 188, // varUint64(60) }, LastUnixtime: 1780777200, LastDelta: 20, H: 128, // literal, length = 1 }, { Nums: []uint32{ 1780777000, 1780777060, // +60 }, RepeatLastDeltaNTimes: 128, CompressionWay: incrementRun, // h byte full filled RequiredSpace: 6, Buf: []byte{ 0, 0, 0, 0, 0, 0, 0, 0, }, LastUnixtime: 1780777060 + 128*60, LastDelta: 60, H: 127, // run, length = 129 }, { Nums: []uint32{ 1780777000, 1780777060, // +60 }, RepeatLastDeltaNTimes: 129, CompressionWay: endSeries, // run, h byte overflowed RequiredSpace: 8, Buf: []byte{ 0, 0, 0, 0, 0, 0, 127, // h - end of run (length = 129) 188, // varUint64(60) }, LastUnixtime: 1780777060 + 129*60, LastDelta: 60, H: 128, // literal, length = 1 }, { Nums: []uint32{ 1780777000, 1780777060, // +60 1780777122, // +62 }, CompressionWay: incrementLiteral, RequiredSpace: 7, Buf: []byte{ 0, 0, 0, 0, 0, 0, 0, 188, // varUint64(60) }, LastUnixtime: 1780777122, LastDelta: 62, H: 129, // literal, length = 2 }, } ) for caseIdx, testCase := range testCases { //fmt.Println("----------") var ( buf = make([]byte, 8) compressionWay int requiredSpace int ) c := NewTimeDeltaCompressor(buf) for _, num := range testCase.Nums { compressionWay, requiredSpace = c.Evaluate(num) c.Compress(compressionWay, num) } if testCase.RepeatLastDeltaNTimes > 0 { for range testCase.RepeatLastDeltaNTimes { num := c.lastUnixtime + c.lastDelta compressionWay, requiredSpace = c.Evaluate(num) c.Compress(compressionWay, num) } } if compressionWay != testCase.CompressionWay { t.Fatalf("%d: got code %d are not equal expected %d", caseIdx, compressionWay, testCase.CompressionWay) } if requiredSpace != testCase.RequiredSpace { t.Fatalf("%d: got requiredSpace %d are not equal expected %d", caseIdx, requiredSpace, testCase.RequiredSpace) } if !bytes.Equal(buf, testCase.Buf) { t.Fatalf("%d: got buf %v are not equal expected %v", caseIdx, buf, testCase.Buf) } if c.lastUnixtime != testCase.LastUnixtime { t.Fatalf("%d: got lastUnixtime %d are not equal expected %d", caseIdx, c.lastUnixtime, testCase.LastUnixtime) } if c.lastDelta != testCase.LastDelta { t.Fatalf("%d: got lastDelta %d are not equal expected %d", caseIdx, c.lastDelta, testCase.LastDelta) } if c.h != testCase.H { t.Fatalf("%d: got h %d are not equal expected %d", caseIdx, c.h, testCase.H) } } } func TestTimeDeltaDecompressorFromState(t *testing.T) { var ( testCases = []struct { Nums []uint32 //RepeatLastDeltaNTimes int }{ { Nums: []uint32{ 1780777000, }, }, { // add1stDelta Nums: []uint32{ 1780777000, 1780777060, // +60 }, }, { // startRun Nums: []uint32{ 1780777000, 1780777060, // +60 1780777120, // +60 }, }, { // incrementRun Nums: []uint32{ 1780777000, 1780777060, // +60 1780777120, // +60 1780777180, // +60 }, }, { // endRun Nums: []uint32{ 1780777000, 1780777060, // +60 1780777120, // +60 1780777180, // +60 1780777200, // +20 }, }, { // incrementLiteral Nums: []uint32{ 1780777000, 1780777060, // +60 1780777122, // +62 }, }, { // ... Nums: []uint32{ 1780777000, 1780777010, // +10 1780777120, // +10 1780777130, // +10 1780777135, // +5 1780777141, // +6 1780777148, // +7 1780777156, // +8 1780777165, // +9 1780777174, // +9 1780777183, // +9 1780777190, // +7 }, }, } ) for caseIdx, testCase := range testCases { var ( buf = make([]byte, 16) decodedNums []uint32 ) c := NewTimeDeltaCompressor(buf) for _, num := range testCase.Nums { compressionWay, _ := c.Evaluate(num) c.Compress(compressionWay, num) } // c.CaptureState() d := c.CreateDecompressor() //fmt.Println(testCase.Nums) // fmt.Println("---------------") // fmt.Println(buf) for { num, done := d.NextValue() //fmt.Println(num, done) if done { break } decodedNums = append(decodedNums, num) } slices.Reverse(decodedNums) if !slices.Equal(testCase.Nums, decodedNums) { t.Fatalf("%d: got nums %v not equal expected %v", caseIdx, decodedNums, testCase.Nums) } } } func TestVarUint64(t *testing.T) { arr := make([]byte, 9) n, err := bin.PutVarInt64(arr, 15) if err != nil { t.Fatal(err) } fmt.Println(arr[:n]) }