wp

.gitignore

@@ -0,0 +1 @@
+/database_linux

client/client.go

@@ -209,6 +209,9 @@ func (s *Connection) AppendMeasures(req proto.AppendMeasuresReq) (result AppendM
 		if err != nil {
 			return
 		}
+	case proto.RespError:
+		err = s.onMaybeError()
+		return
 	default:
 		err = fmt.Errorf("unknown reponse code %d", code)
 		return

database/database.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 	"time"
 
+	"gordenko.dev/dima/qb"
 	"gordenko.dev/dima/qb/inbox"
 	"gordenko.dev/dima/qb/recovery"
 	"gordenko.dev/dima/qb/storage"
@@ -85,6 +86,25 @@ func New(opt Options) (_ *Database, err error) {
 	storageInbox := inbox.New()
 	s.workerInbox = inbox.New()
 
+	// create separately, because needed for page cache
+	dataFile, err := os.OpenFile(
+		qb.GetDataFilePath(opt.Dir, opt.DatabaseName),
+		os.O_CREATE|os.O_RDWR,
+		0666,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	indexFile, err := os.OpenFile(
+		qb.GetIndexFilePath(opt.Dir, opt.DatabaseName),
+		os.O_CREATE|os.O_RDWR,
+		0666,
+	)
+	if err != nil {
+		return nil, err
+	}
+
 	s.storage, err = storage.NewWriter(storage.WriterOptions{
 		Inbox:         storageInbox,
 		WorkerInbox:   s.workerInbox,
@@ -93,6 +113,8 @@ func New(opt Options) (_ *Database, err error) {
 		WAL:           recoveryReport.WAL,
 		IndexFreeList: recoveryReport.IndexFreeList,
 		DataFreeList:  recoveryReport.DataFreeList,
+		DataFile:      dataFile,
+		IndexFile:     indexFile,
 		ExitCh:        s.exitCh,
 		WaitGroup:     s.waitGroup,
 	})

enc/time_delta.go

@@ -162,7 +162,7 @@ func (s *TimeDeltaCompressor) Evaluate(tmp []byte, timestamp uint32) qb.TimeEval
 		RewindOffset: rewindOffset,
 		Offset:       len(s.buf) - s.pos - rewindOffset,
 		ChangeSize:   i,
-		TotalSpace:   s.pos - rewindOffset + i,
+		TotalSpace:   len(s.buf) - s.pos - rewindOffset + i,
 	}
 }
 
@@ -279,6 +279,7 @@ func (s *TimeDeltaCompressor) WriteCommitedTo(w io.Writer) (err error) {
 }
 
 func (s *TimeDeltaCompressor) ReplaceBuffer(newbuf []byte) {
+	//fmt.Printf("time replace buffer: new size %d\n", len(newbuf))
 	s.buf = newbuf
 	s.pos = len(s.buf)
 	s.lastUnixtime = 0

enc/value_delta.go

@@ -1,7 +1,6 @@
 package enc
 
 import (
-	"fmt"
 	"io"
 	"log"
 	"math"
@@ -136,7 +135,7 @@ func (s *ValueDeltaCompressor) Evaluate(tmp []byte, value float64) qb.ValueEvalu
 		RewindOffset: rewindOffset,
 		Offset:       s.pos - rewindOffset,
 		ChangeSize:   i,
-		TotalSpace:   s.pos + i - rewindOffset,
+		TotalSpace:   s.pos - rewindOffset + i,
 		Delta:        delta,
 	}
 }
@@ -147,7 +146,9 @@ func (s *ValueDeltaCompressor) Append(rewindOffset int, change []byte, value flo
 	// fmt.Printf("rewind offset: %d\n", rewindOffset)
 	// fmt.Printf("change: % x\n", change)
 	// fmt.Printf("value: %v\n", value)
-	// fmt.Printf("delta: %d\n", delta)
+	// fmt.Printf("bufsize: %d\n", len(s.buf))
+	// fmt.Printf("s.pos: %d\n", s.pos)
+	//fmt.Printf("delta: %d\n", delta)
 	if s.pos > 0 {
 		s.lastDelta = delta
 	} else {
@@ -242,6 +243,7 @@ func (s *ValueDeltaCompressor) WriteCommitedTo(w io.Writer) (err error) {
 }
 
 func (s *ValueDeltaCompressor) ReplaceBuffer(newbuf []byte) {
+	//fmt.Printf("value replace buffer: new size %d\n", len(newbuf))
 	s.buf = newbuf
 	s.pos = 0
 	s.baseValue = 0
@@ -318,7 +320,7 @@ func (s *ValueDeltaDecompressor) RestoreFromEnd(buf []byte) {
 }
 
 func (s *ValueDeltaDecompressor) RestoreFromState(state qb.ValueDeltaCapturedState) {
-	fmt.Printf("%#v\n", state)
+	//fmt.Printf("%#v\n", state)
 	s.buf = state.Payload
 	s.pos = len(s.buf) // first free
 	if len(s.buf) > 0 {

examples/play/main.go

@@ -324,7 +324,7 @@ func sendRequests(conn *client.Connection) {
 
 	result, err := conn.AppendMeasures(proto.AppendMeasuresReq{
 		MetricID: cumulativeMetricID,
-		Measures: cumulativeMeasures[:1],
+		Measures: cumulativeMeasures[:1000],
 	})
 	if err != nil {
 		log.Fatalf("conn.AppendMeasures: %s\n", err)

go.mod

@@ -5,5 +5,5 @@ go 1.24.2
 require (
 	gopkg.in/ini.v1 v1.67.1
 	gordenko.dev/dima/bin v0.0.0-20260612161453-4ee9be3474fb
-	gordenko.dev/dima/pretty v0.0.0-20221225212746-0c27d8c0ac69
+	gordenko.dev/dima/textutil v0.0.0-20221225052909-270513dcbbbf
 )

go.sum

@@ -20,5 +20,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gordenko.dev/dima/bin v0.0.0-20260612161453-4ee9be3474fb h1:XBYZbK5Z+yfpKmSR2wuPLyW14ReCRSNdOlL5MWro8ns=
 gordenko.dev/dima/bin v0.0.0-20260612161453-4ee9be3474fb/go.mod h1:up64wpJp9xI+HqACtMDBIfPNUH6BYY/b3inKpUfrqDQ=
-gordenko.dev/dima/pretty v0.0.0-20221225212746-0c27d8c0ac69 h1:nyJ3mzTQ46yUeMZCdLyYcs7B5JCS54c67v84miyhq2E=
-gordenko.dev/dima/pretty v0.0.0-20221225212746-0c27d8c0ac69/go.mod h1:AxgKDktpqBVyIOhIcP+nlCpK+EsJyjN5kPdqyd8euVU=
+gordenko.dev/dima/textutil v0.0.0-20221225052909-270513dcbbbf h1:GCL/m736iGDy3LgOm6iuvkxlGhUEyjfP7zgwZRkTYOg=
+gordenko.dev/dima/textutil v0.0.0-20221225052909-270513dcbbbf/go.mod h1:M3e8S3N1USCMYmL6Q5uDQJaxfzBlWHnTX7rxCJQ6iJs=

storage/wal_reader.go

@@ -53,6 +53,7 @@ func (s *WALReader) NextPacket() (_ []any, _ bool, err error) {
 	} else {
 		current, err = s.readPacket()
 		if err != nil {
+			err = fmt.Errorf("readPacket: %s", err)
 			return
 		}
 		if current == nil {
@@ -61,6 +62,7 @@ func (s *WALReader) NextPacket() (_ []any, _ bool, err error) {
 	}
 	s.next, err = s.readPacket()
 	if err != nil {
+		err = fmt.Errorf("readPacket: %s", err)
 		return
 	}
 	return current, s.next == nil, nil
@@ -85,9 +87,12 @@ func (s *WALReader) readPacket() (_ []any, err error) {
 	}
 	calculatedCRC := util.CalculateCRC32(body)
 	if calculatedCRC != writtenCRC {
+		//fmt.Printf("bodySize: %d\n", bodySize)
+		//fmt.Printf("body: % x\n", body)
 		return nil, fmt.Errorf("written CRC %d != calculated CRC %d",
 			writtenCRC, calculatedCRC)
 	}
+	//fmt.Println("SAME CRC")
 	return s.parseRecords(body)
 }
 

storage/write_preparer.go

@@ -49,6 +49,7 @@ type PreparedData struct {
 // )
 
 func (s *WritePreparer) Prepare(input []any) PreparedData {
+	s.w.Reset()
 	s.w.Write([]byte{
 		0, 0, 0, 0, 0, 0, 0, 0, 0, // size (max 9 byte)
 	})
@@ -249,7 +250,7 @@ func (s *WritePreparer) Reset() {
 }
 
 func getIndexRecords(buf []byte, count int) (list []IndexRecord) {
-	fmt.Printf("ipage: % x\n", buf)
+	//fmt.Printf("ipage: % x\n", buf)
 	i := 0
 	for range count {
 		var rec IndexRecord
@@ -268,5 +269,11 @@ func finalizePacket(w *bytes.Buffer, checksum uint32) []byte {
 	// cut empty bytes at start
 	start := 9 - bin.CountVarSize(bodySize)
 	bin.PutVarSize(packet[start:], bodySize)
+
+	fmt.Printf("checksum: %d\n", checksum)
+	fmt.Printf("bodySize: %d\n", bodySize)
+	//fmt.Printf("start: %d\n", start)
+	fmt.Printf("body: % x\n", packet[9:len(packet)-4])
+
 	return packet[start:]
 }

storage/writer.go

@@ -89,6 +89,8 @@ type WriterOptions struct {
 	WAL           string
 	DataFreeList  *freelist.FreeList
 	IndexFreeList *freelist.FreeList
+	DataFile      *os.File
+	IndexFile     *os.File
 	//Atree               *atree.Atree
 	ExitCh    chan struct{}
 	WaitGroup *sync.WaitGroup
@@ -122,9 +124,12 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
 	if opt.IndexFreeList == nil {
 		return nil, errors.New("IndexFreeList option is required")
 	}
-	// if opt.Atree == nil {
-	// 	return nil, errors.New("Atree option is required")
-	// }
+	if opt.DataFile == nil {
+		return nil, errors.New("DataFile option is required")
+	}
+	if opt.IndexFile == nil {
+		return nil, errors.New("IndexFile option is required")
+	}
 	if opt.ExitCh == nil {
 		return nil, errors.New("ExitCh option is required")
 	}
@@ -162,6 +167,8 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
 		workerInbox:   opt.WorkerInbox,
 		dir:           opt.Dir,
 		databaseName:  opt.DatabaseName,
+		dataFile:      opt.DataFile,
+		indexFile:     opt.IndexFile,
 		dataFreeList:  opt.DataFreeList,
 		indexFreeList: opt.IndexFreeList,
 		writePreparer: writePreparer,
@@ -184,6 +191,7 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
 			return nil, err
 		}
 	}
+
 	return s, nil
 }
 
@@ -220,28 +228,33 @@ func (s *Writer) packAndWrite() (err error) {
 	prepared := s.writePreparer.Prepare(input)
 
 	//fmt.Printf("prepared: %#v\n", prepared)
+	fmt.Println("prepared")
 
 	// 3. Пишу на диск WAL (append)
 	n, err := s.wal.Write(prepared.Packet)
 	if err != nil {
 		return
 	}
+	//fmt.Printf("written to wal: % x\n", prepared.Packet)
 	if n != len(prepared.Packet) {
 		return fmt.Errorf("written %d != packet size %d", n, len(prepared.Packet))
 	}
 	if err = s.wal.Sync(); err != nil {
 		return
 	}
+	fmt.Println("synced to wal")
 
 	// 4. Пишу в atree сторінки
 	err = WriteDataPages(s.dataFile, prepared.WriteToData)
 	if err != nil {
 		return
 	}
+	fmt.Println("written to data")
 	err = WriteIndexPages(s.indexFile, prepared.WriteToIndex)
 	if err != nil {
 		return
 	}
+	fmt.Println("written to index")
 
 	// 6. відправляю input - worker-у
 
@@ -300,6 +313,7 @@ func (s *Writer) packAndWrite() (err error) {
 		s.workerInbox.Push(Changes{
 			Commits: prepared.Commits,
 		})
+		fmt.Println("pushed commits")
 	}
 	return nil
 }
@@ -345,6 +359,9 @@ type PageToWrite struct {
 
 func WriteDataPages(file *os.File, pages []PageToWrite) (err error) {
 	for _, p := range pages {
+		fmt.Println("pageNo: %d\n", p.PageNo)
+	}
+	for _, p := range pages {
 		if len(p.Content) != DataPageSize {
 			return fmt.Errorf("wrong data page size: %d", len(p.Content))
 		}
@@ -354,7 +371,7 @@ func WriteDataPages(file *os.File, pages []PageToWrite) (err error) {
 		)
 		n, err = file.WriteAt(p.Content, int64(off))
 		if err != nil {
-			return
+			return fmt.Errorf("file.WriteAt: %s; off=%d", err, off)
 		}
 		if n != DataPageSize {
 			return fmt.Errorf("write %d instead of %d", n, DataPageSize)

transform/raw.go

@@ -1,7 +1,6 @@
 package transform
 
 import (
-	"fmt"
 	"io"
 
 	bin "gordenko.dev/dima/bin/little"
@@ -102,7 +101,7 @@ type CumulativeMeasureWriter struct {
 }
 
 func NewCumulativeMeasureWriter(dst io.Writer, since uint32) *CumulativeMeasureWriter {
-	fmt.Println("NewCumulativeMeasureWriter")
+	//fmt.Println("NewCumulativeMeasureWriter")
 	// 20 - это timestamp, value, total
 	return &CumulativeMeasureWriter{
 		arr:       make([]byte, 20),
@@ -128,13 +127,13 @@ func (s *CumulativeMeasureWriter) feed(timestamp uint32, value float64, isBuffer
 			s.responder.AppendRecord(s.arr)
 		}
 	}
-	fmt.Println("feed:", timestamp, value, isBuffer)
+	//fmt.Println("feed:", timestamp, value, isBuffer)
 	s.endTimestamp = timestamp
 	s.endValue = value
 }
 
 func (s *CumulativeMeasureWriter) pack(total float64) {
-	fmt.Println("PACK")
+	//fmt.Println("PACK")
 	bin.PutUint32(s.arr[0:], s.endTimestamp)
 	bin.PutFloat64(s.arr[4:], s.endValue)
 	bin.PutFloat64(s.arr[12:], total)

transform/responder.go

@@ -38,13 +38,13 @@ func NewChunkedResponder(dst io.Writer) *ChunkedResponder {
 func (s *ChunkedResponder) BufferRecord(rec []byte) {
 	s.buf.Write(rec)
 	s.recordsQty++
-	fmt.Println("BufferRecord", s.recordsQty)
+	//fmt.Println("BufferRecord", s.recordsQty)
 }
 
 func (s *ChunkedResponder) AppendRecord(rec []byte) error {
 	s.buf.Write(rec)
 	s.recordsQty++
-	fmt.Println("AppendRecord", s.recordsQty)
+	//fmt.Println("AppendRecord", s.recordsQty)
 
 	if s.buf.Len() < 1500 {
 		return nil
@@ -63,7 +63,7 @@ func (s *ChunkedResponder) AppendRecord(rec []byte) error {
 }
 
 func (s *ChunkedResponder) Flush() error {
-	fmt.Println("Flush", s.recordsQty)
+	//fmt.Println("Flush", s.recordsQty)
 	if s.recordsQty > 0 {
 		if err := s.sendBuffered(); err != nil {
 			return err
@@ -78,14 +78,14 @@ func (s *ChunkedResponder) Flush() error {
 }
 
 func (s *ChunkedResponder) sendBuffered() (err error) {
-	fmt.Println("sendBuffered", s.recordsQty)
+	//fmt.Println("sendBuffered", s.recordsQty)
 	msg := s.buf.Bytes()
 	bin.PutUint32(msg[1:], uint32(s.recordsQty))
-	fmt.Printf("put uint16: %d\n", msg[:3])
+	//fmt.Printf("put uint16: %d\n", msg[:3])
 
-	fmt.Printf("send %d records\n", s.recordsQty)
+	//fmt.Printf("send %d records\n", s.recordsQty)
 
-	fmt.Printf("send buffered: %d, qty: %d\n", msg, s.recordsQty)
+	//fmt.Printf("send buffered: %d, qty: %d\n", msg, s.recordsQty)
 
 	n, err := s.dst.Write(msg)
 	if err != nil {

util/io_test.go

@@ -1,8 +1,14 @@
 package util
 
 import (
+	"bytes"
+	"fmt"
 	"hash/crc32"
+	"io"
+	"os"
 	"testing"
+
+	"gordenko.dev/dima/textutil"
 )
 
 var page = make([]byte, 4096)
@@ -19,3 +25,38 @@ func BenchmarkCRC32IEEE(b *testing.B) {
 		crc32.Checksum(page, ieeeTable)
 	}
 }
+
+func TestHash(t *testing.T) {
+	raw, err := os.ReadFile("../b")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	buf, err := textutil.ParseHex(string(raw))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Println(CalculateCRC32(buf))
+
+	w := bytes.NewBuffer(nil)
+	w.Write([]byte{
+		0, 0, 0, 0, 0, 0, 0, 0, 0, // size (max 9 byte)
+	})
+
+	hasher := NewHasher()
+
+	multi := io.MultiWriter(w, hasher)
+
+	i := 0
+	for i < len(buf) {
+		end := i + 50
+		if end >= len(buf) {
+			end = len(buf)
+		}
+		multi.Write(buf[i:end])
+		i += 50
+	}
+
+	fmt.Println(hasher.Sum32())
+}

worker/metric.go

@@ -77,6 +77,7 @@ func (s *Metric) OnMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) {
 }
 
 func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox *inbox.Inbox) {
+
 	if s.capturedState != nil {
 		s.WaitQueue = append(s.WaitQueue, req)
 	}
@@ -118,9 +119,9 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
 		}
 
 		tReport := timestamps.Evaluate(tmp[:7], measure.Timestamp)
-		fmt.Printf("tReport: %#v\n", tReport)
+		//fmt.Printf("tReport: %#v\n", tReport)
 		vReport := values.Evaluate(tmp[7:], measure.Value)
-		fmt.Printf("vReport: %#v\n", vReport)
+		//fmt.Printf("vReport: %#v\n", vReport)
 
 		totalRequiredSpace := tReport.TotalSpace + vReport.TotalSpace
 
@@ -136,6 +137,7 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
 				valuesRewindOffset = vReport.RewindOffset
 			}
 		} else {
+			fmt.Println("PAGE FILLED")
 			// сторінка заповнена
 			since := s.timestamps.ReplaceSinceWithUntil()
 
@@ -161,6 +163,11 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
 			// renew
 			s.buffer = buf
 
+			tReport = timestamps.Evaluate(tmp[:7], measure.Timestamp)
+			//fmt.Printf("tReport: %#v\n", tReport)
+			vReport = values.Evaluate(tmp[7:], measure.Value)
+			//fmt.Printf("vReport: %#v\n", vReport)
+
 			timestamps.Append(tReport.RewindOffset, tmp[:tReport.ChangeSize], measure.Timestamp)
 			values.Append(vReport.RewindOffset, tmp[7:7+vReport.ChangeSize], measure.Value, vReport.Delta)
 		}
@@ -181,6 +188,7 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
 	// скопіювати. Причому можна скопіювати зрізи chunks
 
 	if len(pages) > 0 {
+		fmt.Println("push pages")
 		// пишу в storage довгим шляхом через redo файл і запис в data файл
 		storageInbox.Push(storage.MeasuresAppendWithGrow{
 			MetricID:               req.MetricID,
@@ -198,6 +206,7 @@ func (s *Metric) AppendMeasures(req AppendMeasuresReq, tmp []byte, storageInbox
 			ResultCh:               req.ResultCh,
 		})
 	} else {
+		fmt.Println("push simple")
 		// короткий шлях - запис лише в storage
 		storageInbox.Push(storage.MeasuresAppend{
 			MetricID:               req.MetricID,
@@ -329,27 +338,27 @@ func (s *Metric) StartFullScan(req FullScanReq) {
 		if done {
 			break
 		}
-		fmt.Println("ts:", timestamp)
+		//fmt.Println("ts:", timestamp)
 		value, done := valueDecompressor.NextValue()
 		if done {
 			qb.Abort(qb.HasTimestampNoValueBug, ErrNoValueBug)
 		}
-		fmt.Println("value:", value)
+		//fmt.Println("value:", value)
 		req.ResponseWriter.FeedNoSend(timestamp, value)
 	}
 
-	if s.lastPageNo > 0 {
-		req.ResultCh <- FullScanResult{
-			ResultCode: UntilFound,
-			LastPageNo: s.lastPageNo,
-			FracDigits: s.fracDigits,
-		}
-		s.rLocks++
-	} else {
-		req.ResultCh <- FullScanResult{
-			ResultCode: QueryDone,
-		}
+	// if s.lastPageNo > 0 {
+	// 	req.ResultCh <- FullScanResult{
+	// 		ResultCode: UntilFound,
+	// 		LastPageNo: s.lastPageNo,
+	// 		FracDigits: s.fracDigits,
+	// 	}
+	// 	s.rLocks++
+	// } else {
+	req.ResultCh <- FullScanResult{
+		ResultCode: QueryDone,
 	}
+	//}
 }
 
 // індекси