wp
This commit is contained in:
293
database/api.go
293
database/api.go
@@ -11,12 +11,11 @@ import (
|
|||||||
"gordenko.dev/dima/qb/atree"
|
"gordenko.dev/dima/qb/atree"
|
||||||
"gordenko.dev/dima/qb/bufreader"
|
"gordenko.dev/dima/qb/bufreader"
|
||||||
"gordenko.dev/dima/qb/proto"
|
"gordenko.dev/dima/qb/proto"
|
||||||
"gordenko.dev/dima/qb/storage"
|
|
||||||
"gordenko.dev/dima/qb/transform"
|
"gordenko.dev/dima/qb/transform"
|
||||||
|
"gordenko.dev/dima/qb/worker"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
ErrNoValueBug = errors.New("has timestamp but no value")
|
|
||||||
ErrWrongResultCodeBug = errors.New("bug: wrong result code")
|
ErrWrongResultCodeBug = errors.New("bug: wrong result code")
|
||||||
|
|
||||||
successMsg = []byte{
|
successMsg = []byte{
|
||||||
@@ -215,7 +214,7 @@ func (s *Database) AddMetric(req proto.AddMetricReq) uint16 {
|
|||||||
|
|
||||||
fmt.Println("add job")
|
fmt.Println("add job")
|
||||||
|
|
||||||
s.worker.AddJobToQueue(tryAddMetricReq{
|
s.workerInbox.Push(worker.AddMetricReq{
|
||||||
MetricID: req.MetricID,
|
MetricID: req.MetricID,
|
||||||
ResultCh: resultCh,
|
ResultCh: resultCh,
|
||||||
})
|
})
|
||||||
@@ -225,7 +224,7 @@ func (s *Database) AddMetric(req proto.AddMetricReq) uint16 {
|
|||||||
resultCode := <-resultCh
|
resultCode := <-resultCh
|
||||||
|
|
||||||
switch resultCode {
|
switch resultCode {
|
||||||
case Succeed:
|
case worker.Succeed:
|
||||||
fmt.Println("OK")
|
fmt.Println("OK")
|
||||||
// s.storage.Append(storage.MetricAddRecord{
|
// s.storage.Append(storage.MetricAddRecord{
|
||||||
// MetricID: req.MetricID,
|
// MetricID: req.MetricID,
|
||||||
@@ -234,7 +233,7 @@ func (s *Database) AddMetric(req proto.AddMetricReq) uint16 {
|
|||||||
// })
|
// })
|
||||||
//<-waitCh
|
//<-waitCh
|
||||||
|
|
||||||
case MetricDuplicate:
|
case worker.MetricDuplicate:
|
||||||
fmt.Println("ErrDuplicate")
|
fmt.Println("ErrDuplicate")
|
||||||
return proto.ErrDuplicate
|
return proto.ErrDuplicate
|
||||||
|
|
||||||
@@ -245,16 +244,10 @@ func (s *Database) AddMetric(req proto.AddMetricReq) uint16 {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
type Metric struct {
|
|
||||||
MetricType qb.MetricType
|
|
||||||
FracDigits byte
|
|
||||||
ResultCode byte
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Database) GetMetric(conn io.Writer, req proto.GetMetricReq) error {
|
func (s *Database) GetMetric(conn io.Writer, req proto.GetMetricReq) error {
|
||||||
resultCh := make(chan Metric, 1)
|
resultCh := make(chan worker.GetMetricResult, 1)
|
||||||
|
|
||||||
s.worker.AddJobToQueue(tryGetMetricReq{
|
s.workerInbox.Push(worker.GetMetricReq{
|
||||||
MetricID: req.MetricID,
|
MetricID: req.MetricID,
|
||||||
ResultCh: resultCh,
|
ResultCh: resultCh,
|
||||||
})
|
})
|
||||||
@@ -262,7 +255,7 @@ func (s *Database) GetMetric(conn io.Writer, req proto.GetMetricReq) error {
|
|||||||
result := <-resultCh
|
result := <-resultCh
|
||||||
|
|
||||||
switch result.ResultCode {
|
switch result.ResultCode {
|
||||||
case Succeed:
|
case worker.Succeed:
|
||||||
answer := []byte{
|
answer := []byte{
|
||||||
proto.RespValue,
|
proto.RespValue,
|
||||||
0, 0, 0, 0, // metricID
|
0, 0, 0, 0, // metricID
|
||||||
@@ -276,7 +269,7 @@ func (s *Database) GetMetric(conn io.Writer, req proto.GetMetricReq) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
case NoMetric:
|
case worker.NoMetric:
|
||||||
reply(conn, proto.ErrNoMetric)
|
reply(conn, proto.ErrNoMetric)
|
||||||
|
|
||||||
default:
|
default:
|
||||||
@@ -285,15 +278,10 @@ func (s *Database) GetMetric(conn io.Writer, req proto.GetMetricReq) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type tryDeleteMetricResult struct {
|
|
||||||
ResultCode byte
|
|
||||||
RootPageNo uint32
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Database) DeleteMetric(req proto.DeleteMetricReq) uint16 {
|
func (s *Database) DeleteMetric(req proto.DeleteMetricReq) uint16 {
|
||||||
resultCh := make(chan tryDeleteMetricResult, 1)
|
resultCh := make(chan worker.DeleteMetricResult, 1)
|
||||||
|
|
||||||
s.worker.AddJobToQueue(tryDeleteMetricReq{
|
s.workerInbox.Push(worker.DeleteMetricReq{
|
||||||
MetricID: req.MetricID,
|
MetricID: req.MetricID,
|
||||||
ResultCh: resultCh,
|
ResultCh: resultCh,
|
||||||
})
|
})
|
||||||
@@ -301,7 +289,7 @@ func (s *Database) DeleteMetric(req proto.DeleteMetricReq) uint16 {
|
|||||||
result := <-resultCh
|
result := <-resultCh
|
||||||
|
|
||||||
switch result.ResultCode {
|
switch result.ResultCode {
|
||||||
case Succeed:
|
case worker.Succeed:
|
||||||
// var (
|
// var (
|
||||||
// //freePageNumbers []uint32
|
// //freePageNumbers []uint32
|
||||||
// )
|
// )
|
||||||
@@ -318,7 +306,7 @@ func (s *Database) DeleteMetric(req proto.DeleteMetricReq) uint16 {
|
|||||||
// })
|
// })
|
||||||
//<-waitCh
|
//<-waitCh
|
||||||
|
|
||||||
case NoMetric:
|
case worker.NoMetric:
|
||||||
return proto.ErrNoMetric
|
return proto.ErrNoMetric
|
||||||
|
|
||||||
default:
|
default:
|
||||||
@@ -337,127 +325,10 @@ type FilledPage struct {
|
|||||||
ValuesSize uint16
|
ValuesSize uint16
|
||||||
}
|
}
|
||||||
|
|
||||||
// type tryAppendMeasureResult struct {
|
|
||||||
// MetricID uint32
|
|
||||||
// Timestamp uint32
|
|
||||||
// Value float64
|
|
||||||
// FilledPage *FilledPage
|
|
||||||
// ResultCode byte
|
|
||||||
// }
|
|
||||||
|
|
||||||
// func (s *Database) AppendMeasure(req proto.AppendMeasureReq) uint16 {
|
|
||||||
// resultCh := make(chan tryAppendMeasureResult, 1)
|
|
||||||
|
|
||||||
// s.appendJobToWorkerQueue(tryAppendMeasureReq{
|
|
||||||
// MetricID: req.MetricID,
|
|
||||||
// Timestamp: req.Timestamp,
|
|
||||||
// Value: req.Value,
|
|
||||||
// ResultCh: resultCh,
|
|
||||||
// })
|
|
||||||
|
|
||||||
// result := <-resultCh
|
|
||||||
|
|
||||||
// switch result.ResultCode {
|
|
||||||
// case CanAppend:
|
|
||||||
// waitCh := s.storage.Append(storage.AppendedMeasure{
|
|
||||||
// MetricID: req.MetricID,
|
|
||||||
// Timestamp: req.Timestamp,
|
|
||||||
// Value: req.Value,
|
|
||||||
// })
|
|
||||||
// <-waitCh
|
|
||||||
|
|
||||||
// case NewPage:
|
|
||||||
// filled := result.FilledPage
|
|
||||||
|
|
||||||
// var path atree.PathToDataPage
|
|
||||||
// if filled.RootPageNo > 0 {
|
|
||||||
// path, err := s.atree.FindPathToLastPage(filled.RootPageNo)
|
|
||||||
// if err != nil {
|
|
||||||
// // FIX
|
|
||||||
// qb.Abort(qb.WriteToAtreeFailed, err)
|
|
||||||
// }
|
|
||||||
// // for _, leg := range path.Legs {
|
|
||||||
// // pagesToRelease = append(pagesToRelease, leg.PageNo)
|
|
||||||
// // }
|
|
||||||
|
|
||||||
// if path.LastPageNo != filled.PrevPageNo {
|
|
||||||
// qb.Abort(
|
|
||||||
// qb.WrongPrevPageNo,
|
|
||||||
// fmt.Errorf("bug: last pageNo %d in tree != prev pageNo %d in _metric",
|
|
||||||
// path.LastPageNo, filled.PrevPageNo),
|
|
||||||
// )
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// // report, err := s.atree.AppendDataPages(atree.AppendDataPageReq{
|
|
||||||
// // MetricID: req.MetricID,
|
|
||||||
// // Timestamp: req.Timestamp,
|
|
||||||
// // Value: req.Value,
|
|
||||||
// // Since: filled.Since,
|
|
||||||
// // RootPageNo: filled.RootPageNo,
|
|
||||||
// // PrevPageNo: filled.PrevPageNo,
|
|
||||||
// // TimestampsChunks: filled.TimestampsChunks,
|
|
||||||
// // TimestampsSize: filled.TimestampsSize,
|
|
||||||
// // ValuesChunks: filled.ValuesChunks,
|
|
||||||
// // ValuesSize: filled.ValuesSize,
|
|
||||||
// // })
|
|
||||||
// // if err != nil {
|
|
||||||
// // qb.Abort(qb.WriteToAtreeFailed, err)
|
|
||||||
// // }
|
|
||||||
// // _ = report
|
|
||||||
|
|
||||||
// //waitCh := s.storage.WriteAppended(path, report)
|
|
||||||
|
|
||||||
// waitCh := s.storage.Append(storage.AppendedPagesReq{
|
|
||||||
// Legs: path.Legs,
|
|
||||||
// MetricID: req.MetricID,
|
|
||||||
// Timestamp: req.Timestamp,
|
|
||||||
// Value: req.Value,
|
|
||||||
// LastPageNo: filled.PrevPageNo,
|
|
||||||
// //Pages: ,
|
|
||||||
// TimestampsChunks: filled.TimestampsChunks,
|
|
||||||
// TimestampsSize: filled.TimestampsSize,
|
|
||||||
// ValuesChunks: filled.ValuesChunks,
|
|
||||||
// ValuesSize: filled.ValuesSize,
|
|
||||||
// },
|
|
||||||
// )
|
|
||||||
// <-waitCh
|
|
||||||
|
|
||||||
// case NoMetric:
|
|
||||||
// return proto.ErrNoMetric
|
|
||||||
|
|
||||||
// case ExpiredMeasure:
|
|
||||||
// return proto.ErrExpiredMeasure
|
|
||||||
|
|
||||||
// case NonMonotonicValue:
|
|
||||||
// return proto.ErrNonMonotonicValue
|
|
||||||
|
|
||||||
// default:
|
|
||||||
// qb.Abort(qb.WrongResultCodeBug, ErrWrongResultCodeBug)
|
|
||||||
// }
|
|
||||||
// return 0
|
|
||||||
// }
|
|
||||||
|
|
||||||
type tryAppendMeasuresResult struct {
|
|
||||||
ResultCode byte
|
|
||||||
Written int
|
|
||||||
// MetricType qb.MetricType
|
|
||||||
// FracDigits byte
|
|
||||||
// Since uint32
|
|
||||||
// Until uint32
|
|
||||||
// UntilValue float64
|
|
||||||
// RootPageNo uint32
|
|
||||||
// PrevPageNo uint32
|
|
||||||
// TimestampsBuf *conbuf.ContinuousBuffer
|
|
||||||
// ValuesBuf *conbuf.ContinuousBuffer
|
|
||||||
// Timestamps qb.TimestampCompressor
|
|
||||||
// Values qb.ValueCompressor
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
|
func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
|
||||||
resultCh := make(chan tryAppendMeasuresResult, 1)
|
resultCh := make(chan worker.AppendMeasuresResult, 1)
|
||||||
|
|
||||||
s.worker.AddJobToQueue(tryAppendMeasuresReq{
|
s.workerInbox.Push(worker.AppendMeasuresReq{
|
||||||
MetricID: req.MetricID,
|
MetricID: req.MetricID,
|
||||||
Measures: req.Measures,
|
Measures: req.Measures,
|
||||||
ResultCh: resultCh,
|
ResultCh: resultCh,
|
||||||
@@ -466,7 +337,7 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
|
|||||||
result := <-resultCh
|
result := <-resultCh
|
||||||
|
|
||||||
switch result.ResultCode {
|
switch result.ResultCode {
|
||||||
case CanAppend:
|
case worker.CanAppend:
|
||||||
|
|
||||||
// report, err := s.atree.AppendDataPage(atree.AppendDataPageReq{})
|
// report, err := s.atree.AppendDataPage(atree.AppendDataPageReq{})
|
||||||
// if err != nil {
|
// if err != nil {
|
||||||
@@ -485,7 +356,7 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
|
|||||||
// <-waitCh
|
// <-waitCh
|
||||||
// }
|
// }
|
||||||
|
|
||||||
case NoMetric:
|
case worker.NoMetric:
|
||||||
return proto.ErrNoMetric
|
return proto.ErrNoMetric
|
||||||
|
|
||||||
default:
|
default:
|
||||||
@@ -494,15 +365,10 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
type tryDeleteMeasuresResult struct {
|
|
||||||
ResultCode byte
|
|
||||||
RootPageNo uint32
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Database) DeleteMeasures(req proto.DeleteMeasuresReq) uint16 {
|
func (s *Database) DeleteMeasures(req proto.DeleteMeasuresReq) uint16 {
|
||||||
resultCh := make(chan tryDeleteMeasuresResult, 1)
|
resultCh := make(chan worker.DeleteMeasuresResult, 1)
|
||||||
|
|
||||||
s.worker.AddJobToQueue(tryDeleteMeasuresReq{
|
s.workerInbox.Push(worker.DeleteMeasuresReq{
|
||||||
MetricID: req.MetricID,
|
MetricID: req.MetricID,
|
||||||
Since: req.Since,
|
Since: req.Since,
|
||||||
ResultCh: resultCh,
|
ResultCh: resultCh,
|
||||||
@@ -511,17 +377,17 @@ func (s *Database) DeleteMeasures(req proto.DeleteMeasuresReq) uint16 {
|
|||||||
result := <-resultCh
|
result := <-resultCh
|
||||||
|
|
||||||
switch result.ResultCode {
|
switch result.ResultCode {
|
||||||
case NoMeasuresToDelete:
|
case worker.NoMeasuresToDelete:
|
||||||
// ok
|
// ok
|
||||||
|
|
||||||
case DeleteFromAtreeNotNeeded:
|
case worker.DeleteFromAtreeNotNeeded:
|
||||||
// регистрирую удаление в TransactionLog
|
// регистрирую удаление в TransactionLog
|
||||||
s.storage.Append(storage.MeasuresDeleteRecord{
|
// s.storage.Append(storage.MeasuresDeleteRecord{
|
||||||
MetricID: req.MetricID,
|
// MetricID: req.MetricID,
|
||||||
})
|
// })
|
||||||
//<-waitCh
|
//<-waitCh
|
||||||
|
|
||||||
case DeleteFromAtreeRequired:
|
case worker.DeleteFromAtreeRequired:
|
||||||
// собираю номера всех data и index страниц метрики (типа запись REDO лога).
|
// собираю номера всех data и index страниц метрики (типа запись REDO лога).
|
||||||
// pageNumbers, err := s.atree.GetAllPages(req.MetricID)
|
// pageNumbers, err := s.atree.GetAllPages(req.MetricID)
|
||||||
// if err != nil {
|
// if err != nil {
|
||||||
@@ -534,7 +400,7 @@ func (s *Database) DeleteMeasures(req proto.DeleteMeasuresReq) uint16 {
|
|||||||
// })
|
// })
|
||||||
//<-waitCh
|
//<-waitCh
|
||||||
|
|
||||||
case NoMetric:
|
case worker.NoMetric:
|
||||||
return proto.ErrNoMetric
|
return proto.ErrNoMetric
|
||||||
|
|
||||||
default:
|
default:
|
||||||
@@ -545,12 +411,6 @@ func (s *Database) DeleteMeasures(req proto.DeleteMeasuresReq) uint16 {
|
|||||||
|
|
||||||
// SELECT
|
// SELECT
|
||||||
|
|
||||||
type fullScanResult struct {
|
|
||||||
ResultCode byte
|
|
||||||
FracDigits byte
|
|
||||||
LastPageNo uint32
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Database) ListAllInstantMeasures(conn net.Conn, req proto.ListAllInstantMetricMeasuresReq) error {
|
func (s *Database) ListAllInstantMeasures(conn net.Conn, req proto.ListAllInstantMetricMeasuresReq) error {
|
||||||
responseWriter := transform.NewInstantMeasureWriter(conn, 0)
|
responseWriter := transform.NewInstantMeasureWriter(conn, 0)
|
||||||
|
|
||||||
@@ -609,13 +469,6 @@ func (s *Database) ListCumulativeMeasures(conn net.Conn, req proto.ListCumulativ
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
type rangeScanResult struct {
|
|
||||||
ResultCode byte
|
|
||||||
FracDigits byte
|
|
||||||
RootPageNo uint32
|
|
||||||
LastPageNo uint32
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriodsReq) error {
|
func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriodsReq) error {
|
||||||
since, until := timeBoundsOfAggregation(req.Since, req.Until, req.GroupBy, req.FirstHourOfDay)
|
since, until := timeBoundsOfAggregation(req.Since, req.Until, req.GroupBy, req.FirstHourOfDay)
|
||||||
if since.After(until) {
|
if since.After(until) {
|
||||||
@@ -681,9 +534,9 @@ type rangeScanReq struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Database) rangeScan(req rangeScanReq) error {
|
func (s *Database) rangeScan(req rangeScanReq) error {
|
||||||
resultCh := make(chan rangeScanResult, 1)
|
resultCh := make(chan worker.RangeScanResult, 1)
|
||||||
|
|
||||||
s.worker.AddJobToQueue(tryRangeScanReq{
|
s.workerInbox.Push(worker.RangeScanReq{
|
||||||
MetricID: req.MetricID,
|
MetricID: req.MetricID,
|
||||||
Since: req.Since,
|
Since: req.Since,
|
||||||
Until: req.Until,
|
Until: req.Until,
|
||||||
@@ -695,44 +548,44 @@ func (s *Database) rangeScan(req rangeScanReq) error {
|
|||||||
result := <-resultCh
|
result := <-resultCh
|
||||||
|
|
||||||
switch result.ResultCode {
|
switch result.ResultCode {
|
||||||
case QueryDone:
|
case worker.QueryDone:
|
||||||
req.ResponseWriter.Close()
|
req.ResponseWriter.Close()
|
||||||
|
|
||||||
case UntilFound:
|
case worker.UntilFound:
|
||||||
err := s.atree.ContinueRangeScan(atree.ContinueRangeScanReq{
|
// err := s.atree.ContinueRangeScan(atree.ContinueRangeScanReq{
|
||||||
FracDigits: result.FracDigits,
|
// FracDigits: result.FracDigits,
|
||||||
ResponseWriter: req.ResponseWriter,
|
// ResponseWriter: req.ResponseWriter,
|
||||||
LastPageNo: result.LastPageNo,
|
// LastPageNo: result.LastPageNo,
|
||||||
Since: req.Since,
|
// Since: req.Since,
|
||||||
})
|
// })
|
||||||
//s.metricRUnlock(req.MetricID)
|
//s.metricRUnlock(req.MetricID)
|
||||||
|
|
||||||
if err != nil {
|
// if err != nil {
|
||||||
reply(req.Conn, proto.ErrUnexpected)
|
// reply(req.Conn, proto.ErrUnexpected)
|
||||||
} else {
|
// } else {
|
||||||
req.ResponseWriter.Close()
|
// req.ResponseWriter.Close()
|
||||||
}
|
// }
|
||||||
|
|
||||||
case UntilNotFound:
|
case worker.UntilNotFound:
|
||||||
err := s.atree.RangeScan(atree.RangeScanReq{
|
// err := s.atree.RangeScan(atree.RangeScanReq{
|
||||||
FracDigits: result.FracDigits,
|
// FracDigits: result.FracDigits,
|
||||||
ResponseWriter: req.ResponseWriter,
|
// ResponseWriter: req.ResponseWriter,
|
||||||
RootPageNo: result.RootPageNo,
|
// RootPageNo: result.RootPageNo,
|
||||||
Since: req.Since,
|
// Since: req.Since,
|
||||||
Until: req.Until,
|
// Until: req.Until,
|
||||||
})
|
// })
|
||||||
//s.metricRUnlock(req.MetricID)
|
//s.metricRUnlock(req.MetricID)
|
||||||
|
|
||||||
if err != nil {
|
// if err != nil {
|
||||||
reply(req.Conn, proto.ErrUnexpected)
|
// reply(req.Conn, proto.ErrUnexpected)
|
||||||
} else {
|
// } else {
|
||||||
req.ResponseWriter.Close()
|
// req.ResponseWriter.Close()
|
||||||
}
|
// }
|
||||||
|
|
||||||
case NoMetric:
|
case worker.NoMetric:
|
||||||
reply(req.Conn, proto.ErrNoMetric)
|
reply(req.Conn, proto.ErrNoMetric)
|
||||||
|
|
||||||
case WrongMetricType:
|
case worker.WrongMetricType:
|
||||||
reply(req.Conn, proto.ErrWrongMetricType)
|
reply(req.Conn, proto.ErrWrongMetricType)
|
||||||
|
|
||||||
default:
|
default:
|
||||||
@@ -749,9 +602,9 @@ type fullScanReq struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Database) fullScan(req fullScanReq) error {
|
func (s *Database) fullScan(req fullScanReq) error {
|
||||||
resultCh := make(chan fullScanResult, 1)
|
resultCh := make(chan worker.FullScanResult, 1)
|
||||||
|
|
||||||
s.worker.AddJobToQueue(tryFullScanReq{
|
s.workerInbox.Push(worker.FullScanReq{
|
||||||
MetricID: req.MetricID,
|
MetricID: req.MetricID,
|
||||||
MetricType: req.MetricType,
|
MetricType: req.MetricType,
|
||||||
ResponseWriter: req.ResponseWriter,
|
ResponseWriter: req.ResponseWriter,
|
||||||
@@ -761,26 +614,26 @@ func (s *Database) fullScan(req fullScanReq) error {
|
|||||||
result := <-resultCh
|
result := <-resultCh
|
||||||
|
|
||||||
switch result.ResultCode {
|
switch result.ResultCode {
|
||||||
case QueryDone:
|
case worker.QueryDone:
|
||||||
req.ResponseWriter.Close()
|
req.ResponseWriter.Close()
|
||||||
|
|
||||||
case UntilFound:
|
case worker.UntilFound:
|
||||||
err := s.atree.ContinueFullScan(atree.ContinueFullScanReq{
|
// err := s.atree.ContinueFullScan(atree.ContinueFullScanReq{
|
||||||
FracDigits: result.FracDigits,
|
// FracDigits: result.FracDigits,
|
||||||
ResponseWriter: req.ResponseWriter,
|
// ResponseWriter: req.ResponseWriter,
|
||||||
LastPageNo: result.LastPageNo,
|
// LastPageNo: result.LastPageNo,
|
||||||
})
|
// })
|
||||||
s.worker.AddJobToQueue(req.MetricID)
|
// s.worker.AddJobToQueue(req.MetricID)
|
||||||
if err != nil {
|
// if err != nil {
|
||||||
reply(req.Conn, proto.ErrUnexpected)
|
// reply(req.Conn, proto.ErrUnexpected)
|
||||||
} else {
|
// } else {
|
||||||
req.ResponseWriter.Close()
|
// req.ResponseWriter.Close()
|
||||||
}
|
// }
|
||||||
|
|
||||||
case NoMetric:
|
case worker.NoMetric:
|
||||||
reply(req.Conn, proto.ErrNoMetric)
|
reply(req.Conn, proto.ErrNoMetric)
|
||||||
|
|
||||||
case WrongMetricType:
|
case worker.WrongMetricType:
|
||||||
reply(req.Conn, proto.ErrWrongMetricType)
|
reply(req.Conn, proto.ErrWrongMetricType)
|
||||||
|
|
||||||
default:
|
default:
|
||||||
@@ -795,7 +648,7 @@ func (s *Database) ListCurrentValues(conn net.Conn, req proto.ListCurrentValuesR
|
|||||||
|
|
||||||
resultCh := make(chan struct{})
|
resultCh := make(chan struct{})
|
||||||
|
|
||||||
s.worker.AddJobToQueue(tryListCurrentValuesReq{
|
s.workerInbox.Push(worker.ListCurrentValuesReq{
|
||||||
MetricIDs: req.MetricIDs,
|
MetricIDs: req.MetricIDs,
|
||||||
ResponseWriter: responseWriter,
|
ResponseWriter: responseWriter,
|
||||||
ResultCh: resultCh,
|
ResultCh: resultCh,
|
||||||
|
|||||||
@@ -6,39 +6,30 @@ import (
|
|||||||
"log"
|
"log"
|
||||||
"net"
|
"net"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gordenko.dev/dima/qb"
|
"gordenko.dev/dima/qb"
|
||||||
"gordenko.dev/dima/qb/atree"
|
"gordenko.dev/dima/qb/inbox"
|
||||||
|
"gordenko.dev/dima/qb/recovery"
|
||||||
"gordenko.dev/dima/qb/storage"
|
"gordenko.dev/dima/qb/storage"
|
||||||
|
"gordenko.dev/dima/qb/worker"
|
||||||
)
|
)
|
||||||
|
|
||||||
func JoinSnapshotFileName(dir string, snapshotNumber int) string {
|
|
||||||
return filepath.Join(dir, fmt.Sprintf("%d.snapshot", snapshotNumber))
|
|
||||||
}
|
|
||||||
|
|
||||||
func JoinWALFileName(dir string, snapshotNumber int) string {
|
|
||||||
return filepath.Join(dir, fmt.Sprintf("%d.wal", snapshotNumber))
|
|
||||||
}
|
|
||||||
|
|
||||||
// type metricLockEntry struct {
|
// type metricLockEntry struct {
|
||||||
// XLock bool
|
// XLock bool
|
||||||
// RLocks int
|
// RLocks int
|
||||||
// WaitQueue []any
|
// WaitQueue []any
|
||||||
// }
|
// }
|
||||||
|
//metricLockEntries map[uint32]*metricLockEntry
|
||||||
|
|
||||||
type Database struct {
|
type Database struct {
|
||||||
mutex sync.Mutex
|
mutex sync.Mutex
|
||||||
worker *Worker
|
|
||||||
//metricLockEntries map[uint32]*metricLockEntry
|
|
||||||
//dataFreeList *freelist.FreeList
|
|
||||||
//indexFreeList *freelist.FreeList
|
|
||||||
dir string
|
dir string
|
||||||
databaseName string
|
databaseName string
|
||||||
|
workerInbox *inbox.Inbox
|
||||||
|
worker *worker.Worker
|
||||||
storage *storage.Writer
|
storage *storage.Writer
|
||||||
atree *atree.Atree
|
|
||||||
tcpPort int
|
tcpPort int
|
||||||
logfile *os.File
|
logfile *os.File
|
||||||
logger *log.Logger
|
logger *log.Logger
|
||||||
@@ -50,7 +41,6 @@ type Options struct {
|
|||||||
TCPPort int
|
TCPPort int
|
||||||
Dir string
|
Dir string
|
||||||
DatabaseName string
|
DatabaseName string
|
||||||
RedoDir string
|
|
||||||
Logfile *os.File
|
Logfile *os.File
|
||||||
ExitCh chan struct{}
|
ExitCh chan struct{}
|
||||||
WaitGroup *sync.WaitGroup
|
WaitGroup *sync.WaitGroup
|
||||||
@@ -66,9 +56,6 @@ func New(opt Options) (_ *Database, err error) {
|
|||||||
if opt.DatabaseName == "" {
|
if opt.DatabaseName == "" {
|
||||||
return nil, errors.New("DatabaseName option is required")
|
return nil, errors.New("DatabaseName option is required")
|
||||||
}
|
}
|
||||||
if opt.RedoDir == "" {
|
|
||||||
return nil, errors.New("RedoDir option is required")
|
|
||||||
}
|
|
||||||
if opt.Logfile == nil {
|
if opt.Logfile == nil {
|
||||||
return nil, errors.New("Logfile option is required")
|
return nil, errors.New("Logfile option is required")
|
||||||
}
|
}
|
||||||
@@ -79,40 +66,7 @@ func New(opt Options) (_ *Database, err error) {
|
|||||||
return nil, errors.New("WaitGroup option is required")
|
return nil, errors.New("WaitGroup option is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
// deltaFreeListFile, err := os.OpenFile(
|
|
||||||
// filepath.Join(opt.Dir, opt.DatabaseName+".deltafree"),
|
|
||||||
// os.O_RDWR|os.O_CREATE,
|
|
||||||
// 0666,
|
|
||||||
// )
|
|
||||||
// if err != nil {
|
|
||||||
// return nil, err
|
|
||||||
// }
|
|
||||||
|
|
||||||
// dataFreeList, err := freelist.New(freelist.Options{
|
|
||||||
// PageSize: 2048,
|
|
||||||
// BaseFilePath: filepath.Join(opt.Dir, opt.DatabaseName+".free_data"),
|
|
||||||
// DeltaFilePath: filepath.Join(opt.Dir, opt.DatabaseName+".free_data_delta"),
|
|
||||||
// })
|
|
||||||
|
|
||||||
// indexFreeList, err := freelist.New(freelist.Options{
|
|
||||||
// PageSize: 2048,
|
|
||||||
// BaseFilePath: filepath.Join(opt.Dir, opt.DatabaseName+".free_index"),
|
|
||||||
// DeltaFilePath: filepath.Join(opt.Dir, opt.DatabaseName+".free_index_delta"),
|
|
||||||
// })
|
|
||||||
|
|
||||||
worker := NewWorker(WorkerOptions{
|
|
||||||
//
|
|
||||||
Metrics: make(map[uint32]*_metric),
|
|
||||||
Dir: opt.Dir,
|
|
||||||
SaveToStorage: func(in any) {
|
|
||||||
fmt.Println("save to storage")
|
|
||||||
},
|
|
||||||
ExitCh: opt.ExitCh,
|
|
||||||
WaitGroup: opt.WaitGroup,
|
|
||||||
})
|
|
||||||
|
|
||||||
s := &Database{
|
s := &Database{
|
||||||
worker: worker,
|
|
||||||
dir: opt.Dir,
|
dir: opt.Dir,
|
||||||
databaseName: opt.DatabaseName,
|
databaseName: opt.DatabaseName,
|
||||||
//metricLockEntries: make(map[uint32]*metricLockEntry),
|
//metricLockEntries: make(map[uint32]*metricLockEntry),
|
||||||
@@ -124,6 +78,42 @@ func New(opt Options) (_ *Database, err error) {
|
|||||||
exitCh: opt.ExitCh,
|
exitCh: opt.ExitCh,
|
||||||
waitGroup: opt.WaitGroup,
|
waitGroup: opt.WaitGroup,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
recoveryReport, err := recovery.Recovery(s.dir, s.databaseName)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
storageInbox := inbox.New()
|
||||||
|
s.workerInbox = inbox.New()
|
||||||
|
|
||||||
|
s.storage, err = storage.NewWriter(storage.WriterOptions{
|
||||||
|
Inbox: storageInbox,
|
||||||
|
WorkerInbox: s.workerInbox,
|
||||||
|
Dir: s.dir,
|
||||||
|
DatabaseName: s.databaseName,
|
||||||
|
WAL: recoveryReport.WAL,
|
||||||
|
IndexFreeList: recoveryReport.IndexFreeList,
|
||||||
|
DataFreeList: recoveryReport.DataFreeList,
|
||||||
|
ExitCh: s.exitCh,
|
||||||
|
WaitGroup: s.waitGroup,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
qb.Abort(qb.CreateChangesWriterFailed, err)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
s.worker = worker.NewWorker(worker.WorkerOptions{
|
||||||
|
Inbox: s.workerInbox,
|
||||||
|
StorageInbox: storageInbox,
|
||||||
|
//Metrics: make(map[uint32]*_metric), FIX
|
||||||
|
Dir: opt.Dir,
|
||||||
|
// SaveToStorage: func(in any) {
|
||||||
|
// fmt.Println("save to storage")
|
||||||
|
// },
|
||||||
|
ExitCh: opt.ExitCh,
|
||||||
|
WaitGroup: opt.WaitGroup,
|
||||||
|
})
|
||||||
return s, nil
|
return s, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -133,19 +123,6 @@ func (s *Database) ListenAndServe() (err error) {
|
|||||||
return fmt.Errorf("net.Listen: %s; port=%d", err, s.tcpPort)
|
return fmt.Errorf("net.Listen: %s; port=%d", err, s.tcpPort)
|
||||||
}
|
}
|
||||||
|
|
||||||
s.storage, err = storage.NewWriter(storage.WriterOptions{
|
|
||||||
Dir: s.dir,
|
|
||||||
//LogNumber: logNumber,
|
|
||||||
AppendToWorkerQueue: s.worker.AddJobToQueue,
|
|
||||||
//FreeList: s.freeList,
|
|
||||||
//Atree: s.atree,
|
|
||||||
ExitCh: s.exitCh,
|
|
||||||
WaitGroup: s.waitGroup,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
qb.Abort(qb.CreateChangesWriterFailed, err)
|
|
||||||
|
|
||||||
}
|
|
||||||
go s.storage.Run()
|
go s.storage.Run()
|
||||||
|
|
||||||
// s.atree, err = atree.New(atree.Options{
|
// s.atree, err = atree.New(atree.Options{
|
||||||
@@ -177,50 +154,14 @@ func (s *Database) ListenAndServe() (err error) {
|
|||||||
|
|
||||||
// зробити object?
|
// зробити object?
|
||||||
|
|
||||||
func (s *Database) replayChanges(snapshotNumber int) (err error) {
|
// func (s *Database) replayChanges(snapshotNumber int) (err error) {
|
||||||
snapshot, err := readSnapshot(JoinSnapshotFileName(s.dir, snapshotNumber))
|
// snapshot, err := readSnapshot(JoinSnapshotFileName(s.dir, snapshotNumber))
|
||||||
if err != nil {
|
// if err != nil {
|
||||||
return
|
// return
|
||||||
}
|
// }
|
||||||
|
|
||||||
walReader, err := storage.NewWALReader(storage.WALReaderOptions{
|
// return nil
|
||||||
FileName: JoinWALFileName(s.dir, snapshotNumber),
|
// }
|
||||||
BufferSize: 8 * 1024 * 1024,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
walReplayer, err := NewWALReplayer(WALReplayerOptions{
|
|
||||||
WALReader: walReader,
|
|
||||||
Metrics: snapshot.Metrics,
|
|
||||||
FreeIndexPages: snapshot.IndexPageNumbers,
|
|
||||||
FreeDataPages: snapshot.DataPageNumbers,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
err = walReplayer.Replay()
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
err = s.storage.WritePagesToAtree(walReplayer.IndexPages(), walReplayer.DataPages())
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// FIX free pages sync
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Database) relayMetricsToMetrics(replayMetrics map[uint32]*ReplayMetric) {
|
|
||||||
//for metricID, x := range replayMetrics {
|
|
||||||
//s.metrics[metricID] = x.ToMetric() FIX
|
|
||||||
//}
|
|
||||||
}
|
|
||||||
|
|
||||||
//func (s *Database) verifySnapshot(fileName string) (_ bool, err error) {
|
//func (s *Database) verifySnapshot(fileName string) (_ bool, err error) {
|
||||||
// file, err := os.Open(fileName)
|
// file, err := os.Open(fileName)
|
||||||
|
|||||||
@@ -1,113 +1,24 @@
|
|||||||
package database
|
package database
|
||||||
|
|
||||||
import (
|
// func TestComposeHeadIndexPage(t *testing.T) {
|
||||||
"fmt"
|
// var (
|
||||||
"slices"
|
// levelIdx = 0
|
||||||
"testing"
|
// level = storage.IndexLevelTail{
|
||||||
|
// Buffer: []byte{
|
||||||
"gordenko.dev/dima/qb"
|
// 1, 2, 3,
|
||||||
"gordenko.dev/dima/qb/storage"
|
// },
|
||||||
)
|
// RecordsCount: 2,
|
||||||
|
// }
|
||||||
func TestWriteReadSnapshot(t *testing.T) {
|
// head = &storage.IndexPageTail{
|
||||||
metrics := make(map[uint32]*_metric)
|
// PageNo: 100,
|
||||||
metrics[1] = &_metric{
|
// CRC32: 12345,
|
||||||
metricType: qb.Cumulative,
|
// Records: []byte{},
|
||||||
fracDigits: 3,
|
// }
|
||||||
lastPageNo: 1000,
|
// )
|
||||||
buffer: []byte{},
|
// page := composeHeadIndexPage(levelIdx, level, head)
|
||||||
}
|
// if page.PageNo != head.PageNo {
|
||||||
in := writeSnapshotIn{
|
// t.Fatalf("PageNo: got %d are not equal expected %v",
|
||||||
SnapshotNumber: 1,
|
// page.PageNo, head.PageNo)
|
||||||
Dir: ".",
|
// }
|
||||||
WriteBufferSize: 8 * 1024 * 1024,
|
// // fix compare pages
|
||||||
Metrics: metrics,
|
// }
|
||||||
FrozenIndexPagesCount: 7,
|
|
||||||
IndexPageNumbers: []uint32{
|
|
||||||
1, 2, 3, 4, 5,
|
|
||||||
},
|
|
||||||
FrozenDataPagesCount: 8,
|
|
||||||
DataPageNumbers: []uint32{
|
|
||||||
6, 7, 8,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
err := writeSnapshot(in)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("writeSnapshot: %s", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
out, err := readSnapshot("1.snapshot")
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if out.FrozenIndexPagesCount != in.FrozenIndexPagesCount {
|
|
||||||
t.Fatalf("FrozenIndexPagesCount: got %d are not equal expected %d",
|
|
||||||
out.FrozenIndexPagesCount, in.FrozenIndexPagesCount)
|
|
||||||
}
|
|
||||||
if out.FrozenDataPagesCount != in.FrozenDataPagesCount {
|
|
||||||
t.Fatalf("FrozenDataPagesCount: got %d are not equal expected %d",
|
|
||||||
out.FrozenDataPagesCount, in.FrozenDataPagesCount)
|
|
||||||
}
|
|
||||||
if !slices.Equal(out.IndexPageNumbers, in.IndexPageNumbers) {
|
|
||||||
t.Fatalf("IndexPageNumbers: got %v are not equal expected %v",
|
|
||||||
out.IndexPageNumbers, in.IndexPageNumbers)
|
|
||||||
}
|
|
||||||
if !slices.Equal(out.DataPageNumbers, in.DataPageNumbers) {
|
|
||||||
t.Fatalf("DataPageNumbers: got %v are not equal expected %v",
|
|
||||||
out.DataPageNumbers, in.DataPageNumbers)
|
|
||||||
}
|
|
||||||
for metricID, replay := range out.Metrics {
|
|
||||||
origin, ok := metrics[metricID]
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("decoded metricID %d not found in metrics", metricID)
|
|
||||||
}
|
|
||||||
err = cmpMetric(replay, origin)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("metric %d: %s", metricID, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func cmpMetric(replay *ReplayMetric, origin *_metric) error {
|
|
||||||
if replay.metricType != origin.metricType {
|
|
||||||
return fmt.Errorf("metricType: got %d are not equal expected %d",
|
|
||||||
replay.metricType, origin.metricType)
|
|
||||||
}
|
|
||||||
if replay.fracDigits != origin.fracDigits {
|
|
||||||
return fmt.Errorf("fracDigits: got %d are not equal expected %d",
|
|
||||||
replay.fracDigits, origin.fracDigits)
|
|
||||||
}
|
|
||||||
if replay.lastPageNo != origin.lastPageNo {
|
|
||||||
return fmt.Errorf("lastPageNo: got %d are not equal expected %d",
|
|
||||||
replay.lastPageNo, origin.lastPageNo)
|
|
||||||
}
|
|
||||||
// if !refect.DeepEq{
|
|
||||||
// return fmt.Errorf("lastPageNo: got %d are not equal expected %d",
|
|
||||||
// replay.lastPageNo, origin.lastPageNo)
|
|
||||||
// }
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestComposeHeadIndexPage(t *testing.T) {
|
|
||||||
var (
|
|
||||||
levelIdx = 0
|
|
||||||
level = storage.IndexLevelTail{
|
|
||||||
Buffer: []byte{
|
|
||||||
1, 2, 3,
|
|
||||||
},
|
|
||||||
RecordsCount: 2,
|
|
||||||
}
|
|
||||||
head = &storage.IndexPageTail{
|
|
||||||
PageNo: 100,
|
|
||||||
CRC32: 12345,
|
|
||||||
Records: []byte{},
|
|
||||||
}
|
|
||||||
)
|
|
||||||
page := composeHeadIndexPage(levelIdx, level, head)
|
|
||||||
if page.PageNo != head.PageNo {
|
|
||||||
t.Fatalf("PageNo: got %d are not equal expected %v",
|
|
||||||
page.PageNo, head.PageNo)
|
|
||||||
}
|
|
||||||
// fix compare pages
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -42,7 +42,6 @@ func main() {
|
|||||||
TCPPort: config.TcpPort,
|
TCPPort: config.TcpPort,
|
||||||
Dir: config.Dir,
|
Dir: config.Dir,
|
||||||
DatabaseName: config.DatabaseName,
|
DatabaseName: config.DatabaseName,
|
||||||
RedoDir: config.REDODir,
|
|
||||||
Logfile: logfile,
|
Logfile: logfile,
|
||||||
ExitCh: exitCh,
|
ExitCh: exitCh,
|
||||||
WaitGroup: wg,
|
WaitGroup: wg,
|
||||||
|
|||||||
38
inbox/inbox.go
Normal file
38
inbox/inbox.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package inbox
|
||||||
|
|
||||||
|
import "sync"
|
||||||
|
|
||||||
|
type Inbox struct {
|
||||||
|
mutex *sync.Mutex
|
||||||
|
signalCh chan struct{}
|
||||||
|
items []any
|
||||||
|
}
|
||||||
|
|
||||||
|
func New() *Inbox {
|
||||||
|
return &Inbox{
|
||||||
|
mutex: new(sync.Mutex),
|
||||||
|
signalCh: make(chan struct{}, 1),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Inbox) Ready() chan struct{} {
|
||||||
|
return s.signalCh
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Inbox) Push(x any) {
|
||||||
|
s.mutex.Lock()
|
||||||
|
s.items = append(s.items, x)
|
||||||
|
s.mutex.Unlock()
|
||||||
|
select {
|
||||||
|
case s.signalCh <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Inbox) Drain() []any {
|
||||||
|
s.mutex.Lock()
|
||||||
|
items := s.items
|
||||||
|
s.items = nil
|
||||||
|
s.mutex.Unlock()
|
||||||
|
return items
|
||||||
|
}
|
||||||
25
qb.go
25
qb.go
@@ -4,6 +4,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
)
|
)
|
||||||
|
|
||||||
type MetricType byte
|
type MetricType byte
|
||||||
@@ -151,3 +152,27 @@ func Abort(code AbortCode, err error) {
|
|||||||
fmt.Println(err)
|
fmt.Println(err)
|
||||||
os.Exit(int(code))
|
os.Exit(int(code))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func GetIndexFilePath(dir string, databaseName string) string {
|
||||||
|
return filepath.Join(dir, fmt.Sprintf("%s.index", databaseName))
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetDataFilePath(dir string, databaseName string) string {
|
||||||
|
return filepath.Join(dir, fmt.Sprintf("%s.data", databaseName))
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetWALFilePath(dir string, databaseName string, snapshotNumber int) string {
|
||||||
|
return filepath.Join(dir, fmt.Sprintf("%s.wal_%d", databaseName, snapshotNumber))
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetSnapshotFilePath(dir string, databaseName string, snapshotNumber int) string {
|
||||||
|
return filepath.Join(dir, fmt.Sprintf("%s.snapshot_%d", databaseName, snapshotNumber))
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetIndexFreeListFilePath(dir string, databaseName string) string {
|
||||||
|
return filepath.Join(dir, fmt.Sprintf("%s.index_free", databaseName))
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetDataFreeListFilePath(dir string, databaseName string) string {
|
||||||
|
return filepath.Join(dir, fmt.Sprintf("%s.data_free", databaseName))
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,255 +0,0 @@
|
|||||||
package recovery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"regexp"
|
|
||||||
"sort"
|
|
||||||
"strconv"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
reChanges = regexp.MustCompile(`(\d+)\.wal`)
|
|
||||||
reSnapshot = regexp.MustCompile(`(\d+)\.snapshot`)
|
|
||||||
)
|
|
||||||
|
|
||||||
func joinWALFileName(dir string, snapshotNumber int) string {
|
|
||||||
return filepath.Join(dir, fmt.Sprintf("%d.wal", snapshotNumber))
|
|
||||||
}
|
|
||||||
|
|
||||||
type RecoveryRecipe struct {
|
|
||||||
Snapshot string
|
|
||||||
Changes []string
|
|
||||||
SnapshotNumber int
|
|
||||||
ToDelete []string
|
|
||||||
CompleteSnapshot bool // флаг - что нужно завершить создание снапшота
|
|
||||||
}
|
|
||||||
|
|
||||||
type RecoveryAdvisor struct {
|
|
||||||
dir string
|
|
||||||
verifySnapshot func(string) (bool, error) // (fileName) isVerified, error
|
|
||||||
}
|
|
||||||
|
|
||||||
type RecoveryAdvisorOptions struct {
|
|
||||||
Dir string
|
|
||||||
VerifySnapshot func(string) (bool, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewRecoveryAdvisor(opt RecoveryAdvisorOptions) (*RecoveryAdvisor, error) {
|
|
||||||
if opt.Dir == "" {
|
|
||||||
return nil, errors.New("Dir option is required")
|
|
||||||
}
|
|
||||||
if opt.VerifySnapshot == nil {
|
|
||||||
return nil, errors.New("VerifySnapshot option is required")
|
|
||||||
}
|
|
||||||
return &RecoveryAdvisor{
|
|
||||||
dir: opt.Dir,
|
|
||||||
verifySnapshot: opt.VerifySnapshot,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type SnapshotChangesPair struct {
|
|
||||||
SnapshotFileName string
|
|
||||||
WALFileName string
|
|
||||||
SnapshotNumber int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *RecoveryAdvisor) getSnapshotChangesPairs() (*SnapshotChangesPair, *SnapshotChangesPair, error) {
|
|
||||||
var (
|
|
||||||
numSet = make(map[int]bool)
|
|
||||||
changesSet = make(map[int]bool)
|
|
||||||
snapshotsSet = make(map[int]bool)
|
|
||||||
pairs []SnapshotChangesPair
|
|
||||||
)
|
|
||||||
|
|
||||||
entries, err := os.ReadDir(s.dir)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, entry := range entries {
|
|
||||||
if entry.Type().IsRegular() {
|
|
||||||
baseName := entry.Name()
|
|
||||||
groups := reChanges.FindStringSubmatch(baseName)
|
|
||||||
if len(groups) == 2 {
|
|
||||||
num, _ := strconv.Atoi(groups[1])
|
|
||||||
|
|
||||||
numSet[num] = true
|
|
||||||
changesSet[num] = true
|
|
||||||
}
|
|
||||||
groups = reSnapshot.FindStringSubmatch(baseName)
|
|
||||||
if len(groups) == 2 {
|
|
||||||
num, _ := strconv.Atoi(groups[1])
|
|
||||||
|
|
||||||
numSet[num] = true
|
|
||||||
snapshotsSet[num] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for snapshotNumber := range numSet {
|
|
||||||
var (
|
|
||||||
snapshotFileName string
|
|
||||||
changesFileName string
|
|
||||||
)
|
|
||||||
if changesSet[snapshotNumber] {
|
|
||||||
changesFileName = joinWALFileName(s.dir, snapshotNumber)
|
|
||||||
}
|
|
||||||
if snapshotsSet[snapshotNumber] {
|
|
||||||
snapshotFileName = filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", snapshotNumber))
|
|
||||||
}
|
|
||||||
pairs = append(pairs, SnapshotChangesPair{
|
|
||||||
WALFileName: changesFileName,
|
|
||||||
SnapshotFileName: snapshotFileName,
|
|
||||||
SnapshotNumber: snapshotNumber,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(pairs) == 0 {
|
|
||||||
return nil, nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
sort.Slice(pairs, func(i, j int) bool {
|
|
||||||
return pairs[i].SnapshotNumber > pairs[j].SnapshotNumber
|
|
||||||
})
|
|
||||||
|
|
||||||
pair := pairs[0]
|
|
||||||
if pair.WALFileName == "" {
|
|
||||||
return nil, nil, fmt.Errorf("has %d.shapshot file, but %d.changes file not found",
|
|
||||||
pair.SnapshotNumber, pair.SnapshotNumber)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(pairs) > 1 {
|
|
||||||
prevPair := pairs[1]
|
|
||||||
if prevPair.SnapshotFileName == "" && prevPair.SnapshotNumber != 1 {
|
|
||||||
return &pair, nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if prevPair.WALFileName == "" && pair.SnapshotFileName == "" {
|
|
||||||
return &pair, nil, nil
|
|
||||||
}
|
|
||||||
return &pair, &prevPair, nil
|
|
||||||
} else {
|
|
||||||
return &pair, nil, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *RecoveryAdvisor) GetRecipe() (*RecoveryRecipe, error) {
|
|
||||||
pair, prevPair, err := s.getSnapshotChangesPairs()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if pair == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if pair.SnapshotFileName != "" {
|
|
||||||
isVerified, err := s.verifySnapshot(pair.SnapshotFileName)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("verifySnapshot %s: %s",
|
|
||||||
pair.SnapshotFileName, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if isVerified {
|
|
||||||
recipe := &RecoveryRecipe{
|
|
||||||
Snapshot: pair.SnapshotFileName,
|
|
||||||
Changes: []string{
|
|
||||||
pair.WALFileName,
|
|
||||||
},
|
|
||||||
SnapshotNumber: pair.SnapshotNumber,
|
|
||||||
}
|
|
||||||
|
|
||||||
if prevPair != nil {
|
|
||||||
if prevPair.WALFileName != "" {
|
|
||||||
recipe.ToDelete = append(recipe.ToDelete, prevPair.WALFileName)
|
|
||||||
}
|
|
||||||
if prevPair.SnapshotFileName != "" {
|
|
||||||
recipe.ToDelete = append(recipe.ToDelete, prevPair.SnapshotFileName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return recipe, nil
|
|
||||||
}
|
|
||||||
if prevPair != nil {
|
|
||||||
return s.tryPrevPair(pair, prevPair)
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("%d.shapshot is corrupted", pair.SnapshotNumber)
|
|
||||||
} else {
|
|
||||||
if prevPair != nil {
|
|
||||||
return s.tryPrevPair(pair, prevPair)
|
|
||||||
} else {
|
|
||||||
if pair.SnapshotNumber == 1 {
|
|
||||||
return &RecoveryRecipe{
|
|
||||||
Changes: []string{
|
|
||||||
pair.WALFileName,
|
|
||||||
},
|
|
||||||
SnapshotNumber: pair.SnapshotNumber,
|
|
||||||
}, nil
|
|
||||||
} else {
|
|
||||||
return nil, fmt.Errorf("%d.snapshot not found", pair.SnapshotNumber)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *RecoveryAdvisor) tryPrevPair(pair, prevPair *SnapshotChangesPair) (*RecoveryRecipe, error) {
|
|
||||||
if prevPair.WALFileName == "" {
|
|
||||||
if pair.SnapshotFileName != "" {
|
|
||||||
return nil, fmt.Errorf("%d.shapshot is corrupted and %d.changes not found",
|
|
||||||
pair.SnapshotNumber, prevPair.SnapshotNumber)
|
|
||||||
} else {
|
|
||||||
return nil, fmt.Errorf("%d.changes not found", prevPair.SnapshotNumber)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if prevPair.SnapshotFileName == "" {
|
|
||||||
if prevPair.SnapshotNumber == 1 {
|
|
||||||
recipe := &RecoveryRecipe{
|
|
||||||
Changes: []string{
|
|
||||||
prevPair.WALFileName,
|
|
||||||
pair.WALFileName,
|
|
||||||
},
|
|
||||||
SnapshotNumber: pair.SnapshotNumber,
|
|
||||||
CompleteSnapshot: true,
|
|
||||||
ToDelete: []string{
|
|
||||||
prevPair.WALFileName,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
return recipe, nil
|
|
||||||
} else {
|
|
||||||
if pair.SnapshotFileName != "" {
|
|
||||||
return nil, fmt.Errorf("%d.shapshot is corrupted and %d.snapshot not found",
|
|
||||||
pair.SnapshotNumber, prevPair.SnapshotNumber)
|
|
||||||
} else {
|
|
||||||
return nil, fmt.Errorf("%d.snapshot not found", pair.SnapshotNumber)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
isVerified, err := s.verifySnapshot(prevPair.SnapshotFileName)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("verifySnapshot %s: %s",
|
|
||||||
prevPair.SnapshotFileName, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !isVerified {
|
|
||||||
return nil, fmt.Errorf("%d.shapshot is corrupted", prevPair.SnapshotNumber)
|
|
||||||
}
|
|
||||||
|
|
||||||
recipe := &RecoveryRecipe{
|
|
||||||
Snapshot: prevPair.SnapshotFileName,
|
|
||||||
Changes: []string{
|
|
||||||
prevPair.WALFileName,
|
|
||||||
pair.WALFileName,
|
|
||||||
},
|
|
||||||
SnapshotNumber: pair.SnapshotNumber,
|
|
||||||
CompleteSnapshot: true,
|
|
||||||
ToDelete: []string{
|
|
||||||
prevPair.WALFileName,
|
|
||||||
prevPair.SnapshotFileName,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
return recipe, nil
|
|
||||||
}
|
|
||||||
221
recovery/recovery.go
Normal file
221
recovery/recovery.go
Normal file
@@ -0,0 +1,221 @@
|
|||||||
|
package recovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gordenko.dev/dima/qb"
|
||||||
|
"gordenko.dev/dima/qb/freelist"
|
||||||
|
"gordenko.dev/dima/qb/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Result зберігає результати для обох типів файлів
|
||||||
|
type FileVersion struct {
|
||||||
|
Name string
|
||||||
|
Version int
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindMaxSnapshotFiles шукає файли з розширеннями .snapshot_XXX та .wal_XXX
|
||||||
|
// і повертає структуру з максимальними номерами.
|
||||||
|
func findFileLatestVersion(dir string, extPrefix string) (FileVersion, error) {
|
||||||
|
var x FileVersion
|
||||||
|
if !strings.HasPrefix(extPrefix, ".") {
|
||||||
|
extPrefix = "." + extPrefix
|
||||||
|
}
|
||||||
|
entries, err := os.ReadDir(dir)
|
||||||
|
if err != nil {
|
||||||
|
return x, err
|
||||||
|
}
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := entry.Name()
|
||||||
|
ext := filepath.Ext(name)
|
||||||
|
|
||||||
|
if strings.HasPrefix(ext, extPrefix) {
|
||||||
|
version, err := strconv.Atoi(strings.TrimPrefix(ext, extPrefix))
|
||||||
|
if err == nil {
|
||||||
|
if version > x.Version || x.Name == "" {
|
||||||
|
x.Name = name
|
||||||
|
x.Version = version
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return x, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrLostSnapshot = errors.New("lost snapshot")
|
||||||
|
)
|
||||||
|
|
||||||
|
type RecoveryFiles struct {
|
||||||
|
Snapshot string
|
||||||
|
WAL string
|
||||||
|
SnapshotNumber int
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveRecoveryFiles(dir string) (_ RecoveryFiles, err error) {
|
||||||
|
wal, err := findFileLatestVersion(dir, "wal_")
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
snapshot, err := findFileLatestVersion(dir, "snapshot_")
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if snapshot.Name != "" {
|
||||||
|
if wal.Name != "" {
|
||||||
|
if snapshot.Version == wal.Version {
|
||||||
|
return RecoveryFiles{
|
||||||
|
Snapshot: snapshot.Name,
|
||||||
|
WAL: wal.Name,
|
||||||
|
SnapshotNumber: snapshot.Version,
|
||||||
|
}, nil
|
||||||
|
} else if snapshot.Version > wal.Version {
|
||||||
|
return RecoveryFiles{
|
||||||
|
Snapshot: snapshot.Name,
|
||||||
|
SnapshotNumber: snapshot.Version,
|
||||||
|
}, nil
|
||||||
|
} else {
|
||||||
|
// WAL version > snapshot version
|
||||||
|
err = ErrLostSnapshot
|
||||||
|
return
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return RecoveryFiles{
|
||||||
|
Snapshot: snapshot.Name,
|
||||||
|
SnapshotNumber: snapshot.Version,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if wal.Name != "" {
|
||||||
|
if wal.Version == 0 {
|
||||||
|
return RecoveryFiles{
|
||||||
|
WAL: wal.Name,
|
||||||
|
}, nil
|
||||||
|
} else {
|
||||||
|
err = ErrLostSnapshot
|
||||||
|
return
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// no files - ok, 1st start of database
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type RecoveryReport struct {
|
||||||
|
//Metrics []
|
||||||
|
SnapshotNumber int
|
||||||
|
WAL string
|
||||||
|
IndexFreeList *freelist.FreeList
|
||||||
|
DataFreeList *freelist.FreeList
|
||||||
|
}
|
||||||
|
|
||||||
|
func Recovery(dir string, databaseName string) (_ RecoveryReport, err error) {
|
||||||
|
files, err := resolveRecoveryFiles(dir)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var (
|
||||||
|
freeIndexPages []uint32
|
||||||
|
freeDataPages []uint32
|
||||||
|
)
|
||||||
|
if files.Snapshot != "" {
|
||||||
|
var out storage.ReadSnapshotOut
|
||||||
|
out, err = storage.ReadSnapshot(files.Snapshot)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
//
|
||||||
|
freeIndexPages = out.IndexPageNumbers
|
||||||
|
freeDataPages = out.DataPageNumbers
|
||||||
|
}
|
||||||
|
|
||||||
|
if files.WAL != "" {
|
||||||
|
var (
|
||||||
|
walReader *storage.WALReader
|
||||||
|
walReplayer *storage.WALReplayer
|
||||||
|
)
|
||||||
|
walReader, err = storage.NewWALReader(storage.WALReaderOptions{
|
||||||
|
//FileName: JoinWALFileName(s.dir, files.WAL),
|
||||||
|
BufferSize: 8 * 1024 * 1024,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
walReplayer, err = storage.NewWALReplayer(storage.WALReplayerOptions{
|
||||||
|
WALReader: walReader,
|
||||||
|
Metrics: snapshot.Metrics,
|
||||||
|
FreeIndexPages: freeIndexPages,
|
||||||
|
FreeDataPages: freeDataPages,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
err = walReplayer.Replay()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// перезаписую сторінки із останнього комміта
|
||||||
|
var dataFile *os.File
|
||||||
|
dataFile, err = os.OpenFile(qb.GetDataFilePath(dir, databaseName), os.O_WRONLY, 0666)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = storage.WriteDataPages(dataFile, walReplayer.DataPages())
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = dataFile.Close()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var indexFile *os.File
|
||||||
|
indexFile, err = os.OpenFile(qb.GetIndexFilePath(dir, databaseName), os.O_WRONLY, 0666)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = storage.WriteIndexPages(indexFile, walReplayer.IndexPages())
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = indexFile.Close()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
indexFreeList, err := freelist.New(freelist.Options{
|
||||||
|
PageSize: 2048,
|
||||||
|
BaseFilePath: filepath.Join(dir, databaseName+".free_index"),
|
||||||
|
DeltaFilePath: filepath.Join(dir, databaseName+".free_index_delta"),
|
||||||
|
})
|
||||||
|
|
||||||
|
// FIX free pages sync
|
||||||
|
dataFreeList, err := freelist.New(freelist.Options{
|
||||||
|
PageSize: 2048,
|
||||||
|
BaseFilePath: filepath.Join(dir, databaseName+".free_data"),
|
||||||
|
DeltaFilePath: filepath.Join(dir, databaseName+".free_data_delta"),
|
||||||
|
})
|
||||||
|
|
||||||
|
return RecoveryReport{
|
||||||
|
SnapshotNumber: files.SnapshotNumber,
|
||||||
|
WAL: files.WAL,
|
||||||
|
IndexFreeList: indexFreeList,
|
||||||
|
DataFreeList: dataFreeList,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
//func (s *Database) relayMetricsToMetrics(replayMetrics map[uint32]*ReplayMetric) {
|
||||||
|
//for metricID, x := range replayMetrics {
|
||||||
|
//s.metrics[metricID] = x.ToMetric() FIX
|
||||||
|
//}
|
||||||
|
//}
|
||||||
18
recovery/recovery_test.go
Normal file
18
recovery/recovery_test.go
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
package recovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFindMaxVersion(t *testing.T) {
|
||||||
|
result, err := findFileLatestVersion(".", "wal_")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Name != "test.wal_123" {
|
||||||
|
t.Fatalf("wrong file name %s instead of test.wal_123", result.Name)
|
||||||
|
}
|
||||||
|
if result.Version != 123 {
|
||||||
|
t.Fatalf("wrong file version %d instead of 123", result.Version)
|
||||||
|
}
|
||||||
|
}
|
||||||
0
recovery/test.wal_0
Normal file
0
recovery/test.wal_0
Normal file
0
recovery/test.wal_123
Normal file
0
recovery/test.wal_123
Normal file
@@ -236,7 +236,7 @@ func appendIndexRecord(in appendIndexRecordIn) []byte {
|
|||||||
buf = in.Records
|
buf = in.Records
|
||||||
)
|
)
|
||||||
if in.RecordsCount > 0 {
|
if in.RecordsCount > 0 {
|
||||||
pos = in.RecordsCount * indexRecordSize
|
pos = in.RecordsCount * IndexRecordSize
|
||||||
//fmt.Printf("pos: %d\n", pos)
|
//fmt.Printf("pos: %d\n", pos)
|
||||||
} else {
|
} else {
|
||||||
buf = make([]byte, IndexPageSize) // IndexPageIncSize
|
buf = make([]byte, IndexPageSize) // IndexPageIncSize
|
||||||
|
|||||||
55
storage/pageno_provider.go
Normal file
55
storage/pageno_provider.go
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"math"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"gordenko.dev/dima/qb/freelist"
|
||||||
|
)
|
||||||
|
|
||||||
|
type PageNoProvider struct {
|
||||||
|
freeList *freelist.FreeList
|
||||||
|
pagesCount uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
type PageNoProviderOptions struct {
|
||||||
|
FilePath string
|
||||||
|
PageSize int
|
||||||
|
FreeList *freelist.FreeList
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewPageNoProvider(opt PageNoProviderOptions) (_ *PageNoProvider, err error) {
|
||||||
|
if opt.FreeList == nil {
|
||||||
|
return nil, errors.New("FreeList option is required")
|
||||||
|
}
|
||||||
|
file, err := os.Open(opt.FilePath)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
info, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pagesCount := info.Size() / int64(opt.PageSize)
|
||||||
|
return &PageNoProvider{
|
||||||
|
freeList: opt.FreeList,
|
||||||
|
pagesCount: uint32(pagesCount),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PageNoProvider) GetPageNumber() (uint32, bool, error) {
|
||||||
|
pageNo, err := s.freeList.GetPageNumber()
|
||||||
|
if err != nil {
|
||||||
|
return 0, false, err
|
||||||
|
}
|
||||||
|
if pageNo > 0 {
|
||||||
|
return pageNo, true, nil
|
||||||
|
}
|
||||||
|
if s.pagesCount < math.MaxUint32 {
|
||||||
|
s.pagesCount++
|
||||||
|
return s.pagesCount, false, nil
|
||||||
|
}
|
||||||
|
return 0, false, errors.New("no space")
|
||||||
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package database
|
package storage
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -6,8 +6,6 @@ import (
|
|||||||
|
|
||||||
bin "gordenko.dev/dima/bin/little"
|
bin "gordenko.dev/dima/bin/little"
|
||||||
"gordenko.dev/dima/qb"
|
"gordenko.dev/dima/qb"
|
||||||
"gordenko.dev/dima/qb/enc"
|
|
||||||
"gordenko.dev/dima/qb/storage"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type ReplayMetric struct {
|
type ReplayMetric struct {
|
||||||
@@ -18,7 +16,7 @@ type ReplayMetric struct {
|
|||||||
databuf []byte // якщо розмір buf == DataPageSize, то databuf буде менше (мінус футер)
|
databuf []byte // якщо розмір buf == DataPageSize, то databuf буде менше (мінус футер)
|
||||||
tSize int
|
tSize int
|
||||||
vSize int
|
vSize int
|
||||||
indexLevelTails []storage.IndexLevelTail // root - last element
|
indexLevelTails []IndexLevelTail // root - last element
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) {
|
func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) {
|
||||||
@@ -58,18 +56,18 @@ func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) {
|
|||||||
}
|
}
|
||||||
for range levelsCount {
|
for range levelsCount {
|
||||||
var (
|
var (
|
||||||
buf = make([]byte, storage.IndexPageSize)
|
buf = make([]byte, IndexPageSize)
|
||||||
count int
|
count int
|
||||||
)
|
)
|
||||||
count, err = bin.ReadVarSize(r)
|
count, err = bin.ReadVarSize(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
err = bin.ReadNInto(r, buf[:count*indexRecordSize])
|
err = bin.ReadNInto(r, buf[:count*IndexRecordSize])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.indexLevelTails = append(s.indexLevelTails, storage.IndexLevelTail{
|
s.indexLevelTails = append(s.indexLevelTails, IndexLevelTail{
|
||||||
Buffer: buf,
|
Buffer: buf,
|
||||||
RecordsCount: count,
|
RecordsCount: count,
|
||||||
})
|
})
|
||||||
@@ -77,16 +75,16 @@ func (s *ReplayMetric) ReadFrom(r io.Reader) (err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// func appendNewRecordsToIndexTail(level storage.IndexLevelTail, records []byte) {
|
// func appendNewRecordsToIndexTail(level IndexLevelTail, records []byte) {
|
||||||
|
|
||||||
// }
|
// }
|
||||||
|
|
||||||
// func countReusedIndexPages(changedIndexLevels []storage.ChangedIndexLevel) (count int) {
|
// func countReusedIndexPages(changedIndexLevels []ChangedIndexLevel) (count int) {
|
||||||
|
|
||||||
// return
|
// return
|
||||||
// }
|
// }
|
||||||
|
|
||||||
func (s *ReplayMetric) MeasuresAppend(rec storage.MeasuresAppendRecord) {
|
func (s *ReplayMetric) MeasuresAppend(rec MeasuresAppendRecord) {
|
||||||
timestampsSize := rec.TimestampsRewindOffset - len(rec.Timestamps)
|
timestampsSize := rec.TimestampsRewindOffset - len(rec.Timestamps)
|
||||||
// копіюю нові дані із WAL з урахуванням offset-ів
|
// копіюю нові дані із WAL з урахуванням offset-ів
|
||||||
copy(s.databuf[rec.ValuesRewindOffset:], rec.Values)
|
copy(s.databuf[rec.ValuesRewindOffset:], rec.Values)
|
||||||
@@ -97,24 +95,24 @@ func (s *ReplayMetric) MeasuresAppend(rec storage.MeasuresAppendRecord) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type AppendMeasuresResult struct {
|
type AppendMeasuresResult struct {
|
||||||
IndexPages []storage.PageToWrite
|
IndexPages []PageToWrite
|
||||||
DataPages []storage.PageToWrite
|
DataPages []PageToWrite
|
||||||
ReusedIndexPagesCount int
|
ReusedIndexPagesCount int
|
||||||
ReusedDataPagesCount int
|
ReusedDataPagesCount int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *ReplayMetric) MeasuresAppendWithGrow(rec storage.MeasuresAppendWithGrowRecord, isLastPacket bool) (_ AppendMeasuresResult) {
|
func (s *ReplayMetric) MeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord, isLastPacket bool) (_ AppendMeasuresResult) {
|
||||||
var (
|
var (
|
||||||
reusedIndexPages int
|
reusedIndexPages int
|
||||||
reusedDataPages int
|
reusedDataPages int
|
||||||
indexPages []storage.PageToWrite
|
indexPages []PageToWrite
|
||||||
dataPages []storage.PageToWrite
|
dataPages []PageToWrite
|
||||||
)
|
)
|
||||||
// Додати в index level tails недостаючі дані, або замінити
|
// Додати в index level tails недостаючі дані, або замінити
|
||||||
for levelIdx, change := range rec.ChangedIndexLevels {
|
for levelIdx, change := range rec.ChangedIndexLevels {
|
||||||
var (
|
var (
|
||||||
head = change.IndexPageTail
|
head = change.IndexPageTail
|
||||||
newRecordsCount = len(change.TailRecords) / indexRecordSize
|
newRecordsCount = len(change.TailRecords) / IndexRecordSize
|
||||||
)
|
)
|
||||||
// 1. набиваю сторінки для перезапису в index файлі
|
// 1. набиваю сторінки для перезапису в index файлі
|
||||||
if isLastPacket {
|
if isLastPacket {
|
||||||
@@ -123,7 +121,7 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec storage.MeasuresAppendWithGrow
|
|||||||
composeHeadIndexPage(levelIdx, s.indexLevelTails[levelIdx], head))
|
composeHeadIndexPage(levelIdx, s.indexLevelTails[levelIdx], head))
|
||||||
}
|
}
|
||||||
for _, x := range change.IndexPages {
|
for _, x := range change.IndexPages {
|
||||||
indexPages = append(indexPages, storage.PageToWrite{
|
indexPages = append(indexPages, PageToWrite{
|
||||||
PageNo: x.PageNo,
|
PageNo: x.PageNo,
|
||||||
Content: x.Content,
|
Content: x.Content,
|
||||||
})
|
})
|
||||||
@@ -140,17 +138,17 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec storage.MeasuresAppendWithGrow
|
|||||||
level.RecordsCount = newRecordsCount
|
level.RecordsCount = newRecordsCount
|
||||||
} else {
|
} else {
|
||||||
// TailRecords - це нові дані, які треба додати
|
// TailRecords - це нові дані, які треба додати
|
||||||
pos := level.RecordsCount * indexRecordSize
|
pos := level.RecordsCount * IndexRecordSize
|
||||||
copy(level.Buffer[pos:], change.TailRecords)
|
copy(level.Buffer[pos:], change.TailRecords)
|
||||||
level.RecordsCount += newRecordsCount
|
level.RecordsCount += newRecordsCount
|
||||||
}
|
}
|
||||||
s.indexLevelTails[levelIdx] = level
|
s.indexLevelTails[levelIdx] = level
|
||||||
} else {
|
} else {
|
||||||
// додаю новий індексний рівень
|
// додаю новий індексний рівень
|
||||||
buf := make([]byte, storage.IndexPageSize)
|
buf := make([]byte, IndexPageSize)
|
||||||
copy(buf, change.TailRecords)
|
copy(buf, change.TailRecords)
|
||||||
//
|
//
|
||||||
s.indexLevelTails = append(s.indexLevelTails, storage.IndexLevelTail{
|
s.indexLevelTails = append(s.indexLevelTails, IndexLevelTail{
|
||||||
Buffer: buf,
|
Buffer: buf,
|
||||||
RecordsCount: newRecordsCount,
|
RecordsCount: newRecordsCount,
|
||||||
})
|
})
|
||||||
@@ -174,7 +172,7 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec storage.MeasuresAppendWithGrow
|
|||||||
dataPages = append(dataPages, composeHeadDataPage(s.databuf, rec.DataPageTail))
|
dataPages = append(dataPages, composeHeadDataPage(s.databuf, rec.DataPageTail))
|
||||||
|
|
||||||
for _, x := range rec.DataPages {
|
for _, x := range rec.DataPages {
|
||||||
dataPages = append(dataPages, storage.PageToWrite{
|
dataPages = append(dataPages, PageToWrite{
|
||||||
PageNo: x.PageNo,
|
PageNo: x.PageNo,
|
||||||
Content: x.Content,
|
Content: x.Content,
|
||||||
})
|
})
|
||||||
@@ -215,17 +213,17 @@ func (s *ReplayMetric) MeasuresAppendWithGrow(rec storage.MeasuresAppendWithGrow
|
|||||||
|
|
||||||
// HELPERS
|
// HELPERS
|
||||||
|
|
||||||
func composeHeadIndexPage(levelIdx int, level storage.IndexLevelTail, head *storage.IndexPageTail) storage.PageToWrite {
|
func composeHeadIndexPage(levelIdx int, level IndexLevelTail, head *IndexPageTail) PageToWrite {
|
||||||
// розраховую pos, з якого буду дописувати хвіст
|
// розраховую pos, з якого буду дописувати хвіст
|
||||||
pos := level.RecordsCount * indexRecordSize
|
pos := level.RecordsCount * IndexRecordSize
|
||||||
// створюю копію сторінки
|
// створюю копію сторінки
|
||||||
page := make([]byte, storage.IndexPageSize)
|
page := make([]byte, IndexPageSize)
|
||||||
copy(page, level.Buffer[:pos]) // поточні дані
|
copy(page, level.Buffer[:pos]) // поточні дані
|
||||||
copy(page[pos:], head.Records)
|
copy(page[pos:], head.Records)
|
||||||
// запечатати сторінку
|
// запечатати сторінку
|
||||||
calculatedCRC := storage.SealIndexPage(storage.SealIndexPageIn{
|
calculatedCRC := SealIndexPage(SealIndexPageIn{
|
||||||
Content: page,
|
Content: page,
|
||||||
RecordsCount: level.RecordsCount + len(head.Records)/indexRecordSize,
|
RecordsCount: level.RecordsCount + len(head.Records)/IndexRecordSize,
|
||||||
ZeroLevel: levelIdx == 0,
|
ZeroLevel: levelIdx == 0,
|
||||||
})
|
})
|
||||||
// перевірка CRC
|
// перевірка CRC
|
||||||
@@ -234,16 +232,16 @@ func composeHeadIndexPage(levelIdx int, level storage.IndexLevelTail, head *stor
|
|||||||
fmt.Errorf("calculated CRC %d not equal expected %d of head page %d on index level %d",
|
fmt.Errorf("calculated CRC %d not equal expected %d of head page %d on index level %d",
|
||||||
calculatedCRC, head.CRC32, head.PageNo, levelIdx))
|
calculatedCRC, head.CRC32, head.PageNo, levelIdx))
|
||||||
}
|
}
|
||||||
return storage.PageToWrite{
|
return PageToWrite{
|
||||||
PageNo: head.PageNo,
|
PageNo: head.PageNo,
|
||||||
Content: page,
|
Content: page,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func composeHeadDataPage(databuf []byte, head storage.DataPageTail) storage.PageToWrite {
|
func composeHeadDataPage(databuf []byte, head DataPageTail) PageToWrite {
|
||||||
// створюю копію сторінки
|
// створюю копію сторінки
|
||||||
var (
|
var (
|
||||||
page = make([]byte, storage.DataPageSize)
|
page = make([]byte, DataPageSize)
|
||||||
timestampsSize = head.TimestampsRewindOffset - len(head.Timestamps)
|
timestampsSize = head.TimestampsRewindOffset - len(head.Timestamps)
|
||||||
)
|
)
|
||||||
// копіюю поточні дані
|
// копіюю поточні дані
|
||||||
@@ -252,7 +250,7 @@ func composeHeadDataPage(databuf []byte, head storage.DataPageTail) storage.Page
|
|||||||
copy(page[head.ValuesRewindOffset:], head.Values)
|
copy(page[head.ValuesRewindOffset:], head.Values)
|
||||||
copy(page[len(page)-timestampsSize:], head.Timestamps)
|
copy(page[len(page)-timestampsSize:], head.Timestamps)
|
||||||
// запечатати сторінку
|
// запечатати сторінку
|
||||||
calculatedCRC := storage.SealDataPage(storage.SealDataPageIn{
|
calculatedCRC := SealDataPage(SealDataPageIn{
|
||||||
Content: page,
|
Content: page,
|
||||||
PrevPageNo: head.PrevPageNo,
|
PrevPageNo: head.PrevPageNo,
|
||||||
TimestampsSize: timestampsSize,
|
TimestampsSize: timestampsSize,
|
||||||
@@ -264,24 +262,24 @@ func composeHeadDataPage(databuf []byte, head storage.DataPageTail) storage.Page
|
|||||||
fmt.Errorf("calculated CRC %d not equal expected %d of head data page %d",
|
fmt.Errorf("calculated CRC %d not equal expected %d of head data page %d",
|
||||||
calculatedCRC, head.CRC32, head.PageNo))
|
calculatedCRC, head.CRC32, head.PageNo))
|
||||||
}
|
}
|
||||||
return storage.PageToWrite{
|
return PageToWrite{
|
||||||
PageNo: head.PageNo,
|
PageNo: head.PageNo,
|
||||||
Content: page,
|
Content: page,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *ReplayMetric) ToMetric() *_metric {
|
// func (s *ReplayMetric) ToMetric() *worker.metric {
|
||||||
|
|
||||||
databuf := s.buf[:storage.DataPagePayloadSize]
|
// databuf := s.buf[:storage.DataPagePayloadSize]
|
||||||
values := enc.NewValueDeltaCompressor(s.metricType, s.fracDigits, databuf, s.vSize)
|
// values := enc.NewValueDeltaCompressor(s.metricType, s.fracDigits, databuf, s.vSize)
|
||||||
return &_metric{
|
// return &_metric{
|
||||||
metricType: s.metricType,
|
// metricType: s.metricType,
|
||||||
fracDigits: s.fracDigits,
|
// fracDigits: s.fracDigits,
|
||||||
lastPageNo: s.lastPageNo,
|
// lastPageNo: s.lastPageNo,
|
||||||
lastValue: values.LastValue(),
|
// lastValue: values.LastValue(),
|
||||||
buffer: s.buf,
|
// buffer: s.buf,
|
||||||
timestamps: enc.NewTimeDeltaCompressor(databuf, s.tSize),
|
// timestamps: enc.NewTimeDeltaCompressor(databuf, s.tSize),
|
||||||
values: values,
|
// values: values,
|
||||||
indexLevelTails: s.indexLevelTails,
|
// indexLevelTails: s.indexLevelTails,
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package database
|
package storage
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
bin "gordenko.dev/dima/bin/little"
|
bin "gordenko.dev/dima/bin/little"
|
||||||
"gordenko.dev/dima/qb/storage"
|
|
||||||
"gordenko.dev/dima/qb/util"
|
"gordenko.dev/dima/qb/util"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -29,20 +28,24 @@ dataFreeList - Nb
|
|||||||
CRC32 - 4b
|
CRC32 - 4b
|
||||||
*/
|
*/
|
||||||
|
|
||||||
const readBufferSize = 8 * 1024 * 1024 // 8mb
|
const ReadBufferSize = 8 * 1024 * 1024 // 8mb
|
||||||
|
|
||||||
type writeSnapshotIn struct {
|
type WritableMetric interface {
|
||||||
|
WriteTo(io.Writer) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type WriteSnapshotIn struct {
|
||||||
SnapshotNumber int
|
SnapshotNumber int
|
||||||
Dir string
|
Dir string
|
||||||
WriteBufferSize int
|
WriteBufferSize int
|
||||||
Metrics map[uint32]*_metric
|
Metrics map[uint32]WritableMetric
|
||||||
FrozenIndexPagesCount int
|
FrozenIndexPagesCount int
|
||||||
IndexPageNumbers []uint32
|
IndexPageNumbers []uint32
|
||||||
FrozenDataPagesCount int
|
FrozenDataPagesCount int
|
||||||
DataPageNumbers []uint32
|
DataPageNumbers []uint32
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeSnapshot(in writeSnapshotIn) (err error) {
|
func WriteSnapshot(in WriteSnapshotIn) (err error) {
|
||||||
var (
|
var (
|
||||||
fileName = filepath.Join(in.Dir, fmt.Sprintf("%d.snapshot", in.SnapshotNumber))
|
fileName = filepath.Join(in.Dir, fmt.Sprintf("%d.snapshot", in.SnapshotNumber))
|
||||||
hasher = util.NewHasher()
|
hasher = util.NewHasher()
|
||||||
@@ -118,7 +121,7 @@ func writeSnapshot(in writeSnapshotIn) (err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
type readSnapshotOut struct {
|
type ReadSnapshotOut struct {
|
||||||
Metrics map[uint32]*ReplayMetric
|
Metrics map[uint32]*ReplayMetric
|
||||||
FrozenIndexPagesCount int
|
FrozenIndexPagesCount int
|
||||||
IndexPageNumbers []uint32
|
IndexPageNumbers []uint32
|
||||||
@@ -126,7 +129,7 @@ type readSnapshotOut struct {
|
|||||||
DataPageNumbers []uint32
|
DataPageNumbers []uint32
|
||||||
}
|
}
|
||||||
|
|
||||||
func readSnapshot(fileName string) (out readSnapshotOut, err error) {
|
func ReadSnapshot(fileName string) (out ReadSnapshotOut, err error) {
|
||||||
var (
|
var (
|
||||||
metrics = make(map[uint32]*ReplayMetric)
|
metrics = make(map[uint32]*ReplayMetric)
|
||||||
frozenIndexPagesCount int
|
frozenIndexPagesCount int
|
||||||
@@ -149,7 +152,7 @@ func readSnapshot(fileName string) (out readSnapshotOut, err error) {
|
|||||||
fileSize := stat.Size()
|
fileSize := stat.Size()
|
||||||
|
|
||||||
if fileSize == 0 {
|
if fileSize == 0 {
|
||||||
return readSnapshotOut{
|
return ReadSnapshotOut{
|
||||||
Metrics: metrics,
|
Metrics: metrics,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@@ -177,11 +180,11 @@ func readSnapshot(fileName string) (out readSnapshotOut, err error) {
|
|||||||
for range metricsQty {
|
for range metricsQty {
|
||||||
var (
|
var (
|
||||||
metricID uint32
|
metricID uint32
|
||||||
buf = make([]byte, storage.DataPageSize)
|
buf = make([]byte, DataPageSize)
|
||||||
// для простоти створюю буфер максимального розміру
|
// для простоти створюю буфер максимального розміру
|
||||||
metric = &ReplayMetric{
|
metric = &ReplayMetric{
|
||||||
buf: buf,
|
buf: buf,
|
||||||
databuf: buf[:storage.DataPagePayloadSize],
|
databuf: buf[:DataPagePayloadSize],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
metricID, err = bin.ReadUint32(src)
|
metricID, err = bin.ReadUint32(src)
|
||||||
@@ -218,7 +221,7 @@ func readSnapshot(fileName string) (out readSnapshotOut, err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
return readSnapshotOut{
|
return ReadSnapshotOut{
|
||||||
Metrics: metrics,
|
Metrics: metrics,
|
||||||
FrozenIndexPagesCount: frozenIndexPagesCount,
|
FrozenIndexPagesCount: frozenIndexPagesCount,
|
||||||
IndexPageNumbers: indexPageNumbers,
|
IndexPageNumbers: indexPageNumbers,
|
||||||
@@ -24,7 +24,7 @@ var (
|
|||||||
indexRecordsCountIdx = IndexPageSize - 6
|
indexRecordsCountIdx = IndexPageSize - 6
|
||||||
isZeroLevelIdx = IndexPageSize - 7
|
isZeroLevelIdx = IndexPageSize - 7
|
||||||
|
|
||||||
maxRecordsOnIndexPage = (IndexPageSize - IndexPageFooterSize) / indexRecordSize
|
maxRecordsOnIndexPage = (IndexPageSize - IndexPageFooterSize) / IndexRecordSize
|
||||||
|
|
||||||
// timestampSize = 4
|
// timestampSize = 4
|
||||||
// pairSize = timestampSize + PageNoSize
|
// pairSize = timestampSize + PageNoSize
|
||||||
@@ -35,7 +35,7 @@ var (
|
|||||||
const (
|
const (
|
||||||
DataPageFooterSize = 12
|
DataPageFooterSize = 12
|
||||||
IndexPageFooterSize = 7
|
IndexPageFooterSize = 7
|
||||||
indexRecordSize = 8
|
IndexRecordSize = 8
|
||||||
)
|
)
|
||||||
|
|
||||||
type IndexRecord struct {
|
type IndexRecord struct {
|
||||||
|
|||||||
@@ -867,3 +867,83 @@ func TestWritePreparer(t *testing.T) {
|
|||||||
fmt.Printf("%#v\n", x)
|
fmt.Printf("%#v\n", x)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestWriteReadSnapshot(t *testing.T) {
|
||||||
|
metrics := make(map[uint32]*_metric)
|
||||||
|
metrics[1] = &_metric{
|
||||||
|
metricType: qb.Cumulative,
|
||||||
|
fracDigits: 3,
|
||||||
|
lastPageNo: 1000,
|
||||||
|
buffer: []byte{},
|
||||||
|
}
|
||||||
|
in := WriteSnapshotIn{
|
||||||
|
SnapshotNumber: 1,
|
||||||
|
Dir: ".",
|
||||||
|
WriteBufferSize: 8 * 1024 * 1024,
|
||||||
|
Metrics: metrics,
|
||||||
|
FrozenIndexPagesCount: 7,
|
||||||
|
IndexPageNumbers: []uint32{
|
||||||
|
1, 2, 3, 4, 5,
|
||||||
|
},
|
||||||
|
FrozenDataPagesCount: 8,
|
||||||
|
DataPageNumbers: []uint32{
|
||||||
|
6, 7, 8,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
err := WriteSnapshot(in)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("writeSnapshot: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := ReadSnapshot("1.snapshot")
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if out.FrozenIndexPagesCount != in.FrozenIndexPagesCount {
|
||||||
|
t.Fatalf("FrozenIndexPagesCount: got %d are not equal expected %d",
|
||||||
|
out.FrozenIndexPagesCount, in.FrozenIndexPagesCount)
|
||||||
|
}
|
||||||
|
if out.FrozenDataPagesCount != in.FrozenDataPagesCount {
|
||||||
|
t.Fatalf("FrozenDataPagesCount: got %d are not equal expected %d",
|
||||||
|
out.FrozenDataPagesCount, in.FrozenDataPagesCount)
|
||||||
|
}
|
||||||
|
if !slices.Equal(out.IndexPageNumbers, in.IndexPageNumbers) {
|
||||||
|
t.Fatalf("IndexPageNumbers: got %v are not equal expected %v",
|
||||||
|
out.IndexPageNumbers, in.IndexPageNumbers)
|
||||||
|
}
|
||||||
|
if !slices.Equal(out.DataPageNumbers, in.DataPageNumbers) {
|
||||||
|
t.Fatalf("DataPageNumbers: got %v are not equal expected %v",
|
||||||
|
out.DataPageNumbers, in.DataPageNumbers)
|
||||||
|
}
|
||||||
|
for metricID, replay := range out.Metrics {
|
||||||
|
origin, ok := metrics[metricID]
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("decoded metricID %d not found in metrics", metricID)
|
||||||
|
}
|
||||||
|
err = cmpMetric(replay, origin)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("metric %d: %s", metricID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cmpMetric(replay *ReplayMetric, origin *_metric) error {
|
||||||
|
if replay.metricType != origin.metricType {
|
||||||
|
return fmt.Errorf("metricType: got %d are not equal expected %d",
|
||||||
|
replay.metricType, origin.metricType)
|
||||||
|
}
|
||||||
|
if replay.fracDigits != origin.fracDigits {
|
||||||
|
return fmt.Errorf("fracDigits: got %d are not equal expected %d",
|
||||||
|
replay.fracDigits, origin.fracDigits)
|
||||||
|
}
|
||||||
|
if replay.lastPageNo != origin.lastPageNo {
|
||||||
|
return fmt.Errorf("lastPageNo: got %d are not equal expected %d",
|
||||||
|
replay.lastPageNo, origin.lastPageNo)
|
||||||
|
}
|
||||||
|
// if !refect.DeepEq{
|
||||||
|
// return fmt.Errorf("lastPageNo: got %d are not equal expected %d",
|
||||||
|
// replay.lastPageNo, origin.lastPageNo)
|
||||||
|
// }
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -194,7 +194,7 @@ func (s MeasuresAppendWithGrowRecord) Pack(w io.Writer) {
|
|||||||
bin.WriteUint32(w, tail.PageNo)
|
bin.WriteUint32(w, tail.PageNo)
|
||||||
bin.WriteBool(w, tail.Reused)
|
bin.WriteBool(w, tail.Reused)
|
||||||
bin.WriteUint32(w, tail.CRC32)
|
bin.WriteUint32(w, tail.CRC32)
|
||||||
bin.WriteVarSize(w, len(tail.Records)/indexRecordSize)
|
bin.WriteVarSize(w, len(tail.Records)/IndexRecordSize)
|
||||||
w.Write(tail.Records)
|
w.Write(tail.Records)
|
||||||
} else {
|
} else {
|
||||||
bin.WriteBool(w, false)
|
bin.WriteBool(w, false)
|
||||||
@@ -207,7 +207,7 @@ func (s MeasuresAppendWithGrowRecord) Pack(w io.Writer) {
|
|||||||
w.Write(p.Content)
|
w.Write(p.Content)
|
||||||
}
|
}
|
||||||
// level tail
|
// level tail
|
||||||
bin.WriteVarSize(w, len(level.TailRecords)/indexRecordSize)
|
bin.WriteVarSize(w, len(level.TailRecords)/IndexRecordSize)
|
||||||
w.Write(level.TailRecords)
|
w.Write(level.TailRecords)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -313,7 +313,7 @@ func (s *MeasuresAppendWithGrowRecord) Parse(r io.Reader) (err error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
tail.Records, err = bin.ReadN(r, recordsCount*indexRecordSize)
|
tail.Records, err = bin.ReadN(r, recordsCount*IndexRecordSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -344,7 +344,7 @@ func (s *MeasuresAppendWithGrowRecord) Parse(r io.Reader) (err error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
level.TailRecords, err = bin.ReadN(r, recordsCount*indexRecordSize)
|
level.TailRecords, err = bin.ReadN(r, recordsCount*IndexRecordSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,23 +1,21 @@
|
|||||||
package database
|
package storage
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"gordenko.dev/dima/qb/storage"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type WALReplayer struct {
|
type WALReplayer struct {
|
||||||
walReader *storage.WALReader
|
walReader *WALReader
|
||||||
metrics map[uint32]*ReplayMetric
|
metrics map[uint32]*ReplayMetric
|
||||||
freeIndexPages []uint32
|
freeIndexPages []uint32
|
||||||
freeDataPages []uint32
|
freeDataPages []uint32
|
||||||
indexPages []storage.PageToWrite
|
indexPages []PageToWrite
|
||||||
dataPages []storage.PageToWrite
|
dataPages []PageToWrite
|
||||||
}
|
}
|
||||||
|
|
||||||
type WALReplayerOptions struct {
|
type WALReplayerOptions struct {
|
||||||
WALReader *storage.WALReader
|
WALReader *WALReader
|
||||||
Metrics map[uint32]*ReplayMetric // from snapshot
|
Metrics map[uint32]*ReplayMetric // from snapshot
|
||||||
FreeIndexPages []uint32 // from snapshot
|
FreeIndexPages []uint32 // from snapshot
|
||||||
FreeDataPages []uint32 // from snapshot
|
FreeDataPages []uint32 // from snapshot
|
||||||
@@ -38,11 +36,11 @@ func NewWALReplayer(opt WALReplayerOptions) (*WALReplayer, error) {
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *WALReplayer) IndexPages() []storage.PageToWrite {
|
func (s *WALReplayer) IndexPages() []PageToWrite {
|
||||||
return s.indexPages
|
return s.indexPages
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *WALReplayer) DataPages() []storage.PageToWrite {
|
func (s *WALReplayer) DataPages() []PageToWrite {
|
||||||
return s.dataPages
|
return s.dataPages
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -62,23 +60,23 @@ func (s *WALReplayer) Replay() error {
|
|||||||
|
|
||||||
func (s *WALReplayer) replayRecord(untyped any, isLastPacket bool) (err error) {
|
func (s *WALReplayer) replayRecord(untyped any, isLastPacket bool) (err error) {
|
||||||
switch rec := untyped.(type) {
|
switch rec := untyped.(type) {
|
||||||
case storage.MeasuresAppendRecord:
|
case MeasuresAppendRecord:
|
||||||
if err = s.onMeasuresAppend(rec); err != nil {
|
if err = s.onMeasuresAppend(rec); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
case storage.MeasuresAppendWithGrowRecord:
|
case MeasuresAppendWithGrowRecord:
|
||||||
if err = s.onMeasuresAppendWithGrow(rec, isLastPacket); err != nil {
|
if err = s.onMeasuresAppendWithGrow(rec, isLastPacket); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
case storage.MetricAddRecord:
|
case MetricAddRecord:
|
||||||
if err = s.onMetricAdd(rec); err != nil {
|
if err = s.onMetricAdd(rec); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
case storage.MetricDeleteRecord:
|
case MetricDeleteRecord:
|
||||||
if err = s.onMetricDelete(rec); err != nil {
|
if err = s.onMetricDelete(rec); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// case storage.DeletedMeasures:
|
// case DeletedMeasures:
|
||||||
// metric, ok := s.metrics[rec.MetricID]
|
// metric, ok := s.metrics[rec.MetricID]
|
||||||
// if ok {
|
// if ok {
|
||||||
// metric.DeleteMeasures()
|
// metric.DeleteMeasures()
|
||||||
@@ -94,24 +92,24 @@ func (s *WALReplayer) replayRecord(untyped any, isLastPacket bool) (err error) {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *WALReplayer) onMetricAdd(rec storage.MetricAddRecord) (err error) {
|
func (s *WALReplayer) onMetricAdd(rec MetricAddRecord) (err error) {
|
||||||
_, ok := s.metrics[rec.MetricID]
|
_, ok := s.metrics[rec.MetricID]
|
||||||
if ok {
|
if ok {
|
||||||
return fmt.Errorf("metric %d add failed: already added", rec.MetricID)
|
return fmt.Errorf("metric %d add failed: already added", rec.MetricID)
|
||||||
}
|
}
|
||||||
var (
|
var (
|
||||||
buf = make([]byte, storage.DataPageSize)
|
buf = make([]byte, DataPageSize)
|
||||||
)
|
)
|
||||||
s.metrics[rec.MetricID] = &ReplayMetric{
|
s.metrics[rec.MetricID] = &ReplayMetric{
|
||||||
metricType: rec.MetricType,
|
metricType: rec.MetricType,
|
||||||
fracDigits: byte(rec.FracDigits),
|
fracDigits: byte(rec.FracDigits),
|
||||||
buf: buf,
|
buf: buf,
|
||||||
databuf: buf[:storage.DataPagePayloadSize],
|
databuf: buf[:DataPagePayloadSize],
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *WALReplayer) onMetricDelete(rec storage.MetricDeleteRecord) error {
|
func (s *WALReplayer) onMetricDelete(rec MetricDeleteRecord) error {
|
||||||
_, ok := s.metrics[rec.MetricID]
|
_, ok := s.metrics[rec.MetricID]
|
||||||
if !ok {
|
if !ok {
|
||||||
return fmt.Errorf("metric %d deletion failed: not found", rec.MetricID)
|
return fmt.Errorf("metric %d deletion failed: not found", rec.MetricID)
|
||||||
@@ -122,7 +120,7 @@ func (s *WALReplayer) onMetricDelete(rec storage.MetricDeleteRecord) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *WALReplayer) onMeasuresAppend(rec storage.MeasuresAppendRecord) (err error) {
|
func (s *WALReplayer) onMeasuresAppend(rec MeasuresAppendRecord) (err error) {
|
||||||
metric, ok := s.metrics[rec.MetricID]
|
metric, ok := s.metrics[rec.MetricID]
|
||||||
if !ok {
|
if !ok {
|
||||||
return fmt.Errorf("append measures failed: metric %d not found", rec.MetricID)
|
return fmt.Errorf("append measures failed: metric %d not found", rec.MetricID)
|
||||||
@@ -131,7 +129,7 @@ func (s *WALReplayer) onMeasuresAppend(rec storage.MeasuresAppendRecord) (err er
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *WALReplayer) onMeasuresAppendWithGrow(rec storage.MeasuresAppendWithGrowRecord, isLastPacket bool) (err error) {
|
func (s *WALReplayer) onMeasuresAppendWithGrow(rec MeasuresAppendWithGrowRecord, isLastPacket bool) (err error) {
|
||||||
metric, ok := s.metrics[rec.MetricID]
|
metric, ok := s.metrics[rec.MetricID]
|
||||||
if !ok {
|
if !ok {
|
||||||
return fmt.Errorf("append measures failed: metric %d not found", rec.MetricID)
|
return fmt.Errorf("append measures failed: metric %d not found", rec.MetricID)
|
||||||
@@ -148,16 +146,16 @@ func (s *WALReplayer) onMeasuresAppendWithGrow(rec storage.MeasuresAppendWithGro
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
//func (s *WALReplayer) collectPagesToWrite(metric *ReplayMetric, rec storage.WALRecordAppendMeasures) {
|
//func (s *WALReplayer) collectPagesToWrite(metric *ReplayMetric, rec WALRecordAppendMeasures) {
|
||||||
// metric.ApplyHeadDataPage(rec.CompletedDataPage)
|
// metric.ApplyHeadDataPage(rec.CompletedDataPage)
|
||||||
|
|
||||||
// s.dataPages = append(s.dataPages, storage.PageToWrite{
|
// s.dataPages = append(s.dataPages, PageToWrite{
|
||||||
// PageNo: p.PageNo,
|
// PageNo: p.PageNo,
|
||||||
// Content: metric.buf,
|
// Content: metric.buf,
|
||||||
// })
|
// })
|
||||||
|
|
||||||
// for _, x := range rec.DataPages {
|
// for _, x := range rec.DataPages {
|
||||||
// s.dataPages = append(s.dataPages, storage.PageToWrite{
|
// s.dataPages = append(s.dataPages, PageToWrite{
|
||||||
// PageNo: x.PageNo,
|
// PageNo: x.PageNo,
|
||||||
// Content: x.Content,
|
// Content: x.Content,
|
||||||
// })
|
// })
|
||||||
@@ -169,7 +167,7 @@ func (s *WALReplayer) onMeasuresAppendWithGrow(rec storage.MeasuresAppendWithGro
|
|||||||
// func (s *WALReplayer) replayPacket(records []any, isLastPacket bool) {
|
// func (s *WALReplayer) replayPacket(records []any, isLastPacket bool) {
|
||||||
// for _, untyped := range records {
|
// for _, untyped := range records {
|
||||||
// switch rec := untyped.(type) {
|
// switch rec := untyped.(type) {
|
||||||
// case storage.WALRecordAppendMeasures:
|
// case WALRecordAppendMeasures:
|
||||||
// s.onWALRecordAppendMeasures(rec, isLastPacket)
|
// s.onWALRecordAppendMeasures(rec, isLastPacket)
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
@@ -4,7 +4,6 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
|
||||||
"math"
|
"math"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -12,6 +11,7 @@ import (
|
|||||||
|
|
||||||
"gordenko.dev/dima/qb"
|
"gordenko.dev/dima/qb"
|
||||||
"gordenko.dev/dima/qb/freelist"
|
"gordenko.dev/dima/qb/freelist"
|
||||||
|
"gordenko.dev/dima/qb/inbox"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -42,10 +42,6 @@ type FreeList interface {
|
|||||||
GetPageNumber() (uint32, error)
|
GetPageNumber() (uint32, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
func JoinChangesFileName(dir string, logNumber int) string {
|
|
||||||
return filepath.Join(dir, fmt.Sprintf("%d.changes", logNumber))
|
|
||||||
}
|
|
||||||
|
|
||||||
type MetricsState struct {
|
type MetricsState struct {
|
||||||
//Metrics []Metric
|
//Metrics []Metric
|
||||||
WaitCh chan struct{}
|
WaitCh chan struct{}
|
||||||
@@ -63,30 +59,35 @@ type Changes struct {
|
|||||||
|
|
||||||
type Writer struct {
|
type Writer struct {
|
||||||
mutex sync.Mutex
|
mutex sync.Mutex
|
||||||
|
inbox *inbox.Inbox
|
||||||
|
workerInbox *inbox.Inbox
|
||||||
dataPagesCount uint32
|
dataPagesCount uint32
|
||||||
indexPagesCount uint32
|
indexPagesCount uint32
|
||||||
dataFreeList *freelist.FreeList
|
dataFreeList *freelist.FreeList
|
||||||
indexFreeList *freelist.FreeList
|
indexFreeList *freelist.FreeList
|
||||||
//atree *atree.Atree
|
//atree *atree.Atree
|
||||||
dir string
|
dir string
|
||||||
|
databaseName string
|
||||||
w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
|
w *bytes.Buffer // wal буфер для упаковки даних перед записом на диск
|
||||||
wal *os.File
|
wal *os.File
|
||||||
dataFile *os.File
|
dataFile *os.File
|
||||||
indexFile *os.File
|
indexFile *os.File
|
||||||
input []any
|
input []any
|
||||||
writePreparer *WritePreparer
|
writePreparer *WritePreparer
|
||||||
appendToWorkerQueue func(any)
|
|
||||||
written int64
|
written int64
|
||||||
isExited bool
|
isExited bool
|
||||||
exitCh chan struct{}
|
exitCh chan struct{}
|
||||||
waitGroup *sync.WaitGroup
|
waitGroup *sync.WaitGroup
|
||||||
signalCh chan struct{}
|
//signalCh chan struct{}
|
||||||
}
|
}
|
||||||
|
|
||||||
type WriterOptions struct {
|
type WriterOptions struct {
|
||||||
|
Inbox *inbox.Inbox
|
||||||
|
WorkerInbox *inbox.Inbox
|
||||||
Dir string
|
Dir string
|
||||||
|
DatabaseName string
|
||||||
//SnapshotNumber int // номер журнала
|
//SnapshotNumber int // номер журнала
|
||||||
AppendToWorkerQueue func(any)
|
WAL string
|
||||||
DataFreeList *freelist.FreeList
|
DataFreeList *freelist.FreeList
|
||||||
IndexFreeList *freelist.FreeList
|
IndexFreeList *freelist.FreeList
|
||||||
//Atree *atree.Atree
|
//Atree *atree.Atree
|
||||||
@@ -101,12 +102,21 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
|
|||||||
// if (opt.DataPageSize % 2) != 0 {
|
// if (opt.DataPageSize % 2) != 0 {
|
||||||
// return nil, errors.New("DataPageSize must be multiple of 2")
|
// return nil, errors.New("DataPageSize must be multiple of 2")
|
||||||
// }
|
// }
|
||||||
|
if opt.Inbox == nil {
|
||||||
|
return nil, errors.New("Inbox option is required")
|
||||||
|
}
|
||||||
|
if opt.WorkerInbox == nil {
|
||||||
|
return nil, errors.New("WorkerInbox option is required")
|
||||||
|
}
|
||||||
if opt.Dir == "" {
|
if opt.Dir == "" {
|
||||||
return nil, errors.New("Dir option is required")
|
return nil, errors.New("Dir option is required")
|
||||||
}
|
}
|
||||||
if opt.AppendToWorkerQueue == nil {
|
if opt.DatabaseName == "" {
|
||||||
return nil, errors.New("AppendToWorkerQueue option is required")
|
return nil, errors.New("DatabaseName option is required")
|
||||||
}
|
}
|
||||||
|
// if opt.AppendToWorkerQueue == nil {
|
||||||
|
// return nil, errors.New("AppendToWorkerQueue option is required")
|
||||||
|
// }
|
||||||
if opt.DataFreeList == nil {
|
if opt.DataFreeList == nil {
|
||||||
return nil, errors.New("DataFreeList option is required")
|
return nil, errors.New("DataFreeList option is required")
|
||||||
}
|
}
|
||||||
@@ -123,9 +133,21 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
|
|||||||
return nil, errors.New("WaitGroup option is required")
|
return nil, errors.New("WaitGroup option is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
indexPageNoProvider, err := NewPageNoProvider(PageNoProviderOptions{
|
||||||
|
FilePath: qb.GetIndexFilePath(opt.Dir, opt.DatabaseName),
|
||||||
|
PageSize: IndexPageSize,
|
||||||
|
FreeList: opt.IndexFreeList,
|
||||||
|
})
|
||||||
|
|
||||||
|
dataPageNoProvider, err := NewPageNoProvider(PageNoProviderOptions{
|
||||||
|
FilePath: qb.GetDataFilePath(opt.Dir, opt.DatabaseName),
|
||||||
|
PageSize: DataPageSize,
|
||||||
|
FreeList: opt.DataFreeList,
|
||||||
|
})
|
||||||
|
|
||||||
pageIngester, err := NewPageIngester(PageIngesterOptions{
|
pageIngester, err := NewPageIngester(PageIngesterOptions{
|
||||||
GetIndexPageNumber: s.getIndexPageNumber,
|
GetIndexPageNumber: indexPageNoProvider.GetPageNumber,
|
||||||
GetDataPageNumber: s.getDataPageNumber,
|
GetDataPageNumber: dataPageNoProvider.GetPageNumber,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -137,35 +159,39 @@ func NewWriter(opt WriterOptions) (*Writer, error) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
s := &Writer{
|
s := &Writer{
|
||||||
|
inbox: opt.Inbox,
|
||||||
|
workerInbox: opt.WorkerInbox,
|
||||||
dir: opt.Dir,
|
dir: opt.Dir,
|
||||||
appendToWorkerQueue: opt.AppendToWorkerQueue,
|
databaseName: opt.DatabaseName,
|
||||||
dataFreeList: opt.DataFreeList,
|
dataFreeList: opt.DataFreeList,
|
||||||
indexFreeList: opt.IndexFreeList,
|
indexFreeList: opt.IndexFreeList,
|
||||||
writePreparer: writePreparer,
|
writePreparer: writePreparer,
|
||||||
//atree: opt.Atree,
|
|
||||||
exitCh: opt.ExitCh,
|
exitCh: opt.ExitCh,
|
||||||
waitGroup: opt.WaitGroup,
|
waitGroup: opt.WaitGroup,
|
||||||
signalCh: make(chan struct{}, 1),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if opt.SnapshotNumber <= 0 {
|
if opt.WAL != "" {
|
||||||
log.Fatalln("FIX FUCK NUMBER")
|
s.wal, err = os.OpenFile(filepath.Join(opt.Dir, opt.WAL), os.O_APPEND|os.O_WRONLY, filePerm)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
s.wal, err = os.OpenFile(
|
s.wal, err = os.OpenFile(
|
||||||
JoinChangesFileName(opt.Dir, opt.SnapshotNumber),
|
qb.GetWALFilePath(opt.Dir, opt.DatabaseName, 0), // .wal_0
|
||||||
os.O_APPEND|os.O_WRONLY,
|
os.O_CREATE|os.O_APPEND|os.O_WRONLY,
|
||||||
filePerm,
|
filePerm,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
}
|
||||||
return s, nil
|
return s, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Writer) Run() {
|
func (s *Writer) Run() {
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-s.signalCh:
|
case <-s.inbox.Ready():
|
||||||
if err := s.packAndWrite(); err != nil {
|
if err := s.packAndWrite(); err != nil {
|
||||||
qb.Abort(qb.FailedWriteToTxLog, err)
|
qb.Abort(qb.FailedWriteToTxLog, err)
|
||||||
}
|
}
|
||||||
@@ -208,16 +234,16 @@ func (s *Writer) getIndexPageNumber() (uint32, bool, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Writer) packAndWrite() (err error) {
|
func (s *Writer) packAndWrite() (err error) {
|
||||||
s.mutex.Lock()
|
//s.mutex.Lock()
|
||||||
//isExited := s.isExited
|
//isExited := s.isExited
|
||||||
input := s.input
|
|
||||||
s.input = nil
|
|
||||||
|
|
||||||
// var exitWaitGroup *sync.WaitGroup
|
// var exitWaitGroup *sync.WaitGroup
|
||||||
// if s.isExited {
|
// if s.isExited {
|
||||||
// exitWaitGroup = s.waitGroup
|
// exitWaitGroup = s.waitGroup
|
||||||
// }
|
// }
|
||||||
s.mutex.Unlock()
|
//s.mutex.Unlock()
|
||||||
|
|
||||||
|
input := s.inbox.Drain()
|
||||||
|
|
||||||
prepared := s.writePreparer.Prepare(input)
|
prepared := s.writePreparer.Prepare(input)
|
||||||
|
|
||||||
@@ -234,7 +260,11 @@ func (s *Writer) packAndWrite() (err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 4. Пишу в atree сторінки
|
// 4. Пишу в atree сторінки
|
||||||
err = s.WritePagesToAtree(prepared.WriteToIndex, prepared.WriteToData)
|
err = WriteDataPages(s.dataFile, prepared.WriteToData)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = WriteIndexPages(s.indexFile, prepared.WriteToIndex)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -262,7 +292,7 @@ func (s *Writer) packAndWrite() (err error) {
|
|||||||
|
|
||||||
snapshotNumberCh := make(chan int, 1)
|
snapshotNumberCh := make(chan int, 1)
|
||||||
|
|
||||||
s.appendToWorkerQueue(Changes{
|
s.workerInbox.Push(Changes{
|
||||||
Commits: prepared.Commits,
|
Commits: prepared.Commits,
|
||||||
SnapshotNumberCh: snapshotNumberCh,
|
SnapshotNumberCh: snapshotNumberCh,
|
||||||
FrozenIndexPagesCount: s.indexFreeList.Pages(),
|
FrozenIndexPagesCount: s.indexFreeList.Pages(),
|
||||||
@@ -285,7 +315,7 @@ func (s *Writer) packAndWrite() (err error) {
|
|||||||
|
|
||||||
var err error
|
var err error
|
||||||
s.wal, err = os.OpenFile(
|
s.wal, err = os.OpenFile(
|
||||||
JoinChangesFileName(s.dir, snapshotNumber),
|
qb.GetWALFilePath(s.dir, s.databaseName, snapshotNumber),
|
||||||
os.O_CREATE|os.O_WRONLY,
|
os.O_CREATE|os.O_WRONLY,
|
||||||
filePerm,
|
filePerm,
|
||||||
)
|
)
|
||||||
@@ -293,24 +323,13 @@ func (s *Writer) packAndWrite() (err error) {
|
|||||||
return fmt.Errorf("create new changes file: %s", err)
|
return fmt.Errorf("create new changes file: %s", err)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
s.appendToWorkerQueue(Changes{
|
s.workerInbox.Push(Changes{
|
||||||
Commits: prepared.Commits,
|
Commits: prepared.Commits,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Writer) Append(req any) {
|
|
||||||
s.mutex.Lock()
|
|
||||||
s.input = append(s.input, req)
|
|
||||||
s.mutex.Unlock()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case s.signalCh <- struct{}{}:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// func (s *Writer) reset() {
|
// func (s *Writer) reset() {
|
||||||
// s.buf.Reset()
|
// s.buf.Reset()
|
||||||
// s.buf.Write([]byte{
|
// s.buf.Write([]byte{
|
||||||
@@ -350,9 +369,8 @@ type PageToWrite struct {
|
|||||||
Content []byte
|
Content []byte
|
||||||
}
|
}
|
||||||
|
|
||||||
// writePagesToAtree - записує сторінки в .data та .index файли
|
func WriteDataPages(file *os.File, pages []PageToWrite) (err error) {
|
||||||
func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToWrite) (err error) {
|
for _, p := range pages {
|
||||||
for _, p := range dataPages {
|
|
||||||
if len(p.Content) != DataPageSize {
|
if len(p.Content) != DataPageSize {
|
||||||
return fmt.Errorf("wrong data page size: %d", len(p.Content))
|
return fmt.Errorf("wrong data page size: %d", len(p.Content))
|
||||||
}
|
}
|
||||||
@@ -360,7 +378,7 @@ func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToW
|
|||||||
off = int(p.PageNo-1) * DataPageSize
|
off = int(p.PageNo-1) * DataPageSize
|
||||||
n int
|
n int
|
||||||
)
|
)
|
||||||
n, err = s.dataFile.WriteAt(p.Content, int64(off))
|
n, err = file.WriteAt(p.Content, int64(off))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -368,7 +386,11 @@ func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToW
|
|||||||
return fmt.Errorf("write %d instead of %d", n, DataPageSize)
|
return fmt.Errorf("write %d instead of %d", n, DataPageSize)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, p := range indexPages {
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func WriteIndexPages(file *os.File, pages []PageToWrite) (err error) {
|
||||||
|
for _, p := range pages {
|
||||||
if len(p.Content) != IndexPageSize {
|
if len(p.Content) != IndexPageSize {
|
||||||
return fmt.Errorf("wrong index page size: %d", len(p.Content))
|
return fmt.Errorf("wrong index page size: %d", len(p.Content))
|
||||||
}
|
}
|
||||||
@@ -376,7 +398,7 @@ func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToW
|
|||||||
off = int(p.PageNo-1) * IndexPageSize
|
off = int(p.PageNo-1) * IndexPageSize
|
||||||
n int
|
n int
|
||||||
)
|
)
|
||||||
n, err = s.indexFile.WriteAt(p.Content, int64(off))
|
n, err = file.WriteAt(p.Content, int64(off))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -384,7 +406,7 @@ func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToW
|
|||||||
return fmt.Errorf("write %d instead of %d", n, IndexPageSize)
|
return fmt.Errorf("write %d instead of %d", n, IndexPageSize)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// API
|
// API
|
||||||
@@ -401,13 +423,6 @@ func (s *Writer) WritePagesToAtree(indexPages []PageToWrite, dataPages []PageToW
|
|||||||
// ValuesBuf []byte
|
// ValuesBuf []byte
|
||||||
// }
|
// }
|
||||||
|
|
||||||
func (s *Writer) sendSignal() {
|
|
||||||
select {
|
|
||||||
case s.signalCh <- struct{}{}:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Якщо (Pages) > 0 - незаповнену сторінку кодую в стисненому вигляді.
|
// Якщо (Pages) > 0 - незаповнену сторінку кодую в стисненому вигляді.
|
||||||
// Якщо (Pages) = 0 - кодую просто пари timestamp / value
|
// Якщо (Pages) = 0 - кодую просто пари timestamp / value
|
||||||
// func (s *Writer) packWriteAppended(w *bytes.Buffer, req AppendedPagesReq) {
|
// func (s *Writer) packWriteAppended(w *bytes.Buffer, req AppendedPagesReq) {
|
||||||
|
|||||||
@@ -140,13 +140,13 @@ func (s *WritePreparer) Prepare(input []any) PreparedData {
|
|||||||
if len(level.IndexPages) > 0 {
|
if len(level.IndexPages) > 0 {
|
||||||
first := level.IndexPages[0]
|
first := level.IndexPages[0]
|
||||||
firstCRC32, _ := bin.GetUint32(first.Content[indexCRC32Idx:])
|
firstCRC32, _ := bin.GetUint32(first.Content[indexCRC32Idx:])
|
||||||
skipSize := level.SkipRecords * indexRecordSize
|
skipSize := level.SkipRecords * IndexRecordSize
|
||||||
indexPageTail = &IndexPageTail{
|
indexPageTail = &IndexPageTail{
|
||||||
PageNo: first.PageNo,
|
PageNo: first.PageNo,
|
||||||
Reused: first.Reused,
|
Reused: first.Reused,
|
||||||
CRC32: firstCRC32,
|
CRC32: firstCRC32,
|
||||||
// в WAL файл попадає лише payload
|
// в WAL файл попадає лише payload
|
||||||
Records: first.Content[skipSize : maxRecordsOnIndexPage*indexRecordSize],
|
Records: first.Content[skipSize : maxRecordsOnIndexPage*IndexRecordSize],
|
||||||
}
|
}
|
||||||
indexPages = level.IndexPages[1:]
|
indexPages = level.IndexPages[1:]
|
||||||
} else {
|
} else {
|
||||||
@@ -156,7 +156,7 @@ func (s *WritePreparer) Prepare(input []any) PreparedData {
|
|||||||
IndexPageTail: indexPageTail,
|
IndexPageTail: indexPageTail,
|
||||||
IndexPages: indexPages,
|
IndexPages: indexPages,
|
||||||
// в WAL файл попадає лише payload
|
// в WAL файл попадає лише payload
|
||||||
TailRecords: level.TailRecords[:level.TailRecordsCount*indexRecordSize],
|
TailRecords: level.TailRecords[:level.TailRecordsCount*IndexRecordSize],
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
// Worker-у відправляю повний index
|
// Worker-у відправляю повний index
|
||||||
@@ -256,7 +256,7 @@ func getIndexRecords(buf []byte, count int) (list []IndexRecord) {
|
|||||||
rec.Timestamp, _ = bin.GetUint32(buf[i:])
|
rec.Timestamp, _ = bin.GetUint32(buf[i:])
|
||||||
rec.PageNo, _ = bin.GetUint32(buf[i+4:])
|
rec.PageNo, _ = bin.GetUint32(buf[i+4:])
|
||||||
list = append(list, rec)
|
list = append(list, rec)
|
||||||
i += indexRecordSize
|
i += IndexRecordSize
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package database
|
package worker
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
bin "gordenko.dev/dima/bin/little"
|
bin "gordenko.dev/dima/bin/little"
|
||||||
@@ -10,18 +11,14 @@ import (
|
|||||||
|
|
||||||
// METRIC
|
// METRIC
|
||||||
|
|
||||||
//const minBufferSize = 1024
|
var ErrNoValueBug = errors.New("has timestamp but no value")
|
||||||
|
|
||||||
var (
|
|
||||||
indexRecordSize = 8
|
|
||||||
)
|
|
||||||
|
|
||||||
type CapturedState struct {
|
type CapturedState struct {
|
||||||
LastTimestamp uint32
|
LastTimestamp uint32
|
||||||
LastValue float64
|
LastValue float64
|
||||||
}
|
}
|
||||||
|
|
||||||
type _metric struct {
|
type Metric struct {
|
||||||
metricType qb.MetricType
|
metricType qb.MetricType
|
||||||
fracDigits byte
|
fracDigits byte
|
||||||
lastPageNo uint32
|
lastPageNo uint32
|
||||||
@@ -39,21 +36,21 @@ type _metric struct {
|
|||||||
capturedState *CapturedState
|
capturedState *CapturedState
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *_metric) LastValue() float64 {
|
func (s *Metric) LastValue() float64 {
|
||||||
if s.capturedState != nil {
|
if s.capturedState != nil {
|
||||||
return s.capturedState.LastValue
|
return s.capturedState.LastValue
|
||||||
}
|
}
|
||||||
return s.lastValue
|
return s.lastValue
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *_metric) LastTimestamp() uint32 {
|
func (s *Metric) LastTimestamp() uint32 {
|
||||||
if s.capturedState != nil {
|
if s.capturedState != nil {
|
||||||
return s.capturedState.LastTimestamp
|
return s.capturedState.LastTimestamp
|
||||||
}
|
}
|
||||||
return s.timestamps.LastTimestamp()
|
return s.timestamps.LastTimestamp()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *_metric) OnMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) {
|
func (s *Metric) OnMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) {
|
||||||
//s.timestamps.Reset()
|
//s.timestamps.Reset()
|
||||||
//s.values.Reset()
|
//s.values.Reset()
|
||||||
s.XLock = false
|
s.XLock = false
|
||||||
@@ -70,7 +67,7 @@ func (s *_metric) OnMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// func (s *_metric) startAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(storage.AppendedMeasures)) {
|
// func (s *_metric) startAppendMeasures(req tryAppendMeasuresReq, sendToStorage func(storage.AppendedMeasures)) {
|
||||||
func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, tmp []byte, saveToStorage func(any)) {
|
func (s *Metric) StartAppendMeasures(req AppendMeasuresReq, tmp []byte, saveToStorage func(any)) {
|
||||||
if s.capturedState != nil {
|
if s.capturedState != nil {
|
||||||
s.WaitQueue = append(s.WaitQueue, req)
|
s.WaitQueue = append(s.WaitQueue, req)
|
||||||
}
|
}
|
||||||
@@ -163,7 +160,7 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, tmp []byte, save
|
|||||||
|
|
||||||
if written == 0 {
|
if written == 0 {
|
||||||
s.capturedState = nil
|
s.capturedState = nil
|
||||||
req.ResultCh <- tryAppendMeasuresResult{
|
req.ResultCh <- AppendMeasuresResult{
|
||||||
ResultCode: resultCode,
|
ResultCode: resultCode,
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
@@ -204,14 +201,14 @@ func (s *_metric) StartAppendMeasures(req tryAppendMeasuresReq, tmp []byte, save
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *_metric) OnMeasuresAppendCommited(rec storage.MeasuresAppendCommited) {
|
func (s *Metric) OnMeasuresAppendCommited(rec storage.MeasuresAppendCommited) {
|
||||||
// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
|
// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
|
||||||
s.capturedState = nil
|
s.capturedState = nil
|
||||||
s.timestamps.ForgetCapturedState()
|
s.timestamps.ForgetCapturedState()
|
||||||
s.values.ForgetCapturedState()
|
s.values.ForgetCapturedState()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *_metric) OnMeasuresAppendWithGrowCommited(rec storage.MeasuresAppendWithGrowCommited) {
|
func (s *Metric) OnMeasuresAppendWithGrowCommited(rec storage.MeasuresAppendWithGrowCommited) {
|
||||||
// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
|
// Видаляю state. Оригінальні Timestamps і Values вже мають останню версію
|
||||||
s.capturedState = nil
|
s.capturedState = nil
|
||||||
s.timestamps.ForgetCapturedState()
|
s.timestamps.ForgetCapturedState()
|
||||||
@@ -227,7 +224,7 @@ func (s *_metric) OnMeasuresAppendWithGrowCommited(rec storage.MeasuresAppendWit
|
|||||||
|
|
||||||
// READ
|
// READ
|
||||||
|
|
||||||
func (s *_metric) StartRangeScan(req tryRangeScanReq) {
|
func (s *Metric) StartRangeScan(req RangeScanReq) {
|
||||||
// if s.Since == 0 {
|
// if s.Since == 0 {
|
||||||
// req.ResultCh <- rangeScanResult{
|
// req.ResultCh <- rangeScanResult{
|
||||||
// ResultCode: QueryDone,
|
// ResultCode: QueryDone,
|
||||||
@@ -295,7 +292,7 @@ func (s *_metric) StartRangeScan(req tryRangeScanReq) {
|
|||||||
// }
|
// }
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *_metric) StartFullScan(req tryFullScanReq) {
|
func (s *Metric) StartFullScan(req FullScanReq) {
|
||||||
// if s.Since == 0 {
|
// if s.Since == 0 {
|
||||||
// req.ResultCh <- fullScanResult{
|
// req.ResultCh <- fullScanResult{
|
||||||
// ResultCode: QueryDone,
|
// ResultCode: QueryDone,
|
||||||
@@ -319,14 +316,14 @@ func (s *_metric) StartFullScan(req tryFullScanReq) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if s.lastPageNo > 0 {
|
if s.lastPageNo > 0 {
|
||||||
req.ResultCh <- fullScanResult{
|
req.ResultCh <- FullScanResult{
|
||||||
ResultCode: UntilFound,
|
ResultCode: UntilFound,
|
||||||
LastPageNo: s.lastPageNo,
|
LastPageNo: s.lastPageNo,
|
||||||
FracDigits: s.fracDigits,
|
FracDigits: s.fracDigits,
|
||||||
}
|
}
|
||||||
s.RLocks++
|
s.RLocks++
|
||||||
} else {
|
} else {
|
||||||
req.ResultCh <- fullScanResult{
|
req.ResultCh <- FullScanResult{
|
||||||
ResultCode: QueryDone,
|
ResultCode: QueryDone,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -348,7 +345,7 @@ func (s *_metric) StartFullScan(req tryFullScanReq) {
|
|||||||
// records - Nb
|
// records - Nb
|
||||||
// ]
|
// ]
|
||||||
|
|
||||||
func (s *_metric) WriteTo(w io.Writer) (err error) {
|
func (s *Metric) WriteTo(w io.Writer) (err error) {
|
||||||
_, err = w.Write([]byte{
|
_, err = w.Write([]byte{
|
||||||
byte(s.metricType),
|
byte(s.metricType),
|
||||||
s.fracDigits,
|
s.fracDigits,
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package database
|
package worker
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
qb "gordenko.dev/dima/qb"
|
qb "gordenko.dev/dima/qb"
|
||||||
"gordenko.dev/dima/qb/atree"
|
"gordenko.dev/dima/qb/atree"
|
||||||
"gordenko.dev/dima/qb/enc"
|
"gordenko.dev/dima/qb/enc"
|
||||||
|
"gordenko.dev/dima/qb/inbox"
|
||||||
"gordenko.dev/dima/qb/proto"
|
"gordenko.dev/dima/qb/proto"
|
||||||
"gordenko.dev/dima/qb/storage"
|
"gordenko.dev/dima/qb/storage"
|
||||||
"gordenko.dev/dima/qb/transform"
|
"gordenko.dev/dima/qb/transform"
|
||||||
@@ -37,21 +38,22 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type Worker struct {
|
type Worker struct {
|
||||||
mutex sync.Mutex
|
//mutex sync.Mutex
|
||||||
queue []any
|
inbox *inbox.Inbox
|
||||||
signalCh chan struct{}
|
storageInbox *inbox.Inbox
|
||||||
dir string
|
dir string
|
||||||
snapshotNumber int
|
snapshotNumber int
|
||||||
tmp []byte
|
tmp []byte
|
||||||
metrics map[uint32]*_metric
|
metrics map[uint32]*Metric
|
||||||
saveToStorage func(any)
|
saveToStorage func(any)
|
||||||
exitCh chan struct{}
|
exitCh chan struct{}
|
||||||
waitGroup *sync.WaitGroup
|
waitGroup *sync.WaitGroup
|
||||||
}
|
}
|
||||||
|
|
||||||
type WorkerOptions struct {
|
type WorkerOptions struct {
|
||||||
SaveToStorage func(any)
|
Inbox *inbox.Inbox
|
||||||
Metrics map[uint32]*_metric
|
StorageInbox *inbox.Inbox
|
||||||
|
Metrics map[uint32]*Metric
|
||||||
Dir string
|
Dir string
|
||||||
ExitCh chan struct{}
|
ExitCh chan struct{}
|
||||||
WaitGroup *sync.WaitGroup
|
WaitGroup *sync.WaitGroup
|
||||||
@@ -59,40 +61,31 @@ type WorkerOptions struct {
|
|||||||
|
|
||||||
func NewWorker(opt WorkerOptions) *Worker {
|
func NewWorker(opt WorkerOptions) *Worker {
|
||||||
return &Worker{
|
return &Worker{
|
||||||
signalCh: make(chan struct{}, 1),
|
inbox: opt.Inbox,
|
||||||
|
storageInbox: opt.StorageInbox,
|
||||||
dir: opt.Dir,
|
dir: opt.Dir,
|
||||||
tmp: make([]byte, 26),
|
tmp: make([]byte, 26),
|
||||||
metrics: opt.Metrics,
|
metrics: opt.Metrics,
|
||||||
saveToStorage: opt.SaveToStorage,
|
|
||||||
exitCh: opt.ExitCh,
|
exitCh: opt.ExitCh,
|
||||||
waitGroup: opt.WaitGroup,
|
waitGroup: opt.WaitGroup,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Worker) AddJobToQueue(job any) {
|
|
||||||
s.queue = append(s.queue, job)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case s.signalCh <- struct{}{}:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Worker) ReleaseRLock(metricID uint32) {
|
func (s *Worker) ReleaseRLock(metricID uint32) {
|
||||||
s.mutex.Lock()
|
// s.mutex.Lock()
|
||||||
//s.rLocksToRelease = append(s.rLocksToRelease, metricID)
|
// //s.rLocksToRelease = append(s.rLocksToRelease, metricID)
|
||||||
s.mutex.Unlock()
|
// s.mutex.Unlock()
|
||||||
|
|
||||||
select {
|
// select {
|
||||||
case s.signalCh <- struct{}{}:
|
// case s.signalCh <- struct{}{}:
|
||||||
default:
|
// default:
|
||||||
}
|
// }
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Worker) Run() {
|
func (s *Worker) Run() {
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-s.signalCh:
|
case <-s.inbox.Ready():
|
||||||
s.doWork()
|
s.doWork()
|
||||||
|
|
||||||
case <-s.exitCh:
|
case <-s.exitCh:
|
||||||
@@ -104,13 +97,10 @@ func (s *Worker) Run() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Worker) doWork() {
|
func (s *Worker) doWork() {
|
||||||
|
queue := s.inbox.Drain()
|
||||||
|
|
||||||
s.mutex.Lock()
|
|
||||||
//rLocksToRelease := s.rLocksToRelease
|
//rLocksToRelease := s.rLocksToRelease
|
||||||
queue := s.queue
|
|
||||||
//s.rLocksToRelease = nil
|
//s.rLocksToRelease = nil
|
||||||
s.queue = nil
|
|
||||||
s.mutex.Unlock()
|
|
||||||
|
|
||||||
// for _, metricID := range rLocksToRelease {
|
// for _, metricID := range rLocksToRelease {
|
||||||
// metric, ok := s.metrics[metricID]
|
// metric, ok := s.metrics[metricID]
|
||||||
@@ -140,32 +130,32 @@ func (s *Worker) doWork() {
|
|||||||
|
|
||||||
for _, untyped := range queue {
|
for _, untyped := range queue {
|
||||||
switch req := untyped.(type) {
|
switch req := untyped.(type) {
|
||||||
case tryAppendMeasuresReq:
|
case AppendMeasuresReq:
|
||||||
s.tryAppendMeasures(req)
|
s.AppendMeasures(req)
|
||||||
|
|
||||||
case storage.Changes:
|
case storage.Changes:
|
||||||
s.applyCommits(req) // all metrics only
|
s.applyCommits(req) // all metrics only
|
||||||
|
|
||||||
case tryListCurrentValuesReq:
|
case ListCurrentValuesReq:
|
||||||
s.tryListCurrentValues(req) // all metrics only
|
s.tryListCurrentValues(req) // all metrics only
|
||||||
|
|
||||||
case tryRangeScanReq:
|
case RangeScanReq:
|
||||||
s.tryRangeScan(req)
|
s.RangeScan(req)
|
||||||
|
|
||||||
case tryFullScanReq:
|
case FullScanReq:
|
||||||
s.tryFullScan(req)
|
s.tryFullScan(req)
|
||||||
|
|
||||||
case tryAddMetricReq:
|
case AddMetricReq:
|
||||||
s.tryAddMetric(req)
|
s.AddMetric(req)
|
||||||
|
|
||||||
case tryDeleteMetricReq:
|
case DeleteMetricReq:
|
||||||
s.tryDeleteMetric(req)
|
s.DeleteMetric(req)
|
||||||
|
|
||||||
case tryDeleteMeasuresReq:
|
case DeleteMeasuresReq:
|
||||||
s.tryDeleteMeasures(req)
|
s.DeleteMeasures(req)
|
||||||
|
|
||||||
case tryGetMetricReq:
|
case GetMetricReq:
|
||||||
s.tryGetMetric(req)
|
s.GetMetric(req)
|
||||||
|
|
||||||
default:
|
default:
|
||||||
qb.Abort(qb.UnknownWorkerQueueItemBug,
|
qb.Abort(qb.UnknownWorkerQueueItemBug,
|
||||||
@@ -175,29 +165,29 @@ func (s *Worker) doWork() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// суть у тому що треба запускати запити, пока не зустріну XLock
|
// суть у тому що треба запускати запити, пока не зустріну XLock
|
||||||
func (s *Worker) processMetricQueue(metricID uint32, metric *_metric, tmp []byte) {
|
func (s *Worker) processMetricQueue(metricID uint32, metric *Metric, tmp []byte) {
|
||||||
if len(metric.WaitQueue) == 0 {
|
if len(metric.WaitQueue) == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, untyped := range metric.WaitQueue {
|
for _, untyped := range metric.WaitQueue {
|
||||||
switch req := untyped.(type) {
|
switch req := untyped.(type) {
|
||||||
case tryRangeScanReq:
|
case RangeScanReq:
|
||||||
metric.StartRangeScan(req)
|
metric.StartRangeScan(req)
|
||||||
|
|
||||||
case tryFullScanReq:
|
case FullScanReq:
|
||||||
metric.StartFullScan(req)
|
metric.StartFullScan(req)
|
||||||
|
|
||||||
case tryGetMetricReq:
|
case GetMetricReq:
|
||||||
s.tryGetMetric(req)
|
s.GetMetric(req)
|
||||||
|
|
||||||
case tryAppendMeasuresReq:
|
case AppendMeasuresReq:
|
||||||
metric.StartAppendMeasures(req, tmp, s.saveToStorage)
|
metric.StartAppendMeasures(req, tmp, s.saveToStorage)
|
||||||
|
|
||||||
case tryDeleteMetricReq:
|
case DeleteMetricReq:
|
||||||
s.startDeleteMetric(metric, req)
|
s.startDeleteMetric(metric, req)
|
||||||
|
|
||||||
case tryDeleteMeasuresReq:
|
case DeleteMeasuresReq:
|
||||||
s.startDeleteMeasures(metric, req)
|
s.startDeleteMeasures(metric, req)
|
||||||
|
|
||||||
default:
|
default:
|
||||||
@@ -207,12 +197,12 @@ func (s *Worker) processMetricQueue(metricID uint32, metric *_metric, tmp []byte
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type tryAddMetricReq struct {
|
type AddMetricReq struct {
|
||||||
MetricID uint32
|
MetricID uint32
|
||||||
ResultCh chan byte
|
ResultCh chan byte
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Worker) tryAddMetric(req tryAddMetricReq) {
|
func (s *Worker) AddMetric(req AddMetricReq) {
|
||||||
_, ok := s.metrics[req.MetricID]
|
_, ok := s.metrics[req.MetricID]
|
||||||
if ok {
|
if ok {
|
||||||
req.ResultCh <- MetricDuplicate
|
req.ResultCh <- MetricDuplicate
|
||||||
@@ -232,7 +222,7 @@ func (s *Worker) tryAddMetric(req tryAddMetricReq) {
|
|||||||
// }
|
// }
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Worker) processTryAddMetricReqsImmediatelyAfterDelete(reqs []tryAddMetricReq) {
|
func (s *Worker) processTryAddMetricReqsImmediatelyAfterDelete(reqs []AddMetricReq) {
|
||||||
if len(reqs) == 0 {
|
if len(reqs) == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -253,32 +243,43 @@ func (s *Worker) processTryAddMetricReqsImmediatelyAfterDelete(reqs []tryAddMetr
|
|||||||
req.ResultCh <- Succeed
|
req.ResultCh <- Succeed
|
||||||
}
|
}
|
||||||
|
|
||||||
type tryGetMetricReq struct {
|
type GetMetricResult struct {
|
||||||
MetricID uint32
|
MetricType qb.MetricType
|
||||||
ResultCh chan Metric
|
FracDigits byte
|
||||||
|
ResultCode byte
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Worker) tryGetMetric(req tryGetMetricReq) {
|
type GetMetricReq struct {
|
||||||
|
MetricID uint32
|
||||||
|
ResultCh chan GetMetricResult
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Worker) GetMetric(req GetMetricReq) {
|
||||||
metric, ok := s.metrics[req.MetricID]
|
metric, ok := s.metrics[req.MetricID]
|
||||||
if ok {
|
if ok {
|
||||||
req.ResultCh <- Metric{
|
req.ResultCh <- GetMetricResult{
|
||||||
ResultCode: Succeed,
|
ResultCode: Succeed,
|
||||||
MetricType: metric.metricType,
|
MetricType: metric.metricType,
|
||||||
FracDigits: metric.fracDigits,
|
FracDigits: metric.fracDigits,
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
req.ResultCh <- Metric{
|
req.ResultCh <- GetMetricResult{
|
||||||
ResultCode: NoMetric,
|
ResultCode: NoMetric,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type tryDeleteMetricReq struct {
|
type DeleteMetricResult struct {
|
||||||
MetricID uint32
|
ResultCode byte
|
||||||
ResultCh chan tryDeleteMetricResult
|
RootPageNo uint32
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Worker) tryDeleteMetric(req tryDeleteMetricReq) {
|
type DeleteMetricReq struct {
|
||||||
|
MetricID uint32
|
||||||
|
ResultCh chan DeleteMetricResult
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Worker) DeleteMetric(req DeleteMetricReq) {
|
||||||
// FIX
|
// FIX
|
||||||
// metric, ok := s.metrics[req.MetricID]
|
// metric, ok := s.metrics[req.MetricID]
|
||||||
// if !ok {
|
// if !ok {
|
||||||
@@ -296,7 +297,7 @@ func (s *Worker) tryDeleteMetric(req tryDeleteMetricReq) {
|
|||||||
// }
|
// }
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Worker) startDeleteMetric(metric *_metric, req tryDeleteMetricReq) {
|
func (s *Worker) startDeleteMetric(metric *Metric, req DeleteMetricReq) {
|
||||||
// FIX
|
// FIX
|
||||||
// s.metricLockEntries[req.MetricID] = &metricLockEntry{
|
// s.metricLockEntries[req.MetricID] = &metricLockEntry{
|
||||||
// XLock: true,
|
// XLock: true,
|
||||||
@@ -307,13 +308,18 @@ func (s *Worker) startDeleteMetric(metric *_metric, req tryDeleteMetricReq) {
|
|||||||
// }
|
// }
|
||||||
}
|
}
|
||||||
|
|
||||||
type tryDeleteMeasuresReq struct {
|
type DeleteMeasuresResult struct {
|
||||||
MetricID uint32
|
ResultCode byte
|
||||||
Since uint32
|
RootPageNo uint32
|
||||||
ResultCh chan tryDeleteMeasuresResult
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Worker) tryDeleteMeasures(req tryDeleteMeasuresReq) {
|
type DeleteMeasuresReq struct {
|
||||||
|
MetricID uint32
|
||||||
|
Since uint32
|
||||||
|
ResultCh chan DeleteMeasuresResult
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Worker) DeleteMeasures(req DeleteMeasuresReq) {
|
||||||
// FIX
|
// FIX
|
||||||
// metric, ok := s.metrics[req.MetricID]
|
// metric, ok := s.metrics[req.MetricID]
|
||||||
// if !ok {
|
// if !ok {
|
||||||
@@ -337,7 +343,7 @@ func (s *Worker) tryDeleteMeasures(req tryDeleteMeasuresReq) {
|
|||||||
// }
|
// }
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Worker) startDeleteMeasures(metric *_metric, req tryDeleteMeasuresReq) {
|
func (s *Worker) startDeleteMeasures(metric *Metric, req DeleteMeasuresReq) {
|
||||||
// FIX
|
// FIX
|
||||||
// s.metricLockEntries[req.MetricID] = &metricLockEntry{
|
// s.metricLockEntries[req.MetricID] = &metricLockEntry{
|
||||||
// XLock: true,
|
// XLock: true,
|
||||||
@@ -375,16 +381,21 @@ func (s *Worker) onMeasuresAppendWithGrowCommited(rec storage.MeasuresAppendWith
|
|||||||
metric.OnMeasuresAppendWithGrowCommited(rec)
|
metric.OnMeasuresAppendWithGrowCommited(rec)
|
||||||
}
|
}
|
||||||
|
|
||||||
type tryAppendMeasuresReq struct {
|
type AppendMeasuresResult struct {
|
||||||
MetricID uint32
|
ResultCode byte
|
||||||
Measures []proto.Measure
|
Written int
|
||||||
ResultCh chan tryAppendMeasuresResult
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Worker) tryAppendMeasures(req tryAppendMeasuresReq) {
|
type AppendMeasuresReq struct {
|
||||||
|
MetricID uint32
|
||||||
|
Measures []proto.Measure
|
||||||
|
ResultCh chan AppendMeasuresResult
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Worker) AppendMeasures(req AppendMeasuresReq) {
|
||||||
metric, ok := s.metrics[req.MetricID]
|
metric, ok := s.metrics[req.MetricID]
|
||||||
if !ok {
|
if !ok {
|
||||||
req.ResultCh <- tryAppendMeasuresResult{
|
req.ResultCh <- AppendMeasuresResult{
|
||||||
ResultCode: NoMetric,
|
ResultCode: NoMetric,
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
@@ -396,25 +407,32 @@ func (s *Worker) tryAppendMeasures(req tryAppendMeasuresReq) {
|
|||||||
metric.StartAppendMeasures(req, s.tmp, s.saveToStorage)
|
metric.StartAppendMeasures(req, s.tmp, s.saveToStorage)
|
||||||
}
|
}
|
||||||
|
|
||||||
type tryRangeScanReq struct {
|
type RangeScanResult struct {
|
||||||
|
ResultCode byte
|
||||||
|
FracDigits byte
|
||||||
|
RootPageNo uint32
|
||||||
|
LastPageNo uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
type RangeScanReq struct {
|
||||||
MetricID uint32
|
MetricID uint32
|
||||||
Since uint32
|
Since uint32
|
||||||
Until uint32
|
Until uint32
|
||||||
MetricType qb.MetricType
|
MetricType qb.MetricType
|
||||||
ResponseWriter atree.WorkerMeasureConsumer
|
ResponseWriter atree.WorkerMeasureConsumer
|
||||||
ResultCh chan rangeScanResult
|
ResultCh chan RangeScanResult
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Worker) tryRangeScan(req tryRangeScanReq) {
|
func (s *Worker) RangeScan(req RangeScanReq) {
|
||||||
metric, ok := s.metrics[req.MetricID]
|
metric, ok := s.metrics[req.MetricID]
|
||||||
if !ok {
|
if !ok {
|
||||||
req.ResultCh <- rangeScanResult{
|
req.ResultCh <- RangeScanResult{
|
||||||
ResultCode: NoMetric,
|
ResultCode: NoMetric,
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if metric.metricType != req.MetricType {
|
if metric.metricType != req.MetricType {
|
||||||
req.ResultCh <- rangeScanResult{
|
req.ResultCh <- RangeScanResult{
|
||||||
ResultCode: WrongMetricType,
|
ResultCode: WrongMetricType,
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
@@ -428,23 +446,29 @@ func (s *Worker) tryRangeScan(req tryRangeScanReq) {
|
|||||||
metric.StartRangeScan(req)
|
metric.StartRangeScan(req)
|
||||||
}
|
}
|
||||||
|
|
||||||
type tryFullScanReq struct {
|
type FullScanResult struct {
|
||||||
|
ResultCode byte
|
||||||
|
FracDigits byte
|
||||||
|
LastPageNo uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
type FullScanReq struct {
|
||||||
MetricID uint32
|
MetricID uint32
|
||||||
MetricType qb.MetricType
|
MetricType qb.MetricType
|
||||||
ResponseWriter atree.WorkerMeasureConsumer
|
ResponseWriter atree.WorkerMeasureConsumer
|
||||||
ResultCh chan fullScanResult
|
ResultCh chan FullScanResult
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Worker) tryFullScan(req tryFullScanReq) {
|
func (s *Worker) tryFullScan(req FullScanReq) {
|
||||||
metric, ok := s.metrics[req.MetricID]
|
metric, ok := s.metrics[req.MetricID]
|
||||||
if !ok {
|
if !ok {
|
||||||
req.ResultCh <- fullScanResult{
|
req.ResultCh <- FullScanResult{
|
||||||
ResultCode: NoMetric,
|
ResultCode: NoMetric,
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if metric.metricType != req.MetricType {
|
if metric.metricType != req.MetricType {
|
||||||
req.ResultCh <- fullScanResult{
|
req.ResultCh <- FullScanResult{
|
||||||
ResultCode: WrongMetricType,
|
ResultCode: WrongMetricType,
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
@@ -457,13 +481,13 @@ func (s *Worker) tryFullScan(req tryFullScanReq) {
|
|||||||
metric.StartFullScan(req)
|
metric.StartFullScan(req)
|
||||||
}
|
}
|
||||||
|
|
||||||
type tryListCurrentValuesReq struct {
|
type ListCurrentValuesReq struct {
|
||||||
MetricIDs []uint32
|
MetricIDs []uint32
|
||||||
ResponseWriter *transform.CurrentValueWriter
|
ResponseWriter *transform.CurrentValueWriter
|
||||||
ResultCh chan struct{}
|
ResultCh chan struct{}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Worker) tryListCurrentValues(req tryListCurrentValuesReq) {
|
func (s *Worker) tryListCurrentValues(req ListCurrentValuesReq) {
|
||||||
for _, metricID := range req.MetricIDs {
|
for _, metricID := range req.MetricIDs {
|
||||||
metric, ok := s.metrics[metricID]
|
metric, ok := s.metrics[metricID]
|
||||||
if ok {
|
if ok {
|
||||||
@@ -501,7 +525,7 @@ func (s *Worker) applyCommits(req storage.Changes) {
|
|||||||
|
|
||||||
if req.SnapshotNumberCh != nil {
|
if req.SnapshotNumberCh != nil {
|
||||||
s.snapshotNumber++
|
s.snapshotNumber++
|
||||||
err := writeSnapshot(writeSnapshotIn{
|
err := storage.WriteSnapshot(storage.WriteSnapshotIn{
|
||||||
SnapshotNumber: s.snapshotNumber,
|
SnapshotNumber: s.snapshotNumber,
|
||||||
Dir: s.dir,
|
Dir: s.dir,
|
||||||
WriteBufferSize: 4 * 1024 * 1024, // 1mb
|
WriteBufferSize: 4 * 1024 * 1024, // 1mb
|
||||||
@@ -543,7 +567,7 @@ func (s *Worker) addMetric(rec storage.MetricAddRecord) {
|
|||||||
buf = make([]byte, storage.DataPageSize)
|
buf = make([]byte, storage.DataPageSize)
|
||||||
databuf = buf[:storage.DataPagePayloadSize]
|
databuf = buf[:storage.DataPagePayloadSize]
|
||||||
)
|
)
|
||||||
s.metrics[rec.MetricID] = &_metric{
|
s.metrics[rec.MetricID] = &Metric{
|
||||||
metricType: rec.MetricType,
|
metricType: rec.MetricType,
|
||||||
fracDigits: rec.FracDigits,
|
fracDigits: rec.FracDigits,
|
||||||
buffer: buf,
|
buffer: buf,
|
||||||
@@ -566,7 +590,7 @@ func (s *Worker) onMetricDeleteCommited(rec storage.MetricDeleteCommited) {
|
|||||||
rec.MetricID))
|
rec.MetricID))
|
||||||
}
|
}
|
||||||
|
|
||||||
var addMetricReqs []tryAddMetricReq
|
var addMetricReqs []AddMetricReq
|
||||||
|
|
||||||
if len(metric.WaitQueue) > 0 {
|
if len(metric.WaitQueue) > 0 {
|
||||||
for _, untyped := range metric.WaitQueue {
|
for _, untyped := range metric.WaitQueue {
|
||||||
@@ -577,31 +601,31 @@ func (s *Worker) onMetricDeleteCommited(rec storage.MetricDeleteCommited) {
|
|||||||
// ResultCode: NoMetric,
|
// ResultCode: NoMetric,
|
||||||
// }
|
// }
|
||||||
|
|
||||||
case tryRangeScanReq:
|
case RangeScanReq:
|
||||||
req.ResultCh <- rangeScanResult{
|
req.ResultCh <- RangeScanResult{
|
||||||
ResultCode: NoMetric,
|
ResultCode: NoMetric,
|
||||||
}
|
}
|
||||||
|
|
||||||
case tryFullScanReq:
|
case FullScanReq:
|
||||||
req.ResultCh <- fullScanResult{
|
req.ResultCh <- FullScanResult{
|
||||||
ResultCode: NoMetric,
|
ResultCode: NoMetric,
|
||||||
}
|
}
|
||||||
|
|
||||||
case tryAddMetricReq:
|
case AddMetricReq:
|
||||||
addMetricReqs = append(addMetricReqs, req)
|
addMetricReqs = append(addMetricReqs, req)
|
||||||
|
|
||||||
case tryDeleteMetricReq:
|
case DeleteMetricReq:
|
||||||
req.ResultCh <- tryDeleteMetricResult{
|
req.ResultCh <- DeleteMetricResult{
|
||||||
ResultCode: NoMetric,
|
ResultCode: NoMetric,
|
||||||
}
|
}
|
||||||
|
|
||||||
case tryDeleteMeasuresReq:
|
case DeleteMeasuresReq:
|
||||||
req.ResultCh <- tryDeleteMeasuresResult{
|
req.ResultCh <- DeleteMeasuresResult{
|
||||||
ResultCode: NoMetric,
|
ResultCode: NoMetric,
|
||||||
}
|
}
|
||||||
|
|
||||||
case tryGetMetricReq:
|
case GetMetricReq:
|
||||||
req.ResultCh <- Metric{
|
req.ResultCh <- GetMetricResult{
|
||||||
ResultCode: NoMetric,
|
ResultCode: NoMetric,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -650,89 +674,3 @@ func (s *Worker) onMeasuresDeleteCommited(rec storage.MeasuresDeleteCommited) {
|
|||||||
// s.processMetricQueue(metricID, metric)
|
// s.processMetricQueue(metricID, metric)
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
|
|
||||||
func (s *Worker) recovery() {
|
|
||||||
// FIX
|
|
||||||
// advisor, err := recovery.NewRecoveryAdvisor(recovery.RecoveryAdvisorOptions{
|
|
||||||
// Dir: s.dir,
|
|
||||||
// VerifySnapshot: s.verifySnapshot,
|
|
||||||
// })
|
|
||||||
// if err != nil {
|
|
||||||
// panic(err)
|
|
||||||
// }
|
|
||||||
|
|
||||||
// recipe, err := advisor.GetRecipe()
|
|
||||||
// if err != nil {
|
|
||||||
// qb.Abort(qb.GetRecoveryRecipeFailed, err)
|
|
||||||
// }
|
|
||||||
|
|
||||||
// var logNumber int
|
|
||||||
|
|
||||||
// if recipe != nil {
|
|
||||||
// if recipe.Snapshot != "" {
|
|
||||||
// err = s.loadSnapshot(recipe.Snapshot)
|
|
||||||
// if err != nil {
|
|
||||||
// qb.Abort(qb.LoadSnapshotFailed, err)
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// for _, changesFileName := range recipe.Changes {
|
|
||||||
// err = s.replayChanges(changesFileName)
|
|
||||||
// if err != nil {
|
|
||||||
// qb.Abort(qb.ReplayChangesFailed, err)
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// logNumber = recipe.LogNumber
|
|
||||||
// }
|
|
||||||
|
|
||||||
// s.storage, err = storage.NewWriter(storage.WriterOptions{
|
|
||||||
// Dir: s.dir,
|
|
||||||
// LogNumber: logNumber,
|
|
||||||
// AppendToWorkerQueue: s.appendJobToWorkerQueue,
|
|
||||||
// FreeList: s.freeList,
|
|
||||||
// Atree: s.atree,
|
|
||||||
// ExitCh: s.exitCh,
|
|
||||||
// WaitGroup: s.waitGroup,
|
|
||||||
// })
|
|
||||||
// if err != nil {
|
|
||||||
// qb.Abort(qb.CreateChangesWriterFailed, err)
|
|
||||||
|
|
||||||
// }
|
|
||||||
// go s.storage.Run()
|
|
||||||
|
|
||||||
// // fileNames, err := s.searchREDOFiles()
|
|
||||||
// // if err != nil {
|
|
||||||
// // qb.Abort(qb.SearchREDOFilesFailed, err)
|
|
||||||
// // }
|
|
||||||
|
|
||||||
// // if len(fileNames) > 0 {
|
|
||||||
// // for _, fileName := range fileNames {
|
|
||||||
// // err = s.replayREDOFile(fileName)
|
|
||||||
// // if err != nil {
|
|
||||||
// // qb.Abort(qb.ReplayREDOFileFailed, err)
|
|
||||||
// // }
|
|
||||||
// // }
|
|
||||||
|
|
||||||
// // for _, fileName := range fileNames {
|
|
||||||
// // err = os.Remove(fileName)
|
|
||||||
// // if err != nil {
|
|
||||||
// // qb.Abort(qb.RemoveREDOFileFailed, err)
|
|
||||||
// // }
|
|
||||||
// // }
|
|
||||||
// // }
|
|
||||||
|
|
||||||
// if recipe != nil {
|
|
||||||
// if recipe.CompleteSnapshot {
|
|
||||||
// err = s.dumpSnapshot(logNumber)
|
|
||||||
// if err != nil {
|
|
||||||
// qb.Abort(qb.DumpSnapshotFailed, err)
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// for _, fileName := range recipe.ToDelete {
|
|
||||||
// err = os.Remove(fileName)
|
|
||||||
// if err != nil {
|
|
||||||
// qb.Abort(qb.RemoveRecipeFileFailed, err)
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user