dima/diploma
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

rc2

Browse Source
This commit is contained in:
Dima Gordenko
2025-06-22 18:13:59 +03:00
parent 37893fc805 14f1531276
commit c97003bc4f
29 changed files with 1608 additions and 2860 deletions
Download Patch File Download Diff File Expand all files Collapse all files

480
database/api.go
View File

@@ -13,6 +13,7 @@ import (
"gordenko.dev/dima/diploma/chunkenc"
"gordenko.dev/dima/diploma/conbuf"
"gordenko.dev/dima/diploma/proto"
"gordenko.dev/dima/diploma/transform"
"gordenko.dev/dima/diploma/txlog"
)
@@ -326,6 +327,9 @@ type FilledPage struct {
}
type tryAppendMeasureResult struct {
MetricID uint32
Timestamp uint32
Value float64
FilledPage *FilledPage
ResultCode byte
}
@@ -441,7 +445,6 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
)
for idx, measure := range req.Measures {
//fmt.Printf("%d %v\n", measure.Timestamp, measure.Value)
if since == 0 {
since = measure.Timestamp
} else {
@@ -470,7 +473,6 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
)
<-waitCh
}
//fmt.Printf("m.Value: %v < untilValue: %v\n", measure.Value, untilValue)
return proto.ErrNonMonotonicValue
}
}
@@ -497,10 +499,11 @@ func (s *Database) AppendMeasures(req proto.AppendMeasuresReq) uint16 {
toAppendMeasures = nil
}
//fmt.Printf("APPEND DATA PAGE %d, %v\n", measure.Timestamp, measure.Value)
report, err := s.atree.AppendDataPage(atree.AppendDataPageReq{
MetricID: req.MetricID,
Timestamp: until,
Value: untilValue,
Timestamp: measure.Timestamp,
Value: measure.Value,
Since: since,
RootPageNo: rootPageNo,
PrevPageNo: prevPageNo,
@@ -593,23 +596,27 @@ func (s *Database) DeleteMeasures(req proto.DeleteMeasuresReq) uint16 {
result := <-resultCh
switch result.ResultCode {
case Succeed:
var (
freeDataPages []uint32
freeIndexPages []uint32
)
if result.RootPageNo > 0 {
pageLists, err := s.atree.GetAllPages(result.RootPageNo)
if err != nil {
diploma.Abort(diploma.FailedAtreeRequest, err)
}
freeDataPages = pageLists.DataPages
freeIndexPages = pageLists.IndexPages
case NoMeasuresToDelete:
// ok
case DeleteFromAtreeNotNeeded:
// регистрирую удаление в TransactionLog
waitCh := s.txlog.WriteDeletedMeasures(txlog.DeletedMeasures{
MetricID: req.MetricID,
})
<-waitCh
case DeleteFromAtreeRequired:
// собираю номера всех data и index страниц метрики (типа запись REDO лога).
pageLists, err := s.atree.GetAllPages(result.RootPageNo)
if err != nil {
diploma.Abort(diploma.FailedAtreeRequest, err)
}
// регистрирую удаление в TransactionLog
waitCh := s.txlog.WriteDeletedMeasures(txlog.DeletedMeasures{
MetricID: req.MetricID,
FreeDataPages: freeDataPages,
FreeIndexPages: freeIndexPages,
FreeDataPages: pageLists.DataPages,
FreeIndexPages: pageLists.IndexPages,
})
<-waitCh
@@ -624,98 +631,32 @@ func (s *Database) DeleteMeasures(req proto.DeleteMeasuresReq) uint16 {
// SELECT
type instantMeasuresResult struct {
type fullScanResult struct {
ResultCode byte
FracDigits byte
PageNo uint32
LastPageNo uint32
}
func (s *Database) ListAllInstantMeasures(conn net.Conn, req proto.ListAllInstantMetricMeasuresReq) error {
resultCh := make(chan instantMeasuresResult, 1)
responseWriter := transform.NewInstantMeasureWriter(conn, 0)
responseWriter := atree.NewInstantMeasureWriter(conn)
s.appendJobToWorkerQueue(tryListAllInstantMeasuresReq{
return s.fullScan(fullScanReq{
MetricID: req.MetricID,
MetricType: diploma.Instant,
Conn: conn,
ResponseWriter: responseWriter,
ResultCh: resultCh,
})
result := <-resultCh
switch result.ResultCode {
case QueryDone:
responseWriter.Close()
case UntilFound:
err := s.atree.IterateAllInstantByTreeCursor(atree.IterateAllInstantByTreeCursorReq{
FracDigits: result.FracDigits,
PageNo: result.PageNo,
ResponseWriter: responseWriter,
})
s.metricRUnlock(req.MetricID)
if err != nil {
reply(conn, proto.ErrUnexpected)
} else {
responseWriter.Close()
}
case NoMetric:
reply(conn, proto.ErrNoMetric)
case WrongMetricType:
reply(conn, proto.ErrWrongMetricType)
default:
diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug)
}
return nil
}
func (s *Database) ListAllCumulativeMeasures(conn io.Writer, req proto.ListAllCumulativeMeasuresReq) error {
resultCh := make(chan cumulativeMeasuresResult, 1)
responseWriter := transform.NewCumulativeMeasureWriter(conn, 0)
responseWriter := atree.NewCumulativeMeasureWriter(conn)
s.appendJobToWorkerQueue(tryListAllCumulativeMeasuresReq{
return s.fullScan(fullScanReq{
MetricID: req.MetricID,
MetricType: diploma.Cumulative,
Conn: conn,
ResponseWriter: responseWriter,
ResultCh: resultCh,
})
result := <-resultCh
switch result.ResultCode {
case QueryDone:
responseWriter.Close()
case UntilFound:
err := s.atree.IterateAllCumulativeByTreeCursor(atree.IterateAllCumulativeByTreeCursorReq{
FracDigits: result.FracDigits,
PageNo: result.PageNo,
EndTimestamp: result.EndTimestamp,
EndValue: result.EndValue,
ResponseWriter: responseWriter,
})
s.metricRUnlock(req.MetricID)
if err != nil {
reply(conn, proto.ErrUnexpected)
} else {
responseWriter.Close()
}
case NoMetric:
reply(conn, proto.ErrNoMetric)
case WrongMetricType:
reply(conn, proto.ErrWrongMetricType)
default:
diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug)
}
return nil
}
func (s *Database) ListInstantMeasures(conn net.Conn, req proto.ListInstantMeasuresReq) error {
@@ -724,79 +665,16 @@ func (s *Database) ListInstantMeasures(conn net.Conn, req proto.ListInstantMeasu
return nil
}
var (
since = req.Since
until = req.Until
)
if req.FirstHourOfDay > 0 {
since, until = correctToFHD(req.Since, req.Until, req.FirstHourOfDay)
}
responseWriter := transform.NewInstantMeasureWriter(conn, req.Since)
resultCh := make(chan instantMeasuresResult, 1)
responseWriter := atree.NewInstantMeasureWriter(conn)
s.appendJobToWorkerQueue(tryListInstantMeasuresReq{
return s.rangeScan(rangeScanReq{
MetricID: req.MetricID,
Since: since,
Until: until,
MetricType: diploma.Instant,
Since: req.Since,
Until: req.Until,
Conn: conn,
ResponseWriter: responseWriter,
ResultCh: resultCh,
})
result := <-resultCh
switch result.ResultCode {
case QueryDone:
responseWriter.Close()
case UntilFound:
err := s.atree.ContinueIterateInstantByTreeCursor(atree.ContinueIterateInstantByTreeCursorReq{
FracDigits: result.FracDigits,
Since: since,
Until: until,
LastPageNo: result.PageNo,
ResponseWriter: responseWriter,
})
s.metricRUnlock(req.MetricID)
if err != nil {
reply(conn, proto.ErrUnexpected)
} else {
responseWriter.Close()
}
case UntilNotFound:
err := s.atree.FindAndIterateInstantByTreeCursor(atree.FindAndIterateInstantByTreeCursorReq{
FracDigits: result.FracDigits,
Since: since,
Until: until,
RootPageNo: result.PageNo,
ResponseWriter: responseWriter,
})
s.metricRUnlock(req.MetricID)
if err != nil {
reply(conn, proto.ErrUnexpected)
} else {
responseWriter.Close()
}
case NoMetric:
reply(conn, proto.ErrNoMetric)
case WrongMetricType:
reply(conn, proto.ErrWrongMetricType)
default:
diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug)
}
return nil
}
type cumulativeMeasuresResult struct {
ResultCode byte
FracDigits byte
PageNo uint32
EndTimestamp uint32
EndValue float64
}
func (s *Database) ListCumulativeMeasures(conn net.Conn, req proto.ListCumulativeMeasuresReq) error {
@@ -805,99 +683,37 @@ func (s *Database) ListCumulativeMeasures(conn net.Conn, req proto.ListCumulativ
return nil
}
var (
since = req.Since
until = req.Until
)
if req.FirstHourOfDay > 0 {
since, until = correctToFHD(since, until, req.FirstHourOfDay)
}
responseWriter := transform.NewCumulativeMeasureWriter(conn, req.Since)
resultCh := make(chan cumulativeMeasuresResult, 1)
responseWriter := atree.NewCumulativeMeasureWriter(conn)
s.appendJobToWorkerQueue(tryListCumulativeMeasuresReq{
return s.rangeScan(rangeScanReq{
MetricID: req.MetricID,
Since: since,
Until: until,
MetricType: diploma.Cumulative,
Since: req.Since,
Until: req.Until,
Conn: conn,
ResponseWriter: responseWriter,
ResultCh: resultCh,
})
result := <-resultCh
switch result.ResultCode {
case QueryDone:
responseWriter.Close()
case UntilFound:
err := s.atree.ContinueIterateCumulativeByTreeCursor(atree.ContinueIterateCumulativeByTreeCursorReq{
FracDigits: result.FracDigits,
Since: since,
Until: until,
LastPageNo: result.PageNo,
EndTimestamp: result.EndTimestamp,
EndValue: result.EndValue,
ResponseWriter: responseWriter,
})
s.metricRUnlock(req.MetricID)
if err != nil {
reply(conn, proto.ErrUnexpected)
} else {
responseWriter.Close()
}
case UntilNotFound:
err := s.atree.FindAndIterateCumulativeByTreeCursor(atree.FindAndIterateCumulativeByTreeCursorReq{
FracDigits: result.FracDigits,
Since: since,
Until: until,
RootPageNo: result.PageNo,
ResponseWriter: responseWriter,
})
s.metricRUnlock(req.MetricID)
if err != nil {
reply(conn, proto.ErrUnexpected)
} else {
responseWriter.Close()
}
case NoMetric:
reply(conn, proto.ErrNoMetric)
case WrongMetricType:
reply(conn, proto.ErrWrongMetricType)
default:
diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug)
}
return nil
}
type instantPeriodsResult struct {
type rangeScanResult struct {
ResultCode byte
FracDigits byte
PageNo uint32
RootPageNo uint32
LastPageNo uint32
}
func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriodsReq) error {
if req.Since > req.Until {
since, until := timeBoundsOfAggregation(req.Since, req.Until, req.GroupBy, req.FirstHourOfDay)
if since.After(until) {
reply(conn, proto.ErrInvalidRange)
return nil
}
var (
since = req.Since
until = req.Until
)
if req.FirstHourOfDay > 0 {
since, until = correctToFHD(since, until, req.FirstHourOfDay)
}
resultCh := make(chan instantPeriodsResult, 1)
aggregator, err := atree.NewInstantAggregator(atree.InstantAggregatorOptions{
responseWriter, err := transform.NewInstantPeriodsWriter(transform.InstantPeriodsWriterOptions{
Dst: conn,
GroupBy: req.GroupBy,
Since: uint32(since.Unix()),
AggregateFuncs: req.AggregateFuncs,
FirstHourOfDay: req.FirstHourOfDay,
})
if err != nil {
@@ -905,14 +721,62 @@ func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriod
return nil
}
responseWriter := atree.NewInstantPeriodsWriter(conn, req.AggregateFuncs)
s.appendJobToWorkerQueue(tryListInstantPeriodsReq{
return s.rangeScan(rangeScanReq{
MetricID: req.MetricID,
Since: since,
Until: until,
Aggregator: aggregator,
MetricType: diploma.Instant,
Since: uint32(since.Unix()),
Until: uint32(until.Unix()),
Conn: conn,
ResponseWriter: responseWriter,
})
}
func (s *Database) ListCumulativePeriods(conn net.Conn, req proto.ListCumulativePeriodsReq) error {
since, until := timeBoundsOfAggregation(req.Since, req.Until, req.GroupBy, req.FirstHourOfDay)
if since.After(until) {
reply(conn, proto.ErrInvalidRange)
return nil
}
responseWriter, err := transform.NewCumulativePeriodsWriter(transform.CumulativePeriodsWriterOptions{
Dst: conn,
GroupBy: req.GroupBy,
Since: uint32(since.Unix()),
FirstHourOfDay: req.FirstHourOfDay,
})
if err != nil {
reply(conn, proto.ErrUnexpected)
return nil
}
return s.rangeScan(rangeScanReq{
MetricID: req.MetricID,
MetricType: diploma.Cumulative,
Since: uint32(since.Unix()),
Until: uint32(until.Unix()),
Conn: conn,
ResponseWriter: responseWriter,
})
}
type rangeScanReq struct {
MetricID uint32
MetricType diploma.MetricType
Since uint32
Until uint32
Conn io.Writer
ResponseWriter diploma.MeasureConsumer
}
func (s *Database) rangeScan(req rangeScanReq) error {
resultCh := make(chan rangeScanResult, 1)
s.appendJobToWorkerQueue(tryRangeScanReq{
MetricID: req.MetricID,
Since: req.Since,
Until: req.Until,
MetricType: req.MetricType,
ResponseWriter: req.ResponseWriter,
ResultCh: resultCh,
})
@@ -920,48 +784,46 @@ func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriod
switch result.ResultCode {
case QueryDone:
responseWriter.Close()
req.ResponseWriter.Close()
case UntilFound:
err := s.atree.ContinueCollectInstantPeriods(atree.ContinueCollectInstantPeriodsReq{
err := s.atree.ContinueRangeScan(atree.ContinueRangeScanReq{
MetricType: req.MetricType,
FracDigits: result.FracDigits,
Aggregator: aggregator,
ResponseWriter: responseWriter,
LastPageNo: result.PageNo,
Since: since,
Until: until,
ResponseWriter: req.ResponseWriter,
LastPageNo: result.LastPageNo,
Since: req.Since,
})
s.metricRUnlock(req.MetricID)
if err != nil {
reply(conn, proto.ErrUnexpected)
reply(req.Conn, proto.ErrUnexpected)
} else {
responseWriter.Close()
req.ResponseWriter.Close()
}
case UntilNotFound:
err := s.atree.FindInstantPeriods(atree.FindInstantPeriodsReq{
err := s.atree.RangeScan(atree.RangeScanReq{
MetricType: req.MetricType,
FracDigits: result.FracDigits,
ResponseWriter: responseWriter,
RootPageNo: result.PageNo,
Since: since,
Until: until,
GroupBy: req.GroupBy,
FirstHourOfDay: req.FirstHourOfDay,
ResponseWriter: req.ResponseWriter,
RootPageNo: result.RootPageNo,
Since: req.Since,
Until: req.Until,
})
s.metricRUnlock(req.MetricID)
if err != nil {
reply(conn, proto.ErrUnexpected)
reply(req.Conn, proto.ErrUnexpected)
} else {
responseWriter.Close()
req.ResponseWriter.Close()
}
case NoMetric:
reply(conn, proto.ErrNoMetric)
reply(req.Conn, proto.ErrNoMetric)
case WrongMetricType:
reply(conn, proto.ErrWrongMetricType)
reply(req.Conn, proto.ErrWrongMetricType)
default:
diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug)
@@ -969,45 +831,20 @@ func (s *Database) ListInstantPeriods(conn net.Conn, req proto.ListInstantPeriod
return nil
}
type cumulativePeriodsResult struct {
ResultCode byte
FracDigits byte
PageNo uint32
type fullScanReq struct {
MetricID uint32
MetricType diploma.MetricType
Conn io.Writer
ResponseWriter diploma.MeasureConsumer
}
func (s *Database) ListCumulativePeriods(conn net.Conn, req proto.ListCumulativePeriodsReq) error {
if req.Since > req.Until {
reply(conn, proto.ErrInvalidRange)
return nil
}
func (s *Database) fullScan(req fullScanReq) error {
resultCh := make(chan fullScanResult, 1)
var (
since = req.Since
until = req.Until
)
if req.FirstHourOfDay > 0 {
since, until = correctToFHD(since, until, req.FirstHourOfDay)
}
resultCh := make(chan cumulativePeriodsResult, 1)
aggregator, err := atree.NewCumulativeAggregator(atree.CumulativeAggregatorOptions{
GroupBy: req.GroupBy,
FirstHourOfDay: req.FirstHourOfDay,
})
if err != nil {
reply(conn, proto.ErrUnexpected)
return nil
}
responseWriter := atree.NewCumulativePeriodsWriter(conn)
s.appendJobToWorkerQueue(tryListCumulativePeriodsReq{
s.appendJobToWorkerQueue(tryFullScanReq{
MetricID: req.MetricID,
Since: since,
Until: until,
Aggregator: aggregator,
ResponseWriter: responseWriter,
MetricType: req.MetricType,
ResponseWriter: req.ResponseWriter,
ResultCh: resultCh,
})
@@ -1015,48 +852,27 @@ func (s *Database) ListCumulativePeriods(conn net.Conn, req proto.ListCumulative
switch result.ResultCode {
case QueryDone:
responseWriter.Close()
req.ResponseWriter.Close()
case UntilFound:
err := s.atree.ContinueCollectCumulativePeriods(atree.ContinueCollectCumulativePeriodsReq{
err := s.atree.ContinueFullScan(atree.ContinueFullScanReq{
MetricType: req.MetricType,
FracDigits: result.FracDigits,
Aggregator: aggregator,
ResponseWriter: responseWriter,
LastPageNo: result.PageNo,
Since: since,
Until: until,
ResponseWriter: req.ResponseWriter,
LastPageNo: result.LastPageNo,
})
s.metricRUnlock(req.MetricID)
if err != nil {
reply(conn, proto.ErrUnexpected)
reply(req.Conn, proto.ErrUnexpected)
} else {
responseWriter.Close()
}
case UntilNotFound:
err := s.atree.FindCumulativePeriods(atree.FindCumulativePeriodsReq{
FracDigits: result.FracDigits,
ResponseWriter: responseWriter,
RootPageNo: result.PageNo,
Since: since,
Until: until,
GroupBy: req.GroupBy,
FirstHourOfDay: req.FirstHourOfDay,
})
s.metricRUnlock(req.MetricID)
if err != nil {
reply(conn, proto.ErrUnexpected)
} else {
responseWriter.Close()
req.ResponseWriter.Close()
}
case NoMetric:
reply(conn, proto.ErrNoMetric)
reply(req.Conn, proto.ErrNoMetric)
case WrongMetricType:
reply(conn, proto.ErrWrongMetricType)
reply(req.Conn, proto.ErrWrongMetricType)
default:
diploma.Abort(diploma.WrongResultCodeBug, ErrWrongResultCodeBug)
@@ -1065,7 +881,7 @@ func (s *Database) ListCumulativePeriods(conn net.Conn, req proto.ListCumulative
}
func (s *Database) ListCurrentValues(conn net.Conn, req proto.ListCurrentValuesReq) error {
responseWriter := atree.NewCurrentValueWriter(conn)
responseWriter := transform.NewCurrentValueWriter(conn)
defer responseWriter.Close()
resultCh := make(chan struct{})

31
database/helpers.go
View File

@@ -5,8 +5,32 @@ import (
"io/fs"
"os"
"time"
"gordenko.dev/dima/diploma"
"gordenko.dev/dima/diploma/proto"
)
func timeBoundsOfAggregation(since, until proto.TimeBound, groupBy diploma.GroupBy, firstHourOfDay int) (s time.Time, u time.Time) {
switch groupBy {
case diploma.GroupByHour, diploma.GroupByDay:
s = time.Date(since.Year, since.Month, since.Day, 0, 0, 0, 0, time.Local)
u = time.Date(until.Year, until.Month, until.Day, 23, 59, 59, 0, time.Local)
case diploma.GroupByMonth:
s = time.Date(since.Year, since.Month, 1, 0, 0, 0, 0, time.Local)
u = time.Date(until.Year, until.Month, 1, 23, 59, 59, 0, time.Local)
u = u.AddDate(0, 1, 0)
u = u.AddDate(0, 0, -1)
}
if firstHourOfDay > 0 {
duration := time.Duration(firstHourOfDay) * time.Hour
s = s.Add(duration)
u = u.Add(duration)
}
return
}
func isFileExist(fileName string) (bool, error) {
_, err := os.Stat(fileName)
if err != nil {
@@ -41,10 +65,3 @@ func (s *Database) metricRUnlock(metricID uint32) {
default:
}
}
func correctToFHD(since, until uint32, firstHourOfDay int) (uint32, uint32) {
duration := time.Duration(firstHourOfDay) * time.Hour
since = uint32(time.Unix(int64(since), 0).Add(duration).Unix())
until = uint32(time.Unix(int64(until), 0).Add(duration).Unix())
return since, until
}

1011
database/proc.go
View File

File diff suppressed because it is too large Load Diff