wp

recovery/advisor.go

@@ -1,255 +0,0 @@
-package recovery
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
-	"regexp"
-	"sort"
-	"strconv"
-)
-
-var (
-	reChanges  = regexp.MustCompile(`(\d+)\.wal`)
-	reSnapshot = regexp.MustCompile(`(\d+)\.snapshot`)
-)
-
-func joinWALFileName(dir string, snapshotNumber int) string {
-	return filepath.Join(dir, fmt.Sprintf("%d.wal", snapshotNumber))
-}
-
-type RecoveryRecipe struct {
-	Snapshot         string
-	Changes          []string
-	SnapshotNumber   int
-	ToDelete         []string
-	CompleteSnapshot bool // флаг - что нужно завершить создание снапшота
-}
-
-type RecoveryAdvisor struct {
-	dir            string
-	verifySnapshot func(string) (bool, error) // (fileName) isVerified, error
-}
-
-type RecoveryAdvisorOptions struct {
-	Dir            string
-	VerifySnapshot func(string) (bool, error)
-}
-
-func NewRecoveryAdvisor(opt RecoveryAdvisorOptions) (*RecoveryAdvisor, error) {
-	if opt.Dir == "" {
-		return nil, errors.New("Dir option is required")
-	}
-	if opt.VerifySnapshot == nil {
-		return nil, errors.New("VerifySnapshot option is required")
-	}
-	return &RecoveryAdvisor{
-		dir:            opt.Dir,
-		verifySnapshot: opt.VerifySnapshot,
-	}, nil
-}
-
-type SnapshotChangesPair struct {
-	SnapshotFileName string
-	WALFileName      string
-	SnapshotNumber   int
-}
-
-func (s *RecoveryAdvisor) getSnapshotChangesPairs() (*SnapshotChangesPair, *SnapshotChangesPair, error) {
-	var (
-		numSet       = make(map[int]bool)
-		changesSet   = make(map[int]bool)
-		snapshotsSet = make(map[int]bool)
-		pairs        []SnapshotChangesPair
-	)
-
-	entries, err := os.ReadDir(s.dir)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	for _, entry := range entries {
-		if entry.Type().IsRegular() {
-			baseName := entry.Name()
-			groups := reChanges.FindStringSubmatch(baseName)
-			if len(groups) == 2 {
-				num, _ := strconv.Atoi(groups[1])
-
-				numSet[num] = true
-				changesSet[num] = true
-			}
-			groups = reSnapshot.FindStringSubmatch(baseName)
-			if len(groups) == 2 {
-				num, _ := strconv.Atoi(groups[1])
-
-				numSet[num] = true
-				snapshotsSet[num] = true
-			}
-		}
-	}
-
-	for snapshotNumber := range numSet {
-		var (
-			snapshotFileName string
-			changesFileName  string
-		)
-		if changesSet[snapshotNumber] {
-			changesFileName = joinWALFileName(s.dir, snapshotNumber)
-		}
-		if snapshotsSet[snapshotNumber] {
-			snapshotFileName = filepath.Join(s.dir, fmt.Sprintf("%d.snapshot", snapshotNumber))
-		}
-		pairs = append(pairs, SnapshotChangesPair{
-			WALFileName:      changesFileName,
-			SnapshotFileName: snapshotFileName,
-			SnapshotNumber:   snapshotNumber,
-		})
-	}
-
-	if len(pairs) == 0 {
-		return nil, nil, nil
-	}
-
-	sort.Slice(pairs, func(i, j int) bool {
-		return pairs[i].SnapshotNumber > pairs[j].SnapshotNumber
-	})
-
-	pair := pairs[0]
-	if pair.WALFileName == "" {
-		return nil, nil, fmt.Errorf("has %d.shapshot file, but %d.changes file not found",
-			pair.SnapshotNumber, pair.SnapshotNumber)
-	}
-
-	if len(pairs) > 1 {
-		prevPair := pairs[1]
-		if prevPair.SnapshotFileName == "" && prevPair.SnapshotNumber != 1 {
-			return &pair, nil, nil
-		}
-
-		if prevPair.WALFileName == "" && pair.SnapshotFileName == "" {
-			return &pair, nil, nil
-		}
-		return &pair, &prevPair, nil
-	} else {
-		return &pair, nil, nil
-	}
-}
-
-func (s *RecoveryAdvisor) GetRecipe() (*RecoveryRecipe, error) {
-	pair, prevPair, err := s.getSnapshotChangesPairs()
-	if err != nil {
-		return nil, err
-	}
-
-	if pair == nil {
-		return nil, nil
-	}
-
-	if pair.SnapshotFileName != "" {
-		isVerified, err := s.verifySnapshot(pair.SnapshotFileName)
-		if err != nil {
-			return nil, fmt.Errorf("verifySnapshot %s: %s",
-				pair.SnapshotFileName, err)
-		}
-
-		if isVerified {
-			recipe := &RecoveryRecipe{
-				Snapshot: pair.SnapshotFileName,
-				Changes: []string{
-					pair.WALFileName,
-				},
-				SnapshotNumber: pair.SnapshotNumber,
-			}
-
-			if prevPair != nil {
-				if prevPair.WALFileName != "" {
-					recipe.ToDelete = append(recipe.ToDelete, prevPair.WALFileName)
-				}
-				if prevPair.SnapshotFileName != "" {
-					recipe.ToDelete = append(recipe.ToDelete, prevPair.SnapshotFileName)
-				}
-			}
-			return recipe, nil
-		}
-		if prevPair != nil {
-			return s.tryPrevPair(pair, prevPair)
-		}
-		return nil, fmt.Errorf("%d.shapshot is corrupted", pair.SnapshotNumber)
-	} else {
-		if prevPair != nil {
-			return s.tryPrevPair(pair, prevPair)
-		} else {
-			if pair.SnapshotNumber == 1 {
-				return &RecoveryRecipe{
-					Changes: []string{
-						pair.WALFileName,
-					},
-					SnapshotNumber: pair.SnapshotNumber,
-				}, nil
-			} else {
-				return nil, fmt.Errorf("%d.snapshot not found", pair.SnapshotNumber)
-			}
-		}
-	}
-}
-
-func (s *RecoveryAdvisor) tryPrevPair(pair, prevPair *SnapshotChangesPair) (*RecoveryRecipe, error) {
-	if prevPair.WALFileName == "" {
-		if pair.SnapshotFileName != "" {
-			return nil, fmt.Errorf("%d.shapshot is corrupted and %d.changes not found",
-				pair.SnapshotNumber, prevPair.SnapshotNumber)
-		} else {
-			return nil, fmt.Errorf("%d.changes not found", prevPair.SnapshotNumber)
-		}
-	}
-
-	if prevPair.SnapshotFileName == "" {
-		if prevPair.SnapshotNumber == 1 {
-			recipe := &RecoveryRecipe{
-				Changes: []string{
-					prevPair.WALFileName,
-					pair.WALFileName,
-				},
-				SnapshotNumber:   pair.SnapshotNumber,
-				CompleteSnapshot: true,
-				ToDelete: []string{
-					prevPair.WALFileName,
-				},
-			}
-			return recipe, nil
-		} else {
-			if pair.SnapshotFileName != "" {
-				return nil, fmt.Errorf("%d.shapshot is corrupted and %d.snapshot not found",
-					pair.SnapshotNumber, prevPair.SnapshotNumber)
-			} else {
-				return nil, fmt.Errorf("%d.snapshot not found", pair.SnapshotNumber)
-			}
-		}
-	}
-
-	isVerified, err := s.verifySnapshot(prevPair.SnapshotFileName)
-	if err != nil {
-		return nil, fmt.Errorf("verifySnapshot %s: %s",
-			prevPair.SnapshotFileName, err)
-	}
-
-	if !isVerified {
-		return nil, fmt.Errorf("%d.shapshot is corrupted", prevPair.SnapshotNumber)
-	}
-
-	recipe := &RecoveryRecipe{
-		Snapshot: prevPair.SnapshotFileName,
-		Changes: []string{
-			prevPair.WALFileName,
-			pair.WALFileName,
-		},
-		SnapshotNumber:   pair.SnapshotNumber,
-		CompleteSnapshot: true,
-		ToDelete: []string{
-			prevPair.WALFileName,
-			prevPair.SnapshotFileName,
-		},
-	}
-	return recipe, nil
-}

recovery/recovery.go

@@ -0,0 +1,221 @@
+package recovery
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"gordenko.dev/dima/qb"
+	"gordenko.dev/dima/qb/freelist"
+	"gordenko.dev/dima/qb/storage"
+)
+
+// Result зберігає результати для обох типів файлів
+type FileVersion struct {
+	Name    string
+	Version int
+}
+
+// FindMaxSnapshotFiles шукає файли з розширеннями .snapshot_XXX та .wal_XXX
+// і повертає структуру з максимальними номерами.
+func findFileLatestVersion(dir string, extPrefix string) (FileVersion, error) {
+	var x FileVersion
+	if !strings.HasPrefix(extPrefix, ".") {
+		extPrefix = "." + extPrefix
+	}
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return x, err
+	}
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+		name := entry.Name()
+		ext := filepath.Ext(name)
+
+		if strings.HasPrefix(ext, extPrefix) {
+			version, err := strconv.Atoi(strings.TrimPrefix(ext, extPrefix))
+			if err == nil {
+				if version > x.Version || x.Name == "" {
+					x.Name = name
+					x.Version = version
+				}
+			}
+		}
+	}
+	return x, nil
+}
+
+var (
+	ErrLostSnapshot = errors.New("lost snapshot")
+)
+
+type RecoveryFiles struct {
+	Snapshot       string
+	WAL            string
+	SnapshotNumber int
+}
+
+func resolveRecoveryFiles(dir string) (_ RecoveryFiles, err error) {
+	wal, err := findFileLatestVersion(dir, "wal_")
+	if err != nil {
+		return
+	}
+	snapshot, err := findFileLatestVersion(dir, "snapshot_")
+	if err != nil {
+		return
+	}
+	if snapshot.Name != "" {
+		if wal.Name != "" {
+			if snapshot.Version == wal.Version {
+				return RecoveryFiles{
+					Snapshot:       snapshot.Name,
+					WAL:            wal.Name,
+					SnapshotNumber: snapshot.Version,
+				}, nil
+			} else if snapshot.Version > wal.Version {
+				return RecoveryFiles{
+					Snapshot:       snapshot.Name,
+					SnapshotNumber: snapshot.Version,
+				}, nil
+			} else {
+				// WAL version > snapshot version
+				err = ErrLostSnapshot
+				return
+			}
+		} else {
+			return RecoveryFiles{
+				Snapshot:       snapshot.Name,
+				SnapshotNumber: snapshot.Version,
+			}, nil
+		}
+	} else {
+		if wal.Name != "" {
+			if wal.Version == 0 {
+				return RecoveryFiles{
+					WAL: wal.Name,
+				}, nil
+			} else {
+				err = ErrLostSnapshot
+				return
+			}
+		} else {
+			// no files - ok, 1st start of database
+			return
+		}
+	}
+}
+
+type RecoveryReport struct {
+	//Metrics []
+	SnapshotNumber int
+	WAL            string
+	IndexFreeList  *freelist.FreeList
+	DataFreeList   *freelist.FreeList
+}
+
+func Recovery(dir string, databaseName string) (_ RecoveryReport, err error) {
+	files, err := resolveRecoveryFiles(dir)
+	if err != nil {
+		return
+	}
+	var (
+		freeIndexPages []uint32
+		freeDataPages  []uint32
+	)
+	if files.Snapshot != "" {
+		var out storage.ReadSnapshotOut
+		out, err = storage.ReadSnapshot(files.Snapshot)
+		if err != nil {
+			return
+		}
+		//
+		freeIndexPages = out.IndexPageNumbers
+		freeDataPages = out.DataPageNumbers
+	}
+
+	if files.WAL != "" {
+		var (
+			walReader   *storage.WALReader
+			walReplayer *storage.WALReplayer
+		)
+		walReader, err = storage.NewWALReader(storage.WALReaderOptions{
+			//FileName:   JoinWALFileName(s.dir, files.WAL),
+			BufferSize: 8 * 1024 * 1024,
+		})
+		if err != nil {
+			return
+		}
+
+		walReplayer, err = storage.NewWALReplayer(storage.WALReplayerOptions{
+			WALReader:      walReader,
+			Metrics:        snapshot.Metrics,
+			FreeIndexPages: freeIndexPages,
+			FreeDataPages:  freeDataPages,
+		})
+		if err != nil {
+			return
+		}
+
+		err = walReplayer.Replay()
+		if err != nil {
+			return
+		}
+		// перезаписую сторінки із останнього комміта
+		var dataFile *os.File
+		dataFile, err = os.OpenFile(qb.GetDataFilePath(dir, databaseName), os.O_WRONLY, 0666)
+		if err != nil {
+			return
+		}
+		err = storage.WriteDataPages(dataFile, walReplayer.DataPages())
+		if err != nil {
+			return
+		}
+		err = dataFile.Close()
+		if err != nil {
+			return
+		}
+		var indexFile *os.File
+		indexFile, err = os.OpenFile(qb.GetIndexFilePath(dir, databaseName), os.O_WRONLY, 0666)
+		if err != nil {
+			return
+		}
+		err = storage.WriteIndexPages(indexFile, walReplayer.IndexPages())
+		if err != nil {
+			return
+		}
+		err = indexFile.Close()
+		if err != nil {
+			return
+		}
+	}
+
+	indexFreeList, err := freelist.New(freelist.Options{
+		PageSize:      2048,
+		BaseFilePath:  filepath.Join(dir, databaseName+".free_index"),
+		DeltaFilePath: filepath.Join(dir, databaseName+".free_index_delta"),
+	})
+
+	// FIX free pages sync
+	dataFreeList, err := freelist.New(freelist.Options{
+		PageSize:      2048,
+		BaseFilePath:  filepath.Join(dir, databaseName+".free_data"),
+		DeltaFilePath: filepath.Join(dir, databaseName+".free_data_delta"),
+	})
+
+	return RecoveryReport{
+		SnapshotNumber: files.SnapshotNumber,
+		WAL:            files.WAL,
+		IndexFreeList:  indexFreeList,
+		DataFreeList:   dataFreeList,
+	}, nil
+}
+
+//func (s *Database) relayMetricsToMetrics(replayMetrics map[uint32]*ReplayMetric) {
+//for metricID, x := range replayMetrics {
+//s.metrics[metricID] = x.ToMetric() FIX
+//}
+//}

recovery/recovery_test.go

@@ -0,0 +1,18 @@
+package recovery
+
+import (
+	"testing"
+)
+
+func TestFindMaxVersion(t *testing.T) {
+	result, err := findFileLatestVersion(".", "wal_")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Name != "test.wal_123" {
+		t.Fatalf("wrong file name %s instead of test.wal_123", result.Name)
+	}
+	if result.Version != 123 {
+		t.Fatalf("wrong file version %d instead of 123", result.Version)
+	}
+}