verify.go

Documentation: go.etcd.io/etcd/server/v3/verify

     1  // Copyright 2021 The etcd Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package verify
    16  
    17  import (
    18  	"fmt"
    19  	"os"
    20  
    21  	"go.etcd.io/etcd/raft/v3/raftpb"
    22  	"go.etcd.io/etcd/server/v3/datadir"
    23  	"go.etcd.io/etcd/server/v3/etcdserver/cindex"
    24  	"go.etcd.io/etcd/server/v3/mvcc/backend"
    25  	wal2 "go.etcd.io/etcd/server/v3/wal"
    26  	"go.etcd.io/etcd/server/v3/wal/walpb"
    27  	"go.uber.org/zap"
    28  )
    29  
    30  const ENV_VERIFY = "ETCD_VERIFY"
    31  const ENV_VERIFY_ALL_VALUE = "all"
    32  
    33  type Config struct {
    34  	// DataDir is a root directory where the data being verified are stored.
    35  	DataDir string
    36  
    37  	// ExactIndex requires consistent_index in backend exactly match the last committed WAL entry.
    38  	// Usually backend's consistent_index needs to be <= WAL.commit, but for backups the match
    39  	// is expected to be exact.
    40  	ExactIndex bool
    41  
    42  	Logger *zap.Logger
    43  }
    44  
    45  // Verify performs consistency checks of given etcd data-directory.
    46  // The errors are reported as the returned error, but for some situations
    47  // the function can also panic.
    48  // The function is expected to work on not-in-use data model, i.e.
    49  // no file-locks should be taken. Verify does not modified the data.
    50  func Verify(cfg Config) error {
    51  	lg := cfg.Logger
    52  	if lg == nil {
    53  		lg = zap.NewNop()
    54  	}
    55  
    56  	var err error
    57  	lg.Info("verification of persisted state", zap.String("data-dir", cfg.DataDir))
    58  	defer func() {
    59  		if err != nil {
    60  			lg.Error("verification of persisted state failed",
    61  				zap.String("data-dir", cfg.DataDir),
    62  				zap.Error(err))
    63  		} else if r := recover(); r != nil {
    64  			lg.Error("verification of persisted state failed",
    65  				zap.String("data-dir", cfg.DataDir))
    66  			panic(r)
    67  		} else {
    68  			lg.Info("verification of persisted state successful", zap.String("data-dir", cfg.DataDir))
    69  		}
    70  	}()
    71  
    72  	beConfig := backend.DefaultBackendConfig()
    73  	beConfig.Path = datadir.ToBackendFileName(cfg.DataDir)
    74  	beConfig.Logger = cfg.Logger
    75  
    76  	be := backend.New(beConfig)
    77  	defer be.Close()
    78  
    79  	snapshot, hardstate, err := validateWal(cfg)
    80  	if err != nil {
    81  		return err
    82  	}
    83  
    84  	// TODO: Perform validation of consistency of membership between
    85  	// backend/members & WAL confstate (and maybe storev2 if still exists).
    86  
    87  	return validateConsistentIndex(cfg, hardstate, snapshot, be)
    88  }
    89  
    90  // VerifyIfEnabled performs verification according to ETCD_VERIFY env settings.
    91  // See Verify for more information.
    92  func VerifyIfEnabled(cfg Config) error {
    93  	if os.Getenv(ENV_VERIFY) == ENV_VERIFY_ALL_VALUE {
    94  		return Verify(cfg)
    95  	}
    96  	return nil
    97  }
    98  
    99  // MustVerifyIfEnabled performs verification according to ETCD_VERIFY env settings
   100  // and exits in case of found problems.
   101  // See Verify for more information.
   102  func MustVerifyIfEnabled(cfg Config) {
   103  	if err := VerifyIfEnabled(cfg); err != nil {
   104  		cfg.Logger.Fatal("Verification failed",
   105  			zap.String("data-dir", cfg.DataDir),
   106  			zap.Error(err))
   107  	}
   108  }
   109  
   110  func validateConsistentIndex(cfg Config, hardstate *raftpb.HardState, snapshot *walpb.Snapshot, be backend.Backend) error {
   111  	index, term := cindex.ReadConsistentIndex(be.ReadTx())
   112  	if cfg.ExactIndex && index != hardstate.Commit {
   113  		return fmt.Errorf("backend.ConsistentIndex (%v) expected == WAL.HardState.commit (%v)", index, hardstate.Commit)
   114  	}
   115  	if cfg.ExactIndex && term != hardstate.Term {
   116  		return fmt.Errorf("backend.Term (%v) expected == WAL.HardState.term, (%v)", term, hardstate.Term)
   117  	}
   118  	if index > hardstate.Commit {
   119  		return fmt.Errorf("backend.ConsistentIndex (%v) must be <= WAL.HardState.commit (%v)", index, hardstate.Commit)
   120  	}
   121  	if term > hardstate.Term {
   122  		return fmt.Errorf("backend.Term (%v) must be <= WAL.HardState.term, (%v)", term, hardstate.Term)
   123  	}
   124  
   125  	if index < snapshot.Index {
   126  		return fmt.Errorf("backend.ConsistentIndex (%v) must be >= last snapshot index (%v)", index, snapshot.Index)
   127  	}
   128  
   129  	cfg.Logger.Info("verification: consistentIndex OK", zap.Uint64("backend-consistent-index", index), zap.Uint64("hardstate-commit", hardstate.Commit))
   130  	return nil
   131  }
   132  
   133  func validateWal(cfg Config) (*walpb.Snapshot, *raftpb.HardState, error) {
   134  	walDir := datadir.ToWalDir(cfg.DataDir)
   135  
   136  	walSnaps, err := wal2.ValidSnapshotEntries(cfg.Logger, walDir)
   137  	if err != nil {
   138  		return nil, nil, err
   139  	}
   140  
   141  	snapshot := walSnaps[len(walSnaps)-1]
   142  	hardstate, err := wal2.Verify(cfg.Logger, walDir, snapshot)
   143  	if err != nil {
   144  		return nil, nil, err
   145  	}
   146  	return &snapshot, hardstate, nil
   147  }
   148
View as plain text