repair_test.go

Documentation: go.etcd.io/etcd/server/v3/wal

     1  // Copyright 2015 The etcd Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package wal
    16  
    17  import (
    18  	"fmt"
    19  	"io"
    20  	"io/ioutil"
    21  	"os"
    22  	"testing"
    23  
    24  	"go.etcd.io/etcd/raft/v3/raftpb"
    25  	"go.etcd.io/etcd/server/v3/wal/walpb"
    26  
    27  	"go.uber.org/zap"
    28  )
    29  
    30  type corruptFunc func(string, int64) error
    31  
    32  // TestRepairTruncate ensures a truncated file can be repaired
    33  func TestRepairTruncate(t *testing.T) {
    34  	corruptf := func(p string, offset int64) error {
    35  		f, err := openLast(zap.NewExample(), p)
    36  		if err != nil {
    37  			return err
    38  		}
    39  		defer f.Close()
    40  		return f.Truncate(offset - 4)
    41  	}
    42  
    43  	testRepair(t, makeEnts(10), corruptf, 9)
    44  }
    45  
    46  func testRepair(t *testing.T, ents [][]raftpb.Entry, corrupt corruptFunc, expectedEnts int) {
    47  	p, err := ioutil.TempDir(os.TempDir(), "waltest")
    48  	if err != nil {
    49  		t.Fatal(err)
    50  	}
    51  	defer os.RemoveAll(p)
    52  
    53  	// create WAL
    54  	w, err := Create(zap.NewExample(), p, nil)
    55  	defer func() {
    56  		if err = w.Close(); err != nil {
    57  			t.Fatal(err)
    58  		}
    59  	}()
    60  	if err != nil {
    61  		t.Fatal(err)
    62  	}
    63  
    64  	for _, es := range ents {
    65  		if err = w.Save(raftpb.HardState{}, es); err != nil {
    66  			t.Fatal(err)
    67  		}
    68  	}
    69  
    70  	offset, err := w.tail().Seek(0, io.SeekCurrent)
    71  	if err != nil {
    72  		t.Fatal(err)
    73  	}
    74  	w.Close()
    75  
    76  	err = corrupt(p, offset)
    77  	if err != nil {
    78  		t.Fatal(err)
    79  	}
    80  
    81  	// verify we broke the wal
    82  	w, err = Open(zap.NewExample(), p, walpb.Snapshot{})
    83  	if err != nil {
    84  		t.Fatal(err)
    85  	}
    86  	_, _, _, err = w.ReadAll()
    87  	if err != io.ErrUnexpectedEOF {
    88  		t.Fatalf("err = %v, want error %v", err, io.ErrUnexpectedEOF)
    89  	}
    90  	w.Close()
    91  
    92  	// repair the wal
    93  	if ok := Repair(zap.NewExample(), p); !ok {
    94  		t.Fatalf("'Repair' returned '%v', want 'true'", ok)
    95  	}
    96  
    97  	// read it back
    98  	w, err = Open(zap.NewExample(), p, walpb.Snapshot{})
    99  	if err != nil {
   100  		t.Fatal(err)
   101  	}
   102  	_, _, walEnts, err := w.ReadAll()
   103  	if err != nil {
   104  		t.Fatal(err)
   105  	}
   106  	if len(walEnts) != expectedEnts {
   107  		t.Fatalf("len(ents) = %d, want %d", len(walEnts), expectedEnts)
   108  	}
   109  
   110  	// write some more entries to repaired log
   111  	for i := 1; i <= 10; i++ {
   112  		es := []raftpb.Entry{{Index: uint64(expectedEnts + i)}}
   113  		if err = w.Save(raftpb.HardState{}, es); err != nil {
   114  			t.Fatal(err)
   115  		}
   116  	}
   117  	w.Close()
   118  
   119  	// read back entries following repair, ensure it's all there
   120  	w, err = Open(zap.NewExample(), p, walpb.Snapshot{})
   121  	if err != nil {
   122  		t.Fatal(err)
   123  	}
   124  	_, _, walEnts, err = w.ReadAll()
   125  	if err != nil {
   126  		t.Fatal(err)
   127  	}
   128  	if len(walEnts) != expectedEnts+10 {
   129  		t.Fatalf("len(ents) = %d, want %d", len(walEnts), expectedEnts+10)
   130  	}
   131  }
   132  
   133  func makeEnts(ents int) (ret [][]raftpb.Entry) {
   134  	for i := 1; i <= ents; i++ {
   135  		ret = append(ret, []raftpb.Entry{{Index: uint64(i)}})
   136  	}
   137  	return ret
   138  }
   139  
   140  // TestRepairWriteTearLast repairs the WAL in case the last record is a torn write
   141  // that straddled two sectors.
   142  func TestRepairWriteTearLast(t *testing.T) {
   143  	corruptf := func(p string, offset int64) error {
   144  		f, err := openLast(zap.NewExample(), p)
   145  		if err != nil {
   146  			return err
   147  		}
   148  		defer f.Close()
   149  		// 512 bytes perfectly aligns the last record, so use 1024
   150  		if offset < 1024 {
   151  			return fmt.Errorf("got offset %d, expected >1024", offset)
   152  		}
   153  		if terr := f.Truncate(1024); terr != nil {
   154  			return terr
   155  		}
   156  		return f.Truncate(offset)
   157  	}
   158  	testRepair(t, makeEnts(50), corruptf, 40)
   159  }
   160  
   161  // TestRepairWriteTearMiddle repairs the WAL when there is write tearing
   162  // in the middle of a record.
   163  func TestRepairWriteTearMiddle(t *testing.T) {
   164  	corruptf := func(p string, offset int64) error {
   165  		f, err := openLast(zap.NewExample(), p)
   166  		if err != nil {
   167  			return err
   168  		}
   169  		defer f.Close()
   170  		// corrupt middle of 2nd record
   171  		_, werr := f.WriteAt(make([]byte, 512), 4096+512)
   172  		return werr
   173  	}
   174  	ents := makeEnts(5)
   175  	// 4096 bytes of data so a middle sector is easy to corrupt
   176  	dat := make([]byte, 4096)
   177  	for i := range dat {
   178  		dat[i] = byte(i)
   179  	}
   180  	for i := range ents {
   181  		ents[i][0].Data = dat
   182  	}
   183  	testRepair(t, ents, corruptf, 1)
   184  }
   185  
   186  func TestRepairFailDeleteDir(t *testing.T) {
   187  	p, err := ioutil.TempDir(os.TempDir(), "waltest")
   188  	if err != nil {
   189  		t.Fatal(err)
   190  	}
   191  	defer os.RemoveAll(p)
   192  
   193  	w, err := Create(zap.NewExample(), p, nil)
   194  	if err != nil {
   195  		t.Fatal(err)
   196  	}
   197  
   198  	oldSegmentSizeBytes := SegmentSizeBytes
   199  	SegmentSizeBytes = 64
   200  	defer func() {
   201  		SegmentSizeBytes = oldSegmentSizeBytes
   202  	}()
   203  	for _, es := range makeEnts(50) {
   204  		if err = w.Save(raftpb.HardState{}, es); err != nil {
   205  			t.Fatal(err)
   206  		}
   207  	}
   208  
   209  	_, serr := w.tail().Seek(0, io.SeekCurrent)
   210  	if serr != nil {
   211  		t.Fatal(serr)
   212  	}
   213  	w.Close()
   214  
   215  	f, err := openLast(zap.NewExample(), p)
   216  	if err != nil {
   217  		t.Fatal(err)
   218  	}
   219  	if terr := f.Truncate(20); terr != nil {
   220  		t.Fatal(err)
   221  	}
   222  	f.Close()
   223  
   224  	w, err = Open(zap.NewExample(), p, walpb.Snapshot{})
   225  	if err != nil {
   226  		t.Fatal(err)
   227  	}
   228  	_, _, _, err = w.ReadAll()
   229  	if err != io.ErrUnexpectedEOF {
   230  		t.Fatalf("err = %v, want error %v", err, io.ErrUnexpectedEOF)
   231  	}
   232  	w.Close()
   233  
   234  	os.RemoveAll(p)
   235  	if Repair(zap.NewExample(), p) {
   236  		t.Fatal("expect 'Repair' fail on unexpected directory deletion")
   237  	}
   238  }
   239
View as plain text