Diff of the two buildlogs: -- --- b1/build.log 2024-05-15 01:19:01.332823351 +0000 +++ b2/build.log 2024-05-15 01:21:49.789657621 +0000 @@ -1,6 +1,6 @@ I: pbuilder: network access will be disabled during build -I: Current time: Tue May 14 13:15:39 -12 2024 -I: pbuilder-time-stamp: 1715735739 +I: Current time: Tue Jun 17 21:42:04 +14 2025 +I: pbuilder-time-stamp: 1750146124 I: Building the build Environment I: extracting base tarball [/var/cache/pbuilder/trixie-reproducible-base.tgz] I: copying local configuration @@ -29,54 +29,86 @@ dpkg-source: info: applying 0001-Skip-unreliable-tests.patch I: Not using root during the build. I: Installing the build-deps -I: user script /srv/workspace/pbuilder/53785/tmp/hooks/D02_print_environment starting +I: user script /srv/workspace/pbuilder/102804/tmp/hooks/D01_modify_environment starting +debug: Running on ionos6-i386. +I: Changing host+domainname to test build reproducibility +I: Adding a custom variable just for the fun of it... +I: Changing /bin/sh to bash +'/bin/sh' -> '/bin/bash' +lrwxrwxrwx 1 root root 9 Jun 17 07:42 /bin/sh -> /bin/bash +I: Setting pbuilder2's login shell to /bin/bash +I: Setting pbuilder2's GECOS to second user,second room,second work-phone,second home-phone,second other +I: user script /srv/workspace/pbuilder/102804/tmp/hooks/D01_modify_environment finished +I: user script /srv/workspace/pbuilder/102804/tmp/hooks/D02_print_environment starting I: set - BUILDDIR='/build/reproducible-path' - BUILDUSERGECOS='first user,first room,first work-phone,first home-phone,first other' - BUILDUSERNAME='pbuilder1' - BUILD_ARCH='i386' - DEBIAN_FRONTEND='noninteractive' - DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=11 ' - DISTRIBUTION='trixie' - HOME='/root' - HOST_ARCH='i386' + BASH=/bin/sh + BASHOPTS=checkwinsize:cmdhist:complete_fullquote:extquote:force_fignore:globasciiranges:globskipdots:hostcomplete:interactive_comments:patsub_replacement:progcomp:promptvars:sourcepath + BASH_ALIASES=() + BASH_ARGC=() + BASH_ARGV=() + BASH_CMDS=() + BASH_LINENO=([0]="12" [1]="0") + BASH_LOADABLES_PATH=/usr/local/lib/bash:/usr/lib/bash:/opt/local/lib/bash:/usr/pkg/lib/bash:/opt/pkg/lib/bash:. + BASH_SOURCE=([0]="/tmp/hooks/D02_print_environment" [1]="/tmp/hooks/D02_print_environment") + BASH_VERSINFO=([0]="5" [1]="2" [2]="21" [3]="1" [4]="release" [5]="i686-pc-linux-gnu") + BASH_VERSION='5.2.21(1)-release' + BUILDDIR=/build/reproducible-path + BUILDUSERGECOS='second user,second room,second work-phone,second home-phone,second other' + BUILDUSERNAME=pbuilder2 + BUILD_ARCH=i386 + DEBIAN_FRONTEND=noninteractive + DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=21 ' + DIRSTACK=() + DISTRIBUTION=trixie + EUID=0 + FUNCNAME=([0]="Echo" [1]="main") + GROUPS=() + HOME=/root + HOSTNAME=i-capture-the-hostname + HOSTTYPE=i686 + HOST_ARCH=i386 IFS=' ' - INVOCATION_ID='c8ada0996219429e89937c6151418e48' - LANG='C' - LANGUAGE='en_US:en' - LC_ALL='C' - LD_LIBRARY_PATH='/usr/lib/libeatmydata' - LD_PRELOAD='libeatmydata.so' - MAIL='/var/mail/root' - OPTIND='1' - PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/games' - PBCURRENTCOMMANDLINEOPERATION='build' - PBUILDER_OPERATION='build' - PBUILDER_PKGDATADIR='/usr/share/pbuilder' - PBUILDER_PKGLIBDIR='/usr/lib/pbuilder' - PBUILDER_SYSCONFDIR='/etc' - PPID='53785' - PS1='# ' - PS2='> ' + INVOCATION_ID=f0e0a3e8cdcd452bb0a4afd60722ee5f + LANG=C + LANGUAGE=de_CH:de + LC_ALL=C + LD_LIBRARY_PATH=/usr/lib/libeatmydata + LD_PRELOAD=libeatmydata.so + MACHTYPE=i686-pc-linux-gnu + MAIL=/var/mail/root + OPTERR=1 + OPTIND=1 + OSTYPE=linux-gnu + PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path + PBCURRENTCOMMANDLINEOPERATION=build + PBUILDER_OPERATION=build + PBUILDER_PKGDATADIR=/usr/share/pbuilder + PBUILDER_PKGLIBDIR=/usr/lib/pbuilder + PBUILDER_SYSCONFDIR=/etc + PIPESTATUS=([0]="0") + POSIXLY_CORRECT=y + PPID=102804 PS4='+ ' - PWD='/' - SHELL='/bin/bash' - SHLVL='2' - SUDO_COMMAND='/usr/bin/timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.JeU24Gzz/pbuilderrc_eK3C --distribution trixie --hookdir /etc/pbuilder/first-build-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/trixie-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.JeU24Gzz/b1 --logfile b1/build.log golang-github-hashicorp-raft_1.5.0-1.dsc' - SUDO_GID='112' - SUDO_UID='107' - SUDO_USER='jenkins' - TERM='unknown' - TZ='/usr/share/zoneinfo/Etc/GMT+12' - USER='root' - _='/usr/bin/systemd-run' - http_proxy='http://46.16.76.132:3128' + PWD=/ + SHELL=/bin/bash + SHELLOPTS=braceexpand:errexit:hashall:interactive-comments:posix + SHLVL=3 + SUDO_COMMAND='/usr/bin/timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.JeU24Gzz/pbuilderrc_QoBc --distribution trixie --hookdir /etc/pbuilder/rebuild-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/trixie-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.JeU24Gzz/b2 --logfile b2/build.log golang-github-hashicorp-raft_1.5.0-1.dsc' + SUDO_GID=112 + SUDO_UID=107 + SUDO_USER=jenkins + TERM=unknown + TZ=/usr/share/zoneinfo/Etc/GMT-14 + UID=0 + USER=root + _='I: set' + http_proxy=http://213.165.73.152:3128 I: uname -a - Linux ionos12-i386 6.1.0-21-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.90-1 (2024-05-03) x86_64 GNU/Linux + Linux i-capture-the-hostname 6.1.0-21-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.90-1 (2024-05-03) x86_64 GNU/Linux I: ls -l /bin - lrwxrwxrwx 1 root root 7 May 13 11:26 /bin -> usr/bin -I: user script /srv/workspace/pbuilder/53785/tmp/hooks/D02_print_environment finished + lrwxrwxrwx 1 root root 7 Jun 16 17:47 /bin -> usr/bin +I: user script /srv/workspace/pbuilder/102804/tmp/hooks/D02_print_environment finished -> Attempting to satisfy build-dependencies -> Creating pbuilder-satisfydepends-dummy package Package: pbuilder-satisfydepends-dummy @@ -227,7 +259,7 @@ Get: 101 http://deb.debian.org/debian trixie/main i386 golang-gopkg-check.v1-dev all 0.0+git20200902.038fdea-1 [32.1 kB] Get: 102 http://deb.debian.org/debian trixie/main i386 golang-gopkg-mgo.v2-dev all 2016.08.01-7 [316 kB] Get: 103 http://deb.debian.org/debian trixie/main i386 golang-github-hashicorp-go-msgpack-dev all 0.5.5-1 [43.3 kB] -Fetched 88.6 MB in 2s (47.9 MB/s) +Fetched 88.6 MB in 2s (58.1 MB/s) debconf: delaying package configuration, since apt-utils is not installed Selecting previously unselected package sensible-utils. (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 19684 files and directories currently installed.) @@ -654,7 +686,11 @@ Building tag database... -> Finished parsing the build-deps I: Building the package -I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.5.0/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.5.0-1_source.changes +I: user script /srv/workspace/pbuilder/102804/tmp/hooks/A99_set_merged_usr starting +Not re-configuring usrmerge for trixie +I: user script /srv/workspace/pbuilder/102804/tmp/hooks/A99_set_merged_usr finished +hostname: Name or service not known +I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.5.0/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.5.0-1_source.changes dpkg-buildpackage: info: source package golang-github-hashicorp-raft dpkg-buildpackage: info: source version 1.5.0-1 dpkg-buildpackage: info: source distribution unstable @@ -672,81 +708,81 @@ dh_autoreconf -O--buildsystem=golang dh_auto_configure -O--buildsystem=golang dh_auto_build -O--buildsystem=golang - cd obj-i686-linux-gnu && go install -trimpath -v -p 11 github.com/hashicorp/raft github.com/hashicorp/raft/bench -internal/coverage/rtcov -internal/goarch + cd obj-i686-linux-gnu && go install -trimpath -v -p 21 github.com/hashicorp/raft github.com/hashicorp/raft/bench +crypto/internal/alias +cmp +encoding +internal/godebugs internal/unsafeheader +sync/atomic +unicode/utf16 internal/goos -internal/godebugs -internal/race -runtime/internal/atomic -internal/cpu +internal/coverage/rtcov runtime/internal/syscall -sync/atomic -runtime/internal/math -internal/chacha8rand -internal/goexperiment -runtime/internal/sys -internal/abi -unicode -unicode/utf8 container/list +unicode/utf8 internal/itoa -math/bits -crypto/internal/alias crypto/internal/boring/sig -cmp -encoding -unicode/utf16 +runtime/internal/atomic +internal/cpu +internal/goexperiment +internal/race +math/bits +internal/goarch log/internal -vendor/golang.org/x/crypto/cryptobyte/asn1 internal/nettrace +unicode +vendor/golang.org/x/crypto/cryptobyte/asn1 vendor/golang.org/x/crypto/internal/alias -internal/bytealg +runtime/internal/math +internal/chacha8rand +runtime/internal/sys +internal/abi slices math +internal/bytealg runtime crypto/subtle internal/reflectlite sync -internal/testlog internal/singleflight internal/bisect +internal/testlog runtime/cgo errors sort +internal/godebug internal/oserror io -vendor/golang.org/x/net/dns/dnsmessage -internal/safefilepath -internal/godebug path github.com/hashicorp/golang-lru/simplelru +internal/safefilepath strconv +vendor/golang.org/x/net/dns/dnsmessage crypto/internal/nistec/fiat syscall internal/intern math/rand -bytes -strings hash +internal/saferio +bytes crypto/internal/randutil +strings hash/crc32 -internal/saferio hash/crc64 vendor/golang.org/x/text/transform crypto -net/netip crypto/rc4 +net/netip reflect +net/http/internal/ascii bufio github.com/hashicorp/go-immutable-radix -net/http/internal/ascii regexp/syntax html +internal/syscall/unix internal/syscall/execenv time -internal/syscall/unix regexp crypto/internal/nistec context @@ -756,68 +792,68 @@ encoding/binary os encoding/base64 -crypto/internal/edwards25519/field crypto/cipher -crypto/md5 +crypto/internal/edwards25519/field vendor/golang.org/x/crypto/internal/poly1305 +crypto/md5 golang.org/x/sys/unix encoding/pem crypto/internal/edwards25519 crypto/internal/boring -vendor/golang.org/x/crypto/chacha20 crypto/des -crypto/ecdh -crypto/sha512 +vendor/golang.org/x/crypto/chacha20 crypto/hmac -crypto/sha256 -crypto/aes +crypto/sha512 +crypto/ecdh crypto/sha1 -vendor/golang.org/x/crypto/chacha20poly1305 +crypto/aes +crypto/sha256 vendor/golang.org/x/crypto/hkdf -os/signal -path/filepath -fmt +vendor/golang.org/x/crypto/chacha20poly1305 internal/sysinfo io/ioutil +os/signal +fmt +path/filepath net -encoding/json -math/big log encoding/hex net/url -compress/flate -vendor/golang.org/x/text/unicode/norm -vendor/golang.org/x/net/http2/hpack -mime -mime/quotedprintable net/http/internal -vendor/golang.org/x/text/unicode/bidi -encoding/gob -go/token +mime/quotedprintable text/template/parse flag -compress/gzip runtime/debug +go/token runtime/trace +encoding/gob +compress/flate +mime +encoding/json +vendor/golang.org/x/net/http2/hpack +math/big +vendor/golang.org/x/text/unicode/norm +vendor/golang.org/x/text/unicode/bidi +testing +compress/gzip vendor/golang.org/x/text/secure/bidirule +text/template +vendor/golang.org/x/net/idna github.com/mattn/go-isatty -testing github.com/mattn/go-colorable github.com/fatih/color -text/template -github.com/hashicorp/go-hclog -vendor/golang.org/x/net/idna crypto/rand -crypto/elliptic -encoding/asn1 crypto/internal/boring/bbig +crypto/elliptic crypto/internal/bigmod crypto/dsa +encoding/asn1 +github.com/hashicorp/go-hclog crypto/ed25519 crypto/rsa +html/template crypto/x509/pkix vendor/golang.org/x/crypto/cryptobyte -html/template crypto/ecdsa vendor/golang.org/x/net/http/httpproxy net/textproto @@ -835,7 +871,7 @@ debian/rules override_dh_auto_test make[1]: Entering directory '/build/reproducible-path/golang-github-hashicorp-raft-1.5.0' dh_auto_test - cd obj-i686-linux-gnu && go test -vet=off -v -p 11 github.com/hashicorp/raft github.com/hashicorp/raft/bench + cd obj-i686-linux-gnu && go test -vet=off -v -p 21 github.com/hashicorp/raft github.com/hashicorp/raft/bench ? github.com/hashicorp/raft/bench [no test files] === RUN TestCommitment_setVoters --- PASS: TestCommitment_setVoters (0.00s) @@ -878,33 +914,33 @@ === RUN TestFileSnapshotSinkImpl --- PASS: TestFileSnapshotSinkImpl (0.00s) === RUN TestFileSS_CreateSnapshotMissingParentDir -2024-05-15T01:16:59.462Z [INFO] creating new snapshot: path=/tmp/raft652256208/raft206298760/snapshots/3-10-1715735819462.tmp ---- PASS: TestFileSS_CreateSnapshotMissingParentDir (0.01s) +2025-06-17T07:42:51.398Z [INFO] creating new snapshot: path=/tmp/raft48299829/raft2172400474/snapshots/3-10-1750146171398.tmp +--- PASS: TestFileSS_CreateSnapshotMissingParentDir (0.00s) === RUN TestFileSS_CreateSnapshot -2024-05-15T01:16:59.472Z [INFO] creating new snapshot: path=/tmp/raft3517484885/snapshots/3-10-1715735819472.tmp -2024-05-15T01:16:59.475Z [WARN] found temporary snapshot: name=3-10-1715735819472.tmp ---- PASS: TestFileSS_CreateSnapshot (0.01s) +2025-06-17T07:42:51.402Z [INFO] creating new snapshot: path=/tmp/raft3208761822/snapshots/3-10-1750146171402.tmp +2025-06-17T07:42:51.403Z [WARN] found temporary snapshot: name=3-10-1750146171402.tmp +--- PASS: TestFileSS_CreateSnapshot (0.00s) === RUN TestFileSS_CancelSnapshot -2024-05-15T01:16:59.487Z [INFO] creating new snapshot: path=/tmp/raft3105340/snapshots/3-10-1715735819487.tmp ---- PASS: TestFileSS_CancelSnapshot (0.01s) +2025-06-17T07:42:51.406Z [INFO] creating new snapshot: path=/tmp/raft3705871249/snapshots/3-10-1750146171406.tmp +--- PASS: TestFileSS_CancelSnapshot (0.00s) === RUN TestFileSS_Retention -2024-05-15T01:16:59.493Z [INFO] creating new snapshot: path=/tmp/raft928332118/snapshots/3-10-1715735819493.tmp -2024-05-15T01:16:59.501Z [INFO] creating new snapshot: path=/tmp/raft928332118/snapshots/3-11-1715735819501.tmp -2024-05-15T01:16:59.510Z [INFO] creating new snapshot: path=/tmp/raft928332118/snapshots/3-12-1715735819510.tmp -2024-05-15T01:16:59.518Z [INFO] reaping snapshot: path=/tmp/raft928332118/snapshots/3-10-1715735819493 -2024-05-15T01:16:59.518Z [INFO] creating new snapshot: path=/tmp/raft928332118/snapshots/3-13-1715735819518.tmp -2024-05-15T01:16:59.526Z [INFO] reaping snapshot: path=/tmp/raft928332118/snapshots/3-11-1715735819501 -2024-05-15T01:16:59.527Z [INFO] creating new snapshot: path=/tmp/raft928332118/snapshots/3-14-1715735819527.tmp -2024-05-15T01:16:59.536Z [INFO] reaping snapshot: path=/tmp/raft928332118/snapshots/3-12-1715735819510 ---- PASS: TestFileSS_Retention (0.04s) +2025-06-17T07:42:51.407Z [INFO] creating new snapshot: path=/tmp/raft667199878/snapshots/3-10-1750146171407.tmp +2025-06-17T07:42:51.410Z [INFO] creating new snapshot: path=/tmp/raft667199878/snapshots/3-11-1750146171410.tmp +2025-06-17T07:42:51.413Z [INFO] creating new snapshot: path=/tmp/raft667199878/snapshots/3-12-1750146171413.tmp +2025-06-17T07:42:51.416Z [INFO] reaping snapshot: path=/tmp/raft667199878/snapshots/3-10-1750146171407 +2025-06-17T07:42:51.416Z [INFO] creating new snapshot: path=/tmp/raft667199878/snapshots/3-13-1750146171416.tmp +2025-06-17T07:42:51.418Z [INFO] reaping snapshot: path=/tmp/raft667199878/snapshots/3-11-1750146171410 +2025-06-17T07:42:51.418Z [INFO] creating new snapshot: path=/tmp/raft667199878/snapshots/3-14-1750146171418.tmp +2025-06-17T07:42:51.421Z [INFO] reaping snapshot: path=/tmp/raft667199878/snapshots/3-12-1750146171413 +--- PASS: TestFileSS_Retention (0.01s) === RUN TestFileSS_BadPerm --- PASS: TestFileSS_BadPerm (0.00s) === RUN TestFileSS_MissingParentDir --- PASS: TestFileSS_MissingParentDir (0.00s) === RUN TestFileSS_Ordering -2024-05-15T01:16:59.538Z [INFO] creating new snapshot: path=/tmp/raft3864854453/snapshots/5-130350-1715735819538.tmp -2024-05-15T01:16:59.546Z [INFO] creating new snapshot: path=/tmp/raft3864854453/snapshots/36-204917-1715735819546.tmp ---- PASS: TestFileSS_Ordering (0.02s) +2025-06-17T07:42:51.422Z [INFO] creating new snapshot: path=/tmp/raft3173748683/snapshots/5-130350-1750146171422.tmp +2025-06-17T07:42:51.425Z [INFO] creating new snapshot: path=/tmp/raft3173748683/snapshots/36-204917-1750146171425.tmp +--- PASS: TestFileSS_Ordering (0.01s) === RUN TestDeferFutureSuccess --- PASS: TestDeferFutureSuccess (0.00s) === RUN TestDeferFutureError @@ -940,7 +976,7 @@ === RUN TestEmitsLogStoreMetrics --- PASS: TestEmitsLogStoreMetrics (0.01s) === RUN TestNetworkTransport_CloseStreams ---- PASS: TestNetworkTransport_CloseStreams (0.01s) +--- PASS: TestNetworkTransport_CloseStreams (0.00s) === RUN TestNetworkTransport_StartStop --- PASS: TestNetworkTransport_StartStop (0.00s) === RUN TestNetworkTransport_Heartbeat_FastPath @@ -953,13 +989,13 @@ --- PASS: TestNetworkTransport_AppendEntriesPipeline_CloseStreams (0.02s) === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=0 -2024-05-15T01:16:59.663Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=1 +2025-06-17T07:42:51.518Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=2 -2024-05-15T01:16:59.715Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" +2025-06-17T07:42:51.569Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=10 -2024-05-15T01:16:59.768Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" ---- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight (0.16s) +2025-06-17T07:42:51.621Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" +--- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight (0.15s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=0 (0.05s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=1 (0.00s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=2 (0.05s) @@ -973,29 +1009,29 @@ === RUN TestNetworkTransport_EncodeDecode_AddressProvider --- PASS: TestNetworkTransport_EncodeDecode_AddressProvider (0.00s) === RUN TestNetworkTransport_PooledConn ---- PASS: TestNetworkTransport_PooledConn (0.01s) +--- PASS: TestNetworkTransport_PooledConn (0.00s) === RUN TestNetworkTransport_ListenBackoff - net_transport_test.go:839: countingWriter: 2024-05-15T01:16:59.782Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-17T07:42:51.626Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-15T01:16:59.787Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-17T07:42:51.632Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-15T01:16:59.798Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-17T07:42:51.642Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-15T01:16:59.819Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-17T07:42:51.662Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-15T01:16:59.859Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-17T07:42:51.703Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-15T01:16:59.940Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-17T07:42:51.784Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-15T01:17:00.102Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-17T07:42:51.944Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-15T01:17:00.422Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-17T07:42:52.265Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-15T01:17:01.064Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-17T07:42:52.905Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-15T01:17:02.065Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-17T07:42:53.907Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-15T01:17:03.067Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-17T07:42:54.909Z [ERROR] test: failed to accept connection: error="intentional error in test" --- PASS: TestNetworkTransport_ListenBackoff (4.00s) === RUN TestPeersJSON_BadConfiguration @@ -1005,2580 +1041,2581 @@ === RUN TestPeersJSON_ReadConfigJSON --- PASS: TestPeersJSON_ReadConfigJSON (0.00s) === RUN TestRaft_StartStop -2024-05-15T01:17:03.785Z [INFO] server-36e05bf6-8562-6349-5c02-1bedb1086e03: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-36e05bf6-8562-6349-5c02-1bedb1086e03 Address:36e05bf6-8562-6349-5c02-1bedb1086e03}]" +2025-06-17T07:42:55.628Z [INFO] server-72bf6423-02f0-ea47-7927-38f6d3389283: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-72bf6423-02f0-ea47-7927-38f6d3389283 Address:72bf6423-02f0-ea47-7927-38f6d3389283}]" --- PASS: TestRaft_StartStop (0.00s) === RUN TestRaft_AfterShutdown -2024-05-15T01:17:03.786Z [INFO] server-f8dcd1dd-45c1-0dc8-65c8-14f589b3b080: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f8dcd1dd-45c1-0dc8-65c8-14f589b3b080 Address:f8dcd1dd-45c1-0dc8-65c8-14f589b3b080}]" +2025-06-17T07:42:55.629Z [INFO] server-eff7abf6-cc7f-f44b-8066-739fb5c01a31: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eff7abf6-cc7f-f44b-8066-739fb5c01a31 Address:eff7abf6-cc7f-f44b-8066-739fb5c01a31}]" --- PASS: TestRaft_AfterShutdown (0.00s) === RUN TestRaft_LiveBootstrap -2024-05-15T01:17:03.787Z [INFO] server-1f5dd56f-25c6-ecec-4413-cf8d9d07abd2: initial configuration: index=0 servers=[] -2024-05-15T01:17:03.787Z [INFO] server-42b55a94-dda5-173d-2978-1d26ffe35f2a: initial configuration: index=0 servers=[] -2024-05-15T01:17:03.787Z [INFO] server-52e93f9e-76b0-dcc4-56da-dfe5c62eadc1: initial configuration: index=0 servers=[] -2024-05-15T01:17:03.787Z [INFO] server-1f5dd56f-25c6-ecec-4413-cf8d9d07abd2: entering follower state: follower="Node at 1f5dd56f-25c6-ecec-4413-cf8d9d07abd2 [Follower]" leader-address= leader-id= -2024-05-15T01:17:03.787Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:03.787Z [INFO] server-42b55a94-dda5-173d-2978-1d26ffe35f2a: entering follower state: follower="Node at 42b55a94-dda5-173d-2978-1d26ffe35f2a [Follower]" leader-address= leader-id= -2024-05-15T01:17:03.787Z [INFO] server-52e93f9e-76b0-dcc4-56da-dfe5c62eadc1: entering follower state: follower="Node at 52e93f9e-76b0-dcc4-56da-dfe5c62eadc1 [Follower]" leader-address= leader-id= -2024-05-15T01:17:03.847Z [WARN] server-52e93f9e-76b0-dcc4-56da-dfe5c62eadc1: no known peers, aborting election -2024-05-15T01:17:03.858Z [WARN] server-42b55a94-dda5-173d-2978-1d26ffe35f2a: no known peers, aborting election -2024-05-15T01:17:03.876Z [WARN] server-1f5dd56f-25c6-ecec-4413-cf8d9d07abd2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:03.876Z [INFO] server-1f5dd56f-25c6-ecec-4413-cf8d9d07abd2: entering candidate state: node="Node at 1f5dd56f-25c6-ecec-4413-cf8d9d07abd2 [Candidate]" term=2 -2024-05-15T01:17:03.876Z [INFO] server-1f5dd56f-25c6-ecec-4413-cf8d9d07abd2: election won: term=2 tally=2 -2024-05-15T01:17:03.876Z [INFO] server-1f5dd56f-25c6-ecec-4413-cf8d9d07abd2: entering leader state: leader="Node at 1f5dd56f-25c6-ecec-4413-cf8d9d07abd2 [Leader]" -2024-05-15T01:17:03.876Z [INFO] server-1f5dd56f-25c6-ecec-4413-cf8d9d07abd2: added peer, starting replication: peer=server-42b55a94-dda5-173d-2978-1d26ffe35f2a -2024-05-15T01:17:03.876Z [INFO] server-1f5dd56f-25c6-ecec-4413-cf8d9d07abd2: added peer, starting replication: peer=server-52e93f9e-76b0-dcc4-56da-dfe5c62eadc1 -2024-05-15T01:17:03.876Z [WARN] server-52e93f9e-76b0-dcc4-56da-dfe5c62eadc1: failed to get previous log: previous-index=1 last-index=0 error="log not found" -2024-05-15T01:17:03.877Z [WARN] server-1f5dd56f-25c6-ecec-4413-cf8d9d07abd2: appendEntries rejected, sending older logs: peer="{Voter server-52e93f9e-76b0-dcc4-56da-dfe5c62eadc1 52e93f9e-76b0-dcc4-56da-dfe5c62eadc1}" next=1 -2024-05-15T01:17:03.877Z [INFO] server-1f5dd56f-25c6-ecec-4413-cf8d9d07abd2: pipelining replication: peer="{Voter server-52e93f9e-76b0-dcc4-56da-dfe5c62eadc1 52e93f9e-76b0-dcc4-56da-dfe5c62eadc1}" -2024-05-15T01:17:03.877Z [WARN] server-42b55a94-dda5-173d-2978-1d26ffe35f2a: failed to get previous log: previous-index=1 last-index=0 error="log not found" -2024-05-15T01:17:03.877Z [WARN] server-1f5dd56f-25c6-ecec-4413-cf8d9d07abd2: appendEntries rejected, sending older logs: peer="{Voter server-42b55a94-dda5-173d-2978-1d26ffe35f2a 42b55a94-dda5-173d-2978-1d26ffe35f2a}" next=1 -2024-05-15T01:17:03.877Z [INFO] server-1f5dd56f-25c6-ecec-4413-cf8d9d07abd2: pipelining replication: peer="{Voter server-42b55a94-dda5-173d-2978-1d26ffe35f2a 42b55a94-dda5-173d-2978-1d26ffe35f2a}" -2024-05-15T01:17:03.982Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:03.877562094 +0000 UTC m=+4.418946332 (2 nodes), 90.016232ms from start of poll, 90.426911ms from cluster start. Timeout at 2024-05-15 01:17:03.982585878 +0000 UTC m=+4.523970658, 105.024326ms after stability -2024-05-15T01:17:03.982Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:04.088Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:03.982700919 +0000 UTC m=+4.524085648 (1 nodes), 1.801µs from start of poll, 195.566227ms from cluster start. Timeout at 2024-05-15 01:17:04.08826233 +0000 UTC m=+4.629647117, 105.561469ms after stability -2024-05-15T01:17:04.104Z [INFO] server-1f5dd56f-25c6-ecec-4413-cf8d9d07abd2: aborting pipeline replication: peer="{Voter server-52e93f9e-76b0-dcc4-56da-dfe5c62eadc1 52e93f9e-76b0-dcc4-56da-dfe5c62eadc1}" -2024-05-15T01:17:04.104Z [INFO] server-1f5dd56f-25c6-ecec-4413-cf8d9d07abd2: aborting pipeline replication: peer="{Voter server-42b55a94-dda5-173d-2978-1d26ffe35f2a 42b55a94-dda5-173d-2978-1d26ffe35f2a}" ---- PASS: TestRaft_LiveBootstrap (0.32s) +2025-06-17T07:42:55.629Z [INFO] server-7fcf40a7-4508-999b-cd1b-ecaf712c557f: initial configuration: index=0 servers=[] +2025-06-17T07:42:55.629Z [INFO] server-7c7ee5ee-ca51-e9e2-c378-7550710a4f55: initial configuration: index=0 servers=[] +2025-06-17T07:42:55.629Z [INFO] server-7fcf40a7-4508-999b-cd1b-ecaf712c557f: entering follower state: follower="Node at 7fcf40a7-4508-999b-cd1b-ecaf712c557f [Follower]" leader-address= leader-id= +2025-06-17T07:42:55.629Z [INFO] server-58172a0c-7c43-ed02-342d-876a357f7614: initial configuration: index=0 servers=[] +2025-06-17T07:42:55.629Z [INFO] server-7c7ee5ee-ca51-e9e2-c378-7550710a4f55: entering follower state: follower="Node at 7c7ee5ee-ca51-e9e2-c378-7550710a4f55 [Follower]" leader-address= leader-id= +2025-06-17T07:42:55.629Z [INFO] server-58172a0c-7c43-ed02-342d-876a357f7614: entering follower state: follower="Node at 58172a0c-7c43-ed02-342d-876a357f7614 [Follower]" leader-address= leader-id= +2025-06-17T07:42:55.630Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:42:55.692Z [WARN] server-7fcf40a7-4508-999b-cd1b-ecaf712c557f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:42:55.692Z [INFO] server-7fcf40a7-4508-999b-cd1b-ecaf712c557f: entering candidate state: node="Node at 7fcf40a7-4508-999b-cd1b-ecaf712c557f [Candidate]" term=2 +2025-06-17T07:42:55.692Z [INFO] server-7fcf40a7-4508-999b-cd1b-ecaf712c557f: election won: term=2 tally=2 +2025-06-17T07:42:55.692Z [INFO] server-7fcf40a7-4508-999b-cd1b-ecaf712c557f: entering leader state: leader="Node at 7fcf40a7-4508-999b-cd1b-ecaf712c557f [Leader]" +2025-06-17T07:42:55.692Z [INFO] server-7fcf40a7-4508-999b-cd1b-ecaf712c557f: added peer, starting replication: peer=server-7c7ee5ee-ca51-e9e2-c378-7550710a4f55 +2025-06-17T07:42:55.692Z [INFO] server-7fcf40a7-4508-999b-cd1b-ecaf712c557f: added peer, starting replication: peer=server-58172a0c-7c43-ed02-342d-876a357f7614 +2025-06-17T07:42:55.692Z [WARN] server-58172a0c-7c43-ed02-342d-876a357f7614: failed to get previous log: previous-index=1 last-index=0 error="log not found" +2025-06-17T07:42:55.692Z [WARN] server-7c7ee5ee-ca51-e9e2-c378-7550710a4f55: failed to get previous log: previous-index=1 last-index=0 error="log not found" +2025-06-17T07:42:55.692Z [WARN] server-7fcf40a7-4508-999b-cd1b-ecaf712c557f: appendEntries rejected, sending older logs: peer="{Voter server-58172a0c-7c43-ed02-342d-876a357f7614 58172a0c-7c43-ed02-342d-876a357f7614}" next=1 +2025-06-17T07:42:55.692Z [WARN] server-7fcf40a7-4508-999b-cd1b-ecaf712c557f: appendEntries rejected, sending older logs: peer="{Voter server-7c7ee5ee-ca51-e9e2-c378-7550710a4f55 7c7ee5ee-ca51-e9e2-c378-7550710a4f55}" next=1 +2025-06-17T07:42:55.692Z [INFO] server-7fcf40a7-4508-999b-cd1b-ecaf712c557f: pipelining replication: peer="{Voter server-58172a0c-7c43-ed02-342d-876a357f7614 58172a0c-7c43-ed02-342d-876a357f7614}" +2025-06-17T07:42:55.692Z [INFO] server-7fcf40a7-4508-999b-cd1b-ecaf712c557f: pipelining replication: peer="{Voter server-7c7ee5ee-ca51-e9e2-c378-7550710a4f55 7c7ee5ee-ca51-e9e2-c378-7550710a4f55}" +2025-06-17T07:42:55.798Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:42:55.692538579 +0000 UTC m=+4.296067301 (2 nodes), 62.477452ms from start of poll, 62.77181ms from cluster start. Timeout at 2025-06-17 07:42:55.798426869 +0000 UTC m=+4.401955554, 105.888253ms after stability +2025-06-17T07:42:55.798Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:42:55.903Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:42:55.798492472 +0000 UTC m=+4.402021171 (1 nodes), 822ns from start of poll, 168.72568ms from cluster start. Timeout at 2025-06-17 07:42:55.903609125 +0000 UTC m=+4.507137811, 105.11664ms after stability +2025-06-17T07:42:55.914Z [INFO] server-7fcf40a7-4508-999b-cd1b-ecaf712c557f: aborting pipeline replication: peer="{Voter server-58172a0c-7c43-ed02-342d-876a357f7614 58172a0c-7c43-ed02-342d-876a357f7614}" +2025-06-17T07:42:55.914Z [INFO] server-7fcf40a7-4508-999b-cd1b-ecaf712c557f: aborting pipeline replication: peer="{Voter server-7c7ee5ee-ca51-e9e2-c378-7550710a4f55 7c7ee5ee-ca51-e9e2-c378-7550710a4f55}" +--- PASS: TestRaft_LiveBootstrap (0.29s) === RUN TestRaft_LiveBootstrap_From_NonVoter -2024-05-15T01:17:04.106Z [INFO] server-35a306ef-aba8-4591-0010-94ff88dd8274: initial configuration: index=0 servers=[] -2024-05-15T01:17:04.106Z [INFO] server-b8e8c00a-0058-2400-01fa-05bcf3cc5730: initial configuration: index=0 servers=[] -2024-05-15T01:17:04.106Z [INFO] server-35a306ef-aba8-4591-0010-94ff88dd8274: entering follower state: follower="Node at 35a306ef-aba8-4591-0010-94ff88dd8274 [Follower]" leader-address= leader-id= -2024-05-15T01:17:04.106Z [INFO] server-b8e8c00a-0058-2400-01fa-05bcf3cc5730: entering follower state: follower="Node at b8e8c00a-0058-2400-01fa-05bcf3cc5730 [Follower]" leader-address= leader-id= +2025-06-17T07:42:55.915Z [INFO] server-af1212ba-9b95-4c62-8e26-0814e8b548d0: initial configuration: index=0 servers=[] +2025-06-17T07:42:55.915Z [INFO] server-af1212ba-9b95-4c62-8e26-0814e8b548d0: entering follower state: follower="Node at af1212ba-9b95-4c62-8e26-0814e8b548d0 [Follower]" leader-address= leader-id= +2025-06-17T07:42:55.915Z [INFO] server-11acebd9-6763-14a4-cee5-d222361a8ee5: initial configuration: index=0 servers=[] --- PASS: TestRaft_LiveBootstrap_From_NonVoter (0.00s) === RUN TestRaft_RecoverCluster_NoState -2024-05-15T01:17:04.107Z [INFO] server-6818f135-ef4e-b7fd-c990-2148ee8ac739: initial configuration: index=0 servers=[] -2024-05-15T01:17:04.107Z [INFO] server-6818f135-ef4e-b7fd-c990-2148ee8ac739: entering follower state: follower="Node at 6818f135-ef4e-b7fd-c990-2148ee8ac739 [Shutdown]" leader-address= leader-id= +2025-06-17T07:42:55.915Z [INFO] server-b2dff372-1115-b8c5-e7e0-96ce15311503: initial configuration: index=0 servers=[] +2025-06-17T07:42:55.915Z [INFO] server-b2dff372-1115-b8c5-e7e0-96ce15311503: entering follower state: follower="Node at b2dff372-1115-b8c5-e7e0-96ce15311503 [Shutdown]" leader-address= leader-id= --- PASS: TestRaft_RecoverCluster_NoState (0.00s) === RUN TestRaft_RecoverCluster === RUN TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs -2024-05-15T01:17:04.109Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-05271eed-fa24-a2a8-6dbe-ba4fe5745834 Address:05271eed-fa24-a2a8-6dbe-ba4fe5745834} {Suffrage:Voter ID:server-0636077f-2ce7-ad8e-a6db-3a051513f249 Address:0636077f-2ce7-ad8e-a6db-3a051513f249} {Suffrage:Voter ID:server-205aa098-b0bb-6861-9287-dc38ee817c22 Address:205aa098-b0bb-6861-9287-dc38ee817c22}]" -2024-05-15T01:17:04.109Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: entering follower state: follower="Node at 05271eed-fa24-a2a8-6dbe-ba4fe5745834 [Follower]" leader-address= leader-id= -2024-05-15T01:17:04.109Z [INFO] server-0636077f-2ce7-ad8e-a6db-3a051513f249: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-05271eed-fa24-a2a8-6dbe-ba4fe5745834 Address:05271eed-fa24-a2a8-6dbe-ba4fe5745834} {Suffrage:Voter ID:server-0636077f-2ce7-ad8e-a6db-3a051513f249 Address:0636077f-2ce7-ad8e-a6db-3a051513f249} {Suffrage:Voter ID:server-205aa098-b0bb-6861-9287-dc38ee817c22 Address:205aa098-b0bb-6861-9287-dc38ee817c22}]" -2024-05-15T01:17:04.109Z [INFO] server-0636077f-2ce7-ad8e-a6db-3a051513f249: entering follower state: follower="Node at 0636077f-2ce7-ad8e-a6db-3a051513f249 [Follower]" leader-address= leader-id= -2024-05-15T01:17:04.109Z [INFO] server-205aa098-b0bb-6861-9287-dc38ee817c22: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-05271eed-fa24-a2a8-6dbe-ba4fe5745834 Address:05271eed-fa24-a2a8-6dbe-ba4fe5745834} {Suffrage:Voter ID:server-0636077f-2ce7-ad8e-a6db-3a051513f249 Address:0636077f-2ce7-ad8e-a6db-3a051513f249} {Suffrage:Voter ID:server-205aa098-b0bb-6861-9287-dc38ee817c22 Address:205aa098-b0bb-6861-9287-dc38ee817c22}]" -2024-05-15T01:17:04.109Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:04.109Z [INFO] server-205aa098-b0bb-6861-9287-dc38ee817c22: entering follower state: follower="Node at 205aa098-b0bb-6861-9287-dc38ee817c22 [Follower]" leader-address= leader-id= -2024-05-15T01:17:04.183Z [WARN] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:04.183Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: entering candidate state: node="Node at 05271eed-fa24-a2a8-6dbe-ba4fe5745834 [Candidate]" term=2 -2024-05-15T01:17:04.183Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: election won: term=2 tally=2 -2024-05-15T01:17:04.183Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: entering leader state: leader="Node at 05271eed-fa24-a2a8-6dbe-ba4fe5745834 [Leader]" -2024-05-15T01:17:04.183Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: added peer, starting replication: peer=server-0636077f-2ce7-ad8e-a6db-3a051513f249 -2024-05-15T01:17:04.183Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: added peer, starting replication: peer=server-205aa098-b0bb-6861-9287-dc38ee817c22 -2024-05-15T01:17:04.183Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: pipelining replication: peer="{Voter server-205aa098-b0bb-6861-9287-dc38ee817c22 205aa098-b0bb-6861-9287-dc38ee817c22}" -2024-05-15T01:17:04.184Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: pipelining replication: peer="{Voter server-0636077f-2ce7-ad8e-a6db-3a051513f249 0636077f-2ce7-ad8e-a6db-3a051513f249}" -2024-05-15T01:17:04.290Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:04.184341432 +0000 UTC m=+4.725725670 (1 nodes), 74.812322ms from start of poll, 75.46878ms from cluster start. Timeout at 2024-05-15 01:17:04.289958333 +0000 UTC m=+4.831343149, 105.617479ms after stability -2024-05-15T01:17:04.290Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: aborting pipeline replication: peer="{Voter server-0636077f-2ce7-ad8e-a6db-3a051513f249 0636077f-2ce7-ad8e-a6db-3a051513f249}" -2024-05-15T01:17:04.290Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: aborting pipeline replication: peer="{Voter server-205aa098-b0bb-6861-9287-dc38ee817c22 205aa098-b0bb-6861-9287-dc38ee817c22}" -2024-05-15T01:17:04.290Z [INFO] creating new snapshot: path=/tmp/raft366101047/snapshots/2-2-1715735824290.tmp -2024-05-15T01:17:04.291Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: starting restore from snapshot: id=2-2-1715735824290 last-index=2 last-term=2 size-in-bytes=1 -2024-05-15T01:17:04.291Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: snapshot restore progress: id=2-2-1715735824290 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2024-05-15T01:17:04.291Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: restored from snapshot: id=2-2-1715735824290 last-index=2 last-term=2 size-in-bytes=1 -2024-05-15T01:17:04.291Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-05271eed-fa24-a2a8-6dbe-ba4fe5745834 Address:05271eed-fa24-a2a8-6dbe-ba4fe5745834} {Suffrage:Voter ID:server-0636077f-2ce7-ad8e-a6db-3a051513f249 Address:0636077f-2ce7-ad8e-a6db-3a051513f249} {Suffrage:Voter ID:server-205aa098-b0bb-6861-9287-dc38ee817c22 Address:205aa098-b0bb-6861-9287-dc38ee817c22}]" -2024-05-15T01:17:04.291Z [INFO] creating new snapshot: path=/tmp/raft285365559/snapshots/2-2-1715735824291.tmp -2024-05-15T01:17:04.292Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: entering follower state: follower="Node at 05271eed-fa24-a2a8-6dbe-ba4fe5745834 [Follower]" leader-address= leader-id= -2024-05-15T01:17:04.292Z [INFO] server-0636077f-2ce7-ad8e-a6db-3a051513f249: starting restore from snapshot: id=2-2-1715735824291 last-index=2 last-term=2 size-in-bytes=1 -2024-05-15T01:17:04.292Z [INFO] server-0636077f-2ce7-ad8e-a6db-3a051513f249: snapshot restore progress: id=2-2-1715735824291 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2024-05-15T01:17:04.292Z [INFO] server-0636077f-2ce7-ad8e-a6db-3a051513f249: restored from snapshot: id=2-2-1715735824291 last-index=2 last-term=2 size-in-bytes=1 -2024-05-15T01:17:04.292Z [INFO] server-0636077f-2ce7-ad8e-a6db-3a051513f249: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-05271eed-fa24-a2a8-6dbe-ba4fe5745834 Address:05271eed-fa24-a2a8-6dbe-ba4fe5745834} {Suffrage:Voter ID:server-0636077f-2ce7-ad8e-a6db-3a051513f249 Address:0636077f-2ce7-ad8e-a6db-3a051513f249} {Suffrage:Voter ID:server-205aa098-b0bb-6861-9287-dc38ee817c22 Address:205aa098-b0bb-6861-9287-dc38ee817c22}]" -2024-05-15T01:17:04.292Z [INFO] creating new snapshot: path=/tmp/raft1455117591/snapshots/2-2-1715735824292.tmp -2024-05-15T01:17:04.293Z [INFO] server-0636077f-2ce7-ad8e-a6db-3a051513f249: entering follower state: follower="Node at 0636077f-2ce7-ad8e-a6db-3a051513f249 [Follower]" leader-address= leader-id= -2024-05-15T01:17:04.293Z [INFO] server-205aa098-b0bb-6861-9287-dc38ee817c22: starting restore from snapshot: id=2-2-1715735824292 last-index=2 last-term=2 size-in-bytes=1 -2024-05-15T01:17:04.294Z [INFO] server-205aa098-b0bb-6861-9287-dc38ee817c22: snapshot restore progress: id=2-2-1715735824292 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2024-05-15T01:17:04.294Z [INFO] server-205aa098-b0bb-6861-9287-dc38ee817c22: restored from snapshot: id=2-2-1715735824292 last-index=2 last-term=2 size-in-bytes=1 -2024-05-15T01:17:04.294Z [INFO] server-205aa098-b0bb-6861-9287-dc38ee817c22: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-05271eed-fa24-a2a8-6dbe-ba4fe5745834 Address:05271eed-fa24-a2a8-6dbe-ba4fe5745834} {Suffrage:Voter ID:server-0636077f-2ce7-ad8e-a6db-3a051513f249 Address:0636077f-2ce7-ad8e-a6db-3a051513f249} {Suffrage:Voter ID:server-205aa098-b0bb-6861-9287-dc38ee817c22 Address:205aa098-b0bb-6861-9287-dc38ee817c22}]" -2024-05-15T01:17:04.294Z [INFO] server-205aa098-b0bb-6861-9287-dc38ee817c22: entering follower state: follower="Node at 205aa098-b0bb-6861-9287-dc38ee817c22 [Follower]" leader-address= leader-id= -2024-05-15T01:17:04.350Z [WARN] server-205aa098-b0bb-6861-9287-dc38ee817c22: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:04.350Z [INFO] server-205aa098-b0bb-6861-9287-dc38ee817c22: entering candidate state: node="Node at 205aa098-b0bb-6861-9287-dc38ee817c22 [Candidate]" term=3 -2024-05-15T01:17:04.350Z [WARN] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:04.350Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: entering candidate state: node="Node at 05271eed-fa24-a2a8-6dbe-ba4fe5745834 [Candidate]" term=3 -2024-05-15T01:17:04.350Z [INFO] server-205aa098-b0bb-6861-9287-dc38ee817c22: election won: term=3 tally=2 -2024-05-15T01:17:04.351Z [INFO] server-0636077f-2ce7-ad8e-a6db-3a051513f249: duplicate requestVote for same term: term=3 -2024-05-15T01:17:04.351Z [INFO] server-205aa098-b0bb-6861-9287-dc38ee817c22: entering leader state: leader="Node at 205aa098-b0bb-6861-9287-dc38ee817c22 [Leader]" -2024-05-15T01:17:04.350Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: duplicate requestVote for same term: term=3 -2024-05-15T01:17:04.351Z [INFO] server-205aa098-b0bb-6861-9287-dc38ee817c22: added peer, starting replication: peer=server-05271eed-fa24-a2a8-6dbe-ba4fe5745834 -2024-05-15T01:17:04.351Z [INFO] server-205aa098-b0bb-6861-9287-dc38ee817c22: added peer, starting replication: peer=server-0636077f-2ce7-ad8e-a6db-3a051513f249 -2024-05-15T01:17:04.351Z [INFO] server-05271eed-fa24-a2a8-6dbe-ba4fe5745834: entering follower state: follower="Node at 05271eed-fa24-a2a8-6dbe-ba4fe5745834 [Follower]" leader-address=205aa098-b0bb-6861-9287-dc38ee817c22 leader-id=server-205aa098-b0bb-6861-9287-dc38ee817c22 -2024-05-15T01:17:04.351Z [WARN] server-205aa098-b0bb-6861-9287-dc38ee817c22: rejecting vote request since we have a leader: from=05271eed-fa24-a2a8-6dbe-ba4fe5745834 leader=205aa098-b0bb-6861-9287-dc38ee817c22 leader-id=server-205aa098-b0bb-6861-9287-dc38ee817c22 -2024-05-15T01:17:04.351Z [INFO] server-205aa098-b0bb-6861-9287-dc38ee817c22: pipelining replication: peer="{Voter server-05271eed-fa24-a2a8-6dbe-ba4fe5745834 05271eed-fa24-a2a8-6dbe-ba4fe5745834}" -2024-05-15T01:17:04.351Z [INFO] server-205aa098-b0bb-6861-9287-dc38ee817c22: pipelining replication: peer="{Voter server-0636077f-2ce7-ad8e-a6db-3a051513f249 0636077f-2ce7-ad8e-a6db-3a051513f249}" -2024-05-15T01:17:04.610Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:04.715Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:04.610436702 +0000 UTC m=+5.151821585 (1 nodes), 74.253µs from start of poll, 501.564695ms from cluster start. Timeout at 2024-05-15 01:17:04.715640554 +0000 UTC m=+5.257025388, 105.203803ms after stability -2024-05-15T01:17:04.715Z [INFO] server-205aa098-b0bb-6861-9287-dc38ee817c22: aborting pipeline replication: peer="{Voter server-0636077f-2ce7-ad8e-a6db-3a051513f249 0636077f-2ce7-ad8e-a6db-3a051513f249}" -2024-05-15T01:17:04.715Z [INFO] server-205aa098-b0bb-6861-9287-dc38ee817c22: aborting pipeline replication: peer="{Voter server-05271eed-fa24-a2a8-6dbe-ba4fe5745834 05271eed-fa24-a2a8-6dbe-ba4fe5745834}" +2025-06-17T07:42:55.916Z [INFO] server-ce8c9035-719c-f27f-8f8c-98affc3053b4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ce8c9035-719c-f27f-8f8c-98affc3053b4 Address:ce8c9035-719c-f27f-8f8c-98affc3053b4} {Suffrage:Voter ID:server-6d114c06-bd60-1652-f643-e9910a6a84cf Address:6d114c06-bd60-1652-f643-e9910a6a84cf} {Suffrage:Voter ID:server-75b4f31d-74f2-fde0-6f17-b6af61947de4 Address:75b4f31d-74f2-fde0-6f17-b6af61947de4}]" +2025-06-17T07:42:55.916Z [INFO] server-ce8c9035-719c-f27f-8f8c-98affc3053b4: entering follower state: follower="Node at ce8c9035-719c-f27f-8f8c-98affc3053b4 [Follower]" leader-address= leader-id= +2025-06-17T07:42:55.916Z [INFO] server-6d114c06-bd60-1652-f643-e9910a6a84cf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ce8c9035-719c-f27f-8f8c-98affc3053b4 Address:ce8c9035-719c-f27f-8f8c-98affc3053b4} {Suffrage:Voter ID:server-6d114c06-bd60-1652-f643-e9910a6a84cf Address:6d114c06-bd60-1652-f643-e9910a6a84cf} {Suffrage:Voter ID:server-75b4f31d-74f2-fde0-6f17-b6af61947de4 Address:75b4f31d-74f2-fde0-6f17-b6af61947de4}]" +2025-06-17T07:42:55.916Z [INFO] server-6d114c06-bd60-1652-f643-e9910a6a84cf: entering follower state: follower="Node at 6d114c06-bd60-1652-f643-e9910a6a84cf [Follower]" leader-address= leader-id= +2025-06-17T07:42:55.916Z [INFO] server-75b4f31d-74f2-fde0-6f17-b6af61947de4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ce8c9035-719c-f27f-8f8c-98affc3053b4 Address:ce8c9035-719c-f27f-8f8c-98affc3053b4} {Suffrage:Voter ID:server-6d114c06-bd60-1652-f643-e9910a6a84cf Address:6d114c06-bd60-1652-f643-e9910a6a84cf} {Suffrage:Voter ID:server-75b4f31d-74f2-fde0-6f17-b6af61947de4 Address:75b4f31d-74f2-fde0-6f17-b6af61947de4}]" +2025-06-17T07:42:55.916Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:42:55.916Z [INFO] server-75b4f31d-74f2-fde0-6f17-b6af61947de4: entering follower state: follower="Node at 75b4f31d-74f2-fde0-6f17-b6af61947de4 [Follower]" leader-address= leader-id= +2025-06-17T07:42:55.972Z [WARN] server-6d114c06-bd60-1652-f643-e9910a6a84cf: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:42:55.972Z [INFO] server-6d114c06-bd60-1652-f643-e9910a6a84cf: entering candidate state: node="Node at 6d114c06-bd60-1652-f643-e9910a6a84cf [Candidate]" term=2 +2025-06-17T07:42:55.972Z [INFO] server-6d114c06-bd60-1652-f643-e9910a6a84cf: election won: term=2 tally=2 +2025-06-17T07:42:55.972Z [INFO] server-6d114c06-bd60-1652-f643-e9910a6a84cf: entering leader state: leader="Node at 6d114c06-bd60-1652-f643-e9910a6a84cf [Leader]" +2025-06-17T07:42:55.972Z [INFO] server-6d114c06-bd60-1652-f643-e9910a6a84cf: added peer, starting replication: peer=server-ce8c9035-719c-f27f-8f8c-98affc3053b4 +2025-06-17T07:42:55.972Z [INFO] server-6d114c06-bd60-1652-f643-e9910a6a84cf: added peer, starting replication: peer=server-75b4f31d-74f2-fde0-6f17-b6af61947de4 +2025-06-17T07:42:55.972Z [INFO] server-6d114c06-bd60-1652-f643-e9910a6a84cf: pipelining replication: peer="{Voter server-ce8c9035-719c-f27f-8f8c-98affc3053b4 ce8c9035-719c-f27f-8f8c-98affc3053b4}" +2025-06-17T07:42:55.972Z [INFO] server-6d114c06-bd60-1652-f643-e9910a6a84cf: pipelining replication: peer="{Voter server-75b4f31d-74f2-fde0-6f17-b6af61947de4 75b4f31d-74f2-fde0-6f17-b6af61947de4}" +2025-06-17T07:42:56.078Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:42:55.972792845 +0000 UTC m=+4.576321567 (1 nodes), 55.962691ms from start of poll, 56.260978ms from cluster start. Timeout at 2025-06-17 07:42:56.078115805 +0000 UTC m=+4.681644500, 105.322933ms after stability +2025-06-17T07:42:56.078Z [INFO] server-6d114c06-bd60-1652-f643-e9910a6a84cf: aborting pipeline replication: peer="{Voter server-ce8c9035-719c-f27f-8f8c-98affc3053b4 ce8c9035-719c-f27f-8f8c-98affc3053b4}" +2025-06-17T07:42:56.078Z [INFO] server-6d114c06-bd60-1652-f643-e9910a6a84cf: aborting pipeline replication: peer="{Voter server-75b4f31d-74f2-fde0-6f17-b6af61947de4 75b4f31d-74f2-fde0-6f17-b6af61947de4}" +2025-06-17T07:42:56.078Z [INFO] creating new snapshot: path=/tmp/raft1567624679/snapshots/2-2-1750146176078.tmp +2025-06-17T07:42:56.078Z [INFO] server-ce8c9035-719c-f27f-8f8c-98affc3053b4: starting restore from snapshot: id=2-2-1750146176078 last-index=2 last-term=2 size-in-bytes=1 +2025-06-17T07:42:56.078Z [INFO] server-ce8c9035-719c-f27f-8f8c-98affc3053b4: snapshot restore progress: id=2-2-1750146176078 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-06-17T07:42:56.078Z [INFO] server-ce8c9035-719c-f27f-8f8c-98affc3053b4: restored from snapshot: id=2-2-1750146176078 last-index=2 last-term=2 size-in-bytes=1 +2025-06-17T07:42:56.078Z [INFO] server-ce8c9035-719c-f27f-8f8c-98affc3053b4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ce8c9035-719c-f27f-8f8c-98affc3053b4 Address:ce8c9035-719c-f27f-8f8c-98affc3053b4} {Suffrage:Voter ID:server-6d114c06-bd60-1652-f643-e9910a6a84cf Address:6d114c06-bd60-1652-f643-e9910a6a84cf} {Suffrage:Voter ID:server-75b4f31d-74f2-fde0-6f17-b6af61947de4 Address:75b4f31d-74f2-fde0-6f17-b6af61947de4}]" +2025-06-17T07:42:56.078Z [INFO] server-ce8c9035-719c-f27f-8f8c-98affc3053b4: entering follower state: follower="Node at ce8c9035-719c-f27f-8f8c-98affc3053b4 [Follower]" leader-address= leader-id= +2025-06-17T07:42:56.079Z [INFO] creating new snapshot: path=/tmp/raft670520819/snapshots/2-2-1750146176079.tmp +2025-06-17T07:42:56.079Z [INFO] server-6d114c06-bd60-1652-f643-e9910a6a84cf: starting restore from snapshot: id=2-2-1750146176079 last-index=2 last-term=2 size-in-bytes=1 +2025-06-17T07:42:56.079Z [INFO] server-6d114c06-bd60-1652-f643-e9910a6a84cf: snapshot restore progress: id=2-2-1750146176079 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-06-17T07:42:56.079Z [INFO] server-6d114c06-bd60-1652-f643-e9910a6a84cf: restored from snapshot: id=2-2-1750146176079 last-index=2 last-term=2 size-in-bytes=1 +2025-06-17T07:42:56.079Z [INFO] server-6d114c06-bd60-1652-f643-e9910a6a84cf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ce8c9035-719c-f27f-8f8c-98affc3053b4 Address:ce8c9035-719c-f27f-8f8c-98affc3053b4} {Suffrage:Voter ID:server-6d114c06-bd60-1652-f643-e9910a6a84cf Address:6d114c06-bd60-1652-f643-e9910a6a84cf} {Suffrage:Voter ID:server-75b4f31d-74f2-fde0-6f17-b6af61947de4 Address:75b4f31d-74f2-fde0-6f17-b6af61947de4}]" +2025-06-17T07:42:56.079Z [INFO] server-6d114c06-bd60-1652-f643-e9910a6a84cf: entering follower state: follower="Node at 6d114c06-bd60-1652-f643-e9910a6a84cf [Follower]" leader-address= leader-id= +2025-06-17T07:42:56.079Z [INFO] creating new snapshot: path=/tmp/raft3024310752/snapshots/2-2-1750146176079.tmp +2025-06-17T07:42:56.080Z [INFO] server-75b4f31d-74f2-fde0-6f17-b6af61947de4: starting restore from snapshot: id=2-2-1750146176079 last-index=2 last-term=2 size-in-bytes=1 +2025-06-17T07:42:56.080Z [INFO] server-75b4f31d-74f2-fde0-6f17-b6af61947de4: snapshot restore progress: id=2-2-1750146176079 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-06-17T07:42:56.080Z [INFO] server-75b4f31d-74f2-fde0-6f17-b6af61947de4: restored from snapshot: id=2-2-1750146176079 last-index=2 last-term=2 size-in-bytes=1 +2025-06-17T07:42:56.080Z [INFO] server-75b4f31d-74f2-fde0-6f17-b6af61947de4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ce8c9035-719c-f27f-8f8c-98affc3053b4 Address:ce8c9035-719c-f27f-8f8c-98affc3053b4} {Suffrage:Voter ID:server-6d114c06-bd60-1652-f643-e9910a6a84cf Address:6d114c06-bd60-1652-f643-e9910a6a84cf} {Suffrage:Voter ID:server-75b4f31d-74f2-fde0-6f17-b6af61947de4 Address:75b4f31d-74f2-fde0-6f17-b6af61947de4}]" +2025-06-17T07:42:56.080Z [INFO] server-75b4f31d-74f2-fde0-6f17-b6af61947de4: entering follower state: follower="Node at 75b4f31d-74f2-fde0-6f17-b6af61947de4 [Follower]" leader-address= leader-id= +2025-06-17T07:42:56.134Z [WARN] server-ce8c9035-719c-f27f-8f8c-98affc3053b4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:42:56.134Z [INFO] server-ce8c9035-719c-f27f-8f8c-98affc3053b4: entering candidate state: node="Node at ce8c9035-719c-f27f-8f8c-98affc3053b4 [Candidate]" term=3 +2025-06-17T07:42:56.134Z [INFO] server-ce8c9035-719c-f27f-8f8c-98affc3053b4: election won: term=3 tally=2 +2025-06-17T07:42:56.134Z [INFO] server-ce8c9035-719c-f27f-8f8c-98affc3053b4: entering leader state: leader="Node at ce8c9035-719c-f27f-8f8c-98affc3053b4 [Leader]" +2025-06-17T07:42:56.134Z [INFO] server-ce8c9035-719c-f27f-8f8c-98affc3053b4: added peer, starting replication: peer=server-6d114c06-bd60-1652-f643-e9910a6a84cf +2025-06-17T07:42:56.134Z [INFO] server-ce8c9035-719c-f27f-8f8c-98affc3053b4: added peer, starting replication: peer=server-75b4f31d-74f2-fde0-6f17-b6af61947de4 +2025-06-17T07:42:56.134Z [INFO] server-ce8c9035-719c-f27f-8f8c-98affc3053b4: pipelining replication: peer="{Voter server-75b4f31d-74f2-fde0-6f17-b6af61947de4 75b4f31d-74f2-fde0-6f17-b6af61947de4}" +2025-06-17T07:42:56.134Z [INFO] server-ce8c9035-719c-f27f-8f8c-98affc3053b4: pipelining replication: peer="{Voter server-6d114c06-bd60-1652-f643-e9910a6a84cf 6d114c06-bd60-1652-f643-e9910a6a84cf}" +2025-06-17T07:42:56.395Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:42:56.500Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:42:56.395698875 +0000 UTC m=+4.999227564 (1 nodes), 18.883µs from start of poll, 479.166975ms from cluster start. Timeout at 2025-06-17 07:42:56.500895322 +0000 UTC m=+5.104424017, 105.196453ms after stability +2025-06-17T07:42:56.500Z [INFO] server-ce8c9035-719c-f27f-8f8c-98affc3053b4: aborting pipeline replication: peer="{Voter server-75b4f31d-74f2-fde0-6f17-b6af61947de4 75b4f31d-74f2-fde0-6f17-b6af61947de4}" +2025-06-17T07:42:56.500Z [INFO] server-ce8c9035-719c-f27f-8f8c-98affc3053b4: aborting pipeline replication: peer="{Voter server-6d114c06-bd60-1652-f643-e9910a6a84cf 6d114c06-bd60-1652-f643-e9910a6a84cf}" === RUN TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs -2024-05-15T01:17:04.717Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0 Address:f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0} {Suffrage:Voter ID:server-ddbd8007-c1dc-f749-8693-e710f529904d Address:ddbd8007-c1dc-f749-8693-e710f529904d} {Suffrage:Voter ID:server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b Address:7ad036c4-5b54-8fbc-1862-7626c5aeaf2b}]" -2024-05-15T01:17:04.718Z [INFO] server-ddbd8007-c1dc-f749-8693-e710f529904d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0 Address:f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0} {Suffrage:Voter ID:server-ddbd8007-c1dc-f749-8693-e710f529904d Address:ddbd8007-c1dc-f749-8693-e710f529904d} {Suffrage:Voter ID:server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b Address:7ad036c4-5b54-8fbc-1862-7626c5aeaf2b}]" -2024-05-15T01:17:04.718Z [INFO] server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0 Address:f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0} {Suffrage:Voter ID:server-ddbd8007-c1dc-f749-8693-e710f529904d Address:ddbd8007-c1dc-f749-8693-e710f529904d} {Suffrage:Voter ID:server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b Address:7ad036c4-5b54-8fbc-1862-7626c5aeaf2b}]" -2024-05-15T01:17:04.718Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:04.718Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: entering follower state: follower="Node at f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0 [Follower]" leader-address= leader-id= -2024-05-15T01:17:04.718Z [INFO] server-ddbd8007-c1dc-f749-8693-e710f529904d: entering follower state: follower="Node at ddbd8007-c1dc-f749-8693-e710f529904d [Follower]" leader-address= leader-id= -2024-05-15T01:17:04.718Z [INFO] server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b: entering follower state: follower="Node at 7ad036c4-5b54-8fbc-1862-7626c5aeaf2b [Follower]" leader-address= leader-id= -2024-05-15T01:17:04.805Z [WARN] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:04.805Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: entering candidate state: node="Node at f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0 [Candidate]" term=2 -2024-05-15T01:17:04.805Z [WARN] server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:04.805Z [INFO] server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b: entering candidate state: node="Node at 7ad036c4-5b54-8fbc-1862-7626c5aeaf2b [Candidate]" term=2 -2024-05-15T01:17:04.805Z [INFO] server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b: duplicate requestVote for same term: term=2 -2024-05-15T01:17:04.805Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: election won: term=2 tally=2 -2024-05-15T01:17:04.805Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: entering leader state: leader="Node at f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0 [Leader]" -2024-05-15T01:17:04.805Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: added peer, starting replication: peer=server-ddbd8007-c1dc-f749-8693-e710f529904d -2024-05-15T01:17:04.805Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: added peer, starting replication: peer=server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b -2024-05-15T01:17:04.805Z [INFO] server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b: entering follower state: follower="Node at 7ad036c4-5b54-8fbc-1862-7626c5aeaf2b [Follower]" leader-address=f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0 leader-id=server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0 -2024-05-15T01:17:04.805Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: pipelining replication: peer="{Voter server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b 7ad036c4-5b54-8fbc-1862-7626c5aeaf2b}" -2024-05-15T01:17:04.805Z [INFO] server-ddbd8007-c1dc-f749-8693-e710f529904d: duplicate requestVote for same term: term=2 -2024-05-15T01:17:04.806Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: pipelining replication: peer="{Voter server-ddbd8007-c1dc-f749-8693-e710f529904d ddbd8007-c1dc-f749-8693-e710f529904d}" -2024-05-15T01:17:04.806Z [WARN] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: rejecting vote request since we have a leader: from=7ad036c4-5b54-8fbc-1862-7626c5aeaf2b leader=f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0 leader-id=server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0 -2024-05-15T01:17:04.912Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:04.806179981 +0000 UTC m=+5.347564714 (1 nodes), 87.972893ms from start of poll, 88.458362ms from cluster start. Timeout at 2024-05-15 01:17:04.911995175 +0000 UTC m=+5.453379923, 105.815209ms after stability -2024-05-15T01:17:04.913Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: aborting pipeline replication: peer="{Voter server-ddbd8007-c1dc-f749-8693-e710f529904d ddbd8007-c1dc-f749-8693-e710f529904d}" -2024-05-15T01:17:04.913Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: aborting pipeline replication: peer="{Voter server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b 7ad036c4-5b54-8fbc-1862-7626c5aeaf2b}" -2024-05-15T01:17:04.913Z [INFO] creating new snapshot: path=/tmp/raft3424405536/snapshots/2-6-1715735824913.tmp -2024-05-15T01:17:04.914Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: starting restore from snapshot: id=2-6-1715735824913 last-index=6 last-term=2 size-in-bytes=25 -2024-05-15T01:17:04.914Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: snapshot restore progress: id=2-6-1715735824913 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2024-05-15T01:17:04.914Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: restored from snapshot: id=2-6-1715735824913 last-index=6 last-term=2 size-in-bytes=25 -2024-05-15T01:17:04.914Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0 Address:f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0} {Suffrage:Voter ID:server-ddbd8007-c1dc-f749-8693-e710f529904d Address:ddbd8007-c1dc-f749-8693-e710f529904d} {Suffrage:Voter ID:server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b Address:7ad036c4-5b54-8fbc-1862-7626c5aeaf2b}]" -2024-05-15T01:17:04.914Z [INFO] creating new snapshot: path=/tmp/raft3047088635/snapshots/2-6-1715735824914.tmp -2024-05-15T01:17:04.915Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: entering follower state: follower="Node at f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0 [Follower]" leader-address= leader-id= -2024-05-15T01:17:04.915Z [INFO] server-ddbd8007-c1dc-f749-8693-e710f529904d: starting restore from snapshot: id=2-6-1715735824914 last-index=6 last-term=2 size-in-bytes=25 -2024-05-15T01:17:04.915Z [INFO] server-ddbd8007-c1dc-f749-8693-e710f529904d: snapshot restore progress: id=2-6-1715735824914 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2024-05-15T01:17:04.915Z [INFO] server-ddbd8007-c1dc-f749-8693-e710f529904d: restored from snapshot: id=2-6-1715735824914 last-index=6 last-term=2 size-in-bytes=25 -2024-05-15T01:17:04.915Z [INFO] server-ddbd8007-c1dc-f749-8693-e710f529904d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0 Address:f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0} {Suffrage:Voter ID:server-ddbd8007-c1dc-f749-8693-e710f529904d Address:ddbd8007-c1dc-f749-8693-e710f529904d} {Suffrage:Voter ID:server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b Address:7ad036c4-5b54-8fbc-1862-7626c5aeaf2b}]" -2024-05-15T01:17:04.915Z [INFO] creating new snapshot: path=/tmp/raft1923554761/snapshots/2-6-1715735824915.tmp -2024-05-15T01:17:04.916Z [INFO] server-ddbd8007-c1dc-f749-8693-e710f529904d: entering follower state: follower="Node at ddbd8007-c1dc-f749-8693-e710f529904d [Follower]" leader-address= leader-id= -2024-05-15T01:17:04.916Z [INFO] server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b: starting restore from snapshot: id=2-6-1715735824915 last-index=6 last-term=2 size-in-bytes=25 -2024-05-15T01:17:04.916Z [INFO] server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b: snapshot restore progress: id=2-6-1715735824915 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2024-05-15T01:17:04.917Z [INFO] server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b: restored from snapshot: id=2-6-1715735824915 last-index=6 last-term=2 size-in-bytes=25 -2024-05-15T01:17:04.917Z [INFO] server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0 Address:f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0} {Suffrage:Voter ID:server-ddbd8007-c1dc-f749-8693-e710f529904d Address:ddbd8007-c1dc-f749-8693-e710f529904d} {Suffrage:Voter ID:server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b Address:7ad036c4-5b54-8fbc-1862-7626c5aeaf2b}]" -2024-05-15T01:17:04.917Z [INFO] server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b: entering follower state: follower="Node at 7ad036c4-5b54-8fbc-1862-7626c5aeaf2b [Follower]" leader-address= leader-id= -2024-05-15T01:17:04.975Z [WARN] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:04.975Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: entering candidate state: node="Node at f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0 [Candidate]" term=3 -2024-05-15T01:17:04.975Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: election won: term=3 tally=2 -2024-05-15T01:17:04.975Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: entering leader state: leader="Node at f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0 [Leader]" -2024-05-15T01:17:04.975Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: added peer, starting replication: peer=server-ddbd8007-c1dc-f749-8693-e710f529904d -2024-05-15T01:17:04.975Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: added peer, starting replication: peer=server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b -2024-05-15T01:17:04.975Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: pipelining replication: peer="{Voter server-ddbd8007-c1dc-f749-8693-e710f529904d ddbd8007-c1dc-f749-8693-e710f529904d}" -2024-05-15T01:17:04.975Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: pipelining replication: peer="{Voter server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b 7ad036c4-5b54-8fbc-1862-7626c5aeaf2b}" -2024-05-15T01:17:05.232Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:05.338Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:05.232950219 +0000 UTC m=+5.774334973 (1 nodes), 42.281µs from start of poll, 515.228621ms from cluster start. Timeout at 2024-05-15 01:17:05.338815508 +0000 UTC m=+5.880200281, 105.865308ms after stability -2024-05-15T01:17:05.338Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: aborting pipeline replication: peer="{Voter server-7ad036c4-5b54-8fbc-1862-7626c5aeaf2b 7ad036c4-5b54-8fbc-1862-7626c5aeaf2b}" -2024-05-15T01:17:05.339Z [INFO] server-f1e7e046-079c-bf34-2fd0-4ada7b7ce0b0: aborting pipeline replication: peer="{Voter server-ddbd8007-c1dc-f749-8693-e710f529904d ddbd8007-c1dc-f749-8693-e710f529904d}" +2025-06-17T07:42:56.503Z [INFO] server-bd3a5da2-6705-1230-f7f9-763b28364752: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bd3a5da2-6705-1230-f7f9-763b28364752 Address:bd3a5da2-6705-1230-f7f9-763b28364752} {Suffrage:Voter ID:server-bad5664c-a8d3-5a52-5418-f718ef14dfa9 Address:bad5664c-a8d3-5a52-5418-f718ef14dfa9} {Suffrage:Voter ID:server-095034c7-8402-155b-01f8-5b093a7edfea Address:095034c7-8402-155b-01f8-5b093a7edfea}]" +2025-06-17T07:42:56.503Z [INFO] server-bd3a5da2-6705-1230-f7f9-763b28364752: entering follower state: follower="Node at bd3a5da2-6705-1230-f7f9-763b28364752 [Follower]" leader-address= leader-id= +2025-06-17T07:42:56.503Z [INFO] server-bad5664c-a8d3-5a52-5418-f718ef14dfa9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bd3a5da2-6705-1230-f7f9-763b28364752 Address:bd3a5da2-6705-1230-f7f9-763b28364752} {Suffrage:Voter ID:server-bad5664c-a8d3-5a52-5418-f718ef14dfa9 Address:bad5664c-a8d3-5a52-5418-f718ef14dfa9} {Suffrage:Voter ID:server-095034c7-8402-155b-01f8-5b093a7edfea Address:095034c7-8402-155b-01f8-5b093a7edfea}]" +2025-06-17T07:42:56.503Z [INFO] server-bad5664c-a8d3-5a52-5418-f718ef14dfa9: entering follower state: follower="Node at bad5664c-a8d3-5a52-5418-f718ef14dfa9 [Follower]" leader-address= leader-id= +2025-06-17T07:42:56.503Z [INFO] server-095034c7-8402-155b-01f8-5b093a7edfea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bd3a5da2-6705-1230-f7f9-763b28364752 Address:bd3a5da2-6705-1230-f7f9-763b28364752} {Suffrage:Voter ID:server-bad5664c-a8d3-5a52-5418-f718ef14dfa9 Address:bad5664c-a8d3-5a52-5418-f718ef14dfa9} {Suffrage:Voter ID:server-095034c7-8402-155b-01f8-5b093a7edfea Address:095034c7-8402-155b-01f8-5b093a7edfea}]" +2025-06-17T07:42:56.503Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:42:56.503Z [INFO] server-095034c7-8402-155b-01f8-5b093a7edfea: entering follower state: follower="Node at 095034c7-8402-155b-01f8-5b093a7edfea [Follower]" leader-address= leader-id= +2025-06-17T07:42:56.559Z [WARN] server-bd3a5da2-6705-1230-f7f9-763b28364752: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:42:56.559Z [INFO] server-bd3a5da2-6705-1230-f7f9-763b28364752: entering candidate state: node="Node at bd3a5da2-6705-1230-f7f9-763b28364752 [Candidate]" term=2 +2025-06-17T07:42:56.559Z [INFO] server-bd3a5da2-6705-1230-f7f9-763b28364752: election won: term=2 tally=2 +2025-06-17T07:42:56.559Z [INFO] server-bd3a5da2-6705-1230-f7f9-763b28364752: entering leader state: leader="Node at bd3a5da2-6705-1230-f7f9-763b28364752 [Leader]" +2025-06-17T07:42:56.559Z [INFO] server-bd3a5da2-6705-1230-f7f9-763b28364752: added peer, starting replication: peer=server-bad5664c-a8d3-5a52-5418-f718ef14dfa9 +2025-06-17T07:42:56.559Z [INFO] server-bd3a5da2-6705-1230-f7f9-763b28364752: added peer, starting replication: peer=server-095034c7-8402-155b-01f8-5b093a7edfea +2025-06-17T07:42:56.559Z [INFO] server-bd3a5da2-6705-1230-f7f9-763b28364752: pipelining replication: peer="{Voter server-095034c7-8402-155b-01f8-5b093a7edfea 095034c7-8402-155b-01f8-5b093a7edfea}" +2025-06-17T07:42:56.559Z [INFO] server-bd3a5da2-6705-1230-f7f9-763b28364752: pipelining replication: peer="{Voter server-bad5664c-a8d3-5a52-5418-f718ef14dfa9 bad5664c-a8d3-5a52-5418-f718ef14dfa9}" +2025-06-17T07:42:56.665Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:42:56.559629195 +0000 UTC m=+5.163157881 (1 nodes), 55.970573ms from start of poll, 56.323595ms from cluster start. Timeout at 2025-06-17 07:42:56.665482598 +0000 UTC m=+5.269011292, 105.853411ms after stability +2025-06-17T07:42:56.665Z [INFO] server-bd3a5da2-6705-1230-f7f9-763b28364752: aborting pipeline replication: peer="{Voter server-095034c7-8402-155b-01f8-5b093a7edfea 095034c7-8402-155b-01f8-5b093a7edfea}" +2025-06-17T07:42:56.665Z [INFO] server-bd3a5da2-6705-1230-f7f9-763b28364752: aborting pipeline replication: peer="{Voter server-bad5664c-a8d3-5a52-5418-f718ef14dfa9 bad5664c-a8d3-5a52-5418-f718ef14dfa9}" +2025-06-17T07:42:56.666Z [INFO] creating new snapshot: path=/tmp/raft3128310977/snapshots/2-6-1750146176666.tmp +2025-06-17T07:42:56.666Z [INFO] server-bd3a5da2-6705-1230-f7f9-763b28364752: starting restore from snapshot: id=2-6-1750146176666 last-index=6 last-term=2 size-in-bytes=25 +2025-06-17T07:42:56.666Z [INFO] server-bd3a5da2-6705-1230-f7f9-763b28364752: snapshot restore progress: id=2-6-1750146176666 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-06-17T07:42:56.666Z [INFO] server-bd3a5da2-6705-1230-f7f9-763b28364752: restored from snapshot: id=2-6-1750146176666 last-index=6 last-term=2 size-in-bytes=25 +2025-06-17T07:42:56.666Z [INFO] server-bd3a5da2-6705-1230-f7f9-763b28364752: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bd3a5da2-6705-1230-f7f9-763b28364752 Address:bd3a5da2-6705-1230-f7f9-763b28364752} {Suffrage:Voter ID:server-bad5664c-a8d3-5a52-5418-f718ef14dfa9 Address:bad5664c-a8d3-5a52-5418-f718ef14dfa9} {Suffrage:Voter ID:server-095034c7-8402-155b-01f8-5b093a7edfea Address:095034c7-8402-155b-01f8-5b093a7edfea}]" +2025-06-17T07:42:56.666Z [INFO] server-bd3a5da2-6705-1230-f7f9-763b28364752: entering follower state: follower="Node at bd3a5da2-6705-1230-f7f9-763b28364752 [Follower]" leader-address= leader-id= +2025-06-17T07:42:56.666Z [INFO] creating new snapshot: path=/tmp/raft3717263419/snapshots/2-6-1750146176666.tmp +2025-06-17T07:42:56.667Z [INFO] server-bad5664c-a8d3-5a52-5418-f718ef14dfa9: starting restore from snapshot: id=2-6-1750146176666 last-index=6 last-term=2 size-in-bytes=25 +2025-06-17T07:42:56.667Z [INFO] server-bad5664c-a8d3-5a52-5418-f718ef14dfa9: snapshot restore progress: id=2-6-1750146176666 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-06-17T07:42:56.667Z [INFO] server-bad5664c-a8d3-5a52-5418-f718ef14dfa9: restored from snapshot: id=2-6-1750146176666 last-index=6 last-term=2 size-in-bytes=25 +2025-06-17T07:42:56.667Z [INFO] server-bad5664c-a8d3-5a52-5418-f718ef14dfa9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bd3a5da2-6705-1230-f7f9-763b28364752 Address:bd3a5da2-6705-1230-f7f9-763b28364752} {Suffrage:Voter ID:server-bad5664c-a8d3-5a52-5418-f718ef14dfa9 Address:bad5664c-a8d3-5a52-5418-f718ef14dfa9} {Suffrage:Voter ID:server-095034c7-8402-155b-01f8-5b093a7edfea Address:095034c7-8402-155b-01f8-5b093a7edfea}]" +2025-06-17T07:42:56.667Z [INFO] server-bad5664c-a8d3-5a52-5418-f718ef14dfa9: entering follower state: follower="Node at bad5664c-a8d3-5a52-5418-f718ef14dfa9 [Follower]" leader-address= leader-id= +2025-06-17T07:42:56.667Z [INFO] creating new snapshot: path=/tmp/raft3524920457/snapshots/2-6-1750146176667.tmp +2025-06-17T07:42:56.668Z [INFO] server-095034c7-8402-155b-01f8-5b093a7edfea: starting restore from snapshot: id=2-6-1750146176667 last-index=6 last-term=2 size-in-bytes=25 +2025-06-17T07:42:56.668Z [INFO] server-095034c7-8402-155b-01f8-5b093a7edfea: snapshot restore progress: id=2-6-1750146176667 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-06-17T07:42:56.668Z [INFO] server-095034c7-8402-155b-01f8-5b093a7edfea: restored from snapshot: id=2-6-1750146176667 last-index=6 last-term=2 size-in-bytes=25 +2025-06-17T07:42:56.668Z [INFO] server-095034c7-8402-155b-01f8-5b093a7edfea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bd3a5da2-6705-1230-f7f9-763b28364752 Address:bd3a5da2-6705-1230-f7f9-763b28364752} {Suffrage:Voter ID:server-bad5664c-a8d3-5a52-5418-f718ef14dfa9 Address:bad5664c-a8d3-5a52-5418-f718ef14dfa9} {Suffrage:Voter ID:server-095034c7-8402-155b-01f8-5b093a7edfea Address:095034c7-8402-155b-01f8-5b093a7edfea}]" +2025-06-17T07:42:56.668Z [INFO] server-095034c7-8402-155b-01f8-5b093a7edfea: entering follower state: follower="Node at 095034c7-8402-155b-01f8-5b093a7edfea [Follower]" leader-address= leader-id= +2025-06-17T07:42:56.730Z [WARN] server-095034c7-8402-155b-01f8-5b093a7edfea: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:42:56.730Z [INFO] server-095034c7-8402-155b-01f8-5b093a7edfea: entering candidate state: node="Node at 095034c7-8402-155b-01f8-5b093a7edfea [Candidate]" term=3 +2025-06-17T07:42:56.730Z [INFO] server-095034c7-8402-155b-01f8-5b093a7edfea: election won: term=3 tally=2 +2025-06-17T07:42:56.730Z [INFO] server-095034c7-8402-155b-01f8-5b093a7edfea: entering leader state: leader="Node at 095034c7-8402-155b-01f8-5b093a7edfea [Leader]" +2025-06-17T07:42:56.730Z [INFO] server-095034c7-8402-155b-01f8-5b093a7edfea: added peer, starting replication: peer=server-bd3a5da2-6705-1230-f7f9-763b28364752 +2025-06-17T07:42:56.730Z [INFO] server-095034c7-8402-155b-01f8-5b093a7edfea: added peer, starting replication: peer=server-bad5664c-a8d3-5a52-5418-f718ef14dfa9 +2025-06-17T07:42:56.730Z [INFO] server-095034c7-8402-155b-01f8-5b093a7edfea: pipelining replication: peer="{Voter server-bad5664c-a8d3-5a52-5418-f718ef14dfa9 bad5664c-a8d3-5a52-5418-f718ef14dfa9}" +2025-06-17T07:42:56.730Z [INFO] server-095034c7-8402-155b-01f8-5b093a7edfea: pipelining replication: peer="{Voter server-bd3a5da2-6705-1230-f7f9-763b28364752 bd3a5da2-6705-1230-f7f9-763b28364752}" +2025-06-17T07:42:56.984Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:42:57.089Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:42:56.984318187 +0000 UTC m=+5.587846889 (1 nodes), 41.77µs from start of poll, 481.012603ms from cluster start. Timeout at 2025-06-17 07:42:57.089729653 +0000 UTC m=+5.693258335, 105.411446ms after stability +2025-06-17T07:42:57.089Z [INFO] server-095034c7-8402-155b-01f8-5b093a7edfea: aborting pipeline replication: peer="{Voter server-bad5664c-a8d3-5a52-5418-f718ef14dfa9 bad5664c-a8d3-5a52-5418-f718ef14dfa9}" +2025-06-17T07:42:57.089Z [INFO] server-095034c7-8402-155b-01f8-5b093a7edfea: aborting pipeline replication: peer="{Voter server-bd3a5da2-6705-1230-f7f9-763b28364752 bd3a5da2-6705-1230-f7f9-763b28364752}" === RUN TestRaft_RecoverCluster/snapshot,_with_trailing_logs -2024-05-15T01:17:05.341Z [INFO] server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389 Address:1b4e3c02-d5b5-bf90-82b2-09a92f8ea389} {Suffrage:Voter ID:server-5ca1766d-944b-1315-477e-8beb3263ad6e Address:5ca1766d-944b-1315-477e-8beb3263ad6e} {Suffrage:Voter ID:server-5f2bd6d2-bd9d-8524-e381-061d30ee8143 Address:5f2bd6d2-bd9d-8524-e381-061d30ee8143}]" -2024-05-15T01:17:05.341Z [INFO] server-5ca1766d-944b-1315-477e-8beb3263ad6e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389 Address:1b4e3c02-d5b5-bf90-82b2-09a92f8ea389} {Suffrage:Voter ID:server-5ca1766d-944b-1315-477e-8beb3263ad6e Address:5ca1766d-944b-1315-477e-8beb3263ad6e} {Suffrage:Voter ID:server-5f2bd6d2-bd9d-8524-e381-061d30ee8143 Address:5f2bd6d2-bd9d-8524-e381-061d30ee8143}]" -2024-05-15T01:17:05.341Z [INFO] server-5f2bd6d2-bd9d-8524-e381-061d30ee8143: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389 Address:1b4e3c02-d5b5-bf90-82b2-09a92f8ea389} {Suffrage:Voter ID:server-5ca1766d-944b-1315-477e-8beb3263ad6e Address:5ca1766d-944b-1315-477e-8beb3263ad6e} {Suffrage:Voter ID:server-5f2bd6d2-bd9d-8524-e381-061d30ee8143 Address:5f2bd6d2-bd9d-8524-e381-061d30ee8143}]" -2024-05-15T01:17:05.341Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:05.341Z [INFO] server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389: entering follower state: follower="Node at 1b4e3c02-d5b5-bf90-82b2-09a92f8ea389 [Follower]" leader-address= leader-id= -2024-05-15T01:17:05.341Z [INFO] server-5ca1766d-944b-1315-477e-8beb3263ad6e: entering follower state: follower="Node at 5ca1766d-944b-1315-477e-8beb3263ad6e [Follower]" leader-address= leader-id= -2024-05-15T01:17:05.341Z [INFO] server-5f2bd6d2-bd9d-8524-e381-061d30ee8143: entering follower state: follower="Node at 5f2bd6d2-bd9d-8524-e381-061d30ee8143 [Follower]" leader-address= leader-id= -2024-05-15T01:17:05.401Z [WARN] server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:05.401Z [INFO] server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389: entering candidate state: node="Node at 1b4e3c02-d5b5-bf90-82b2-09a92f8ea389 [Candidate]" term=2 -2024-05-15T01:17:05.402Z [INFO] server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389: election won: term=2 tally=2 -2024-05-15T01:17:05.402Z [INFO] server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389: entering leader state: leader="Node at 1b4e3c02-d5b5-bf90-82b2-09a92f8ea389 [Leader]" -2024-05-15T01:17:05.402Z [INFO] server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389: added peer, starting replication: peer=server-5ca1766d-944b-1315-477e-8beb3263ad6e -2024-05-15T01:17:05.402Z [INFO] server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389: added peer, starting replication: peer=server-5f2bd6d2-bd9d-8524-e381-061d30ee8143 -2024-05-15T01:17:05.402Z [INFO] server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389: pipelining replication: peer="{Voter server-5f2bd6d2-bd9d-8524-e381-061d30ee8143 5f2bd6d2-bd9d-8524-e381-061d30ee8143}" -2024-05-15T01:17:05.402Z [INFO] server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389: pipelining replication: peer="{Voter server-5ca1766d-944b-1315-477e-8beb3263ad6e 5ca1766d-944b-1315-477e-8beb3263ad6e}" -2024-05-15T01:17:05.507Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:05.402608823 +0000 UTC m=+5.943993600 (1 nodes), 61.195548ms from start of poll, 61.737003ms from cluster start. Timeout at 2024-05-15 01:17:05.507643362 +0000 UTC m=+6.049028099, 105.034499ms after stability -2024-05-15T01:17:05.514Z [INFO] server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389: aborting pipeline replication: peer="{Voter server-5f2bd6d2-bd9d-8524-e381-061d30ee8143 5f2bd6d2-bd9d-8524-e381-061d30ee8143}" -2024-05-15T01:17:05.514Z [INFO] server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389: aborting pipeline replication: peer="{Voter server-5ca1766d-944b-1315-477e-8beb3263ad6e 5ca1766d-944b-1315-477e-8beb3263ad6e}" -2024-05-15T01:17:05.514Z [INFO] creating new snapshot: path=/tmp/raft3800435329/snapshots/2-27-1715735825514.tmp -2024-05-15T01:17:05.517Z [INFO] server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389: starting restore from snapshot: id=2-27-1715735825514 last-index=27 last-term=2 size-in-bytes=168 -2024-05-15T01:17:05.517Z [INFO] server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389: snapshot restore progress: id=2-27-1715735825514 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2024-05-15T01:17:05.517Z [INFO] server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389: restored from snapshot: id=2-27-1715735825514 last-index=27 last-term=2 size-in-bytes=168 -2024-05-15T01:17:05.517Z [INFO] server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389 Address:1b4e3c02-d5b5-bf90-82b2-09a92f8ea389} {Suffrage:Voter ID:server-5ca1766d-944b-1315-477e-8beb3263ad6e Address:5ca1766d-944b-1315-477e-8beb3263ad6e} {Suffrage:Voter ID:server-5f2bd6d2-bd9d-8524-e381-061d30ee8143 Address:5f2bd6d2-bd9d-8524-e381-061d30ee8143}]" -2024-05-15T01:17:05.517Z [INFO] server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389: entering follower state: follower="Node at 1b4e3c02-d5b5-bf90-82b2-09a92f8ea389 [Follower]" leader-address= leader-id= -2024-05-15T01:17:05.517Z [INFO] creating new snapshot: path=/tmp/raft2158362538/snapshots/2-27-1715735825517.tmp -2024-05-15T01:17:05.518Z [INFO] server-5ca1766d-944b-1315-477e-8beb3263ad6e: starting restore from snapshot: id=2-27-1715735825517 last-index=27 last-term=2 size-in-bytes=168 -2024-05-15T01:17:05.518Z [INFO] server-5ca1766d-944b-1315-477e-8beb3263ad6e: snapshot restore progress: id=2-27-1715735825517 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2024-05-15T01:17:05.518Z [INFO] server-5ca1766d-944b-1315-477e-8beb3263ad6e: restored from snapshot: id=2-27-1715735825517 last-index=27 last-term=2 size-in-bytes=168 -2024-05-15T01:17:05.518Z [INFO] server-5ca1766d-944b-1315-477e-8beb3263ad6e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389 Address:1b4e3c02-d5b5-bf90-82b2-09a92f8ea389} {Suffrage:Voter ID:server-5ca1766d-944b-1315-477e-8beb3263ad6e Address:5ca1766d-944b-1315-477e-8beb3263ad6e} {Suffrage:Voter ID:server-5f2bd6d2-bd9d-8524-e381-061d30ee8143 Address:5f2bd6d2-bd9d-8524-e381-061d30ee8143}]" -2024-05-15T01:17:05.519Z [INFO] server-5ca1766d-944b-1315-477e-8beb3263ad6e: entering follower state: follower="Node at 5ca1766d-944b-1315-477e-8beb3263ad6e [Follower]" leader-address= leader-id= -2024-05-15T01:17:05.519Z [INFO] creating new snapshot: path=/tmp/raft287435701/snapshots/2-27-1715735825519.tmp -2024-05-15T01:17:05.520Z [INFO] server-5f2bd6d2-bd9d-8524-e381-061d30ee8143: starting restore from snapshot: id=2-27-1715735825519 last-index=27 last-term=2 size-in-bytes=168 -2024-05-15T01:17:05.520Z [INFO] server-5f2bd6d2-bd9d-8524-e381-061d30ee8143: snapshot restore progress: id=2-27-1715735825519 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2024-05-15T01:17:05.520Z [INFO] server-5f2bd6d2-bd9d-8524-e381-061d30ee8143: restored from snapshot: id=2-27-1715735825519 last-index=27 last-term=2 size-in-bytes=168 -2024-05-15T01:17:05.520Z [INFO] server-5f2bd6d2-bd9d-8524-e381-061d30ee8143: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389 Address:1b4e3c02-d5b5-bf90-82b2-09a92f8ea389} {Suffrage:Voter ID:server-5ca1766d-944b-1315-477e-8beb3263ad6e Address:5ca1766d-944b-1315-477e-8beb3263ad6e} {Suffrage:Voter ID:server-5f2bd6d2-bd9d-8524-e381-061d30ee8143 Address:5f2bd6d2-bd9d-8524-e381-061d30ee8143}]" -2024-05-15T01:17:05.520Z [INFO] server-5f2bd6d2-bd9d-8524-e381-061d30ee8143: entering follower state: follower="Node at 5f2bd6d2-bd9d-8524-e381-061d30ee8143 [Follower]" leader-address= leader-id= -2024-05-15T01:17:05.586Z [WARN] server-5f2bd6d2-bd9d-8524-e381-061d30ee8143: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:05.586Z [INFO] server-5f2bd6d2-bd9d-8524-e381-061d30ee8143: entering candidate state: node="Node at 5f2bd6d2-bd9d-8524-e381-061d30ee8143 [Candidate]" term=3 -2024-05-15T01:17:05.586Z [INFO] server-5f2bd6d2-bd9d-8524-e381-061d30ee8143: election won: term=3 tally=2 -2024-05-15T01:17:05.586Z [INFO] server-5f2bd6d2-bd9d-8524-e381-061d30ee8143: entering leader state: leader="Node at 5f2bd6d2-bd9d-8524-e381-061d30ee8143 [Leader]" -2024-05-15T01:17:05.586Z [INFO] server-5f2bd6d2-bd9d-8524-e381-061d30ee8143: added peer, starting replication: peer=server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389 -2024-05-15T01:17:05.586Z [INFO] server-5f2bd6d2-bd9d-8524-e381-061d30ee8143: added peer, starting replication: peer=server-5ca1766d-944b-1315-477e-8beb3263ad6e -2024-05-15T01:17:05.587Z [INFO] server-5f2bd6d2-bd9d-8524-e381-061d30ee8143: pipelining replication: peer="{Voter server-5ca1766d-944b-1315-477e-8beb3263ad6e 5ca1766d-944b-1315-477e-8beb3263ad6e}" -2024-05-15T01:17:05.587Z [INFO] server-5f2bd6d2-bd9d-8524-e381-061d30ee8143: pipelining replication: peer="{Voter server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389 1b4e3c02-d5b5-bf90-82b2-09a92f8ea389}" -2024-05-15T01:17:05.836Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:05.941Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:05.836110843 +0000 UTC m=+6.377495581 (1 nodes), 40.723µs from start of poll, 495.238984ms from cluster start. Timeout at 2024-05-15 01:17:05.941760156 +0000 UTC m=+6.483144902, 105.649321ms after stability -2024-05-15T01:17:05.941Z [INFO] server-5f2bd6d2-bd9d-8524-e381-061d30ee8143: aborting pipeline replication: peer="{Voter server-5ca1766d-944b-1315-477e-8beb3263ad6e 5ca1766d-944b-1315-477e-8beb3263ad6e}" -2024-05-15T01:17:05.942Z [INFO] server-5f2bd6d2-bd9d-8524-e381-061d30ee8143: aborting pipeline replication: peer="{Voter server-1b4e3c02-d5b5-bf90-82b2-09a92f8ea389 1b4e3c02-d5b5-bf90-82b2-09a92f8ea389}" ---- PASS: TestRaft_RecoverCluster (1.83s) - --- PASS: TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs (0.61s) - --- PASS: TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs (0.62s) - --- PASS: TestRaft_RecoverCluster/snapshot,_with_trailing_logs (0.60s) +2025-06-17T07:42:57.090Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727 Address:0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727} {Suffrage:Voter ID:server-b3fe844f-da43-a0d9-7d77-4f242df9413d Address:b3fe844f-da43-a0d9-7d77-4f242df9413d} {Suffrage:Voter ID:server-9becbada-4913-cc3b-cd39-23dc43f5e983 Address:9becbada-4913-cc3b-cd39-23dc43f5e983}]" +2025-06-17T07:42:57.090Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: entering follower state: follower="Node at 0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727 [Follower]" leader-address= leader-id= +2025-06-17T07:42:57.090Z [INFO] server-b3fe844f-da43-a0d9-7d77-4f242df9413d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727 Address:0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727} {Suffrage:Voter ID:server-b3fe844f-da43-a0d9-7d77-4f242df9413d Address:b3fe844f-da43-a0d9-7d77-4f242df9413d} {Suffrage:Voter ID:server-9becbada-4913-cc3b-cd39-23dc43f5e983 Address:9becbada-4913-cc3b-cd39-23dc43f5e983}]" +2025-06-17T07:42:57.090Z [INFO] server-b3fe844f-da43-a0d9-7d77-4f242df9413d: entering follower state: follower="Node at b3fe844f-da43-a0d9-7d77-4f242df9413d [Follower]" leader-address= leader-id= +2025-06-17T07:42:57.090Z [INFO] server-9becbada-4913-cc3b-cd39-23dc43f5e983: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727 Address:0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727} {Suffrage:Voter ID:server-b3fe844f-da43-a0d9-7d77-4f242df9413d Address:b3fe844f-da43-a0d9-7d77-4f242df9413d} {Suffrage:Voter ID:server-9becbada-4913-cc3b-cd39-23dc43f5e983 Address:9becbada-4913-cc3b-cd39-23dc43f5e983}]" +2025-06-17T07:42:57.090Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:42:57.090Z [INFO] server-9becbada-4913-cc3b-cd39-23dc43f5e983: entering follower state: follower="Node at 9becbada-4913-cc3b-cd39-23dc43f5e983 [Follower]" leader-address= leader-id= +2025-06-17T07:42:57.141Z [WARN] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:42:57.141Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: entering candidate state: node="Node at 0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727 [Candidate]" term=2 +2025-06-17T07:42:57.141Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: election won: term=2 tally=2 +2025-06-17T07:42:57.141Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: entering leader state: leader="Node at 0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727 [Leader]" +2025-06-17T07:42:57.141Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: added peer, starting replication: peer=server-b3fe844f-da43-a0d9-7d77-4f242df9413d +2025-06-17T07:42:57.141Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: added peer, starting replication: peer=server-9becbada-4913-cc3b-cd39-23dc43f5e983 +2025-06-17T07:42:57.141Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: pipelining replication: peer="{Voter server-b3fe844f-da43-a0d9-7d77-4f242df9413d b3fe844f-da43-a0d9-7d77-4f242df9413d}" +2025-06-17T07:42:57.141Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: pipelining replication: peer="{Voter server-9becbada-4913-cc3b-cd39-23dc43f5e983 9becbada-4913-cc3b-cd39-23dc43f5e983}" +2025-06-17T07:42:57.247Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:42:57.141262985 +0000 UTC m=+5.744791686 (1 nodes), 50.290132ms from start of poll, 50.552436ms from cluster start. Timeout at 2025-06-17 07:42:57.247096735 +0000 UTC m=+5.850625427, 105.833741ms after stability +2025-06-17T07:42:57.249Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: aborting pipeline replication: peer="{Voter server-9becbada-4913-cc3b-cd39-23dc43f5e983 9becbada-4913-cc3b-cd39-23dc43f5e983}" +2025-06-17T07:42:57.249Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: aborting pipeline replication: peer="{Voter server-b3fe844f-da43-a0d9-7d77-4f242df9413d b3fe844f-da43-a0d9-7d77-4f242df9413d}" +2025-06-17T07:42:57.249Z [INFO] creating new snapshot: path=/tmp/raft2389018813/snapshots/2-27-1750146177249.tmp +2025-06-17T07:42:57.250Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: starting restore from snapshot: id=2-27-1750146177249 last-index=27 last-term=2 size-in-bytes=168 +2025-06-17T07:42:57.250Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: snapshot restore progress: id=2-27-1750146177249 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-06-17T07:42:57.250Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: restored from snapshot: id=2-27-1750146177249 last-index=27 last-term=2 size-in-bytes=168 +2025-06-17T07:42:57.250Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727 Address:0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727} {Suffrage:Voter ID:server-b3fe844f-da43-a0d9-7d77-4f242df9413d Address:b3fe844f-da43-a0d9-7d77-4f242df9413d} {Suffrage:Voter ID:server-9becbada-4913-cc3b-cd39-23dc43f5e983 Address:9becbada-4913-cc3b-cd39-23dc43f5e983}]" +2025-06-17T07:42:57.250Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: entering follower state: follower="Node at 0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727 [Follower]" leader-address= leader-id= +2025-06-17T07:42:57.250Z [INFO] creating new snapshot: path=/tmp/raft449266425/snapshots/2-27-1750146177250.tmp +2025-06-17T07:42:57.250Z [INFO] server-b3fe844f-da43-a0d9-7d77-4f242df9413d: starting restore from snapshot: id=2-27-1750146177250 last-index=27 last-term=2 size-in-bytes=168 +2025-06-17T07:42:57.251Z [INFO] server-b3fe844f-da43-a0d9-7d77-4f242df9413d: snapshot restore progress: id=2-27-1750146177250 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-06-17T07:42:57.251Z [INFO] server-b3fe844f-da43-a0d9-7d77-4f242df9413d: restored from snapshot: id=2-27-1750146177250 last-index=27 last-term=2 size-in-bytes=168 +2025-06-17T07:42:57.251Z [INFO] server-b3fe844f-da43-a0d9-7d77-4f242df9413d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727 Address:0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727} {Suffrage:Voter ID:server-b3fe844f-da43-a0d9-7d77-4f242df9413d Address:b3fe844f-da43-a0d9-7d77-4f242df9413d} {Suffrage:Voter ID:server-9becbada-4913-cc3b-cd39-23dc43f5e983 Address:9becbada-4913-cc3b-cd39-23dc43f5e983}]" +2025-06-17T07:42:57.251Z [INFO] server-b3fe844f-da43-a0d9-7d77-4f242df9413d: entering follower state: follower="Node at b3fe844f-da43-a0d9-7d77-4f242df9413d [Follower]" leader-address= leader-id= +2025-06-17T07:42:57.251Z [INFO] creating new snapshot: path=/tmp/raft3442519395/snapshots/2-27-1750146177251.tmp +2025-06-17T07:42:57.251Z [INFO] server-9becbada-4913-cc3b-cd39-23dc43f5e983: starting restore from snapshot: id=2-27-1750146177251 last-index=27 last-term=2 size-in-bytes=168 +2025-06-17T07:42:57.251Z [INFO] server-9becbada-4913-cc3b-cd39-23dc43f5e983: snapshot restore progress: id=2-27-1750146177251 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-06-17T07:42:57.251Z [INFO] server-9becbada-4913-cc3b-cd39-23dc43f5e983: restored from snapshot: id=2-27-1750146177251 last-index=27 last-term=2 size-in-bytes=168 +2025-06-17T07:42:57.251Z [INFO] server-9becbada-4913-cc3b-cd39-23dc43f5e983: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727 Address:0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727} {Suffrage:Voter ID:server-b3fe844f-da43-a0d9-7d77-4f242df9413d Address:b3fe844f-da43-a0d9-7d77-4f242df9413d} {Suffrage:Voter ID:server-9becbada-4913-cc3b-cd39-23dc43f5e983 Address:9becbada-4913-cc3b-cd39-23dc43f5e983}]" +2025-06-17T07:42:57.251Z [INFO] server-9becbada-4913-cc3b-cd39-23dc43f5e983: entering follower state: follower="Node at 9becbada-4913-cc3b-cd39-23dc43f5e983 [Follower]" leader-address= leader-id= +2025-06-17T07:42:57.314Z [WARN] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:42:57.314Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: entering candidate state: node="Node at 0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727 [Candidate]" term=3 +2025-06-17T07:42:57.314Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: election won: term=3 tally=2 +2025-06-17T07:42:57.314Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: entering leader state: leader="Node at 0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727 [Leader]" +2025-06-17T07:42:57.314Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: added peer, starting replication: peer=server-b3fe844f-da43-a0d9-7d77-4f242df9413d +2025-06-17T07:42:57.314Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: added peer, starting replication: peer=server-9becbada-4913-cc3b-cd39-23dc43f5e983 +2025-06-17T07:42:57.315Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: pipelining replication: peer="{Voter server-9becbada-4913-cc3b-cd39-23dc43f5e983 9becbada-4913-cc3b-cd39-23dc43f5e983}" +2025-06-17T07:42:57.315Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: pipelining replication: peer="{Voter server-b3fe844f-da43-a0d9-7d77-4f242df9413d b3fe844f-da43-a0d9-7d77-4f242df9413d}" +2025-06-17T07:42:57.567Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:42:57.672Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:42:57.567241904 +0000 UTC m=+6.170770602 (1 nodes), 20.761µs from start of poll, 476.531352ms from cluster start. Timeout at 2025-06-17 07:42:57.672813483 +0000 UTC m=+6.276342166, 105.571564ms after stability +2025-06-17T07:42:57.672Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: aborting pipeline replication: peer="{Voter server-9becbada-4913-cc3b-cd39-23dc43f5e983 9becbada-4913-cc3b-cd39-23dc43f5e983}" +2025-06-17T07:42:57.672Z [INFO] server-0c9ad62a-c4f6-0aa3-d1a7-6f22e4588727: aborting pipeline replication: peer="{Voter server-b3fe844f-da43-a0d9-7d77-4f242df9413d b3fe844f-da43-a0d9-7d77-4f242df9413d}" +--- PASS: TestRaft_RecoverCluster (1.76s) + --- PASS: TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs (0.59s) + --- PASS: TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs (0.59s) + --- PASS: TestRaft_RecoverCluster/snapshot,_with_trailing_logs (0.58s) === RUN TestRaft_HasExistingState raft_test.go:258: Skipping unreliable test --- SKIP: TestRaft_HasExistingState (0.00s) === RUN TestRaft_SingleNode -2024-05-15T01:17:05.943Z [INFO] server-fbd22acc-3eef-66f8-a7d5-3e57020534c4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fbd22acc-3eef-66f8-a7d5-3e57020534c4 Address:fbd22acc-3eef-66f8-a7d5-3e57020534c4}]" -2024-05-15T01:17:05.943Z [INFO] server-fbd22acc-3eef-66f8-a7d5-3e57020534c4: entering follower state: follower="Node at fbd22acc-3eef-66f8-a7d5-3e57020534c4 [Follower]" leader-address= leader-id= -2024-05-15T01:17:06.008Z [WARN] server-fbd22acc-3eef-66f8-a7d5-3e57020534c4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:06.008Z [INFO] server-fbd22acc-3eef-66f8-a7d5-3e57020534c4: entering candidate state: node="Node at fbd22acc-3eef-66f8-a7d5-3e57020534c4 [Candidate]" term=2 -2024-05-15T01:17:06.008Z [INFO] server-fbd22acc-3eef-66f8-a7d5-3e57020534c4: election won: term=2 tally=1 -2024-05-15T01:17:06.008Z [INFO] server-fbd22acc-3eef-66f8-a7d5-3e57020534c4: entering leader state: leader="Node at fbd22acc-3eef-66f8-a7d5-3e57020534c4 [Leader]" ---- PASS: TestRaft_SingleNode (0.07s) +2025-06-17T07:42:57.673Z [INFO] server-e43676f4-eda2-4dbe-7ba7-dbe4713f8258: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e43676f4-eda2-4dbe-7ba7-dbe4713f8258 Address:e43676f4-eda2-4dbe-7ba7-dbe4713f8258}]" +2025-06-17T07:42:57.673Z [INFO] server-e43676f4-eda2-4dbe-7ba7-dbe4713f8258: entering follower state: follower="Node at e43676f4-eda2-4dbe-7ba7-dbe4713f8258 [Follower]" leader-address= leader-id= +2025-06-17T07:42:57.734Z [WARN] server-e43676f4-eda2-4dbe-7ba7-dbe4713f8258: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:42:57.734Z [INFO] server-e43676f4-eda2-4dbe-7ba7-dbe4713f8258: entering candidate state: node="Node at e43676f4-eda2-4dbe-7ba7-dbe4713f8258 [Candidate]" term=2 +2025-06-17T07:42:57.734Z [INFO] server-e43676f4-eda2-4dbe-7ba7-dbe4713f8258: election won: term=2 tally=1 +2025-06-17T07:42:57.734Z [INFO] server-e43676f4-eda2-4dbe-7ba7-dbe4713f8258: entering leader state: leader="Node at e43676f4-eda2-4dbe-7ba7-dbe4713f8258 [Leader]" +--- PASS: TestRaft_SingleNode (0.06s) === RUN TestRaft_TripleNode -2024-05-15T01:17:06.010Z [INFO] server-4d559cea-8196-3f0e-e64b-d07fa95ce0d5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d559cea-8196-3f0e-e64b-d07fa95ce0d5 Address:4d559cea-8196-3f0e-e64b-d07fa95ce0d5} {Suffrage:Voter ID:server-fe04b44e-a7b9-5620-130a-119b940353db Address:fe04b44e-a7b9-5620-130a-119b940353db} {Suffrage:Voter ID:server-5e25c1c1-d50c-c34b-538a-42ed44cdde47 Address:5e25c1c1-d50c-c34b-538a-42ed44cdde47}]" -2024-05-15T01:17:06.010Z [INFO] server-fe04b44e-a7b9-5620-130a-119b940353db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d559cea-8196-3f0e-e64b-d07fa95ce0d5 Address:4d559cea-8196-3f0e-e64b-d07fa95ce0d5} {Suffrage:Voter ID:server-fe04b44e-a7b9-5620-130a-119b940353db Address:fe04b44e-a7b9-5620-130a-119b940353db} {Suffrage:Voter ID:server-5e25c1c1-d50c-c34b-538a-42ed44cdde47 Address:5e25c1c1-d50c-c34b-538a-42ed44cdde47}]" -2024-05-15T01:17:06.010Z [INFO] server-5e25c1c1-d50c-c34b-538a-42ed44cdde47: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d559cea-8196-3f0e-e64b-d07fa95ce0d5 Address:4d559cea-8196-3f0e-e64b-d07fa95ce0d5} {Suffrage:Voter ID:server-fe04b44e-a7b9-5620-130a-119b940353db Address:fe04b44e-a7b9-5620-130a-119b940353db} {Suffrage:Voter ID:server-5e25c1c1-d50c-c34b-538a-42ed44cdde47 Address:5e25c1c1-d50c-c34b-538a-42ed44cdde47}]" -2024-05-15T01:17:06.010Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:06.010Z [INFO] server-4d559cea-8196-3f0e-e64b-d07fa95ce0d5: entering follower state: follower="Node at 4d559cea-8196-3f0e-e64b-d07fa95ce0d5 [Follower]" leader-address= leader-id= -2024-05-15T01:17:06.010Z [INFO] server-fe04b44e-a7b9-5620-130a-119b940353db: entering follower state: follower="Node at fe04b44e-a7b9-5620-130a-119b940353db [Follower]" leader-address= leader-id= -2024-05-15T01:17:06.010Z [INFO] server-5e25c1c1-d50c-c34b-538a-42ed44cdde47: entering follower state: follower="Node at 5e25c1c1-d50c-c34b-538a-42ed44cdde47 [Follower]" leader-address= leader-id= -2024-05-15T01:17:06.084Z [WARN] server-4d559cea-8196-3f0e-e64b-d07fa95ce0d5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:06.084Z [INFO] server-4d559cea-8196-3f0e-e64b-d07fa95ce0d5: entering candidate state: node="Node at 4d559cea-8196-3f0e-e64b-d07fa95ce0d5 [Candidate]" term=2 -2024-05-15T01:17:06.084Z [INFO] server-4d559cea-8196-3f0e-e64b-d07fa95ce0d5: election won: term=2 tally=2 -2024-05-15T01:17:06.084Z [INFO] server-4d559cea-8196-3f0e-e64b-d07fa95ce0d5: entering leader state: leader="Node at 4d559cea-8196-3f0e-e64b-d07fa95ce0d5 [Leader]" -2024-05-15T01:17:06.084Z [INFO] server-4d559cea-8196-3f0e-e64b-d07fa95ce0d5: added peer, starting replication: peer=server-fe04b44e-a7b9-5620-130a-119b940353db -2024-05-15T01:17:06.084Z [INFO] server-4d559cea-8196-3f0e-e64b-d07fa95ce0d5: added peer, starting replication: peer=server-5e25c1c1-d50c-c34b-538a-42ed44cdde47 -2024-05-15T01:17:06.084Z [INFO] server-4d559cea-8196-3f0e-e64b-d07fa95ce0d5: pipelining replication: peer="{Voter server-5e25c1c1-d50c-c34b-538a-42ed44cdde47 5e25c1c1-d50c-c34b-538a-42ed44cdde47}" -2024-05-15T01:17:06.085Z [INFO] server-4d559cea-8196-3f0e-e64b-d07fa95ce0d5: pipelining replication: peer="{Voter server-fe04b44e-a7b9-5620-130a-119b940353db fe04b44e-a7b9-5620-130a-119b940353db}" -2024-05-15T01:17:06.191Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:06.085186028 +0000 UTC m=+6.626570940 (2 nodes), 74.607391ms from start of poll, 75.15161ms from cluster start. Timeout at 2024-05-15 01:17:06.191225772 +0000 UTC m=+6.732610010, 106.03907ms after stability -2024-05-15T01:17:06.191Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:06.296Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:06.191348195 +0000 UTC m=+6.732734784 (1 nodes), 3.164µs from start of poll, 181.315454ms from cluster start. Timeout at 2024-05-15 01:17:06.296763629 +0000 UTC m=+6.838148517, 105.413733ms after stability -2024-05-15T01:17:06.308Z [INFO] server-4d559cea-8196-3f0e-e64b-d07fa95ce0d5: aborting pipeline replication: peer="{Voter server-5e25c1c1-d50c-c34b-538a-42ed44cdde47 5e25c1c1-d50c-c34b-538a-42ed44cdde47}" -2024-05-15T01:17:06.308Z [INFO] server-4d559cea-8196-3f0e-e64b-d07fa95ce0d5: aborting pipeline replication: peer="{Voter server-fe04b44e-a7b9-5620-130a-119b940353db fe04b44e-a7b9-5620-130a-119b940353db}" ---- PASS: TestRaft_TripleNode (0.30s) +2025-06-17T07:42:57.735Z [INFO] server-bdc6051e-236d-7028-ebd7-09bcca756ced: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bdc6051e-236d-7028-ebd7-09bcca756ced Address:bdc6051e-236d-7028-ebd7-09bcca756ced} {Suffrage:Voter ID:server-3561e53a-fbfa-227b-605b-b3db5dcdf8f9 Address:3561e53a-fbfa-227b-605b-b3db5dcdf8f9} {Suffrage:Voter ID:server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4 Address:cb7338ea-5cbb-64e5-b2b6-b1efa67beae4}]" +2025-06-17T07:42:57.735Z [INFO] server-bdc6051e-236d-7028-ebd7-09bcca756ced: entering follower state: follower="Node at bdc6051e-236d-7028-ebd7-09bcca756ced [Follower]" leader-address= leader-id= +2025-06-17T07:42:57.735Z [INFO] server-3561e53a-fbfa-227b-605b-b3db5dcdf8f9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bdc6051e-236d-7028-ebd7-09bcca756ced Address:bdc6051e-236d-7028-ebd7-09bcca756ced} {Suffrage:Voter ID:server-3561e53a-fbfa-227b-605b-b3db5dcdf8f9 Address:3561e53a-fbfa-227b-605b-b3db5dcdf8f9} {Suffrage:Voter ID:server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4 Address:cb7338ea-5cbb-64e5-b2b6-b1efa67beae4}]" +2025-06-17T07:42:57.735Z [INFO] server-3561e53a-fbfa-227b-605b-b3db5dcdf8f9: entering follower state: follower="Node at 3561e53a-fbfa-227b-605b-b3db5dcdf8f9 [Follower]" leader-address= leader-id= +2025-06-17T07:42:57.735Z [INFO] server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bdc6051e-236d-7028-ebd7-09bcca756ced Address:bdc6051e-236d-7028-ebd7-09bcca756ced} {Suffrage:Voter ID:server-3561e53a-fbfa-227b-605b-b3db5dcdf8f9 Address:3561e53a-fbfa-227b-605b-b3db5dcdf8f9} {Suffrage:Voter ID:server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4 Address:cb7338ea-5cbb-64e5-b2b6-b1efa67beae4}]" +2025-06-17T07:42:57.735Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:42:57.735Z [INFO] server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4: entering follower state: follower="Node at cb7338ea-5cbb-64e5-b2b6-b1efa67beae4 [Follower]" leader-address= leader-id= +2025-06-17T07:42:57.787Z [WARN] server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:42:57.787Z [INFO] server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4: entering candidate state: node="Node at cb7338ea-5cbb-64e5-b2b6-b1efa67beae4 [Candidate]" term=2 +2025-06-17T07:42:57.787Z [WARN] server-bdc6051e-236d-7028-ebd7-09bcca756ced: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:42:57.787Z [INFO] server-bdc6051e-236d-7028-ebd7-09bcca756ced: entering candidate state: node="Node at bdc6051e-236d-7028-ebd7-09bcca756ced [Candidate]" term=2 +2025-06-17T07:42:57.787Z [INFO] server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4: election won: term=2 tally=2 +2025-06-17T07:42:57.787Z [INFO] server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4: entering leader state: leader="Node at cb7338ea-5cbb-64e5-b2b6-b1efa67beae4 [Leader]" +2025-06-17T07:42:57.787Z [INFO] server-bdc6051e-236d-7028-ebd7-09bcca756ced: duplicate requestVote for same term: term=2 +2025-06-17T07:42:57.787Z [INFO] server-3561e53a-fbfa-227b-605b-b3db5dcdf8f9: duplicate requestVote for same term: term=2 +2025-06-17T07:42:57.787Z [INFO] server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4: added peer, starting replication: peer=server-bdc6051e-236d-7028-ebd7-09bcca756ced +2025-06-17T07:42:57.787Z [INFO] server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4: added peer, starting replication: peer=server-3561e53a-fbfa-227b-605b-b3db5dcdf8f9 +2025-06-17T07:42:57.787Z [INFO] server-bdc6051e-236d-7028-ebd7-09bcca756ced: entering follower state: follower="Node at bdc6051e-236d-7028-ebd7-09bcca756ced [Follower]" leader-address=cb7338ea-5cbb-64e5-b2b6-b1efa67beae4 leader-id=server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4 +2025-06-17T07:42:57.787Z [WARN] server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4: rejecting vote request since we have a leader: from=bdc6051e-236d-7028-ebd7-09bcca756ced leader=cb7338ea-5cbb-64e5-b2b6-b1efa67beae4 leader-id=server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4 +2025-06-17T07:42:57.787Z [INFO] server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4: pipelining replication: peer="{Voter server-bdc6051e-236d-7028-ebd7-09bcca756ced bdc6051e-236d-7028-ebd7-09bcca756ced}" +2025-06-17T07:42:57.787Z [INFO] server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4: pipelining replication: peer="{Voter server-3561e53a-fbfa-227b-605b-b3db5dcdf8f9 3561e53a-fbfa-227b-605b-b3db5dcdf8f9}" +2025-06-17T07:42:57.893Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:42:57.787751531 +0000 UTC m=+6.391280223 (2 nodes), 52.343482ms from start of poll, 52.613208ms from cluster start. Timeout at 2025-06-17 07:42:57.893773338 +0000 UTC m=+6.497302020, 106.021797ms after stability +2025-06-17T07:42:57.893Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:42:57.999Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:42:57.893821056 +0000 UTC m=+6.497349741 (1 nodes), 560ns from start of poll, 158.682726ms from cluster start. Timeout at 2025-06-17 07:42:57.999322207 +0000 UTC m=+6.602850892, 105.501151ms after stability +2025-06-17T07:42:58.010Z [INFO] server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4: aborting pipeline replication: peer="{Voter server-3561e53a-fbfa-227b-605b-b3db5dcdf8f9 3561e53a-fbfa-227b-605b-b3db5dcdf8f9}" +2025-06-17T07:42:58.010Z [INFO] server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4: aborting pipeline replication: peer="{Voter server-bdc6051e-236d-7028-ebd7-09bcca756ced bdc6051e-236d-7028-ebd7-09bcca756ced}" +2025-06-17T07:42:58.510Z [ERROR] server-cb7338ea-5cbb-64e5-b2b6-b1efa67beae4: failed to heartbeat to: peer=bdc6051e-236d-7028-ebd7-09bcca756ced backoff time=10ms error="command timed out" +--- PASS: TestRaft_TripleNode (0.78s) === RUN TestRaft_LeaderFail -2024-05-15T01:17:06.310Z [INFO] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-30e34a87-7dc8-512c-ba0d-4404fe120fc3 Address:30e34a87-7dc8-512c-ba0d-4404fe120fc3} {Suffrage:Voter ID:server-ecbba42c-35ff-c566-9c03-0a2e6422992b Address:ecbba42c-35ff-c566-9c03-0a2e6422992b} {Suffrage:Voter ID:server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 Address:b135f45e-cb7d-0b1a-4b80-1815584f45a9}]" -2024-05-15T01:17:06.310Z [INFO] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: entering follower state: follower="Node at 30e34a87-7dc8-512c-ba0d-4404fe120fc3 [Follower]" leader-address= leader-id= -2024-05-15T01:17:06.310Z [INFO] server-ecbba42c-35ff-c566-9c03-0a2e6422992b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-30e34a87-7dc8-512c-ba0d-4404fe120fc3 Address:30e34a87-7dc8-512c-ba0d-4404fe120fc3} {Suffrage:Voter ID:server-ecbba42c-35ff-c566-9c03-0a2e6422992b Address:ecbba42c-35ff-c566-9c03-0a2e6422992b} {Suffrage:Voter ID:server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 Address:b135f45e-cb7d-0b1a-4b80-1815584f45a9}]" -2024-05-15T01:17:06.312Z [INFO] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-30e34a87-7dc8-512c-ba0d-4404fe120fc3 Address:30e34a87-7dc8-512c-ba0d-4404fe120fc3} {Suffrage:Voter ID:server-ecbba42c-35ff-c566-9c03-0a2e6422992b Address:ecbba42c-35ff-c566-9c03-0a2e6422992b} {Suffrage:Voter ID:server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 Address:b135f45e-cb7d-0b1a-4b80-1815584f45a9}]" -2024-05-15T01:17:06.312Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:06.312Z [INFO] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: entering follower state: follower="Node at b135f45e-cb7d-0b1a-4b80-1815584f45a9 [Follower]" leader-address= leader-id= -2024-05-15T01:17:06.312Z [INFO] server-ecbba42c-35ff-c566-9c03-0a2e6422992b: entering follower state: follower="Node at ecbba42c-35ff-c566-9c03-0a2e6422992b [Follower]" leader-address= leader-id= -2024-05-15T01:17:06.364Z [WARN] server-ecbba42c-35ff-c566-9c03-0a2e6422992b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:06.364Z [WARN] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:06.364Z [INFO] server-ecbba42c-35ff-c566-9c03-0a2e6422992b: entering candidate state: node="Node at ecbba42c-35ff-c566-9c03-0a2e6422992b [Candidate]" term=2 -2024-05-15T01:17:06.364Z [INFO] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: entering candidate state: node="Node at b135f45e-cb7d-0b1a-4b80-1815584f45a9 [Candidate]" term=2 -2024-05-15T01:17:06.364Z [INFO] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: duplicate requestVote for same term: term=2 -2024-05-15T01:17:06.364Z [INFO] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: election won: term=2 tally=2 -2024-05-15T01:17:06.364Z [INFO] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: entering leader state: leader="Node at b135f45e-cb7d-0b1a-4b80-1815584f45a9 [Leader]" -2024-05-15T01:17:06.364Z [INFO] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: added peer, starting replication: peer=server-30e34a87-7dc8-512c-ba0d-4404fe120fc3 -2024-05-15T01:17:06.364Z [INFO] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: duplicate requestVote for same term: term=2 -2024-05-15T01:17:06.364Z [INFO] server-ecbba42c-35ff-c566-9c03-0a2e6422992b: duplicate requestVote for same term: term=2 -2024-05-15T01:17:06.364Z [INFO] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: added peer, starting replication: peer=server-ecbba42c-35ff-c566-9c03-0a2e6422992b -2024-05-15T01:17:06.365Z [INFO] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: pipelining replication: peer="{Voter server-30e34a87-7dc8-512c-ba0d-4404fe120fc3 30e34a87-7dc8-512c-ba0d-4404fe120fc3}" -2024-05-15T01:17:06.365Z [INFO] server-ecbba42c-35ff-c566-9c03-0a2e6422992b: entering follower state: follower="Node at ecbba42c-35ff-c566-9c03-0a2e6422992b [Follower]" leader-address=b135f45e-cb7d-0b1a-4b80-1815584f45a9 leader-id=server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 -2024-05-15T01:17:06.365Z [INFO] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: pipelining replication: peer="{Voter server-ecbba42c-35ff-c566-9c03-0a2e6422992b ecbba42c-35ff-c566-9c03-0a2e6422992b}" -2024-05-15T01:17:06.471Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:06.365331018 +0000 UTC m=+6.906715256 (2 nodes), 53.210164ms from start of poll, 55.218121ms from cluster start. Timeout at 2024-05-15 01:17:06.471436402 +0000 UTC m=+7.012821177, 106.105921ms after stability -2024-05-15T01:17:06.471Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:06.577Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:06.471625563 +0000 UTC m=+7.013009801 (1 nodes), 2.762µs from start of poll, 161.512666ms from cluster start. Timeout at 2024-05-15 01:17:06.577501682 +0000 UTC m=+7.118886450, 105.876649ms after stability - raft_test.go:378: [INFO] Disconnecting Node at b135f45e-cb7d-0b1a-4b80-1815584f45a9 [Leader] -2024-05-15T01:17:06.592Z [ERROR] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to heartbeat to: peer=ecbba42c-35ff-c566-9c03-0a2e6422992b backoff time=10ms error="failed to connect to peer: ecbba42c-35ff-c566-9c03-0a2e6422992b" -2024-05-15T01:17:06.592Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:06.593Z [ERROR] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to pipeline appendEntries: peer="{Voter server-30e34a87-7dc8-512c-ba0d-4404fe120fc3 30e34a87-7dc8-512c-ba0d-4404fe120fc3}" error="append pipeline closed" -2024-05-15T01:17:06.593Z [INFO] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: aborting pipeline replication: peer="{Voter server-30e34a87-7dc8-512c-ba0d-4404fe120fc3 30e34a87-7dc8-512c-ba0d-4404fe120fc3}" -2024-05-15T01:17:06.597Z [ERROR] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to heartbeat to: peer=30e34a87-7dc8-512c-ba0d-4404fe120fc3 backoff time=10ms error="failed to connect to peer: 30e34a87-7dc8-512c-ba0d-4404fe120fc3" -2024-05-15T01:17:06.598Z [ERROR] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to pipeline appendEntries: peer="{Voter server-ecbba42c-35ff-c566-9c03-0a2e6422992b ecbba42c-35ff-c566-9c03-0a2e6422992b}" error="append pipeline closed" -2024-05-15T01:17:06.598Z [INFO] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: aborting pipeline replication: peer="{Voter server-ecbba42c-35ff-c566-9c03-0a2e6422992b ecbba42c-35ff-c566-9c03-0a2e6422992b}" -2024-05-15T01:17:06.602Z [ERROR] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to appendEntries to: peer="{Voter server-30e34a87-7dc8-512c-ba0d-4404fe120fc3 30e34a87-7dc8-512c-ba0d-4404fe120fc3}" error="failed to connect to peer: 30e34a87-7dc8-512c-ba0d-4404fe120fc3" -2024-05-15T01:17:06.606Z [ERROR] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to appendEntries to: peer="{Voter server-ecbba42c-35ff-c566-9c03-0a2e6422992b ecbba42c-35ff-c566-9c03-0a2e6422992b}" error="failed to connect to peer: ecbba42c-35ff-c566-9c03-0a2e6422992b" -2024-05-15T01:17:06.610Z [ERROR] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to heartbeat to: peer=ecbba42c-35ff-c566-9c03-0a2e6422992b backoff time=10ms error="failed to connect to peer: ecbba42c-35ff-c566-9c03-0a2e6422992b" -2024-05-15T01:17:06.618Z [ERROR] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to heartbeat to: peer=30e34a87-7dc8-512c-ba0d-4404fe120fc3 backoff time=10ms error="failed to connect to peer: 30e34a87-7dc8-512c-ba0d-4404fe120fc3" -2024-05-15T01:17:06.619Z [ERROR] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to appendEntries to: peer="{Voter server-30e34a87-7dc8-512c-ba0d-4404fe120fc3 30e34a87-7dc8-512c-ba0d-4404fe120fc3}" error="failed to connect to peer: 30e34a87-7dc8-512c-ba0d-4404fe120fc3" -2024-05-15T01:17:06.627Z [ERROR] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to heartbeat to: peer=ecbba42c-35ff-c566-9c03-0a2e6422992b backoff time=10ms error="failed to connect to peer: ecbba42c-35ff-c566-9c03-0a2e6422992b" -2024-05-15T01:17:06.627Z [ERROR] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to appendEntries to: peer="{Voter server-ecbba42c-35ff-c566-9c03-0a2e6422992b ecbba42c-35ff-c566-9c03-0a2e6422992b}" error="failed to connect to peer: ecbba42c-35ff-c566-9c03-0a2e6422992b" -2024-05-15T01:17:06.639Z [ERROR] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to heartbeat to: peer=30e34a87-7dc8-512c-ba0d-4404fe120fc3 backoff time=10ms error="failed to connect to peer: 30e34a87-7dc8-512c-ba0d-4404fe120fc3" -2024-05-15T01:17:06.639Z [ERROR] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to appendEntries to: peer="{Voter server-30e34a87-7dc8-512c-ba0d-4404fe120fc3 30e34a87-7dc8-512c-ba0d-4404fe120fc3}" error="failed to connect to peer: 30e34a87-7dc8-512c-ba0d-4404fe120fc3" -2024-05-15T01:17:06.639Z [WARN] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to contact: server-id=server-30e34a87-7dc8-512c-ba0d-4404fe120fc3 time=51.343607ms -2024-05-15T01:17:06.639Z [WARN] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to contact: server-id=server-ecbba42c-35ff-c566-9c03-0a2e6422992b time=51.476755ms -2024-05-15T01:17:06.639Z [WARN] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to contact quorum of nodes, stepping down -2024-05-15T01:17:06.639Z [INFO] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: entering follower state: follower="Node at b135f45e-cb7d-0b1a-4b80-1815584f45a9 [Follower]" leader-address= leader-id= -2024-05-15T01:17:06.645Z [ERROR] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to appendEntries to: peer="{Voter server-ecbba42c-35ff-c566-9c03-0a2e6422992b ecbba42c-35ff-c566-9c03-0a2e6422992b}" error="failed to connect to peer: ecbba42c-35ff-c566-9c03-0a2e6422992b" -2024-05-15T01:17:06.664Z [WARN] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: heartbeat timeout reached, starting election: last-leader-addr=b135f45e-cb7d-0b1a-4b80-1815584f45a9 last-leader-id=server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 -2024-05-15T01:17:06.664Z [INFO] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: entering candidate state: node="Node at 30e34a87-7dc8-512c-ba0d-4404fe120fc3 [Candidate]" term=3 -2024-05-15T01:17:06.664Z [ERROR] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: failed to make requestVote RPC: target="{Voter server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 b135f45e-cb7d-0b1a-4b80-1815584f45a9}" error="failed to connect to peer: b135f45e-cb7d-0b1a-4b80-1815584f45a9" term=3 -2024-05-15T01:17:06.664Z [WARN] server-ecbba42c-35ff-c566-9c03-0a2e6422992b: rejecting vote request since we have a leader: from=30e34a87-7dc8-512c-ba0d-4404fe120fc3 leader=b135f45e-cb7d-0b1a-4b80-1815584f45a9 leader-id=server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 -2024-05-15T01:17:06.724Z [WARN] server-ecbba42c-35ff-c566-9c03-0a2e6422992b: heartbeat timeout reached, starting election: last-leader-addr=b135f45e-cb7d-0b1a-4b80-1815584f45a9 last-leader-id=server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 -2024-05-15T01:17:06.724Z [INFO] server-ecbba42c-35ff-c566-9c03-0a2e6422992b: entering candidate state: node="Node at ecbba42c-35ff-c566-9c03-0a2e6422992b [Candidate]" term=3 -2024-05-15T01:17:06.724Z [ERROR] server-ecbba42c-35ff-c566-9c03-0a2e6422992b: failed to make requestVote RPC: target="{Voter server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 b135f45e-cb7d-0b1a-4b80-1815584f45a9}" error="failed to connect to peer: b135f45e-cb7d-0b1a-4b80-1815584f45a9" term=3 -2024-05-15T01:17:06.724Z [INFO] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: duplicate requestVote for same term: term=3 -2024-05-15T01:17:06.739Z [WARN] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:06.739Z [INFO] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: entering candidate state: node="Node at b135f45e-cb7d-0b1a-4b80-1815584f45a9 [Candidate]" term=3 -2024-05-15T01:17:06.739Z [ERROR] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to make requestVote RPC: target="{Voter server-ecbba42c-35ff-c566-9c03-0a2e6422992b ecbba42c-35ff-c566-9c03-0a2e6422992b}" error="failed to connect to peer: ecbba42c-35ff-c566-9c03-0a2e6422992b" term=3 -2024-05-15T01:17:06.740Z [ERROR] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to make requestVote RPC: target="{Voter server-30e34a87-7dc8-512c-ba0d-4404fe120fc3 30e34a87-7dc8-512c-ba0d-4404fe120fc3}" error="failed to connect to peer: 30e34a87-7dc8-512c-ba0d-4404fe120fc3" term=3 -2024-05-15T01:17:06.743Z [WARN] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: Election timeout reached, restarting election -2024-05-15T01:17:06.743Z [INFO] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: entering candidate state: node="Node at 30e34a87-7dc8-512c-ba0d-4404fe120fc3 [Candidate]" term=4 -2024-05-15T01:17:06.743Z [ERROR] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: failed to make requestVote RPC: target="{Voter server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 b135f45e-cb7d-0b1a-4b80-1815584f45a9}" error="failed to connect to peer: b135f45e-cb7d-0b1a-4b80-1815584f45a9" term=4 -2024-05-15T01:17:06.743Z [INFO] server-ecbba42c-35ff-c566-9c03-0a2e6422992b: entering follower state: follower="Node at ecbba42c-35ff-c566-9c03-0a2e6422992b [Follower]" leader-address= leader-id= -2024-05-15T01:17:06.743Z [INFO] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: election won: term=4 tally=2 -2024-05-15T01:17:06.743Z [INFO] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: entering leader state: leader="Node at 30e34a87-7dc8-512c-ba0d-4404fe120fc3 [Leader]" -2024-05-15T01:17:06.743Z [INFO] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: added peer, starting replication: peer=server-ecbba42c-35ff-c566-9c03-0a2e6422992b -2024-05-15T01:17:06.743Z [INFO] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: added peer, starting replication: peer=server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 -2024-05-15T01:17:06.743Z [ERROR] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: failed to appendEntries to: peer="{Voter server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 b135f45e-cb7d-0b1a-4b80-1815584f45a9}" error="failed to connect to peer: b135f45e-cb7d-0b1a-4b80-1815584f45a9" -2024-05-15T01:17:06.743Z [INFO] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: pipelining replication: peer="{Voter server-ecbba42c-35ff-c566-9c03-0a2e6422992b ecbba42c-35ff-c566-9c03-0a2e6422992b}" -2024-05-15T01:17:06.753Z [ERROR] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: failed to heartbeat to: peer=b135f45e-cb7d-0b1a-4b80-1815584f45a9 backoff time=10ms error="failed to connect to peer: b135f45e-cb7d-0b1a-4b80-1815584f45a9" -2024-05-15T01:17:06.760Z [ERROR] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: failed to appendEntries to: peer="{Voter server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 b135f45e-cb7d-0b1a-4b80-1815584f45a9}" error="failed to connect to peer: b135f45e-cb7d-0b1a-4b80-1815584f45a9" -2024-05-15T01:17:06.771Z [ERROR] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: failed to heartbeat to: peer=b135f45e-cb7d-0b1a-4b80-1815584f45a9 backoff time=10ms error="failed to connect to peer: b135f45e-cb7d-0b1a-4b80-1815584f45a9" -2024-05-15T01:17:06.777Z [ERROR] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: failed to appendEntries to: peer="{Voter server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 b135f45e-cb7d-0b1a-4b80-1815584f45a9}" error="failed to connect to peer: b135f45e-cb7d-0b1a-4b80-1815584f45a9" -2024-05-15T01:17:06.791Z [ERROR] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: failed to heartbeat to: peer=b135f45e-cb7d-0b1a-4b80-1815584f45a9 backoff time=10ms error="failed to connect to peer: b135f45e-cb7d-0b1a-4b80-1815584f45a9" -2024-05-15T01:17:06.794Z [WARN] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: failed to contact: server-id=server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 time=50.864018ms -2024-05-15T01:17:06.808Z [ERROR] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: failed to heartbeat to: peer=b135f45e-cb7d-0b1a-4b80-1815584f45a9 backoff time=20ms error="failed to connect to peer: b135f45e-cb7d-0b1a-4b80-1815584f45a9" -2024-05-15T01:17:06.808Z [ERROR] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: failed to appendEntries to: peer="{Voter server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 b135f45e-cb7d-0b1a-4b80-1815584f45a9}" error="failed to connect to peer: b135f45e-cb7d-0b1a-4b80-1815584f45a9" -2024-05-15T01:17:06.818Z [WARN] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: Election timeout reached, restarting election -2024-05-15T01:17:06.818Z [INFO] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: entering candidate state: node="Node at b135f45e-cb7d-0b1a-4b80-1815584f45a9 [Candidate]" term=4 -2024-05-15T01:17:06.818Z [ERROR] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to make requestVote RPC: target="{Voter server-ecbba42c-35ff-c566-9c03-0a2e6422992b ecbba42c-35ff-c566-9c03-0a2e6422992b}" error="failed to connect to peer: ecbba42c-35ff-c566-9c03-0a2e6422992b" term=4 -2024-05-15T01:17:06.818Z [ERROR] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: failed to make requestVote RPC: target="{Voter server-30e34a87-7dc8-512c-ba0d-4404fe120fc3 30e34a87-7dc8-512c-ba0d-4404fe120fc3}" error="failed to connect to peer: 30e34a87-7dc8-512c-ba0d-4404fe120fc3" term=4 -2024-05-15T01:17:06.837Z [ERROR] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: failed to heartbeat to: peer=b135f45e-cb7d-0b1a-4b80-1815584f45a9 backoff time=25ms error="failed to connect to peer: b135f45e-cb7d-0b1a-4b80-1815584f45a9" -2024-05-15T01:17:06.841Z [WARN] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: failed to contact: server-id=server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 time=98.165798ms -2024-05-15T01:17:06.849Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:06.743556013 +0000 UTC m=+7.284940251 (1 nodes), 151.488852ms from start of poll, 433.443116ms from cluster start. Timeout at 2024-05-15 01:17:06.848970611 +0000 UTC m=+7.390355398, 105.415147ms after stability - raft_test.go:413: [INFO] Reconnecting Node at b135f45e-cb7d-0b1a-4b80-1815584f45a9 [Candidate] -2024-05-15T01:17:06.855Z [INFO] server-b135f45e-cb7d-0b1a-4b80-1815584f45a9: entering follower state: follower="Node at b135f45e-cb7d-0b1a-4b80-1815584f45a9 [Follower]" leader-address=30e34a87-7dc8-512c-ba0d-4404fe120fc3 leader-id=server-30e34a87-7dc8-512c-ba0d-4404fe120fc3 -2024-05-15T01:17:06.856Z [INFO] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: pipelining replication: peer="{Voter server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 b135f45e-cb7d-0b1a-4b80-1815584f45a9}" -2024-05-15T01:17:06.861Z [INFO] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: aborting pipeline replication: peer="{Voter server-b135f45e-cb7d-0b1a-4b80-1815584f45a9 b135f45e-cb7d-0b1a-4b80-1815584f45a9}" -2024-05-15T01:17:06.861Z [INFO] server-30e34a87-7dc8-512c-ba0d-4404fe120fc3: aborting pipeline replication: peer="{Voter server-ecbba42c-35ff-c566-9c03-0a2e6422992b ecbba42c-35ff-c566-9c03-0a2e6422992b}" ---- PASS: TestRaft_LeaderFail (0.55s) +2025-06-17T07:42:58.511Z [INFO] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-29a889ec-a70f-2d45-4e25-79aaf25b8aff Address:29a889ec-a70f-2d45-4e25-79aaf25b8aff} {Suffrage:Voter ID:server-8109375d-fbf3-793d-4080-fd218e10f9a4 Address:8109375d-fbf3-793d-4080-fd218e10f9a4} {Suffrage:Voter ID:server-3311b05e-d15c-7e0c-0575-f83a787dcd34 Address:3311b05e-d15c-7e0c-0575-f83a787dcd34}]" +2025-06-17T07:42:58.511Z [INFO] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: entering follower state: follower="Node at 29a889ec-a70f-2d45-4e25-79aaf25b8aff [Follower]" leader-address= leader-id= +2025-06-17T07:42:58.511Z [INFO] server-8109375d-fbf3-793d-4080-fd218e10f9a4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-29a889ec-a70f-2d45-4e25-79aaf25b8aff Address:29a889ec-a70f-2d45-4e25-79aaf25b8aff} {Suffrage:Voter ID:server-8109375d-fbf3-793d-4080-fd218e10f9a4 Address:8109375d-fbf3-793d-4080-fd218e10f9a4} {Suffrage:Voter ID:server-3311b05e-d15c-7e0c-0575-f83a787dcd34 Address:3311b05e-d15c-7e0c-0575-f83a787dcd34}]" +2025-06-17T07:42:58.511Z [INFO] server-8109375d-fbf3-793d-4080-fd218e10f9a4: entering follower state: follower="Node at 8109375d-fbf3-793d-4080-fd218e10f9a4 [Follower]" leader-address= leader-id= +2025-06-17T07:42:58.511Z [INFO] server-3311b05e-d15c-7e0c-0575-f83a787dcd34: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-29a889ec-a70f-2d45-4e25-79aaf25b8aff Address:29a889ec-a70f-2d45-4e25-79aaf25b8aff} {Suffrage:Voter ID:server-8109375d-fbf3-793d-4080-fd218e10f9a4 Address:8109375d-fbf3-793d-4080-fd218e10f9a4} {Suffrage:Voter ID:server-3311b05e-d15c-7e0c-0575-f83a787dcd34 Address:3311b05e-d15c-7e0c-0575-f83a787dcd34}]" +2025-06-17T07:42:58.511Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:42:58.511Z [INFO] server-3311b05e-d15c-7e0c-0575-f83a787dcd34: entering follower state: follower="Node at 3311b05e-d15c-7e0c-0575-f83a787dcd34 [Follower]" leader-address= leader-id= +2025-06-17T07:42:58.566Z [WARN] server-8109375d-fbf3-793d-4080-fd218e10f9a4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:42:58.566Z [INFO] server-8109375d-fbf3-793d-4080-fd218e10f9a4: entering candidate state: node="Node at 8109375d-fbf3-793d-4080-fd218e10f9a4 [Candidate]" term=2 +2025-06-17T07:42:58.566Z [INFO] server-8109375d-fbf3-793d-4080-fd218e10f9a4: election won: term=2 tally=2 +2025-06-17T07:42:58.566Z [INFO] server-8109375d-fbf3-793d-4080-fd218e10f9a4: entering leader state: leader="Node at 8109375d-fbf3-793d-4080-fd218e10f9a4 [Leader]" +2025-06-17T07:42:58.566Z [INFO] server-8109375d-fbf3-793d-4080-fd218e10f9a4: added peer, starting replication: peer=server-29a889ec-a70f-2d45-4e25-79aaf25b8aff +2025-06-17T07:42:58.566Z [INFO] server-8109375d-fbf3-793d-4080-fd218e10f9a4: added peer, starting replication: peer=server-3311b05e-d15c-7e0c-0575-f83a787dcd34 +2025-06-17T07:42:58.566Z [INFO] server-8109375d-fbf3-793d-4080-fd218e10f9a4: pipelining replication: peer="{Voter server-3311b05e-d15c-7e0c-0575-f83a787dcd34 3311b05e-d15c-7e0c-0575-f83a787dcd34}" +2025-06-17T07:42:58.566Z [INFO] server-8109375d-fbf3-793d-4080-fd218e10f9a4: pipelining replication: peer="{Voter server-29a889ec-a70f-2d45-4e25-79aaf25b8aff 29a889ec-a70f-2d45-4e25-79aaf25b8aff}" +2025-06-17T07:42:58.672Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:42:58.566460608 +0000 UTC m=+7.169989303 (2 nodes), 54.641319ms from start of poll, 54.91546ms from cluster start. Timeout at 2025-06-17 07:42:58.672515196 +0000 UTC m=+7.276043888, 106.054585ms after stability +2025-06-17T07:42:58.672Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:42:58.778Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:42:58.672562407 +0000 UTC m=+7.276091094 (1 nodes), 486ns from start of poll, 161.017251ms from cluster start. Timeout at 2025-06-17 07:42:58.778116864 +0000 UTC m=+7.381645547, 105.554453ms after stability + raft_test.go:378: [INFO] Disconnecting Node at 8109375d-fbf3-793d-4080-fd218e10f9a4 [Leader] +2025-06-17T07:42:58.789Z [ERROR] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to heartbeat to: peer=29a889ec-a70f-2d45-4e25-79aaf25b8aff backoff time=10ms error="failed to connect to peer: 29a889ec-a70f-2d45-4e25-79aaf25b8aff" +2025-06-17T07:42:58.790Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:42:58.791Z [ERROR] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to heartbeat to: peer=3311b05e-d15c-7e0c-0575-f83a787dcd34 backoff time=10ms error="failed to connect to peer: 3311b05e-d15c-7e0c-0575-f83a787dcd34" +2025-06-17T07:42:58.791Z [ERROR] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to pipeline appendEntries: peer="{Voter server-3311b05e-d15c-7e0c-0575-f83a787dcd34 3311b05e-d15c-7e0c-0575-f83a787dcd34}" error="append pipeline closed" +2025-06-17T07:42:58.791Z [INFO] server-8109375d-fbf3-793d-4080-fd218e10f9a4: aborting pipeline replication: peer="{Voter server-3311b05e-d15c-7e0c-0575-f83a787dcd34 3311b05e-d15c-7e0c-0575-f83a787dcd34}" +2025-06-17T07:42:58.793Z [ERROR] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to pipeline appendEntries: peer="{Voter server-29a889ec-a70f-2d45-4e25-79aaf25b8aff 29a889ec-a70f-2d45-4e25-79aaf25b8aff}" error="append pipeline closed" +2025-06-17T07:42:58.793Z [INFO] server-8109375d-fbf3-793d-4080-fd218e10f9a4: aborting pipeline replication: peer="{Voter server-29a889ec-a70f-2d45-4e25-79aaf25b8aff 29a889ec-a70f-2d45-4e25-79aaf25b8aff}" +2025-06-17T07:42:58.800Z [ERROR] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to appendEntries to: peer="{Voter server-29a889ec-a70f-2d45-4e25-79aaf25b8aff 29a889ec-a70f-2d45-4e25-79aaf25b8aff}" error="failed to connect to peer: 29a889ec-a70f-2d45-4e25-79aaf25b8aff" +2025-06-17T07:42:58.800Z [ERROR] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to appendEntries to: peer="{Voter server-3311b05e-d15c-7e0c-0575-f83a787dcd34 3311b05e-d15c-7e0c-0575-f83a787dcd34}" error="failed to connect to peer: 3311b05e-d15c-7e0c-0575-f83a787dcd34" +2025-06-17T07:42:58.807Z [ERROR] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to heartbeat to: peer=29a889ec-a70f-2d45-4e25-79aaf25b8aff backoff time=10ms error="failed to connect to peer: 29a889ec-a70f-2d45-4e25-79aaf25b8aff" +2025-06-17T07:42:58.811Z [ERROR] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to heartbeat to: peer=3311b05e-d15c-7e0c-0575-f83a787dcd34 backoff time=10ms error="failed to connect to peer: 3311b05e-d15c-7e0c-0575-f83a787dcd34" +2025-06-17T07:42:58.818Z [ERROR] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to appendEntries to: peer="{Voter server-3311b05e-d15c-7e0c-0575-f83a787dcd34 3311b05e-d15c-7e0c-0575-f83a787dcd34}" error="failed to connect to peer: 3311b05e-d15c-7e0c-0575-f83a787dcd34" +2025-06-17T07:42:58.819Z [ERROR] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to appendEntries to: peer="{Voter server-29a889ec-a70f-2d45-4e25-79aaf25b8aff 29a889ec-a70f-2d45-4e25-79aaf25b8aff}" error="failed to connect to peer: 29a889ec-a70f-2d45-4e25-79aaf25b8aff" +2025-06-17T07:42:58.826Z [ERROR] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to heartbeat to: peer=29a889ec-a70f-2d45-4e25-79aaf25b8aff backoff time=10ms error="failed to connect to peer: 29a889ec-a70f-2d45-4e25-79aaf25b8aff" +2025-06-17T07:42:58.830Z [ERROR] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to heartbeat to: peer=3311b05e-d15c-7e0c-0575-f83a787dcd34 backoff time=10ms error="failed to connect to peer: 3311b05e-d15c-7e0c-0575-f83a787dcd34" +2025-06-17T07:42:58.835Z [WARN] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to contact: server-id=server-29a889ec-a70f-2d45-4e25-79aaf25b8aff time=50.284928ms +2025-06-17T07:42:58.835Z [WARN] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to contact: server-id=server-3311b05e-d15c-7e0c-0575-f83a787dcd34 time=50.365668ms +2025-06-17T07:42:58.835Z [WARN] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to contact quorum of nodes, stepping down +2025-06-17T07:42:58.835Z [INFO] server-8109375d-fbf3-793d-4080-fd218e10f9a4: entering follower state: follower="Node at 8109375d-fbf3-793d-4080-fd218e10f9a4 [Follower]" leader-address= leader-id= +2025-06-17T07:42:58.838Z [ERROR] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to appendEntries to: peer="{Voter server-3311b05e-d15c-7e0c-0575-f83a787dcd34 3311b05e-d15c-7e0c-0575-f83a787dcd34}" error="failed to connect to peer: 3311b05e-d15c-7e0c-0575-f83a787dcd34" +2025-06-17T07:42:58.840Z [ERROR] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to appendEntries to: peer="{Voter server-29a889ec-a70f-2d45-4e25-79aaf25b8aff 29a889ec-a70f-2d45-4e25-79aaf25b8aff}" error="failed to connect to peer: 29a889ec-a70f-2d45-4e25-79aaf25b8aff" +2025-06-17T07:42:58.841Z [WARN] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: heartbeat timeout reached, starting election: last-leader-addr=8109375d-fbf3-793d-4080-fd218e10f9a4 last-leader-id=server-8109375d-fbf3-793d-4080-fd218e10f9a4 +2025-06-17T07:42:58.841Z [INFO] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: entering candidate state: node="Node at 29a889ec-a70f-2d45-4e25-79aaf25b8aff [Candidate]" term=3 +2025-06-17T07:42:58.841Z [WARN] server-3311b05e-d15c-7e0c-0575-f83a787dcd34: rejecting vote request since we have a leader: from=29a889ec-a70f-2d45-4e25-79aaf25b8aff leader=8109375d-fbf3-793d-4080-fd218e10f9a4 leader-id=server-8109375d-fbf3-793d-4080-fd218e10f9a4 +2025-06-17T07:42:58.841Z [ERROR] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: failed to make requestVote RPC: target="{Voter server-8109375d-fbf3-793d-4080-fd218e10f9a4 8109375d-fbf3-793d-4080-fd218e10f9a4}" error="failed to connect to peer: 8109375d-fbf3-793d-4080-fd218e10f9a4" term=3 +2025-06-17T07:42:58.890Z [WARN] server-3311b05e-d15c-7e0c-0575-f83a787dcd34: heartbeat timeout reached, starting election: last-leader-addr=8109375d-fbf3-793d-4080-fd218e10f9a4 last-leader-id=server-8109375d-fbf3-793d-4080-fd218e10f9a4 +2025-06-17T07:42:58.890Z [INFO] server-3311b05e-d15c-7e0c-0575-f83a787dcd34: entering candidate state: node="Node at 3311b05e-d15c-7e0c-0575-f83a787dcd34 [Candidate]" term=3 +2025-06-17T07:42:58.890Z [ERROR] server-3311b05e-d15c-7e0c-0575-f83a787dcd34: failed to make requestVote RPC: target="{Voter server-8109375d-fbf3-793d-4080-fd218e10f9a4 8109375d-fbf3-793d-4080-fd218e10f9a4}" error="failed to connect to peer: 8109375d-fbf3-793d-4080-fd218e10f9a4" term=3 +2025-06-17T07:42:58.890Z [INFO] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: duplicate requestVote for same term: term=3 +2025-06-17T07:42:58.891Z [WARN] server-8109375d-fbf3-793d-4080-fd218e10f9a4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:42:58.891Z [INFO] server-8109375d-fbf3-793d-4080-fd218e10f9a4: entering candidate state: node="Node at 8109375d-fbf3-793d-4080-fd218e10f9a4 [Candidate]" term=3 +2025-06-17T07:42:58.891Z [ERROR] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to make requestVote RPC: target="{Voter server-3311b05e-d15c-7e0c-0575-f83a787dcd34 3311b05e-d15c-7e0c-0575-f83a787dcd34}" error="failed to connect to peer: 3311b05e-d15c-7e0c-0575-f83a787dcd34" term=3 +2025-06-17T07:42:58.891Z [ERROR] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to make requestVote RPC: target="{Voter server-29a889ec-a70f-2d45-4e25-79aaf25b8aff 29a889ec-a70f-2d45-4e25-79aaf25b8aff}" error="failed to connect to peer: 29a889ec-a70f-2d45-4e25-79aaf25b8aff" term=3 +2025-06-17T07:42:58.900Z [WARN] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: Election timeout reached, restarting election +2025-06-17T07:42:58.900Z [INFO] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: entering candidate state: node="Node at 29a889ec-a70f-2d45-4e25-79aaf25b8aff [Candidate]" term=4 +2025-06-17T07:42:58.900Z [ERROR] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: failed to make requestVote RPC: target="{Voter server-8109375d-fbf3-793d-4080-fd218e10f9a4 8109375d-fbf3-793d-4080-fd218e10f9a4}" error="failed to connect to peer: 8109375d-fbf3-793d-4080-fd218e10f9a4" term=4 +2025-06-17T07:42:58.900Z [INFO] server-3311b05e-d15c-7e0c-0575-f83a787dcd34: entering follower state: follower="Node at 3311b05e-d15c-7e0c-0575-f83a787dcd34 [Follower]" leader-address= leader-id= +2025-06-17T07:42:58.900Z [INFO] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: election won: term=4 tally=2 +2025-06-17T07:42:58.900Z [INFO] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: entering leader state: leader="Node at 29a889ec-a70f-2d45-4e25-79aaf25b8aff [Leader]" +2025-06-17T07:42:58.900Z [INFO] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: added peer, starting replication: peer=server-8109375d-fbf3-793d-4080-fd218e10f9a4 +2025-06-17T07:42:58.900Z [INFO] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: added peer, starting replication: peer=server-3311b05e-d15c-7e0c-0575-f83a787dcd34 +2025-06-17T07:42:58.900Z [ERROR] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: failed to appendEntries to: peer="{Voter server-8109375d-fbf3-793d-4080-fd218e10f9a4 8109375d-fbf3-793d-4080-fd218e10f9a4}" error="failed to connect to peer: 8109375d-fbf3-793d-4080-fd218e10f9a4" +2025-06-17T07:42:58.901Z [INFO] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: pipelining replication: peer="{Voter server-3311b05e-d15c-7e0c-0575-f83a787dcd34 3311b05e-d15c-7e0c-0575-f83a787dcd34}" +2025-06-17T07:42:58.909Z [ERROR] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: failed to heartbeat to: peer=8109375d-fbf3-793d-4080-fd218e10f9a4 backoff time=10ms error="failed to connect to peer: 8109375d-fbf3-793d-4080-fd218e10f9a4" +2025-06-17T07:42:58.919Z [ERROR] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: failed to appendEntries to: peer="{Voter server-8109375d-fbf3-793d-4080-fd218e10f9a4 8109375d-fbf3-793d-4080-fd218e10f9a4}" error="failed to connect to peer: 8109375d-fbf3-793d-4080-fd218e10f9a4" +2025-06-17T07:42:58.926Z [ERROR] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: failed to heartbeat to: peer=8109375d-fbf3-793d-4080-fd218e10f9a4 backoff time=10ms error="failed to connect to peer: 8109375d-fbf3-793d-4080-fd218e10f9a4" +2025-06-17T07:42:58.940Z [ERROR] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: failed to appendEntries to: peer="{Voter server-8109375d-fbf3-793d-4080-fd218e10f9a4 8109375d-fbf3-793d-4080-fd218e10f9a4}" error="failed to connect to peer: 8109375d-fbf3-793d-4080-fd218e10f9a4" +2025-06-17T07:42:58.946Z [ERROR] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: failed to heartbeat to: peer=8109375d-fbf3-793d-4080-fd218e10f9a4 backoff time=10ms error="failed to connect to peer: 8109375d-fbf3-793d-4080-fd218e10f9a4" +2025-06-17T07:42:58.951Z [WARN] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: failed to contact: server-id=server-8109375d-fbf3-793d-4080-fd218e10f9a4 time=50.75193ms +2025-06-17T07:42:58.966Z [ERROR] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: failed to heartbeat to: peer=8109375d-fbf3-793d-4080-fd218e10f9a4 backoff time=20ms error="failed to connect to peer: 8109375d-fbf3-793d-4080-fd218e10f9a4" +2025-06-17T07:42:58.969Z [ERROR] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: failed to appendEntries to: peer="{Voter server-8109375d-fbf3-793d-4080-fd218e10f9a4 8109375d-fbf3-793d-4080-fd218e10f9a4}" error="failed to connect to peer: 8109375d-fbf3-793d-4080-fd218e10f9a4" +2025-06-17T07:42:58.991Z [WARN] server-8109375d-fbf3-793d-4080-fd218e10f9a4: Election timeout reached, restarting election +2025-06-17T07:42:58.991Z [INFO] server-8109375d-fbf3-793d-4080-fd218e10f9a4: entering candidate state: node="Node at 8109375d-fbf3-793d-4080-fd218e10f9a4 [Candidate]" term=4 +2025-06-17T07:42:58.991Z [ERROR] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to make requestVote RPC: target="{Voter server-3311b05e-d15c-7e0c-0575-f83a787dcd34 3311b05e-d15c-7e0c-0575-f83a787dcd34}" error="failed to connect to peer: 3311b05e-d15c-7e0c-0575-f83a787dcd34" term=4 +2025-06-17T07:42:58.991Z [ERROR] server-8109375d-fbf3-793d-4080-fd218e10f9a4: failed to make requestVote RPC: target="{Voter server-29a889ec-a70f-2d45-4e25-79aaf25b8aff 29a889ec-a70f-2d45-4e25-79aaf25b8aff}" error="failed to connect to peer: 29a889ec-a70f-2d45-4e25-79aaf25b8aff" term=4 +2025-06-17T07:42:58.996Z [WARN] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: failed to contact: server-id=server-8109375d-fbf3-793d-4080-fd218e10f9a4 time=95.492327ms +2025-06-17T07:42:58.997Z [ERROR] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: failed to heartbeat to: peer=8109375d-fbf3-793d-4080-fd218e10f9a4 backoff time=25ms error="failed to connect to peer: 8109375d-fbf3-793d-4080-fd218e10f9a4" +2025-06-17T07:42:59.006Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:42:58.900959878 +0000 UTC m=+7.504488565 (1 nodes), 110.927864ms from start of poll, 389.414722ms from cluster start. Timeout at 2025-06-17 07:42:59.00599396 +0000 UTC m=+7.609522656, 105.034091ms after stability + raft_test.go:413: [INFO] Reconnecting Node at 8109375d-fbf3-793d-4080-fd218e10f9a4 [Candidate] +2025-06-17T07:42:59.017Z [INFO] server-8109375d-fbf3-793d-4080-fd218e10f9a4: entering follower state: follower="Node at 8109375d-fbf3-793d-4080-fd218e10f9a4 [Follower]" leader-address=29a889ec-a70f-2d45-4e25-79aaf25b8aff leader-id=server-29a889ec-a70f-2d45-4e25-79aaf25b8aff +2025-06-17T07:42:59.017Z [INFO] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: pipelining replication: peer="{Voter server-8109375d-fbf3-793d-4080-fd218e10f9a4 8109375d-fbf3-793d-4080-fd218e10f9a4}" +2025-06-17T07:42:59.022Z [INFO] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: aborting pipeline replication: peer="{Voter server-3311b05e-d15c-7e0c-0575-f83a787dcd34 3311b05e-d15c-7e0c-0575-f83a787dcd34}" +2025-06-17T07:42:59.022Z [INFO] server-29a889ec-a70f-2d45-4e25-79aaf25b8aff: aborting pipeline replication: peer="{Voter server-8109375d-fbf3-793d-4080-fd218e10f9a4 8109375d-fbf3-793d-4080-fd218e10f9a4}" +--- PASS: TestRaft_LeaderFail (0.51s) === RUN TestRaft_BehindFollower -2024-05-15T01:17:06.863Z [INFO] server-ce002f9a-938a-db88-130e-c7a8144f312c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ce002f9a-938a-db88-130e-c7a8144f312c Address:ce002f9a-938a-db88-130e-c7a8144f312c} {Suffrage:Voter ID:server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d Address:ff9b13c9-bca6-f529-2e1b-f50e5a308a0d} {Suffrage:Voter ID:server-695f8f78-e8cc-96be-2f24-f6bd1cea5594 Address:695f8f78-e8cc-96be-2f24-f6bd1cea5594}]" -2024-05-15T01:17:06.863Z [INFO] server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ce002f9a-938a-db88-130e-c7a8144f312c Address:ce002f9a-938a-db88-130e-c7a8144f312c} {Suffrage:Voter ID:server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d Address:ff9b13c9-bca6-f529-2e1b-f50e5a308a0d} {Suffrage:Voter ID:server-695f8f78-e8cc-96be-2f24-f6bd1cea5594 Address:695f8f78-e8cc-96be-2f24-f6bd1cea5594}]" -2024-05-15T01:17:06.863Z [INFO] server-695f8f78-e8cc-96be-2f24-f6bd1cea5594: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ce002f9a-938a-db88-130e-c7a8144f312c Address:ce002f9a-938a-db88-130e-c7a8144f312c} {Suffrage:Voter ID:server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d Address:ff9b13c9-bca6-f529-2e1b-f50e5a308a0d} {Suffrage:Voter ID:server-695f8f78-e8cc-96be-2f24-f6bd1cea5594 Address:695f8f78-e8cc-96be-2f24-f6bd1cea5594}]" -2024-05-15T01:17:06.863Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:06.863Z [INFO] server-ce002f9a-938a-db88-130e-c7a8144f312c: entering follower state: follower="Node at ce002f9a-938a-db88-130e-c7a8144f312c [Follower]" leader-address= leader-id= -2024-05-15T01:17:06.863Z [INFO] server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d: entering follower state: follower="Node at ff9b13c9-bca6-f529-2e1b-f50e5a308a0d [Follower]" leader-address= leader-id= -2024-05-15T01:17:06.863Z [INFO] server-695f8f78-e8cc-96be-2f24-f6bd1cea5594: entering follower state: follower="Node at 695f8f78-e8cc-96be-2f24-f6bd1cea5594 [Follower]" leader-address= leader-id= -2024-05-15T01:17:06.919Z [WARN] server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:06.919Z [INFO] server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d: entering candidate state: node="Node at ff9b13c9-bca6-f529-2e1b-f50e5a308a0d [Candidate]" term=2 -2024-05-15T01:17:06.919Z [INFO] server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d: election won: term=2 tally=2 -2024-05-15T01:17:06.919Z [INFO] server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d: entering leader state: leader="Node at ff9b13c9-bca6-f529-2e1b-f50e5a308a0d [Leader]" -2024-05-15T01:17:06.919Z [INFO] server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d: added peer, starting replication: peer=server-ce002f9a-938a-db88-130e-c7a8144f312c -2024-05-15T01:17:06.919Z [INFO] server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d: added peer, starting replication: peer=server-695f8f78-e8cc-96be-2f24-f6bd1cea5594 -2024-05-15T01:17:06.919Z [INFO] server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d: pipelining replication: peer="{Voter server-695f8f78-e8cc-96be-2f24-f6bd1cea5594 695f8f78-e8cc-96be-2f24-f6bd1cea5594}" -2024-05-15T01:17:06.920Z [INFO] server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d: pipelining replication: peer="{Voter server-ce002f9a-938a-db88-130e-c7a8144f312c ce002f9a-938a-db88-130e-c7a8144f312c}" -2024-05-15T01:17:07.025Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:06.920135619 +0000 UTC m=+7.461520362 (1 nodes), 56.628334ms from start of poll, 57.130952ms from cluster start. Timeout at 2024-05-15 01:17:07.025193675 +0000 UTC m=+7.566578407, 105.058045ms after stability -2024-05-15T01:17:07.025Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:07.131Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:07.025295738 +0000 UTC m=+7.566680573 (2 nodes), 2.244µs from start of poll, 162.291163ms from cluster start. Timeout at 2024-05-15 01:17:07.131053415 +0000 UTC m=+7.672438231, 105.757658ms after stability -2024-05-15T01:17:07.131Z [ERROR] server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d: failed to pipeline appendEntries: peer="{Voter server-ce002f9a-938a-db88-130e-c7a8144f312c ce002f9a-938a-db88-130e-c7a8144f312c}" error="append pipeline closed" -2024-05-15T01:17:07.131Z [INFO] server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d: aborting pipeline replication: peer="{Voter server-ce002f9a-938a-db88-130e-c7a8144f312c ce002f9a-938a-db88-130e-c7a8144f312c}" -2024-05-15T01:17:07.131Z [ERROR] server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d: failed to appendEntries to: peer="{Voter server-ce002f9a-938a-db88-130e-c7a8144f312c ce002f9a-938a-db88-130e-c7a8144f312c}" error="failed to connect to peer: ce002f9a-938a-db88-130e-c7a8144f312c" -2024-05-15T01:17:07.133Z [ERROR] server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d: failed to heartbeat to: peer=ce002f9a-938a-db88-130e-c7a8144f312c backoff time=10ms error="failed to connect to peer: ce002f9a-938a-db88-130e-c7a8144f312c" +2025-06-17T07:42:59.023Z [INFO] server-28c64436-471b-05a8-7365-dbfb9ddc6cb6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-28c64436-471b-05a8-7365-dbfb9ddc6cb6 Address:28c64436-471b-05a8-7365-dbfb9ddc6cb6} {Suffrage:Voter ID:server-197c510f-5e6e-7495-1131-5f314d66e306 Address:197c510f-5e6e-7495-1131-5f314d66e306} {Suffrage:Voter ID:server-e22c0f07-789a-562a-4dc8-feceb0ad4e13 Address:e22c0f07-789a-562a-4dc8-feceb0ad4e13}]" +2025-06-17T07:42:59.023Z [INFO] server-28c64436-471b-05a8-7365-dbfb9ddc6cb6: entering follower state: follower="Node at 28c64436-471b-05a8-7365-dbfb9ddc6cb6 [Follower]" leader-address= leader-id= +2025-06-17T07:42:59.023Z [INFO] server-197c510f-5e6e-7495-1131-5f314d66e306: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-28c64436-471b-05a8-7365-dbfb9ddc6cb6 Address:28c64436-471b-05a8-7365-dbfb9ddc6cb6} {Suffrage:Voter ID:server-197c510f-5e6e-7495-1131-5f314d66e306 Address:197c510f-5e6e-7495-1131-5f314d66e306} {Suffrage:Voter ID:server-e22c0f07-789a-562a-4dc8-feceb0ad4e13 Address:e22c0f07-789a-562a-4dc8-feceb0ad4e13}]" +2025-06-17T07:42:59.023Z [INFO] server-197c510f-5e6e-7495-1131-5f314d66e306: entering follower state: follower="Node at 197c510f-5e6e-7495-1131-5f314d66e306 [Follower]" leader-address= leader-id= +2025-06-17T07:42:59.024Z [INFO] server-e22c0f07-789a-562a-4dc8-feceb0ad4e13: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-28c64436-471b-05a8-7365-dbfb9ddc6cb6 Address:28c64436-471b-05a8-7365-dbfb9ddc6cb6} {Suffrage:Voter ID:server-197c510f-5e6e-7495-1131-5f314d66e306 Address:197c510f-5e6e-7495-1131-5f314d66e306} {Suffrage:Voter ID:server-e22c0f07-789a-562a-4dc8-feceb0ad4e13 Address:e22c0f07-789a-562a-4dc8-feceb0ad4e13}]" +2025-06-17T07:42:59.024Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:42:59.024Z [INFO] server-e22c0f07-789a-562a-4dc8-feceb0ad4e13: entering follower state: follower="Node at e22c0f07-789a-562a-4dc8-feceb0ad4e13 [Follower]" leader-address= leader-id= +2025-06-17T07:42:59.076Z [WARN] server-28c64436-471b-05a8-7365-dbfb9ddc6cb6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:42:59.076Z [INFO] server-28c64436-471b-05a8-7365-dbfb9ddc6cb6: entering candidate state: node="Node at 28c64436-471b-05a8-7365-dbfb9ddc6cb6 [Candidate]" term=2 +2025-06-17T07:42:59.076Z [INFO] server-28c64436-471b-05a8-7365-dbfb9ddc6cb6: election won: term=2 tally=2 +2025-06-17T07:42:59.076Z [INFO] server-28c64436-471b-05a8-7365-dbfb9ddc6cb6: entering leader state: leader="Node at 28c64436-471b-05a8-7365-dbfb9ddc6cb6 [Leader]" +2025-06-17T07:42:59.076Z [INFO] server-28c64436-471b-05a8-7365-dbfb9ddc6cb6: added peer, starting replication: peer=server-197c510f-5e6e-7495-1131-5f314d66e306 +2025-06-17T07:42:59.076Z [INFO] server-28c64436-471b-05a8-7365-dbfb9ddc6cb6: added peer, starting replication: peer=server-e22c0f07-789a-562a-4dc8-feceb0ad4e13 +2025-06-17T07:42:59.076Z [INFO] server-28c64436-471b-05a8-7365-dbfb9ddc6cb6: pipelining replication: peer="{Voter server-197c510f-5e6e-7495-1131-5f314d66e306 197c510f-5e6e-7495-1131-5f314d66e306}" +2025-06-17T07:42:59.076Z [INFO] server-28c64436-471b-05a8-7365-dbfb9ddc6cb6: pipelining replication: peer="{Voter server-e22c0f07-789a-562a-4dc8-feceb0ad4e13 e22c0f07-789a-562a-4dc8-feceb0ad4e13}" +2025-06-17T07:42:59.182Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:42:59.076448655 +0000 UTC m=+7.679977351 (1 nodes), 52.340404ms from start of poll, 52.687899ms from cluster start. Timeout at 2025-06-17 07:42:59.182295381 +0000 UTC m=+7.785824063, 105.846712ms after stability +2025-06-17T07:42:59.182Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:42:59.287Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:42:59.18234358 +0000 UTC m=+7.785872267 (2 nodes), 723ns from start of poll, 158.582815ms from cluster start. Timeout at 2025-06-17 07:42:59.287523571 +0000 UTC m=+7.891052261, 105.179994ms after stability +2025-06-17T07:42:59.287Z [ERROR] server-28c64436-471b-05a8-7365-dbfb9ddc6cb6: failed to pipeline appendEntries: peer="{Voter server-197c510f-5e6e-7495-1131-5f314d66e306 197c510f-5e6e-7495-1131-5f314d66e306}" error="append pipeline closed" +2025-06-17T07:42:59.287Z [INFO] server-28c64436-471b-05a8-7365-dbfb9ddc6cb6: aborting pipeline replication: peer="{Voter server-197c510f-5e6e-7495-1131-5f314d66e306 197c510f-5e6e-7495-1131-5f314d66e306}" +2025-06-17T07:42:59.287Z [ERROR] server-28c64436-471b-05a8-7365-dbfb9ddc6cb6: failed to appendEntries to: peer="{Voter server-197c510f-5e6e-7495-1131-5f314d66e306 197c510f-5e6e-7495-1131-5f314d66e306}" error="failed to connect to peer: 197c510f-5e6e-7495-1131-5f314d66e306" raft_test.go:459: [INFO] Finished apply without behind follower -2024-05-15T01:17:07.142Z [INFO] server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d: pipelining replication: peer="{Voter server-ce002f9a-938a-db88-130e-c7a8144f312c ce002f9a-938a-db88-130e-c7a8144f312c}" -2024-05-15T01:17:07.153Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:07.258Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:07.153516341 +0000 UTC m=+7.694901066 (1 nodes), 1.82µs from start of poll, 290.511656ms from cluster start. Timeout at 2024-05-15 01:17:07.258542678 +0000 UTC m=+7.799927418, 105.026352ms after stability -2024-05-15T01:17:07.258Z [INFO] server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d: aborting pipeline replication: peer="{Voter server-ce002f9a-938a-db88-130e-c7a8144f312c ce002f9a-938a-db88-130e-c7a8144f312c}" -2024-05-15T01:17:07.258Z [INFO] server-ff9b13c9-bca6-f529-2e1b-f50e5a308a0d: aborting pipeline replication: peer="{Voter server-695f8f78-e8cc-96be-2f24-f6bd1cea5594 695f8f78-e8cc-96be-2f24-f6bd1cea5594}" ---- PASS: TestRaft_BehindFollower (0.40s) +2025-06-17T07:42:59.298Z [INFO] server-28c64436-471b-05a8-7365-dbfb9ddc6cb6: pipelining replication: peer="{Voter server-197c510f-5e6e-7495-1131-5f314d66e306 197c510f-5e6e-7495-1131-5f314d66e306}" +2025-06-17T07:42:59.307Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:42:59.412Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:42:59.307071495 +0000 UTC m=+7.910600180 (1 nodes), 588ns from start of poll, 283.310728ms from cluster start. Timeout at 2025-06-17 07:42:59.412933839 +0000 UTC m=+8.016462521, 105.862341ms after stability +2025-06-17T07:42:59.412Z [INFO] server-28c64436-471b-05a8-7365-dbfb9ddc6cb6: aborting pipeline replication: peer="{Voter server-e22c0f07-789a-562a-4dc8-feceb0ad4e13 e22c0f07-789a-562a-4dc8-feceb0ad4e13}" +2025-06-17T07:42:59.412Z [INFO] server-28c64436-471b-05a8-7365-dbfb9ddc6cb6: aborting pipeline replication: peer="{Voter server-197c510f-5e6e-7495-1131-5f314d66e306 197c510f-5e6e-7495-1131-5f314d66e306}" +--- PASS: TestRaft_BehindFollower (0.39s) === RUN TestRaft_ApplyNonLeader -2024-05-15T01:17:07.260Z [INFO] server-efb16a39-45aa-c12a-07cb-459721fcadb2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-efb16a39-45aa-c12a-07cb-459721fcadb2 Address:efb16a39-45aa-c12a-07cb-459721fcadb2} {Suffrage:Voter ID:server-db0b30de-a34a-081a-08b4-cc2d29808f2f Address:db0b30de-a34a-081a-08b4-cc2d29808f2f} {Suffrage:Voter ID:server-0d7bad93-8788-1875-2a02-08bfb94223da Address:0d7bad93-8788-1875-2a02-08bfb94223da}]" -2024-05-15T01:17:07.260Z [INFO] server-db0b30de-a34a-081a-08b4-cc2d29808f2f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-efb16a39-45aa-c12a-07cb-459721fcadb2 Address:efb16a39-45aa-c12a-07cb-459721fcadb2} {Suffrage:Voter ID:server-db0b30de-a34a-081a-08b4-cc2d29808f2f Address:db0b30de-a34a-081a-08b4-cc2d29808f2f} {Suffrage:Voter ID:server-0d7bad93-8788-1875-2a02-08bfb94223da Address:0d7bad93-8788-1875-2a02-08bfb94223da}]" -2024-05-15T01:17:07.260Z [INFO] server-0d7bad93-8788-1875-2a02-08bfb94223da: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-efb16a39-45aa-c12a-07cb-459721fcadb2 Address:efb16a39-45aa-c12a-07cb-459721fcadb2} {Suffrage:Voter ID:server-db0b30de-a34a-081a-08b4-cc2d29808f2f Address:db0b30de-a34a-081a-08b4-cc2d29808f2f} {Suffrage:Voter ID:server-0d7bad93-8788-1875-2a02-08bfb94223da Address:0d7bad93-8788-1875-2a02-08bfb94223da}]" -2024-05-15T01:17:07.260Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:07.260Z [INFO] server-efb16a39-45aa-c12a-07cb-459721fcadb2: entering follower state: follower="Node at efb16a39-45aa-c12a-07cb-459721fcadb2 [Follower]" leader-address= leader-id= -2024-05-15T01:17:07.260Z [INFO] server-db0b30de-a34a-081a-08b4-cc2d29808f2f: entering follower state: follower="Node at db0b30de-a34a-081a-08b4-cc2d29808f2f [Follower]" leader-address= leader-id= -2024-05-15T01:17:07.260Z [INFO] server-0d7bad93-8788-1875-2a02-08bfb94223da: entering follower state: follower="Node at 0d7bad93-8788-1875-2a02-08bfb94223da [Follower]" leader-address= leader-id= -2024-05-15T01:17:07.313Z [WARN] server-0d7bad93-8788-1875-2a02-08bfb94223da: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:07.313Z [INFO] server-0d7bad93-8788-1875-2a02-08bfb94223da: entering candidate state: node="Node at 0d7bad93-8788-1875-2a02-08bfb94223da [Candidate]" term=2 -2024-05-15T01:17:07.314Z [INFO] server-0d7bad93-8788-1875-2a02-08bfb94223da: election won: term=2 tally=2 -2024-05-15T01:17:07.314Z [INFO] server-0d7bad93-8788-1875-2a02-08bfb94223da: entering leader state: leader="Node at 0d7bad93-8788-1875-2a02-08bfb94223da [Leader]" -2024-05-15T01:17:07.314Z [INFO] server-0d7bad93-8788-1875-2a02-08bfb94223da: added peer, starting replication: peer=server-efb16a39-45aa-c12a-07cb-459721fcadb2 -2024-05-15T01:17:07.314Z [INFO] server-0d7bad93-8788-1875-2a02-08bfb94223da: added peer, starting replication: peer=server-db0b30de-a34a-081a-08b4-cc2d29808f2f -2024-05-15T01:17:07.314Z [INFO] server-0d7bad93-8788-1875-2a02-08bfb94223da: pipelining replication: peer="{Voter server-efb16a39-45aa-c12a-07cb-459721fcadb2 efb16a39-45aa-c12a-07cb-459721fcadb2}" -2024-05-15T01:17:07.314Z [INFO] server-0d7bad93-8788-1875-2a02-08bfb94223da: pipelining replication: peer="{Voter server-db0b30de-a34a-081a-08b4-cc2d29808f2f db0b30de-a34a-081a-08b4-cc2d29808f2f}" -2024-05-15T01:17:07.419Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:07.314186579 +0000 UTC m=+7.855571493 (1 nodes), 53.348918ms from start of poll, 53.877455ms from cluster start. Timeout at 2024-05-15 01:17:07.419883711 +0000 UTC m=+7.961268445, 105.696952ms after stability -2024-05-15T01:17:07.419Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:07.525Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:07.420013645 +0000 UTC m=+7.961398375 (2 nodes), 2.393µs from start of poll, 159.704337ms from cluster start. Timeout at 2024-05-15 01:17:07.525604357 +0000 UTC m=+8.066989122, 105.590747ms after stability -2024-05-15T01:17:07.525Z [INFO] server-0d7bad93-8788-1875-2a02-08bfb94223da: aborting pipeline replication: peer="{Voter server-db0b30de-a34a-081a-08b4-cc2d29808f2f db0b30de-a34a-081a-08b4-cc2d29808f2f}" -2024-05-15T01:17:07.525Z [INFO] server-0d7bad93-8788-1875-2a02-08bfb94223da: aborting pipeline replication: peer="{Voter server-efb16a39-45aa-c12a-07cb-459721fcadb2 efb16a39-45aa-c12a-07cb-459721fcadb2}" +2025-06-17T07:42:59.414Z [INFO] server-8ea296cc-22c7-f48f-108f-c671e8e974d5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8ea296cc-22c7-f48f-108f-c671e8e974d5 Address:8ea296cc-22c7-f48f-108f-c671e8e974d5} {Suffrage:Voter ID:server-f4a55314-cc63-e26f-d240-bbe8389a804c Address:f4a55314-cc63-e26f-d240-bbe8389a804c} {Suffrage:Voter ID:server-15628282-53b9-d420-32b7-144aa55e226f Address:15628282-53b9-d420-32b7-144aa55e226f}]" +2025-06-17T07:42:59.414Z [INFO] server-8ea296cc-22c7-f48f-108f-c671e8e974d5: entering follower state: follower="Node at 8ea296cc-22c7-f48f-108f-c671e8e974d5 [Follower]" leader-address= leader-id= +2025-06-17T07:42:59.414Z [INFO] server-f4a55314-cc63-e26f-d240-bbe8389a804c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8ea296cc-22c7-f48f-108f-c671e8e974d5 Address:8ea296cc-22c7-f48f-108f-c671e8e974d5} {Suffrage:Voter ID:server-f4a55314-cc63-e26f-d240-bbe8389a804c Address:f4a55314-cc63-e26f-d240-bbe8389a804c} {Suffrage:Voter ID:server-15628282-53b9-d420-32b7-144aa55e226f Address:15628282-53b9-d420-32b7-144aa55e226f}]" +2025-06-17T07:42:59.414Z [INFO] server-f4a55314-cc63-e26f-d240-bbe8389a804c: entering follower state: follower="Node at f4a55314-cc63-e26f-d240-bbe8389a804c [Follower]" leader-address= leader-id= +2025-06-17T07:42:59.414Z [INFO] server-15628282-53b9-d420-32b7-144aa55e226f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8ea296cc-22c7-f48f-108f-c671e8e974d5 Address:8ea296cc-22c7-f48f-108f-c671e8e974d5} {Suffrage:Voter ID:server-f4a55314-cc63-e26f-d240-bbe8389a804c Address:f4a55314-cc63-e26f-d240-bbe8389a804c} {Suffrage:Voter ID:server-15628282-53b9-d420-32b7-144aa55e226f Address:15628282-53b9-d420-32b7-144aa55e226f}]" +2025-06-17T07:42:59.414Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:42:59.414Z [INFO] server-15628282-53b9-d420-32b7-144aa55e226f: entering follower state: follower="Node at 15628282-53b9-d420-32b7-144aa55e226f [Follower]" leader-address= leader-id= +2025-06-17T07:42:59.474Z [WARN] server-8ea296cc-22c7-f48f-108f-c671e8e974d5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:42:59.474Z [INFO] server-8ea296cc-22c7-f48f-108f-c671e8e974d5: entering candidate state: node="Node at 8ea296cc-22c7-f48f-108f-c671e8e974d5 [Candidate]" term=2 +2025-06-17T07:42:59.474Z [INFO] server-8ea296cc-22c7-f48f-108f-c671e8e974d5: election won: term=2 tally=2 +2025-06-17T07:42:59.474Z [INFO] server-8ea296cc-22c7-f48f-108f-c671e8e974d5: entering leader state: leader="Node at 8ea296cc-22c7-f48f-108f-c671e8e974d5 [Leader]" +2025-06-17T07:42:59.474Z [INFO] server-8ea296cc-22c7-f48f-108f-c671e8e974d5: added peer, starting replication: peer=server-f4a55314-cc63-e26f-d240-bbe8389a804c +2025-06-17T07:42:59.474Z [INFO] server-8ea296cc-22c7-f48f-108f-c671e8e974d5: added peer, starting replication: peer=server-15628282-53b9-d420-32b7-144aa55e226f +2025-06-17T07:42:59.474Z [INFO] server-8ea296cc-22c7-f48f-108f-c671e8e974d5: pipelining replication: peer="{Voter server-f4a55314-cc63-e26f-d240-bbe8389a804c f4a55314-cc63-e26f-d240-bbe8389a804c}" +2025-06-17T07:42:59.474Z [INFO] server-8ea296cc-22c7-f48f-108f-c671e8e974d5: pipelining replication: peer="{Voter server-15628282-53b9-d420-32b7-144aa55e226f 15628282-53b9-d420-32b7-144aa55e226f}" +2025-06-17T07:42:59.580Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:42:59.474601231 +0000 UTC m=+8.078129926 (1 nodes), 60.039401ms from start of poll, 60.356556ms from cluster start. Timeout at 2025-06-17 07:42:59.580551017 +0000 UTC m=+8.184079699, 105.949773ms after stability +2025-06-17T07:42:59.580Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:42:59.685Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:42:59.580625639 +0000 UTC m=+8.184154324 (2 nodes), 824ns from start of poll, 166.380954ms from cluster start. Timeout at 2025-06-17 07:42:59.685857617 +0000 UTC m=+8.289386312, 105.231988ms after stability +2025-06-17T07:42:59.685Z [INFO] server-8ea296cc-22c7-f48f-108f-c671e8e974d5: aborting pipeline replication: peer="{Voter server-15628282-53b9-d420-32b7-144aa55e226f 15628282-53b9-d420-32b7-144aa55e226f}" +2025-06-17T07:42:59.685Z [INFO] server-8ea296cc-22c7-f48f-108f-c671e8e974d5: aborting pipeline replication: peer="{Voter server-f4a55314-cc63-e26f-d240-bbe8389a804c f4a55314-cc63-e26f-d240-bbe8389a804c}" --- PASS: TestRaft_ApplyNonLeader (0.27s) === RUN TestRaft_ApplyConcurrent -2024-05-15T01:17:07.527Z [INFO] server-c490efd0-9256-0c7f-ed29-0adf80a3d31f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c490efd0-9256-0c7f-ed29-0adf80a3d31f Address:c490efd0-9256-0c7f-ed29-0adf80a3d31f} {Suffrage:Voter ID:server-1e16675c-1862-2590-db3c-e833776b2420 Address:1e16675c-1862-2590-db3c-e833776b2420} {Suffrage:Voter ID:server-20092957-effa-c21c-644f-365e35d911ad Address:20092957-effa-c21c-644f-365e35d911ad}]" -2024-05-15T01:17:07.527Z [INFO] server-c490efd0-9256-0c7f-ed29-0adf80a3d31f: entering follower state: follower="Node at c490efd0-9256-0c7f-ed29-0adf80a3d31f [Follower]" leader-address= leader-id= -2024-05-15T01:17:07.527Z [INFO] server-1e16675c-1862-2590-db3c-e833776b2420: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c490efd0-9256-0c7f-ed29-0adf80a3d31f Address:c490efd0-9256-0c7f-ed29-0adf80a3d31f} {Suffrage:Voter ID:server-1e16675c-1862-2590-db3c-e833776b2420 Address:1e16675c-1862-2590-db3c-e833776b2420} {Suffrage:Voter ID:server-20092957-effa-c21c-644f-365e35d911ad Address:20092957-effa-c21c-644f-365e35d911ad}]" -2024-05-15T01:17:07.527Z [INFO] server-1e16675c-1862-2590-db3c-e833776b2420: entering follower state: follower="Node at 1e16675c-1862-2590-db3c-e833776b2420 [Follower]" leader-address= leader-id= -2024-05-15T01:17:07.527Z [INFO] server-20092957-effa-c21c-644f-365e35d911ad: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c490efd0-9256-0c7f-ed29-0adf80a3d31f Address:c490efd0-9256-0c7f-ed29-0adf80a3d31f} {Suffrage:Voter ID:server-1e16675c-1862-2590-db3c-e833776b2420 Address:1e16675c-1862-2590-db3c-e833776b2420} {Suffrage:Voter ID:server-20092957-effa-c21c-644f-365e35d911ad Address:20092957-effa-c21c-644f-365e35d911ad}]" -2024-05-15T01:17:07.527Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:07.528Z [INFO] server-20092957-effa-c21c-644f-365e35d911ad: entering follower state: follower="Node at 20092957-effa-c21c-644f-365e35d911ad [Follower]" leader-address= leader-id= -2024-05-15T01:17:07.628Z [WARN] server-c490efd0-9256-0c7f-ed29-0adf80a3d31f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:07.629Z [INFO] server-c490efd0-9256-0c7f-ed29-0adf80a3d31f: entering candidate state: node="Node at c490efd0-9256-0c7f-ed29-0adf80a3d31f [Candidate]" term=2 -2024-05-15T01:17:07.629Z [INFO] server-c490efd0-9256-0c7f-ed29-0adf80a3d31f: election won: term=2 tally=2 -2024-05-15T01:17:07.629Z [INFO] server-c490efd0-9256-0c7f-ed29-0adf80a3d31f: entering leader state: leader="Node at c490efd0-9256-0c7f-ed29-0adf80a3d31f [Leader]" -2024-05-15T01:17:07.629Z [INFO] server-c490efd0-9256-0c7f-ed29-0adf80a3d31f: added peer, starting replication: peer=server-1e16675c-1862-2590-db3c-e833776b2420 -2024-05-15T01:17:07.629Z [INFO] server-c490efd0-9256-0c7f-ed29-0adf80a3d31f: added peer, starting replication: peer=server-20092957-effa-c21c-644f-365e35d911ad -2024-05-15T01:17:07.629Z [INFO] server-c490efd0-9256-0c7f-ed29-0adf80a3d31f: pipelining replication: peer="{Voter server-20092957-effa-c21c-644f-365e35d911ad 20092957-effa-c21c-644f-365e35d911ad}" -2024-05-15T01:17:07.629Z [INFO] server-c490efd0-9256-0c7f-ed29-0adf80a3d31f: pipelining replication: peer="{Voter server-1e16675c-1862-2590-db3c-e833776b2420 1e16675c-1862-2590-db3c-e833776b2420}" -2024-05-15T01:17:07.835Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:07.629711092 +0000 UTC m=+8.171095330 (1 nodes), 101.684196ms from start of poll, 102.269728ms from cluster start. Timeout at 2024-05-15 01:17:07.835406097 +0000 UTC m=+8.376790864, 205.695534ms after stability -2024-05-15T01:17:07.854Z [INFO] server-c490efd0-9256-0c7f-ed29-0adf80a3d31f: aborting pipeline replication: peer="{Voter server-20092957-effa-c21c-644f-365e35d911ad 20092957-effa-c21c-644f-365e35d911ad}" -2024-05-15T01:17:07.855Z [INFO] server-c490efd0-9256-0c7f-ed29-0adf80a3d31f: aborting pipeline replication: peer="{Voter server-1e16675c-1862-2590-db3c-e833776b2420 1e16675c-1862-2590-db3c-e833776b2420}" ---- PASS: TestRaft_ApplyConcurrent (0.33s) +2025-06-17T07:42:59.686Z [INFO] server-c6580545-4b44-5c53-60b0-bd694493cfba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c6580545-4b44-5c53-60b0-bd694493cfba Address:c6580545-4b44-5c53-60b0-bd694493cfba} {Suffrage:Voter ID:server-44a54577-6186-57d2-a4ac-61079aa0f276 Address:44a54577-6186-57d2-a4ac-61079aa0f276} {Suffrage:Voter ID:server-728315d2-8176-370a-2865-fe3b4b46f13d Address:728315d2-8176-370a-2865-fe3b4b46f13d}]" +2025-06-17T07:42:59.686Z [INFO] server-c6580545-4b44-5c53-60b0-bd694493cfba: entering follower state: follower="Node at c6580545-4b44-5c53-60b0-bd694493cfba [Follower]" leader-address= leader-id= +2025-06-17T07:42:59.686Z [INFO] server-44a54577-6186-57d2-a4ac-61079aa0f276: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c6580545-4b44-5c53-60b0-bd694493cfba Address:c6580545-4b44-5c53-60b0-bd694493cfba} {Suffrage:Voter ID:server-44a54577-6186-57d2-a4ac-61079aa0f276 Address:44a54577-6186-57d2-a4ac-61079aa0f276} {Suffrage:Voter ID:server-728315d2-8176-370a-2865-fe3b4b46f13d Address:728315d2-8176-370a-2865-fe3b4b46f13d}]" +2025-06-17T07:42:59.686Z [INFO] server-44a54577-6186-57d2-a4ac-61079aa0f276: entering follower state: follower="Node at 44a54577-6186-57d2-a4ac-61079aa0f276 [Follower]" leader-address= leader-id= +2025-06-17T07:42:59.686Z [INFO] server-728315d2-8176-370a-2865-fe3b4b46f13d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c6580545-4b44-5c53-60b0-bd694493cfba Address:c6580545-4b44-5c53-60b0-bd694493cfba} {Suffrage:Voter ID:server-44a54577-6186-57d2-a4ac-61079aa0f276 Address:44a54577-6186-57d2-a4ac-61079aa0f276} {Suffrage:Voter ID:server-728315d2-8176-370a-2865-fe3b4b46f13d Address:728315d2-8176-370a-2865-fe3b4b46f13d}]" +2025-06-17T07:42:59.686Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:42:59.686Z [INFO] server-728315d2-8176-370a-2865-fe3b4b46f13d: entering follower state: follower="Node at 728315d2-8176-370a-2865-fe3b4b46f13d [Follower]" leader-address= leader-id= +2025-06-17T07:42:59.820Z [WARN] server-44a54577-6186-57d2-a4ac-61079aa0f276: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:42:59.820Z [INFO] server-44a54577-6186-57d2-a4ac-61079aa0f276: entering candidate state: node="Node at 44a54577-6186-57d2-a4ac-61079aa0f276 [Candidate]" term=2 +2025-06-17T07:42:59.820Z [INFO] server-44a54577-6186-57d2-a4ac-61079aa0f276: election won: term=2 tally=2 +2025-06-17T07:42:59.820Z [INFO] server-44a54577-6186-57d2-a4ac-61079aa0f276: entering leader state: leader="Node at 44a54577-6186-57d2-a4ac-61079aa0f276 [Leader]" +2025-06-17T07:42:59.820Z [INFO] server-44a54577-6186-57d2-a4ac-61079aa0f276: added peer, starting replication: peer=server-c6580545-4b44-5c53-60b0-bd694493cfba +2025-06-17T07:42:59.820Z [INFO] server-44a54577-6186-57d2-a4ac-61079aa0f276: added peer, starting replication: peer=server-728315d2-8176-370a-2865-fe3b4b46f13d +2025-06-17T07:42:59.820Z [INFO] server-44a54577-6186-57d2-a4ac-61079aa0f276: pipelining replication: peer="{Voter server-728315d2-8176-370a-2865-fe3b4b46f13d 728315d2-8176-370a-2865-fe3b4b46f13d}" +2025-06-17T07:42:59.820Z [INFO] server-44a54577-6186-57d2-a4ac-61079aa0f276: pipelining replication: peer="{Voter server-c6580545-4b44-5c53-60b0-bd694493cfba c6580545-4b44-5c53-60b0-bd694493cfba}" +2025-06-17T07:43:00.025Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:42:59.820713154 +0000 UTC m=+8.424241877 (1 nodes), 133.800152ms from start of poll, 134.072283ms from cluster start. Timeout at 2025-06-17 07:43:00.02587048 +0000 UTC m=+8.629399175, 205.157298ms after stability +2025-06-17T07:43:00.039Z [INFO] server-44a54577-6186-57d2-a4ac-61079aa0f276: aborting pipeline replication: peer="{Voter server-728315d2-8176-370a-2865-fe3b4b46f13d 728315d2-8176-370a-2865-fe3b4b46f13d}" +2025-06-17T07:43:00.039Z [INFO] server-44a54577-6186-57d2-a4ac-61079aa0f276: aborting pipeline replication: peer="{Voter server-c6580545-4b44-5c53-60b0-bd694493cfba c6580545-4b44-5c53-60b0-bd694493cfba}" +--- PASS: TestRaft_ApplyConcurrent (0.35s) === RUN TestRaft_ApplyConcurrent_Timeout -2024-05-15T01:17:07.856Z [INFO] server-35e06bf4-687d-78ca-ef1f-953dbb295f3e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35e06bf4-687d-78ca-ef1f-953dbb295f3e Address:35e06bf4-687d-78ca-ef1f-953dbb295f3e}]" -2024-05-15T01:17:07.856Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:07.858Z [INFO] server-35e06bf4-687d-78ca-ef1f-953dbb295f3e: entering follower state: follower="Node at 35e06bf4-687d-78ca-ef1f-953dbb295f3e [Follower]" leader-address= leader-id= -2024-05-15T01:17:08.043Z [WARN] server-35e06bf4-687d-78ca-ef1f-953dbb295f3e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:08.043Z [INFO] server-35e06bf4-687d-78ca-ef1f-953dbb295f3e: entering candidate state: node="Node at 35e06bf4-687d-78ca-ef1f-953dbb295f3e [Candidate]" term=2 -2024-05-15T01:17:08.043Z [INFO] server-35e06bf4-687d-78ca-ef1f-953dbb295f3e: election won: term=2 tally=1 -2024-05-15T01:17:08.043Z [INFO] server-35e06bf4-687d-78ca-ef1f-953dbb295f3e: entering leader state: leader="Node at 35e06bf4-687d-78ca-ef1f-953dbb295f3e [Leader]" -2024-05-15T01:17:08.245Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:08.043426643 +0000 UTC m=+8.584812502 (1 nodes), 186.775204ms from start of poll, 186.998047ms from cluster start. Timeout at 2024-05-15 01:17:08.245301011 +0000 UTC m=+8.786685249, 201.872747ms after stability ---- PASS: TestRaft_ApplyConcurrent_Timeout (0.40s) +2025-06-17T07:43:00.039Z [INFO] server-cf83713a-6c96-b9d9-75d0-6138d569cda9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf83713a-6c96-b9d9-75d0-6138d569cda9 Address:cf83713a-6c96-b9d9-75d0-6138d569cda9}]" +2025-06-17T07:43:00.039Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:00.039Z [INFO] server-cf83713a-6c96-b9d9-75d0-6138d569cda9: entering follower state: follower="Node at cf83713a-6c96-b9d9-75d0-6138d569cda9 [Follower]" leader-address= leader-id= +2025-06-17T07:43:00.211Z [WARN] server-cf83713a-6c96-b9d9-75d0-6138d569cda9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:00.211Z [INFO] server-cf83713a-6c96-b9d9-75d0-6138d569cda9: entering candidate state: node="Node at cf83713a-6c96-b9d9-75d0-6138d569cda9 [Candidate]" term=2 +2025-06-17T07:43:00.211Z [INFO] server-cf83713a-6c96-b9d9-75d0-6138d569cda9: election won: term=2 tally=1 +2025-06-17T07:43:00.211Z [INFO] server-cf83713a-6c96-b9d9-75d0-6138d569cda9: entering leader state: leader="Node at cf83713a-6c96-b9d9-75d0-6138d569cda9 [Leader]" +2025-06-17T07:43:00.412Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:00.211538388 +0000 UTC m=+8.815067090 (1 nodes), 171.719914ms from start of poll, 171.857386ms from cluster start. Timeout at 2025-06-17 07:43:00.412788204 +0000 UTC m=+9.016316887, 201.249797ms after stability +--- PASS: TestRaft_ApplyConcurrent_Timeout (0.37s) === RUN TestRaft_JoinNode -2024-05-15T01:17:08.252Z [INFO] server-58c367b1-ca2b-cac2-d3e0-2ccef6fe68cd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-58c367b1-ca2b-cac2-d3e0-2ccef6fe68cd Address:58c367b1-ca2b-cac2-d3e0-2ccef6fe68cd} {Suffrage:Voter ID:server-ff0263f6-479e-1792-2998-08a5af7ebdcd Address:ff0263f6-479e-1792-2998-08a5af7ebdcd}]" -2024-05-15T01:17:08.252Z [INFO] server-ff0263f6-479e-1792-2998-08a5af7ebdcd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-58c367b1-ca2b-cac2-d3e0-2ccef6fe68cd Address:58c367b1-ca2b-cac2-d3e0-2ccef6fe68cd} {Suffrage:Voter ID:server-ff0263f6-479e-1792-2998-08a5af7ebdcd Address:ff0263f6-479e-1792-2998-08a5af7ebdcd}]" -2024-05-15T01:17:08.253Z [INFO] server-5c6ba903-5001-fcf3-448e-5bfaf2985493: initial configuration: index=0 servers=[] -2024-05-15T01:17:08.253Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:08.253Z [INFO] server-58c367b1-ca2b-cac2-d3e0-2ccef6fe68cd: entering follower state: follower="Node at 58c367b1-ca2b-cac2-d3e0-2ccef6fe68cd [Follower]" leader-address= leader-id= -2024-05-15T01:17:08.253Z [INFO] server-ff0263f6-479e-1792-2998-08a5af7ebdcd: entering follower state: follower="Node at ff0263f6-479e-1792-2998-08a5af7ebdcd [Follower]" leader-address= leader-id= -2024-05-15T01:17:08.253Z [INFO] server-5c6ba903-5001-fcf3-448e-5bfaf2985493: entering follower state: follower="Node at 5c6ba903-5001-fcf3-448e-5bfaf2985493 [Follower]" leader-address= leader-id= -2024-05-15T01:17:08.306Z [WARN] server-ff0263f6-479e-1792-2998-08a5af7ebdcd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:08.306Z [INFO] server-ff0263f6-479e-1792-2998-08a5af7ebdcd: entering candidate state: node="Node at ff0263f6-479e-1792-2998-08a5af7ebdcd [Candidate]" term=2 -2024-05-15T01:17:08.307Z [INFO] server-ff0263f6-479e-1792-2998-08a5af7ebdcd: election won: term=2 tally=2 -2024-05-15T01:17:08.307Z [INFO] server-ff0263f6-479e-1792-2998-08a5af7ebdcd: entering leader state: leader="Node at ff0263f6-479e-1792-2998-08a5af7ebdcd [Leader]" -2024-05-15T01:17:08.307Z [INFO] server-ff0263f6-479e-1792-2998-08a5af7ebdcd: added peer, starting replication: peer=server-58c367b1-ca2b-cac2-d3e0-2ccef6fe68cd -2024-05-15T01:17:08.307Z [INFO] server-ff0263f6-479e-1792-2998-08a5af7ebdcd: pipelining replication: peer="{Voter server-58c367b1-ca2b-cac2-d3e0-2ccef6fe68cd 58c367b1-ca2b-cac2-d3e0-2ccef6fe68cd}" -2024-05-15T01:17:08.316Z [WARN] server-5c6ba903-5001-fcf3-448e-5bfaf2985493: no known peers, aborting election -2024-05-15T01:17:08.413Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:08.307384363 +0000 UTC m=+8.848769129 (1 nodes), 54.047163ms from start of poll, 54.694805ms from cluster start. Timeout at 2024-05-15 01:17:08.413088607 +0000 UTC m=+8.954473368, 105.704239ms after stability -2024-05-15T01:17:08.413Z [INFO] server-ff0263f6-479e-1792-2998-08a5af7ebdcd: updating configuration: command=AddVoter server-id=server-5c6ba903-5001-fcf3-448e-5bfaf2985493 server-addr=5c6ba903-5001-fcf3-448e-5bfaf2985493 servers="[{Suffrage:Voter ID:server-58c367b1-ca2b-cac2-d3e0-2ccef6fe68cd Address:58c367b1-ca2b-cac2-d3e0-2ccef6fe68cd} {Suffrage:Voter ID:server-ff0263f6-479e-1792-2998-08a5af7ebdcd Address:ff0263f6-479e-1792-2998-08a5af7ebdcd} {Suffrage:Voter ID:server-5c6ba903-5001-fcf3-448e-5bfaf2985493 Address:5c6ba903-5001-fcf3-448e-5bfaf2985493}]" -2024-05-15T01:17:08.413Z [INFO] server-ff0263f6-479e-1792-2998-08a5af7ebdcd: added peer, starting replication: peer=server-5c6ba903-5001-fcf3-448e-5bfaf2985493 -2024-05-15T01:17:08.413Z [WARN] server-5c6ba903-5001-fcf3-448e-5bfaf2985493: failed to get previous log: previous-index=3 last-index=0 error="log not found" -2024-05-15T01:17:08.413Z [WARN] server-ff0263f6-479e-1792-2998-08a5af7ebdcd: appendEntries rejected, sending older logs: peer="{Voter server-5c6ba903-5001-fcf3-448e-5bfaf2985493 5c6ba903-5001-fcf3-448e-5bfaf2985493}" next=1 -2024-05-15T01:17:08.413Z [INFO] server-ff0263f6-479e-1792-2998-08a5af7ebdcd: pipelining replication: peer="{Voter server-5c6ba903-5001-fcf3-448e-5bfaf2985493 5c6ba903-5001-fcf3-448e-5bfaf2985493}" -2024-05-15T01:17:08.413Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:08.519Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:08.413850895 +0000 UTC m=+8.955235639 (1 nodes), 1.813µs from start of poll, 161.161315ms from cluster start. Timeout at 2024-05-15 01:17:08.519585551 +0000 UTC m=+9.060970330, 105.734691ms after stability -2024-05-15T01:17:08.519Z [INFO] server-ff0263f6-479e-1792-2998-08a5af7ebdcd: aborting pipeline replication: peer="{Voter server-5c6ba903-5001-fcf3-448e-5bfaf2985493 5c6ba903-5001-fcf3-448e-5bfaf2985493}" -2024-05-15T01:17:08.519Z [INFO] server-ff0263f6-479e-1792-2998-08a5af7ebdcd: aborting pipeline replication: peer="{Voter server-58c367b1-ca2b-cac2-d3e0-2ccef6fe68cd 58c367b1-ca2b-cac2-d3e0-2ccef6fe68cd}" ---- PASS: TestRaft_JoinNode (0.27s) +2025-06-17T07:43:00.413Z [INFO] server-d0f5436f-a7cb-287b-e359-18e6ba96ddb6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d0f5436f-a7cb-287b-e359-18e6ba96ddb6 Address:d0f5436f-a7cb-287b-e359-18e6ba96ddb6} {Suffrage:Voter ID:server-67714710-076a-f35a-35e9-75bdaf84d468 Address:67714710-076a-f35a-35e9-75bdaf84d468}]" +2025-06-17T07:43:00.413Z [INFO] server-d0f5436f-a7cb-287b-e359-18e6ba96ddb6: entering follower state: follower="Node at d0f5436f-a7cb-287b-e359-18e6ba96ddb6 [Follower]" leader-address= leader-id= +2025-06-17T07:43:00.413Z [INFO] server-67714710-076a-f35a-35e9-75bdaf84d468: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d0f5436f-a7cb-287b-e359-18e6ba96ddb6 Address:d0f5436f-a7cb-287b-e359-18e6ba96ddb6} {Suffrage:Voter ID:server-67714710-076a-f35a-35e9-75bdaf84d468 Address:67714710-076a-f35a-35e9-75bdaf84d468}]" +2025-06-17T07:43:00.413Z [INFO] server-67714710-076a-f35a-35e9-75bdaf84d468: entering follower state: follower="Node at 67714710-076a-f35a-35e9-75bdaf84d468 [Follower]" leader-address= leader-id= +2025-06-17T07:43:00.413Z [INFO] server-c5100d20-f914-3bf3-41c1-85be6acf64e0: initial configuration: index=0 servers=[] +2025-06-17T07:43:00.413Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:00.413Z [INFO] server-c5100d20-f914-3bf3-41c1-85be6acf64e0: entering follower state: follower="Node at c5100d20-f914-3bf3-41c1-85be6acf64e0 [Follower]" leader-address= leader-id= +2025-06-17T07:43:00.466Z [WARN] server-c5100d20-f914-3bf3-41c1-85be6acf64e0: no known peers, aborting election +2025-06-17T07:43:00.488Z [WARN] server-d0f5436f-a7cb-287b-e359-18e6ba96ddb6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:00.488Z [INFO] server-d0f5436f-a7cb-287b-e359-18e6ba96ddb6: entering candidate state: node="Node at d0f5436f-a7cb-287b-e359-18e6ba96ddb6 [Candidate]" term=2 +2025-06-17T07:43:00.488Z [INFO] server-d0f5436f-a7cb-287b-e359-18e6ba96ddb6: election won: term=2 tally=2 +2025-06-17T07:43:00.488Z [INFO] server-d0f5436f-a7cb-287b-e359-18e6ba96ddb6: entering leader state: leader="Node at d0f5436f-a7cb-287b-e359-18e6ba96ddb6 [Leader]" +2025-06-17T07:43:00.488Z [INFO] server-d0f5436f-a7cb-287b-e359-18e6ba96ddb6: added peer, starting replication: peer=server-67714710-076a-f35a-35e9-75bdaf84d468 +2025-06-17T07:43:00.488Z [INFO] server-d0f5436f-a7cb-287b-e359-18e6ba96ddb6: pipelining replication: peer="{Voter server-67714710-076a-f35a-35e9-75bdaf84d468 67714710-076a-f35a-35e9-75bdaf84d468}" +2025-06-17T07:43:00.594Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:00.488865291 +0000 UTC m=+9.092393978 (1 nodes), 75.13128ms from start of poll, 75.49381ms from cluster start. Timeout at 2025-06-17 07:43:00.594318287 +0000 UTC m=+9.197846982, 105.453004ms after stability +2025-06-17T07:43:00.594Z [INFO] server-d0f5436f-a7cb-287b-e359-18e6ba96ddb6: updating configuration: command=AddVoter server-id=server-c5100d20-f914-3bf3-41c1-85be6acf64e0 server-addr=c5100d20-f914-3bf3-41c1-85be6acf64e0 servers="[{Suffrage:Voter ID:server-d0f5436f-a7cb-287b-e359-18e6ba96ddb6 Address:d0f5436f-a7cb-287b-e359-18e6ba96ddb6} {Suffrage:Voter ID:server-67714710-076a-f35a-35e9-75bdaf84d468 Address:67714710-076a-f35a-35e9-75bdaf84d468} {Suffrage:Voter ID:server-c5100d20-f914-3bf3-41c1-85be6acf64e0 Address:c5100d20-f914-3bf3-41c1-85be6acf64e0}]" +2025-06-17T07:43:00.594Z [INFO] server-d0f5436f-a7cb-287b-e359-18e6ba96ddb6: added peer, starting replication: peer=server-c5100d20-f914-3bf3-41c1-85be6acf64e0 +2025-06-17T07:43:00.594Z [WARN] server-c5100d20-f914-3bf3-41c1-85be6acf64e0: failed to get previous log: previous-index=3 last-index=0 error="log not found" +2025-06-17T07:43:00.594Z [WARN] server-d0f5436f-a7cb-287b-e359-18e6ba96ddb6: appendEntries rejected, sending older logs: peer="{Voter server-c5100d20-f914-3bf3-41c1-85be6acf64e0 c5100d20-f914-3bf3-41c1-85be6acf64e0}" next=1 +2025-06-17T07:43:00.594Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:00.594Z [INFO] server-d0f5436f-a7cb-287b-e359-18e6ba96ddb6: pipelining replication: peer="{Voter server-c5100d20-f914-3bf3-41c1-85be6acf64e0 c5100d20-f914-3bf3-41c1-85be6acf64e0}" +2025-06-17T07:43:00.700Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:00.594673002 +0000 UTC m=+9.198201689 (1 nodes), 544ns from start of poll, 181.301521ms from cluster start. Timeout at 2025-06-17 07:43:00.699998975 +0000 UTC m=+9.303527658, 105.325969ms after stability +2025-06-17T07:43:00.700Z [INFO] server-d0f5436f-a7cb-287b-e359-18e6ba96ddb6: aborting pipeline replication: peer="{Voter server-c5100d20-f914-3bf3-41c1-85be6acf64e0 c5100d20-f914-3bf3-41c1-85be6acf64e0}" +2025-06-17T07:43:00.700Z [INFO] server-d0f5436f-a7cb-287b-e359-18e6ba96ddb6: aborting pipeline replication: peer="{Voter server-67714710-076a-f35a-35e9-75bdaf84d468 67714710-076a-f35a-35e9-75bdaf84d468}" +--- PASS: TestRaft_JoinNode (0.29s) === RUN TestRaft_JoinNode_ConfigStore -2024-05-15T01:17:08.521Z [INFO] server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487 Address:d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487}]" -2024-05-15T01:17:08.521Z [INFO] server-bfc3aa5b-d73a-46b0-757f-4c6ddddb5d0b: initial configuration: index=0 servers=[] -2024-05-15T01:17:08.521Z [INFO] server-af281fed-cf90-aa70-49c2-f346ef210342: initial configuration: index=0 servers=[] -2024-05-15T01:17:08.521Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:08.521Z [INFO] server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487: entering follower state: follower="Node at d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487 [Follower]" leader-address= leader-id= -2024-05-15T01:17:08.521Z [INFO] server-bfc3aa5b-d73a-46b0-757f-4c6ddddb5d0b: entering follower state: follower="Node at bfc3aa5b-d73a-46b0-757f-4c6ddddb5d0b [Follower]" leader-address= leader-id= -2024-05-15T01:17:08.521Z [INFO] server-af281fed-cf90-aa70-49c2-f346ef210342: entering follower state: follower="Node at af281fed-cf90-aa70-49c2-f346ef210342 [Follower]" leader-address= leader-id= -2024-05-15T01:17:08.602Z [WARN] server-bfc3aa5b-d73a-46b0-757f-4c6ddddb5d0b: no known peers, aborting election -2024-05-15T01:17:08.615Z [WARN] server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:08.615Z [INFO] server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487: entering candidate state: node="Node at d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487 [Candidate]" term=2 -2024-05-15T01:17:08.615Z [INFO] server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487: election won: term=2 tally=1 -2024-05-15T01:17:08.615Z [INFO] server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487: entering leader state: leader="Node at d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487 [Leader]" -2024-05-15T01:17:08.620Z [WARN] server-af281fed-cf90-aa70-49c2-f346ef210342: no known peers, aborting election -2024-05-15T01:17:08.721Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:08.616128556 +0000 UTC m=+9.157513310 (1 nodes), 94.402875ms from start of poll, 95.199894ms from cluster start. Timeout at 2024-05-15 01:17:08.721373065 +0000 UTC m=+9.262757848, 105.244538ms after stability -2024-05-15T01:17:08.721Z [INFO] server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487: updating configuration: command=AddVoter server-id=server-bfc3aa5b-d73a-46b0-757f-4c6ddddb5d0b server-addr=bfc3aa5b-d73a-46b0-757f-4c6ddddb5d0b servers="[{Suffrage:Voter ID:server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487 Address:d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487} {Suffrage:Voter ID:server-bfc3aa5b-d73a-46b0-757f-4c6ddddb5d0b Address:bfc3aa5b-d73a-46b0-757f-4c6ddddb5d0b}]" -2024-05-15T01:17:08.721Z [INFO] server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487: added peer, starting replication: peer=server-bfc3aa5b-d73a-46b0-757f-4c6ddddb5d0b -2024-05-15T01:17:08.721Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:08.721Z [WARN] server-bfc3aa5b-d73a-46b0-757f-4c6ddddb5d0b: failed to get previous log: previous-index=3 last-index=0 error="log not found" -2024-05-15T01:17:08.722Z [WARN] server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487: appendEntries rejected, sending older logs: peer="{Voter server-bfc3aa5b-d73a-46b0-757f-4c6ddddb5d0b bfc3aa5b-d73a-46b0-757f-4c6ddddb5d0b}" next=1 -2024-05-15T01:17:08.722Z [INFO] server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487: pipelining replication: peer="{Voter server-bfc3aa5b-d73a-46b0-757f-4c6ddddb5d0b bfc3aa5b-d73a-46b0-757f-4c6ddddb5d0b}" -2024-05-15T01:17:08.827Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:08.721796183 +0000 UTC m=+9.263180914 (1 nodes), 2.1µs from start of poll, 200.867498ms from cluster start. Timeout at 2024-05-15 01:17:08.826943455 +0000 UTC m=+9.368328206, 105.147292ms after stability -2024-05-15T01:17:08.827Z [INFO] server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487: updating configuration: command=AddVoter server-id=server-af281fed-cf90-aa70-49c2-f346ef210342 server-addr=af281fed-cf90-aa70-49c2-f346ef210342 servers="[{Suffrage:Voter ID:server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487 Address:d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487} {Suffrage:Voter ID:server-bfc3aa5b-d73a-46b0-757f-4c6ddddb5d0b Address:bfc3aa5b-d73a-46b0-757f-4c6ddddb5d0b} {Suffrage:Voter ID:server-af281fed-cf90-aa70-49c2-f346ef210342 Address:af281fed-cf90-aa70-49c2-f346ef210342}]" -2024-05-15T01:17:08.827Z [INFO] server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487: added peer, starting replication: peer=server-af281fed-cf90-aa70-49c2-f346ef210342 -2024-05-15T01:17:08.827Z [WARN] server-af281fed-cf90-aa70-49c2-f346ef210342: failed to get previous log: previous-index=4 last-index=0 error="log not found" -2024-05-15T01:17:08.827Z [WARN] server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487: appendEntries rejected, sending older logs: peer="{Voter server-af281fed-cf90-aa70-49c2-f346ef210342 af281fed-cf90-aa70-49c2-f346ef210342}" next=1 -2024-05-15T01:17:08.827Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:08.827Z [INFO] server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487: pipelining replication: peer="{Voter server-af281fed-cf90-aa70-49c2-f346ef210342 af281fed-cf90-aa70-49c2-f346ef210342}" -2024-05-15T01:17:08.933Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:08.827533718 +0000 UTC m=+9.368918713 (1 nodes), 2.715µs from start of poll, 306.605297ms from cluster start. Timeout at 2024-05-15 01:17:08.933384729 +0000 UTC m=+9.474769547, 105.850834ms after stability -2024-05-15T01:17:08.933Z [INFO] server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487: aborting pipeline replication: peer="{Voter server-af281fed-cf90-aa70-49c2-f346ef210342 af281fed-cf90-aa70-49c2-f346ef210342}" -2024-05-15T01:17:08.933Z [INFO] server-d2ea6bc0-16ac-3adb-ca0f-d71cbf3ba487: aborting pipeline replication: peer="{Voter server-bfc3aa5b-d73a-46b0-757f-4c6ddddb5d0b bfc3aa5b-d73a-46b0-757f-4c6ddddb5d0b}" ---- PASS: TestRaft_JoinNode_ConfigStore (0.41s) +2025-06-17T07:43:00.700Z [INFO] server-466a2ba1-1d34-61ae-9657-73e30a24a51e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-466a2ba1-1d34-61ae-9657-73e30a24a51e Address:466a2ba1-1d34-61ae-9657-73e30a24a51e}]" +2025-06-17T07:43:00.700Z [INFO] server-466a2ba1-1d34-61ae-9657-73e30a24a51e: entering follower state: follower="Node at 466a2ba1-1d34-61ae-9657-73e30a24a51e [Follower]" leader-address= leader-id= +2025-06-17T07:43:00.700Z [INFO] server-f31eb2d9-5aed-ddba-c097-19a167a1ee93: initial configuration: index=0 servers=[] +2025-06-17T07:43:00.700Z [INFO] server-f31eb2d9-5aed-ddba-c097-19a167a1ee93: entering follower state: follower="Node at f31eb2d9-5aed-ddba-c097-19a167a1ee93 [Follower]" leader-address= leader-id= +2025-06-17T07:43:00.701Z [INFO] server-d1a876bc-5f75-7030-2ae0-05307ebdf0fd: initial configuration: index=0 servers=[] +2025-06-17T07:43:00.701Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:00.701Z [INFO] server-d1a876bc-5f75-7030-2ae0-05307ebdf0fd: entering follower state: follower="Node at d1a876bc-5f75-7030-2ae0-05307ebdf0fd [Follower]" leader-address= leader-id= +2025-06-17T07:43:00.755Z [WARN] server-466a2ba1-1d34-61ae-9657-73e30a24a51e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:00.755Z [INFO] server-466a2ba1-1d34-61ae-9657-73e30a24a51e: entering candidate state: node="Node at 466a2ba1-1d34-61ae-9657-73e30a24a51e [Candidate]" term=2 +2025-06-17T07:43:00.755Z [INFO] server-466a2ba1-1d34-61ae-9657-73e30a24a51e: election won: term=2 tally=1 +2025-06-17T07:43:00.755Z [INFO] server-466a2ba1-1d34-61ae-9657-73e30a24a51e: entering leader state: leader="Node at 466a2ba1-1d34-61ae-9657-73e30a24a51e [Leader]" +2025-06-17T07:43:00.774Z [WARN] server-f31eb2d9-5aed-ddba-c097-19a167a1ee93: no known peers, aborting election +2025-06-17T07:43:00.776Z [WARN] server-d1a876bc-5f75-7030-2ae0-05307ebdf0fd: no known peers, aborting election +2025-06-17T07:43:00.860Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:00.75511121 +0000 UTC m=+9.358639900 (1 nodes), 54.016906ms from start of poll, 54.530839ms from cluster start. Timeout at 2025-06-17 07:43:00.860641461 +0000 UTC m=+9.464170146, 105.530246ms after stability +2025-06-17T07:43:00.860Z [INFO] server-466a2ba1-1d34-61ae-9657-73e30a24a51e: updating configuration: command=AddVoter server-id=server-f31eb2d9-5aed-ddba-c097-19a167a1ee93 server-addr=f31eb2d9-5aed-ddba-c097-19a167a1ee93 servers="[{Suffrage:Voter ID:server-466a2ba1-1d34-61ae-9657-73e30a24a51e Address:466a2ba1-1d34-61ae-9657-73e30a24a51e} {Suffrage:Voter ID:server-f31eb2d9-5aed-ddba-c097-19a167a1ee93 Address:f31eb2d9-5aed-ddba-c097-19a167a1ee93}]" +2025-06-17T07:43:00.860Z [INFO] server-466a2ba1-1d34-61ae-9657-73e30a24a51e: added peer, starting replication: peer=server-f31eb2d9-5aed-ddba-c097-19a167a1ee93 +2025-06-17T07:43:00.860Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:00.860Z [WARN] server-f31eb2d9-5aed-ddba-c097-19a167a1ee93: failed to get previous log: previous-index=3 last-index=0 error="log not found" +2025-06-17T07:43:00.860Z [WARN] server-466a2ba1-1d34-61ae-9657-73e30a24a51e: appendEntries rejected, sending older logs: peer="{Voter server-f31eb2d9-5aed-ddba-c097-19a167a1ee93 f31eb2d9-5aed-ddba-c097-19a167a1ee93}" next=1 +2025-06-17T07:43:00.861Z [INFO] server-466a2ba1-1d34-61ae-9657-73e30a24a51e: pipelining replication: peer="{Voter server-f31eb2d9-5aed-ddba-c097-19a167a1ee93 f31eb2d9-5aed-ddba-c097-19a167a1ee93}" +2025-06-17T07:43:00.966Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:00.860890337 +0000 UTC m=+9.464419024 (1 nodes), 940ns from start of poll, 160.309963ms from cluster start. Timeout at 2025-06-17 07:43:00.966468751 +0000 UTC m=+9.569997469, 105.578445ms after stability +2025-06-17T07:43:00.966Z [INFO] server-466a2ba1-1d34-61ae-9657-73e30a24a51e: updating configuration: command=AddVoter server-id=server-d1a876bc-5f75-7030-2ae0-05307ebdf0fd server-addr=d1a876bc-5f75-7030-2ae0-05307ebdf0fd servers="[{Suffrage:Voter ID:server-466a2ba1-1d34-61ae-9657-73e30a24a51e Address:466a2ba1-1d34-61ae-9657-73e30a24a51e} {Suffrage:Voter ID:server-f31eb2d9-5aed-ddba-c097-19a167a1ee93 Address:f31eb2d9-5aed-ddba-c097-19a167a1ee93} {Suffrage:Voter ID:server-d1a876bc-5f75-7030-2ae0-05307ebdf0fd Address:d1a876bc-5f75-7030-2ae0-05307ebdf0fd}]" +2025-06-17T07:43:00.966Z [INFO] server-466a2ba1-1d34-61ae-9657-73e30a24a51e: added peer, starting replication: peer=server-d1a876bc-5f75-7030-2ae0-05307ebdf0fd +2025-06-17T07:43:00.966Z [WARN] server-d1a876bc-5f75-7030-2ae0-05307ebdf0fd: failed to get previous log: previous-index=4 last-index=0 error="log not found" +2025-06-17T07:43:00.966Z [WARN] server-466a2ba1-1d34-61ae-9657-73e30a24a51e: appendEntries rejected, sending older logs: peer="{Voter server-d1a876bc-5f75-7030-2ae0-05307ebdf0fd d1a876bc-5f75-7030-2ae0-05307ebdf0fd}" next=1 +2025-06-17T07:43:00.966Z [INFO] server-466a2ba1-1d34-61ae-9657-73e30a24a51e: pipelining replication: peer="{Voter server-d1a876bc-5f75-7030-2ae0-05307ebdf0fd d1a876bc-5f75-7030-2ae0-05307ebdf0fd}" +2025-06-17T07:43:00.966Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:01.072Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:00.966938221 +0000 UTC m=+9.570466913 (1 nodes), 659ns from start of poll, 266.357852ms from cluster start. Timeout at 2025-06-17 07:43:01.072734199 +0000 UTC m=+9.676262879, 105.795966ms after stability +2025-06-17T07:43:01.072Z [INFO] server-466a2ba1-1d34-61ae-9657-73e30a24a51e: aborting pipeline replication: peer="{Voter server-d1a876bc-5f75-7030-2ae0-05307ebdf0fd d1a876bc-5f75-7030-2ae0-05307ebdf0fd}" +2025-06-17T07:43:01.072Z [INFO] server-466a2ba1-1d34-61ae-9657-73e30a24a51e: aborting pipeline replication: peer="{Voter server-f31eb2d9-5aed-ddba-c097-19a167a1ee93 f31eb2d9-5aed-ddba-c097-19a167a1ee93}" +--- PASS: TestRaft_JoinNode_ConfigStore (0.37s) === RUN TestRaft_RemoveFollower -2024-05-15T01:17:08.935Z [INFO] server-d6bc1ba2-03cd-51f1-9fc2-da3eaeb6c847: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d6bc1ba2-03cd-51f1-9fc2-da3eaeb6c847 Address:d6bc1ba2-03cd-51f1-9fc2-da3eaeb6c847} {Suffrage:Voter ID:server-a87b575d-6b58-4fc4-bd9d-df381b445c35 Address:a87b575d-6b58-4fc4-bd9d-df381b445c35} {Suffrage:Voter ID:server-49fcd411-7a3c-f432-9576-12e9da1f28ae Address:49fcd411-7a3c-f432-9576-12e9da1f28ae}]" -2024-05-15T01:17:08.935Z [INFO] server-d6bc1ba2-03cd-51f1-9fc2-da3eaeb6c847: entering follower state: follower="Node at d6bc1ba2-03cd-51f1-9fc2-da3eaeb6c847 [Follower]" leader-address= leader-id= -2024-05-15T01:17:08.935Z [INFO] server-a87b575d-6b58-4fc4-bd9d-df381b445c35: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d6bc1ba2-03cd-51f1-9fc2-da3eaeb6c847 Address:d6bc1ba2-03cd-51f1-9fc2-da3eaeb6c847} {Suffrage:Voter ID:server-a87b575d-6b58-4fc4-bd9d-df381b445c35 Address:a87b575d-6b58-4fc4-bd9d-df381b445c35} {Suffrage:Voter ID:server-49fcd411-7a3c-f432-9576-12e9da1f28ae Address:49fcd411-7a3c-f432-9576-12e9da1f28ae}]" -2024-05-15T01:17:08.935Z [INFO] server-a87b575d-6b58-4fc4-bd9d-df381b445c35: entering follower state: follower="Node at a87b575d-6b58-4fc4-bd9d-df381b445c35 [Follower]" leader-address= leader-id= -2024-05-15T01:17:08.935Z [INFO] server-49fcd411-7a3c-f432-9576-12e9da1f28ae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d6bc1ba2-03cd-51f1-9fc2-da3eaeb6c847 Address:d6bc1ba2-03cd-51f1-9fc2-da3eaeb6c847} {Suffrage:Voter ID:server-a87b575d-6b58-4fc4-bd9d-df381b445c35 Address:a87b575d-6b58-4fc4-bd9d-df381b445c35} {Suffrage:Voter ID:server-49fcd411-7a3c-f432-9576-12e9da1f28ae Address:49fcd411-7a3c-f432-9576-12e9da1f28ae}]" -2024-05-15T01:17:08.935Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:08.935Z [INFO] server-49fcd411-7a3c-f432-9576-12e9da1f28ae: entering follower state: follower="Node at 49fcd411-7a3c-f432-9576-12e9da1f28ae [Follower]" leader-address= leader-id= -2024-05-15T01:17:08.986Z [WARN] server-a87b575d-6b58-4fc4-bd9d-df381b445c35: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:08.986Z [INFO] server-a87b575d-6b58-4fc4-bd9d-df381b445c35: entering candidate state: node="Node at a87b575d-6b58-4fc4-bd9d-df381b445c35 [Candidate]" term=2 -2024-05-15T01:17:08.986Z [INFO] server-a87b575d-6b58-4fc4-bd9d-df381b445c35: election won: term=2 tally=2 -2024-05-15T01:17:08.986Z [INFO] server-a87b575d-6b58-4fc4-bd9d-df381b445c35: entering leader state: leader="Node at a87b575d-6b58-4fc4-bd9d-df381b445c35 [Leader]" -2024-05-15T01:17:08.986Z [INFO] server-a87b575d-6b58-4fc4-bd9d-df381b445c35: added peer, starting replication: peer=server-d6bc1ba2-03cd-51f1-9fc2-da3eaeb6c847 -2024-05-15T01:17:08.986Z [INFO] server-a87b575d-6b58-4fc4-bd9d-df381b445c35: added peer, starting replication: peer=server-49fcd411-7a3c-f432-9576-12e9da1f28ae -2024-05-15T01:17:08.986Z [INFO] server-a87b575d-6b58-4fc4-bd9d-df381b445c35: pipelining replication: peer="{Voter server-49fcd411-7a3c-f432-9576-12e9da1f28ae 49fcd411-7a3c-f432-9576-12e9da1f28ae}" -2024-05-15T01:17:08.987Z [INFO] server-a87b575d-6b58-4fc4-bd9d-df381b445c35: pipelining replication: peer="{Voter server-d6bc1ba2-03cd-51f1-9fc2-da3eaeb6c847 d6bc1ba2-03cd-51f1-9fc2-da3eaeb6c847}" -2024-05-15T01:17:09.092Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:08.987224956 +0000 UTC m=+9.528609716 (1 nodes), 51.343281ms from start of poll, 51.958746ms from cluster start. Timeout at 2024-05-15 01:17:09.092328841 +0000 UTC m=+9.633713616, 105.1039ms after stability -2024-05-15T01:17:09.098Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:09.204Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:09.098100939 +0000 UTC m=+9.639485664 (2 nodes), 2.805µs from start of poll, 162.834694ms from cluster start. Timeout at 2024-05-15 01:17:09.203998943 +0000 UTC m=+9.745383694, 105.89803ms after stability -2024-05-15T01:17:09.204Z [INFO] server-a87b575d-6b58-4fc4-bd9d-df381b445c35: updating configuration: command=RemoveServer server-id=server-d6bc1ba2-03cd-51f1-9fc2-da3eaeb6c847 server-addr= servers="[{Suffrage:Voter ID:server-a87b575d-6b58-4fc4-bd9d-df381b445c35 Address:a87b575d-6b58-4fc4-bd9d-df381b445c35} {Suffrage:Voter ID:server-49fcd411-7a3c-f432-9576-12e9da1f28ae Address:49fcd411-7a3c-f432-9576-12e9da1f28ae}]" -2024-05-15T01:17:09.204Z [INFO] server-a87b575d-6b58-4fc4-bd9d-df381b445c35: removed peer, stopping replication: peer=server-d6bc1ba2-03cd-51f1-9fc2-da3eaeb6c847 last-index=3 -2024-05-15T01:17:09.204Z [INFO] server-a87b575d-6b58-4fc4-bd9d-df381b445c35: aborting pipeline replication: peer="{Voter server-d6bc1ba2-03cd-51f1-9fc2-da3eaeb6c847 d6bc1ba2-03cd-51f1-9fc2-da3eaeb6c847}" -2024-05-15T01:17:09.296Z [WARN] server-d6bc1ba2-03cd-51f1-9fc2-da3eaeb6c847: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2024-05-15T01:17:09.310Z [INFO] server-a87b575d-6b58-4fc4-bd9d-df381b445c35: aborting pipeline replication: peer="{Voter server-49fcd411-7a3c-f432-9576-12e9da1f28ae 49fcd411-7a3c-f432-9576-12e9da1f28ae}" ---- PASS: TestRaft_RemoveFollower (0.38s) +2025-06-17T07:43:01.074Z [INFO] server-a422f3f1-7e07-8513-dcea-7780030d3dbd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a422f3f1-7e07-8513-dcea-7780030d3dbd Address:a422f3f1-7e07-8513-dcea-7780030d3dbd} {Suffrage:Voter ID:server-11238980-fd33-72bf-1741-1957df500d77 Address:11238980-fd33-72bf-1741-1957df500d77} {Suffrage:Voter ID:server-57ab72ed-a7fb-a23f-f42b-6fffc31b7698 Address:57ab72ed-a7fb-a23f-f42b-6fffc31b7698}]" +2025-06-17T07:43:01.074Z [INFO] server-a422f3f1-7e07-8513-dcea-7780030d3dbd: entering follower state: follower="Node at a422f3f1-7e07-8513-dcea-7780030d3dbd [Follower]" leader-address= leader-id= +2025-06-17T07:43:01.074Z [INFO] server-11238980-fd33-72bf-1741-1957df500d77: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a422f3f1-7e07-8513-dcea-7780030d3dbd Address:a422f3f1-7e07-8513-dcea-7780030d3dbd} {Suffrage:Voter ID:server-11238980-fd33-72bf-1741-1957df500d77 Address:11238980-fd33-72bf-1741-1957df500d77} {Suffrage:Voter ID:server-57ab72ed-a7fb-a23f-f42b-6fffc31b7698 Address:57ab72ed-a7fb-a23f-f42b-6fffc31b7698}]" +2025-06-17T07:43:01.074Z [INFO] server-11238980-fd33-72bf-1741-1957df500d77: entering follower state: follower="Node at 11238980-fd33-72bf-1741-1957df500d77 [Follower]" leader-address= leader-id= +2025-06-17T07:43:01.074Z [INFO] server-57ab72ed-a7fb-a23f-f42b-6fffc31b7698: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a422f3f1-7e07-8513-dcea-7780030d3dbd Address:a422f3f1-7e07-8513-dcea-7780030d3dbd} {Suffrage:Voter ID:server-11238980-fd33-72bf-1741-1957df500d77 Address:11238980-fd33-72bf-1741-1957df500d77} {Suffrage:Voter ID:server-57ab72ed-a7fb-a23f-f42b-6fffc31b7698 Address:57ab72ed-a7fb-a23f-f42b-6fffc31b7698}]" +2025-06-17T07:43:01.074Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:01.074Z [INFO] server-57ab72ed-a7fb-a23f-f42b-6fffc31b7698: entering follower state: follower="Node at 57ab72ed-a7fb-a23f-f42b-6fffc31b7698 [Follower]" leader-address= leader-id= +2025-06-17T07:43:01.138Z [WARN] server-57ab72ed-a7fb-a23f-f42b-6fffc31b7698: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:01.138Z [INFO] server-57ab72ed-a7fb-a23f-f42b-6fffc31b7698: entering candidate state: node="Node at 57ab72ed-a7fb-a23f-f42b-6fffc31b7698 [Candidate]" term=2 +2025-06-17T07:43:01.138Z [INFO] server-57ab72ed-a7fb-a23f-f42b-6fffc31b7698: election won: term=2 tally=2 +2025-06-17T07:43:01.138Z [INFO] server-57ab72ed-a7fb-a23f-f42b-6fffc31b7698: entering leader state: leader="Node at 57ab72ed-a7fb-a23f-f42b-6fffc31b7698 [Leader]" +2025-06-17T07:43:01.138Z [INFO] server-57ab72ed-a7fb-a23f-f42b-6fffc31b7698: added peer, starting replication: peer=server-a422f3f1-7e07-8513-dcea-7780030d3dbd +2025-06-17T07:43:01.138Z [INFO] server-57ab72ed-a7fb-a23f-f42b-6fffc31b7698: added peer, starting replication: peer=server-11238980-fd33-72bf-1741-1957df500d77 +2025-06-17T07:43:01.138Z [INFO] server-57ab72ed-a7fb-a23f-f42b-6fffc31b7698: pipelining replication: peer="{Voter server-a422f3f1-7e07-8513-dcea-7780030d3dbd a422f3f1-7e07-8513-dcea-7780030d3dbd}" +2025-06-17T07:43:01.138Z [INFO] server-57ab72ed-a7fb-a23f-f42b-6fffc31b7698: pipelining replication: peer="{Voter server-11238980-fd33-72bf-1741-1957df500d77 11238980-fd33-72bf-1741-1957df500d77}" +2025-06-17T07:43:01.244Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:01.138830788 +0000 UTC m=+9.742359502 (1 nodes), 64.474386ms from start of poll, 64.885448ms from cluster start. Timeout at 2025-06-17 07:43:01.2440929 +0000 UTC m=+9.847621589, 105.262087ms after stability +2025-06-17T07:43:01.249Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:01.354Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:01.249508755 +0000 UTC m=+9.853037457 (2 nodes), 1.388µs from start of poll, 175.563403ms from cluster start. Timeout at 2025-06-17 07:43:01.354897985 +0000 UTC m=+9.958426668, 105.389211ms after stability +2025-06-17T07:43:01.354Z [INFO] server-57ab72ed-a7fb-a23f-f42b-6fffc31b7698: updating configuration: command=RemoveServer server-id=server-a422f3f1-7e07-8513-dcea-7780030d3dbd server-addr= servers="[{Suffrage:Voter ID:server-11238980-fd33-72bf-1741-1957df500d77 Address:11238980-fd33-72bf-1741-1957df500d77} {Suffrage:Voter ID:server-57ab72ed-a7fb-a23f-f42b-6fffc31b7698 Address:57ab72ed-a7fb-a23f-f42b-6fffc31b7698}]" +2025-06-17T07:43:01.355Z [INFO] server-57ab72ed-a7fb-a23f-f42b-6fffc31b7698: removed peer, stopping replication: peer=server-a422f3f1-7e07-8513-dcea-7780030d3dbd last-index=3 +2025-06-17T07:43:01.355Z [INFO] server-57ab72ed-a7fb-a23f-f42b-6fffc31b7698: aborting pipeline replication: peer="{Voter server-a422f3f1-7e07-8513-dcea-7780030d3dbd a422f3f1-7e07-8513-dcea-7780030d3dbd}" +2025-06-17T07:43:01.441Z [WARN] server-a422f3f1-7e07-8513-dcea-7780030d3dbd: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-06-17T07:43:01.460Z [INFO] server-57ab72ed-a7fb-a23f-f42b-6fffc31b7698: aborting pipeline replication: peer="{Voter server-11238980-fd33-72bf-1741-1957df500d77 11238980-fd33-72bf-1741-1957df500d77}" +--- PASS: TestRaft_RemoveFollower (0.39s) === RUN TestRaft_RemoveLeader -2024-05-15T01:17:09.311Z [INFO] server-02e75ba1-0021-962b-2f2f-58be065255af: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-02e75ba1-0021-962b-2f2f-58be065255af Address:02e75ba1-0021-962b-2f2f-58be065255af} {Suffrage:Voter ID:server-46b88a12-d023-96f6-2a57-81fd87d846e3 Address:46b88a12-d023-96f6-2a57-81fd87d846e3} {Suffrage:Voter ID:server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05 Address:a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05}]" -2024-05-15T01:17:09.311Z [INFO] server-46b88a12-d023-96f6-2a57-81fd87d846e3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-02e75ba1-0021-962b-2f2f-58be065255af Address:02e75ba1-0021-962b-2f2f-58be065255af} {Suffrage:Voter ID:server-46b88a12-d023-96f6-2a57-81fd87d846e3 Address:46b88a12-d023-96f6-2a57-81fd87d846e3} {Suffrage:Voter ID:server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05 Address:a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05}]" -2024-05-15T01:17:09.312Z [INFO] server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-02e75ba1-0021-962b-2f2f-58be065255af Address:02e75ba1-0021-962b-2f2f-58be065255af} {Suffrage:Voter ID:server-46b88a12-d023-96f6-2a57-81fd87d846e3 Address:46b88a12-d023-96f6-2a57-81fd87d846e3} {Suffrage:Voter ID:server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05 Address:a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05}]" -2024-05-15T01:17:09.312Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:09.312Z [INFO] server-02e75ba1-0021-962b-2f2f-58be065255af: entering follower state: follower="Node at 02e75ba1-0021-962b-2f2f-58be065255af [Follower]" leader-address= leader-id= -2024-05-15T01:17:09.312Z [INFO] server-46b88a12-d023-96f6-2a57-81fd87d846e3: entering follower state: follower="Node at 46b88a12-d023-96f6-2a57-81fd87d846e3 [Follower]" leader-address= leader-id= -2024-05-15T01:17:09.312Z [INFO] server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05: entering follower state: follower="Node at a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05 [Follower]" leader-address= leader-id= -2024-05-15T01:17:09.381Z [WARN] server-02e75ba1-0021-962b-2f2f-58be065255af: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:09.381Z [INFO] server-02e75ba1-0021-962b-2f2f-58be065255af: entering candidate state: node="Node at 02e75ba1-0021-962b-2f2f-58be065255af [Candidate]" term=2 -2024-05-15T01:17:09.381Z [INFO] server-02e75ba1-0021-962b-2f2f-58be065255af: election won: term=2 tally=2 -2024-05-15T01:17:09.381Z [INFO] server-02e75ba1-0021-962b-2f2f-58be065255af: entering leader state: leader="Node at 02e75ba1-0021-962b-2f2f-58be065255af [Leader]" -2024-05-15T01:17:09.381Z [INFO] server-02e75ba1-0021-962b-2f2f-58be065255af: added peer, starting replication: peer=server-46b88a12-d023-96f6-2a57-81fd87d846e3 -2024-05-15T01:17:09.381Z [INFO] server-02e75ba1-0021-962b-2f2f-58be065255af: added peer, starting replication: peer=server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05 -2024-05-15T01:17:09.381Z [INFO] server-02e75ba1-0021-962b-2f2f-58be065255af: pipelining replication: peer="{Voter server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05 a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05}" -2024-05-15T01:17:09.381Z [INFO] server-02e75ba1-0021-962b-2f2f-58be065255af: pipelining replication: peer="{Voter server-46b88a12-d023-96f6-2a57-81fd87d846e3 46b88a12-d023-96f6-2a57-81fd87d846e3}" -2024-05-15T01:17:09.487Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:09.381760781 +0000 UTC m=+9.923145525 (1 nodes), 69.676541ms from start of poll, 70.187748ms from cluster start. Timeout at 2024-05-15 01:17:09.487170742 +0000 UTC m=+10.028555498, 105.409973ms after stability -2024-05-15T01:17:09.493Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:09.599Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:09.493252145 +0000 UTC m=+10.034636901 (2 nodes), 2.342µs from start of poll, 181.679124ms from cluster start. Timeout at 2024-05-15 01:17:09.599029021 +0000 UTC m=+10.140413771, 105.77687ms after stability -2024-05-15T01:17:09.599Z [INFO] server-02e75ba1-0021-962b-2f2f-58be065255af: updating configuration: command=RemoveServer server-id=server-02e75ba1-0021-962b-2f2f-58be065255af server-addr= servers="[{Suffrage:Voter ID:server-46b88a12-d023-96f6-2a57-81fd87d846e3 Address:46b88a12-d023-96f6-2a57-81fd87d846e3} {Suffrage:Voter ID:server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05 Address:a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05}]" -2024-05-15T01:17:09.599Z [INFO] server-02e75ba1-0021-962b-2f2f-58be065255af: removed ourself, shutting down -2024-05-15T01:17:09.599Z [INFO] server-02e75ba1-0021-962b-2f2f-58be065255af: aborting pipeline replication: peer="{Voter server-46b88a12-d023-96f6-2a57-81fd87d846e3 46b88a12-d023-96f6-2a57-81fd87d846e3}" -2024-05-15T01:17:09.599Z [INFO] server-02e75ba1-0021-962b-2f2f-58be065255af: aborting pipeline replication: peer="{Voter server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05 a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05}" -2024-05-15T01:17:09.653Z [WARN] server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05: heartbeat timeout reached, starting election: last-leader-addr=02e75ba1-0021-962b-2f2f-58be065255af last-leader-id=server-02e75ba1-0021-962b-2f2f-58be065255af -2024-05-15T01:17:09.653Z [INFO] server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05: entering candidate state: node="Node at a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05 [Candidate]" term=3 -2024-05-15T01:17:09.653Z [WARN] server-46b88a12-d023-96f6-2a57-81fd87d846e3: rejecting vote request since we have a leader: from=a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05 leader=02e75ba1-0021-962b-2f2f-58be065255af leader-id=server-02e75ba1-0021-962b-2f2f-58be065255af -2024-05-15T01:17:09.698Z [WARN] server-46b88a12-d023-96f6-2a57-81fd87d846e3: heartbeat timeout reached, starting election: last-leader-addr=02e75ba1-0021-962b-2f2f-58be065255af last-leader-id=server-02e75ba1-0021-962b-2f2f-58be065255af -2024-05-15T01:17:09.698Z [INFO] server-46b88a12-d023-96f6-2a57-81fd87d846e3: entering candidate state: node="Node at 46b88a12-d023-96f6-2a57-81fd87d846e3 [Candidate]" term=3 -2024-05-15T01:17:09.698Z [INFO] server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05: duplicate requestVote for same term: term=3 -2024-05-15T01:17:09.731Z [WARN] server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05: Election timeout reached, restarting election -2024-05-15T01:17:09.731Z [INFO] server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05: entering candidate state: node="Node at a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05 [Candidate]" term=4 -2024-05-15T01:17:09.731Z [INFO] server-46b88a12-d023-96f6-2a57-81fd87d846e3: entering follower state: follower="Node at 46b88a12-d023-96f6-2a57-81fd87d846e3 [Follower]" leader-address= leader-id= -2024-05-15T01:17:09.731Z [INFO] server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05: election won: term=4 tally=2 -2024-05-15T01:17:09.731Z [INFO] server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05: entering leader state: leader="Node at a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05 [Leader]" -2024-05-15T01:17:09.731Z [INFO] server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05: added peer, starting replication: peer=server-46b88a12-d023-96f6-2a57-81fd87d846e3 -2024-05-15T01:17:09.731Z [INFO] server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05: pipelining replication: peer="{Voter server-46b88a12-d023-96f6-2a57-81fd87d846e3 46b88a12-d023-96f6-2a57-81fd87d846e3}" -2024-05-15T01:17:09.810Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:09.916Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:09.811114036 +0000 UTC m=+10.352498808 (1 nodes), 76.609µs from start of poll, 499.541031ms from cluster start. Timeout at 2024-05-15 01:17:09.916949162 +0000 UTC m=+10.458333915, 105.835107ms after stability -2024-05-15T01:17:09.917Z [INFO] server-a43fef5b-2ba9-f0a7-d13b-8fe9b14bdb05: aborting pipeline replication: peer="{Voter server-46b88a12-d023-96f6-2a57-81fd87d846e3 46b88a12-d023-96f6-2a57-81fd87d846e3}" ---- PASS: TestRaft_RemoveLeader (0.61s) +2025-06-17T07:43:01.461Z [INFO] server-2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0 Address:2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0} {Suffrage:Voter ID:server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd Address:0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd} {Suffrage:Voter ID:server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8 Address:2483e7ce-0b0e-a2e6-9182-0c159479e7e8}]" +2025-06-17T07:43:01.461Z [INFO] server-2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0: entering follower state: follower="Node at 2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0 [Follower]" leader-address= leader-id= +2025-06-17T07:43:01.461Z [INFO] server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0 Address:2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0} {Suffrage:Voter ID:server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd Address:0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd} {Suffrage:Voter ID:server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8 Address:2483e7ce-0b0e-a2e6-9182-0c159479e7e8}]" +2025-06-17T07:43:01.461Z [INFO] server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd: entering follower state: follower="Node at 0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd [Follower]" leader-address= leader-id= +2025-06-17T07:43:01.461Z [INFO] server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0 Address:2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0} {Suffrage:Voter ID:server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd Address:0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd} {Suffrage:Voter ID:server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8 Address:2483e7ce-0b0e-a2e6-9182-0c159479e7e8}]" +2025-06-17T07:43:01.461Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:01.461Z [INFO] server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8: entering follower state: follower="Node at 2483e7ce-0b0e-a2e6-9182-0c159479e7e8 [Follower]" leader-address= leader-id= +2025-06-17T07:43:01.517Z [WARN] server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:01.517Z [INFO] server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8: entering candidate state: node="Node at 2483e7ce-0b0e-a2e6-9182-0c159479e7e8 [Candidate]" term=2 +2025-06-17T07:43:01.517Z [INFO] server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8: election won: term=2 tally=2 +2025-06-17T07:43:01.517Z [INFO] server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8: entering leader state: leader="Node at 2483e7ce-0b0e-a2e6-9182-0c159479e7e8 [Leader]" +2025-06-17T07:43:01.517Z [INFO] server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8: added peer, starting replication: peer=server-2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0 +2025-06-17T07:43:01.517Z [INFO] server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8: added peer, starting replication: peer=server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd +2025-06-17T07:43:01.517Z [INFO] server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8: pipelining replication: peer="{Voter server-2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0 2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0}" +2025-06-17T07:43:01.517Z [INFO] server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8: pipelining replication: peer="{Voter server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd 0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd}" +2025-06-17T07:43:01.622Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:01.517243935 +0000 UTC m=+10.120772651 (1 nodes), 55.581551ms from start of poll, 55.921755ms from cluster start. Timeout at 2025-06-17 07:43:01.622641516 +0000 UTC m=+10.226170201, 105.39755ms after stability +2025-06-17T07:43:01.628Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:01.733Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:01.628035437 +0000 UTC m=+10.231564124 (2 nodes), 743ns from start of poll, 166.713228ms from cluster start. Timeout at 2025-06-17 07:43:01.733669275 +0000 UTC m=+10.337197959, 105.633835ms after stability +2025-06-17T07:43:01.733Z [INFO] server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8: updating configuration: command=RemoveServer server-id=server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8 server-addr= servers="[{Suffrage:Voter ID:server-2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0 Address:2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0} {Suffrage:Voter ID:server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd Address:0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd}]" +2025-06-17T07:43:01.733Z [INFO] server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8: removed ourself, shutting down +2025-06-17T07:43:01.733Z [INFO] server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8: aborting pipeline replication: peer="{Voter server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd 0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd}" +2025-06-17T07:43:01.733Z [INFO] server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8: aborting pipeline replication: peer="{Voter server-2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0 2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0}" +2025-06-17T07:43:01.840Z [WARN] server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd: heartbeat timeout reached, starting election: last-leader-addr=2483e7ce-0b0e-a2e6-9182-0c159479e7e8 last-leader-id=server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8 +2025-06-17T07:43:01.840Z [INFO] server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd: entering candidate state: node="Node at 0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd [Candidate]" term=3 +2025-06-17T07:43:01.840Z [WARN] server-2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0: rejecting vote request since we have a leader: from=0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd leader=2483e7ce-0b0e-a2e6-9182-0c159479e7e8 leader-id=server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8 +2025-06-17T07:43:01.858Z [WARN] server-2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0: heartbeat timeout reached, starting election: last-leader-addr=2483e7ce-0b0e-a2e6-9182-0c159479e7e8 last-leader-id=server-2483e7ce-0b0e-a2e6-9182-0c159479e7e8 +2025-06-17T07:43:01.858Z [INFO] server-2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0: entering candidate state: node="Node at 2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0 [Candidate]" term=3 +2025-06-17T07:43:01.858Z [INFO] server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd: duplicate requestVote for same term: term=3 +2025-06-17T07:43:01.939Z [WARN] server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd: Election timeout reached, restarting election +2025-06-17T07:43:01.939Z [INFO] server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd: entering candidate state: node="Node at 0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd [Candidate]" term=4 +2025-06-17T07:43:01.939Z [INFO] server-2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0: entering follower state: follower="Node at 2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0 [Follower]" leader-address= leader-id= +2025-06-17T07:43:01.939Z [INFO] server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd: election won: term=4 tally=2 +2025-06-17T07:43:01.939Z [INFO] server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd: entering leader state: leader="Node at 0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd [Leader]" +2025-06-17T07:43:01.939Z [INFO] server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd: added peer, starting replication: peer=server-2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0 +2025-06-17T07:43:01.939Z [INFO] server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd: pipelining replication: peer="{Voter server-2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0 2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0}" +2025-06-17T07:43:01.944Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:02.050Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:01.944623971 +0000 UTC m=+10.548152667 (1 nodes), 51.581µs from start of poll, 483.301771ms from cluster start. Timeout at 2025-06-17 07:43:02.050414623 +0000 UTC m=+10.653943306, 105.790639ms after stability +2025-06-17T07:43:02.050Z [INFO] server-0c43e7e7-c6f9-7e06-a326-eedfb7a89cfd: aborting pipeline replication: peer="{Voter server-2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0 2e760fe3-e81b-c58a-33ab-9c6d7d1c81d0}" +--- PASS: TestRaft_RemoveLeader (0.59s) === RUN TestRaft_RemoveLeader_NoShutdown -2024-05-15T01:17:09.918Z [INFO] server-cc8d7394-e736-17d8-63cb-7f860b794e84: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cc8d7394-e736-17d8-63cb-7f860b794e84 Address:cc8d7394-e736-17d8-63cb-7f860b794e84} {Suffrage:Voter ID:server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4 Address:4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4} {Suffrage:Voter ID:server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a Address:e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a}]" -2024-05-15T01:17:09.918Z [INFO] server-cc8d7394-e736-17d8-63cb-7f860b794e84: entering follower state: follower="Node at cc8d7394-e736-17d8-63cb-7f860b794e84 [Follower]" leader-address= leader-id= -2024-05-15T01:17:09.919Z [INFO] server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cc8d7394-e736-17d8-63cb-7f860b794e84 Address:cc8d7394-e736-17d8-63cb-7f860b794e84} {Suffrage:Voter ID:server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4 Address:4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4} {Suffrage:Voter ID:server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a Address:e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a}]" -2024-05-15T01:17:09.919Z [INFO] server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4: entering follower state: follower="Node at 4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4 [Follower]" leader-address= leader-id= -2024-05-15T01:17:09.919Z [INFO] server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cc8d7394-e736-17d8-63cb-7f860b794e84 Address:cc8d7394-e736-17d8-63cb-7f860b794e84} {Suffrage:Voter ID:server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4 Address:4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4} {Suffrage:Voter ID:server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a Address:e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a}]" -2024-05-15T01:17:09.919Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:09.919Z [INFO] server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a: entering follower state: follower="Node at e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a [Follower]" leader-address= leader-id= -2024-05-15T01:17:09.970Z [WARN] server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:09.970Z [INFO] server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a: entering candidate state: node="Node at e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a [Candidate]" term=2 -2024-05-15T01:17:09.970Z [INFO] server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a: election won: term=2 tally=2 -2024-05-15T01:17:09.970Z [INFO] server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a: entering leader state: leader="Node at e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a [Leader]" -2024-05-15T01:17:09.970Z [INFO] server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a: added peer, starting replication: peer=server-cc8d7394-e736-17d8-63cb-7f860b794e84 -2024-05-15T01:17:09.970Z [INFO] server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a: added peer, starting replication: peer=server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4 -2024-05-15T01:17:09.971Z [INFO] server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a: pipelining replication: peer="{Voter server-cc8d7394-e736-17d8-63cb-7f860b794e84 cc8d7394-e736-17d8-63cb-7f860b794e84}" -2024-05-15T01:17:09.971Z [INFO] server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a: pipelining replication: peer="{Voter server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4 4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4}" -2024-05-15T01:17:10.076Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:09.971126006 +0000 UTC m=+10.512510824 (2 nodes), 51.864825ms from start of poll, 52.410631ms from cluster start. Timeout at 2024-05-15 01:17:10.07653698 +0000 UTC m=+10.617921736, 105.410912ms after stability -2024-05-15T01:17:10.076Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:10.182Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:10.076687688 +0000 UTC m=+10.618072418 (1 nodes), 2.437µs from start of poll, 157.972225ms from cluster start. Timeout at 2024-05-15 01:17:10.182425762 +0000 UTC m=+10.723810870, 105.738452ms after stability -2024-05-15T01:17:10.191Z [INFO] server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a: updating configuration: command=RemoveServer server-id=server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a server-addr= servers="[{Suffrage:Voter ID:server-cc8d7394-e736-17d8-63cb-7f860b794e84 Address:cc8d7394-e736-17d8-63cb-7f860b794e84} {Suffrage:Voter ID:server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4 Address:4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4}]" -2024-05-15T01:17:10.192Z [INFO] server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a: removed ourself, transitioning to follower -2024-05-15T01:17:10.192Z [INFO] server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a: entering follower state: follower="Node at e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a [Follower]" leader-address= leader-id= -2024-05-15T01:17:10.192Z [INFO] server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a: aborting pipeline replication: peer="{Voter server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4 4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4}" -2024-05-15T01:17:10.192Z [INFO] server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a: aborting pipeline replication: peer="{Voter server-cc8d7394-e736-17d8-63cb-7f860b794e84 cc8d7394-e736-17d8-63cb-7f860b794e84}" -2024-05-15T01:17:10.251Z [WARN] server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4: heartbeat timeout reached, starting election: last-leader-addr=e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a last-leader-id=server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a -2024-05-15T01:17:10.251Z [INFO] server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4: entering candidate state: node="Node at 4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4 [Candidate]" term=3 -2024-05-15T01:17:10.251Z [WARN] server-cc8d7394-e736-17d8-63cb-7f860b794e84: rejecting vote request since we have a leader: from=4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4 leader=e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a leader-id=server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a -2024-05-15T01:17:10.256Z [WARN] server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a: not part of stable configuration, aborting election -2024-05-15T01:17:10.298Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:10.319Z [WARN] server-cc8d7394-e736-17d8-63cb-7f860b794e84: heartbeat timeout reached, starting election: last-leader-addr=e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a last-leader-id=server-e4cba34d-7f49-32a8-e0cf-bc5e236b3a6a -2024-05-15T01:17:10.319Z [INFO] server-cc8d7394-e736-17d8-63cb-7f860b794e84: entering candidate state: node="Node at cc8d7394-e736-17d8-63cb-7f860b794e84 [Candidate]" term=3 -2024-05-15T01:17:10.319Z [INFO] server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4: duplicate requestVote for same term: term=3 -2024-05-15T01:17:10.319Z [WARN] server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4: Election timeout reached, restarting election -2024-05-15T01:17:10.319Z [INFO] server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4: entering candidate state: node="Node at 4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4 [Candidate]" term=4 -2024-05-15T01:17:10.319Z [INFO] server-cc8d7394-e736-17d8-63cb-7f860b794e84: entering follower state: follower="Node at cc8d7394-e736-17d8-63cb-7f860b794e84 [Follower]" leader-address= leader-id= -2024-05-15T01:17:10.319Z [INFO] server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4: election won: term=4 tally=2 -2024-05-15T01:17:10.319Z [INFO] server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4: entering leader state: leader="Node at 4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4 [Leader]" -2024-05-15T01:17:10.319Z [INFO] server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4: added peer, starting replication: peer=server-cc8d7394-e736-17d8-63cb-7f860b794e84 -2024-05-15T01:17:10.320Z [INFO] server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4: pipelining replication: peer="{Voter server-cc8d7394-e736-17d8-63cb-7f860b794e84 cc8d7394-e736-17d8-63cb-7f860b794e84}" -2024-05-15T01:17:10.425Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:10.319937122 +0000 UTC m=+10.861322109 (1 nodes), 21.457997ms from start of poll, 401.221916ms from cluster start. Timeout at 2024-05-15 01:17:10.425507188 +0000 UTC m=+10.966891995, 105.569886ms after stability -2024-05-15T01:17:10.530Z [INFO] server-4cc80f35-04b3-09c8-8676-cc5cbd4ce7f4: aborting pipeline replication: peer="{Voter server-cc8d7394-e736-17d8-63cb-7f860b794e84 cc8d7394-e736-17d8-63cb-7f860b794e84}" ---- PASS: TestRaft_RemoveLeader_NoShutdown (0.61s) +2025-06-17T07:43:02.052Z [INFO] server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7 Address:ee2663da-5696-c5aa-66ca-b5f5d4d867f7} {Suffrage:Voter ID:server-6baddb7a-1e46-f922-fc3f-677d2e2dac43 Address:6baddb7a-1e46-f922-fc3f-677d2e2dac43} {Suffrage:Voter ID:server-867190c3-3229-94eb-1ad0-f8f815d94ea0 Address:867190c3-3229-94eb-1ad0-f8f815d94ea0}]" +2025-06-17T07:43:02.052Z [INFO] server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7: entering follower state: follower="Node at ee2663da-5696-c5aa-66ca-b5f5d4d867f7 [Follower]" leader-address= leader-id= +2025-06-17T07:43:02.052Z [INFO] server-6baddb7a-1e46-f922-fc3f-677d2e2dac43: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7 Address:ee2663da-5696-c5aa-66ca-b5f5d4d867f7} {Suffrage:Voter ID:server-6baddb7a-1e46-f922-fc3f-677d2e2dac43 Address:6baddb7a-1e46-f922-fc3f-677d2e2dac43} {Suffrage:Voter ID:server-867190c3-3229-94eb-1ad0-f8f815d94ea0 Address:867190c3-3229-94eb-1ad0-f8f815d94ea0}]" +2025-06-17T07:43:02.052Z [INFO] server-6baddb7a-1e46-f922-fc3f-677d2e2dac43: entering follower state: follower="Node at 6baddb7a-1e46-f922-fc3f-677d2e2dac43 [Follower]" leader-address= leader-id= +2025-06-17T07:43:02.052Z [INFO] server-867190c3-3229-94eb-1ad0-f8f815d94ea0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7 Address:ee2663da-5696-c5aa-66ca-b5f5d4d867f7} {Suffrage:Voter ID:server-6baddb7a-1e46-f922-fc3f-677d2e2dac43 Address:6baddb7a-1e46-f922-fc3f-677d2e2dac43} {Suffrage:Voter ID:server-867190c3-3229-94eb-1ad0-f8f815d94ea0 Address:867190c3-3229-94eb-1ad0-f8f815d94ea0}]" +2025-06-17T07:43:02.052Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:02.052Z [INFO] server-867190c3-3229-94eb-1ad0-f8f815d94ea0: entering follower state: follower="Node at 867190c3-3229-94eb-1ad0-f8f815d94ea0 [Follower]" leader-address= leader-id= +2025-06-17T07:43:02.107Z [WARN] server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:02.107Z [INFO] server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7: entering candidate state: node="Node at ee2663da-5696-c5aa-66ca-b5f5d4d867f7 [Candidate]" term=2 +2025-06-17T07:43:02.108Z [INFO] server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7: election won: term=2 tally=2 +2025-06-17T07:43:02.108Z [INFO] server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7: entering leader state: leader="Node at ee2663da-5696-c5aa-66ca-b5f5d4d867f7 [Leader]" +2025-06-17T07:43:02.108Z [INFO] server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7: added peer, starting replication: peer=server-6baddb7a-1e46-f922-fc3f-677d2e2dac43 +2025-06-17T07:43:02.108Z [INFO] server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7: added peer, starting replication: peer=server-867190c3-3229-94eb-1ad0-f8f815d94ea0 +2025-06-17T07:43:02.108Z [INFO] server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7: pipelining replication: peer="{Voter server-867190c3-3229-94eb-1ad0-f8f815d94ea0 867190c3-3229-94eb-1ad0-f8f815d94ea0}" +2025-06-17T07:43:02.108Z [INFO] server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7: pipelining replication: peer="{Voter server-6baddb7a-1e46-f922-fc3f-677d2e2dac43 6baddb7a-1e46-f922-fc3f-677d2e2dac43}" +2025-06-17T07:43:02.213Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:02.108119626 +0000 UTC m=+10.711648328 (2 nodes), 55.551182ms from start of poll, 55.837273ms from cluster start. Timeout at 2025-06-17 07:43:02.213557492 +0000 UTC m=+10.817086175, 105.437847ms after stability +2025-06-17T07:43:02.213Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:02.319Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:02.213609689 +0000 UTC m=+10.817138376 (1 nodes), 529ns from start of poll, 161.327321ms from cluster start. Timeout at 2025-06-17 07:43:02.319232234 +0000 UTC m=+10.922760919, 105.622543ms after stability +2025-06-17T07:43:02.323Z [INFO] server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7: updating configuration: command=RemoveServer server-id=server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7 server-addr= servers="[{Suffrage:Voter ID:server-6baddb7a-1e46-f922-fc3f-677d2e2dac43 Address:6baddb7a-1e46-f922-fc3f-677d2e2dac43} {Suffrage:Voter ID:server-867190c3-3229-94eb-1ad0-f8f815d94ea0 Address:867190c3-3229-94eb-1ad0-f8f815d94ea0}]" +2025-06-17T07:43:02.323Z [INFO] server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7: removed ourself, transitioning to follower +2025-06-17T07:43:02.323Z [INFO] server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7: entering follower state: follower="Node at ee2663da-5696-c5aa-66ca-b5f5d4d867f7 [Follower]" leader-address= leader-id= +2025-06-17T07:43:02.323Z [INFO] server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7: aborting pipeline replication: peer="{Voter server-867190c3-3229-94eb-1ad0-f8f815d94ea0 867190c3-3229-94eb-1ad0-f8f815d94ea0}" +2025-06-17T07:43:02.323Z [INFO] server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7: aborting pipeline replication: peer="{Voter server-6baddb7a-1e46-f922-fc3f-677d2e2dac43 6baddb7a-1e46-f922-fc3f-677d2e2dac43}" +2025-06-17T07:43:02.374Z [WARN] server-867190c3-3229-94eb-1ad0-f8f815d94ea0: heartbeat timeout reached, starting election: last-leader-addr=ee2663da-5696-c5aa-66ca-b5f5d4d867f7 last-leader-id=server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7 +2025-06-17T07:43:02.374Z [INFO] server-867190c3-3229-94eb-1ad0-f8f815d94ea0: entering candidate state: node="Node at 867190c3-3229-94eb-1ad0-f8f815d94ea0 [Candidate]" term=3 +2025-06-17T07:43:02.374Z [WARN] server-6baddb7a-1e46-f922-fc3f-677d2e2dac43: rejecting vote request since we have a leader: from=867190c3-3229-94eb-1ad0-f8f815d94ea0 leader=ee2663da-5696-c5aa-66ca-b5f5d4d867f7 leader-id=server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7 +2025-06-17T07:43:02.389Z [WARN] server-6baddb7a-1e46-f922-fc3f-677d2e2dac43: heartbeat timeout reached, starting election: last-leader-addr=ee2663da-5696-c5aa-66ca-b5f5d4d867f7 last-leader-id=server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7 +2025-06-17T07:43:02.389Z [INFO] server-6baddb7a-1e46-f922-fc3f-677d2e2dac43: entering candidate state: node="Node at 6baddb7a-1e46-f922-fc3f-677d2e2dac43 [Candidate]" term=3 +2025-06-17T07:43:02.390Z [INFO] server-867190c3-3229-94eb-1ad0-f8f815d94ea0: duplicate requestVote for same term: term=3 +2025-06-17T07:43:02.419Z [WARN] server-ee2663da-5696-c5aa-66ca-b5f5d4d867f7: not part of stable configuration, aborting election +2025-06-17T07:43:02.429Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:02.436Z [WARN] server-867190c3-3229-94eb-1ad0-f8f815d94ea0: Election timeout reached, restarting election +2025-06-17T07:43:02.436Z [INFO] server-867190c3-3229-94eb-1ad0-f8f815d94ea0: entering candidate state: node="Node at 867190c3-3229-94eb-1ad0-f8f815d94ea0 [Candidate]" term=4 +2025-06-17T07:43:02.436Z [INFO] server-6baddb7a-1e46-f922-fc3f-677d2e2dac43: entering follower state: follower="Node at 6baddb7a-1e46-f922-fc3f-677d2e2dac43 [Follower]" leader-address= leader-id= +2025-06-17T07:43:02.436Z [INFO] server-867190c3-3229-94eb-1ad0-f8f815d94ea0: election won: term=4 tally=2 +2025-06-17T07:43:02.436Z [INFO] server-867190c3-3229-94eb-1ad0-f8f815d94ea0: entering leader state: leader="Node at 867190c3-3229-94eb-1ad0-f8f815d94ea0 [Leader]" +2025-06-17T07:43:02.436Z [INFO] server-867190c3-3229-94eb-1ad0-f8f815d94ea0: added peer, starting replication: peer=server-6baddb7a-1e46-f922-fc3f-677d2e2dac43 +2025-06-17T07:43:02.436Z [INFO] server-867190c3-3229-94eb-1ad0-f8f815d94ea0: pipelining replication: peer="{Voter server-6baddb7a-1e46-f922-fc3f-677d2e2dac43 6baddb7a-1e46-f922-fc3f-677d2e2dac43}" +2025-06-17T07:43:02.542Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:02.436630636 +0000 UTC m=+11.040159323 (1 nodes), 7.269301ms from start of poll, 384.348268ms from cluster start. Timeout at 2025-06-17 07:43:02.542548703 +0000 UTC m=+11.146077418, 105.918095ms after stability +2025-06-17T07:43:02.648Z [INFO] server-867190c3-3229-94eb-1ad0-f8f815d94ea0: aborting pipeline replication: peer="{Voter server-6baddb7a-1e46-f922-fc3f-677d2e2dac43 6baddb7a-1e46-f922-fc3f-677d2e2dac43}" +--- PASS: TestRaft_RemoveLeader_NoShutdown (0.60s) === RUN TestRaft_RemoveFollower_SplitCluster -2024-05-15T01:17:10.532Z [INFO] server-fe3a707c-c0ff-1f4a-926f-eec3d306b5a7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fe3a707c-c0ff-1f4a-926f-eec3d306b5a7 Address:fe3a707c-c0ff-1f4a-926f-eec3d306b5a7} {Suffrage:Voter ID:server-eb16721b-cf09-cdfe-2fc8-630c958e20f6 Address:eb16721b-cf09-cdfe-2fc8-630c958e20f6} {Suffrage:Voter ID:server-20adf00e-88b0-647e-d2ac-f1936036abc2 Address:20adf00e-88b0-647e-d2ac-f1936036abc2} {Suffrage:Voter ID:server-fa7b6e8c-b40d-feaa-6611-e767e27d8123 Address:fa7b6e8c-b40d-feaa-6611-e767e27d8123}]" -2024-05-15T01:17:10.533Z [INFO] server-eb16721b-cf09-cdfe-2fc8-630c958e20f6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fe3a707c-c0ff-1f4a-926f-eec3d306b5a7 Address:fe3a707c-c0ff-1f4a-926f-eec3d306b5a7} {Suffrage:Voter ID:server-eb16721b-cf09-cdfe-2fc8-630c958e20f6 Address:eb16721b-cf09-cdfe-2fc8-630c958e20f6} {Suffrage:Voter ID:server-20adf00e-88b0-647e-d2ac-f1936036abc2 Address:20adf00e-88b0-647e-d2ac-f1936036abc2} {Suffrage:Voter ID:server-fa7b6e8c-b40d-feaa-6611-e767e27d8123 Address:fa7b6e8c-b40d-feaa-6611-e767e27d8123}]" -2024-05-15T01:17:10.533Z [INFO] server-fe3a707c-c0ff-1f4a-926f-eec3d306b5a7: entering follower state: follower="Node at fe3a707c-c0ff-1f4a-926f-eec3d306b5a7 [Follower]" leader-address= leader-id= -2024-05-15T01:17:10.533Z [INFO] server-eb16721b-cf09-cdfe-2fc8-630c958e20f6: entering follower state: follower="Node at eb16721b-cf09-cdfe-2fc8-630c958e20f6 [Follower]" leader-address= leader-id= -2024-05-15T01:17:10.533Z [INFO] server-20adf00e-88b0-647e-d2ac-f1936036abc2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fe3a707c-c0ff-1f4a-926f-eec3d306b5a7 Address:fe3a707c-c0ff-1f4a-926f-eec3d306b5a7} {Suffrage:Voter ID:server-eb16721b-cf09-cdfe-2fc8-630c958e20f6 Address:eb16721b-cf09-cdfe-2fc8-630c958e20f6} {Suffrage:Voter ID:server-20adf00e-88b0-647e-d2ac-f1936036abc2 Address:20adf00e-88b0-647e-d2ac-f1936036abc2} {Suffrage:Voter ID:server-fa7b6e8c-b40d-feaa-6611-e767e27d8123 Address:fa7b6e8c-b40d-feaa-6611-e767e27d8123}]" -2024-05-15T01:17:10.533Z [INFO] server-20adf00e-88b0-647e-d2ac-f1936036abc2: entering follower state: follower="Node at 20adf00e-88b0-647e-d2ac-f1936036abc2 [Follower]" leader-address= leader-id= -2024-05-15T01:17:10.533Z [INFO] server-fa7b6e8c-b40d-feaa-6611-e767e27d8123: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fe3a707c-c0ff-1f4a-926f-eec3d306b5a7 Address:fe3a707c-c0ff-1f4a-926f-eec3d306b5a7} {Suffrage:Voter ID:server-eb16721b-cf09-cdfe-2fc8-630c958e20f6 Address:eb16721b-cf09-cdfe-2fc8-630c958e20f6} {Suffrage:Voter ID:server-20adf00e-88b0-647e-d2ac-f1936036abc2 Address:20adf00e-88b0-647e-d2ac-f1936036abc2} {Suffrage:Voter ID:server-fa7b6e8c-b40d-feaa-6611-e767e27d8123 Address:fa7b6e8c-b40d-feaa-6611-e767e27d8123}]" -2024-05-15T01:17:10.533Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:10.533Z [INFO] server-fa7b6e8c-b40d-feaa-6611-e767e27d8123: entering follower state: follower="Node at fa7b6e8c-b40d-feaa-6611-e767e27d8123 [Follower]" leader-address= leader-id= -2024-05-15T01:17:10.586Z [WARN] server-20adf00e-88b0-647e-d2ac-f1936036abc2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:10.586Z [INFO] server-20adf00e-88b0-647e-d2ac-f1936036abc2: entering candidate state: node="Node at 20adf00e-88b0-647e-d2ac-f1936036abc2 [Candidate]" term=2 -2024-05-15T01:17:10.587Z [INFO] server-20adf00e-88b0-647e-d2ac-f1936036abc2: election won: term=2 tally=3 -2024-05-15T01:17:10.587Z [INFO] server-20adf00e-88b0-647e-d2ac-f1936036abc2: entering leader state: leader="Node at 20adf00e-88b0-647e-d2ac-f1936036abc2 [Leader]" -2024-05-15T01:17:10.587Z [INFO] server-20adf00e-88b0-647e-d2ac-f1936036abc2: added peer, starting replication: peer=server-fe3a707c-c0ff-1f4a-926f-eec3d306b5a7 -2024-05-15T01:17:10.587Z [INFO] server-20adf00e-88b0-647e-d2ac-f1936036abc2: added peer, starting replication: peer=server-eb16721b-cf09-cdfe-2fc8-630c958e20f6 -2024-05-15T01:17:10.587Z [INFO] server-20adf00e-88b0-647e-d2ac-f1936036abc2: added peer, starting replication: peer=server-fa7b6e8c-b40d-feaa-6611-e767e27d8123 -2024-05-15T01:17:10.587Z [INFO] server-20adf00e-88b0-647e-d2ac-f1936036abc2: pipelining replication: peer="{Voter server-fa7b6e8c-b40d-feaa-6611-e767e27d8123 fa7b6e8c-b40d-feaa-6611-e767e27d8123}" -2024-05-15T01:17:10.587Z [INFO] server-20adf00e-88b0-647e-d2ac-f1936036abc2: pipelining replication: peer="{Voter server-fe3a707c-c0ff-1f4a-926f-eec3d306b5a7 fe3a707c-c0ff-1f4a-926f-eec3d306b5a7}" -2024-05-15T01:17:10.587Z [INFO] server-20adf00e-88b0-647e-d2ac-f1936036abc2: pipelining replication: peer="{Voter server-eb16721b-cf09-cdfe-2fc8-630c958e20f6 eb16721b-cf09-cdfe-2fc8-630c958e20f6}" -2024-05-15T01:17:10.693Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:10.587373621 +0000 UTC m=+11.128758373 (1 nodes), 53.441903ms from start of poll, 54.684599ms from cluster start. Timeout at 2024-05-15 01:17:10.693220203 +0000 UTC m=+11.234606943, 105.84857ms after stability -2024-05-15T01:17:10.799Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:10.905Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:10.799488224 +0000 UTC m=+11.340872948 (3 nodes), 2.116µs from start of poll, 266.799174ms from cluster start. Timeout at 2024-05-15 01:17:10.905741488 +0000 UTC m=+11.447126501, 106.253553ms after stability -2024-05-15T01:17:10.905Z [INFO] server-20adf00e-88b0-647e-d2ac-f1936036abc2: updating configuration: command=RemoveServer server-id=server-fa7b6e8c-b40d-feaa-6611-e767e27d8123 server-addr= servers="[{Suffrage:Voter ID:server-fe3a707c-c0ff-1f4a-926f-eec3d306b5a7 Address:fe3a707c-c0ff-1f4a-926f-eec3d306b5a7} {Suffrage:Voter ID:server-eb16721b-cf09-cdfe-2fc8-630c958e20f6 Address:eb16721b-cf09-cdfe-2fc8-630c958e20f6} {Suffrage:Voter ID:server-20adf00e-88b0-647e-d2ac-f1936036abc2 Address:20adf00e-88b0-647e-d2ac-f1936036abc2}]" -2024-05-15T01:17:10.905Z [INFO] server-20adf00e-88b0-647e-d2ac-f1936036abc2: removed peer, stopping replication: peer=server-fa7b6e8c-b40d-feaa-6611-e767e27d8123 last-index=3 -2024-05-15T01:17:10.906Z [INFO] server-20adf00e-88b0-647e-d2ac-f1936036abc2: aborting pipeline replication: peer="{Voter server-fa7b6e8c-b40d-feaa-6611-e767e27d8123 fa7b6e8c-b40d-feaa-6611-e767e27d8123}" -2024-05-15T01:17:10.906Z [ERROR] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to pipeline appendEntries: peer="{Voter server-fe3a707c-c0ff-1f4a-926f-eec3d306b5a7 fe3a707c-c0ff-1f4a-926f-eec3d306b5a7}" error="append pipeline closed" -2024-05-15T01:17:10.906Z [INFO] server-20adf00e-88b0-647e-d2ac-f1936036abc2: aborting pipeline replication: peer="{Voter server-fe3a707c-c0ff-1f4a-926f-eec3d306b5a7 fe3a707c-c0ff-1f4a-926f-eec3d306b5a7}" -2024-05-15T01:17:10.906Z [ERROR] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to pipeline appendEntries: peer="{Voter server-eb16721b-cf09-cdfe-2fc8-630c958e20f6 eb16721b-cf09-cdfe-2fc8-630c958e20f6}" error="append pipeline closed" -2024-05-15T01:17:10.906Z [INFO] server-20adf00e-88b0-647e-d2ac-f1936036abc2: aborting pipeline replication: peer="{Voter server-eb16721b-cf09-cdfe-2fc8-630c958e20f6 eb16721b-cf09-cdfe-2fc8-630c958e20f6}" -2024-05-15T01:17:10.909Z [ERROR] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to heartbeat to: peer=fe3a707c-c0ff-1f4a-926f-eec3d306b5a7 backoff time=10ms error="failed to connect to peer: fe3a707c-c0ff-1f4a-926f-eec3d306b5a7" -2024-05-15T01:17:10.912Z [ERROR] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to appendEntries to: peer="{Voter server-eb16721b-cf09-cdfe-2fc8-630c958e20f6 eb16721b-cf09-cdfe-2fc8-630c958e20f6}" error="failed to connect to peer: eb16721b-cf09-cdfe-2fc8-630c958e20f6" -2024-05-15T01:17:10.915Z [ERROR] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to heartbeat to: peer=eb16721b-cf09-cdfe-2fc8-630c958e20f6 backoff time=10ms error="failed to connect to peer: eb16721b-cf09-cdfe-2fc8-630c958e20f6" -2024-05-15T01:17:10.916Z [ERROR] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to appendEntries to: peer="{Voter server-fe3a707c-c0ff-1f4a-926f-eec3d306b5a7 fe3a707c-c0ff-1f4a-926f-eec3d306b5a7}" error="failed to connect to peer: fe3a707c-c0ff-1f4a-926f-eec3d306b5a7" -2024-05-15T01:17:10.928Z [ERROR] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to heartbeat to: peer=fe3a707c-c0ff-1f4a-926f-eec3d306b5a7 backoff time=10ms error="failed to connect to peer: fe3a707c-c0ff-1f4a-926f-eec3d306b5a7" -2024-05-15T01:17:10.928Z [ERROR] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to appendEntries to: peer="{Voter server-eb16721b-cf09-cdfe-2fc8-630c958e20f6 eb16721b-cf09-cdfe-2fc8-630c958e20f6}" error="failed to connect to peer: eb16721b-cf09-cdfe-2fc8-630c958e20f6" -2024-05-15T01:17:10.931Z [ERROR] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to heartbeat to: peer=eb16721b-cf09-cdfe-2fc8-630c958e20f6 backoff time=10ms error="failed to connect to peer: eb16721b-cf09-cdfe-2fc8-630c958e20f6" -2024-05-15T01:17:10.934Z [ERROR] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to appendEntries to: peer="{Voter server-fe3a707c-c0ff-1f4a-926f-eec3d306b5a7 fe3a707c-c0ff-1f4a-926f-eec3d306b5a7}" error="failed to connect to peer: fe3a707c-c0ff-1f4a-926f-eec3d306b5a7" -2024-05-15T01:17:10.946Z [ERROR] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to heartbeat to: peer=fe3a707c-c0ff-1f4a-926f-eec3d306b5a7 backoff time=10ms error="failed to connect to peer: fe3a707c-c0ff-1f4a-926f-eec3d306b5a7" -2024-05-15T01:17:10.946Z [ERROR] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to appendEntries to: peer="{Voter server-eb16721b-cf09-cdfe-2fc8-630c958e20f6 eb16721b-cf09-cdfe-2fc8-630c958e20f6}" error="failed to connect to peer: eb16721b-cf09-cdfe-2fc8-630c958e20f6" -2024-05-15T01:17:10.950Z [ERROR] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to heartbeat to: peer=eb16721b-cf09-cdfe-2fc8-630c958e20f6 backoff time=10ms error="failed to connect to peer: eb16721b-cf09-cdfe-2fc8-630c958e20f6" -2024-05-15T01:17:10.951Z [WARN] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to contact: server-id=server-fe3a707c-c0ff-1f4a-926f-eec3d306b5a7 time=50.924009ms -2024-05-15T01:17:10.954Z [ERROR] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to appendEntries to: peer="{Voter server-fe3a707c-c0ff-1f4a-926f-eec3d306b5a7 fe3a707c-c0ff-1f4a-926f-eec3d306b5a7}" error="failed to connect to peer: fe3a707c-c0ff-1f4a-926f-eec3d306b5a7" -2024-05-15T01:17:10.962Z [WARN] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to contact: server-id=server-fe3a707c-c0ff-1f4a-926f-eec3d306b5a7 time=62.014259ms -2024-05-15T01:17:10.962Z [WARN] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to contact: server-id=server-eb16721b-cf09-cdfe-2fc8-630c958e20f6 time=56.380999ms -2024-05-15T01:17:10.962Z [WARN] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to contact quorum of nodes, stepping down -2024-05-15T01:17:10.962Z [INFO] server-20adf00e-88b0-647e-d2ac-f1936036abc2: entering follower state: follower="Node at 20adf00e-88b0-647e-d2ac-f1936036abc2 [Follower]" leader-address= leader-id= -2024-05-15T01:17:10.962Z [ERROR] server-20adf00e-88b0-647e-d2ac-f1936036abc2: failed to appendEntries to: peer="{Voter server-eb16721b-cf09-cdfe-2fc8-630c958e20f6 eb16721b-cf09-cdfe-2fc8-630c958e20f6}" error="failed to connect to peer: eb16721b-cf09-cdfe-2fc8-630c958e20f6" +2025-06-17T07:43:02.649Z [INFO] server-f2b98340-74d7-6816-7948-182aa61f2b8f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f2b98340-74d7-6816-7948-182aa61f2b8f Address:f2b98340-74d7-6816-7948-182aa61f2b8f} {Suffrage:Voter ID:server-5a73a730-5fd6-8650-e980-41e63fdc09b3 Address:5a73a730-5fd6-8650-e980-41e63fdc09b3} {Suffrage:Voter ID:server-b23c0a8a-2a38-a77c-7529-5d42fe17048b Address:b23c0a8a-2a38-a77c-7529-5d42fe17048b} {Suffrage:Voter ID:server-4dd2928b-79a0-777e-fb03-a2be7f49e96a Address:4dd2928b-79a0-777e-fb03-a2be7f49e96a}]" +2025-06-17T07:43:02.649Z [INFO] server-f2b98340-74d7-6816-7948-182aa61f2b8f: entering follower state: follower="Node at f2b98340-74d7-6816-7948-182aa61f2b8f [Follower]" leader-address= leader-id= +2025-06-17T07:43:02.649Z [INFO] server-5a73a730-5fd6-8650-e980-41e63fdc09b3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f2b98340-74d7-6816-7948-182aa61f2b8f Address:f2b98340-74d7-6816-7948-182aa61f2b8f} {Suffrage:Voter ID:server-5a73a730-5fd6-8650-e980-41e63fdc09b3 Address:5a73a730-5fd6-8650-e980-41e63fdc09b3} {Suffrage:Voter ID:server-b23c0a8a-2a38-a77c-7529-5d42fe17048b Address:b23c0a8a-2a38-a77c-7529-5d42fe17048b} {Suffrage:Voter ID:server-4dd2928b-79a0-777e-fb03-a2be7f49e96a Address:4dd2928b-79a0-777e-fb03-a2be7f49e96a}]" +2025-06-17T07:43:02.649Z [INFO] server-5a73a730-5fd6-8650-e980-41e63fdc09b3: entering follower state: follower="Node at 5a73a730-5fd6-8650-e980-41e63fdc09b3 [Follower]" leader-address= leader-id= +2025-06-17T07:43:02.649Z [INFO] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f2b98340-74d7-6816-7948-182aa61f2b8f Address:f2b98340-74d7-6816-7948-182aa61f2b8f} {Suffrage:Voter ID:server-5a73a730-5fd6-8650-e980-41e63fdc09b3 Address:5a73a730-5fd6-8650-e980-41e63fdc09b3} {Suffrage:Voter ID:server-b23c0a8a-2a38-a77c-7529-5d42fe17048b Address:b23c0a8a-2a38-a77c-7529-5d42fe17048b} {Suffrage:Voter ID:server-4dd2928b-79a0-777e-fb03-a2be7f49e96a Address:4dd2928b-79a0-777e-fb03-a2be7f49e96a}]" +2025-06-17T07:43:02.649Z [INFO] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: entering follower state: follower="Node at b23c0a8a-2a38-a77c-7529-5d42fe17048b [Follower]" leader-address= leader-id= +2025-06-17T07:43:02.649Z [INFO] server-4dd2928b-79a0-777e-fb03-a2be7f49e96a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f2b98340-74d7-6816-7948-182aa61f2b8f Address:f2b98340-74d7-6816-7948-182aa61f2b8f} {Suffrage:Voter ID:server-5a73a730-5fd6-8650-e980-41e63fdc09b3 Address:5a73a730-5fd6-8650-e980-41e63fdc09b3} {Suffrage:Voter ID:server-b23c0a8a-2a38-a77c-7529-5d42fe17048b Address:b23c0a8a-2a38-a77c-7529-5d42fe17048b} {Suffrage:Voter ID:server-4dd2928b-79a0-777e-fb03-a2be7f49e96a Address:4dd2928b-79a0-777e-fb03-a2be7f49e96a}]" +2025-06-17T07:43:02.649Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:02.649Z [INFO] server-4dd2928b-79a0-777e-fb03-a2be7f49e96a: entering follower state: follower="Node at 4dd2928b-79a0-777e-fb03-a2be7f49e96a [Follower]" leader-address= leader-id= +2025-06-17T07:43:02.707Z [WARN] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:02.708Z [INFO] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: entering candidate state: node="Node at b23c0a8a-2a38-a77c-7529-5d42fe17048b [Candidate]" term=2 +2025-06-17T07:43:02.708Z [INFO] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: election won: term=2 tally=3 +2025-06-17T07:43:02.708Z [INFO] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: entering leader state: leader="Node at b23c0a8a-2a38-a77c-7529-5d42fe17048b [Leader]" +2025-06-17T07:43:02.708Z [INFO] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: added peer, starting replication: peer=server-f2b98340-74d7-6816-7948-182aa61f2b8f +2025-06-17T07:43:02.708Z [INFO] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: added peer, starting replication: peer=server-5a73a730-5fd6-8650-e980-41e63fdc09b3 +2025-06-17T07:43:02.708Z [INFO] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: added peer, starting replication: peer=server-4dd2928b-79a0-777e-fb03-a2be7f49e96a +2025-06-17T07:43:02.708Z [INFO] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: pipelining replication: peer="{Voter server-4dd2928b-79a0-777e-fb03-a2be7f49e96a 4dd2928b-79a0-777e-fb03-a2be7f49e96a}" +2025-06-17T07:43:02.708Z [INFO] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: pipelining replication: peer="{Voter server-5a73a730-5fd6-8650-e980-41e63fdc09b3 5a73a730-5fd6-8650-e980-41e63fdc09b3}" +2025-06-17T07:43:02.708Z [INFO] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: pipelining replication: peer="{Voter server-f2b98340-74d7-6816-7948-182aa61f2b8f f2b98340-74d7-6816-7948-182aa61f2b8f}" +2025-06-17T07:43:02.813Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:02.708152489 +0000 UTC m=+11.311681191 (1 nodes), 58.665694ms from start of poll, 59.063386ms from cluster start. Timeout at 2025-06-17 07:43:02.813543649 +0000 UTC m=+11.417072331, 105.39114ms after stability +2025-06-17T07:43:02.918Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:03.024Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:02.91888964 +0000 UTC m=+11.522418331 (3 nodes), 1.115µs from start of poll, 269.800526ms from cluster start. Timeout at 2025-06-17 07:43:03.024000732 +0000 UTC m=+11.627529414, 105.111083ms after stability +2025-06-17T07:43:03.024Z [INFO] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: updating configuration: command=RemoveServer server-id=server-4dd2928b-79a0-777e-fb03-a2be7f49e96a server-addr= servers="[{Suffrage:Voter ID:server-f2b98340-74d7-6816-7948-182aa61f2b8f Address:f2b98340-74d7-6816-7948-182aa61f2b8f} {Suffrage:Voter ID:server-5a73a730-5fd6-8650-e980-41e63fdc09b3 Address:5a73a730-5fd6-8650-e980-41e63fdc09b3} {Suffrage:Voter ID:server-b23c0a8a-2a38-a77c-7529-5d42fe17048b Address:b23c0a8a-2a38-a77c-7529-5d42fe17048b}]" +2025-06-17T07:43:03.024Z [INFO] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: removed peer, stopping replication: peer=server-4dd2928b-79a0-777e-fb03-a2be7f49e96a last-index=3 +2025-06-17T07:43:03.024Z [ERROR] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to pipeline appendEntries: peer="{Voter server-f2b98340-74d7-6816-7948-182aa61f2b8f f2b98340-74d7-6816-7948-182aa61f2b8f}" error="append pipeline closed" +2025-06-17T07:43:03.024Z [ERROR] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to pipeline appendEntries: peer="{Voter server-5a73a730-5fd6-8650-e980-41e63fdc09b3 5a73a730-5fd6-8650-e980-41e63fdc09b3}" error="append pipeline closed" +2025-06-17T07:43:03.024Z [INFO] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: aborting pipeline replication: peer="{Voter server-5a73a730-5fd6-8650-e980-41e63fdc09b3 5a73a730-5fd6-8650-e980-41e63fdc09b3}" +2025-06-17T07:43:03.024Z [INFO] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: aborting pipeline replication: peer="{Voter server-f2b98340-74d7-6816-7948-182aa61f2b8f f2b98340-74d7-6816-7948-182aa61f2b8f}" +2025-06-17T07:43:03.024Z [INFO] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: aborting pipeline replication: peer="{Voter server-4dd2928b-79a0-777e-fb03-a2be7f49e96a 4dd2928b-79a0-777e-fb03-a2be7f49e96a}" +2025-06-17T07:43:03.026Z [ERROR] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to heartbeat to: peer=f2b98340-74d7-6816-7948-182aa61f2b8f backoff time=10ms error="failed to connect to peer: f2b98340-74d7-6816-7948-182aa61f2b8f" +2025-06-17T07:43:03.027Z [ERROR] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to heartbeat to: peer=5a73a730-5fd6-8650-e980-41e63fdc09b3 backoff time=10ms error="failed to connect to peer: 5a73a730-5fd6-8650-e980-41e63fdc09b3" +2025-06-17T07:43:03.029Z [ERROR] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to appendEntries to: peer="{Voter server-5a73a730-5fd6-8650-e980-41e63fdc09b3 5a73a730-5fd6-8650-e980-41e63fdc09b3}" error="failed to connect to peer: 5a73a730-5fd6-8650-e980-41e63fdc09b3" +2025-06-17T07:43:03.030Z [ERROR] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to appendEntries to: peer="{Voter server-f2b98340-74d7-6816-7948-182aa61f2b8f f2b98340-74d7-6816-7948-182aa61f2b8f}" error="failed to connect to peer: f2b98340-74d7-6816-7948-182aa61f2b8f" +2025-06-17T07:43:03.046Z [ERROR] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to heartbeat to: peer=f2b98340-74d7-6816-7948-182aa61f2b8f backoff time=10ms error="failed to connect to peer: f2b98340-74d7-6816-7948-182aa61f2b8f" +2025-06-17T07:43:03.046Z [ERROR] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to appendEntries to: peer="{Voter server-5a73a730-5fd6-8650-e980-41e63fdc09b3 5a73a730-5fd6-8650-e980-41e63fdc09b3}" error="failed to connect to peer: 5a73a730-5fd6-8650-e980-41e63fdc09b3" +2025-06-17T07:43:03.047Z [ERROR] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to heartbeat to: peer=5a73a730-5fd6-8650-e980-41e63fdc09b3 backoff time=10ms error="failed to connect to peer: 5a73a730-5fd6-8650-e980-41e63fdc09b3" +2025-06-17T07:43:03.050Z [ERROR] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to appendEntries to: peer="{Voter server-f2b98340-74d7-6816-7948-182aa61f2b8f f2b98340-74d7-6816-7948-182aa61f2b8f}" error="failed to connect to peer: f2b98340-74d7-6816-7948-182aa61f2b8f" +2025-06-17T07:43:03.062Z [ERROR] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to heartbeat to: peer=f2b98340-74d7-6816-7948-182aa61f2b8f backoff time=10ms error="failed to connect to peer: f2b98340-74d7-6816-7948-182aa61f2b8f" +2025-06-17T07:43:03.063Z [ERROR] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to appendEntries to: peer="{Voter server-5a73a730-5fd6-8650-e980-41e63fdc09b3 5a73a730-5fd6-8650-e980-41e63fdc09b3}" error="failed to connect to peer: 5a73a730-5fd6-8650-e980-41e63fdc09b3" +2025-06-17T07:43:03.065Z [ERROR] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to heartbeat to: peer=5a73a730-5fd6-8650-e980-41e63fdc09b3 backoff time=10ms error="failed to connect to peer: 5a73a730-5fd6-8650-e980-41e63fdc09b3" +2025-06-17T07:43:03.067Z [ERROR] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to appendEntries to: peer="{Voter server-f2b98340-74d7-6816-7948-182aa61f2b8f f2b98340-74d7-6816-7948-182aa61f2b8f}" error="failed to connect to peer: f2b98340-74d7-6816-7948-182aa61f2b8f" +2025-06-17T07:43:03.070Z [WARN] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to contact: server-id=server-f2b98340-74d7-6816-7948-182aa61f2b8f time=50.359347ms +2025-06-17T07:43:03.077Z [WARN] server-f2b98340-74d7-6816-7948-182aa61f2b8f: heartbeat timeout reached, starting election: last-leader-addr=b23c0a8a-2a38-a77c-7529-5d42fe17048b last-leader-id=server-b23c0a8a-2a38-a77c-7529-5d42fe17048b +2025-06-17T07:43:03.077Z [INFO] server-f2b98340-74d7-6816-7948-182aa61f2b8f: entering candidate state: node="Node at f2b98340-74d7-6816-7948-182aa61f2b8f [Candidate]" term=3 +2025-06-17T07:43:03.077Z [ERROR] server-f2b98340-74d7-6816-7948-182aa61f2b8f: failed to make requestVote RPC: target="{Voter server-4dd2928b-79a0-777e-fb03-a2be7f49e96a 4dd2928b-79a0-777e-fb03-a2be7f49e96a}" error="failed to connect to peer: 4dd2928b-79a0-777e-fb03-a2be7f49e96a" term=3 +2025-06-17T07:43:03.077Z [WARN] server-5a73a730-5fd6-8650-e980-41e63fdc09b3: rejecting vote request since we have a leader: from=f2b98340-74d7-6816-7948-182aa61f2b8f leader=b23c0a8a-2a38-a77c-7529-5d42fe17048b leader-id=server-b23c0a8a-2a38-a77c-7529-5d42fe17048b +2025-06-17T07:43:03.077Z [ERROR] server-f2b98340-74d7-6816-7948-182aa61f2b8f: failed to make requestVote RPC: target="{Voter server-b23c0a8a-2a38-a77c-7529-5d42fe17048b b23c0a8a-2a38-a77c-7529-5d42fe17048b}" error="failed to connect to peer: b23c0a8a-2a38-a77c-7529-5d42fe17048b" term=3 +2025-06-17T07:43:03.080Z [WARN] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to contact: server-id=server-f2b98340-74d7-6816-7948-182aa61f2b8f time=61.035298ms +2025-06-17T07:43:03.080Z [WARN] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to contact: server-id=server-5a73a730-5fd6-8650-e980-41e63fdc09b3 time=60.646975ms +2025-06-17T07:43:03.080Z [WARN] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to contact quorum of nodes, stepping down +2025-06-17T07:43:03.080Z [INFO] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: entering follower state: follower="Node at b23c0a8a-2a38-a77c-7529-5d42fe17048b [Follower]" leader-address= leader-id= +2025-06-17T07:43:03.080Z [ERROR] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to appendEntries to: peer="{Voter server-f2b98340-74d7-6816-7948-182aa61f2b8f f2b98340-74d7-6816-7948-182aa61f2b8f}" error="failed to connect to peer: f2b98340-74d7-6816-7948-182aa61f2b8f" +2025-06-17T07:43:03.080Z [ERROR] server-b23c0a8a-2a38-a77c-7529-5d42fe17048b: failed to appendEntries to: peer="{Voter server-5a73a730-5fd6-8650-e980-41e63fdc09b3 5a73a730-5fd6-8650-e980-41e63fdc09b3}" error="failed to connect to peer: 5a73a730-5fd6-8650-e980-41e63fdc09b3" --- PASS: TestRaft_RemoveFollower_SplitCluster (0.43s) === RUN TestRaft_AddKnownPeer -2024-05-15T01:17:10.964Z [INFO] server-fb4cb339-270a-c8e4-7f05-d668347f671a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fb4cb339-270a-c8e4-7f05-d668347f671a Address:fb4cb339-270a-c8e4-7f05-d668347f671a} {Suffrage:Voter ID:server-b3707e86-cd7c-653b-f769-05a87653af44 Address:b3707e86-cd7c-653b-f769-05a87653af44} {Suffrage:Voter ID:server-ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb Address:ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb}]" -2024-05-15T01:17:10.964Z [INFO] server-fb4cb339-270a-c8e4-7f05-d668347f671a: entering follower state: follower="Node at fb4cb339-270a-c8e4-7f05-d668347f671a [Follower]" leader-address= leader-id= -2024-05-15T01:17:10.964Z [INFO] server-b3707e86-cd7c-653b-f769-05a87653af44: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fb4cb339-270a-c8e4-7f05-d668347f671a Address:fb4cb339-270a-c8e4-7f05-d668347f671a} {Suffrage:Voter ID:server-b3707e86-cd7c-653b-f769-05a87653af44 Address:b3707e86-cd7c-653b-f769-05a87653af44} {Suffrage:Voter ID:server-ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb Address:ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb}]" -2024-05-15T01:17:10.964Z [INFO] server-b3707e86-cd7c-653b-f769-05a87653af44: entering follower state: follower="Node at b3707e86-cd7c-653b-f769-05a87653af44 [Follower]" leader-address= leader-id= -2024-05-15T01:17:10.964Z [INFO] server-ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fb4cb339-270a-c8e4-7f05-d668347f671a Address:fb4cb339-270a-c8e4-7f05-d668347f671a} {Suffrage:Voter ID:server-b3707e86-cd7c-653b-f769-05a87653af44 Address:b3707e86-cd7c-653b-f769-05a87653af44} {Suffrage:Voter ID:server-ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb Address:ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb}]" -2024-05-15T01:17:10.964Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:10.964Z [INFO] server-ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb: entering follower state: follower="Node at ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb [Follower]" leader-address= leader-id= -2024-05-15T01:17:11.036Z [WARN] server-ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:11.036Z [INFO] server-ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb: entering candidate state: node="Node at ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb [Candidate]" term=2 -2024-05-15T01:17:11.037Z [INFO] server-ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb: election won: term=2 tally=2 -2024-05-15T01:17:11.037Z [INFO] server-ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb: entering leader state: leader="Node at ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb [Leader]" -2024-05-15T01:17:11.037Z [INFO] server-ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb: added peer, starting replication: peer=server-fb4cb339-270a-c8e4-7f05-d668347f671a -2024-05-15T01:17:11.037Z [INFO] server-ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb: added peer, starting replication: peer=server-b3707e86-cd7c-653b-f769-05a87653af44 -2024-05-15T01:17:11.037Z [INFO] server-ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb: pipelining replication: peer="{Voter server-b3707e86-cd7c-653b-f769-05a87653af44 b3707e86-cd7c-653b-f769-05a87653af44}" -2024-05-15T01:17:11.037Z [INFO] server-ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb: pipelining replication: peer="{Voter server-fb4cb339-270a-c8e4-7f05-d668347f671a fb4cb339-270a-c8e4-7f05-d668347f671a}" -2024-05-15T01:17:11.142Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:11.037424672 +0000 UTC m=+11.578809683 (1 nodes), 72.836936ms from start of poll, 73.352631ms from cluster start. Timeout at 2024-05-15 01:17:11.142520544 +0000 UTC m=+11.683905322, 105.095639ms after stability -2024-05-15T01:17:11.142Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:11.248Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:11.142653188 +0000 UTC m=+11.684037426 (2 nodes), 3.801µs from start of poll, 178.580374ms from cluster start. Timeout at 2024-05-15 01:17:11.248239969 +0000 UTC m=+11.789624754, 105.587328ms after stability -2024-05-15T01:17:11.248Z [INFO] server-ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb: updating configuration: command=AddVoter server-id=server-fb4cb339-270a-c8e4-7f05-d668347f671a server-addr=fb4cb339-270a-c8e4-7f05-d668347f671a servers="[{Suffrage:Voter ID:server-fb4cb339-270a-c8e4-7f05-d668347f671a Address:fb4cb339-270a-c8e4-7f05-d668347f671a} {Suffrage:Voter ID:server-b3707e86-cd7c-653b-f769-05a87653af44 Address:b3707e86-cd7c-653b-f769-05a87653af44} {Suffrage:Voter ID:server-ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb Address:ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb}]" -2024-05-15T01:17:11.248Z [INFO] server-ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb: aborting pipeline replication: peer="{Voter server-b3707e86-cd7c-653b-f769-05a87653af44 b3707e86-cd7c-653b-f769-05a87653af44}" -2024-05-15T01:17:11.248Z [INFO] server-ec3b0a06-6fba-3ec0-d7f9-096b0e51c6cb: aborting pipeline replication: peer="{Voter server-fb4cb339-270a-c8e4-7f05-d668347f671a fb4cb339-270a-c8e4-7f05-d668347f671a}" ---- PASS: TestRaft_AddKnownPeer (0.29s) +2025-06-17T07:43:03.081Z [INFO] server-8bb8b103-2132-2c95-7da5-b8dafe44500a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8bb8b103-2132-2c95-7da5-b8dafe44500a Address:8bb8b103-2132-2c95-7da5-b8dafe44500a} {Suffrage:Voter ID:server-a3c4ccf3-5865-a245-c296-25a39d663a11 Address:a3c4ccf3-5865-a245-c296-25a39d663a11} {Suffrage:Voter ID:server-b9a9f34b-bed9-bfd1-feb7-2d5a1c5e7199 Address:b9a9f34b-bed9-bfd1-feb7-2d5a1c5e7199}]" +2025-06-17T07:43:03.081Z [INFO] server-8bb8b103-2132-2c95-7da5-b8dafe44500a: entering follower state: follower="Node at 8bb8b103-2132-2c95-7da5-b8dafe44500a [Follower]" leader-address= leader-id= +2025-06-17T07:43:03.082Z [INFO] server-a3c4ccf3-5865-a245-c296-25a39d663a11: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8bb8b103-2132-2c95-7da5-b8dafe44500a Address:8bb8b103-2132-2c95-7da5-b8dafe44500a} {Suffrage:Voter ID:server-a3c4ccf3-5865-a245-c296-25a39d663a11 Address:a3c4ccf3-5865-a245-c296-25a39d663a11} {Suffrage:Voter ID:server-b9a9f34b-bed9-bfd1-feb7-2d5a1c5e7199 Address:b9a9f34b-bed9-bfd1-feb7-2d5a1c5e7199}]" +2025-06-17T07:43:03.082Z [INFO] server-a3c4ccf3-5865-a245-c296-25a39d663a11: entering follower state: follower="Node at a3c4ccf3-5865-a245-c296-25a39d663a11 [Follower]" leader-address= leader-id= +2025-06-17T07:43:03.082Z [INFO] server-b9a9f34b-bed9-bfd1-feb7-2d5a1c5e7199: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8bb8b103-2132-2c95-7da5-b8dafe44500a Address:8bb8b103-2132-2c95-7da5-b8dafe44500a} {Suffrage:Voter ID:server-a3c4ccf3-5865-a245-c296-25a39d663a11 Address:a3c4ccf3-5865-a245-c296-25a39d663a11} {Suffrage:Voter ID:server-b9a9f34b-bed9-bfd1-feb7-2d5a1c5e7199 Address:b9a9f34b-bed9-bfd1-feb7-2d5a1c5e7199}]" +2025-06-17T07:43:03.082Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:03.082Z [INFO] server-b9a9f34b-bed9-bfd1-feb7-2d5a1c5e7199: entering follower state: follower="Node at b9a9f34b-bed9-bfd1-feb7-2d5a1c5e7199 [Follower]" leader-address= leader-id= +2025-06-17T07:43:03.153Z [WARN] server-8bb8b103-2132-2c95-7da5-b8dafe44500a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:03.153Z [INFO] server-8bb8b103-2132-2c95-7da5-b8dafe44500a: entering candidate state: node="Node at 8bb8b103-2132-2c95-7da5-b8dafe44500a [Candidate]" term=2 +2025-06-17T07:43:03.153Z [INFO] server-8bb8b103-2132-2c95-7da5-b8dafe44500a: election won: term=2 tally=2 +2025-06-17T07:43:03.153Z [INFO] server-8bb8b103-2132-2c95-7da5-b8dafe44500a: entering leader state: leader="Node at 8bb8b103-2132-2c95-7da5-b8dafe44500a [Leader]" +2025-06-17T07:43:03.153Z [INFO] server-8bb8b103-2132-2c95-7da5-b8dafe44500a: added peer, starting replication: peer=server-a3c4ccf3-5865-a245-c296-25a39d663a11 +2025-06-17T07:43:03.153Z [INFO] server-8bb8b103-2132-2c95-7da5-b8dafe44500a: added peer, starting replication: peer=server-b9a9f34b-bed9-bfd1-feb7-2d5a1c5e7199 +2025-06-17T07:43:03.153Z [INFO] server-8bb8b103-2132-2c95-7da5-b8dafe44500a: pipelining replication: peer="{Voter server-b9a9f34b-bed9-bfd1-feb7-2d5a1c5e7199 b9a9f34b-bed9-bfd1-feb7-2d5a1c5e7199}" +2025-06-17T07:43:03.153Z [INFO] server-8bb8b103-2132-2c95-7da5-b8dafe44500a: pipelining replication: peer="{Voter server-a3c4ccf3-5865-a245-c296-25a39d663a11 a3c4ccf3-5865-a245-c296-25a39d663a11}" +2025-06-17T07:43:03.259Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:03.153407015 +0000 UTC m=+11.756935743 (1 nodes), 71.19102ms from start of poll, 71.571012ms from cluster start. Timeout at 2025-06-17 07:43:03.259133915 +0000 UTC m=+11.862662606, 105.726863ms after stability +2025-06-17T07:43:03.259Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:03.365Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:03.259186414 +0000 UTC m=+11.862715110 (2 nodes), 979ns from start of poll, 177.350379ms from cluster start. Timeout at 2025-06-17 07:43:03.365155367 +0000 UTC m=+11.968684049, 105.968939ms after stability +2025-06-17T07:43:03.365Z [INFO] server-8bb8b103-2132-2c95-7da5-b8dafe44500a: updating configuration: command=AddVoter server-id=server-a3c4ccf3-5865-a245-c296-25a39d663a11 server-addr=a3c4ccf3-5865-a245-c296-25a39d663a11 servers="[{Suffrage:Voter ID:server-8bb8b103-2132-2c95-7da5-b8dafe44500a Address:8bb8b103-2132-2c95-7da5-b8dafe44500a} {Suffrage:Voter ID:server-a3c4ccf3-5865-a245-c296-25a39d663a11 Address:a3c4ccf3-5865-a245-c296-25a39d663a11} {Suffrage:Voter ID:server-b9a9f34b-bed9-bfd1-feb7-2d5a1c5e7199 Address:b9a9f34b-bed9-bfd1-feb7-2d5a1c5e7199}]" +2025-06-17T07:43:03.365Z [INFO] server-8bb8b103-2132-2c95-7da5-b8dafe44500a: aborting pipeline replication: peer="{Voter server-b9a9f34b-bed9-bfd1-feb7-2d5a1c5e7199 b9a9f34b-bed9-bfd1-feb7-2d5a1c5e7199}" +2025-06-17T07:43:03.365Z [INFO] server-8bb8b103-2132-2c95-7da5-b8dafe44500a: aborting pipeline replication: peer="{Voter server-a3c4ccf3-5865-a245-c296-25a39d663a11 a3c4ccf3-5865-a245-c296-25a39d663a11}" +--- PASS: TestRaft_AddKnownPeer (0.28s) === RUN TestRaft_RemoveUnknownPeer -2024-05-15T01:17:11.250Z [INFO] server-8ba02e96-4e96-2903-3468-7b15c20faaf6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8ba02e96-4e96-2903-3468-7b15c20faaf6 Address:8ba02e96-4e96-2903-3468-7b15c20faaf6} {Suffrage:Voter ID:server-01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d Address:01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d} {Suffrage:Voter ID:server-e375f5f3-ef2f-f965-1cab-69bb8123f6cf Address:e375f5f3-ef2f-f965-1cab-69bb8123f6cf}]" -2024-05-15T01:17:11.250Z [INFO] server-01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8ba02e96-4e96-2903-3468-7b15c20faaf6 Address:8ba02e96-4e96-2903-3468-7b15c20faaf6} {Suffrage:Voter ID:server-01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d Address:01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d} {Suffrage:Voter ID:server-e375f5f3-ef2f-f965-1cab-69bb8123f6cf Address:e375f5f3-ef2f-f965-1cab-69bb8123f6cf}]" -2024-05-15T01:17:11.250Z [INFO] server-e375f5f3-ef2f-f965-1cab-69bb8123f6cf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8ba02e96-4e96-2903-3468-7b15c20faaf6 Address:8ba02e96-4e96-2903-3468-7b15c20faaf6} {Suffrage:Voter ID:server-01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d Address:01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d} {Suffrage:Voter ID:server-e375f5f3-ef2f-f965-1cab-69bb8123f6cf Address:e375f5f3-ef2f-f965-1cab-69bb8123f6cf}]" -2024-05-15T01:17:11.250Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:11.250Z [INFO] server-8ba02e96-4e96-2903-3468-7b15c20faaf6: entering follower state: follower="Node at 8ba02e96-4e96-2903-3468-7b15c20faaf6 [Follower]" leader-address= leader-id= -2024-05-15T01:17:11.250Z [INFO] server-01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d: entering follower state: follower="Node at 01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d [Follower]" leader-address= leader-id= -2024-05-15T01:17:11.250Z [INFO] server-e375f5f3-ef2f-f965-1cab-69bb8123f6cf: entering follower state: follower="Node at e375f5f3-ef2f-f965-1cab-69bb8123f6cf [Follower]" leader-address= leader-id= -2024-05-15T01:17:11.319Z [WARN] server-01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:11.319Z [INFO] server-01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d: entering candidate state: node="Node at 01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d [Candidate]" term=2 -2024-05-15T01:17:11.319Z [INFO] server-01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d: election won: term=2 tally=2 -2024-05-15T01:17:11.319Z [INFO] server-01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d: entering leader state: leader="Node at 01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d [Leader]" -2024-05-15T01:17:11.319Z [INFO] server-01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d: added peer, starting replication: peer=server-8ba02e96-4e96-2903-3468-7b15c20faaf6 -2024-05-15T01:17:11.319Z [INFO] server-01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d: added peer, starting replication: peer=server-e375f5f3-ef2f-f965-1cab-69bb8123f6cf -2024-05-15T01:17:11.320Z [INFO] server-01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d: pipelining replication: peer="{Voter server-e375f5f3-ef2f-f965-1cab-69bb8123f6cf e375f5f3-ef2f-f965-1cab-69bb8123f6cf}" -2024-05-15T01:17:11.320Z [INFO] server-01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d: pipelining replication: peer="{Voter server-8ba02e96-4e96-2903-3468-7b15c20faaf6 8ba02e96-4e96-2903-3468-7b15c20faaf6}" -2024-05-15T01:17:11.426Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:11.320438097 +0000 UTC m=+11.861822335 (1 nodes), 69.618365ms from start of poll, 70.144242ms from cluster start. Timeout at 2024-05-15 01:17:11.426348972 +0000 UTC m=+11.967733210, 105.910875ms after stability -2024-05-15T01:17:11.426Z [INFO] server-01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d: updating configuration: command=RemoveServer server-id=89244db2-92da-3df9-f869-533d51252539 server-addr= servers="[{Suffrage:Voter ID:server-8ba02e96-4e96-2903-3468-7b15c20faaf6 Address:8ba02e96-4e96-2903-3468-7b15c20faaf6} {Suffrage:Voter ID:server-01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d Address:01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d} {Suffrage:Voter ID:server-e375f5f3-ef2f-f965-1cab-69bb8123f6cf Address:e375f5f3-ef2f-f965-1cab-69bb8123f6cf}]" -2024-05-15T01:17:11.426Z [INFO] server-01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d: aborting pipeline replication: peer="{Voter server-e375f5f3-ef2f-f965-1cab-69bb8123f6cf e375f5f3-ef2f-f965-1cab-69bb8123f6cf}" -2024-05-15T01:17:11.427Z [INFO] server-01b0ab1c-ed5f-4ca6-e870-c7ba18446c5d: aborting pipeline replication: peer="{Voter server-8ba02e96-4e96-2903-3468-7b15c20faaf6 8ba02e96-4e96-2903-3468-7b15c20faaf6}" ---- PASS: TestRaft_RemoveUnknownPeer (0.18s) +2025-06-17T07:43:03.366Z [INFO] server-1ece00b9-6550-9e4d-30eb-8e71c91177b8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ece00b9-6550-9e4d-30eb-8e71c91177b8 Address:1ece00b9-6550-9e4d-30eb-8e71c91177b8} {Suffrage:Voter ID:server-4559d7fc-9fd1-3774-007a-19e35c62bce8 Address:4559d7fc-9fd1-3774-007a-19e35c62bce8} {Suffrage:Voter ID:server-ed61b977-eef9-c88c-fac0-dad7fb56906a Address:ed61b977-eef9-c88c-fac0-dad7fb56906a}]" +2025-06-17T07:43:03.366Z [INFO] server-1ece00b9-6550-9e4d-30eb-8e71c91177b8: entering follower state: follower="Node at 1ece00b9-6550-9e4d-30eb-8e71c91177b8 [Follower]" leader-address= leader-id= +2025-06-17T07:43:03.366Z [INFO] server-4559d7fc-9fd1-3774-007a-19e35c62bce8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ece00b9-6550-9e4d-30eb-8e71c91177b8 Address:1ece00b9-6550-9e4d-30eb-8e71c91177b8} {Suffrage:Voter ID:server-4559d7fc-9fd1-3774-007a-19e35c62bce8 Address:4559d7fc-9fd1-3774-007a-19e35c62bce8} {Suffrage:Voter ID:server-ed61b977-eef9-c88c-fac0-dad7fb56906a Address:ed61b977-eef9-c88c-fac0-dad7fb56906a}]" +2025-06-17T07:43:03.366Z [INFO] server-4559d7fc-9fd1-3774-007a-19e35c62bce8: entering follower state: follower="Node at 4559d7fc-9fd1-3774-007a-19e35c62bce8 [Follower]" leader-address= leader-id= +2025-06-17T07:43:03.366Z [INFO] server-ed61b977-eef9-c88c-fac0-dad7fb56906a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ece00b9-6550-9e4d-30eb-8e71c91177b8 Address:1ece00b9-6550-9e4d-30eb-8e71c91177b8} {Suffrage:Voter ID:server-4559d7fc-9fd1-3774-007a-19e35c62bce8 Address:4559d7fc-9fd1-3774-007a-19e35c62bce8} {Suffrage:Voter ID:server-ed61b977-eef9-c88c-fac0-dad7fb56906a Address:ed61b977-eef9-c88c-fac0-dad7fb56906a}]" +2025-06-17T07:43:03.366Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:03.366Z [INFO] server-ed61b977-eef9-c88c-fac0-dad7fb56906a: entering follower state: follower="Node at ed61b977-eef9-c88c-fac0-dad7fb56906a [Follower]" leader-address= leader-id= +2025-06-17T07:43:03.446Z [WARN] server-1ece00b9-6550-9e4d-30eb-8e71c91177b8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:03.446Z [INFO] server-1ece00b9-6550-9e4d-30eb-8e71c91177b8: entering candidate state: node="Node at 1ece00b9-6550-9e4d-30eb-8e71c91177b8 [Candidate]" term=2 +2025-06-17T07:43:03.446Z [WARN] server-ed61b977-eef9-c88c-fac0-dad7fb56906a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:03.446Z [INFO] server-ed61b977-eef9-c88c-fac0-dad7fb56906a: entering candidate state: node="Node at ed61b977-eef9-c88c-fac0-dad7fb56906a [Candidate]" term=2 +2025-06-17T07:43:03.446Z [INFO] server-1ece00b9-6550-9e4d-30eb-8e71c91177b8: election won: term=2 tally=2 +2025-06-17T07:43:03.446Z [INFO] server-1ece00b9-6550-9e4d-30eb-8e71c91177b8: entering leader state: leader="Node at 1ece00b9-6550-9e4d-30eb-8e71c91177b8 [Leader]" +2025-06-17T07:43:03.446Z [INFO] server-4559d7fc-9fd1-3774-007a-19e35c62bce8: duplicate requestVote for same term: term=2 +2025-06-17T07:43:03.446Z [INFO] server-1ece00b9-6550-9e4d-30eb-8e71c91177b8: added peer, starting replication: peer=server-4559d7fc-9fd1-3774-007a-19e35c62bce8 +2025-06-17T07:43:03.446Z [INFO] server-ed61b977-eef9-c88c-fac0-dad7fb56906a: duplicate requestVote for same term: term=2 +2025-06-17T07:43:03.446Z [INFO] server-1ece00b9-6550-9e4d-30eb-8e71c91177b8: added peer, starting replication: peer=server-ed61b977-eef9-c88c-fac0-dad7fb56906a +2025-06-17T07:43:03.446Z [WARN] server-1ece00b9-6550-9e4d-30eb-8e71c91177b8: rejecting vote request since we have a leader: from=ed61b977-eef9-c88c-fac0-dad7fb56906a leader=1ece00b9-6550-9e4d-30eb-8e71c91177b8 leader-id=server-1ece00b9-6550-9e4d-30eb-8e71c91177b8 +2025-06-17T07:43:03.446Z [INFO] server-ed61b977-eef9-c88c-fac0-dad7fb56906a: entering follower state: follower="Node at ed61b977-eef9-c88c-fac0-dad7fb56906a [Follower]" leader-address=1ece00b9-6550-9e4d-30eb-8e71c91177b8 leader-id=server-1ece00b9-6550-9e4d-30eb-8e71c91177b8 +2025-06-17T07:43:03.446Z [INFO] server-1ece00b9-6550-9e4d-30eb-8e71c91177b8: pipelining replication: peer="{Voter server-4559d7fc-9fd1-3774-007a-19e35c62bce8 4559d7fc-9fd1-3774-007a-19e35c62bce8}" +2025-06-17T07:43:03.446Z [INFO] server-1ece00b9-6550-9e4d-30eb-8e71c91177b8: pipelining replication: peer="{Voter server-ed61b977-eef9-c88c-fac0-dad7fb56906a ed61b977-eef9-c88c-fac0-dad7fb56906a}" +2025-06-17T07:43:03.552Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:03.446512657 +0000 UTC m=+12.050041354 (1 nodes), 79.89667ms from start of poll, 80.220364ms from cluster start. Timeout at 2025-06-17 07:43:03.55217413 +0000 UTC m=+12.155702835, 105.661481ms after stability +2025-06-17T07:43:03.552Z [INFO] server-1ece00b9-6550-9e4d-30eb-8e71c91177b8: updating configuration: command=RemoveServer server-id=eb30a08b-8548-7ff3-e74d-647875a97ce4 server-addr= servers="[{Suffrage:Voter ID:server-1ece00b9-6550-9e4d-30eb-8e71c91177b8 Address:1ece00b9-6550-9e4d-30eb-8e71c91177b8} {Suffrage:Voter ID:server-4559d7fc-9fd1-3774-007a-19e35c62bce8 Address:4559d7fc-9fd1-3774-007a-19e35c62bce8} {Suffrage:Voter ID:server-ed61b977-eef9-c88c-fac0-dad7fb56906a Address:ed61b977-eef9-c88c-fac0-dad7fb56906a}]" +2025-06-17T07:43:03.552Z [INFO] server-1ece00b9-6550-9e4d-30eb-8e71c91177b8: aborting pipeline replication: peer="{Voter server-ed61b977-eef9-c88c-fac0-dad7fb56906a ed61b977-eef9-c88c-fac0-dad7fb56906a}" +2025-06-17T07:43:03.552Z [INFO] server-1ece00b9-6550-9e4d-30eb-8e71c91177b8: aborting pipeline replication: peer="{Voter server-4559d7fc-9fd1-3774-007a-19e35c62bce8 4559d7fc-9fd1-3774-007a-19e35c62bce8}" +--- PASS: TestRaft_RemoveUnknownPeer (0.19s) === RUN TestRaft_SnapshotRestore -2024-05-15T01:17:11.428Z [INFO] server-873bce67-85cf-7dce-aec9-60aedd29ea3a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-873bce67-85cf-7dce-aec9-60aedd29ea3a Address:873bce67-85cf-7dce-aec9-60aedd29ea3a}]" -2024-05-15T01:17:11.428Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:11.428Z [INFO] server-873bce67-85cf-7dce-aec9-60aedd29ea3a: entering follower state: follower="Node at 873bce67-85cf-7dce-aec9-60aedd29ea3a [Follower]" leader-address= leader-id= -2024-05-15T01:17:11.523Z [WARN] server-873bce67-85cf-7dce-aec9-60aedd29ea3a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:11.523Z [INFO] server-873bce67-85cf-7dce-aec9-60aedd29ea3a: entering candidate state: node="Node at 873bce67-85cf-7dce-aec9-60aedd29ea3a [Candidate]" term=2 -2024-05-15T01:17:11.523Z [INFO] server-873bce67-85cf-7dce-aec9-60aedd29ea3a: election won: term=2 tally=1 -2024-05-15T01:17:11.523Z [INFO] server-873bce67-85cf-7dce-aec9-60aedd29ea3a: entering leader state: leader="Node at 873bce67-85cf-7dce-aec9-60aedd29ea3a [Leader]" -2024-05-15T01:17:11.629Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:11.523475818 +0000 UTC m=+12.064860056 (1 nodes), 95.202793ms from start of poll, 95.398115ms from cluster start. Timeout at 2024-05-15 01:17:11.629370283 +0000 UTC m=+12.170755072, 105.895016ms after stability -2024-05-15T01:17:11.635Z [INFO] server-873bce67-85cf-7dce-aec9-60aedd29ea3a: starting snapshot up to: index=102 -2024-05-15T01:17:11.635Z [INFO] creating new snapshot: path=/tmp/raft3221358303/snapshots/2-102-1715735831635.tmp -2024-05-15T01:17:11.636Z [INFO] server-873bce67-85cf-7dce-aec9-60aedd29ea3a: compacting logs: from=1 to=92 -2024-05-15T01:17:11.636Z [INFO] server-873bce67-85cf-7dce-aec9-60aedd29ea3a: snapshot complete up to: index=102 -2024-05-15T01:17:11.636Z [INFO] server-873bce67-85cf-7dce-aec9-60aedd29ea3a: starting restore from snapshot: id=2-102-1715735831635 last-index=102 last-term=2 size-in-bytes=693 -2024-05-15T01:17:11.637Z [INFO] server-873bce67-85cf-7dce-aec9-60aedd29ea3a: snapshot restore progress: id=2-102-1715735831635 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2024-05-15T01:17:11.637Z [INFO] server-873bce67-85cf-7dce-aec9-60aedd29ea3a: restored from snapshot: id=2-102-1715735831635 last-index=102 last-term=2 size-in-bytes=693 -2024-05-15T01:17:11.637Z [INFO] server-873bce67-85cf-7dce-aec9-60aedd29ea3a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-873bce67-85cf-7dce-aec9-60aedd29ea3a Address:873bce67-85cf-7dce-aec9-60aedd29ea3a}]" +2025-06-17T07:43:03.553Z [INFO] server-637bcdbb-9d65-89ee-d135-df72351f871b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-637bcdbb-9d65-89ee-d135-df72351f871b Address:637bcdbb-9d65-89ee-d135-df72351f871b}]" +2025-06-17T07:43:03.553Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:03.553Z [INFO] server-637bcdbb-9d65-89ee-d135-df72351f871b: entering follower state: follower="Node at 637bcdbb-9d65-89ee-d135-df72351f871b [Follower]" leader-address= leader-id= +2025-06-17T07:43:03.651Z [WARN] server-637bcdbb-9d65-89ee-d135-df72351f871b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:03.651Z [INFO] server-637bcdbb-9d65-89ee-d135-df72351f871b: entering candidate state: node="Node at 637bcdbb-9d65-89ee-d135-df72351f871b [Candidate]" term=2 +2025-06-17T07:43:03.651Z [INFO] server-637bcdbb-9d65-89ee-d135-df72351f871b: election won: term=2 tally=1 +2025-06-17T07:43:03.651Z [INFO] server-637bcdbb-9d65-89ee-d135-df72351f871b: entering leader state: leader="Node at 637bcdbb-9d65-89ee-d135-df72351f871b [Leader]" +2025-06-17T07:43:03.757Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:03.651614605 +0000 UTC m=+12.255143301 (1 nodes), 98.516363ms from start of poll, 98.629598ms from cluster start. Timeout at 2025-06-17 07:43:03.757173033 +0000 UTC m=+12.360701715, 105.558414ms after stability +2025-06-17T07:43:03.759Z [INFO] server-637bcdbb-9d65-89ee-d135-df72351f871b: starting snapshot up to: index=102 +2025-06-17T07:43:03.759Z [INFO] creating new snapshot: path=/tmp/raft1841969819/snapshots/2-102-1750146183759.tmp +2025-06-17T07:43:03.759Z [INFO] server-637bcdbb-9d65-89ee-d135-df72351f871b: compacting logs: from=1 to=92 +2025-06-17T07:43:03.759Z [INFO] server-637bcdbb-9d65-89ee-d135-df72351f871b: snapshot complete up to: index=102 +2025-06-17T07:43:03.760Z [INFO] server-637bcdbb-9d65-89ee-d135-df72351f871b: starting restore from snapshot: id=2-102-1750146183759 last-index=102 last-term=2 size-in-bytes=693 +2025-06-17T07:43:03.760Z [INFO] server-637bcdbb-9d65-89ee-d135-df72351f871b: snapshot restore progress: id=2-102-1750146183759 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-06-17T07:43:03.760Z [INFO] server-637bcdbb-9d65-89ee-d135-df72351f871b: restored from snapshot: id=2-102-1750146183759 last-index=102 last-term=2 size-in-bytes=693 +2025-06-17T07:43:03.760Z [INFO] server-637bcdbb-9d65-89ee-d135-df72351f871b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-637bcdbb-9d65-89ee-d135-df72351f871b Address:637bcdbb-9d65-89ee-d135-df72351f871b}]" --- PASS: TestRaft_SnapshotRestore (0.21s) === RUN TestRaft_RestoreSnapshotOnStartup_Monotonic -2024-05-15T01:17:11.639Z [INFO] server-e0d193e4-84fc-c675-30ce-fa1fe84ee1bc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e0d193e4-84fc-c675-30ce-fa1fe84ee1bc Address:e0d193e4-84fc-c675-30ce-fa1fe84ee1bc}]" -2024-05-15T01:17:11.639Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:11.639Z [INFO] server-e0d193e4-84fc-c675-30ce-fa1fe84ee1bc: entering follower state: follower="Node at e0d193e4-84fc-c675-30ce-fa1fe84ee1bc [Follower]" leader-address= leader-id= -2024-05-15T01:17:11.704Z [WARN] server-e0d193e4-84fc-c675-30ce-fa1fe84ee1bc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:11.704Z [INFO] server-e0d193e4-84fc-c675-30ce-fa1fe84ee1bc: entering candidate state: node="Node at e0d193e4-84fc-c675-30ce-fa1fe84ee1bc [Candidate]" term=2 -2024-05-15T01:17:11.705Z [INFO] server-e0d193e4-84fc-c675-30ce-fa1fe84ee1bc: election won: term=2 tally=1 -2024-05-15T01:17:11.705Z [INFO] server-e0d193e4-84fc-c675-30ce-fa1fe84ee1bc: entering leader state: leader="Node at e0d193e4-84fc-c675-30ce-fa1fe84ee1bc [Leader]" -2024-05-15T01:17:11.810Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:11.705175481 +0000 UTC m=+12.246579176 (1 nodes), 65.760594ms from start of poll, 65.970763ms from cluster start. Timeout at 2024-05-15 01:17:11.810916099 +0000 UTC m=+12.352300892, 105.721716ms after stability -2024-05-15T01:17:11.816Z [INFO] server-e0d193e4-84fc-c675-30ce-fa1fe84ee1bc: starting snapshot up to: index=102 -2024-05-15T01:17:11.816Z [INFO] creating new snapshot: path=/tmp/raft4176731083/snapshots/2-102-1715735831816.tmp -2024-05-15T01:17:11.816Z [INFO] server-e0d193e4-84fc-c675-30ce-fa1fe84ee1bc: compacting logs: from=1 to=92 -2024-05-15T01:17:11.816Z [INFO] server-e0d193e4-84fc-c675-30ce-fa1fe84ee1bc: snapshot complete up to: index=102 -2024-05-15T01:17:11.817Z [INFO] server-e0d193e4-84fc-c675-30ce-fa1fe84ee1bc: starting restore from snapshot: id=2-102-1715735831816 last-index=102 last-term=2 size-in-bytes=693 -2024-05-15T01:17:11.817Z [INFO] server-e0d193e4-84fc-c675-30ce-fa1fe84ee1bc: snapshot restore progress: id=2-102-1715735831816 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2024-05-15T01:17:11.817Z [INFO] server-e0d193e4-84fc-c675-30ce-fa1fe84ee1bc: restored from snapshot: id=2-102-1715735831816 last-index=102 last-term=2 size-in-bytes=693 -2024-05-15T01:17:11.817Z [INFO] server-e0d193e4-84fc-c675-30ce-fa1fe84ee1bc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e0d193e4-84fc-c675-30ce-fa1fe84ee1bc Address:e0d193e4-84fc-c675-30ce-fa1fe84ee1bc}]" ---- PASS: TestRaft_RestoreSnapshotOnStartup_Monotonic (0.18s) +2025-06-17T07:43:03.760Z [INFO] server-a122949b-500f-ce0b-5e2a-a505e22afaa4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a122949b-500f-ce0b-5e2a-a505e22afaa4 Address:a122949b-500f-ce0b-5e2a-a505e22afaa4}]" +2025-06-17T07:43:03.760Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:03.760Z [INFO] server-a122949b-500f-ce0b-5e2a-a505e22afaa4: entering follower state: follower="Node at a122949b-500f-ce0b-5e2a-a505e22afaa4 [Follower]" leader-address= leader-id= +2025-06-17T07:43:03.814Z [WARN] server-a122949b-500f-ce0b-5e2a-a505e22afaa4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:03.814Z [INFO] server-a122949b-500f-ce0b-5e2a-a505e22afaa4: entering candidate state: node="Node at a122949b-500f-ce0b-5e2a-a505e22afaa4 [Candidate]" term=2 +2025-06-17T07:43:03.814Z [INFO] server-a122949b-500f-ce0b-5e2a-a505e22afaa4: election won: term=2 tally=1 +2025-06-17T07:43:03.814Z [INFO] server-a122949b-500f-ce0b-5e2a-a505e22afaa4: entering leader state: leader="Node at a122949b-500f-ce0b-5e2a-a505e22afaa4 [Leader]" +2025-06-17T07:43:03.920Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:03.814139103 +0000 UTC m=+12.417667801 (1 nodes), 53.157501ms from start of poll, 53.275952ms from cluster start. Timeout at 2025-06-17 07:43:03.920189935 +0000 UTC m=+12.523718620, 106.050819ms after stability +2025-06-17T07:43:03.923Z [INFO] server-a122949b-500f-ce0b-5e2a-a505e22afaa4: starting snapshot up to: index=102 +2025-06-17T07:43:03.923Z [INFO] creating new snapshot: path=/tmp/raft1137874585/snapshots/2-102-1750146183923.tmp +2025-06-17T07:43:03.923Z [INFO] server-a122949b-500f-ce0b-5e2a-a505e22afaa4: compacting logs: from=1 to=92 +2025-06-17T07:43:03.923Z [INFO] server-a122949b-500f-ce0b-5e2a-a505e22afaa4: snapshot complete up to: index=102 +2025-06-17T07:43:03.924Z [INFO] server-a122949b-500f-ce0b-5e2a-a505e22afaa4: starting restore from snapshot: id=2-102-1750146183923 last-index=102 last-term=2 size-in-bytes=693 +2025-06-17T07:43:03.924Z [INFO] server-a122949b-500f-ce0b-5e2a-a505e22afaa4: snapshot restore progress: id=2-102-1750146183923 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-06-17T07:43:03.924Z [INFO] server-a122949b-500f-ce0b-5e2a-a505e22afaa4: restored from snapshot: id=2-102-1750146183923 last-index=102 last-term=2 size-in-bytes=693 +2025-06-17T07:43:03.924Z [INFO] server-a122949b-500f-ce0b-5e2a-a505e22afaa4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a122949b-500f-ce0b-5e2a-a505e22afaa4 Address:a122949b-500f-ce0b-5e2a-a505e22afaa4}]" +2025-06-17T07:43:03.924Z [INFO] server-a122949b-500f-ce0b-5e2a-a505e22afaa4: entering follower state: follower="Node at a122949b-500f-ce0b-5e2a-a505e22afaa4 [Shutdown]" leader-address= leader-id= +--- PASS: TestRaft_RestoreSnapshotOnStartup_Monotonic (0.16s) === RUN TestRaft_SnapshotRestore_Progress -2024-05-15T01:17:11.818Z [INFO] server-ad46f188-8136-06b9-638d-f06737d2e3c5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ad46f188-8136-06b9-638d-f06737d2e3c5 Address:ad46f188-8136-06b9-638d-f06737d2e3c5}]" -2024-05-15T01:17:11.818Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:11.818Z [INFO] server-ad46f188-8136-06b9-638d-f06737d2e3c5: entering follower state: follower="Node at ad46f188-8136-06b9-638d-f06737d2e3c5 [Follower]" leader-address= leader-id= -2024-05-15T01:17:11.875Z [WARN] server-ad46f188-8136-06b9-638d-f06737d2e3c5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:11.876Z [INFO] server-ad46f188-8136-06b9-638d-f06737d2e3c5: entering candidate state: node="Node at ad46f188-8136-06b9-638d-f06737d2e3c5 [Candidate]" term=2 -2024-05-15T01:17:11.876Z [INFO] server-ad46f188-8136-06b9-638d-f06737d2e3c5: election won: term=2 tally=1 -2024-05-15T01:17:11.876Z [INFO] server-ad46f188-8136-06b9-638d-f06737d2e3c5: entering leader state: leader="Node at ad46f188-8136-06b9-638d-f06737d2e3c5 [Leader]" -2024-05-15T01:17:11.981Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:11.876250288 +0000 UTC m=+12.417635031 (1 nodes), 57.835237ms from start of poll, 58.035014ms from cluster start. Timeout at 2024-05-15 01:17:11.981798635 +0000 UTC m=+12.523183409, 105.548378ms after stability -2024-05-15T01:17:11.989Z [INFO] server-ad46f188-8136-06b9-638d-f06737d2e3c5: starting snapshot up to: index=102 -2024-05-15T01:17:11.990Z [INFO] creating new snapshot: path=/tmp/raft1506767398/snapshots/2-102-1715735831989.tmp -2024-05-15T01:17:11.990Z [INFO] server-ad46f188-8136-06b9-638d-f06737d2e3c5: compacting logs: from=1 to=92 -2024-05-15T01:17:11.990Z [INFO] server-ad46f188-8136-06b9-638d-f06737d2e3c5: snapshot complete up to: index=102 ---- PASS: TestRaft_SnapshotRestore_Progress (0.17s) +2025-06-17T07:43:03.924Z [INFO] server-2ad726d6-0eca-62a9-e162-f87b8bc4dbc8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2ad726d6-0eca-62a9-e162-f87b8bc4dbc8 Address:2ad726d6-0eca-62a9-e162-f87b8bc4dbc8}]" +2025-06-17T07:43:03.924Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:03.924Z [INFO] server-2ad726d6-0eca-62a9-e162-f87b8bc4dbc8: entering follower state: follower="Node at 2ad726d6-0eca-62a9-e162-f87b8bc4dbc8 [Follower]" leader-address= leader-id= +2025-06-17T07:43:03.994Z [WARN] server-2ad726d6-0eca-62a9-e162-f87b8bc4dbc8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:03.994Z [INFO] server-2ad726d6-0eca-62a9-e162-f87b8bc4dbc8: entering candidate state: node="Node at 2ad726d6-0eca-62a9-e162-f87b8bc4dbc8 [Candidate]" term=2 +2025-06-17T07:43:03.994Z [INFO] server-2ad726d6-0eca-62a9-e162-f87b8bc4dbc8: election won: term=2 tally=1 +2025-06-17T07:43:03.994Z [INFO] server-2ad726d6-0eca-62a9-e162-f87b8bc4dbc8: entering leader state: leader="Node at 2ad726d6-0eca-62a9-e162-f87b8bc4dbc8 [Leader]" +2025-06-17T07:43:04.099Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:03.99438307 +0000 UTC m=+12.597911758 (1 nodes), 69.397356ms from start of poll, 69.523804ms from cluster start. Timeout at 2025-06-17 07:43:04.099739828 +0000 UTC m=+12.703268521, 105.356763ms after stability +2025-06-17T07:43:04.101Z [INFO] server-2ad726d6-0eca-62a9-e162-f87b8bc4dbc8: starting snapshot up to: index=102 +2025-06-17T07:43:04.101Z [INFO] creating new snapshot: path=/tmp/raft2551621848/snapshots/2-102-1750146184101.tmp +2025-06-17T07:43:04.102Z [INFO] server-2ad726d6-0eca-62a9-e162-f87b8bc4dbc8: compacting logs: from=1 to=92 +2025-06-17T07:43:04.102Z [INFO] server-2ad726d6-0eca-62a9-e162-f87b8bc4dbc8: snapshot complete up to: index=102 +--- PASS: TestRaft_SnapshotRestore_Progress (0.18s) === RUN TestRaft_NoRestoreOnStart -2024-05-15T01:17:11.993Z [INFO] server-1af1132a-28ac-ff12-ef82-79b63b0a548d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1af1132a-28ac-ff12-ef82-79b63b0a548d Address:1af1132a-28ac-ff12-ef82-79b63b0a548d}]" -2024-05-15T01:17:11.993Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:11.993Z [INFO] server-1af1132a-28ac-ff12-ef82-79b63b0a548d: entering follower state: follower="Node at 1af1132a-28ac-ff12-ef82-79b63b0a548d [Follower]" leader-address= leader-id= -2024-05-15T01:17:12.077Z [WARN] server-1af1132a-28ac-ff12-ef82-79b63b0a548d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:12.077Z [INFO] server-1af1132a-28ac-ff12-ef82-79b63b0a548d: entering candidate state: node="Node at 1af1132a-28ac-ff12-ef82-79b63b0a548d [Candidate]" term=2 -2024-05-15T01:17:12.077Z [INFO] server-1af1132a-28ac-ff12-ef82-79b63b0a548d: election won: term=2 tally=1 -2024-05-15T01:17:12.077Z [INFO] server-1af1132a-28ac-ff12-ef82-79b63b0a548d: entering leader state: leader="Node at 1af1132a-28ac-ff12-ef82-79b63b0a548d [Leader]" -2024-05-15T01:17:12.183Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:12.077983294 +0000 UTC m=+12.619368142 (1 nodes), 84.673997ms from start of poll, 84.901838ms from cluster start. Timeout at 2024-05-15 01:17:12.18347756 +0000 UTC m=+12.724862349, 105.494207ms after stability -2024-05-15T01:17:12.188Z [INFO] server-1af1132a-28ac-ff12-ef82-79b63b0a548d: starting snapshot up to: index=102 -2024-05-15T01:17:12.188Z [INFO] creating new snapshot: path=/tmp/raft2154635237/snapshots/2-102-1715735832188.tmp -2024-05-15T01:17:12.190Z [INFO] server-1af1132a-28ac-ff12-ef82-79b63b0a548d: compacting logs: from=1 to=92 -2024-05-15T01:17:12.190Z [INFO] server-1af1132a-28ac-ff12-ef82-79b63b0a548d: snapshot complete up to: index=102 -2024-05-15T01:17:12.190Z [INFO] server-1af1132a-28ac-ff12-ef82-79b63b0a548d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1af1132a-28ac-ff12-ef82-79b63b0a548d Address:1af1132a-28ac-ff12-ef82-79b63b0a548d}]" +2025-06-17T07:43:04.103Z [INFO] server-4951dbfc-dd98-6101-547b-374dde6ab33e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4951dbfc-dd98-6101-547b-374dde6ab33e Address:4951dbfc-dd98-6101-547b-374dde6ab33e}]" +2025-06-17T07:43:04.103Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:04.103Z [INFO] server-4951dbfc-dd98-6101-547b-374dde6ab33e: entering follower state: follower="Node at 4951dbfc-dd98-6101-547b-374dde6ab33e [Follower]" leader-address= leader-id= +2025-06-17T07:43:04.191Z [WARN] server-4951dbfc-dd98-6101-547b-374dde6ab33e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:04.191Z [INFO] server-4951dbfc-dd98-6101-547b-374dde6ab33e: entering candidate state: node="Node at 4951dbfc-dd98-6101-547b-374dde6ab33e [Candidate]" term=2 +2025-06-17T07:43:04.191Z [INFO] server-4951dbfc-dd98-6101-547b-374dde6ab33e: election won: term=2 tally=1 +2025-06-17T07:43:04.191Z [INFO] server-4951dbfc-dd98-6101-547b-374dde6ab33e: entering leader state: leader="Node at 4951dbfc-dd98-6101-547b-374dde6ab33e [Leader]" +2025-06-17T07:43:04.297Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:04.191752137 +0000 UTC m=+12.795280825 (1 nodes), 88.435533ms from start of poll, 88.534681ms from cluster start. Timeout at 2025-06-17 07:43:04.297672071 +0000 UTC m=+12.901200753, 105.919928ms after stability +2025-06-17T07:43:04.299Z [INFO] server-4951dbfc-dd98-6101-547b-374dde6ab33e: starting snapshot up to: index=102 +2025-06-17T07:43:04.299Z [INFO] creating new snapshot: path=/tmp/raft1204181695/snapshots/2-102-1750146184299.tmp +2025-06-17T07:43:04.300Z [INFO] server-4951dbfc-dd98-6101-547b-374dde6ab33e: compacting logs: from=1 to=92 +2025-06-17T07:43:04.300Z [INFO] server-4951dbfc-dd98-6101-547b-374dde6ab33e: snapshot complete up to: index=102 +2025-06-17T07:43:04.300Z [INFO] server-4951dbfc-dd98-6101-547b-374dde6ab33e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4951dbfc-dd98-6101-547b-374dde6ab33e Address:4951dbfc-dd98-6101-547b-374dde6ab33e}]" --- PASS: TestRaft_NoRestoreOnStart (0.20s) === RUN TestRaft_SnapshotRestore_PeerChange +2025-06-17T07:43:04.300Z [INFO] server-4951dbfc-dd98-6101-547b-374dde6ab33e: entering follower state: follower="Node at 4951dbfc-dd98-6101-547b-374dde6ab33e [Follower]" leader-address= leader-id= raft_test.go:1266: Skipping unreliable test --- SKIP: TestRaft_SnapshotRestore_PeerChange (0.00s) === RUN TestRaft_AutoSnapshot -2024-05-15T01:17:12.191Z [INFO] server-90cf02d4-9234-a3a9-e661-4fb9b59d2a11: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90cf02d4-9234-a3a9-e661-4fb9b59d2a11 Address:90cf02d4-9234-a3a9-e661-4fb9b59d2a11}]" -2024-05-15T01:17:12.191Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:12.191Z [INFO] server-1af1132a-28ac-ff12-ef82-79b63b0a548d: entering follower state: follower="Node at 1af1132a-28ac-ff12-ef82-79b63b0a548d [Follower]" leader-address= leader-id= -2024-05-15T01:17:12.191Z [INFO] server-90cf02d4-9234-a3a9-e661-4fb9b59d2a11: entering follower state: follower="Node at 90cf02d4-9234-a3a9-e661-4fb9b59d2a11 [Follower]" leader-address= leader-id= -2024-05-15T01:17:12.255Z [WARN] server-90cf02d4-9234-a3a9-e661-4fb9b59d2a11: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:12.255Z [INFO] server-90cf02d4-9234-a3a9-e661-4fb9b59d2a11: entering candidate state: node="Node at 90cf02d4-9234-a3a9-e661-4fb9b59d2a11 [Candidate]" term=2 -2024-05-15T01:17:12.255Z [INFO] server-90cf02d4-9234-a3a9-e661-4fb9b59d2a11: election won: term=2 tally=1 -2024-05-15T01:17:12.255Z [INFO] server-90cf02d4-9234-a3a9-e661-4fb9b59d2a11: entering leader state: leader="Node at 90cf02d4-9234-a3a9-e661-4fb9b59d2a11 [Leader]" -2024-05-15T01:17:12.258Z [WARN] server-1af1132a-28ac-ff12-ef82-79b63b0a548d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:12.258Z [INFO] server-1af1132a-28ac-ff12-ef82-79b63b0a548d: entering candidate state: node="Node at 1af1132a-28ac-ff12-ef82-79b63b0a548d [Candidate]" term=3 -2024-05-15T01:17:12.259Z [INFO] server-1af1132a-28ac-ff12-ef82-79b63b0a548d: election won: term=3 tally=1 -2024-05-15T01:17:12.259Z [INFO] server-1af1132a-28ac-ff12-ef82-79b63b0a548d: entering leader state: leader="Node at 1af1132a-28ac-ff12-ef82-79b63b0a548d [Leader]" -2024-05-15T01:17:12.361Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:12.255722919 +0000 UTC m=+12.797107794 (1 nodes), 64.393639ms from start of poll, 64.58831ms from cluster start. Timeout at 2024-05-15 01:17:12.361766773 +0000 UTC m=+12.903151544, 106.04375ms after stability -2024-05-15T01:17:12.365Z [INFO] server-90cf02d4-9234-a3a9-e661-4fb9b59d2a11: starting snapshot up to: index=66 -2024-05-15T01:17:12.365Z [INFO] creating new snapshot: path=/tmp/raft3394942132/snapshots/2-66-1715735832365.tmp -2024-05-15T01:17:12.365Z [INFO] server-90cf02d4-9234-a3a9-e661-4fb9b59d2a11: compacting logs: from=1 to=56 -2024-05-15T01:17:12.366Z [INFO] server-90cf02d4-9234-a3a9-e661-4fb9b59d2a11: snapshot complete up to: index=66 ---- PASS: TestRaft_AutoSnapshot (0.28s) +2025-06-17T07:43:04.300Z [INFO] server-1ff9d6f9-b0f7-ef4a-d0d2-7122a40f861f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ff9d6f9-b0f7-ef4a-d0d2-7122a40f861f Address:1ff9d6f9-b0f7-ef4a-d0d2-7122a40f861f}]" +2025-06-17T07:43:04.300Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:04.300Z [INFO] server-1ff9d6f9-b0f7-ef4a-d0d2-7122a40f861f: entering follower state: follower="Node at 1ff9d6f9-b0f7-ef4a-d0d2-7122a40f861f [Follower]" leader-address= leader-id= +2025-06-17T07:43:04.374Z [WARN] server-4951dbfc-dd98-6101-547b-374dde6ab33e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:04.374Z [INFO] server-4951dbfc-dd98-6101-547b-374dde6ab33e: entering candidate state: node="Node at 4951dbfc-dd98-6101-547b-374dde6ab33e [Candidate]" term=3 +2025-06-17T07:43:04.374Z [INFO] server-4951dbfc-dd98-6101-547b-374dde6ab33e: election won: term=3 tally=1 +2025-06-17T07:43:04.374Z [INFO] server-4951dbfc-dd98-6101-547b-374dde6ab33e: entering leader state: leader="Node at 4951dbfc-dd98-6101-547b-374dde6ab33e [Leader]" +2025-06-17T07:43:04.379Z [WARN] server-1ff9d6f9-b0f7-ef4a-d0d2-7122a40f861f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:04.379Z [INFO] server-1ff9d6f9-b0f7-ef4a-d0d2-7122a40f861f: entering candidate state: node="Node at 1ff9d6f9-b0f7-ef4a-d0d2-7122a40f861f [Candidate]" term=2 +2025-06-17T07:43:04.379Z [INFO] server-1ff9d6f9-b0f7-ef4a-d0d2-7122a40f861f: election won: term=2 tally=1 +2025-06-17T07:43:04.379Z [INFO] server-1ff9d6f9-b0f7-ef4a-d0d2-7122a40f861f: entering leader state: leader="Node at 1ff9d6f9-b0f7-ef4a-d0d2-7122a40f861f [Leader]" +2025-06-17T07:43:04.485Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:04.379805246 +0000 UTC m=+12.983333938 (1 nodes), 79.153182ms from start of poll, 79.270986ms from cluster start. Timeout at 2025-06-17 07:43:04.485288744 +0000 UTC m=+13.088817436, 105.483498ms after stability +2025-06-17T07:43:04.502Z [INFO] server-1ff9d6f9-b0f7-ef4a-d0d2-7122a40f861f: starting snapshot up to: index=102 +2025-06-17T07:43:04.502Z [INFO] creating new snapshot: path=/tmp/raft158422425/snapshots/2-102-1750146184502.tmp +2025-06-17T07:43:04.502Z [INFO] server-1ff9d6f9-b0f7-ef4a-d0d2-7122a40f861f: compacting logs: from=1 to=92 +2025-06-17T07:43:04.502Z [INFO] server-1ff9d6f9-b0f7-ef4a-d0d2-7122a40f861f: snapshot complete up to: index=102 +--- PASS: TestRaft_AutoSnapshot (0.29s) === RUN TestRaft_UserSnapshot -2024-05-15T01:17:12.474Z [INFO] server-0c0e9e90-05c8-2f7e-9604-3cb91448757b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c0e9e90-05c8-2f7e-9604-3cb91448757b Address:0c0e9e90-05c8-2f7e-9604-3cb91448757b}]" -2024-05-15T01:17:12.474Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:12.474Z [INFO] server-0c0e9e90-05c8-2f7e-9604-3cb91448757b: entering follower state: follower="Node at 0c0e9e90-05c8-2f7e-9604-3cb91448757b [Follower]" leader-address= leader-id= -2024-05-15T01:17:12.541Z [WARN] server-0c0e9e90-05c8-2f7e-9604-3cb91448757b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:12.541Z [INFO] server-0c0e9e90-05c8-2f7e-9604-3cb91448757b: entering candidate state: node="Node at 0c0e9e90-05c8-2f7e-9604-3cb91448757b [Candidate]" term=2 -2024-05-15T01:17:12.541Z [INFO] server-0c0e9e90-05c8-2f7e-9604-3cb91448757b: election won: term=2 tally=1 -2024-05-15T01:17:12.542Z [INFO] server-0c0e9e90-05c8-2f7e-9604-3cb91448757b: entering leader state: leader="Node at 0c0e9e90-05c8-2f7e-9604-3cb91448757b [Leader]" -2024-05-15T01:17:12.648Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:12.542115454 +0000 UTC m=+13.083500218 (1 nodes), 67.242757ms from start of poll, 67.483307ms from cluster start. Timeout at 2024-05-15 01:17:12.648073045 +0000 UTC m=+13.189457824, 105.957606ms after stability -2024-05-15T01:17:12.648Z [ERROR] server-0c0e9e90-05c8-2f7e-9604-3cb91448757b: failed to take snapshot: error="nothing new to snapshot" -2024-05-15T01:17:12.648Z [INFO] server-0c0e9e90-05c8-2f7e-9604-3cb91448757b: starting snapshot up to: index=12 -2024-05-15T01:17:12.648Z [INFO] creating new snapshot: path=/tmp/raft3230005263/snapshots/2-12-1715735832648.tmp -2024-05-15T01:17:12.649Z [INFO] server-0c0e9e90-05c8-2f7e-9604-3cb91448757b: compacting logs: from=1 to=2 -2024-05-15T01:17:12.649Z [INFO] server-0c0e9e90-05c8-2f7e-9604-3cb91448757b: snapshot complete up to: index=12 ---- PASS: TestRaft_UserSnapshot (0.18s) +2025-06-17T07:43:04.593Z [INFO] server-4b6c5e47-1a6e-c294-d162-c546b8f99fb5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4b6c5e47-1a6e-c294-d162-c546b8f99fb5 Address:4b6c5e47-1a6e-c294-d162-c546b8f99fb5}]" +2025-06-17T07:43:04.593Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:04.593Z [INFO] server-4b6c5e47-1a6e-c294-d162-c546b8f99fb5: entering follower state: follower="Node at 4b6c5e47-1a6e-c294-d162-c546b8f99fb5 [Follower]" leader-address= leader-id= +2025-06-17T07:43:04.685Z [WARN] server-4b6c5e47-1a6e-c294-d162-c546b8f99fb5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:04.685Z [INFO] server-4b6c5e47-1a6e-c294-d162-c546b8f99fb5: entering candidate state: node="Node at 4b6c5e47-1a6e-c294-d162-c546b8f99fb5 [Candidate]" term=2 +2025-06-17T07:43:04.685Z [INFO] server-4b6c5e47-1a6e-c294-d162-c546b8f99fb5: election won: term=2 tally=1 +2025-06-17T07:43:04.685Z [INFO] server-4b6c5e47-1a6e-c294-d162-c546b8f99fb5: entering leader state: leader="Node at 4b6c5e47-1a6e-c294-d162-c546b8f99fb5 [Leader]" +2025-06-17T07:43:04.790Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:04.685531696 +0000 UTC m=+13.289060383 (1 nodes), 92.462747ms from start of poll, 92.569265ms from cluster start. Timeout at 2025-06-17 07:43:04.790883 +0000 UTC m=+13.394411682, 105.351299ms after stability +2025-06-17T07:43:04.790Z [ERROR] server-4b6c5e47-1a6e-c294-d162-c546b8f99fb5: failed to take snapshot: error="nothing new to snapshot" +2025-06-17T07:43:04.791Z [INFO] server-4b6c5e47-1a6e-c294-d162-c546b8f99fb5: starting snapshot up to: index=12 +2025-06-17T07:43:04.791Z [INFO] creating new snapshot: path=/tmp/raft464459469/snapshots/2-12-1750146184791.tmp +2025-06-17T07:43:04.791Z [INFO] server-4b6c5e47-1a6e-c294-d162-c546b8f99fb5: compacting logs: from=1 to=2 +2025-06-17T07:43:04.791Z [INFO] server-4b6c5e47-1a6e-c294-d162-c546b8f99fb5: snapshot complete up to: index=12 +--- PASS: TestRaft_UserSnapshot (0.20s) === RUN TestRaft_UserRestore === RUN TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false -2024-05-15T01:17:12.653Z [INFO] server-b7c38cfb-4e18-f631-7a42-dc33067a3fa0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b7c38cfb-4e18-f631-7a42-dc33067a3fa0 Address:b7c38cfb-4e18-f631-7a42-dc33067a3fa0} {Suffrage:Voter ID:server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441 Address:cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441} {Suffrage:Voter ID:server-96d8ff17-baba-ba12-a0e0-20ee285ba1ac Address:96d8ff17-baba-ba12-a0e0-20ee285ba1ac}]" -2024-05-15T01:17:12.654Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b7c38cfb-4e18-f631-7a42-dc33067a3fa0 Address:b7c38cfb-4e18-f631-7a42-dc33067a3fa0} {Suffrage:Voter ID:server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441 Address:cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441} {Suffrage:Voter ID:server-96d8ff17-baba-ba12-a0e0-20ee285ba1ac Address:96d8ff17-baba-ba12-a0e0-20ee285ba1ac}]" -2024-05-15T01:17:12.654Z [INFO] server-96d8ff17-baba-ba12-a0e0-20ee285ba1ac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b7c38cfb-4e18-f631-7a42-dc33067a3fa0 Address:b7c38cfb-4e18-f631-7a42-dc33067a3fa0} {Suffrage:Voter ID:server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441 Address:cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441} {Suffrage:Voter ID:server-96d8ff17-baba-ba12-a0e0-20ee285ba1ac Address:96d8ff17-baba-ba12-a0e0-20ee285ba1ac}]" -2024-05-15T01:17:12.654Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:12.654Z [INFO] server-b7c38cfb-4e18-f631-7a42-dc33067a3fa0: entering follower state: follower="Node at b7c38cfb-4e18-f631-7a42-dc33067a3fa0 [Follower]" leader-address= leader-id= -2024-05-15T01:17:12.654Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: entering follower state: follower="Node at cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441 [Follower]" leader-address= leader-id= -2024-05-15T01:17:12.654Z [INFO] server-96d8ff17-baba-ba12-a0e0-20ee285ba1ac: entering follower state: follower="Node at 96d8ff17-baba-ba12-a0e0-20ee285ba1ac [Follower]" leader-address= leader-id= -2024-05-15T01:17:13.234Z [WARN] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:13.234Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: entering candidate state: node="Node at cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441 [Candidate]" term=2 -2024-05-15T01:17:13.235Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: election won: term=2 tally=2 -2024-05-15T01:17:13.235Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: entering leader state: leader="Node at cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441 [Leader]" -2024-05-15T01:17:13.235Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: added peer, starting replication: peer=server-b7c38cfb-4e18-f631-7a42-dc33067a3fa0 -2024-05-15T01:17:13.235Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: added peer, starting replication: peer=server-96d8ff17-baba-ba12-a0e0-20ee285ba1ac -2024-05-15T01:17:13.235Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: pipelining replication: peer="{Voter server-96d8ff17-baba-ba12-a0e0-20ee285ba1ac 96d8ff17-baba-ba12-a0e0-20ee285ba1ac}" -2024-05-15T01:17:13.235Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: pipelining replication: peer="{Voter server-b7c38cfb-4e18-f631-7a42-dc33067a3fa0 b7c38cfb-4e18-f631-7a42-dc33067a3fa0}" -2024-05-15T01:17:14.241Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:13.235666323 +0000 UTC m=+13.777050561 (1 nodes), 581.415259ms from start of poll, 581.974838ms from cluster start. Timeout at 2024-05-15 01:17:14.241446866 +0000 UTC m=+14.782831626, 1.005781065s after stability -2024-05-15T01:17:14.242Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: starting snapshot up to: index=12 -2024-05-15T01:17:14.243Z [INFO] creating new snapshot: path=/tmp/raft1276692006/snapshots/2-12-1715735834243.tmp -2024-05-15T01:17:14.244Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: snapshot complete up to: index=12 -2024-05-15T01:17:14.249Z [INFO] creating new snapshot: path=/tmp/raft1276692006/snapshots/2-23-1715735834249.tmp -2024-05-15T01:17:14.250Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: copied to local snapshot: bytes=71 -2024-05-15T01:17:14.250Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: snapshot restore progress: id=2-23-1715735834249 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:14.250Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: restored user snapshot: index=23 -2024-05-15T01:17:14.250Z [ERROR] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: failed to get log: index=23 error="log not found" -2024-05-15T01:17:14.250Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: aborting pipeline replication: peer="{Voter server-96d8ff17-baba-ba12-a0e0-20ee285ba1ac 96d8ff17-baba-ba12-a0e0-20ee285ba1ac}" -2024-05-15T01:17:14.250Z [ERROR] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: failed to get log: index=23 error="log not found" -2024-05-15T01:17:14.250Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: aborting pipeline replication: peer="{Voter server-b7c38cfb-4e18-f631-7a42-dc33067a3fa0 b7c38cfb-4e18-f631-7a42-dc33067a3fa0}" -2024-05-15T01:17:14.257Z [ERROR] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: failed to get log: index=23 error="log not found" -2024-05-15T01:17:14.257Z [INFO] creating new snapshot: path=/tmp/raft2215147011/snapshots/2-23-1715735834257.tmp -2024-05-15T01:17:14.258Z [INFO] server-b7c38cfb-4e18-f631-7a42-dc33067a3fa0: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:14.258Z [INFO] server-b7c38cfb-4e18-f631-7a42-dc33067a3fa0: copied to local snapshot: bytes=71 -2024-05-15T01:17:14.259Z [INFO] server-b7c38cfb-4e18-f631-7a42-dc33067a3fa0: snapshot restore progress: id=2-23-1715735834257 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:14.259Z [INFO] server-b7c38cfb-4e18-f631-7a42-dc33067a3fa0: Installed remote snapshot -2024-05-15T01:17:14.259Z [WARN] server-b7c38cfb-4e18-f631-7a42-dc33067a3fa0: skipping application of old log: index=22 -2024-05-15T01:17:14.259Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: pipelining replication: peer="{Voter server-b7c38cfb-4e18-f631-7a42-dc33067a3fa0 b7c38cfb-4e18-f631-7a42-dc33067a3fa0}" -2024-05-15T01:17:14.259Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:14.261Z [ERROR] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: failed to get log: index=23 error="log not found" -2024-05-15T01:17:14.262Z [INFO] creating new snapshot: path=/tmp/raft3607939196/snapshots/2-23-1715735834262.tmp -2024-05-15T01:17:14.262Z [INFO] server-96d8ff17-baba-ba12-a0e0-20ee285ba1ac: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:14.262Z [INFO] server-96d8ff17-baba-ba12-a0e0-20ee285ba1ac: copied to local snapshot: bytes=71 -2024-05-15T01:17:14.263Z [INFO] server-96d8ff17-baba-ba12-a0e0-20ee285ba1ac: snapshot restore progress: id=2-23-1715735834262 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:14.263Z [INFO] server-96d8ff17-baba-ba12-a0e0-20ee285ba1ac: Installed remote snapshot -2024-05-15T01:17:14.263Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: pipelining replication: peer="{Voter server-96d8ff17-baba-ba12-a0e0-20ee285ba1ac 96d8ff17-baba-ba12-a0e0-20ee285ba1ac}" -2024-05-15T01:17:15.264Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:14.259375264 +0000 UTC m=+14.800759502 (2 nodes), 0s from start of poll, 1.605683779s from cluster start. Timeout at 2024-05-15 01:17:15.26449925 +0000 UTC m=+15.805884032, 1.00512453s after stability -2024-05-15T01:17:15.277Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: aborting pipeline replication: peer="{Voter server-96d8ff17-baba-ba12-a0e0-20ee285ba1ac 96d8ff17-baba-ba12-a0e0-20ee285ba1ac}" -2024-05-15T01:17:15.277Z [INFO] server-cdb14442-3dd3-c6f1-43a8-2a5e7a9c4441: aborting pipeline replication: peer="{Voter server-b7c38cfb-4e18-f631-7a42-dc33067a3fa0 b7c38cfb-4e18-f631-7a42-dc33067a3fa0}" +2025-06-17T07:43:04.792Z [INFO] server-b5b26191-53fd-d408-85a7-61fb8bea87b3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b5b26191-53fd-d408-85a7-61fb8bea87b3 Address:b5b26191-53fd-d408-85a7-61fb8bea87b3} {Suffrage:Voter ID:server-2809de0f-8086-9521-b425-41e46563d5a9 Address:2809de0f-8086-9521-b425-41e46563d5a9} {Suffrage:Voter ID:server-5b8d36ca-3603-542c-47ad-1f6fe4ca3903 Address:5b8d36ca-3603-542c-47ad-1f6fe4ca3903}]" +2025-06-17T07:43:04.792Z [INFO] server-b5b26191-53fd-d408-85a7-61fb8bea87b3: entering follower state: follower="Node at b5b26191-53fd-d408-85a7-61fb8bea87b3 [Follower]" leader-address= leader-id= +2025-06-17T07:43:04.792Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b5b26191-53fd-d408-85a7-61fb8bea87b3 Address:b5b26191-53fd-d408-85a7-61fb8bea87b3} {Suffrage:Voter ID:server-2809de0f-8086-9521-b425-41e46563d5a9 Address:2809de0f-8086-9521-b425-41e46563d5a9} {Suffrage:Voter ID:server-5b8d36ca-3603-542c-47ad-1f6fe4ca3903 Address:5b8d36ca-3603-542c-47ad-1f6fe4ca3903}]" +2025-06-17T07:43:04.792Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: entering follower state: follower="Node at 2809de0f-8086-9521-b425-41e46563d5a9 [Follower]" leader-address= leader-id= +2025-06-17T07:43:04.792Z [INFO] server-5b8d36ca-3603-542c-47ad-1f6fe4ca3903: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b5b26191-53fd-d408-85a7-61fb8bea87b3 Address:b5b26191-53fd-d408-85a7-61fb8bea87b3} {Suffrage:Voter ID:server-2809de0f-8086-9521-b425-41e46563d5a9 Address:2809de0f-8086-9521-b425-41e46563d5a9} {Suffrage:Voter ID:server-5b8d36ca-3603-542c-47ad-1f6fe4ca3903 Address:5b8d36ca-3603-542c-47ad-1f6fe4ca3903}]" +2025-06-17T07:43:04.792Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:04.792Z [INFO] server-5b8d36ca-3603-542c-47ad-1f6fe4ca3903: entering follower state: follower="Node at 5b8d36ca-3603-542c-47ad-1f6fe4ca3903 [Follower]" leader-address= leader-id= +2025-06-17T07:43:05.494Z [WARN] server-2809de0f-8086-9521-b425-41e46563d5a9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:05.494Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: entering candidate state: node="Node at 2809de0f-8086-9521-b425-41e46563d5a9 [Candidate]" term=2 +2025-06-17T07:43:05.494Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: election won: term=2 tally=2 +2025-06-17T07:43:05.494Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: entering leader state: leader="Node at 2809de0f-8086-9521-b425-41e46563d5a9 [Leader]" +2025-06-17T07:43:05.494Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: added peer, starting replication: peer=server-b5b26191-53fd-d408-85a7-61fb8bea87b3 +2025-06-17T07:43:05.494Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: added peer, starting replication: peer=server-5b8d36ca-3603-542c-47ad-1f6fe4ca3903 +2025-06-17T07:43:05.494Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: pipelining replication: peer="{Voter server-b5b26191-53fd-d408-85a7-61fb8bea87b3 b5b26191-53fd-d408-85a7-61fb8bea87b3}" +2025-06-17T07:43:05.494Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: pipelining replication: peer="{Voter server-5b8d36ca-3603-542c-47ad-1f6fe4ca3903 5b8d36ca-3603-542c-47ad-1f6fe4ca3903}" +2025-06-17T07:43:06.499Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:05.494374389 +0000 UTC m=+14.097903107 (1 nodes), 701.579624ms from start of poll, 701.874076ms from cluster start. Timeout at 2025-06-17 07:43:06.499871673 +0000 UTC m=+15.103400364, 1.005497257s after stability +2025-06-17T07:43:06.500Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: starting snapshot up to: index=12 +2025-06-17T07:43:06.500Z [INFO] creating new snapshot: path=/tmp/raft2572183750/snapshots/2-12-1750146186500.tmp +2025-06-17T07:43:06.500Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: snapshot complete up to: index=12 +2025-06-17T07:43:06.501Z [INFO] creating new snapshot: path=/tmp/raft2572183750/snapshots/2-23-1750146186501.tmp +2025-06-17T07:43:06.501Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: copied to local snapshot: bytes=71 +2025-06-17T07:43:06.502Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: snapshot restore progress: id=2-23-1750146186501 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:06.502Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: restored user snapshot: index=23 +2025-06-17T07:43:06.502Z [ERROR] server-2809de0f-8086-9521-b425-41e46563d5a9: failed to get log: index=23 error="log not found" +2025-06-17T07:43:06.502Z [ERROR] server-2809de0f-8086-9521-b425-41e46563d5a9: failed to get log: index=23 error="log not found" +2025-06-17T07:43:06.502Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: aborting pipeline replication: peer="{Voter server-b5b26191-53fd-d408-85a7-61fb8bea87b3 b5b26191-53fd-d408-85a7-61fb8bea87b3}" +2025-06-17T07:43:06.502Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: aborting pipeline replication: peer="{Voter server-5b8d36ca-3603-542c-47ad-1f6fe4ca3903 5b8d36ca-3603-542c-47ad-1f6fe4ca3903}" +2025-06-17T07:43:06.511Z [ERROR] server-2809de0f-8086-9521-b425-41e46563d5a9: failed to get log: index=23 error="log not found" +2025-06-17T07:43:06.511Z [ERROR] server-2809de0f-8086-9521-b425-41e46563d5a9: failed to get log: index=23 error="log not found" +2025-06-17T07:43:06.512Z [INFO] creating new snapshot: path=/tmp/raft2716668931/snapshots/2-23-1750146186512.tmp +2025-06-17T07:43:06.512Z [INFO] creating new snapshot: path=/tmp/raft4294527437/snapshots/2-23-1750146186512.tmp +2025-06-17T07:43:06.512Z [INFO] server-5b8d36ca-3603-542c-47ad-1f6fe4ca3903: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:06.513Z [INFO] server-b5b26191-53fd-d408-85a7-61fb8bea87b3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:06.513Z [INFO] server-5b8d36ca-3603-542c-47ad-1f6fe4ca3903: copied to local snapshot: bytes=71 +2025-06-17T07:43:06.513Z [INFO] server-b5b26191-53fd-d408-85a7-61fb8bea87b3: copied to local snapshot: bytes=71 +2025-06-17T07:43:06.513Z [INFO] server-5b8d36ca-3603-542c-47ad-1f6fe4ca3903: snapshot restore progress: id=2-23-1750146186512 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:06.513Z [INFO] server-5b8d36ca-3603-542c-47ad-1f6fe4ca3903: Installed remote snapshot +2025-06-17T07:43:06.513Z [INFO] server-b5b26191-53fd-d408-85a7-61fb8bea87b3: snapshot restore progress: id=2-23-1750146186512 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:06.513Z [INFO] server-b5b26191-53fd-d408-85a7-61fb8bea87b3: Installed remote snapshot +2025-06-17T07:43:06.513Z [WARN] server-5b8d36ca-3603-542c-47ad-1f6fe4ca3903: skipping application of old log: index=22 +2025-06-17T07:43:06.513Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: pipelining replication: peer="{Voter server-5b8d36ca-3603-542c-47ad-1f6fe4ca3903 5b8d36ca-3603-542c-47ad-1f6fe4ca3903}" +2025-06-17T07:43:06.513Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:06.513Z [WARN] server-b5b26191-53fd-d408-85a7-61fb8bea87b3: skipping application of old log: index=22 +2025-06-17T07:43:06.513Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: pipelining replication: peer="{Voter server-b5b26191-53fd-d408-85a7-61fb8bea87b3 b5b26191-53fd-d408-85a7-61fb8bea87b3}" +2025-06-17T07:43:07.518Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:06.513733452 +0000 UTC m=+15.117262141 (2 nodes), 619ns from start of poll, 1.72123311s from cluster start. Timeout at 2025-06-17 07:43:07.518911344 +0000 UTC m=+16.122440028, 1.005177887s after stability +2025-06-17T07:43:07.531Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: aborting pipeline replication: peer="{Voter server-5b8d36ca-3603-542c-47ad-1f6fe4ca3903 5b8d36ca-3603-542c-47ad-1f6fe4ca3903}" +2025-06-17T07:43:07.531Z [INFO] server-2809de0f-8086-9521-b425-41e46563d5a9: aborting pipeline replication: peer="{Voter server-b5b26191-53fd-d408-85a7-61fb8bea87b3 b5b26191-53fd-d408-85a7-61fb8bea87b3}" === RUN TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false -2024-05-15T01:17:15.280Z [INFO] server-32ad5b19-8c01-a125-81a4-7b0577c4f996: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-32ad5b19-8c01-a125-81a4-7b0577c4f996 Address:32ad5b19-8c01-a125-81a4-7b0577c4f996} {Suffrage:Voter ID:server-91c027ba-27cc-f862-e03c-93418d1f90c9 Address:91c027ba-27cc-f862-e03c-93418d1f90c9} {Suffrage:Voter ID:server-a11df365-d116-25be-403f-5c6d7d530015 Address:a11df365-d116-25be-403f-5c6d7d530015}]" -2024-05-15T01:17:15.280Z [INFO] server-91c027ba-27cc-f862-e03c-93418d1f90c9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-32ad5b19-8c01-a125-81a4-7b0577c4f996 Address:32ad5b19-8c01-a125-81a4-7b0577c4f996} {Suffrage:Voter ID:server-91c027ba-27cc-f862-e03c-93418d1f90c9 Address:91c027ba-27cc-f862-e03c-93418d1f90c9} {Suffrage:Voter ID:server-a11df365-d116-25be-403f-5c6d7d530015 Address:a11df365-d116-25be-403f-5c6d7d530015}]" -2024-05-15T01:17:15.280Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-32ad5b19-8c01-a125-81a4-7b0577c4f996 Address:32ad5b19-8c01-a125-81a4-7b0577c4f996} {Suffrage:Voter ID:server-91c027ba-27cc-f862-e03c-93418d1f90c9 Address:91c027ba-27cc-f862-e03c-93418d1f90c9} {Suffrage:Voter ID:server-a11df365-d116-25be-403f-5c6d7d530015 Address:a11df365-d116-25be-403f-5c6d7d530015}]" -2024-05-15T01:17:15.280Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:15.282Z [INFO] server-91c027ba-27cc-f862-e03c-93418d1f90c9: entering follower state: follower="Node at 91c027ba-27cc-f862-e03c-93418d1f90c9 [Follower]" leader-address= leader-id= -2024-05-15T01:17:15.282Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: entering follower state: follower="Node at a11df365-d116-25be-403f-5c6d7d530015 [Follower]" leader-address= leader-id= -2024-05-15T01:17:15.282Z [INFO] server-32ad5b19-8c01-a125-81a4-7b0577c4f996: entering follower state: follower="Node at 32ad5b19-8c01-a125-81a4-7b0577c4f996 [Follower]" leader-address= leader-id= -2024-05-15T01:17:15.896Z [WARN] server-a11df365-d116-25be-403f-5c6d7d530015: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:15.896Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: entering candidate state: node="Node at a11df365-d116-25be-403f-5c6d7d530015 [Candidate]" term=2 -2024-05-15T01:17:15.896Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: election won: term=2 tally=2 -2024-05-15T01:17:15.897Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: entering leader state: leader="Node at a11df365-d116-25be-403f-5c6d7d530015 [Leader]" -2024-05-15T01:17:15.897Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: added peer, starting replication: peer=server-32ad5b19-8c01-a125-81a4-7b0577c4f996 -2024-05-15T01:17:15.897Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: added peer, starting replication: peer=server-91c027ba-27cc-f862-e03c-93418d1f90c9 -2024-05-15T01:17:15.897Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: pipelining replication: peer="{Voter server-91c027ba-27cc-f862-e03c-93418d1f90c9 91c027ba-27cc-f862-e03c-93418d1f90c9}" -2024-05-15T01:17:15.897Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: pipelining replication: peer="{Voter server-32ad5b19-8c01-a125-81a4-7b0577c4f996 32ad5b19-8c01-a125-81a4-7b0577c4f996}" -2024-05-15T01:17:16.902Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:15.897595673 +0000 UTC m=+16.438979911 (1 nodes), 616.799066ms from start of poll, 617.753721ms from cluster start. Timeout at 2024-05-15 01:17:16.902772642 +0000 UTC m=+17.444156880, 1.005176969s after stability -2024-05-15T01:17:16.904Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: starting snapshot up to: index=12 -2024-05-15T01:17:16.904Z [INFO] creating new snapshot: path=/tmp/raft2836445937/snapshots/2-12-1715735836904.tmp -2024-05-15T01:17:16.905Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: snapshot complete up to: index=12 -2024-05-15T01:17:16.907Z [INFO] creating new snapshot: path=/tmp/raft2836445937/snapshots/2-23-1715735836907.tmp -2024-05-15T01:17:16.907Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: copied to local snapshot: bytes=71 -2024-05-15T01:17:16.908Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: snapshot restore progress: id=2-23-1715735836907 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:16.908Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: removing all old logs from log store -2024-05-15T01:17:16.908Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: compacting logs: from=1 to=22 -2024-05-15T01:17:16.908Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: restored user snapshot: index=23 -2024-05-15T01:17:16.908Z [ERROR] server-a11df365-d116-25be-403f-5c6d7d530015: failed to get log: index=22 error="log not found" -2024-05-15T01:17:16.908Z [ERROR] server-a11df365-d116-25be-403f-5c6d7d530015: failed to get log: index=22 error="log not found" -2024-05-15T01:17:16.908Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: aborting pipeline replication: peer="{Voter server-32ad5b19-8c01-a125-81a4-7b0577c4f996 32ad5b19-8c01-a125-81a4-7b0577c4f996}" -2024-05-15T01:17:16.908Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: aborting pipeline replication: peer="{Voter server-91c027ba-27cc-f862-e03c-93418d1f90c9 91c027ba-27cc-f862-e03c-93418d1f90c9}" -2024-05-15T01:17:16.918Z [ERROR] server-a11df365-d116-25be-403f-5c6d7d530015: failed to get log: index=22 error="log not found" -2024-05-15T01:17:16.918Z [ERROR] server-a11df365-d116-25be-403f-5c6d7d530015: failed to get log: index=22 error="log not found" -2024-05-15T01:17:16.919Z [INFO] creating new snapshot: path=/tmp/raft3608125343/snapshots/2-23-1715735836919.tmp -2024-05-15T01:17:16.921Z [INFO] creating new snapshot: path=/tmp/raft250183118/snapshots/2-23-1715735836921.tmp -2024-05-15T01:17:16.922Z [INFO] server-91c027ba-27cc-f862-e03c-93418d1f90c9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:16.921Z [INFO] server-32ad5b19-8c01-a125-81a4-7b0577c4f996: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:16.922Z [INFO] server-91c027ba-27cc-f862-e03c-93418d1f90c9: copied to local snapshot: bytes=71 -2024-05-15T01:17:16.922Z [INFO] server-91c027ba-27cc-f862-e03c-93418d1f90c9: snapshot restore progress: id=2-23-1715735836921 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:16.922Z [INFO] server-91c027ba-27cc-f862-e03c-93418d1f90c9: removing all old logs from log store -2024-05-15T01:17:16.922Z [INFO] server-91c027ba-27cc-f862-e03c-93418d1f90c9: compacting logs: from=1 to=22 -2024-05-15T01:17:16.922Z [INFO] server-91c027ba-27cc-f862-e03c-93418d1f90c9: Installed remote snapshot -2024-05-15T01:17:16.922Z [INFO] server-32ad5b19-8c01-a125-81a4-7b0577c4f996: copied to local snapshot: bytes=71 -2024-05-15T01:17:16.923Z [WARN] server-91c027ba-27cc-f862-e03c-93418d1f90c9: skipping application of old log: index=22 -2024-05-15T01:17:16.923Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: pipelining replication: peer="{Voter server-91c027ba-27cc-f862-e03c-93418d1f90c9 91c027ba-27cc-f862-e03c-93418d1f90c9}" -2024-05-15T01:17:16.923Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:16.923Z [INFO] server-32ad5b19-8c01-a125-81a4-7b0577c4f996: snapshot restore progress: id=2-23-1715735836919 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:16.923Z [INFO] server-32ad5b19-8c01-a125-81a4-7b0577c4f996: removing all old logs from log store -2024-05-15T01:17:16.923Z [INFO] server-32ad5b19-8c01-a125-81a4-7b0577c4f996: compacting logs: from=1 to=22 -2024-05-15T01:17:16.923Z [INFO] server-32ad5b19-8c01-a125-81a4-7b0577c4f996: Installed remote snapshot -2024-05-15T01:17:16.924Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: pipelining replication: peer="{Voter server-32ad5b19-8c01-a125-81a4-7b0577c4f996 32ad5b19-8c01-a125-81a4-7b0577c4f996}" -2024-05-15T01:17:17.929Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:16.92343718 +0000 UTC m=+17.464822115 (2 nodes), 2.646µs from start of poll, 1.643595925s from cluster start. Timeout at 2024-05-15 01:17:17.929070779 +0000 UTC m=+18.470455692, 1.005633577s after stability -2024-05-15T01:17:17.941Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: aborting pipeline replication: peer="{Voter server-91c027ba-27cc-f862-e03c-93418d1f90c9 91c027ba-27cc-f862-e03c-93418d1f90c9}" -2024-05-15T01:17:17.942Z [INFO] server-a11df365-d116-25be-403f-5c6d7d530015: aborting pipeline replication: peer="{Voter server-32ad5b19-8c01-a125-81a4-7b0577c4f996 32ad5b19-8c01-a125-81a4-7b0577c4f996}" +2025-06-17T07:43:07.533Z [INFO] server-9c967c08-920a-c3b8-118e-7479bf558dd4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9c967c08-920a-c3b8-118e-7479bf558dd4 Address:9c967c08-920a-c3b8-118e-7479bf558dd4} {Suffrage:Voter ID:server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c Address:c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c} {Suffrage:Voter ID:server-238ad258-29eb-90f3-38e9-1aba685807dd Address:238ad258-29eb-90f3-38e9-1aba685807dd}]" +2025-06-17T07:43:07.533Z [INFO] server-9c967c08-920a-c3b8-118e-7479bf558dd4: entering follower state: follower="Node at 9c967c08-920a-c3b8-118e-7479bf558dd4 [Follower]" leader-address= leader-id= +2025-06-17T07:43:07.533Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9c967c08-920a-c3b8-118e-7479bf558dd4 Address:9c967c08-920a-c3b8-118e-7479bf558dd4} {Suffrage:Voter ID:server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c Address:c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c} {Suffrage:Voter ID:server-238ad258-29eb-90f3-38e9-1aba685807dd Address:238ad258-29eb-90f3-38e9-1aba685807dd}]" +2025-06-17T07:43:07.533Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: entering follower state: follower="Node at c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c [Follower]" leader-address= leader-id= +2025-06-17T07:43:07.533Z [INFO] server-238ad258-29eb-90f3-38e9-1aba685807dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9c967c08-920a-c3b8-118e-7479bf558dd4 Address:9c967c08-920a-c3b8-118e-7479bf558dd4} {Suffrage:Voter ID:server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c Address:c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c} {Suffrage:Voter ID:server-238ad258-29eb-90f3-38e9-1aba685807dd Address:238ad258-29eb-90f3-38e9-1aba685807dd}]" +2025-06-17T07:43:07.533Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:07.533Z [INFO] server-238ad258-29eb-90f3-38e9-1aba685807dd: entering follower state: follower="Node at 238ad258-29eb-90f3-38e9-1aba685807dd [Follower]" leader-address= leader-id= +2025-06-17T07:43:08.324Z [WARN] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:08.324Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: entering candidate state: node="Node at c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c [Candidate]" term=2 +2025-06-17T07:43:08.324Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: election won: term=2 tally=2 +2025-06-17T07:43:08.324Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: entering leader state: leader="Node at c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c [Leader]" +2025-06-17T07:43:08.324Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: added peer, starting replication: peer=server-9c967c08-920a-c3b8-118e-7479bf558dd4 +2025-06-17T07:43:08.324Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: added peer, starting replication: peer=server-238ad258-29eb-90f3-38e9-1aba685807dd +2025-06-17T07:43:08.325Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: pipelining replication: peer="{Voter server-238ad258-29eb-90f3-38e9-1aba685807dd 238ad258-29eb-90f3-38e9-1aba685807dd}" +2025-06-17T07:43:08.325Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: pipelining replication: peer="{Voter server-9c967c08-920a-c3b8-118e-7479bf558dd4 9c967c08-920a-c3b8-118e-7479bf558dd4}" +2025-06-17T07:43:09.330Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:08.324984947 +0000 UTC m=+16.928513672 (1 nodes), 791.122964ms from start of poll, 791.397579ms from cluster start. Timeout at 2025-06-17 07:43:09.330065621 +0000 UTC m=+17.933594304, 1.005080632s after stability +2025-06-17T07:43:09.331Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: starting snapshot up to: index=12 +2025-06-17T07:43:09.331Z [INFO] creating new snapshot: path=/tmp/raft4046153188/snapshots/2-12-1750146189331.tmp +2025-06-17T07:43:09.331Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: snapshot complete up to: index=12 +2025-06-17T07:43:09.332Z [INFO] creating new snapshot: path=/tmp/raft4046153188/snapshots/2-23-1750146189332.tmp +2025-06-17T07:43:09.333Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: copied to local snapshot: bytes=71 +2025-06-17T07:43:09.333Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: snapshot restore progress: id=2-23-1750146189332 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:09.333Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: removing all old logs from log store +2025-06-17T07:43:09.333Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: compacting logs: from=1 to=22 +2025-06-17T07:43:09.333Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: restored user snapshot: index=23 +2025-06-17T07:43:09.333Z [ERROR] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: failed to get log: index=22 error="log not found" +2025-06-17T07:43:09.333Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: aborting pipeline replication: peer="{Voter server-238ad258-29eb-90f3-38e9-1aba685807dd 238ad258-29eb-90f3-38e9-1aba685807dd}" +2025-06-17T07:43:09.333Z [ERROR] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: failed to get log: index=22 error="log not found" +2025-06-17T07:43:09.333Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: aborting pipeline replication: peer="{Voter server-9c967c08-920a-c3b8-118e-7479bf558dd4 9c967c08-920a-c3b8-118e-7479bf558dd4}" +2025-06-17T07:43:09.338Z [ERROR] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: failed to get log: index=22 error="log not found" +2025-06-17T07:43:09.339Z [INFO] creating new snapshot: path=/tmp/raft3933860234/snapshots/2-23-1750146189339.tmp +2025-06-17T07:43:09.339Z [INFO] server-238ad258-29eb-90f3-38e9-1aba685807dd: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:09.339Z [INFO] server-238ad258-29eb-90f3-38e9-1aba685807dd: copied to local snapshot: bytes=71 +2025-06-17T07:43:09.339Z [INFO] server-238ad258-29eb-90f3-38e9-1aba685807dd: snapshot restore progress: id=2-23-1750146189339 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:09.339Z [INFO] server-238ad258-29eb-90f3-38e9-1aba685807dd: removing all old logs from log store +2025-06-17T07:43:09.339Z [INFO] server-238ad258-29eb-90f3-38e9-1aba685807dd: compacting logs: from=1 to=22 +2025-06-17T07:43:09.339Z [INFO] server-238ad258-29eb-90f3-38e9-1aba685807dd: Installed remote snapshot +2025-06-17T07:43:09.339Z [WARN] server-238ad258-29eb-90f3-38e9-1aba685807dd: skipping application of old log: index=22 +2025-06-17T07:43:09.339Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: pipelining replication: peer="{Voter server-238ad258-29eb-90f3-38e9-1aba685807dd 238ad258-29eb-90f3-38e9-1aba685807dd}" +2025-06-17T07:43:09.339Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:09.340Z [ERROR] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: failed to get log: index=22 error="log not found" +2025-06-17T07:43:09.341Z [INFO] creating new snapshot: path=/tmp/raft3011322827/snapshots/2-23-1750146189341.tmp +2025-06-17T07:43:09.341Z [INFO] server-9c967c08-920a-c3b8-118e-7479bf558dd4: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:09.341Z [INFO] server-9c967c08-920a-c3b8-118e-7479bf558dd4: copied to local snapshot: bytes=71 +2025-06-17T07:43:09.341Z [INFO] server-9c967c08-920a-c3b8-118e-7479bf558dd4: snapshot restore progress: id=2-23-1750146189341 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:09.341Z [INFO] server-9c967c08-920a-c3b8-118e-7479bf558dd4: removing all old logs from log store +2025-06-17T07:43:09.341Z [INFO] server-9c967c08-920a-c3b8-118e-7479bf558dd4: compacting logs: from=1 to=22 +2025-06-17T07:43:09.341Z [INFO] server-9c967c08-920a-c3b8-118e-7479bf558dd4: Installed remote snapshot +2025-06-17T07:43:09.342Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: pipelining replication: peer="{Voter server-9c967c08-920a-c3b8-118e-7479bf558dd4 9c967c08-920a-c3b8-118e-7479bf558dd4}" +2025-06-17T07:43:10.345Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:09.339832394 +0000 UTC m=+17.943361081 (2 nodes), 650ns from start of poll, 1.806244988s from cluster start. Timeout at 2025-06-17 07:43:10.345854104 +0000 UTC m=+18.949382786, 1.006021705s after stability +2025-06-17T07:43:10.358Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: aborting pipeline replication: peer="{Voter server-238ad258-29eb-90f3-38e9-1aba685807dd 238ad258-29eb-90f3-38e9-1aba685807dd}" +2025-06-17T07:43:10.358Z [INFO] server-c1f2f7a4-54b2-7d5c-ba83-dea5f84c6d2c: aborting pipeline replication: peer="{Voter server-9c967c08-920a-c3b8-118e-7479bf558dd4 9c967c08-920a-c3b8-118e-7479bf558dd4}" === RUN TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true -2024-05-15T01:17:17.944Z [INFO] server-b82bd7e0-18d8-5e30-da29-41ad7281edea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b82bd7e0-18d8-5e30-da29-41ad7281edea Address:b82bd7e0-18d8-5e30-da29-41ad7281edea} {Suffrage:Voter ID:server-2d3be6e3-c209-902e-6f3d-490515995df4 Address:2d3be6e3-c209-902e-6f3d-490515995df4} {Suffrage:Voter ID:server-21b54f48-26f4-63e7-81ea-aeeaa4820532 Address:21b54f48-26f4-63e7-81ea-aeeaa4820532}]" -2024-05-15T01:17:17.944Z [INFO] server-2d3be6e3-c209-902e-6f3d-490515995df4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b82bd7e0-18d8-5e30-da29-41ad7281edea Address:b82bd7e0-18d8-5e30-da29-41ad7281edea} {Suffrage:Voter ID:server-2d3be6e3-c209-902e-6f3d-490515995df4 Address:2d3be6e3-c209-902e-6f3d-490515995df4} {Suffrage:Voter ID:server-21b54f48-26f4-63e7-81ea-aeeaa4820532 Address:21b54f48-26f4-63e7-81ea-aeeaa4820532}]" -2024-05-15T01:17:17.944Z [INFO] server-21b54f48-26f4-63e7-81ea-aeeaa4820532: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b82bd7e0-18d8-5e30-da29-41ad7281edea Address:b82bd7e0-18d8-5e30-da29-41ad7281edea} {Suffrage:Voter ID:server-2d3be6e3-c209-902e-6f3d-490515995df4 Address:2d3be6e3-c209-902e-6f3d-490515995df4} {Suffrage:Voter ID:server-21b54f48-26f4-63e7-81ea-aeeaa4820532 Address:21b54f48-26f4-63e7-81ea-aeeaa4820532}]" -2024-05-15T01:17:17.944Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:17.944Z [INFO] server-b82bd7e0-18d8-5e30-da29-41ad7281edea: entering follower state: follower="Node at b82bd7e0-18d8-5e30-da29-41ad7281edea [Follower]" leader-address= leader-id= -2024-05-15T01:17:17.944Z [INFO] server-2d3be6e3-c209-902e-6f3d-490515995df4: entering follower state: follower="Node at 2d3be6e3-c209-902e-6f3d-490515995df4 [Follower]" leader-address= leader-id= -2024-05-15T01:17:17.944Z [INFO] server-21b54f48-26f4-63e7-81ea-aeeaa4820532: entering follower state: follower="Node at 21b54f48-26f4-63e7-81ea-aeeaa4820532 [Follower]" leader-address= leader-id= -2024-05-15T01:17:18.677Z [WARN] server-21b54f48-26f4-63e7-81ea-aeeaa4820532: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:18.677Z [INFO] server-21b54f48-26f4-63e7-81ea-aeeaa4820532: entering candidate state: node="Node at 21b54f48-26f4-63e7-81ea-aeeaa4820532 [Candidate]" term=2 -2024-05-15T01:17:18.677Z [INFO] server-21b54f48-26f4-63e7-81ea-aeeaa4820532: election won: term=2 tally=2 -2024-05-15T01:17:18.677Z [INFO] server-21b54f48-26f4-63e7-81ea-aeeaa4820532: entering leader state: leader="Node at 21b54f48-26f4-63e7-81ea-aeeaa4820532 [Leader]" -2024-05-15T01:17:18.677Z [INFO] server-21b54f48-26f4-63e7-81ea-aeeaa4820532: added peer, starting replication: peer=server-b82bd7e0-18d8-5e30-da29-41ad7281edea -2024-05-15T01:17:18.677Z [INFO] server-21b54f48-26f4-63e7-81ea-aeeaa4820532: added peer, starting replication: peer=server-2d3be6e3-c209-902e-6f3d-490515995df4 -2024-05-15T01:17:18.678Z [INFO] server-21b54f48-26f4-63e7-81ea-aeeaa4820532: pipelining replication: peer="{Voter server-2d3be6e3-c209-902e-6f3d-490515995df4 2d3be6e3-c209-902e-6f3d-490515995df4}" -2024-05-15T01:17:18.678Z [INFO] server-21b54f48-26f4-63e7-81ea-aeeaa4820532: pipelining replication: peer="{Voter server-b82bd7e0-18d8-5e30-da29-41ad7281edea b82bd7e0-18d8-5e30-da29-41ad7281edea}" -2024-05-15T01:17:19.684Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:18.678363334 +0000 UTC m=+19.219748092 (1 nodes), 733.679718ms from start of poll, 734.237183ms from cluster start. Timeout at 2024-05-15 01:17:19.684076952 +0000 UTC m=+20.225461708, 1.005713616s after stability -2024-05-15T01:17:19.685Z [INFO] server-21b54f48-26f4-63e7-81ea-aeeaa4820532: starting snapshot up to: index=12 -2024-05-15T01:17:19.685Z [INFO] creating new snapshot: path=/tmp/raft2142762904/snapshots/2-12-1715735839685.tmp -2024-05-15T01:17:19.686Z [INFO] server-21b54f48-26f4-63e7-81ea-aeeaa4820532: snapshot complete up to: index=12 -2024-05-15T01:17:19.689Z [INFO] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8 Address:16f7e4db-c763-9a11-b4cb-074c1b1b82e8} {Suffrage:Voter ID:server-fbe633fc-bfd4-317f-2061-05c643b8345f Address:fbe633fc-bfd4-317f-2061-05c643b8345f} {Suffrage:Voter ID:server-33d51e77-803b-719b-23a2-f2f74af19ab0 Address:33d51e77-803b-719b-23a2-f2f74af19ab0}]" -2024-05-15T01:17:19.690Z [INFO] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: entering follower state: follower="Node at 16f7e4db-c763-9a11-b4cb-074c1b1b82e8 [Follower]" leader-address= leader-id= -2024-05-15T01:17:19.690Z [INFO] server-fbe633fc-bfd4-317f-2061-05c643b8345f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8 Address:16f7e4db-c763-9a11-b4cb-074c1b1b82e8} {Suffrage:Voter ID:server-fbe633fc-bfd4-317f-2061-05c643b8345f Address:fbe633fc-bfd4-317f-2061-05c643b8345f} {Suffrage:Voter ID:server-33d51e77-803b-719b-23a2-f2f74af19ab0 Address:33d51e77-803b-719b-23a2-f2f74af19ab0}]" -2024-05-15T01:17:19.690Z [INFO] server-fbe633fc-bfd4-317f-2061-05c643b8345f: entering follower state: follower="Node at fbe633fc-bfd4-317f-2061-05c643b8345f [Follower]" leader-address= leader-id= -2024-05-15T01:17:19.690Z [INFO] server-33d51e77-803b-719b-23a2-f2f74af19ab0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8 Address:16f7e4db-c763-9a11-b4cb-074c1b1b82e8} {Suffrage:Voter ID:server-fbe633fc-bfd4-317f-2061-05c643b8345f Address:fbe633fc-bfd4-317f-2061-05c643b8345f} {Suffrage:Voter ID:server-33d51e77-803b-719b-23a2-f2f74af19ab0 Address:33d51e77-803b-719b-23a2-f2f74af19ab0}]" -2024-05-15T01:17:19.690Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:19.690Z [INFO] server-33d51e77-803b-719b-23a2-f2f74af19ab0: entering follower state: follower="Node at 33d51e77-803b-719b-23a2-f2f74af19ab0 [Follower]" leader-address= leader-id= -2024-05-15T01:17:20.333Z [WARN] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:20.333Z [INFO] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: entering candidate state: node="Node at 16f7e4db-c763-9a11-b4cb-074c1b1b82e8 [Candidate]" term=2 -2024-05-15T01:17:20.333Z [INFO] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: election won: term=2 tally=2 -2024-05-15T01:17:20.333Z [INFO] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: entering leader state: leader="Node at 16f7e4db-c763-9a11-b4cb-074c1b1b82e8 [Leader]" -2024-05-15T01:17:20.334Z [INFO] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: added peer, starting replication: peer=server-fbe633fc-bfd4-317f-2061-05c643b8345f -2024-05-15T01:17:20.334Z [INFO] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: added peer, starting replication: peer=server-33d51e77-803b-719b-23a2-f2f74af19ab0 -2024-05-15T01:17:20.334Z [INFO] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: pipelining replication: peer="{Voter server-33d51e77-803b-719b-23a2-f2f74af19ab0 33d51e77-803b-719b-23a2-f2f74af19ab0}" -2024-05-15T01:17:20.334Z [INFO] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: pipelining replication: peer="{Voter server-fbe633fc-bfd4-317f-2061-05c643b8345f fbe633fc-bfd4-317f-2061-05c643b8345f}" -2024-05-15T01:17:21.339Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:20.334503261 +0000 UTC m=+20.875887499 (1 nodes), 644.185406ms from start of poll, 644.730841ms from cluster start. Timeout at 2024-05-15 01:17:21.339873591 +0000 UTC m=+21.881258361, 1.005370862s after stability -2024-05-15T01:17:21.340Z [INFO] creating new snapshot: path=/tmp/raft1616569903/snapshots/2-13-1715735841340.tmp -2024-05-15T01:17:21.340Z [INFO] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: copied to local snapshot: bytes=71 -2024-05-15T01:17:21.341Z [INFO] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: snapshot restore progress: id=2-13-1715735841340 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:21.341Z [INFO] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: restored user snapshot: index=13 -2024-05-15T01:17:21.341Z [ERROR] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: failed to get log: index=3 error="log not found" -2024-05-15T01:17:21.341Z [INFO] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: aborting pipeline replication: peer="{Voter server-33d51e77-803b-719b-23a2-f2f74af19ab0 33d51e77-803b-719b-23a2-f2f74af19ab0}" -2024-05-15T01:17:21.341Z [ERROR] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: failed to get log: index=3 error="log not found" -2024-05-15T01:17:21.341Z [INFO] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: aborting pipeline replication: peer="{Voter server-fbe633fc-bfd4-317f-2061-05c643b8345f fbe633fc-bfd4-317f-2061-05c643b8345f}" -2024-05-15T01:17:21.347Z [ERROR] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: failed to get log: index=3 error="log not found" -2024-05-15T01:17:21.347Z [INFO] creating new snapshot: path=/tmp/raft2641015910/snapshots/2-13-1715735841347.tmp -2024-05-15T01:17:21.348Z [INFO] server-33d51e77-803b-719b-23a2-f2f74af19ab0: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:21.348Z [INFO] server-33d51e77-803b-719b-23a2-f2f74af19ab0: copied to local snapshot: bytes=71 -2024-05-15T01:17:21.348Z [INFO] server-33d51e77-803b-719b-23a2-f2f74af19ab0: snapshot restore progress: id=2-13-1715735841347 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:21.348Z [INFO] server-33d51e77-803b-719b-23a2-f2f74af19ab0: Installed remote snapshot -2024-05-15T01:17:21.348Z [INFO] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: pipelining replication: peer="{Voter server-33d51e77-803b-719b-23a2-f2f74af19ab0 33d51e77-803b-719b-23a2-f2f74af19ab0}" -2024-05-15T01:17:21.348Z [ERROR] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: failed to get log: index=3 error="log not found" -2024-05-15T01:17:21.349Z [INFO] creating new snapshot: path=/tmp/raft4095998925/snapshots/2-13-1715735841349.tmp -2024-05-15T01:17:21.349Z [INFO] server-fbe633fc-bfd4-317f-2061-05c643b8345f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:21.349Z [INFO] server-fbe633fc-bfd4-317f-2061-05c643b8345f: copied to local snapshot: bytes=71 -2024-05-15T01:17:21.350Z [INFO] server-fbe633fc-bfd4-317f-2061-05c643b8345f: snapshot restore progress: id=2-13-1715735841349 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:21.350Z [INFO] server-fbe633fc-bfd4-317f-2061-05c643b8345f: Installed remote snapshot -2024-05-15T01:17:21.350Z [INFO] server-16f7e4db-c763-9a11-b4cb-074c1b1b82e8: pipelining replication: peer="{Voter server-fbe633fc-bfd4-317f-2061-05c643b8345f fbe633fc-bfd4-317f-2061-05c643b8345f}" -2024-05-15T01:17:21.350Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:22.355Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:21.350472608 +0000 UTC m=+21.891857351 (2 nodes), 2.03µs from start of poll, 1.660700693s from cluster start. Timeout at 2024-05-15 01:17:22.355663401 +0000 UTC m=+22.897048134, 1.005190783s after stability -2024-05-15T01:17:22.369Z [INFO] server-21b54f48-26f4-63e7-81ea-aeeaa4820532: aborting pipeline replication: peer="{Voter server-2d3be6e3-c209-902e-6f3d-490515995df4 2d3be6e3-c209-902e-6f3d-490515995df4}" -2024-05-15T01:17:22.369Z [INFO] server-21b54f48-26f4-63e7-81ea-aeeaa4820532: aborting pipeline replication: peer="{Voter server-b82bd7e0-18d8-5e30-da29-41ad7281edea b82bd7e0-18d8-5e30-da29-41ad7281edea}" +2025-06-17T07:43:10.359Z [INFO] server-4896f5d8-73de-9752-da25-d37d402f5951: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4896f5d8-73de-9752-da25-d37d402f5951 Address:4896f5d8-73de-9752-da25-d37d402f5951} {Suffrage:Voter ID:server-53708e8b-85b6-d3bd-8db9-e046563bb0f2 Address:53708e8b-85b6-d3bd-8db9-e046563bb0f2} {Suffrage:Voter ID:server-7c080e07-1fd1-bdbb-70cc-7413164595d0 Address:7c080e07-1fd1-bdbb-70cc-7413164595d0}]" +2025-06-17T07:43:10.359Z [INFO] server-4896f5d8-73de-9752-da25-d37d402f5951: entering follower state: follower="Node at 4896f5d8-73de-9752-da25-d37d402f5951 [Follower]" leader-address= leader-id= +2025-06-17T07:43:10.359Z [INFO] server-53708e8b-85b6-d3bd-8db9-e046563bb0f2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4896f5d8-73de-9752-da25-d37d402f5951 Address:4896f5d8-73de-9752-da25-d37d402f5951} {Suffrage:Voter ID:server-53708e8b-85b6-d3bd-8db9-e046563bb0f2 Address:53708e8b-85b6-d3bd-8db9-e046563bb0f2} {Suffrage:Voter ID:server-7c080e07-1fd1-bdbb-70cc-7413164595d0 Address:7c080e07-1fd1-bdbb-70cc-7413164595d0}]" +2025-06-17T07:43:10.359Z [INFO] server-53708e8b-85b6-d3bd-8db9-e046563bb0f2: entering follower state: follower="Node at 53708e8b-85b6-d3bd-8db9-e046563bb0f2 [Follower]" leader-address= leader-id= +2025-06-17T07:43:10.359Z [INFO] server-7c080e07-1fd1-bdbb-70cc-7413164595d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4896f5d8-73de-9752-da25-d37d402f5951 Address:4896f5d8-73de-9752-da25-d37d402f5951} {Suffrage:Voter ID:server-53708e8b-85b6-d3bd-8db9-e046563bb0f2 Address:53708e8b-85b6-d3bd-8db9-e046563bb0f2} {Suffrage:Voter ID:server-7c080e07-1fd1-bdbb-70cc-7413164595d0 Address:7c080e07-1fd1-bdbb-70cc-7413164595d0}]" +2025-06-17T07:43:10.359Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:10.359Z [INFO] server-7c080e07-1fd1-bdbb-70cc-7413164595d0: entering follower state: follower="Node at 7c080e07-1fd1-bdbb-70cc-7413164595d0 [Follower]" leader-address= leader-id= +2025-06-17T07:43:10.896Z [WARN] server-4896f5d8-73de-9752-da25-d37d402f5951: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:10.896Z [INFO] server-4896f5d8-73de-9752-da25-d37d402f5951: entering candidate state: node="Node at 4896f5d8-73de-9752-da25-d37d402f5951 [Candidate]" term=2 +2025-06-17T07:43:10.896Z [INFO] server-4896f5d8-73de-9752-da25-d37d402f5951: election won: term=2 tally=2 +2025-06-17T07:43:10.896Z [INFO] server-4896f5d8-73de-9752-da25-d37d402f5951: entering leader state: leader="Node at 4896f5d8-73de-9752-da25-d37d402f5951 [Leader]" +2025-06-17T07:43:10.896Z [INFO] server-4896f5d8-73de-9752-da25-d37d402f5951: added peer, starting replication: peer=server-53708e8b-85b6-d3bd-8db9-e046563bb0f2 +2025-06-17T07:43:10.896Z [INFO] server-4896f5d8-73de-9752-da25-d37d402f5951: added peer, starting replication: peer=server-7c080e07-1fd1-bdbb-70cc-7413164595d0 +2025-06-17T07:43:10.896Z [INFO] server-4896f5d8-73de-9752-da25-d37d402f5951: pipelining replication: peer="{Voter server-53708e8b-85b6-d3bd-8db9-e046563bb0f2 53708e8b-85b6-d3bd-8db9-e046563bb0f2}" +2025-06-17T07:43:10.896Z [INFO] server-4896f5d8-73de-9752-da25-d37d402f5951: pipelining replication: peer="{Voter server-7c080e07-1fd1-bdbb-70cc-7413164595d0 7c080e07-1fd1-bdbb-70cc-7413164595d0}" +2025-06-17T07:43:11.901Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:10.896224677 +0000 UTC m=+19.499753393 (1 nodes), 536.738451ms from start of poll, 537.016529ms from cluster start. Timeout at 2025-06-17 07:43:11.901637118 +0000 UTC m=+20.505165805, 1.005412412s after stability +2025-06-17T07:43:11.902Z [INFO] server-4896f5d8-73de-9752-da25-d37d402f5951: starting snapshot up to: index=12 +2025-06-17T07:43:11.902Z [INFO] creating new snapshot: path=/tmp/raft1052765205/snapshots/2-12-1750146191902.tmp +2025-06-17T07:43:11.903Z [INFO] server-4896f5d8-73de-9752-da25-d37d402f5951: snapshot complete up to: index=12 +2025-06-17T07:43:11.904Z [INFO] server-f531adc0-87e0-1df2-4d51-0197aa06d3d8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f531adc0-87e0-1df2-4d51-0197aa06d3d8 Address:f531adc0-87e0-1df2-4d51-0197aa06d3d8} {Suffrage:Voter ID:server-c6187a6d-0380-943b-edbe-826551ad2226 Address:c6187a6d-0380-943b-edbe-826551ad2226} {Suffrage:Voter ID:server-6b40e24b-2ff2-e3a9-b85d-4a387748d04c Address:6b40e24b-2ff2-e3a9-b85d-4a387748d04c}]" +2025-06-17T07:43:11.904Z [INFO] server-f531adc0-87e0-1df2-4d51-0197aa06d3d8: entering follower state: follower="Node at f531adc0-87e0-1df2-4d51-0197aa06d3d8 [Follower]" leader-address= leader-id= +2025-06-17T07:43:11.904Z [INFO] server-c6187a6d-0380-943b-edbe-826551ad2226: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f531adc0-87e0-1df2-4d51-0197aa06d3d8 Address:f531adc0-87e0-1df2-4d51-0197aa06d3d8} {Suffrage:Voter ID:server-c6187a6d-0380-943b-edbe-826551ad2226 Address:c6187a6d-0380-943b-edbe-826551ad2226} {Suffrage:Voter ID:server-6b40e24b-2ff2-e3a9-b85d-4a387748d04c Address:6b40e24b-2ff2-e3a9-b85d-4a387748d04c}]" +2025-06-17T07:43:11.904Z [INFO] server-c6187a6d-0380-943b-edbe-826551ad2226: entering follower state: follower="Node at c6187a6d-0380-943b-edbe-826551ad2226 [Follower]" leader-address= leader-id= +2025-06-17T07:43:11.904Z [INFO] server-6b40e24b-2ff2-e3a9-b85d-4a387748d04c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f531adc0-87e0-1df2-4d51-0197aa06d3d8 Address:f531adc0-87e0-1df2-4d51-0197aa06d3d8} {Suffrage:Voter ID:server-c6187a6d-0380-943b-edbe-826551ad2226 Address:c6187a6d-0380-943b-edbe-826551ad2226} {Suffrage:Voter ID:server-6b40e24b-2ff2-e3a9-b85d-4a387748d04c Address:6b40e24b-2ff2-e3a9-b85d-4a387748d04c}]" +2025-06-17T07:43:11.904Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:11.904Z [INFO] server-6b40e24b-2ff2-e3a9-b85d-4a387748d04c: entering follower state: follower="Node at 6b40e24b-2ff2-e3a9-b85d-4a387748d04c [Follower]" leader-address= leader-id= +2025-06-17T07:43:12.533Z [WARN] server-c6187a6d-0380-943b-edbe-826551ad2226: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:12.533Z [INFO] server-c6187a6d-0380-943b-edbe-826551ad2226: entering candidate state: node="Node at c6187a6d-0380-943b-edbe-826551ad2226 [Candidate]" term=2 +2025-06-17T07:43:12.533Z [INFO] server-c6187a6d-0380-943b-edbe-826551ad2226: election won: term=2 tally=2 +2025-06-17T07:43:12.533Z [INFO] server-c6187a6d-0380-943b-edbe-826551ad2226: entering leader state: leader="Node at c6187a6d-0380-943b-edbe-826551ad2226 [Leader]" +2025-06-17T07:43:12.533Z [INFO] server-c6187a6d-0380-943b-edbe-826551ad2226: added peer, starting replication: peer=server-f531adc0-87e0-1df2-4d51-0197aa06d3d8 +2025-06-17T07:43:12.533Z [INFO] server-c6187a6d-0380-943b-edbe-826551ad2226: added peer, starting replication: peer=server-6b40e24b-2ff2-e3a9-b85d-4a387748d04c +2025-06-17T07:43:12.533Z [INFO] server-c6187a6d-0380-943b-edbe-826551ad2226: pipelining replication: peer="{Voter server-6b40e24b-2ff2-e3a9-b85d-4a387748d04c 6b40e24b-2ff2-e3a9-b85d-4a387748d04c}" +2025-06-17T07:43:12.533Z [INFO] server-c6187a6d-0380-943b-edbe-826551ad2226: pipelining replication: peer="{Voter server-f531adc0-87e0-1df2-4d51-0197aa06d3d8 f531adc0-87e0-1df2-4d51-0197aa06d3d8}" +2025-06-17T07:43:13.538Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:12.533203467 +0000 UTC m=+21.136732184 (1 nodes), 628.985536ms from start of poll, 629.286927ms from cluster start. Timeout at 2025-06-17 07:43:13.538771348 +0000 UTC m=+22.142300030, 1.005567846s after stability +2025-06-17T07:43:13.539Z [INFO] creating new snapshot: path=/tmp/raft3249592807/snapshots/2-13-1750146193539.tmp +2025-06-17T07:43:13.539Z [INFO] server-c6187a6d-0380-943b-edbe-826551ad2226: copied to local snapshot: bytes=71 +2025-06-17T07:43:13.539Z [INFO] server-c6187a6d-0380-943b-edbe-826551ad2226: snapshot restore progress: id=2-13-1750146193539 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:13.539Z [INFO] server-c6187a6d-0380-943b-edbe-826551ad2226: restored user snapshot: index=13 +2025-06-17T07:43:13.539Z [ERROR] server-c6187a6d-0380-943b-edbe-826551ad2226: failed to get log: index=3 error="log not found" +2025-06-17T07:43:13.539Z [INFO] server-c6187a6d-0380-943b-edbe-826551ad2226: aborting pipeline replication: peer="{Voter server-6b40e24b-2ff2-e3a9-b85d-4a387748d04c 6b40e24b-2ff2-e3a9-b85d-4a387748d04c}" +2025-06-17T07:43:13.539Z [ERROR] server-c6187a6d-0380-943b-edbe-826551ad2226: failed to get log: index=3 error="log not found" +2025-06-17T07:43:13.539Z [INFO] server-c6187a6d-0380-943b-edbe-826551ad2226: aborting pipeline replication: peer="{Voter server-f531adc0-87e0-1df2-4d51-0197aa06d3d8 f531adc0-87e0-1df2-4d51-0197aa06d3d8}" +2025-06-17T07:43:13.548Z [ERROR] server-c6187a6d-0380-943b-edbe-826551ad2226: failed to get log: index=3 error="log not found" +2025-06-17T07:43:13.548Z [INFO] creating new snapshot: path=/tmp/raft3854787771/snapshots/2-13-1750146193548.tmp +2025-06-17T07:43:13.548Z [INFO] server-f531adc0-87e0-1df2-4d51-0197aa06d3d8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:13.549Z [INFO] server-f531adc0-87e0-1df2-4d51-0197aa06d3d8: copied to local snapshot: bytes=71 +2025-06-17T07:43:13.549Z [ERROR] server-c6187a6d-0380-943b-edbe-826551ad2226: failed to get log: index=3 error="log not found" +2025-06-17T07:43:13.549Z [INFO] server-f531adc0-87e0-1df2-4d51-0197aa06d3d8: snapshot restore progress: id=2-13-1750146193548 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:13.549Z [INFO] server-f531adc0-87e0-1df2-4d51-0197aa06d3d8: Installed remote snapshot +2025-06-17T07:43:13.549Z [INFO] creating new snapshot: path=/tmp/raft567853058/snapshots/2-13-1750146193549.tmp +2025-06-17T07:43:13.549Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:13.549Z [INFO] server-c6187a6d-0380-943b-edbe-826551ad2226: pipelining replication: peer="{Voter server-f531adc0-87e0-1df2-4d51-0197aa06d3d8 f531adc0-87e0-1df2-4d51-0197aa06d3d8}" +2025-06-17T07:43:13.549Z [INFO] server-6b40e24b-2ff2-e3a9-b85d-4a387748d04c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:13.549Z [INFO] server-6b40e24b-2ff2-e3a9-b85d-4a387748d04c: copied to local snapshot: bytes=71 +2025-06-17T07:43:13.550Z [INFO] server-6b40e24b-2ff2-e3a9-b85d-4a387748d04c: snapshot restore progress: id=2-13-1750146193549 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:13.550Z [INFO] server-6b40e24b-2ff2-e3a9-b85d-4a387748d04c: Installed remote snapshot +2025-06-17T07:43:13.550Z [INFO] server-c6187a6d-0380-943b-edbe-826551ad2226: pipelining replication: peer="{Voter server-6b40e24b-2ff2-e3a9-b85d-4a387748d04c 6b40e24b-2ff2-e3a9-b85d-4a387748d04c}" +2025-06-17T07:43:14.554Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:13.549458071 +0000 UTC m=+22.152986776 (2 nodes), 712ns from start of poll, 1.645541519s from cluster start. Timeout at 2025-06-17 07:43:14.55481219 +0000 UTC m=+23.158340884, 1.005354108s after stability +2025-06-17T07:43:14.566Z [INFO] server-4896f5d8-73de-9752-da25-d37d402f5951: aborting pipeline replication: peer="{Voter server-53708e8b-85b6-d3bd-8db9-e046563bb0f2 53708e8b-85b6-d3bd-8db9-e046563bb0f2}" +2025-06-17T07:43:14.566Z [INFO] server-4896f5d8-73de-9752-da25-d37d402f5951: aborting pipeline replication: peer="{Voter server-7c080e07-1fd1-bdbb-70cc-7413164595d0 7c080e07-1fd1-bdbb-70cc-7413164595d0}" === RUN TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true -2024-05-15T01:17:22.371Z [INFO] server-4d72f09f-28d6-d6ae-3463-40d25dd27338: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d72f09f-28d6-d6ae-3463-40d25dd27338 Address:4d72f09f-28d6-d6ae-3463-40d25dd27338} {Suffrage:Voter ID:server-476372f9-f23a-6fec-0eb7-bbf1318ba96f Address:476372f9-f23a-6fec-0eb7-bbf1318ba96f} {Suffrage:Voter ID:server-d4a77520-5e02-6e84-f2dd-1194c037733d Address:d4a77520-5e02-6e84-f2dd-1194c037733d}]" -2024-05-15T01:17:22.371Z [INFO] server-476372f9-f23a-6fec-0eb7-bbf1318ba96f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d72f09f-28d6-d6ae-3463-40d25dd27338 Address:4d72f09f-28d6-d6ae-3463-40d25dd27338} {Suffrage:Voter ID:server-476372f9-f23a-6fec-0eb7-bbf1318ba96f Address:476372f9-f23a-6fec-0eb7-bbf1318ba96f} {Suffrage:Voter ID:server-d4a77520-5e02-6e84-f2dd-1194c037733d Address:d4a77520-5e02-6e84-f2dd-1194c037733d}]" -2024-05-15T01:17:22.371Z [INFO] server-d4a77520-5e02-6e84-f2dd-1194c037733d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d72f09f-28d6-d6ae-3463-40d25dd27338 Address:4d72f09f-28d6-d6ae-3463-40d25dd27338} {Suffrage:Voter ID:server-476372f9-f23a-6fec-0eb7-bbf1318ba96f Address:476372f9-f23a-6fec-0eb7-bbf1318ba96f} {Suffrage:Voter ID:server-d4a77520-5e02-6e84-f2dd-1194c037733d Address:d4a77520-5e02-6e84-f2dd-1194c037733d}]" -2024-05-15T01:17:22.371Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:22.371Z [INFO] server-4d72f09f-28d6-d6ae-3463-40d25dd27338: entering follower state: follower="Node at 4d72f09f-28d6-d6ae-3463-40d25dd27338 [Follower]" leader-address= leader-id= -2024-05-15T01:17:22.371Z [INFO] server-476372f9-f23a-6fec-0eb7-bbf1318ba96f: entering follower state: follower="Node at 476372f9-f23a-6fec-0eb7-bbf1318ba96f [Follower]" leader-address= leader-id= -2024-05-15T01:17:22.371Z [INFO] server-d4a77520-5e02-6e84-f2dd-1194c037733d: entering follower state: follower="Node at d4a77520-5e02-6e84-f2dd-1194c037733d [Follower]" leader-address= leader-id= -2024-05-15T01:17:22.889Z [WARN] server-4d72f09f-28d6-d6ae-3463-40d25dd27338: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:22.889Z [INFO] server-4d72f09f-28d6-d6ae-3463-40d25dd27338: entering candidate state: node="Node at 4d72f09f-28d6-d6ae-3463-40d25dd27338 [Candidate]" term=2 -2024-05-15T01:17:22.889Z [INFO] server-4d72f09f-28d6-d6ae-3463-40d25dd27338: election won: term=2 tally=2 -2024-05-15T01:17:22.889Z [INFO] server-4d72f09f-28d6-d6ae-3463-40d25dd27338: entering leader state: leader="Node at 4d72f09f-28d6-d6ae-3463-40d25dd27338 [Leader]" -2024-05-15T01:17:22.889Z [INFO] server-4d72f09f-28d6-d6ae-3463-40d25dd27338: added peer, starting replication: peer=server-476372f9-f23a-6fec-0eb7-bbf1318ba96f -2024-05-15T01:17:22.889Z [INFO] server-4d72f09f-28d6-d6ae-3463-40d25dd27338: added peer, starting replication: peer=server-d4a77520-5e02-6e84-f2dd-1194c037733d -2024-05-15T01:17:22.890Z [INFO] server-4d72f09f-28d6-d6ae-3463-40d25dd27338: pipelining replication: peer="{Voter server-d4a77520-5e02-6e84-f2dd-1194c037733d d4a77520-5e02-6e84-f2dd-1194c037733d}" -2024-05-15T01:17:22.892Z [INFO] server-4d72f09f-28d6-d6ae-3463-40d25dd27338: pipelining replication: peer="{Voter server-476372f9-f23a-6fec-0eb7-bbf1318ba96f 476372f9-f23a-6fec-0eb7-bbf1318ba96f}" -2024-05-15T01:17:22.892Z [WARN] server-476372f9-f23a-6fec-0eb7-bbf1318ba96f: rejecting vote request since our last term is greater: candidate=4d72f09f-28d6-d6ae-3463-40d25dd27338 last-term=2 last-candidate-term=1 -2024-05-15T01:17:23.897Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:22.892676137 +0000 UTC m=+23.434060893 (1 nodes), 521.002294ms from start of poll, 521.482892ms from cluster start. Timeout at 2024-05-15 01:17:23.897696934 +0000 UTC m=+24.439081684, 1.005020791s after stability -2024-05-15T01:17:23.903Z [INFO] server-4d72f09f-28d6-d6ae-3463-40d25dd27338: starting snapshot up to: index=12 -2024-05-15T01:17:23.903Z [INFO] creating new snapshot: path=/tmp/raft1114389671/snapshots/2-12-1715735843903.tmp -2024-05-15T01:17:23.904Z [INFO] server-4d72f09f-28d6-d6ae-3463-40d25dd27338: snapshot complete up to: index=12 -2024-05-15T01:17:23.907Z [INFO] server-468b3a50-e809-e8b0-17b0-53dbe7d0f215: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-468b3a50-e809-e8b0-17b0-53dbe7d0f215 Address:468b3a50-e809-e8b0-17b0-53dbe7d0f215} {Suffrage:Voter ID:server-f9d35305-2eb7-dc72-26e2-78da263cced9 Address:f9d35305-2eb7-dc72-26e2-78da263cced9} {Suffrage:Voter ID:server-ab47f68f-4aae-93d5-18ef-d3d7469070f1 Address:ab47f68f-4aae-93d5-18ef-d3d7469070f1}]" -2024-05-15T01:17:23.907Z [INFO] server-f9d35305-2eb7-dc72-26e2-78da263cced9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-468b3a50-e809-e8b0-17b0-53dbe7d0f215 Address:468b3a50-e809-e8b0-17b0-53dbe7d0f215} {Suffrage:Voter ID:server-f9d35305-2eb7-dc72-26e2-78da263cced9 Address:f9d35305-2eb7-dc72-26e2-78da263cced9} {Suffrage:Voter ID:server-ab47f68f-4aae-93d5-18ef-d3d7469070f1 Address:ab47f68f-4aae-93d5-18ef-d3d7469070f1}]" -2024-05-15T01:17:23.907Z [INFO] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-468b3a50-e809-e8b0-17b0-53dbe7d0f215 Address:468b3a50-e809-e8b0-17b0-53dbe7d0f215} {Suffrage:Voter ID:server-f9d35305-2eb7-dc72-26e2-78da263cced9 Address:f9d35305-2eb7-dc72-26e2-78da263cced9} {Suffrage:Voter ID:server-ab47f68f-4aae-93d5-18ef-d3d7469070f1 Address:ab47f68f-4aae-93d5-18ef-d3d7469070f1}]" -2024-05-15T01:17:23.907Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:23.907Z [INFO] server-468b3a50-e809-e8b0-17b0-53dbe7d0f215: entering follower state: follower="Node at 468b3a50-e809-e8b0-17b0-53dbe7d0f215 [Follower]" leader-address= leader-id= -2024-05-15T01:17:23.907Z [INFO] server-f9d35305-2eb7-dc72-26e2-78da263cced9: entering follower state: follower="Node at f9d35305-2eb7-dc72-26e2-78da263cced9 [Follower]" leader-address= leader-id= -2024-05-15T01:17:23.907Z [INFO] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: entering follower state: follower="Node at ab47f68f-4aae-93d5-18ef-d3d7469070f1 [Follower]" leader-address= leader-id= -2024-05-15T01:17:24.496Z [WARN] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:24.496Z [INFO] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: entering candidate state: node="Node at ab47f68f-4aae-93d5-18ef-d3d7469070f1 [Candidate]" term=2 -2024-05-15T01:17:24.497Z [INFO] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: election won: term=2 tally=2 -2024-05-15T01:17:24.497Z [INFO] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: entering leader state: leader="Node at ab47f68f-4aae-93d5-18ef-d3d7469070f1 [Leader]" -2024-05-15T01:17:24.497Z [INFO] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: added peer, starting replication: peer=server-468b3a50-e809-e8b0-17b0-53dbe7d0f215 -2024-05-15T01:17:24.497Z [INFO] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: added peer, starting replication: peer=server-f9d35305-2eb7-dc72-26e2-78da263cced9 -2024-05-15T01:17:24.497Z [INFO] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: pipelining replication: peer="{Voter server-f9d35305-2eb7-dc72-26e2-78da263cced9 f9d35305-2eb7-dc72-26e2-78da263cced9}" -2024-05-15T01:17:24.497Z [INFO] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: pipelining replication: peer="{Voter server-468b3a50-e809-e8b0-17b0-53dbe7d0f215 468b3a50-e809-e8b0-17b0-53dbe7d0f215}" -2024-05-15T01:17:25.503Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:24.497692556 +0000 UTC m=+25.039077300 (1 nodes), 590.205791ms from start of poll, 590.734565ms from cluster start. Timeout at 2024-05-15 01:17:25.503267852 +0000 UTC m=+26.044652607, 1.005575307s after stability -2024-05-15T01:17:25.503Z [INFO] creating new snapshot: path=/tmp/raft1859485428/snapshots/2-13-1715735845503.tmp -2024-05-15T01:17:25.504Z [INFO] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: copied to local snapshot: bytes=71 -2024-05-15T01:17:25.504Z [INFO] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: snapshot restore progress: id=2-13-1715735845503 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:25.504Z [INFO] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: removing all old logs from log store -2024-05-15T01:17:25.504Z [INFO] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: compacting logs: from=1 to=2 -2024-05-15T01:17:25.504Z [INFO] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: restored user snapshot: index=13 -2024-05-15T01:17:25.504Z [ERROR] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: failed to get log: index=2 error="log not found" -2024-05-15T01:17:25.504Z [INFO] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: aborting pipeline replication: peer="{Voter server-f9d35305-2eb7-dc72-26e2-78da263cced9 f9d35305-2eb7-dc72-26e2-78da263cced9}" -2024-05-15T01:17:25.505Z [ERROR] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: failed to get log: index=2 error="log not found" -2024-05-15T01:17:25.505Z [INFO] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: aborting pipeline replication: peer="{Voter server-468b3a50-e809-e8b0-17b0-53dbe7d0f215 468b3a50-e809-e8b0-17b0-53dbe7d0f215}" -2024-05-15T01:17:25.511Z [ERROR] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: failed to get log: index=2 error="log not found" -2024-05-15T01:17:25.512Z [INFO] creating new snapshot: path=/tmp/raft1262785378/snapshots/2-13-1715735845512.tmp -2024-05-15T01:17:25.512Z [INFO] server-f9d35305-2eb7-dc72-26e2-78da263cced9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:25.512Z [INFO] server-f9d35305-2eb7-dc72-26e2-78da263cced9: copied to local snapshot: bytes=71 -2024-05-15T01:17:25.513Z [ERROR] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: failed to get log: index=2 error="log not found" -2024-05-15T01:17:25.513Z [INFO] creating new snapshot: path=/tmp/raft618494990/snapshots/2-13-1715735845513.tmp -2024-05-15T01:17:25.513Z [INFO] server-468b3a50-e809-e8b0-17b0-53dbe7d0f215: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:25.514Z [INFO] server-468b3a50-e809-e8b0-17b0-53dbe7d0f215: copied to local snapshot: bytes=71 -2024-05-15T01:17:25.514Z [INFO] server-468b3a50-e809-e8b0-17b0-53dbe7d0f215: snapshot restore progress: id=2-13-1715735845513 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:25.514Z [INFO] server-468b3a50-e809-e8b0-17b0-53dbe7d0f215: removing all old logs from log store -2024-05-15T01:17:25.514Z [INFO] server-468b3a50-e809-e8b0-17b0-53dbe7d0f215: compacting logs: from=1 to=2 -2024-05-15T01:17:25.514Z [INFO] server-468b3a50-e809-e8b0-17b0-53dbe7d0f215: Installed remote snapshot -2024-05-15T01:17:25.515Z [INFO] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: pipelining replication: peer="{Voter server-468b3a50-e809-e8b0-17b0-53dbe7d0f215 468b3a50-e809-e8b0-17b0-53dbe7d0f215}" -2024-05-15T01:17:25.515Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:25.515Z [INFO] server-f9d35305-2eb7-dc72-26e2-78da263cced9: snapshot restore progress: id=2-13-1715735845512 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:25.515Z [INFO] server-f9d35305-2eb7-dc72-26e2-78da263cced9: removing all old logs from log store -2024-05-15T01:17:25.515Z [INFO] server-f9d35305-2eb7-dc72-26e2-78da263cced9: compacting logs: from=1 to=2 -2024-05-15T01:17:25.515Z [INFO] server-f9d35305-2eb7-dc72-26e2-78da263cced9: Installed remote snapshot -2024-05-15T01:17:25.515Z [INFO] server-ab47f68f-4aae-93d5-18ef-d3d7469070f1: pipelining replication: peer="{Voter server-f9d35305-2eb7-dc72-26e2-78da263cced9 f9d35305-2eb7-dc72-26e2-78da263cced9}" -2024-05-15T01:17:26.520Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:25.515209934 +0000 UTC m=+26.056595020 (2 nodes), 2.605µs from start of poll, 1.608252285s from cluster start. Timeout at 2024-05-15 01:17:26.52025712 +0000 UTC m=+27.061641358, 1.005046338s after stability -2024-05-15T01:17:26.533Z [INFO] server-4d72f09f-28d6-d6ae-3463-40d25dd27338: aborting pipeline replication: peer="{Voter server-d4a77520-5e02-6e84-f2dd-1194c037733d d4a77520-5e02-6e84-f2dd-1194c037733d}" -2024-05-15T01:17:26.533Z [INFO] server-4d72f09f-28d6-d6ae-3463-40d25dd27338: aborting pipeline replication: peer="{Voter server-476372f9-f23a-6fec-0eb7-bbf1318ba96f 476372f9-f23a-6fec-0eb7-bbf1318ba96f}" +2025-06-17T07:43:14.567Z [INFO] server-a5490a4e-c2a7-5c34-ea82-299d30f6a088: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5490a4e-c2a7-5c34-ea82-299d30f6a088 Address:a5490a4e-c2a7-5c34-ea82-299d30f6a088} {Suffrage:Voter ID:server-ed259f19-baad-c2ec-7848-27caa7d1001f Address:ed259f19-baad-c2ec-7848-27caa7d1001f} {Suffrage:Voter ID:server-e8a768ae-dfff-f4dd-eeba-4804eb0319e9 Address:e8a768ae-dfff-f4dd-eeba-4804eb0319e9}]" +2025-06-17T07:43:14.567Z [INFO] server-a5490a4e-c2a7-5c34-ea82-299d30f6a088: entering follower state: follower="Node at a5490a4e-c2a7-5c34-ea82-299d30f6a088 [Follower]" leader-address= leader-id= +2025-06-17T07:43:14.567Z [INFO] server-ed259f19-baad-c2ec-7848-27caa7d1001f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5490a4e-c2a7-5c34-ea82-299d30f6a088 Address:a5490a4e-c2a7-5c34-ea82-299d30f6a088} {Suffrage:Voter ID:server-ed259f19-baad-c2ec-7848-27caa7d1001f Address:ed259f19-baad-c2ec-7848-27caa7d1001f} {Suffrage:Voter ID:server-e8a768ae-dfff-f4dd-eeba-4804eb0319e9 Address:e8a768ae-dfff-f4dd-eeba-4804eb0319e9}]" +2025-06-17T07:43:14.567Z [INFO] server-ed259f19-baad-c2ec-7848-27caa7d1001f: entering follower state: follower="Node at ed259f19-baad-c2ec-7848-27caa7d1001f [Follower]" leader-address= leader-id= +2025-06-17T07:43:14.567Z [INFO] server-e8a768ae-dfff-f4dd-eeba-4804eb0319e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5490a4e-c2a7-5c34-ea82-299d30f6a088 Address:a5490a4e-c2a7-5c34-ea82-299d30f6a088} {Suffrage:Voter ID:server-ed259f19-baad-c2ec-7848-27caa7d1001f Address:ed259f19-baad-c2ec-7848-27caa7d1001f} {Suffrage:Voter ID:server-e8a768ae-dfff-f4dd-eeba-4804eb0319e9 Address:e8a768ae-dfff-f4dd-eeba-4804eb0319e9}]" +2025-06-17T07:43:14.567Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:14.567Z [INFO] server-e8a768ae-dfff-f4dd-eeba-4804eb0319e9: entering follower state: follower="Node at e8a768ae-dfff-f4dd-eeba-4804eb0319e9 [Follower]" leader-address= leader-id= +2025-06-17T07:43:15.198Z [WARN] server-ed259f19-baad-c2ec-7848-27caa7d1001f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:15.198Z [INFO] server-ed259f19-baad-c2ec-7848-27caa7d1001f: entering candidate state: node="Node at ed259f19-baad-c2ec-7848-27caa7d1001f [Candidate]" term=2 +2025-06-17T07:43:15.198Z [INFO] server-ed259f19-baad-c2ec-7848-27caa7d1001f: election won: term=2 tally=2 +2025-06-17T07:43:15.198Z [INFO] server-ed259f19-baad-c2ec-7848-27caa7d1001f: entering leader state: leader="Node at ed259f19-baad-c2ec-7848-27caa7d1001f [Leader]" +2025-06-17T07:43:15.198Z [INFO] server-ed259f19-baad-c2ec-7848-27caa7d1001f: added peer, starting replication: peer=server-a5490a4e-c2a7-5c34-ea82-299d30f6a088 +2025-06-17T07:43:15.198Z [INFO] server-ed259f19-baad-c2ec-7848-27caa7d1001f: added peer, starting replication: peer=server-e8a768ae-dfff-f4dd-eeba-4804eb0319e9 +2025-06-17T07:43:15.198Z [INFO] server-ed259f19-baad-c2ec-7848-27caa7d1001f: pipelining replication: peer="{Voter server-e8a768ae-dfff-f4dd-eeba-4804eb0319e9 e8a768ae-dfff-f4dd-eeba-4804eb0319e9}" +2025-06-17T07:43:15.198Z [INFO] server-ed259f19-baad-c2ec-7848-27caa7d1001f: pipelining replication: peer="{Voter server-a5490a4e-c2a7-5c34-ea82-299d30f6a088 a5490a4e-c2a7-5c34-ea82-299d30f6a088}" +2025-06-17T07:43:16.203Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:15.198470492 +0000 UTC m=+23.801999181 (1 nodes), 630.572626ms from start of poll, 630.838975ms from cluster start. Timeout at 2025-06-17 07:43:16.203589889 +0000 UTC m=+24.807118603, 1.005119422s after stability +2025-06-17T07:43:16.204Z [INFO] server-ed259f19-baad-c2ec-7848-27caa7d1001f: starting snapshot up to: index=12 +2025-06-17T07:43:16.204Z [INFO] creating new snapshot: path=/tmp/raft1368138440/snapshots/2-12-1750146196204.tmp +2025-06-17T07:43:16.204Z [INFO] server-ed259f19-baad-c2ec-7848-27caa7d1001f: snapshot complete up to: index=12 +2025-06-17T07:43:16.205Z [INFO] server-ac2b635d-10bc-8a62-ca28-d78034374c26: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ac2b635d-10bc-8a62-ca28-d78034374c26 Address:ac2b635d-10bc-8a62-ca28-d78034374c26} {Suffrage:Voter ID:server-edc41963-97fd-6cc7-6d65-8546cbf40b76 Address:edc41963-97fd-6cc7-6d65-8546cbf40b76} {Suffrage:Voter ID:server-130158ef-d525-815a-1ff6-57e1738c3240 Address:130158ef-d525-815a-1ff6-57e1738c3240}]" +2025-06-17T07:43:16.205Z [INFO] server-ac2b635d-10bc-8a62-ca28-d78034374c26: entering follower state: follower="Node at ac2b635d-10bc-8a62-ca28-d78034374c26 [Follower]" leader-address= leader-id= +2025-06-17T07:43:16.205Z [INFO] server-edc41963-97fd-6cc7-6d65-8546cbf40b76: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ac2b635d-10bc-8a62-ca28-d78034374c26 Address:ac2b635d-10bc-8a62-ca28-d78034374c26} {Suffrage:Voter ID:server-edc41963-97fd-6cc7-6d65-8546cbf40b76 Address:edc41963-97fd-6cc7-6d65-8546cbf40b76} {Suffrage:Voter ID:server-130158ef-d525-815a-1ff6-57e1738c3240 Address:130158ef-d525-815a-1ff6-57e1738c3240}]" +2025-06-17T07:43:16.205Z [INFO] server-edc41963-97fd-6cc7-6d65-8546cbf40b76: entering follower state: follower="Node at edc41963-97fd-6cc7-6d65-8546cbf40b76 [Follower]" leader-address= leader-id= +2025-06-17T07:43:16.205Z [INFO] server-130158ef-d525-815a-1ff6-57e1738c3240: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ac2b635d-10bc-8a62-ca28-d78034374c26 Address:ac2b635d-10bc-8a62-ca28-d78034374c26} {Suffrage:Voter ID:server-edc41963-97fd-6cc7-6d65-8546cbf40b76 Address:edc41963-97fd-6cc7-6d65-8546cbf40b76} {Suffrage:Voter ID:server-130158ef-d525-815a-1ff6-57e1738c3240 Address:130158ef-d525-815a-1ff6-57e1738c3240}]" +2025-06-17T07:43:16.205Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:16.205Z [INFO] server-130158ef-d525-815a-1ff6-57e1738c3240: entering follower state: follower="Node at 130158ef-d525-815a-1ff6-57e1738c3240 [Follower]" leader-address= leader-id= +2025-06-17T07:43:17.098Z [WARN] server-ac2b635d-10bc-8a62-ca28-d78034374c26: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:17.098Z [INFO] server-ac2b635d-10bc-8a62-ca28-d78034374c26: entering candidate state: node="Node at ac2b635d-10bc-8a62-ca28-d78034374c26 [Candidate]" term=2 +2025-06-17T07:43:17.098Z [INFO] server-ac2b635d-10bc-8a62-ca28-d78034374c26: election won: term=2 tally=2 +2025-06-17T07:43:17.098Z [INFO] server-ac2b635d-10bc-8a62-ca28-d78034374c26: entering leader state: leader="Node at ac2b635d-10bc-8a62-ca28-d78034374c26 [Leader]" +2025-06-17T07:43:17.098Z [INFO] server-ac2b635d-10bc-8a62-ca28-d78034374c26: added peer, starting replication: peer=server-edc41963-97fd-6cc7-6d65-8546cbf40b76 +2025-06-17T07:43:17.098Z [INFO] server-ac2b635d-10bc-8a62-ca28-d78034374c26: added peer, starting replication: peer=server-130158ef-d525-815a-1ff6-57e1738c3240 +2025-06-17T07:43:17.098Z [INFO] server-ac2b635d-10bc-8a62-ca28-d78034374c26: pipelining replication: peer="{Voter server-130158ef-d525-815a-1ff6-57e1738c3240 130158ef-d525-815a-1ff6-57e1738c3240}" +2025-06-17T07:43:17.098Z [INFO] server-ac2b635d-10bc-8a62-ca28-d78034374c26: pipelining replication: peer="{Voter server-edc41963-97fd-6cc7-6d65-8546cbf40b76 edc41963-97fd-6cc7-6d65-8546cbf40b76}" +2025-06-17T07:43:18.104Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:17.09829913 +0000 UTC m=+25.701827857 (1 nodes), 892.413855ms from start of poll, 892.781447ms from cluster start. Timeout at 2025-06-17 07:43:18.104168793 +0000 UTC m=+26.707697475, 1.005869618s after stability +2025-06-17T07:43:18.104Z [INFO] creating new snapshot: path=/tmp/raft329399010/snapshots/2-13-1750146198104.tmp +2025-06-17T07:43:18.105Z [INFO] server-ac2b635d-10bc-8a62-ca28-d78034374c26: copied to local snapshot: bytes=71 +2025-06-17T07:43:18.105Z [INFO] server-ac2b635d-10bc-8a62-ca28-d78034374c26: snapshot restore progress: id=2-13-1750146198104 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:18.105Z [INFO] server-ac2b635d-10bc-8a62-ca28-d78034374c26: removing all old logs from log store +2025-06-17T07:43:18.105Z [INFO] server-ac2b635d-10bc-8a62-ca28-d78034374c26: compacting logs: from=1 to=2 +2025-06-17T07:43:18.105Z [INFO] server-ac2b635d-10bc-8a62-ca28-d78034374c26: restored user snapshot: index=13 +2025-06-17T07:43:18.105Z [ERROR] server-ac2b635d-10bc-8a62-ca28-d78034374c26: failed to get log: index=2 error="log not found" +2025-06-17T07:43:18.105Z [INFO] server-ac2b635d-10bc-8a62-ca28-d78034374c26: aborting pipeline replication: peer="{Voter server-130158ef-d525-815a-1ff6-57e1738c3240 130158ef-d525-815a-1ff6-57e1738c3240}" +2025-06-17T07:43:18.105Z [ERROR] server-ac2b635d-10bc-8a62-ca28-d78034374c26: failed to get log: index=2 error="log not found" +2025-06-17T07:43:18.105Z [INFO] server-ac2b635d-10bc-8a62-ca28-d78034374c26: aborting pipeline replication: peer="{Voter server-edc41963-97fd-6cc7-6d65-8546cbf40b76 edc41963-97fd-6cc7-6d65-8546cbf40b76}" +2025-06-17T07:43:18.111Z [ERROR] server-ac2b635d-10bc-8a62-ca28-d78034374c26: failed to get log: index=2 error="log not found" +2025-06-17T07:43:18.112Z [INFO] creating new snapshot: path=/tmp/raft3103240808/snapshots/2-13-1750146198112.tmp +2025-06-17T07:43:18.112Z [INFO] server-edc41963-97fd-6cc7-6d65-8546cbf40b76: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:18.112Z [INFO] server-edc41963-97fd-6cc7-6d65-8546cbf40b76: copied to local snapshot: bytes=71 +2025-06-17T07:43:18.112Z [INFO] server-edc41963-97fd-6cc7-6d65-8546cbf40b76: snapshot restore progress: id=2-13-1750146198112 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:18.112Z [INFO] server-edc41963-97fd-6cc7-6d65-8546cbf40b76: removing all old logs from log store +2025-06-17T07:43:18.112Z [INFO] server-edc41963-97fd-6cc7-6d65-8546cbf40b76: compacting logs: from=1 to=2 +2025-06-17T07:43:18.112Z [INFO] server-edc41963-97fd-6cc7-6d65-8546cbf40b76: Installed remote snapshot +2025-06-17T07:43:18.112Z [INFO] server-ac2b635d-10bc-8a62-ca28-d78034374c26: pipelining replication: peer="{Voter server-edc41963-97fd-6cc7-6d65-8546cbf40b76 edc41963-97fd-6cc7-6d65-8546cbf40b76}" +2025-06-17T07:43:18.112Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:18.113Z [ERROR] server-ac2b635d-10bc-8a62-ca28-d78034374c26: failed to get log: index=2 error="log not found" +2025-06-17T07:43:18.114Z [INFO] creating new snapshot: path=/tmp/raft2137311833/snapshots/2-13-1750146198114.tmp +2025-06-17T07:43:18.114Z [INFO] server-130158ef-d525-815a-1ff6-57e1738c3240: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:18.114Z [INFO] server-130158ef-d525-815a-1ff6-57e1738c3240: copied to local snapshot: bytes=71 +2025-06-17T07:43:18.114Z [INFO] server-130158ef-d525-815a-1ff6-57e1738c3240: snapshot restore progress: id=2-13-1750146198114 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:18.114Z [INFO] server-130158ef-d525-815a-1ff6-57e1738c3240: removing all old logs from log store +2025-06-17T07:43:18.114Z [INFO] server-130158ef-d525-815a-1ff6-57e1738c3240: compacting logs: from=1 to=2 +2025-06-17T07:43:18.114Z [INFO] server-130158ef-d525-815a-1ff6-57e1738c3240: Installed remote snapshot +2025-06-17T07:43:18.115Z [INFO] server-ac2b635d-10bc-8a62-ca28-d78034374c26: pipelining replication: peer="{Voter server-130158ef-d525-815a-1ff6-57e1738c3240 130158ef-d525-815a-1ff6-57e1738c3240}" +2025-06-17T07:43:19.118Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:18.112911599 +0000 UTC m=+26.716440302 (2 nodes), 591ns from start of poll, 1.907393892s from cluster start. Timeout at 2025-06-17 07:43:19.1188997 +0000 UTC m=+27.722428383, 1.005988081s after stability +2025-06-17T07:43:19.130Z [INFO] server-ed259f19-baad-c2ec-7848-27caa7d1001f: aborting pipeline replication: peer="{Voter server-e8a768ae-dfff-f4dd-eeba-4804eb0319e9 e8a768ae-dfff-f4dd-eeba-4804eb0319e9}" +2025-06-17T07:43:19.130Z [INFO] server-ed259f19-baad-c2ec-7848-27caa7d1001f: aborting pipeline replication: peer="{Voter server-a5490a4e-c2a7-5c34-ea82-299d30f6a088 a5490a4e-c2a7-5c34-ea82-299d30f6a088}" === RUN TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false -2024-05-15T01:17:26.535Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb Address:8f61f22e-047b-9e0c-a7db-fbf6940f9efb} {Suffrage:Voter ID:server-ee2da21b-b108-3da2-ebcb-d89405cf52c5 Address:ee2da21b-b108-3da2-ebcb-d89405cf52c5} {Suffrage:Voter ID:server-9194d294-3abd-0c5d-c4f2-b27267426035 Address:9194d294-3abd-0c5d-c4f2-b27267426035}]" -2024-05-15T01:17:26.535Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: entering follower state: follower="Node at 8f61f22e-047b-9e0c-a7db-fbf6940f9efb [Follower]" leader-address= leader-id= -2024-05-15T01:17:26.535Z [INFO] server-ee2da21b-b108-3da2-ebcb-d89405cf52c5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb Address:8f61f22e-047b-9e0c-a7db-fbf6940f9efb} {Suffrage:Voter ID:server-ee2da21b-b108-3da2-ebcb-d89405cf52c5 Address:ee2da21b-b108-3da2-ebcb-d89405cf52c5} {Suffrage:Voter ID:server-9194d294-3abd-0c5d-c4f2-b27267426035 Address:9194d294-3abd-0c5d-c4f2-b27267426035}]" -2024-05-15T01:17:26.536Z [INFO] server-ee2da21b-b108-3da2-ebcb-d89405cf52c5: entering follower state: follower="Node at ee2da21b-b108-3da2-ebcb-d89405cf52c5 [Follower]" leader-address= leader-id= -2024-05-15T01:17:26.536Z [INFO] server-9194d294-3abd-0c5d-c4f2-b27267426035: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb Address:8f61f22e-047b-9e0c-a7db-fbf6940f9efb} {Suffrage:Voter ID:server-ee2da21b-b108-3da2-ebcb-d89405cf52c5 Address:ee2da21b-b108-3da2-ebcb-d89405cf52c5} {Suffrage:Voter ID:server-9194d294-3abd-0c5d-c4f2-b27267426035 Address:9194d294-3abd-0c5d-c4f2-b27267426035}]" -2024-05-15T01:17:26.536Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:26.536Z [INFO] server-9194d294-3abd-0c5d-c4f2-b27267426035: entering follower state: follower="Node at 9194d294-3abd-0c5d-c4f2-b27267426035 [Follower]" leader-address= leader-id= -2024-05-15T01:17:27.068Z [WARN] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:27.068Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: entering candidate state: node="Node at 8f61f22e-047b-9e0c-a7db-fbf6940f9efb [Candidate]" term=2 -2024-05-15T01:17:27.069Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: election won: term=2 tally=2 -2024-05-15T01:17:27.069Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: entering leader state: leader="Node at 8f61f22e-047b-9e0c-a7db-fbf6940f9efb [Leader]" -2024-05-15T01:17:27.069Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: added peer, starting replication: peer=server-ee2da21b-b108-3da2-ebcb-d89405cf52c5 -2024-05-15T01:17:27.069Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: added peer, starting replication: peer=server-9194d294-3abd-0c5d-c4f2-b27267426035 -2024-05-15T01:17:27.069Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: pipelining replication: peer="{Voter server-ee2da21b-b108-3da2-ebcb-d89405cf52c5 ee2da21b-b108-3da2-ebcb-d89405cf52c5}" -2024-05-15T01:17:27.069Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: pipelining replication: peer="{Voter server-9194d294-3abd-0c5d-c4f2-b27267426035 9194d294-3abd-0c5d-c4f2-b27267426035}" -2024-05-15T01:17:28.074Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:27.069096512 +0000 UTC m=+27.610481260 (1 nodes), 532.839693ms from start of poll, 533.650231ms from cluster start. Timeout at 2024-05-15 01:17:28.074484747 +0000 UTC m=+28.615869491, 1.005388231s after stability -2024-05-15T01:17:28.076Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: starting snapshot up to: index=12 -2024-05-15T01:17:28.076Z [INFO] creating new snapshot: path=/tmp/raft989996576/snapshots/2-12-1715735848076.tmp -2024-05-15T01:17:28.136Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: snapshot complete up to: index=12 -2024-05-15T01:17:28.138Z [INFO] creating new snapshot: path=/tmp/raft989996576/snapshots/2-23-1715735848138.tmp -2024-05-15T01:17:28.205Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: copied to local snapshot: bytes=71 -2024-05-15T01:17:28.205Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: snapshot restore progress: id=2-23-1715735848138 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:28.206Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: restored user snapshot: index=23 -2024-05-15T01:17:28.206Z [ERROR] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: failed to get log: index=23 error="log not found" -2024-05-15T01:17:28.206Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: aborting pipeline replication: peer="{Voter server-9194d294-3abd-0c5d-c4f2-b27267426035 9194d294-3abd-0c5d-c4f2-b27267426035}" -2024-05-15T01:17:28.206Z [ERROR] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: failed to get log: index=23 error="log not found" -2024-05-15T01:17:28.206Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: aborting pipeline replication: peer="{Voter server-ee2da21b-b108-3da2-ebcb-d89405cf52c5 ee2da21b-b108-3da2-ebcb-d89405cf52c5}" -2024-05-15T01:17:28.214Z [ERROR] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: failed to get log: index=23 error="log not found" -2024-05-15T01:17:28.214Z [INFO] creating new snapshot: path=/tmp/raft313968380/snapshots/2-23-1715735848214.tmp -2024-05-15T01:17:28.215Z [INFO] server-9194d294-3abd-0c5d-c4f2-b27267426035: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:28.216Z [ERROR] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: failed to get log: index=23 error="log not found" -2024-05-15T01:17:28.217Z [INFO] creating new snapshot: path=/tmp/raft1822850656/snapshots/2-23-1715735848217.tmp -2024-05-15T01:17:28.217Z [INFO] server-ee2da21b-b108-3da2-ebcb-d89405cf52c5: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:28.269Z [INFO] server-9194d294-3abd-0c5d-c4f2-b27267426035: copied to local snapshot: bytes=71 -2024-05-15T01:17:28.270Z [INFO] server-ee2da21b-b108-3da2-ebcb-d89405cf52c5: copied to local snapshot: bytes=71 -2024-05-15T01:17:28.270Z [INFO] server-ee2da21b-b108-3da2-ebcb-d89405cf52c5: snapshot restore progress: id=2-23-1715735848217 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:28.270Z [INFO] server-ee2da21b-b108-3da2-ebcb-d89405cf52c5: Installed remote snapshot -2024-05-15T01:17:28.270Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: pipelining replication: peer="{Voter server-ee2da21b-b108-3da2-ebcb-d89405cf52c5 ee2da21b-b108-3da2-ebcb-d89405cf52c5}" -2024-05-15T01:17:28.271Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:28.271Z [INFO] server-9194d294-3abd-0c5d-c4f2-b27267426035: snapshot restore progress: id=2-23-1715735848214 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:28.271Z [INFO] server-9194d294-3abd-0c5d-c4f2-b27267426035: Installed remote snapshot -2024-05-15T01:17:28.271Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: pipelining replication: peer="{Voter server-9194d294-3abd-0c5d-c4f2-b27267426035 9194d294-3abd-0c5d-c4f2-b27267426035}" -2024-05-15T01:17:29.276Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:28.271059357 +0000 UTC m=+28.812484028 (2 nodes), 41.96µs from start of poll, 1.735652999s from cluster start. Timeout at 2024-05-15 01:17:29.27630369 +0000 UTC m=+29.817688433, 1.005204405s after stability -2024-05-15T01:17:29.288Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: aborting pipeline replication: peer="{Voter server-9194d294-3abd-0c5d-c4f2-b27267426035 9194d294-3abd-0c5d-c4f2-b27267426035}" -2024-05-15T01:17:29.288Z [INFO] server-8f61f22e-047b-9e0c-a7db-fbf6940f9efb: aborting pipeline replication: peer="{Voter server-ee2da21b-b108-3da2-ebcb-d89405cf52c5 ee2da21b-b108-3da2-ebcb-d89405cf52c5}" +2025-06-17T07:43:19.131Z [INFO] server-fdbe5e7d-a5c7-542f-a54b-f0537a906165: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fdbe5e7d-a5c7-542f-a54b-f0537a906165 Address:fdbe5e7d-a5c7-542f-a54b-f0537a906165} {Suffrage:Voter ID:server-e73006cc-a0bf-fc43-227a-397d09002df8 Address:e73006cc-a0bf-fc43-227a-397d09002df8} {Suffrage:Voter ID:server-cfd320c4-06ce-1f06-dfc5-83f1f029d712 Address:cfd320c4-06ce-1f06-dfc5-83f1f029d712}]" +2025-06-17T07:43:19.131Z [INFO] server-fdbe5e7d-a5c7-542f-a54b-f0537a906165: entering follower state: follower="Node at fdbe5e7d-a5c7-542f-a54b-f0537a906165 [Follower]" leader-address= leader-id= +2025-06-17T07:43:19.131Z [INFO] server-e73006cc-a0bf-fc43-227a-397d09002df8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fdbe5e7d-a5c7-542f-a54b-f0537a906165 Address:fdbe5e7d-a5c7-542f-a54b-f0537a906165} {Suffrage:Voter ID:server-e73006cc-a0bf-fc43-227a-397d09002df8 Address:e73006cc-a0bf-fc43-227a-397d09002df8} {Suffrage:Voter ID:server-cfd320c4-06ce-1f06-dfc5-83f1f029d712 Address:cfd320c4-06ce-1f06-dfc5-83f1f029d712}]" +2025-06-17T07:43:19.131Z [INFO] server-e73006cc-a0bf-fc43-227a-397d09002df8: entering follower state: follower="Node at e73006cc-a0bf-fc43-227a-397d09002df8 [Follower]" leader-address= leader-id= +2025-06-17T07:43:19.131Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fdbe5e7d-a5c7-542f-a54b-f0537a906165 Address:fdbe5e7d-a5c7-542f-a54b-f0537a906165} {Suffrage:Voter ID:server-e73006cc-a0bf-fc43-227a-397d09002df8 Address:e73006cc-a0bf-fc43-227a-397d09002df8} {Suffrage:Voter ID:server-cfd320c4-06ce-1f06-dfc5-83f1f029d712 Address:cfd320c4-06ce-1f06-dfc5-83f1f029d712}]" +2025-06-17T07:43:19.131Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:19.131Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: entering follower state: follower="Node at cfd320c4-06ce-1f06-dfc5-83f1f029d712 [Follower]" leader-address= leader-id= +2025-06-17T07:43:19.847Z [WARN] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:19.847Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: entering candidate state: node="Node at cfd320c4-06ce-1f06-dfc5-83f1f029d712 [Candidate]" term=2 +2025-06-17T07:43:19.847Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: election won: term=2 tally=2 +2025-06-17T07:43:19.847Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: entering leader state: leader="Node at cfd320c4-06ce-1f06-dfc5-83f1f029d712 [Leader]" +2025-06-17T07:43:19.847Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: added peer, starting replication: peer=server-fdbe5e7d-a5c7-542f-a54b-f0537a906165 +2025-06-17T07:43:19.847Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: added peer, starting replication: peer=server-e73006cc-a0bf-fc43-227a-397d09002df8 +2025-06-17T07:43:19.847Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: pipelining replication: peer="{Voter server-fdbe5e7d-a5c7-542f-a54b-f0537a906165 fdbe5e7d-a5c7-542f-a54b-f0537a906165}" +2025-06-17T07:43:19.847Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: pipelining replication: peer="{Voter server-e73006cc-a0bf-fc43-227a-397d09002df8 e73006cc-a0bf-fc43-227a-397d09002df8}" +2025-06-17T07:43:20.852Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:19.84728955 +0000 UTC m=+28.450818267 (1 nodes), 715.761001ms from start of poll, 716.053234ms from cluster start. Timeout at 2025-06-17 07:43:20.852753135 +0000 UTC m=+29.456281817, 1.00546355s after stability +2025-06-17T07:43:20.853Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: starting snapshot up to: index=12 +2025-06-17T07:43:20.853Z [INFO] creating new snapshot: path=/tmp/raft1220977082/snapshots/2-12-1750146200853.tmp +2025-06-17T07:43:20.853Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: snapshot complete up to: index=12 +2025-06-17T07:43:20.854Z [INFO] creating new snapshot: path=/tmp/raft1220977082/snapshots/2-23-1750146200854.tmp +2025-06-17T07:43:20.855Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: copied to local snapshot: bytes=71 +2025-06-17T07:43:20.855Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: snapshot restore progress: id=2-23-1750146200854 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:20.855Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: restored user snapshot: index=23 +2025-06-17T07:43:20.855Z [ERROR] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: failed to get log: index=23 error="log not found" +2025-06-17T07:43:20.855Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: aborting pipeline replication: peer="{Voter server-e73006cc-a0bf-fc43-227a-397d09002df8 e73006cc-a0bf-fc43-227a-397d09002df8}" +2025-06-17T07:43:20.855Z [ERROR] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: failed to get log: index=23 error="log not found" +2025-06-17T07:43:20.855Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: aborting pipeline replication: peer="{Voter server-fdbe5e7d-a5c7-542f-a54b-f0537a906165 fdbe5e7d-a5c7-542f-a54b-f0537a906165}" +2025-06-17T07:43:20.861Z [ERROR] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: failed to get log: index=23 error="log not found" +2025-06-17T07:43:20.862Z [INFO] creating new snapshot: path=/tmp/raft3325884934/snapshots/2-23-1750146200862.tmp +2025-06-17T07:43:20.862Z [ERROR] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: failed to get log: index=23 error="log not found" +2025-06-17T07:43:20.862Z [INFO] server-e73006cc-a0bf-fc43-227a-397d09002df8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:20.862Z [INFO] server-e73006cc-a0bf-fc43-227a-397d09002df8: copied to local snapshot: bytes=71 +2025-06-17T07:43:20.862Z [INFO] creating new snapshot: path=/tmp/raft1004591192/snapshots/2-23-1750146200862.tmp +2025-06-17T07:43:20.862Z [INFO] server-e73006cc-a0bf-fc43-227a-397d09002df8: snapshot restore progress: id=2-23-1750146200862 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:20.862Z [INFO] server-e73006cc-a0bf-fc43-227a-397d09002df8: Installed remote snapshot +2025-06-17T07:43:20.862Z [WARN] server-e73006cc-a0bf-fc43-227a-397d09002df8: skipping application of old log: index=22 +2025-06-17T07:43:20.862Z [INFO] server-fdbe5e7d-a5c7-542f-a54b-f0537a906165: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:20.862Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: pipelining replication: peer="{Voter server-e73006cc-a0bf-fc43-227a-397d09002df8 e73006cc-a0bf-fc43-227a-397d09002df8}" +2025-06-17T07:43:20.862Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:20.862Z [INFO] server-fdbe5e7d-a5c7-542f-a54b-f0537a906165: copied to local snapshot: bytes=71 +2025-06-17T07:43:20.863Z [INFO] server-fdbe5e7d-a5c7-542f-a54b-f0537a906165: snapshot restore progress: id=2-23-1750146200862 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:20.863Z [INFO] server-fdbe5e7d-a5c7-542f-a54b-f0537a906165: Installed remote snapshot +2025-06-17T07:43:20.863Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: pipelining replication: peer="{Voter server-fdbe5e7d-a5c7-542f-a54b-f0537a906165 fdbe5e7d-a5c7-542f-a54b-f0537a906165}" +2025-06-17T07:43:21.868Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:20.862888536 +0000 UTC m=+29.466417223 (2 nodes), 629ns from start of poll, 1.73165219s from cluster start. Timeout at 2025-06-17 07:43:21.868451476 +0000 UTC m=+30.471980172, 1.005562949s after stability +2025-06-17T07:43:21.879Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: aborting pipeline replication: peer="{Voter server-e73006cc-a0bf-fc43-227a-397d09002df8 e73006cc-a0bf-fc43-227a-397d09002df8}" +2025-06-17T07:43:21.879Z [INFO] server-cfd320c4-06ce-1f06-dfc5-83f1f029d712: aborting pipeline replication: peer="{Voter server-fdbe5e7d-a5c7-542f-a54b-f0537a906165 fdbe5e7d-a5c7-542f-a54b-f0537a906165}" === RUN TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false -2024-05-15T01:17:29.291Z [INFO] server-f75613c5-7c48-f4ae-6b2c-833b2f36b3ff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f75613c5-7c48-f4ae-6b2c-833b2f36b3ff Address:f75613c5-7c48-f4ae-6b2c-833b2f36b3ff} {Suffrage:Voter ID:server-2e1e6fcb-f545-9d77-d383-d41ed3b05921 Address:2e1e6fcb-f545-9d77-d383-d41ed3b05921} {Suffrage:Voter ID:server-e550bf12-f402-16d0-a5a2-45df3e24d77e Address:e550bf12-f402-16d0-a5a2-45df3e24d77e}]" -2024-05-15T01:17:29.291Z [INFO] server-2e1e6fcb-f545-9d77-d383-d41ed3b05921: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f75613c5-7c48-f4ae-6b2c-833b2f36b3ff Address:f75613c5-7c48-f4ae-6b2c-833b2f36b3ff} {Suffrage:Voter ID:server-2e1e6fcb-f545-9d77-d383-d41ed3b05921 Address:2e1e6fcb-f545-9d77-d383-d41ed3b05921} {Suffrage:Voter ID:server-e550bf12-f402-16d0-a5a2-45df3e24d77e Address:e550bf12-f402-16d0-a5a2-45df3e24d77e}]" -2024-05-15T01:17:29.291Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f75613c5-7c48-f4ae-6b2c-833b2f36b3ff Address:f75613c5-7c48-f4ae-6b2c-833b2f36b3ff} {Suffrage:Voter ID:server-2e1e6fcb-f545-9d77-d383-d41ed3b05921 Address:2e1e6fcb-f545-9d77-d383-d41ed3b05921} {Suffrage:Voter ID:server-e550bf12-f402-16d0-a5a2-45df3e24d77e Address:e550bf12-f402-16d0-a5a2-45df3e24d77e}]" -2024-05-15T01:17:29.291Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:29.291Z [INFO] server-f75613c5-7c48-f4ae-6b2c-833b2f36b3ff: entering follower state: follower="Node at f75613c5-7c48-f4ae-6b2c-833b2f36b3ff [Follower]" leader-address= leader-id= -2024-05-15T01:17:29.291Z [INFO] server-2e1e6fcb-f545-9d77-d383-d41ed3b05921: entering follower state: follower="Node at 2e1e6fcb-f545-9d77-d383-d41ed3b05921 [Follower]" leader-address= leader-id= -2024-05-15T01:17:29.291Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: entering follower state: follower="Node at e550bf12-f402-16d0-a5a2-45df3e24d77e [Follower]" leader-address= leader-id= -2024-05-15T01:17:29.849Z [WARN] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:29.849Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: entering candidate state: node="Node at e550bf12-f402-16d0-a5a2-45df3e24d77e [Candidate]" term=2 -2024-05-15T01:17:29.850Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: election won: term=2 tally=2 -2024-05-15T01:17:29.850Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: entering leader state: leader="Node at e550bf12-f402-16d0-a5a2-45df3e24d77e [Leader]" -2024-05-15T01:17:29.850Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: added peer, starting replication: peer=server-f75613c5-7c48-f4ae-6b2c-833b2f36b3ff -2024-05-15T01:17:29.850Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: added peer, starting replication: peer=server-2e1e6fcb-f545-9d77-d383-d41ed3b05921 -2024-05-15T01:17:29.850Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: pipelining replication: peer="{Voter server-2e1e6fcb-f545-9d77-d383-d41ed3b05921 2e1e6fcb-f545-9d77-d383-d41ed3b05921}" -2024-05-15T01:17:29.850Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: pipelining replication: peer="{Voter server-f75613c5-7c48-f4ae-6b2c-833b2f36b3ff f75613c5-7c48-f4ae-6b2c-833b2f36b3ff}" -2024-05-15T01:17:30.856Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:29.850675362 +0000 UTC m=+30.392059600 (1 nodes), 559.175123ms from start of poll, 559.729595ms from cluster start. Timeout at 2024-05-15 01:17:30.85658857 +0000 UTC m=+31.397973334, 1.005913734s after stability -2024-05-15T01:17:30.858Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: starting snapshot up to: index=12 -2024-05-15T01:17:30.858Z [INFO] creating new snapshot: path=/tmp/raft1464048626/snapshots/2-12-1715735850858.tmp -2024-05-15T01:17:30.859Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: snapshot complete up to: index=12 -2024-05-15T01:17:30.860Z [INFO] creating new snapshot: path=/tmp/raft1464048626/snapshots/2-23-1715735850860.tmp -2024-05-15T01:17:30.861Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: copied to local snapshot: bytes=71 -2024-05-15T01:17:30.862Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: snapshot restore progress: id=2-23-1715735850860 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:30.862Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: removing all old logs from log store -2024-05-15T01:17:30.862Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: compacting logs: from=1 to=22 -2024-05-15T01:17:30.862Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: restored user snapshot: index=23 -2024-05-15T01:17:30.862Z [ERROR] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: failed to get log: index=22 error="log not found" -2024-05-15T01:17:30.862Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: aborting pipeline replication: peer="{Voter server-2e1e6fcb-f545-9d77-d383-d41ed3b05921 2e1e6fcb-f545-9d77-d383-d41ed3b05921}" -2024-05-15T01:17:30.862Z [ERROR] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: failed to get log: index=22 error="log not found" -2024-05-15T01:17:30.862Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: aborting pipeline replication: peer="{Voter server-f75613c5-7c48-f4ae-6b2c-833b2f36b3ff f75613c5-7c48-f4ae-6b2c-833b2f36b3ff}" -2024-05-15T01:17:30.871Z [ERROR] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: failed to get log: index=22 error="log not found" -2024-05-15T01:17:30.871Z [INFO] creating new snapshot: path=/tmp/raft470712994/snapshots/2-23-1715735850871.tmp -2024-05-15T01:17:30.872Z [INFO] server-f75613c5-7c48-f4ae-6b2c-833b2f36b3ff: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:30.872Z [ERROR] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: failed to get log: index=22 error="log not found" -2024-05-15T01:17:30.872Z [INFO] server-f75613c5-7c48-f4ae-6b2c-833b2f36b3ff: copied to local snapshot: bytes=71 -2024-05-15T01:17:30.872Z [INFO] creating new snapshot: path=/tmp/raft1981125900/snapshots/2-23-1715735850872.tmp -2024-05-15T01:17:30.873Z [INFO] server-f75613c5-7c48-f4ae-6b2c-833b2f36b3ff: snapshot restore progress: id=2-23-1715735850871 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:30.873Z [INFO] server-f75613c5-7c48-f4ae-6b2c-833b2f36b3ff: removing all old logs from log store -2024-05-15T01:17:30.873Z [INFO] server-f75613c5-7c48-f4ae-6b2c-833b2f36b3ff: compacting logs: from=1 to=22 -2024-05-15T01:17:30.873Z [INFO] server-f75613c5-7c48-f4ae-6b2c-833b2f36b3ff: Installed remote snapshot -2024-05-15T01:17:30.873Z [INFO] server-2e1e6fcb-f545-9d77-d383-d41ed3b05921: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:30.873Z [WARN] server-f75613c5-7c48-f4ae-6b2c-833b2f36b3ff: skipping application of old log: index=22 -2024-05-15T01:17:30.873Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: pipelining replication: peer="{Voter server-f75613c5-7c48-f4ae-6b2c-833b2f36b3ff f75613c5-7c48-f4ae-6b2c-833b2f36b3ff}" -2024-05-15T01:17:30.873Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:30.873Z [INFO] server-2e1e6fcb-f545-9d77-d383-d41ed3b05921: copied to local snapshot: bytes=71 -2024-05-15T01:17:30.876Z [INFO] server-2e1e6fcb-f545-9d77-d383-d41ed3b05921: snapshot restore progress: id=2-23-1715735850872 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:30.876Z [INFO] server-2e1e6fcb-f545-9d77-d383-d41ed3b05921: removing all old logs from log store -2024-05-15T01:17:30.876Z [INFO] server-2e1e6fcb-f545-9d77-d383-d41ed3b05921: compacting logs: from=1 to=22 -2024-05-15T01:17:30.876Z [INFO] server-2e1e6fcb-f545-9d77-d383-d41ed3b05921: Installed remote snapshot -2024-05-15T01:17:30.876Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: pipelining replication: peer="{Voter server-2e1e6fcb-f545-9d77-d383-d41ed3b05921 2e1e6fcb-f545-9d77-d383-d41ed3b05921}" -2024-05-15T01:17:31.878Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:30.873555191 +0000 UTC m=+31.414940173 (2 nodes), 2.397µs from start of poll, 1.582610168s from cluster start. Timeout at 2024-05-15 01:17:31.878881677 +0000 UTC m=+32.420266455, 1.005326282s after stability -2024-05-15T01:17:31.891Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: aborting pipeline replication: peer="{Voter server-2e1e6fcb-f545-9d77-d383-d41ed3b05921 2e1e6fcb-f545-9d77-d383-d41ed3b05921}" -2024-05-15T01:17:31.891Z [INFO] server-e550bf12-f402-16d0-a5a2-45df3e24d77e: aborting pipeline replication: peer="{Voter server-f75613c5-7c48-f4ae-6b2c-833b2f36b3ff f75613c5-7c48-f4ae-6b2c-833b2f36b3ff}" +2025-06-17T07:43:21.881Z [INFO] server-209900c3-7284-ea90-7c23-0eb1b49b8ae0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-209900c3-7284-ea90-7c23-0eb1b49b8ae0 Address:209900c3-7284-ea90-7c23-0eb1b49b8ae0} {Suffrage:Voter ID:server-05b454e2-86e9-b22a-d969-cab86ed0d64e Address:05b454e2-86e9-b22a-d969-cab86ed0d64e} {Suffrage:Voter ID:server-bbf24344-eb2a-487d-d155-2b7e0b1d0177 Address:bbf24344-eb2a-487d-d155-2b7e0b1d0177}]" +2025-06-17T07:43:21.881Z [INFO] server-209900c3-7284-ea90-7c23-0eb1b49b8ae0: entering follower state: follower="Node at 209900c3-7284-ea90-7c23-0eb1b49b8ae0 [Follower]" leader-address= leader-id= +2025-06-17T07:43:21.881Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-209900c3-7284-ea90-7c23-0eb1b49b8ae0 Address:209900c3-7284-ea90-7c23-0eb1b49b8ae0} {Suffrage:Voter ID:server-05b454e2-86e9-b22a-d969-cab86ed0d64e Address:05b454e2-86e9-b22a-d969-cab86ed0d64e} {Suffrage:Voter ID:server-bbf24344-eb2a-487d-d155-2b7e0b1d0177 Address:bbf24344-eb2a-487d-d155-2b7e0b1d0177}]" +2025-06-17T07:43:21.881Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: entering follower state: follower="Node at 05b454e2-86e9-b22a-d969-cab86ed0d64e [Follower]" leader-address= leader-id= +2025-06-17T07:43:21.881Z [INFO] server-bbf24344-eb2a-487d-d155-2b7e0b1d0177: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-209900c3-7284-ea90-7c23-0eb1b49b8ae0 Address:209900c3-7284-ea90-7c23-0eb1b49b8ae0} {Suffrage:Voter ID:server-05b454e2-86e9-b22a-d969-cab86ed0d64e Address:05b454e2-86e9-b22a-d969-cab86ed0d64e} {Suffrage:Voter ID:server-bbf24344-eb2a-487d-d155-2b7e0b1d0177 Address:bbf24344-eb2a-487d-d155-2b7e0b1d0177}]" +2025-06-17T07:43:21.881Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:21.881Z [INFO] server-bbf24344-eb2a-487d-d155-2b7e0b1d0177: entering follower state: follower="Node at bbf24344-eb2a-487d-d155-2b7e0b1d0177 [Follower]" leader-address= leader-id= +2025-06-17T07:43:22.411Z [WARN] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:22.411Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: entering candidate state: node="Node at 05b454e2-86e9-b22a-d969-cab86ed0d64e [Candidate]" term=2 +2025-06-17T07:43:22.411Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: election won: term=2 tally=2 +2025-06-17T07:43:22.411Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: entering leader state: leader="Node at 05b454e2-86e9-b22a-d969-cab86ed0d64e [Leader]" +2025-06-17T07:43:22.411Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: added peer, starting replication: peer=server-209900c3-7284-ea90-7c23-0eb1b49b8ae0 +2025-06-17T07:43:22.411Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: added peer, starting replication: peer=server-bbf24344-eb2a-487d-d155-2b7e0b1d0177 +2025-06-17T07:43:22.411Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: pipelining replication: peer="{Voter server-bbf24344-eb2a-487d-d155-2b7e0b1d0177 bbf24344-eb2a-487d-d155-2b7e0b1d0177}" +2025-06-17T07:43:22.411Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: pipelining replication: peer="{Voter server-209900c3-7284-ea90-7c23-0eb1b49b8ae0 209900c3-7284-ea90-7c23-0eb1b49b8ae0}" +2025-06-17T07:43:23.417Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:22.41142153 +0000 UTC m=+31.014950218 (1 nodes), 530.174772ms from start of poll, 530.483166ms from cluster start. Timeout at 2025-06-17 07:43:23.417215533 +0000 UTC m=+32.020744248, 1.00579403s after stability +2025-06-17T07:43:23.417Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: starting snapshot up to: index=12 +2025-06-17T07:43:23.417Z [INFO] creating new snapshot: path=/tmp/raft2787383562/snapshots/2-12-1750146203417.tmp +2025-06-17T07:43:23.418Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: snapshot complete up to: index=12 +2025-06-17T07:43:23.418Z [INFO] creating new snapshot: path=/tmp/raft2787383562/snapshots/2-23-1750146203418.tmp +2025-06-17T07:43:23.419Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: copied to local snapshot: bytes=71 +2025-06-17T07:43:23.419Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: snapshot restore progress: id=2-23-1750146203418 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:23.419Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: removing all old logs from log store +2025-06-17T07:43:23.419Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: compacting logs: from=1 to=22 +2025-06-17T07:43:23.419Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: restored user snapshot: index=23 +2025-06-17T07:43:23.419Z [ERROR] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: failed to get log: index=22 error="log not found" +2025-06-17T07:43:23.419Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: aborting pipeline replication: peer="{Voter server-bbf24344-eb2a-487d-d155-2b7e0b1d0177 bbf24344-eb2a-487d-d155-2b7e0b1d0177}" +2025-06-17T07:43:23.419Z [ERROR] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: failed to get log: index=22 error="log not found" +2025-06-17T07:43:23.419Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: aborting pipeline replication: peer="{Voter server-209900c3-7284-ea90-7c23-0eb1b49b8ae0 209900c3-7284-ea90-7c23-0eb1b49b8ae0}" +2025-06-17T07:43:23.428Z [ERROR] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: failed to get log: index=22 error="log not found" +2025-06-17T07:43:23.428Z [ERROR] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: failed to get log: index=22 error="log not found" +2025-06-17T07:43:23.428Z [INFO] creating new snapshot: path=/tmp/raft3331412983/snapshots/2-23-1750146203428.tmp +2025-06-17T07:43:23.428Z [INFO] creating new snapshot: path=/tmp/raft3163761328/snapshots/2-23-1750146203428.tmp +2025-06-17T07:43:23.428Z [INFO] server-bbf24344-eb2a-487d-d155-2b7e0b1d0177: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:23.428Z [INFO] server-209900c3-7284-ea90-7c23-0eb1b49b8ae0: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:23.429Z [INFO] server-bbf24344-eb2a-487d-d155-2b7e0b1d0177: copied to local snapshot: bytes=71 +2025-06-17T07:43:23.429Z [INFO] server-209900c3-7284-ea90-7c23-0eb1b49b8ae0: copied to local snapshot: bytes=71 +2025-06-17T07:43:23.429Z [INFO] server-bbf24344-eb2a-487d-d155-2b7e0b1d0177: snapshot restore progress: id=2-23-1750146203428 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:23.429Z [INFO] server-bbf24344-eb2a-487d-d155-2b7e0b1d0177: removing all old logs from log store +2025-06-17T07:43:23.429Z [INFO] server-bbf24344-eb2a-487d-d155-2b7e0b1d0177: compacting logs: from=1 to=22 +2025-06-17T07:43:23.429Z [INFO] server-bbf24344-eb2a-487d-d155-2b7e0b1d0177: Installed remote snapshot +2025-06-17T07:43:23.429Z [INFO] server-209900c3-7284-ea90-7c23-0eb1b49b8ae0: snapshot restore progress: id=2-23-1750146203428 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:23.430Z [INFO] server-209900c3-7284-ea90-7c23-0eb1b49b8ae0: removing all old logs from log store +2025-06-17T07:43:23.430Z [INFO] server-209900c3-7284-ea90-7c23-0eb1b49b8ae0: compacting logs: from=1 to=22 +2025-06-17T07:43:23.430Z [INFO] server-209900c3-7284-ea90-7c23-0eb1b49b8ae0: Installed remote snapshot +2025-06-17T07:43:23.430Z [WARN] server-bbf24344-eb2a-487d-d155-2b7e0b1d0177: skipping application of old log: index=22 +2025-06-17T07:43:23.430Z [WARN] server-209900c3-7284-ea90-7c23-0eb1b49b8ae0: skipping application of old log: index=23 +2025-06-17T07:43:23.430Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: pipelining replication: peer="{Voter server-bbf24344-eb2a-487d-d155-2b7e0b1d0177 bbf24344-eb2a-487d-d155-2b7e0b1d0177}" +2025-06-17T07:43:23.430Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: pipelining replication: peer="{Voter server-209900c3-7284-ea90-7c23-0eb1b49b8ae0 209900c3-7284-ea90-7c23-0eb1b49b8ae0}" +2025-06-17T07:43:23.430Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:24.435Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:23.430415089 +0000 UTC m=+32.033943789 (2 nodes), 821ns from start of poll, 1.549476737s from cluster start. Timeout at 2025-06-17 07:43:24.435603432 +0000 UTC m=+33.039132113, 1.005188324s after stability +2025-06-17T07:43:24.446Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: aborting pipeline replication: peer="{Voter server-bbf24344-eb2a-487d-d155-2b7e0b1d0177 bbf24344-eb2a-487d-d155-2b7e0b1d0177}" +2025-06-17T07:43:24.446Z [INFO] server-05b454e2-86e9-b22a-d969-cab86ed0d64e: aborting pipeline replication: peer="{Voter server-209900c3-7284-ea90-7c23-0eb1b49b8ae0 209900c3-7284-ea90-7c23-0eb1b49b8ae0}" === RUN TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true -2024-05-15T01:17:31.893Z [INFO] server-943b9cec-5398-da31-a10f-320f3b3ffe00: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-943b9cec-5398-da31-a10f-320f3b3ffe00 Address:943b9cec-5398-da31-a10f-320f3b3ffe00} {Suffrage:Voter ID:server-f1f51d34-265f-754a-4388-e84aee0ec3b8 Address:f1f51d34-265f-754a-4388-e84aee0ec3b8} {Suffrage:Voter ID:server-c94ac717-952d-566b-f808-6df8ff1ef918 Address:c94ac717-952d-566b-f808-6df8ff1ef918}]" -2024-05-15T01:17:31.893Z [INFO] server-f1f51d34-265f-754a-4388-e84aee0ec3b8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-943b9cec-5398-da31-a10f-320f3b3ffe00 Address:943b9cec-5398-da31-a10f-320f3b3ffe00} {Suffrage:Voter ID:server-f1f51d34-265f-754a-4388-e84aee0ec3b8 Address:f1f51d34-265f-754a-4388-e84aee0ec3b8} {Suffrage:Voter ID:server-c94ac717-952d-566b-f808-6df8ff1ef918 Address:c94ac717-952d-566b-f808-6df8ff1ef918}]" -2024-05-15T01:17:31.894Z [INFO] server-c94ac717-952d-566b-f808-6df8ff1ef918: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-943b9cec-5398-da31-a10f-320f3b3ffe00 Address:943b9cec-5398-da31-a10f-320f3b3ffe00} {Suffrage:Voter ID:server-f1f51d34-265f-754a-4388-e84aee0ec3b8 Address:f1f51d34-265f-754a-4388-e84aee0ec3b8} {Suffrage:Voter ID:server-c94ac717-952d-566b-f808-6df8ff1ef918 Address:c94ac717-952d-566b-f808-6df8ff1ef918}]" -2024-05-15T01:17:31.894Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:31.894Z [INFO] server-943b9cec-5398-da31-a10f-320f3b3ffe00: entering follower state: follower="Node at 943b9cec-5398-da31-a10f-320f3b3ffe00 [Follower]" leader-address= leader-id= -2024-05-15T01:17:31.894Z [INFO] server-f1f51d34-265f-754a-4388-e84aee0ec3b8: entering follower state: follower="Node at f1f51d34-265f-754a-4388-e84aee0ec3b8 [Follower]" leader-address= leader-id= -2024-05-15T01:17:31.894Z [INFO] server-c94ac717-952d-566b-f808-6df8ff1ef918: entering follower state: follower="Node at c94ac717-952d-566b-f808-6df8ff1ef918 [Follower]" leader-address= leader-id= -2024-05-15T01:17:32.434Z [WARN] server-f1f51d34-265f-754a-4388-e84aee0ec3b8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:32.434Z [INFO] server-f1f51d34-265f-754a-4388-e84aee0ec3b8: entering candidate state: node="Node at f1f51d34-265f-754a-4388-e84aee0ec3b8 [Candidate]" term=2 -2024-05-15T01:17:32.434Z [INFO] server-f1f51d34-265f-754a-4388-e84aee0ec3b8: election won: term=2 tally=2 -2024-05-15T01:17:32.434Z [INFO] server-f1f51d34-265f-754a-4388-e84aee0ec3b8: entering leader state: leader="Node at f1f51d34-265f-754a-4388-e84aee0ec3b8 [Leader]" -2024-05-15T01:17:32.434Z [INFO] server-f1f51d34-265f-754a-4388-e84aee0ec3b8: added peer, starting replication: peer=server-943b9cec-5398-da31-a10f-320f3b3ffe00 -2024-05-15T01:17:32.434Z [INFO] server-f1f51d34-265f-754a-4388-e84aee0ec3b8: added peer, starting replication: peer=server-c94ac717-952d-566b-f808-6df8ff1ef918 -2024-05-15T01:17:32.435Z [INFO] server-f1f51d34-265f-754a-4388-e84aee0ec3b8: pipelining replication: peer="{Voter server-c94ac717-952d-566b-f808-6df8ff1ef918 c94ac717-952d-566b-f808-6df8ff1ef918}" -2024-05-15T01:17:32.435Z [INFO] server-f1f51d34-265f-754a-4388-e84aee0ec3b8: pipelining replication: peer="{Voter server-943b9cec-5398-da31-a10f-320f3b3ffe00 943b9cec-5398-da31-a10f-320f3b3ffe00}" -2024-05-15T01:17:33.440Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:32.435423333 +0000 UTC m=+32.976807571 (1 nodes), 541.229743ms from start of poll, 541.781041ms from cluster start. Timeout at 2024-05-15 01:17:33.440634562 +0000 UTC m=+33.982019329, 1.005211758s after stability -2024-05-15T01:17:33.442Z [INFO] server-f1f51d34-265f-754a-4388-e84aee0ec3b8: starting snapshot up to: index=12 -2024-05-15T01:17:33.442Z [INFO] creating new snapshot: path=/tmp/raft1982993834/snapshots/2-12-1715735853442.tmp -2024-05-15T01:17:33.442Z [INFO] server-f1f51d34-265f-754a-4388-e84aee0ec3b8: snapshot complete up to: index=12 -2024-05-15T01:17:33.445Z [INFO] server-f24c20c2-a58b-47bd-71b3-6f15770caf3b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f24c20c2-a58b-47bd-71b3-6f15770caf3b Address:f24c20c2-a58b-47bd-71b3-6f15770caf3b} {Suffrage:Voter ID:server-5f5084f2-8c6e-70aa-0e38-1cab16e8952d Address:5f5084f2-8c6e-70aa-0e38-1cab16e8952d} {Suffrage:Voter ID:server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9 Address:faf7db6b-0030-714e-f7dd-1bfcc5fb23e9}]" -2024-05-15T01:17:33.445Z [INFO] server-5f5084f2-8c6e-70aa-0e38-1cab16e8952d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f24c20c2-a58b-47bd-71b3-6f15770caf3b Address:f24c20c2-a58b-47bd-71b3-6f15770caf3b} {Suffrage:Voter ID:server-5f5084f2-8c6e-70aa-0e38-1cab16e8952d Address:5f5084f2-8c6e-70aa-0e38-1cab16e8952d} {Suffrage:Voter ID:server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9 Address:faf7db6b-0030-714e-f7dd-1bfcc5fb23e9}]" -2024-05-15T01:17:33.446Z [INFO] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f24c20c2-a58b-47bd-71b3-6f15770caf3b Address:f24c20c2-a58b-47bd-71b3-6f15770caf3b} {Suffrage:Voter ID:server-5f5084f2-8c6e-70aa-0e38-1cab16e8952d Address:5f5084f2-8c6e-70aa-0e38-1cab16e8952d} {Suffrage:Voter ID:server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9 Address:faf7db6b-0030-714e-f7dd-1bfcc5fb23e9}]" -2024-05-15T01:17:33.446Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:33.446Z [INFO] server-f24c20c2-a58b-47bd-71b3-6f15770caf3b: entering follower state: follower="Node at f24c20c2-a58b-47bd-71b3-6f15770caf3b [Follower]" leader-address= leader-id= -2024-05-15T01:17:33.446Z [INFO] server-5f5084f2-8c6e-70aa-0e38-1cab16e8952d: entering follower state: follower="Node at 5f5084f2-8c6e-70aa-0e38-1cab16e8952d [Follower]" leader-address= leader-id= -2024-05-15T01:17:33.446Z [INFO] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: entering follower state: follower="Node at faf7db6b-0030-714e-f7dd-1bfcc5fb23e9 [Follower]" leader-address= leader-id= -2024-05-15T01:17:34.019Z [WARN] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:34.019Z [INFO] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: entering candidate state: node="Node at faf7db6b-0030-714e-f7dd-1bfcc5fb23e9 [Candidate]" term=2 -2024-05-15T01:17:34.019Z [INFO] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: election won: term=2 tally=2 -2024-05-15T01:17:34.019Z [INFO] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: entering leader state: leader="Node at faf7db6b-0030-714e-f7dd-1bfcc5fb23e9 [Leader]" -2024-05-15T01:17:34.019Z [INFO] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: added peer, starting replication: peer=server-f24c20c2-a58b-47bd-71b3-6f15770caf3b -2024-05-15T01:17:34.019Z [INFO] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: added peer, starting replication: peer=server-5f5084f2-8c6e-70aa-0e38-1cab16e8952d -2024-05-15T01:17:34.019Z [INFO] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: pipelining replication: peer="{Voter server-5f5084f2-8c6e-70aa-0e38-1cab16e8952d 5f5084f2-8c6e-70aa-0e38-1cab16e8952d}" -2024-05-15T01:17:34.019Z [INFO] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: pipelining replication: peer="{Voter server-f24c20c2-a58b-47bd-71b3-6f15770caf3b f24c20c2-a58b-47bd-71b3-6f15770caf3b}" -2024-05-15T01:17:35.025Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:34.019838601 +0000 UTC m=+34.561223362 (1 nodes), 573.745926ms from start of poll, 574.276559ms from cluster start. Timeout at 2024-05-15 01:17:35.025935834 +0000 UTC m=+35.567320602, 1.00609724s after stability -2024-05-15T01:17:35.026Z [INFO] creating new snapshot: path=/tmp/raft1407455900/snapshots/2-14-1715735855026.tmp -2024-05-15T01:17:35.114Z [INFO] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: copied to local snapshot: bytes=71 -2024-05-15T01:17:35.115Z [INFO] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: snapshot restore progress: id=2-14-1715735855026 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:35.115Z [INFO] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: restored user snapshot: index=14 -2024-05-15T01:17:35.115Z [ERROR] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: failed to get log: index=3 error="log not found" -2024-05-15T01:17:35.115Z [INFO] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: aborting pipeline replication: peer="{Voter server-5f5084f2-8c6e-70aa-0e38-1cab16e8952d 5f5084f2-8c6e-70aa-0e38-1cab16e8952d}" -2024-05-15T01:17:35.115Z [ERROR] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: failed to get log: index=3 error="log not found" -2024-05-15T01:17:35.115Z [INFO] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: aborting pipeline replication: peer="{Voter server-f24c20c2-a58b-47bd-71b3-6f15770caf3b f24c20c2-a58b-47bd-71b3-6f15770caf3b}" -2024-05-15T01:17:35.125Z [ERROR] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: failed to get log: index=3 error="log not found" -2024-05-15T01:17:35.125Z [ERROR] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: failed to get log: index=3 error="log not found" -2024-05-15T01:17:35.126Z [INFO] creating new snapshot: path=/tmp/raft4255509498/snapshots/2-14-1715735855126.tmp -2024-05-15T01:17:35.126Z [INFO] server-f24c20c2-a58b-47bd-71b3-6f15770caf3b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:35.126Z [INFO] creating new snapshot: path=/tmp/raft2568909686/snapshots/2-14-1715735855126.tmp -2024-05-15T01:17:35.127Z [INFO] server-5f5084f2-8c6e-70aa-0e38-1cab16e8952d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:35.338Z [INFO] server-5f5084f2-8c6e-70aa-0e38-1cab16e8952d: copied to local snapshot: bytes=71 -2024-05-15T01:17:35.339Z [INFO] server-5f5084f2-8c6e-70aa-0e38-1cab16e8952d: snapshot restore progress: id=2-14-1715735855126 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:35.339Z [INFO] server-5f5084f2-8c6e-70aa-0e38-1cab16e8952d: Installed remote snapshot -2024-05-15T01:17:35.339Z [INFO] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: pipelining replication: peer="{Voter server-5f5084f2-8c6e-70aa-0e38-1cab16e8952d 5f5084f2-8c6e-70aa-0e38-1cab16e8952d}" -2024-05-15T01:17:35.339Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:35.339Z [INFO] server-f24c20c2-a58b-47bd-71b3-6f15770caf3b: copied to local snapshot: bytes=71 -2024-05-15T01:17:35.340Z [INFO] server-f24c20c2-a58b-47bd-71b3-6f15770caf3b: snapshot restore progress: id=2-14-1715735855126 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:35.340Z [INFO] server-f24c20c2-a58b-47bd-71b3-6f15770caf3b: Installed remote snapshot -2024-05-15T01:17:35.340Z [INFO] server-faf7db6b-0030-714e-f7dd-1bfcc5fb23e9: pipelining replication: peer="{Voter server-f24c20c2-a58b-47bd-71b3-6f15770caf3b f24c20c2-a58b-47bd-71b3-6f15770caf3b}" -2024-05-15T01:17:36.345Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:35.339691261 +0000 UTC m=+35.881076291 (2 nodes), 2.362µs from start of poll, 1.894129488s from cluster start. Timeout at 2024-05-15 01:17:36.345863588 +0000 UTC m=+36.887248537, 1.006172246s after stability -2024-05-15T01:17:36.359Z [INFO] server-f1f51d34-265f-754a-4388-e84aee0ec3b8: aborting pipeline replication: peer="{Voter server-c94ac717-952d-566b-f808-6df8ff1ef918 c94ac717-952d-566b-f808-6df8ff1ef918}" -2024-05-15T01:17:36.359Z [INFO] server-f1f51d34-265f-754a-4388-e84aee0ec3b8: aborting pipeline replication: peer="{Voter server-943b9cec-5398-da31-a10f-320f3b3ffe00 943b9cec-5398-da31-a10f-320f3b3ffe00}" +2025-06-17T07:43:24.449Z [INFO] server-071c04e4-2807-f7a8-4299-13f737b656c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-071c04e4-2807-f7a8-4299-13f737b656c3 Address:071c04e4-2807-f7a8-4299-13f737b656c3} {Suffrage:Voter ID:server-1889f12e-c930-3bf5-45c3-532f6ac43466 Address:1889f12e-c930-3bf5-45c3-532f6ac43466} {Suffrage:Voter ID:server-aad9908d-559a-5872-c21b-0a937e1ecd8c Address:aad9908d-559a-5872-c21b-0a937e1ecd8c}]" +2025-06-17T07:43:24.449Z [INFO] server-071c04e4-2807-f7a8-4299-13f737b656c3: entering follower state: follower="Node at 071c04e4-2807-f7a8-4299-13f737b656c3 [Follower]" leader-address= leader-id= +2025-06-17T07:43:24.449Z [INFO] server-1889f12e-c930-3bf5-45c3-532f6ac43466: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-071c04e4-2807-f7a8-4299-13f737b656c3 Address:071c04e4-2807-f7a8-4299-13f737b656c3} {Suffrage:Voter ID:server-1889f12e-c930-3bf5-45c3-532f6ac43466 Address:1889f12e-c930-3bf5-45c3-532f6ac43466} {Suffrage:Voter ID:server-aad9908d-559a-5872-c21b-0a937e1ecd8c Address:aad9908d-559a-5872-c21b-0a937e1ecd8c}]" +2025-06-17T07:43:24.449Z [INFO] server-1889f12e-c930-3bf5-45c3-532f6ac43466: entering follower state: follower="Node at 1889f12e-c930-3bf5-45c3-532f6ac43466 [Follower]" leader-address= leader-id= +2025-06-17T07:43:24.449Z [INFO] server-aad9908d-559a-5872-c21b-0a937e1ecd8c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-071c04e4-2807-f7a8-4299-13f737b656c3 Address:071c04e4-2807-f7a8-4299-13f737b656c3} {Suffrage:Voter ID:server-1889f12e-c930-3bf5-45c3-532f6ac43466 Address:1889f12e-c930-3bf5-45c3-532f6ac43466} {Suffrage:Voter ID:server-aad9908d-559a-5872-c21b-0a937e1ecd8c Address:aad9908d-559a-5872-c21b-0a937e1ecd8c}]" +2025-06-17T07:43:24.449Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:24.449Z [INFO] server-aad9908d-559a-5872-c21b-0a937e1ecd8c: entering follower state: follower="Node at aad9908d-559a-5872-c21b-0a937e1ecd8c [Follower]" leader-address= leader-id= +2025-06-17T07:43:25.145Z [WARN] server-aad9908d-559a-5872-c21b-0a937e1ecd8c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:25.145Z [INFO] server-aad9908d-559a-5872-c21b-0a937e1ecd8c: entering candidate state: node="Node at aad9908d-559a-5872-c21b-0a937e1ecd8c [Candidate]" term=2 +2025-06-17T07:43:25.145Z [INFO] server-aad9908d-559a-5872-c21b-0a937e1ecd8c: election won: term=2 tally=2 +2025-06-17T07:43:25.145Z [INFO] server-aad9908d-559a-5872-c21b-0a937e1ecd8c: entering leader state: leader="Node at aad9908d-559a-5872-c21b-0a937e1ecd8c [Leader]" +2025-06-17T07:43:25.146Z [INFO] server-aad9908d-559a-5872-c21b-0a937e1ecd8c: added peer, starting replication: peer=server-071c04e4-2807-f7a8-4299-13f737b656c3 +2025-06-17T07:43:25.146Z [INFO] server-aad9908d-559a-5872-c21b-0a937e1ecd8c: added peer, starting replication: peer=server-1889f12e-c930-3bf5-45c3-532f6ac43466 +2025-06-17T07:43:25.146Z [INFO] server-aad9908d-559a-5872-c21b-0a937e1ecd8c: pipelining replication: peer="{Voter server-071c04e4-2807-f7a8-4299-13f737b656c3 071c04e4-2807-f7a8-4299-13f737b656c3}" +2025-06-17T07:43:25.146Z [INFO] server-aad9908d-559a-5872-c21b-0a937e1ecd8c: pipelining replication: peer="{Voter server-1889f12e-c930-3bf5-45c3-532f6ac43466 1889f12e-c930-3bf5-45c3-532f6ac43466}" +2025-06-17T07:43:26.151Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:25.14601157 +0000 UTC m=+33.749540270 (1 nodes), 696.504018ms from start of poll, 696.787434ms from cluster start. Timeout at 2025-06-17 07:43:26.151648147 +0000 UTC m=+34.755176829, 1.005636559s after stability +2025-06-17T07:43:26.152Z [INFO] server-aad9908d-559a-5872-c21b-0a937e1ecd8c: starting snapshot up to: index=12 +2025-06-17T07:43:26.152Z [INFO] creating new snapshot: path=/tmp/raft1615584118/snapshots/2-12-1750146206152.tmp +2025-06-17T07:43:26.152Z [INFO] server-aad9908d-559a-5872-c21b-0a937e1ecd8c: snapshot complete up to: index=12 +2025-06-17T07:43:26.153Z [INFO] server-e75f5bca-9fbc-73b1-3fe3-481bc875b19a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e75f5bca-9fbc-73b1-3fe3-481bc875b19a Address:e75f5bca-9fbc-73b1-3fe3-481bc875b19a} {Suffrage:Voter ID:server-a43b99a8-a181-88a5-b4ea-e97192160efb Address:a43b99a8-a181-88a5-b4ea-e97192160efb} {Suffrage:Voter ID:server-95af5a61-f6df-0609-3adf-dbd36431f9f7 Address:95af5a61-f6df-0609-3adf-dbd36431f9f7}]" +2025-06-17T07:43:26.153Z [INFO] server-e75f5bca-9fbc-73b1-3fe3-481bc875b19a: entering follower state: follower="Node at e75f5bca-9fbc-73b1-3fe3-481bc875b19a [Follower]" leader-address= leader-id= +2025-06-17T07:43:26.153Z [INFO] server-a43b99a8-a181-88a5-b4ea-e97192160efb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e75f5bca-9fbc-73b1-3fe3-481bc875b19a Address:e75f5bca-9fbc-73b1-3fe3-481bc875b19a} {Suffrage:Voter ID:server-a43b99a8-a181-88a5-b4ea-e97192160efb Address:a43b99a8-a181-88a5-b4ea-e97192160efb} {Suffrage:Voter ID:server-95af5a61-f6df-0609-3adf-dbd36431f9f7 Address:95af5a61-f6df-0609-3adf-dbd36431f9f7}]" +2025-06-17T07:43:26.153Z [INFO] server-a43b99a8-a181-88a5-b4ea-e97192160efb: entering follower state: follower="Node at a43b99a8-a181-88a5-b4ea-e97192160efb [Follower]" leader-address= leader-id= +2025-06-17T07:43:26.154Z [INFO] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e75f5bca-9fbc-73b1-3fe3-481bc875b19a Address:e75f5bca-9fbc-73b1-3fe3-481bc875b19a} {Suffrage:Voter ID:server-a43b99a8-a181-88a5-b4ea-e97192160efb Address:a43b99a8-a181-88a5-b4ea-e97192160efb} {Suffrage:Voter ID:server-95af5a61-f6df-0609-3adf-dbd36431f9f7 Address:95af5a61-f6df-0609-3adf-dbd36431f9f7}]" +2025-06-17T07:43:26.154Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:26.154Z [INFO] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: entering follower state: follower="Node at 95af5a61-f6df-0609-3adf-dbd36431f9f7 [Follower]" leader-address= leader-id= +2025-06-17T07:43:26.773Z [WARN] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:26.773Z [INFO] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: entering candidate state: node="Node at 95af5a61-f6df-0609-3adf-dbd36431f9f7 [Candidate]" term=2 +2025-06-17T07:43:26.773Z [INFO] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: election won: term=2 tally=2 +2025-06-17T07:43:26.773Z [INFO] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: entering leader state: leader="Node at 95af5a61-f6df-0609-3adf-dbd36431f9f7 [Leader]" +2025-06-17T07:43:26.773Z [INFO] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: added peer, starting replication: peer=server-e75f5bca-9fbc-73b1-3fe3-481bc875b19a +2025-06-17T07:43:26.773Z [INFO] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: added peer, starting replication: peer=server-a43b99a8-a181-88a5-b4ea-e97192160efb +2025-06-17T07:43:26.773Z [INFO] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: pipelining replication: peer="{Voter server-a43b99a8-a181-88a5-b4ea-e97192160efb a43b99a8-a181-88a5-b4ea-e97192160efb}" +2025-06-17T07:43:26.773Z [INFO] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: pipelining replication: peer="{Voter server-e75f5bca-9fbc-73b1-3fe3-481bc875b19a e75f5bca-9fbc-73b1-3fe3-481bc875b19a}" +2025-06-17T07:43:27.779Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:26.773731158 +0000 UTC m=+35.377259882 (1 nodes), 619.681594ms from start of poll, 620.047086ms from cluster start. Timeout at 2025-06-17 07:43:27.779161244 +0000 UTC m=+36.382689927, 1.005430045s after stability +2025-06-17T07:43:27.779Z [INFO] creating new snapshot: path=/tmp/raft4177089609/snapshots/2-14-1750146207779.tmp +2025-06-17T07:43:27.779Z [INFO] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: copied to local snapshot: bytes=71 +2025-06-17T07:43:27.779Z [INFO] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: snapshot restore progress: id=2-14-1750146207779 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:27.780Z [INFO] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: restored user snapshot: index=14 +2025-06-17T07:43:27.780Z [ERROR] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: failed to get log: index=3 error="log not found" +2025-06-17T07:43:27.780Z [INFO] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: aborting pipeline replication: peer="{Voter server-a43b99a8-a181-88a5-b4ea-e97192160efb a43b99a8-a181-88a5-b4ea-e97192160efb}" +2025-06-17T07:43:27.780Z [ERROR] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: failed to get log: index=3 error="log not found" +2025-06-17T07:43:27.780Z [INFO] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: aborting pipeline replication: peer="{Voter server-e75f5bca-9fbc-73b1-3fe3-481bc875b19a e75f5bca-9fbc-73b1-3fe3-481bc875b19a}" +2025-06-17T07:43:27.786Z [ERROR] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: failed to get log: index=3 error="log not found" +2025-06-17T07:43:27.787Z [INFO] creating new snapshot: path=/tmp/raft441868256/snapshots/2-14-1750146207787.tmp +2025-06-17T07:43:27.787Z [INFO] server-a43b99a8-a181-88a5-b4ea-e97192160efb: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:27.787Z [INFO] server-a43b99a8-a181-88a5-b4ea-e97192160efb: copied to local snapshot: bytes=71 +2025-06-17T07:43:27.787Z [INFO] server-a43b99a8-a181-88a5-b4ea-e97192160efb: snapshot restore progress: id=2-14-1750146207787 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:27.787Z [INFO] server-a43b99a8-a181-88a5-b4ea-e97192160efb: Installed remote snapshot +2025-06-17T07:43:27.787Z [INFO] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: pipelining replication: peer="{Voter server-a43b99a8-a181-88a5-b4ea-e97192160efb a43b99a8-a181-88a5-b4ea-e97192160efb}" +2025-06-17T07:43:27.787Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:27.790Z [ERROR] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: failed to get log: index=3 error="log not found" +2025-06-17T07:43:27.790Z [INFO] creating new snapshot: path=/tmp/raft2838512025/snapshots/2-14-1750146207790.tmp +2025-06-17T07:43:27.790Z [INFO] server-e75f5bca-9fbc-73b1-3fe3-481bc875b19a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:27.790Z [INFO] server-e75f5bca-9fbc-73b1-3fe3-481bc875b19a: copied to local snapshot: bytes=71 +2025-06-17T07:43:27.791Z [INFO] server-e75f5bca-9fbc-73b1-3fe3-481bc875b19a: snapshot restore progress: id=2-14-1750146207790 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:27.791Z [INFO] server-e75f5bca-9fbc-73b1-3fe3-481bc875b19a: Installed remote snapshot +2025-06-17T07:43:27.791Z [INFO] server-95af5a61-f6df-0609-3adf-dbd36431f9f7: pipelining replication: peer="{Voter server-e75f5bca-9fbc-73b1-3fe3-481bc875b19a e75f5bca-9fbc-73b1-3fe3-481bc875b19a}" +2025-06-17T07:43:28.793Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:27.787928653 +0000 UTC m=+36.391457350 (2 nodes), 561ns from start of poll, 1.634244554s from cluster start. Timeout at 2025-06-17 07:43:28.793009018 +0000 UTC m=+37.396537737, 1.005080387s after stability +2025-06-17T07:43:28.809Z [INFO] server-aad9908d-559a-5872-c21b-0a937e1ecd8c: aborting pipeline replication: peer="{Voter server-1889f12e-c930-3bf5-45c3-532f6ac43466 1889f12e-c930-3bf5-45c3-532f6ac43466}" +2025-06-17T07:43:28.809Z [INFO] server-aad9908d-559a-5872-c21b-0a937e1ecd8c: aborting pipeline replication: peer="{Voter server-071c04e4-2807-f7a8-4299-13f737b656c3 071c04e4-2807-f7a8-4299-13f737b656c3}" === RUN TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true -2024-05-15T01:17:36.361Z [INFO] server-87509249-080f-5bd9-2749-19d42474f8b3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-87509249-080f-5bd9-2749-19d42474f8b3 Address:87509249-080f-5bd9-2749-19d42474f8b3} {Suffrage:Voter ID:server-3ee5d55a-f7d0-695c-cc70-f94360b37c8c Address:3ee5d55a-f7d0-695c-cc70-f94360b37c8c} {Suffrage:Voter ID:server-ab51b0e7-da7b-9c8d-77ee-2617055787ed Address:ab51b0e7-da7b-9c8d-77ee-2617055787ed}]" -2024-05-15T01:17:36.361Z [INFO] server-87509249-080f-5bd9-2749-19d42474f8b3: entering follower state: follower="Node at 87509249-080f-5bd9-2749-19d42474f8b3 [Follower]" leader-address= leader-id= -2024-05-15T01:17:36.361Z [INFO] server-3ee5d55a-f7d0-695c-cc70-f94360b37c8c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-87509249-080f-5bd9-2749-19d42474f8b3 Address:87509249-080f-5bd9-2749-19d42474f8b3} {Suffrage:Voter ID:server-3ee5d55a-f7d0-695c-cc70-f94360b37c8c Address:3ee5d55a-f7d0-695c-cc70-f94360b37c8c} {Suffrage:Voter ID:server-ab51b0e7-da7b-9c8d-77ee-2617055787ed Address:ab51b0e7-da7b-9c8d-77ee-2617055787ed}]" -2024-05-15T01:17:36.361Z [INFO] server-ab51b0e7-da7b-9c8d-77ee-2617055787ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-87509249-080f-5bd9-2749-19d42474f8b3 Address:87509249-080f-5bd9-2749-19d42474f8b3} {Suffrage:Voter ID:server-3ee5d55a-f7d0-695c-cc70-f94360b37c8c Address:3ee5d55a-f7d0-695c-cc70-f94360b37c8c} {Suffrage:Voter ID:server-ab51b0e7-da7b-9c8d-77ee-2617055787ed Address:ab51b0e7-da7b-9c8d-77ee-2617055787ed}]" -2024-05-15T01:17:36.361Z [INFO] server-3ee5d55a-f7d0-695c-cc70-f94360b37c8c: entering follower state: follower="Node at 3ee5d55a-f7d0-695c-cc70-f94360b37c8c [Follower]" leader-address= leader-id= -2024-05-15T01:17:36.361Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:36.361Z [INFO] server-ab51b0e7-da7b-9c8d-77ee-2617055787ed: entering follower state: follower="Node at ab51b0e7-da7b-9c8d-77ee-2617055787ed [Follower]" leader-address= leader-id= -2024-05-15T01:17:36.914Z [WARN] server-ab51b0e7-da7b-9c8d-77ee-2617055787ed: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:36.914Z [INFO] server-ab51b0e7-da7b-9c8d-77ee-2617055787ed: entering candidate state: node="Node at ab51b0e7-da7b-9c8d-77ee-2617055787ed [Candidate]" term=2 -2024-05-15T01:17:36.914Z [INFO] server-ab51b0e7-da7b-9c8d-77ee-2617055787ed: election won: term=2 tally=2 -2024-05-15T01:17:36.914Z [INFO] server-ab51b0e7-da7b-9c8d-77ee-2617055787ed: entering leader state: leader="Node at ab51b0e7-da7b-9c8d-77ee-2617055787ed [Leader]" -2024-05-15T01:17:36.914Z [INFO] server-ab51b0e7-da7b-9c8d-77ee-2617055787ed: added peer, starting replication: peer=server-87509249-080f-5bd9-2749-19d42474f8b3 -2024-05-15T01:17:36.914Z [INFO] server-ab51b0e7-da7b-9c8d-77ee-2617055787ed: added peer, starting replication: peer=server-3ee5d55a-f7d0-695c-cc70-f94360b37c8c -2024-05-15T01:17:36.915Z [INFO] server-ab51b0e7-da7b-9c8d-77ee-2617055787ed: pipelining replication: peer="{Voter server-87509249-080f-5bd9-2749-19d42474f8b3 87509249-080f-5bd9-2749-19d42474f8b3}" -2024-05-15T01:17:36.915Z [INFO] server-ab51b0e7-da7b-9c8d-77ee-2617055787ed: pipelining replication: peer="{Voter server-3ee5d55a-f7d0-695c-cc70-f94360b37c8c 3ee5d55a-f7d0-695c-cc70-f94360b37c8c}" -2024-05-15T01:17:37.920Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:36.914896732 +0000 UTC m=+37.456281636 (1 nodes), 553.332998ms from start of poll, 553.958588ms from cluster start. Timeout at 2024-05-15 01:17:37.920824521 +0000 UTC m=+38.462209327, 1.005927691s after stability -2024-05-15T01:17:37.922Z [INFO] server-ab51b0e7-da7b-9c8d-77ee-2617055787ed: starting snapshot up to: index=12 -2024-05-15T01:17:37.922Z [INFO] creating new snapshot: path=/tmp/raft3207597252/snapshots/2-12-1715735857922.tmp -2024-05-15T01:17:37.923Z [INFO] server-ab51b0e7-da7b-9c8d-77ee-2617055787ed: snapshot complete up to: index=12 -2024-05-15T01:17:37.925Z [INFO] server-45ca580f-f01c-31e9-00d3-62c35b44f662: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45ca580f-f01c-31e9-00d3-62c35b44f662 Address:45ca580f-f01c-31e9-00d3-62c35b44f662} {Suffrage:Voter ID:server-16077a32-3b1f-9b57-935e-0c599fdf72f8 Address:16077a32-3b1f-9b57-935e-0c599fdf72f8} {Suffrage:Voter ID:server-5d2726b3-c9c6-86ca-facf-c3523c967221 Address:5d2726b3-c9c6-86ca-facf-c3523c967221}]" -2024-05-15T01:17:37.925Z [INFO] server-16077a32-3b1f-9b57-935e-0c599fdf72f8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45ca580f-f01c-31e9-00d3-62c35b44f662 Address:45ca580f-f01c-31e9-00d3-62c35b44f662} {Suffrage:Voter ID:server-16077a32-3b1f-9b57-935e-0c599fdf72f8 Address:16077a32-3b1f-9b57-935e-0c599fdf72f8} {Suffrage:Voter ID:server-5d2726b3-c9c6-86ca-facf-c3523c967221 Address:5d2726b3-c9c6-86ca-facf-c3523c967221}]" -2024-05-15T01:17:37.926Z [INFO] server-5d2726b3-c9c6-86ca-facf-c3523c967221: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45ca580f-f01c-31e9-00d3-62c35b44f662 Address:45ca580f-f01c-31e9-00d3-62c35b44f662} {Suffrage:Voter ID:server-16077a32-3b1f-9b57-935e-0c599fdf72f8 Address:16077a32-3b1f-9b57-935e-0c599fdf72f8} {Suffrage:Voter ID:server-5d2726b3-c9c6-86ca-facf-c3523c967221 Address:5d2726b3-c9c6-86ca-facf-c3523c967221}]" -2024-05-15T01:17:37.926Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:37.926Z [INFO] server-16077a32-3b1f-9b57-935e-0c599fdf72f8: entering follower state: follower="Node at 16077a32-3b1f-9b57-935e-0c599fdf72f8 [Follower]" leader-address= leader-id= -2024-05-15T01:17:37.926Z [INFO] server-5d2726b3-c9c6-86ca-facf-c3523c967221: entering follower state: follower="Node at 5d2726b3-c9c6-86ca-facf-c3523c967221 [Follower]" leader-address= leader-id= -2024-05-15T01:17:37.926Z [INFO] server-45ca580f-f01c-31e9-00d3-62c35b44f662: entering follower state: follower="Node at 45ca580f-f01c-31e9-00d3-62c35b44f662 [Follower]" leader-address= leader-id= -2024-05-15T01:17:38.566Z [WARN] server-5d2726b3-c9c6-86ca-facf-c3523c967221: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:38.566Z [INFO] server-5d2726b3-c9c6-86ca-facf-c3523c967221: entering candidate state: node="Node at 5d2726b3-c9c6-86ca-facf-c3523c967221 [Candidate]" term=2 -2024-05-15T01:17:38.566Z [INFO] server-5d2726b3-c9c6-86ca-facf-c3523c967221: election won: term=2 tally=2 -2024-05-15T01:17:38.566Z [INFO] server-5d2726b3-c9c6-86ca-facf-c3523c967221: entering leader state: leader="Node at 5d2726b3-c9c6-86ca-facf-c3523c967221 [Leader]" -2024-05-15T01:17:38.566Z [INFO] server-5d2726b3-c9c6-86ca-facf-c3523c967221: added peer, starting replication: peer=server-45ca580f-f01c-31e9-00d3-62c35b44f662 -2024-05-15T01:17:38.566Z [INFO] server-5d2726b3-c9c6-86ca-facf-c3523c967221: added peer, starting replication: peer=server-16077a32-3b1f-9b57-935e-0c599fdf72f8 -2024-05-15T01:17:38.566Z [INFO] server-5d2726b3-c9c6-86ca-facf-c3523c967221: pipelining replication: peer="{Voter server-16077a32-3b1f-9b57-935e-0c599fdf72f8 16077a32-3b1f-9b57-935e-0c599fdf72f8}" -2024-05-15T01:17:38.566Z [INFO] server-5d2726b3-c9c6-86ca-facf-c3523c967221: pipelining replication: peer="{Voter server-45ca580f-f01c-31e9-00d3-62c35b44f662 45ca580f-f01c-31e9-00d3-62c35b44f662}" -2024-05-15T01:17:39.572Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:38.567034769 +0000 UTC m=+39.108419526 (1 nodes), 640.903118ms from start of poll, 641.522045ms from cluster start. Timeout at 2024-05-15 01:17:39.572749402 +0000 UTC m=+40.114134131, 1.005714605s after stability -2024-05-15T01:17:39.573Z [INFO] creating new snapshot: path=/tmp/raft3405704586/snapshots/2-14-1715735859573.tmp -2024-05-15T01:17:39.573Z [INFO] server-5d2726b3-c9c6-86ca-facf-c3523c967221: copied to local snapshot: bytes=71 -2024-05-15T01:17:39.574Z [INFO] server-5d2726b3-c9c6-86ca-facf-c3523c967221: snapshot restore progress: id=2-14-1715735859573 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:39.574Z [INFO] server-5d2726b3-c9c6-86ca-facf-c3523c967221: removing all old logs from log store -2024-05-15T01:17:39.574Z [INFO] server-5d2726b3-c9c6-86ca-facf-c3523c967221: compacting logs: from=1 to=2 -2024-05-15T01:17:39.574Z [INFO] server-5d2726b3-c9c6-86ca-facf-c3523c967221: restored user snapshot: index=14 -2024-05-15T01:17:39.574Z [ERROR] server-5d2726b3-c9c6-86ca-facf-c3523c967221: failed to get log: index=2 error="log not found" -2024-05-15T01:17:39.574Z [INFO] server-5d2726b3-c9c6-86ca-facf-c3523c967221: aborting pipeline replication: peer="{Voter server-16077a32-3b1f-9b57-935e-0c599fdf72f8 16077a32-3b1f-9b57-935e-0c599fdf72f8}" -2024-05-15T01:17:39.574Z [ERROR] server-5d2726b3-c9c6-86ca-facf-c3523c967221: failed to get log: index=2 error="log not found" -2024-05-15T01:17:39.574Z [INFO] server-5d2726b3-c9c6-86ca-facf-c3523c967221: aborting pipeline replication: peer="{Voter server-45ca580f-f01c-31e9-00d3-62c35b44f662 45ca580f-f01c-31e9-00d3-62c35b44f662}" -2024-05-15T01:17:39.581Z [ERROR] server-5d2726b3-c9c6-86ca-facf-c3523c967221: failed to get log: index=2 error="log not found" -2024-05-15T01:17:39.581Z [INFO] creating new snapshot: path=/tmp/raft481331869/snapshots/2-14-1715735859581.tmp -2024-05-15T01:17:39.586Z [ERROR] server-5d2726b3-c9c6-86ca-facf-c3523c967221: failed to get log: index=2 error="log not found" -2024-05-15T01:17:39.587Z [INFO] creating new snapshot: path=/tmp/raft2691155814/snapshots/2-14-1715735859587.tmp -2024-05-15T01:17:39.587Z [INFO] server-45ca580f-f01c-31e9-00d3-62c35b44f662: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:39.587Z [INFO] server-45ca580f-f01c-31e9-00d3-62c35b44f662: copied to local snapshot: bytes=71 -2024-05-15T01:17:39.589Z [INFO] server-16077a32-3b1f-9b57-935e-0c599fdf72f8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:39.589Z [INFO] server-16077a32-3b1f-9b57-935e-0c599fdf72f8: copied to local snapshot: bytes=71 -2024-05-15T01:17:39.589Z [INFO] server-16077a32-3b1f-9b57-935e-0c599fdf72f8: snapshot restore progress: id=2-14-1715735859581 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:39.589Z [INFO] server-16077a32-3b1f-9b57-935e-0c599fdf72f8: removing all old logs from log store -2024-05-15T01:17:39.589Z [INFO] server-16077a32-3b1f-9b57-935e-0c599fdf72f8: compacting logs: from=1 to=2 -2024-05-15T01:17:39.589Z [INFO] server-16077a32-3b1f-9b57-935e-0c599fdf72f8: Installed remote snapshot -2024-05-15T01:17:39.590Z [INFO] server-45ca580f-f01c-31e9-00d3-62c35b44f662: snapshot restore progress: id=2-14-1715735859587 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:39.590Z [INFO] server-45ca580f-f01c-31e9-00d3-62c35b44f662: removing all old logs from log store -2024-05-15T01:17:39.590Z [INFO] server-45ca580f-f01c-31e9-00d3-62c35b44f662: compacting logs: from=1 to=2 -2024-05-15T01:17:39.590Z [INFO] server-45ca580f-f01c-31e9-00d3-62c35b44f662: Installed remote snapshot -2024-05-15T01:17:39.590Z [INFO] server-5d2726b3-c9c6-86ca-facf-c3523c967221: pipelining replication: peer="{Voter server-16077a32-3b1f-9b57-935e-0c599fdf72f8 16077a32-3b1f-9b57-935e-0c599fdf72f8}" -2024-05-15T01:17:39.590Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:39.590Z [INFO] server-5d2726b3-c9c6-86ca-facf-c3523c967221: pipelining replication: peer="{Voter server-45ca580f-f01c-31e9-00d3-62c35b44f662 45ca580f-f01c-31e9-00d3-62c35b44f662}" -2024-05-15T01:17:40.595Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:39.590345539 +0000 UTC m=+40.131730465 (2 nodes), 2.256µs from start of poll, 1.664832984s from cluster start. Timeout at 2024-05-15 01:17:40.59556341 +0000 UTC m=+41.136948202, 1.005217737s after stability -2024-05-15T01:17:40.609Z [INFO] server-ab51b0e7-da7b-9c8d-77ee-2617055787ed: aborting pipeline replication: peer="{Voter server-3ee5d55a-f7d0-695c-cc70-f94360b37c8c 3ee5d55a-f7d0-695c-cc70-f94360b37c8c}" -2024-05-15T01:17:40.609Z [INFO] server-ab51b0e7-da7b-9c8d-77ee-2617055787ed: aborting pipeline replication: peer="{Voter server-87509249-080f-5bd9-2749-19d42474f8b3 87509249-080f-5bd9-2749-19d42474f8b3}" +2025-06-17T07:43:28.810Z [INFO] server-ebd5554a-8535-9c4d-db9d-c36ae00919c4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ebd5554a-8535-9c4d-db9d-c36ae00919c4 Address:ebd5554a-8535-9c4d-db9d-c36ae00919c4} {Suffrage:Voter ID:server-ad7277e0-2e4d-806e-ea1d-733800db3ccd Address:ad7277e0-2e4d-806e-ea1d-733800db3ccd} {Suffrage:Voter ID:server-ea6b215e-462b-de01-26bc-21a6af461999 Address:ea6b215e-462b-de01-26bc-21a6af461999}]" +2025-06-17T07:43:28.811Z [INFO] server-ebd5554a-8535-9c4d-db9d-c36ae00919c4: entering follower state: follower="Node at ebd5554a-8535-9c4d-db9d-c36ae00919c4 [Follower]" leader-address= leader-id= +2025-06-17T07:43:28.811Z [INFO] server-ad7277e0-2e4d-806e-ea1d-733800db3ccd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ebd5554a-8535-9c4d-db9d-c36ae00919c4 Address:ebd5554a-8535-9c4d-db9d-c36ae00919c4} {Suffrage:Voter ID:server-ad7277e0-2e4d-806e-ea1d-733800db3ccd Address:ad7277e0-2e4d-806e-ea1d-733800db3ccd} {Suffrage:Voter ID:server-ea6b215e-462b-de01-26bc-21a6af461999 Address:ea6b215e-462b-de01-26bc-21a6af461999}]" +2025-06-17T07:43:28.811Z [INFO] server-ad7277e0-2e4d-806e-ea1d-733800db3ccd: entering follower state: follower="Node at ad7277e0-2e4d-806e-ea1d-733800db3ccd [Follower]" leader-address= leader-id= +2025-06-17T07:43:28.811Z [INFO] server-ea6b215e-462b-de01-26bc-21a6af461999: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ebd5554a-8535-9c4d-db9d-c36ae00919c4 Address:ebd5554a-8535-9c4d-db9d-c36ae00919c4} {Suffrage:Voter ID:server-ad7277e0-2e4d-806e-ea1d-733800db3ccd Address:ad7277e0-2e4d-806e-ea1d-733800db3ccd} {Suffrage:Voter ID:server-ea6b215e-462b-de01-26bc-21a6af461999 Address:ea6b215e-462b-de01-26bc-21a6af461999}]" +2025-06-17T07:43:28.811Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:28.811Z [INFO] server-ea6b215e-462b-de01-26bc-21a6af461999: entering follower state: follower="Node at ea6b215e-462b-de01-26bc-21a6af461999 [Follower]" leader-address= leader-id= +2025-06-17T07:43:29.336Z [WARN] server-ea6b215e-462b-de01-26bc-21a6af461999: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:29.336Z [INFO] server-ea6b215e-462b-de01-26bc-21a6af461999: entering candidate state: node="Node at ea6b215e-462b-de01-26bc-21a6af461999 [Candidate]" term=2 +2025-06-17T07:43:29.337Z [INFO] server-ea6b215e-462b-de01-26bc-21a6af461999: election won: term=2 tally=2 +2025-06-17T07:43:29.337Z [INFO] server-ea6b215e-462b-de01-26bc-21a6af461999: entering leader state: leader="Node at ea6b215e-462b-de01-26bc-21a6af461999 [Leader]" +2025-06-17T07:43:29.337Z [INFO] server-ea6b215e-462b-de01-26bc-21a6af461999: added peer, starting replication: peer=server-ebd5554a-8535-9c4d-db9d-c36ae00919c4 +2025-06-17T07:43:29.337Z [INFO] server-ea6b215e-462b-de01-26bc-21a6af461999: added peer, starting replication: peer=server-ad7277e0-2e4d-806e-ea1d-733800db3ccd +2025-06-17T07:43:29.337Z [INFO] server-ea6b215e-462b-de01-26bc-21a6af461999: pipelining replication: peer="{Voter server-ad7277e0-2e4d-806e-ea1d-733800db3ccd ad7277e0-2e4d-806e-ea1d-733800db3ccd}" +2025-06-17T07:43:29.337Z [INFO] server-ea6b215e-462b-de01-26bc-21a6af461999: pipelining replication: peer="{Voter server-ebd5554a-8535-9c4d-db9d-c36ae00919c4 ebd5554a-8535-9c4d-db9d-c36ae00919c4}" +2025-06-17T07:43:30.342Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:29.337321699 +0000 UTC m=+37.940850419 (1 nodes), 526.183848ms from start of poll, 526.490537ms from cluster start. Timeout at 2025-06-17 07:43:30.342687593 +0000 UTC m=+38.946216311, 1.005365892s after stability +2025-06-17T07:43:30.343Z [INFO] server-ea6b215e-462b-de01-26bc-21a6af461999: starting snapshot up to: index=12 +2025-06-17T07:43:30.343Z [INFO] creating new snapshot: path=/tmp/raft1015159532/snapshots/2-12-1750146210343.tmp +2025-06-17T07:43:30.343Z [INFO] server-ea6b215e-462b-de01-26bc-21a6af461999: snapshot complete up to: index=12 +2025-06-17T07:43:30.344Z [INFO] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f6756e11-2df7-52ac-43c4-b1956d5035bd Address:f6756e11-2df7-52ac-43c4-b1956d5035bd} {Suffrage:Voter ID:server-dc86a935-33cf-4fa2-568b-106fbcceeee8 Address:dc86a935-33cf-4fa2-568b-106fbcceeee8} {Suffrage:Voter ID:server-8a8b9edb-b8ad-b45e-e075-68839817cd16 Address:8a8b9edb-b8ad-b45e-e075-68839817cd16}]" +2025-06-17T07:43:30.344Z [INFO] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: entering follower state: follower="Node at f6756e11-2df7-52ac-43c4-b1956d5035bd [Follower]" leader-address= leader-id= +2025-06-17T07:43:30.344Z [INFO] server-dc86a935-33cf-4fa2-568b-106fbcceeee8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f6756e11-2df7-52ac-43c4-b1956d5035bd Address:f6756e11-2df7-52ac-43c4-b1956d5035bd} {Suffrage:Voter ID:server-dc86a935-33cf-4fa2-568b-106fbcceeee8 Address:dc86a935-33cf-4fa2-568b-106fbcceeee8} {Suffrage:Voter ID:server-8a8b9edb-b8ad-b45e-e075-68839817cd16 Address:8a8b9edb-b8ad-b45e-e075-68839817cd16}]" +2025-06-17T07:43:30.345Z [INFO] server-dc86a935-33cf-4fa2-568b-106fbcceeee8: entering follower state: follower="Node at dc86a935-33cf-4fa2-568b-106fbcceeee8 [Follower]" leader-address= leader-id= +2025-06-17T07:43:30.345Z [INFO] server-8a8b9edb-b8ad-b45e-e075-68839817cd16: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f6756e11-2df7-52ac-43c4-b1956d5035bd Address:f6756e11-2df7-52ac-43c4-b1956d5035bd} {Suffrage:Voter ID:server-dc86a935-33cf-4fa2-568b-106fbcceeee8 Address:dc86a935-33cf-4fa2-568b-106fbcceeee8} {Suffrage:Voter ID:server-8a8b9edb-b8ad-b45e-e075-68839817cd16 Address:8a8b9edb-b8ad-b45e-e075-68839817cd16}]" +2025-06-17T07:43:30.345Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:30.345Z [INFO] server-8a8b9edb-b8ad-b45e-e075-68839817cd16: entering follower state: follower="Node at 8a8b9edb-b8ad-b45e-e075-68839817cd16 [Follower]" leader-address= leader-id= +2025-06-17T07:43:30.918Z [WARN] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:30.918Z [INFO] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: entering candidate state: node="Node at f6756e11-2df7-52ac-43c4-b1956d5035bd [Candidate]" term=2 +2025-06-17T07:43:30.918Z [INFO] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: election won: term=2 tally=2 +2025-06-17T07:43:30.918Z [INFO] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: entering leader state: leader="Node at f6756e11-2df7-52ac-43c4-b1956d5035bd [Leader]" +2025-06-17T07:43:30.918Z [INFO] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: added peer, starting replication: peer=server-dc86a935-33cf-4fa2-568b-106fbcceeee8 +2025-06-17T07:43:30.918Z [INFO] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: added peer, starting replication: peer=server-8a8b9edb-b8ad-b45e-e075-68839817cd16 +2025-06-17T07:43:30.918Z [INFO] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: pipelining replication: peer="{Voter server-8a8b9edb-b8ad-b45e-e075-68839817cd16 8a8b9edb-b8ad-b45e-e075-68839817cd16}" +2025-06-17T07:43:30.919Z [INFO] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: pipelining replication: peer="{Voter server-dc86a935-33cf-4fa2-568b-106fbcceeee8 dc86a935-33cf-4fa2-568b-106fbcceeee8}" +2025-06-17T07:43:31.924Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:30.91887761 +0000 UTC m=+39.522406300 (1 nodes), 573.735188ms from start of poll, 574.15919ms from cluster start. Timeout at 2025-06-17 07:43:31.924433407 +0000 UTC m=+40.527962112, 1.005555812s after stability +2025-06-17T07:43:31.924Z [INFO] creating new snapshot: path=/tmp/raft553156693/snapshots/2-14-1750146211924.tmp +2025-06-17T07:43:31.925Z [INFO] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: copied to local snapshot: bytes=71 +2025-06-17T07:43:31.925Z [INFO] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: snapshot restore progress: id=2-14-1750146211924 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:31.925Z [INFO] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: removing all old logs from log store +2025-06-17T07:43:31.925Z [INFO] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: compacting logs: from=1 to=2 +2025-06-17T07:43:31.925Z [INFO] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: restored user snapshot: index=14 +2025-06-17T07:43:31.925Z [ERROR] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: failed to get log: index=2 error="log not found" +2025-06-17T07:43:31.925Z [INFO] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: aborting pipeline replication: peer="{Voter server-8a8b9edb-b8ad-b45e-e075-68839817cd16 8a8b9edb-b8ad-b45e-e075-68839817cd16}" +2025-06-17T07:43:31.925Z [ERROR] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: failed to get log: index=2 error="log not found" +2025-06-17T07:43:31.925Z [INFO] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: aborting pipeline replication: peer="{Voter server-dc86a935-33cf-4fa2-568b-106fbcceeee8 dc86a935-33cf-4fa2-568b-106fbcceeee8}" +2025-06-17T07:43:31.932Z [ERROR] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: failed to get log: index=2 error="log not found" +2025-06-17T07:43:31.932Z [INFO] creating new snapshot: path=/tmp/raft3153712279/snapshots/2-14-1750146211932.tmp +2025-06-17T07:43:31.932Z [INFO] server-8a8b9edb-b8ad-b45e-e075-68839817cd16: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:31.932Z [INFO] server-8a8b9edb-b8ad-b45e-e075-68839817cd16: copied to local snapshot: bytes=71 +2025-06-17T07:43:31.933Z [INFO] server-8a8b9edb-b8ad-b45e-e075-68839817cd16: snapshot restore progress: id=2-14-1750146211932 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:31.933Z [INFO] server-8a8b9edb-b8ad-b45e-e075-68839817cd16: removing all old logs from log store +2025-06-17T07:43:31.933Z [INFO] server-8a8b9edb-b8ad-b45e-e075-68839817cd16: compacting logs: from=1 to=2 +2025-06-17T07:43:31.933Z [INFO] server-8a8b9edb-b8ad-b45e-e075-68839817cd16: Installed remote snapshot +2025-06-17T07:43:31.933Z [INFO] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: pipelining replication: peer="{Voter server-8a8b9edb-b8ad-b45e-e075-68839817cd16 8a8b9edb-b8ad-b45e-e075-68839817cd16}" +2025-06-17T07:43:31.933Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:31.933Z [ERROR] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: failed to get log: index=2 error="log not found" +2025-06-17T07:43:31.933Z [INFO] creating new snapshot: path=/tmp/raft3577809826/snapshots/2-14-1750146211933.tmp +2025-06-17T07:43:31.933Z [INFO] server-dc86a935-33cf-4fa2-568b-106fbcceeee8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:31.933Z [INFO] server-dc86a935-33cf-4fa2-568b-106fbcceeee8: copied to local snapshot: bytes=71 +2025-06-17T07:43:31.934Z [INFO] server-dc86a935-33cf-4fa2-568b-106fbcceeee8: snapshot restore progress: id=2-14-1750146211933 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:31.934Z [INFO] server-dc86a935-33cf-4fa2-568b-106fbcceeee8: removing all old logs from log store +2025-06-17T07:43:31.934Z [INFO] server-dc86a935-33cf-4fa2-568b-106fbcceeee8: compacting logs: from=1 to=2 +2025-06-17T07:43:31.934Z [INFO] server-dc86a935-33cf-4fa2-568b-106fbcceeee8: Installed remote snapshot +2025-06-17T07:43:31.934Z [INFO] server-f6756e11-2df7-52ac-43c4-b1956d5035bd: pipelining replication: peer="{Voter server-dc86a935-33cf-4fa2-568b-106fbcceeee8 dc86a935-33cf-4fa2-568b-106fbcceeee8}" +2025-06-17T07:43:32.938Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:31.933238539 +0000 UTC m=+40.536767226 (2 nodes), 632ns from start of poll, 1.588520116s from cluster start. Timeout at 2025-06-17 07:43:32.938586348 +0000 UTC m=+41.542115028, 1.005347802s after stability +2025-06-17T07:43:32.950Z [INFO] server-ea6b215e-462b-de01-26bc-21a6af461999: aborting pipeline replication: peer="{Voter server-ad7277e0-2e4d-806e-ea1d-733800db3ccd ad7277e0-2e4d-806e-ea1d-733800db3ccd}" +2025-06-17T07:43:32.950Z [INFO] server-ea6b215e-462b-de01-26bc-21a6af461999: aborting pipeline replication: peer="{Voter server-ebd5554a-8535-9c4d-db9d-c36ae00919c4 ebd5554a-8535-9c4d-db9d-c36ae00919c4}" === RUN TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false -2024-05-15T01:17:40.610Z [INFO] server-de2b0fb7-6e42-16c0-8d67-ffb8135e11c9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-de2b0fb7-6e42-16c0-8d67-ffb8135e11c9 Address:de2b0fb7-6e42-16c0-8d67-ffb8135e11c9} {Suffrage:Voter ID:server-ad064760-d8dd-c2c9-996b-2fbc28350783 Address:ad064760-d8dd-c2c9-996b-2fbc28350783} {Suffrage:Voter ID:server-e0954c19-0133-1fe9-fb78-7def5457e5d3 Address:e0954c19-0133-1fe9-fb78-7def5457e5d3}]" -2024-05-15T01:17:40.611Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-de2b0fb7-6e42-16c0-8d67-ffb8135e11c9 Address:de2b0fb7-6e42-16c0-8d67-ffb8135e11c9} {Suffrage:Voter ID:server-ad064760-d8dd-c2c9-996b-2fbc28350783 Address:ad064760-d8dd-c2c9-996b-2fbc28350783} {Suffrage:Voter ID:server-e0954c19-0133-1fe9-fb78-7def5457e5d3 Address:e0954c19-0133-1fe9-fb78-7def5457e5d3}]" -2024-05-15T01:17:40.611Z [INFO] server-e0954c19-0133-1fe9-fb78-7def5457e5d3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-de2b0fb7-6e42-16c0-8d67-ffb8135e11c9 Address:de2b0fb7-6e42-16c0-8d67-ffb8135e11c9} {Suffrage:Voter ID:server-ad064760-d8dd-c2c9-996b-2fbc28350783 Address:ad064760-d8dd-c2c9-996b-2fbc28350783} {Suffrage:Voter ID:server-e0954c19-0133-1fe9-fb78-7def5457e5d3 Address:e0954c19-0133-1fe9-fb78-7def5457e5d3}]" -2024-05-15T01:17:40.611Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:40.611Z [INFO] server-de2b0fb7-6e42-16c0-8d67-ffb8135e11c9: entering follower state: follower="Node at de2b0fb7-6e42-16c0-8d67-ffb8135e11c9 [Follower]" leader-address= leader-id= -2024-05-15T01:17:40.611Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: entering follower state: follower="Node at ad064760-d8dd-c2c9-996b-2fbc28350783 [Follower]" leader-address= leader-id= -2024-05-15T01:17:40.611Z [INFO] server-e0954c19-0133-1fe9-fb78-7def5457e5d3: entering follower state: follower="Node at e0954c19-0133-1fe9-fb78-7def5457e5d3 [Follower]" leader-address= leader-id= -2024-05-15T01:17:41.363Z [WARN] server-ad064760-d8dd-c2c9-996b-2fbc28350783: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:41.363Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: entering candidate state: node="Node at ad064760-d8dd-c2c9-996b-2fbc28350783 [Candidate]" term=2 -2024-05-15T01:17:41.363Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: election won: term=2 tally=2 -2024-05-15T01:17:41.363Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: entering leader state: leader="Node at ad064760-d8dd-c2c9-996b-2fbc28350783 [Leader]" -2024-05-15T01:17:41.363Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: added peer, starting replication: peer=server-de2b0fb7-6e42-16c0-8d67-ffb8135e11c9 -2024-05-15T01:17:41.363Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: added peer, starting replication: peer=server-e0954c19-0133-1fe9-fb78-7def5457e5d3 -2024-05-15T01:17:41.363Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: pipelining replication: peer="{Voter server-e0954c19-0133-1fe9-fb78-7def5457e5d3 e0954c19-0133-1fe9-fb78-7def5457e5d3}" -2024-05-15T01:17:41.364Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: pipelining replication: peer="{Voter server-de2b0fb7-6e42-16c0-8d67-ffb8135e11c9 de2b0fb7-6e42-16c0-8d67-ffb8135e11c9}" -2024-05-15T01:17:42.369Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:41.364307595 +0000 UTC m=+41.905692420 (1 nodes), 753.036799ms from start of poll, 753.516582ms from cluster start. Timeout at 2024-05-15 01:17:42.369537207 +0000 UTC m=+42.910922013, 1.005229593s after stability -2024-05-15T01:17:42.370Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: starting snapshot up to: index=12 -2024-05-15T01:17:42.370Z [INFO] creating new snapshot: path=/tmp/raft3239165210/snapshots/2-12-1715735862370.tmp -2024-05-15T01:17:42.371Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: snapshot complete up to: index=12 -2024-05-15T01:17:42.373Z [INFO] creating new snapshot: path=/tmp/raft3239165210/snapshots/2-23-1715735862373.tmp -2024-05-15T01:17:42.374Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: copied to local snapshot: bytes=71 -2024-05-15T01:17:42.375Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: snapshot restore progress: id=2-23-1715735862373 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:42.375Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: restored user snapshot: index=23 -2024-05-15T01:17:42.375Z [ERROR] server-ad064760-d8dd-c2c9-996b-2fbc28350783: failed to get log: index=23 error="log not found" -2024-05-15T01:17:42.375Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: aborting pipeline replication: peer="{Voter server-de2b0fb7-6e42-16c0-8d67-ffb8135e11c9 de2b0fb7-6e42-16c0-8d67-ffb8135e11c9}" -2024-05-15T01:17:42.375Z [ERROR] server-ad064760-d8dd-c2c9-996b-2fbc28350783: failed to get log: index=23 error="log not found" -2024-05-15T01:17:42.375Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: aborting pipeline replication: peer="{Voter server-e0954c19-0133-1fe9-fb78-7def5457e5d3 e0954c19-0133-1fe9-fb78-7def5457e5d3}" -2024-05-15T01:17:42.382Z [ERROR] server-ad064760-d8dd-c2c9-996b-2fbc28350783: failed to get log: index=23 error="log not found" -2024-05-15T01:17:42.383Z [INFO] creating new snapshot: path=/tmp/raft1534278778/snapshots/2-23-1715735862383.tmp -2024-05-15T01:17:42.383Z [INFO] server-e0954c19-0133-1fe9-fb78-7def5457e5d3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:42.384Z [INFO] server-e0954c19-0133-1fe9-fb78-7def5457e5d3: copied to local snapshot: bytes=71 -2024-05-15T01:17:42.384Z [ERROR] server-ad064760-d8dd-c2c9-996b-2fbc28350783: failed to get log: index=23 error="log not found" -2024-05-15T01:17:42.384Z [INFO] server-e0954c19-0133-1fe9-fb78-7def5457e5d3: snapshot restore progress: id=2-23-1715735862383 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:42.385Z [INFO] server-e0954c19-0133-1fe9-fb78-7def5457e5d3: Installed remote snapshot -2024-05-15T01:17:42.385Z [INFO] creating new snapshot: path=/tmp/raft2768070854/snapshots/2-23-1715735862385.tmp -2024-05-15T01:17:42.385Z [WARN] server-e0954c19-0133-1fe9-fb78-7def5457e5d3: skipping application of old log: index=22 -2024-05-15T01:17:42.385Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: pipelining replication: peer="{Voter server-e0954c19-0133-1fe9-fb78-7def5457e5d3 e0954c19-0133-1fe9-fb78-7def5457e5d3}" -2024-05-15T01:17:42.385Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:42.385Z [INFO] server-de2b0fb7-6e42-16c0-8d67-ffb8135e11c9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:42.386Z [INFO] server-de2b0fb7-6e42-16c0-8d67-ffb8135e11c9: copied to local snapshot: bytes=71 -2024-05-15T01:17:42.386Z [INFO] server-de2b0fb7-6e42-16c0-8d67-ffb8135e11c9: snapshot restore progress: id=2-23-1715735862385 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:42.386Z [INFO] server-de2b0fb7-6e42-16c0-8d67-ffb8135e11c9: Installed remote snapshot -2024-05-15T01:17:42.386Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: pipelining replication: peer="{Voter server-de2b0fb7-6e42-16c0-8d67-ffb8135e11c9 de2b0fb7-6e42-16c0-8d67-ffb8135e11c9}" -2024-05-15T01:17:43.393Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:42.385584121 +0000 UTC m=+42.926969337 (2 nodes), 2.655µs from start of poll, 1.774793499s from cluster start. Timeout at 2024-05-15 01:17:43.393409908 +0000 UTC m=+43.934794933, 1.007825596s after stability -2024-05-15T01:17:43.413Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: aborting pipeline replication: peer="{Voter server-e0954c19-0133-1fe9-fb78-7def5457e5d3 e0954c19-0133-1fe9-fb78-7def5457e5d3}" -2024-05-15T01:17:43.413Z [INFO] server-ad064760-d8dd-c2c9-996b-2fbc28350783: aborting pipeline replication: peer="{Voter server-de2b0fb7-6e42-16c0-8d67-ffb8135e11c9 de2b0fb7-6e42-16c0-8d67-ffb8135e11c9}" +2025-06-17T07:43:32.951Z [INFO] server-7a46dd9d-28b9-60ee-76ec-7f081fa22e7c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a46dd9d-28b9-60ee-76ec-7f081fa22e7c Address:7a46dd9d-28b9-60ee-76ec-7f081fa22e7c} {Suffrage:Voter ID:server-d3c1d566-32c3-c8d9-f957-47872dd535db Address:d3c1d566-32c3-c8d9-f957-47872dd535db} {Suffrage:Voter ID:server-20cbff90-4747-86b9-8d39-acb7eb957f0c Address:20cbff90-4747-86b9-8d39-acb7eb957f0c}]" +2025-06-17T07:43:32.951Z [INFO] server-7a46dd9d-28b9-60ee-76ec-7f081fa22e7c: entering follower state: follower="Node at 7a46dd9d-28b9-60ee-76ec-7f081fa22e7c [Follower]" leader-address= leader-id= +2025-06-17T07:43:32.951Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a46dd9d-28b9-60ee-76ec-7f081fa22e7c Address:7a46dd9d-28b9-60ee-76ec-7f081fa22e7c} {Suffrage:Voter ID:server-d3c1d566-32c3-c8d9-f957-47872dd535db Address:d3c1d566-32c3-c8d9-f957-47872dd535db} {Suffrage:Voter ID:server-20cbff90-4747-86b9-8d39-acb7eb957f0c Address:20cbff90-4747-86b9-8d39-acb7eb957f0c}]" +2025-06-17T07:43:32.952Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: entering follower state: follower="Node at d3c1d566-32c3-c8d9-f957-47872dd535db [Follower]" leader-address= leader-id= +2025-06-17T07:43:32.952Z [INFO] server-20cbff90-4747-86b9-8d39-acb7eb957f0c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a46dd9d-28b9-60ee-76ec-7f081fa22e7c Address:7a46dd9d-28b9-60ee-76ec-7f081fa22e7c} {Suffrage:Voter ID:server-d3c1d566-32c3-c8d9-f957-47872dd535db Address:d3c1d566-32c3-c8d9-f957-47872dd535db} {Suffrage:Voter ID:server-20cbff90-4747-86b9-8d39-acb7eb957f0c Address:20cbff90-4747-86b9-8d39-acb7eb957f0c}]" +2025-06-17T07:43:32.952Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:32.952Z [INFO] server-20cbff90-4747-86b9-8d39-acb7eb957f0c: entering follower state: follower="Node at 20cbff90-4747-86b9-8d39-acb7eb957f0c [Follower]" leader-address= leader-id= +2025-06-17T07:43:33.549Z [WARN] server-d3c1d566-32c3-c8d9-f957-47872dd535db: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:33.549Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: entering candidate state: node="Node at d3c1d566-32c3-c8d9-f957-47872dd535db [Candidate]" term=2 +2025-06-17T07:43:33.549Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: election won: term=2 tally=2 +2025-06-17T07:43:33.549Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: entering leader state: leader="Node at d3c1d566-32c3-c8d9-f957-47872dd535db [Leader]" +2025-06-17T07:43:33.549Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: added peer, starting replication: peer=server-7a46dd9d-28b9-60ee-76ec-7f081fa22e7c +2025-06-17T07:43:33.549Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: added peer, starting replication: peer=server-20cbff90-4747-86b9-8d39-acb7eb957f0c +2025-06-17T07:43:33.549Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: pipelining replication: peer="{Voter server-20cbff90-4747-86b9-8d39-acb7eb957f0c 20cbff90-4747-86b9-8d39-acb7eb957f0c}" +2025-06-17T07:43:33.549Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: pipelining replication: peer="{Voter server-7a46dd9d-28b9-60ee-76ec-7f081fa22e7c 7a46dd9d-28b9-60ee-76ec-7f081fa22e7c}" +2025-06-17T07:43:34.554Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:33.549373125 +0000 UTC m=+42.152901840 (1 nodes), 597.295413ms from start of poll, 597.571119ms from cluster start. Timeout at 2025-06-17 07:43:34.554776255 +0000 UTC m=+43.158304973, 1.005403133s after stability +2025-06-17T07:43:34.555Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: starting snapshot up to: index=12 +2025-06-17T07:43:34.555Z [INFO] creating new snapshot: path=/tmp/raft3518395512/snapshots/2-12-1750146214555.tmp +2025-06-17T07:43:34.556Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: snapshot complete up to: index=12 +2025-06-17T07:43:34.557Z [INFO] creating new snapshot: path=/tmp/raft3518395512/snapshots/2-23-1750146214557.tmp +2025-06-17T07:43:34.557Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: copied to local snapshot: bytes=71 +2025-06-17T07:43:34.558Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: snapshot restore progress: id=2-23-1750146214557 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:34.558Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: restored user snapshot: index=23 +2025-06-17T07:43:34.558Z [ERROR] server-d3c1d566-32c3-c8d9-f957-47872dd535db: failed to get log: index=23 error="log not found" +2025-06-17T07:43:34.558Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: aborting pipeline replication: peer="{Voter server-20cbff90-4747-86b9-8d39-acb7eb957f0c 20cbff90-4747-86b9-8d39-acb7eb957f0c}" +2025-06-17T07:43:34.558Z [ERROR] server-d3c1d566-32c3-c8d9-f957-47872dd535db: failed to get log: index=23 error="log not found" +2025-06-17T07:43:34.558Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: aborting pipeline replication: peer="{Voter server-7a46dd9d-28b9-60ee-76ec-7f081fa22e7c 7a46dd9d-28b9-60ee-76ec-7f081fa22e7c}" +2025-06-17T07:43:34.565Z [ERROR] server-d3c1d566-32c3-c8d9-f957-47872dd535db: failed to get log: index=23 error="log not found" +2025-06-17T07:43:34.565Z [INFO] creating new snapshot: path=/tmp/raft3759747065/snapshots/2-23-1750146214565.tmp +2025-06-17T07:43:34.565Z [ERROR] server-d3c1d566-32c3-c8d9-f957-47872dd535db: failed to get log: index=23 error="log not found" +2025-06-17T07:43:34.565Z [INFO] server-7a46dd9d-28b9-60ee-76ec-7f081fa22e7c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:34.566Z [INFO] creating new snapshot: path=/tmp/raft2020867690/snapshots/2-23-1750146214566.tmp +2025-06-17T07:43:34.566Z [INFO] server-7a46dd9d-28b9-60ee-76ec-7f081fa22e7c: copied to local snapshot: bytes=71 +2025-06-17T07:43:34.566Z [INFO] server-20cbff90-4747-86b9-8d39-acb7eb957f0c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:34.566Z [INFO] server-7a46dd9d-28b9-60ee-76ec-7f081fa22e7c: snapshot restore progress: id=2-23-1750146214565 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:34.566Z [INFO] server-7a46dd9d-28b9-60ee-76ec-7f081fa22e7c: Installed remote snapshot +2025-06-17T07:43:34.566Z [WARN] server-7a46dd9d-28b9-60ee-76ec-7f081fa22e7c: skipping application of old log: index=22 +2025-06-17T07:43:34.566Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: pipelining replication: peer="{Voter server-7a46dd9d-28b9-60ee-76ec-7f081fa22e7c 7a46dd9d-28b9-60ee-76ec-7f081fa22e7c}" +2025-06-17T07:43:34.566Z [INFO] server-20cbff90-4747-86b9-8d39-acb7eb957f0c: copied to local snapshot: bytes=71 +2025-06-17T07:43:34.566Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:34.566Z [INFO] server-20cbff90-4747-86b9-8d39-acb7eb957f0c: snapshot restore progress: id=2-23-1750146214566 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:34.567Z [INFO] server-20cbff90-4747-86b9-8d39-acb7eb957f0c: Installed remote snapshot +2025-06-17T07:43:34.567Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: pipelining replication: peer="{Voter server-20cbff90-4747-86b9-8d39-acb7eb957f0c 20cbff90-4747-86b9-8d39-acb7eb957f0c}" +2025-06-17T07:43:35.572Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:34.566879178 +0000 UTC m=+43.170407865 (2 nodes), 1.125µs from start of poll, 1.615077144s from cluster start. Timeout at 2025-06-17 07:43:35.572594026 +0000 UTC m=+44.176122718, 1.005714853s after stability +2025-06-17T07:43:35.583Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: aborting pipeline replication: peer="{Voter server-20cbff90-4747-86b9-8d39-acb7eb957f0c 20cbff90-4747-86b9-8d39-acb7eb957f0c}" +2025-06-17T07:43:35.583Z [INFO] server-d3c1d566-32c3-c8d9-f957-47872dd535db: aborting pipeline replication: peer="{Voter server-7a46dd9d-28b9-60ee-76ec-7f081fa22e7c 7a46dd9d-28b9-60ee-76ec-7f081fa22e7c}" === RUN TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false -2024-05-15T01:17:43.415Z [INFO] server-f21db385-69b0-b2cc-41ee-252a92b79dd5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f21db385-69b0-b2cc-41ee-252a92b79dd5 Address:f21db385-69b0-b2cc-41ee-252a92b79dd5} {Suffrage:Voter ID:server-e323e934-0c86-9083-5d8b-44bb650636d6 Address:e323e934-0c86-9083-5d8b-44bb650636d6} {Suffrage:Voter ID:server-85bd68c9-6d05-bcab-d357-6693d9d2661c Address:85bd68c9-6d05-bcab-d357-6693d9d2661c}]" -2024-05-15T01:17:43.416Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f21db385-69b0-b2cc-41ee-252a92b79dd5 Address:f21db385-69b0-b2cc-41ee-252a92b79dd5} {Suffrage:Voter ID:server-e323e934-0c86-9083-5d8b-44bb650636d6 Address:e323e934-0c86-9083-5d8b-44bb650636d6} {Suffrage:Voter ID:server-85bd68c9-6d05-bcab-d357-6693d9d2661c Address:85bd68c9-6d05-bcab-d357-6693d9d2661c}]" -2024-05-15T01:17:43.416Z [INFO] server-f21db385-69b0-b2cc-41ee-252a92b79dd5: entering follower state: follower="Node at f21db385-69b0-b2cc-41ee-252a92b79dd5 [Follower]" leader-address= leader-id= -2024-05-15T01:17:43.416Z [INFO] server-85bd68c9-6d05-bcab-d357-6693d9d2661c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f21db385-69b0-b2cc-41ee-252a92b79dd5 Address:f21db385-69b0-b2cc-41ee-252a92b79dd5} {Suffrage:Voter ID:server-e323e934-0c86-9083-5d8b-44bb650636d6 Address:e323e934-0c86-9083-5d8b-44bb650636d6} {Suffrage:Voter ID:server-85bd68c9-6d05-bcab-d357-6693d9d2661c Address:85bd68c9-6d05-bcab-d357-6693d9d2661c}]" -2024-05-15T01:17:43.416Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:43.416Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: entering follower state: follower="Node at e323e934-0c86-9083-5d8b-44bb650636d6 [Follower]" leader-address= leader-id= -2024-05-15T01:17:43.416Z [INFO] server-85bd68c9-6d05-bcab-d357-6693d9d2661c: entering follower state: follower="Node at 85bd68c9-6d05-bcab-d357-6693d9d2661c [Follower]" leader-address= leader-id= -2024-05-15T01:17:44.168Z [WARN] server-e323e934-0c86-9083-5d8b-44bb650636d6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:44.168Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: entering candidate state: node="Node at e323e934-0c86-9083-5d8b-44bb650636d6 [Candidate]" term=2 -2024-05-15T01:17:44.168Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: election won: term=2 tally=2 -2024-05-15T01:17:44.168Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: entering leader state: leader="Node at e323e934-0c86-9083-5d8b-44bb650636d6 [Leader]" -2024-05-15T01:17:44.168Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: added peer, starting replication: peer=server-f21db385-69b0-b2cc-41ee-252a92b79dd5 -2024-05-15T01:17:44.168Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: added peer, starting replication: peer=server-85bd68c9-6d05-bcab-d357-6693d9d2661c -2024-05-15T01:17:44.169Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: pipelining replication: peer="{Voter server-f21db385-69b0-b2cc-41ee-252a92b79dd5 f21db385-69b0-b2cc-41ee-252a92b79dd5}" -2024-05-15T01:17:44.169Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: pipelining replication: peer="{Voter server-85bd68c9-6d05-bcab-d357-6693d9d2661c 85bd68c9-6d05-bcab-d357-6693d9d2661c}" -2024-05-15T01:17:45.174Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:44.168838275 +0000 UTC m=+44.710223715 (1 nodes), 752.550598ms from start of poll, 753.058312ms from cluster start. Timeout at 2024-05-15 01:17:45.174387742 +0000 UTC m=+45.715772504, 1.005548789s after stability -2024-05-15T01:17:45.179Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: starting snapshot up to: index=12 -2024-05-15T01:17:45.179Z [INFO] creating new snapshot: path=/tmp/raft1618228473/snapshots/2-12-1715735865179.tmp -2024-05-15T01:17:45.180Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: snapshot complete up to: index=12 -2024-05-15T01:17:45.182Z [INFO] creating new snapshot: path=/tmp/raft1618228473/snapshots/2-23-1715735865182.tmp -2024-05-15T01:17:45.183Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: copied to local snapshot: bytes=71 -2024-05-15T01:17:45.183Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: snapshot restore progress: id=2-23-1715735865182 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:45.183Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: removing all old logs from log store -2024-05-15T01:17:45.183Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: compacting logs: from=1 to=22 -2024-05-15T01:17:45.183Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: restored user snapshot: index=23 -2024-05-15T01:17:45.183Z [ERROR] server-e323e934-0c86-9083-5d8b-44bb650636d6: failed to get log: index=22 error="log not found" -2024-05-15T01:17:45.183Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: aborting pipeline replication: peer="{Voter server-f21db385-69b0-b2cc-41ee-252a92b79dd5 f21db385-69b0-b2cc-41ee-252a92b79dd5}" -2024-05-15T01:17:45.183Z [ERROR] server-e323e934-0c86-9083-5d8b-44bb650636d6: failed to get log: index=22 error="log not found" -2024-05-15T01:17:45.183Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: aborting pipeline replication: peer="{Voter server-85bd68c9-6d05-bcab-d357-6693d9d2661c 85bd68c9-6d05-bcab-d357-6693d9d2661c}" -2024-05-15T01:17:45.189Z [ERROR] server-e323e934-0c86-9083-5d8b-44bb650636d6: failed to get log: index=22 error="log not found" -2024-05-15T01:17:45.189Z [INFO] creating new snapshot: path=/tmp/raft244268490/snapshots/2-23-1715735865189.tmp -2024-05-15T01:17:45.190Z [INFO] server-f21db385-69b0-b2cc-41ee-252a92b79dd5: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:45.190Z [INFO] server-f21db385-69b0-b2cc-41ee-252a92b79dd5: copied to local snapshot: bytes=71 -2024-05-15T01:17:45.190Z [ERROR] server-e323e934-0c86-9083-5d8b-44bb650636d6: failed to get log: index=22 error="log not found" -2024-05-15T01:17:45.191Z [INFO] creating new snapshot: path=/tmp/raft616361839/snapshots/2-23-1715735865191.tmp -2024-05-15T01:17:45.191Z [INFO] server-f21db385-69b0-b2cc-41ee-252a92b79dd5: snapshot restore progress: id=2-23-1715735865189 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:45.191Z [INFO] server-f21db385-69b0-b2cc-41ee-252a92b79dd5: removing all old logs from log store -2024-05-15T01:17:45.191Z [INFO] server-f21db385-69b0-b2cc-41ee-252a92b79dd5: compacting logs: from=1 to=22 -2024-05-15T01:17:45.191Z [INFO] server-f21db385-69b0-b2cc-41ee-252a92b79dd5: Installed remote snapshot -2024-05-15T01:17:45.191Z [WARN] server-f21db385-69b0-b2cc-41ee-252a92b79dd5: skipping application of old log: index=22 -2024-05-15T01:17:45.191Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: pipelining replication: peer="{Voter server-f21db385-69b0-b2cc-41ee-252a92b79dd5 f21db385-69b0-b2cc-41ee-252a92b79dd5}" -2024-05-15T01:17:45.191Z [INFO] server-85bd68c9-6d05-bcab-d357-6693d9d2661c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:45.191Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:45.192Z [INFO] server-85bd68c9-6d05-bcab-d357-6693d9d2661c: copied to local snapshot: bytes=71 -2024-05-15T01:17:45.192Z [INFO] server-85bd68c9-6d05-bcab-d357-6693d9d2661c: snapshot restore progress: id=2-23-1715735865191 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:45.192Z [INFO] server-85bd68c9-6d05-bcab-d357-6693d9d2661c: removing all old logs from log store -2024-05-15T01:17:45.192Z [INFO] server-85bd68c9-6d05-bcab-d357-6693d9d2661c: compacting logs: from=1 to=22 -2024-05-15T01:17:45.192Z [INFO] server-85bd68c9-6d05-bcab-d357-6693d9d2661c: Installed remote snapshot -2024-05-15T01:17:45.192Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: pipelining replication: peer="{Voter server-85bd68c9-6d05-bcab-d357-6693d9d2661c 85bd68c9-6d05-bcab-d357-6693d9d2661c}" -2024-05-15T01:17:46.197Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:45.191630574 +0000 UTC m=+45.733015317 (2 nodes), 2.085µs from start of poll, 1.775849914s from cluster start. Timeout at 2024-05-15 01:17:46.197334867 +0000 UTC m=+46.738719769, 1.005704452s after stability -2024-05-15T01:17:46.210Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: aborting pipeline replication: peer="{Voter server-85bd68c9-6d05-bcab-d357-6693d9d2661c 85bd68c9-6d05-bcab-d357-6693d9d2661c}" -2024-05-15T01:17:46.210Z [INFO] server-e323e934-0c86-9083-5d8b-44bb650636d6: aborting pipeline replication: peer="{Voter server-f21db385-69b0-b2cc-41ee-252a92b79dd5 f21db385-69b0-b2cc-41ee-252a92b79dd5}" +2025-06-17T07:43:35.586Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037 Address:8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037} {Suffrage:Voter ID:server-5338a734-6dc6-e911-f92e-c3d65e364c5e Address:5338a734-6dc6-e911-f92e-c3d65e364c5e} {Suffrage:Voter ID:server-e1044ce9-e21c-c919-822a-5a93d49054fa Address:e1044ce9-e21c-c919-822a-5a93d49054fa}]" +2025-06-17T07:43:35.586Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: entering follower state: follower="Node at 8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037 [Follower]" leader-address= leader-id= +2025-06-17T07:43:35.586Z [INFO] server-5338a734-6dc6-e911-f92e-c3d65e364c5e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037 Address:8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037} {Suffrage:Voter ID:server-5338a734-6dc6-e911-f92e-c3d65e364c5e Address:5338a734-6dc6-e911-f92e-c3d65e364c5e} {Suffrage:Voter ID:server-e1044ce9-e21c-c919-822a-5a93d49054fa Address:e1044ce9-e21c-c919-822a-5a93d49054fa}]" +2025-06-17T07:43:35.586Z [INFO] server-5338a734-6dc6-e911-f92e-c3d65e364c5e: entering follower state: follower="Node at 5338a734-6dc6-e911-f92e-c3d65e364c5e [Follower]" leader-address= leader-id= +2025-06-17T07:43:35.586Z [INFO] server-e1044ce9-e21c-c919-822a-5a93d49054fa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037 Address:8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037} {Suffrage:Voter ID:server-5338a734-6dc6-e911-f92e-c3d65e364c5e Address:5338a734-6dc6-e911-f92e-c3d65e364c5e} {Suffrage:Voter ID:server-e1044ce9-e21c-c919-822a-5a93d49054fa Address:e1044ce9-e21c-c919-822a-5a93d49054fa}]" +2025-06-17T07:43:35.586Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:35.586Z [INFO] server-e1044ce9-e21c-c919-822a-5a93d49054fa: entering follower state: follower="Node at e1044ce9-e21c-c919-822a-5a93d49054fa [Follower]" leader-address= leader-id= +2025-06-17T07:43:36.278Z [WARN] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:36.278Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: entering candidate state: node="Node at 8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037 [Candidate]" term=2 +2025-06-17T07:43:36.278Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: election won: term=2 tally=2 +2025-06-17T07:43:36.278Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: entering leader state: leader="Node at 8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037 [Leader]" +2025-06-17T07:43:36.278Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: added peer, starting replication: peer=server-5338a734-6dc6-e911-f92e-c3d65e364c5e +2025-06-17T07:43:36.279Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: added peer, starting replication: peer=server-e1044ce9-e21c-c919-822a-5a93d49054fa +2025-06-17T07:43:36.279Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: pipelining replication: peer="{Voter server-5338a734-6dc6-e911-f92e-c3d65e364c5e 5338a734-6dc6-e911-f92e-c3d65e364c5e}" +2025-06-17T07:43:36.279Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: pipelining replication: peer="{Voter server-e1044ce9-e21c-c919-822a-5a93d49054fa e1044ce9-e21c-c919-822a-5a93d49054fa}" +2025-06-17T07:43:37.285Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:36.279800956 +0000 UTC m=+44.883329736 (1 nodes), 693.425842ms from start of poll, 693.702797ms from cluster start. Timeout at 2025-06-17 07:43:37.285590538 +0000 UTC m=+45.889119233, 1.005789497s after stability +2025-06-17T07:43:37.286Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: starting snapshot up to: index=12 +2025-06-17T07:43:37.286Z [INFO] creating new snapshot: path=/tmp/raft980407236/snapshots/2-12-1750146217286.tmp +2025-06-17T07:43:37.287Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: snapshot complete up to: index=12 +2025-06-17T07:43:37.288Z [INFO] creating new snapshot: path=/tmp/raft980407236/snapshots/2-23-1750146217288.tmp +2025-06-17T07:43:37.288Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: copied to local snapshot: bytes=71 +2025-06-17T07:43:37.289Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: snapshot restore progress: id=2-23-1750146217288 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:37.289Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: removing all old logs from log store +2025-06-17T07:43:37.289Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: compacting logs: from=1 to=22 +2025-06-17T07:43:37.289Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: restored user snapshot: index=23 +2025-06-17T07:43:37.289Z [ERROR] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: failed to get log: index=22 error="log not found" +2025-06-17T07:43:37.289Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: aborting pipeline replication: peer="{Voter server-5338a734-6dc6-e911-f92e-c3d65e364c5e 5338a734-6dc6-e911-f92e-c3d65e364c5e}" +2025-06-17T07:43:37.289Z [ERROR] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: failed to get log: index=22 error="log not found" +2025-06-17T07:43:37.289Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: aborting pipeline replication: peer="{Voter server-e1044ce9-e21c-c919-822a-5a93d49054fa e1044ce9-e21c-c919-822a-5a93d49054fa}" +2025-06-17T07:43:37.295Z [ERROR] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: failed to get log: index=22 error="log not found" +2025-06-17T07:43:37.296Z [INFO] creating new snapshot: path=/tmp/raft3649169870/snapshots/2-23-1750146217296.tmp +2025-06-17T07:43:37.296Z [INFO] server-e1044ce9-e21c-c919-822a-5a93d49054fa: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:37.296Z [INFO] server-e1044ce9-e21c-c919-822a-5a93d49054fa: copied to local snapshot: bytes=71 +2025-06-17T07:43:37.296Z [INFO] server-e1044ce9-e21c-c919-822a-5a93d49054fa: snapshot restore progress: id=2-23-1750146217296 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:37.296Z [INFO] server-e1044ce9-e21c-c919-822a-5a93d49054fa: removing all old logs from log store +2025-06-17T07:43:37.296Z [INFO] server-e1044ce9-e21c-c919-822a-5a93d49054fa: compacting logs: from=1 to=22 +2025-06-17T07:43:37.296Z [INFO] server-e1044ce9-e21c-c919-822a-5a93d49054fa: Installed remote snapshot +2025-06-17T07:43:37.296Z [WARN] server-e1044ce9-e21c-c919-822a-5a93d49054fa: skipping application of old log: index=22 +2025-06-17T07:43:37.296Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: pipelining replication: peer="{Voter server-e1044ce9-e21c-c919-822a-5a93d49054fa e1044ce9-e21c-c919-822a-5a93d49054fa}" +2025-06-17T07:43:37.296Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:37.299Z [ERROR] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: failed to get log: index=22 error="log not found" +2025-06-17T07:43:37.299Z [INFO] creating new snapshot: path=/tmp/raft815823513/snapshots/2-23-1750146217299.tmp +2025-06-17T07:43:37.299Z [INFO] server-5338a734-6dc6-e911-f92e-c3d65e364c5e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:37.299Z [INFO] server-5338a734-6dc6-e911-f92e-c3d65e364c5e: copied to local snapshot: bytes=71 +2025-06-17T07:43:37.300Z [INFO] server-5338a734-6dc6-e911-f92e-c3d65e364c5e: snapshot restore progress: id=2-23-1750146217299 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:37.300Z [INFO] server-5338a734-6dc6-e911-f92e-c3d65e364c5e: removing all old logs from log store +2025-06-17T07:43:37.300Z [INFO] server-5338a734-6dc6-e911-f92e-c3d65e364c5e: compacting logs: from=1 to=22 +2025-06-17T07:43:37.300Z [INFO] server-5338a734-6dc6-e911-f92e-c3d65e364c5e: Installed remote snapshot +2025-06-17T07:43:37.300Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: pipelining replication: peer="{Voter server-5338a734-6dc6-e911-f92e-c3d65e364c5e 5338a734-6dc6-e911-f92e-c3d65e364c5e}" +2025-06-17T07:43:38.302Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:37.296897247 +0000 UTC m=+45.900425959 (2 nodes), 879ns from start of poll, 1.71079902s from cluster start. Timeout at 2025-06-17 07:43:38.302039625 +0000 UTC m=+46.905568309, 1.00514235s after stability +2025-06-17T07:43:38.313Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: aborting pipeline replication: peer="{Voter server-e1044ce9-e21c-c919-822a-5a93d49054fa e1044ce9-e21c-c919-822a-5a93d49054fa}" +2025-06-17T07:43:38.313Z [INFO] server-8bcda61d-3621-ba9f-9cf7-7c7c6f3eb037: aborting pipeline replication: peer="{Voter server-5338a734-6dc6-e911-f92e-c3d65e364c5e 5338a734-6dc6-e911-f92e-c3d65e364c5e}" === RUN TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true -2024-05-15T01:17:46.212Z [INFO] server-ea3edd4b-84f7-b263-9e44-0755657880fc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ea3edd4b-84f7-b263-9e44-0755657880fc Address:ea3edd4b-84f7-b263-9e44-0755657880fc} {Suffrage:Voter ID:server-c32901e1-d992-a6a2-bcfd-d5825792d8da Address:c32901e1-d992-a6a2-bcfd-d5825792d8da} {Suffrage:Voter ID:server-21758940-b1a0-c74c-1ef9-a94b1370509d Address:21758940-b1a0-c74c-1ef9-a94b1370509d}]" -2024-05-15T01:17:46.212Z [INFO] server-c32901e1-d992-a6a2-bcfd-d5825792d8da: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ea3edd4b-84f7-b263-9e44-0755657880fc Address:ea3edd4b-84f7-b263-9e44-0755657880fc} {Suffrage:Voter ID:server-c32901e1-d992-a6a2-bcfd-d5825792d8da Address:c32901e1-d992-a6a2-bcfd-d5825792d8da} {Suffrage:Voter ID:server-21758940-b1a0-c74c-1ef9-a94b1370509d Address:21758940-b1a0-c74c-1ef9-a94b1370509d}]" -2024-05-15T01:17:46.213Z [INFO] server-ea3edd4b-84f7-b263-9e44-0755657880fc: entering follower state: follower="Node at ea3edd4b-84f7-b263-9e44-0755657880fc [Follower]" leader-address= leader-id= -2024-05-15T01:17:46.213Z [INFO] server-c32901e1-d992-a6a2-bcfd-d5825792d8da: entering follower state: follower="Node at c32901e1-d992-a6a2-bcfd-d5825792d8da [Follower]" leader-address= leader-id= -2024-05-15T01:17:46.213Z [INFO] server-21758940-b1a0-c74c-1ef9-a94b1370509d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ea3edd4b-84f7-b263-9e44-0755657880fc Address:ea3edd4b-84f7-b263-9e44-0755657880fc} {Suffrage:Voter ID:server-c32901e1-d992-a6a2-bcfd-d5825792d8da Address:c32901e1-d992-a6a2-bcfd-d5825792d8da} {Suffrage:Voter ID:server-21758940-b1a0-c74c-1ef9-a94b1370509d Address:21758940-b1a0-c74c-1ef9-a94b1370509d}]" -2024-05-15T01:17:46.215Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:46.216Z [INFO] server-21758940-b1a0-c74c-1ef9-a94b1370509d: entering follower state: follower="Node at 21758940-b1a0-c74c-1ef9-a94b1370509d [Follower]" leader-address= leader-id= -2024-05-15T01:17:46.860Z [WARN] server-c32901e1-d992-a6a2-bcfd-d5825792d8da: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:46.860Z [INFO] server-c32901e1-d992-a6a2-bcfd-d5825792d8da: entering candidate state: node="Node at c32901e1-d992-a6a2-bcfd-d5825792d8da [Candidate]" term=2 -2024-05-15T01:17:46.860Z [INFO] server-c32901e1-d992-a6a2-bcfd-d5825792d8da: election won: term=2 tally=2 -2024-05-15T01:17:46.860Z [INFO] server-c32901e1-d992-a6a2-bcfd-d5825792d8da: entering leader state: leader="Node at c32901e1-d992-a6a2-bcfd-d5825792d8da [Leader]" -2024-05-15T01:17:46.860Z [INFO] server-c32901e1-d992-a6a2-bcfd-d5825792d8da: added peer, starting replication: peer=server-ea3edd4b-84f7-b263-9e44-0755657880fc -2024-05-15T01:17:46.860Z [INFO] server-c32901e1-d992-a6a2-bcfd-d5825792d8da: added peer, starting replication: peer=server-21758940-b1a0-c74c-1ef9-a94b1370509d -2024-05-15T01:17:46.861Z [INFO] server-c32901e1-d992-a6a2-bcfd-d5825792d8da: pipelining replication: peer="{Voter server-ea3edd4b-84f7-b263-9e44-0755657880fc ea3edd4b-84f7-b263-9e44-0755657880fc}" -2024-05-15T01:17:46.861Z [INFO] server-c32901e1-d992-a6a2-bcfd-d5825792d8da: pipelining replication: peer="{Voter server-21758940-b1a0-c74c-1ef9-a94b1370509d 21758940-b1a0-c74c-1ef9-a94b1370509d}" -2024-05-15T01:17:47.866Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:46.860804918 +0000 UTC m=+47.402190014 (1 nodes), 644.778103ms from start of poll, 648.206251ms from cluster start. Timeout at 2024-05-15 01:17:47.866829932 +0000 UTC m=+48.408215653, 1.006025639s after stability -2024-05-15T01:17:47.868Z [INFO] server-c32901e1-d992-a6a2-bcfd-d5825792d8da: starting snapshot up to: index=12 -2024-05-15T01:17:47.868Z [INFO] creating new snapshot: path=/tmp/raft893222813/snapshots/2-12-1715735867868.tmp -2024-05-15T01:17:47.869Z [INFO] server-c32901e1-d992-a6a2-bcfd-d5825792d8da: snapshot complete up to: index=12 -2024-05-15T01:17:47.882Z [INFO] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d0d70c55-c048-cb45-1d45-c1fcaed747e2 Address:d0d70c55-c048-cb45-1d45-c1fcaed747e2} {Suffrage:Voter ID:server-3eee9cbc-7438-c4f9-6c47-2fa74aadd989 Address:3eee9cbc-7438-c4f9-6c47-2fa74aadd989} {Suffrage:Voter ID:server-cd8c875c-bd7d-fe09-7004-e1b7b310a79f Address:cd8c875c-bd7d-fe09-7004-e1b7b310a79f}]" -2024-05-15T01:17:47.882Z [INFO] server-3eee9cbc-7438-c4f9-6c47-2fa74aadd989: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d0d70c55-c048-cb45-1d45-c1fcaed747e2 Address:d0d70c55-c048-cb45-1d45-c1fcaed747e2} {Suffrage:Voter ID:server-3eee9cbc-7438-c4f9-6c47-2fa74aadd989 Address:3eee9cbc-7438-c4f9-6c47-2fa74aadd989} {Suffrage:Voter ID:server-cd8c875c-bd7d-fe09-7004-e1b7b310a79f Address:cd8c875c-bd7d-fe09-7004-e1b7b310a79f}]" -2024-05-15T01:17:47.882Z [INFO] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: entering follower state: follower="Node at d0d70c55-c048-cb45-1d45-c1fcaed747e2 [Follower]" leader-address= leader-id= -2024-05-15T01:17:47.882Z [INFO] server-3eee9cbc-7438-c4f9-6c47-2fa74aadd989: entering follower state: follower="Node at 3eee9cbc-7438-c4f9-6c47-2fa74aadd989 [Follower]" leader-address= leader-id= -2024-05-15T01:17:47.882Z [INFO] server-cd8c875c-bd7d-fe09-7004-e1b7b310a79f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d0d70c55-c048-cb45-1d45-c1fcaed747e2 Address:d0d70c55-c048-cb45-1d45-c1fcaed747e2} {Suffrage:Voter ID:server-3eee9cbc-7438-c4f9-6c47-2fa74aadd989 Address:3eee9cbc-7438-c4f9-6c47-2fa74aadd989} {Suffrage:Voter ID:server-cd8c875c-bd7d-fe09-7004-e1b7b310a79f Address:cd8c875c-bd7d-fe09-7004-e1b7b310a79f}]" -2024-05-15T01:17:47.882Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:47.882Z [INFO] server-cd8c875c-bd7d-fe09-7004-e1b7b310a79f: entering follower state: follower="Node at cd8c875c-bd7d-fe09-7004-e1b7b310a79f [Follower]" leader-address= leader-id= -2024-05-15T01:17:48.455Z [WARN] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:48.455Z [INFO] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: entering candidate state: node="Node at d0d70c55-c048-cb45-1d45-c1fcaed747e2 [Candidate]" term=2 -2024-05-15T01:17:48.456Z [INFO] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: election won: term=2 tally=2 -2024-05-15T01:17:48.456Z [INFO] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: entering leader state: leader="Node at d0d70c55-c048-cb45-1d45-c1fcaed747e2 [Leader]" -2024-05-15T01:17:48.456Z [INFO] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: added peer, starting replication: peer=server-3eee9cbc-7438-c4f9-6c47-2fa74aadd989 -2024-05-15T01:17:48.456Z [INFO] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: added peer, starting replication: peer=server-cd8c875c-bd7d-fe09-7004-e1b7b310a79f -2024-05-15T01:17:48.456Z [INFO] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: pipelining replication: peer="{Voter server-cd8c875c-bd7d-fe09-7004-e1b7b310a79f cd8c875c-bd7d-fe09-7004-e1b7b310a79f}" -2024-05-15T01:17:48.456Z [INFO] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: pipelining replication: peer="{Voter server-3eee9cbc-7438-c4f9-6c47-2fa74aadd989 3eee9cbc-7438-c4f9-6c47-2fa74aadd989}" -2024-05-15T01:17:49.462Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:48.456629909 +0000 UTC m=+48.998014792 (1 nodes), 573.701524ms from start of poll, 574.341507ms from cluster start. Timeout at 2024-05-15 01:17:49.462434791 +0000 UTC m=+50.003819625, 1.005804833s after stability -2024-05-15T01:17:49.462Z [INFO] creating new snapshot: path=/tmp/raft77608596/snapshots/2-15-1715735869462.tmp -2024-05-15T01:17:49.463Z [INFO] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: copied to local snapshot: bytes=71 -2024-05-15T01:17:49.464Z [INFO] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: snapshot restore progress: id=2-15-1715735869462 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:49.464Z [INFO] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: restored user snapshot: index=15 -2024-05-15T01:17:49.464Z [ERROR] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: failed to get log: index=3 error="log not found" -2024-05-15T01:17:49.464Z [INFO] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: aborting pipeline replication: peer="{Voter server-cd8c875c-bd7d-fe09-7004-e1b7b310a79f cd8c875c-bd7d-fe09-7004-e1b7b310a79f}" -2024-05-15T01:17:49.464Z [ERROR] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: failed to get log: index=3 error="log not found" -2024-05-15T01:17:49.464Z [INFO] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: aborting pipeline replication: peer="{Voter server-3eee9cbc-7438-c4f9-6c47-2fa74aadd989 3eee9cbc-7438-c4f9-6c47-2fa74aadd989}" -2024-05-15T01:17:49.470Z [ERROR] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: failed to get log: index=3 error="log not found" -2024-05-15T01:17:49.471Z [INFO] creating new snapshot: path=/tmp/raft3435922732/snapshots/2-15-1715735869471.tmp -2024-05-15T01:17:49.471Z [INFO] server-cd8c875c-bd7d-fe09-7004-e1b7b310a79f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:49.471Z [INFO] server-cd8c875c-bd7d-fe09-7004-e1b7b310a79f: copied to local snapshot: bytes=71 -2024-05-15T01:17:49.472Z [INFO] server-cd8c875c-bd7d-fe09-7004-e1b7b310a79f: snapshot restore progress: id=2-15-1715735869471 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:49.472Z [INFO] server-cd8c875c-bd7d-fe09-7004-e1b7b310a79f: Installed remote snapshot -2024-05-15T01:17:49.472Z [INFO] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: pipelining replication: peer="{Voter server-cd8c875c-bd7d-fe09-7004-e1b7b310a79f cd8c875c-bd7d-fe09-7004-e1b7b310a79f}" -2024-05-15T01:17:49.472Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:49.474Z [ERROR] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: failed to get log: index=3 error="log not found" -2024-05-15T01:17:49.474Z [INFO] creating new snapshot: path=/tmp/raft629300379/snapshots/2-15-1715735869474.tmp -2024-05-15T01:17:49.474Z [INFO] server-3eee9cbc-7438-c4f9-6c47-2fa74aadd989: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:49.475Z [INFO] server-3eee9cbc-7438-c4f9-6c47-2fa74aadd989: copied to local snapshot: bytes=71 -2024-05-15T01:17:49.475Z [INFO] server-3eee9cbc-7438-c4f9-6c47-2fa74aadd989: snapshot restore progress: id=2-15-1715735869474 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:49.475Z [INFO] server-3eee9cbc-7438-c4f9-6c47-2fa74aadd989: Installed remote snapshot -2024-05-15T01:17:49.475Z [INFO] server-d0d70c55-c048-cb45-1d45-c1fcaed747e2: pipelining replication: peer="{Voter server-3eee9cbc-7438-c4f9-6c47-2fa74aadd989 3eee9cbc-7438-c4f9-6c47-2fa74aadd989}" -2024-05-15T01:17:50.477Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:49.47258689 +0000 UTC m=+50.013971629 (2 nodes), 2.08µs from start of poll, 1.590298344s from cluster start. Timeout at 2024-05-15 01:17:50.477813654 +0000 UTC m=+51.019198418, 1.005226789s after stability -2024-05-15T01:17:50.488Z [INFO] server-c32901e1-d992-a6a2-bcfd-d5825792d8da: aborting pipeline replication: peer="{Voter server-21758940-b1a0-c74c-1ef9-a94b1370509d 21758940-b1a0-c74c-1ef9-a94b1370509d}" -2024-05-15T01:17:50.488Z [INFO] server-c32901e1-d992-a6a2-bcfd-d5825792d8da: aborting pipeline replication: peer="{Voter server-ea3edd4b-84f7-b263-9e44-0755657880fc ea3edd4b-84f7-b263-9e44-0755657880fc}" +2025-06-17T07:43:38.314Z [INFO] server-ff812348-1eed-6cc1-a53a-cbff63b70ec3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff812348-1eed-6cc1-a53a-cbff63b70ec3 Address:ff812348-1eed-6cc1-a53a-cbff63b70ec3} {Suffrage:Voter ID:server-5dce1e35-2025-6191-dd51-6b854f02459f Address:5dce1e35-2025-6191-dd51-6b854f02459f} {Suffrage:Voter ID:server-a36224b9-c54b-f159-d533-6fb4f2eb3579 Address:a36224b9-c54b-f159-d533-6fb4f2eb3579}]" +2025-06-17T07:43:38.314Z [INFO] server-ff812348-1eed-6cc1-a53a-cbff63b70ec3: entering follower state: follower="Node at ff812348-1eed-6cc1-a53a-cbff63b70ec3 [Follower]" leader-address= leader-id= +2025-06-17T07:43:38.314Z [INFO] server-5dce1e35-2025-6191-dd51-6b854f02459f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff812348-1eed-6cc1-a53a-cbff63b70ec3 Address:ff812348-1eed-6cc1-a53a-cbff63b70ec3} {Suffrage:Voter ID:server-5dce1e35-2025-6191-dd51-6b854f02459f Address:5dce1e35-2025-6191-dd51-6b854f02459f} {Suffrage:Voter ID:server-a36224b9-c54b-f159-d533-6fb4f2eb3579 Address:a36224b9-c54b-f159-d533-6fb4f2eb3579}]" +2025-06-17T07:43:38.314Z [INFO] server-5dce1e35-2025-6191-dd51-6b854f02459f: entering follower state: follower="Node at 5dce1e35-2025-6191-dd51-6b854f02459f [Follower]" leader-address= leader-id= +2025-06-17T07:43:38.314Z [INFO] server-a36224b9-c54b-f159-d533-6fb4f2eb3579: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff812348-1eed-6cc1-a53a-cbff63b70ec3 Address:ff812348-1eed-6cc1-a53a-cbff63b70ec3} {Suffrage:Voter ID:server-5dce1e35-2025-6191-dd51-6b854f02459f Address:5dce1e35-2025-6191-dd51-6b854f02459f} {Suffrage:Voter ID:server-a36224b9-c54b-f159-d533-6fb4f2eb3579 Address:a36224b9-c54b-f159-d533-6fb4f2eb3579}]" +2025-06-17T07:43:38.314Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:38.314Z [INFO] server-a36224b9-c54b-f159-d533-6fb4f2eb3579: entering follower state: follower="Node at a36224b9-c54b-f159-d533-6fb4f2eb3579 [Follower]" leader-address= leader-id= +2025-06-17T07:43:38.865Z [WARN] server-ff812348-1eed-6cc1-a53a-cbff63b70ec3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:38.865Z [INFO] server-ff812348-1eed-6cc1-a53a-cbff63b70ec3: entering candidate state: node="Node at ff812348-1eed-6cc1-a53a-cbff63b70ec3 [Candidate]" term=2 +2025-06-17T07:43:38.865Z [INFO] server-ff812348-1eed-6cc1-a53a-cbff63b70ec3: election won: term=2 tally=2 +2025-06-17T07:43:38.865Z [INFO] server-ff812348-1eed-6cc1-a53a-cbff63b70ec3: entering leader state: leader="Node at ff812348-1eed-6cc1-a53a-cbff63b70ec3 [Leader]" +2025-06-17T07:43:38.865Z [INFO] server-ff812348-1eed-6cc1-a53a-cbff63b70ec3: added peer, starting replication: peer=server-5dce1e35-2025-6191-dd51-6b854f02459f +2025-06-17T07:43:38.865Z [INFO] server-ff812348-1eed-6cc1-a53a-cbff63b70ec3: added peer, starting replication: peer=server-a36224b9-c54b-f159-d533-6fb4f2eb3579 +2025-06-17T07:43:38.865Z [INFO] server-ff812348-1eed-6cc1-a53a-cbff63b70ec3: pipelining replication: peer="{Voter server-5dce1e35-2025-6191-dd51-6b854f02459f 5dce1e35-2025-6191-dd51-6b854f02459f}" +2025-06-17T07:43:38.865Z [INFO] server-ff812348-1eed-6cc1-a53a-cbff63b70ec3: pipelining replication: peer="{Voter server-a36224b9-c54b-f159-d533-6fb4f2eb3579 a36224b9-c54b-f159-d533-6fb4f2eb3579}" +2025-06-17T07:43:39.871Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:38.865802845 +0000 UTC m=+47.469331563 (1 nodes), 550.848857ms from start of poll, 551.147051ms from cluster start. Timeout at 2025-06-17 07:43:39.87150789 +0000 UTC m=+48.475036594, 1.005705031s after stability +2025-06-17T07:43:39.872Z [INFO] server-ff812348-1eed-6cc1-a53a-cbff63b70ec3: starting snapshot up to: index=12 +2025-06-17T07:43:39.872Z [INFO] creating new snapshot: path=/tmp/raft2642304904/snapshots/2-12-1750146219872.tmp +2025-06-17T07:43:39.872Z [INFO] server-ff812348-1eed-6cc1-a53a-cbff63b70ec3: snapshot complete up to: index=12 +2025-06-17T07:43:39.873Z [INFO] server-6ce56410-ef6f-42c5-0080-0686a76cfa05: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ce56410-ef6f-42c5-0080-0686a76cfa05 Address:6ce56410-ef6f-42c5-0080-0686a76cfa05} {Suffrage:Voter ID:server-c90d7a95-186a-4d95-bad3-fbd27bc7bed9 Address:c90d7a95-186a-4d95-bad3-fbd27bc7bed9} {Suffrage:Voter ID:server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4 Address:e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4}]" +2025-06-17T07:43:39.873Z [INFO] server-6ce56410-ef6f-42c5-0080-0686a76cfa05: entering follower state: follower="Node at 6ce56410-ef6f-42c5-0080-0686a76cfa05 [Follower]" leader-address= leader-id= +2025-06-17T07:43:39.873Z [INFO] server-c90d7a95-186a-4d95-bad3-fbd27bc7bed9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ce56410-ef6f-42c5-0080-0686a76cfa05 Address:6ce56410-ef6f-42c5-0080-0686a76cfa05} {Suffrage:Voter ID:server-c90d7a95-186a-4d95-bad3-fbd27bc7bed9 Address:c90d7a95-186a-4d95-bad3-fbd27bc7bed9} {Suffrage:Voter ID:server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4 Address:e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4}]" +2025-06-17T07:43:39.873Z [INFO] server-c90d7a95-186a-4d95-bad3-fbd27bc7bed9: entering follower state: follower="Node at c90d7a95-186a-4d95-bad3-fbd27bc7bed9 [Follower]" leader-address= leader-id= +2025-06-17T07:43:39.873Z [INFO] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ce56410-ef6f-42c5-0080-0686a76cfa05 Address:6ce56410-ef6f-42c5-0080-0686a76cfa05} {Suffrage:Voter ID:server-c90d7a95-186a-4d95-bad3-fbd27bc7bed9 Address:c90d7a95-186a-4d95-bad3-fbd27bc7bed9} {Suffrage:Voter ID:server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4 Address:e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4}]" +2025-06-17T07:43:39.873Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:39.873Z [INFO] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: entering follower state: follower="Node at e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4 [Follower]" leader-address= leader-id= +2025-06-17T07:43:40.501Z [WARN] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:40.501Z [INFO] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: entering candidate state: node="Node at e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4 [Candidate]" term=2 +2025-06-17T07:43:40.501Z [INFO] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: election won: term=2 tally=2 +2025-06-17T07:43:40.501Z [INFO] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: entering leader state: leader="Node at e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4 [Leader]" +2025-06-17T07:43:40.501Z [INFO] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: added peer, starting replication: peer=server-6ce56410-ef6f-42c5-0080-0686a76cfa05 +2025-06-17T07:43:40.501Z [INFO] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: added peer, starting replication: peer=server-c90d7a95-186a-4d95-bad3-fbd27bc7bed9 +2025-06-17T07:43:40.501Z [INFO] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: pipelining replication: peer="{Voter server-6ce56410-ef6f-42c5-0080-0686a76cfa05 6ce56410-ef6f-42c5-0080-0686a76cfa05}" +2025-06-17T07:43:40.501Z [INFO] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: pipelining replication: peer="{Voter server-c90d7a95-186a-4d95-bad3-fbd27bc7bed9 c90d7a95-186a-4d95-bad3-fbd27bc7bed9}" +2025-06-17T07:43:41.506Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:40.501115599 +0000 UTC m=+49.104644301 (1 nodes), 627.161232ms from start of poll, 627.54834ms from cluster start. Timeout at 2025-06-17 07:43:41.506130808 +0000 UTC m=+50.109659485, 1.005015184s after stability +2025-06-17T07:43:41.506Z [INFO] creating new snapshot: path=/tmp/raft2635772480/snapshots/2-15-1750146221506.tmp +2025-06-17T07:43:41.506Z [INFO] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: copied to local snapshot: bytes=71 +2025-06-17T07:43:41.507Z [INFO] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: snapshot restore progress: id=2-15-1750146221506 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:41.507Z [INFO] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: restored user snapshot: index=15 +2025-06-17T07:43:41.507Z [ERROR] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: failed to get log: index=3 error="log not found" +2025-06-17T07:43:41.507Z [ERROR] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: failed to get log: index=3 error="log not found" +2025-06-17T07:43:41.507Z [INFO] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: aborting pipeline replication: peer="{Voter server-6ce56410-ef6f-42c5-0080-0686a76cfa05 6ce56410-ef6f-42c5-0080-0686a76cfa05}" +2025-06-17T07:43:41.507Z [INFO] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: aborting pipeline replication: peer="{Voter server-c90d7a95-186a-4d95-bad3-fbd27bc7bed9 c90d7a95-186a-4d95-bad3-fbd27bc7bed9}" +2025-06-17T07:43:41.514Z [ERROR] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: failed to get log: index=3 error="log not found" +2025-06-17T07:43:41.514Z [INFO] creating new snapshot: path=/tmp/raft3249859693/snapshots/2-15-1750146221514.tmp +2025-06-17T07:43:41.515Z [INFO] server-c90d7a95-186a-4d95-bad3-fbd27bc7bed9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:41.515Z [INFO] server-c90d7a95-186a-4d95-bad3-fbd27bc7bed9: copied to local snapshot: bytes=71 +2025-06-17T07:43:41.515Z [INFO] server-c90d7a95-186a-4d95-bad3-fbd27bc7bed9: snapshot restore progress: id=2-15-1750146221514 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:41.515Z [INFO] server-c90d7a95-186a-4d95-bad3-fbd27bc7bed9: Installed remote snapshot +2025-06-17T07:43:41.516Z [INFO] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: pipelining replication: peer="{Voter server-c90d7a95-186a-4d95-bad3-fbd27bc7bed9 c90d7a95-186a-4d95-bad3-fbd27bc7bed9}" +2025-06-17T07:43:41.516Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:41.517Z [ERROR] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: failed to get log: index=3 error="log not found" +2025-06-17T07:43:41.517Z [INFO] creating new snapshot: path=/tmp/raft525896468/snapshots/2-15-1750146221517.tmp +2025-06-17T07:43:41.517Z [INFO] server-6ce56410-ef6f-42c5-0080-0686a76cfa05: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:41.518Z [INFO] server-6ce56410-ef6f-42c5-0080-0686a76cfa05: copied to local snapshot: bytes=71 +2025-06-17T07:43:41.518Z [INFO] server-6ce56410-ef6f-42c5-0080-0686a76cfa05: snapshot restore progress: id=2-15-1750146221517 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:41.518Z [INFO] server-6ce56410-ef6f-42c5-0080-0686a76cfa05: Installed remote snapshot +2025-06-17T07:43:41.518Z [INFO] server-e7a09af8-6ea0-e6b5-fa0a-2b2af28d3db4: pipelining replication: peer="{Voter server-6ce56410-ef6f-42c5-0080-0686a76cfa05 6ce56410-ef6f-42c5-0080-0686a76cfa05}" +2025-06-17T07:43:42.522Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:41.516136796 +0000 UTC m=+50.119665482 (2 nodes), 641ns from start of poll, 1.642569521s from cluster start. Timeout at 2025-06-17 07:43:42.522155636 +0000 UTC m=+51.125684342, 1.00601886s after stability +2025-06-17T07:43:42.534Z [INFO] server-ff812348-1eed-6cc1-a53a-cbff63b70ec3: aborting pipeline replication: peer="{Voter server-a36224b9-c54b-f159-d533-6fb4f2eb3579 a36224b9-c54b-f159-d533-6fb4f2eb3579}" +2025-06-17T07:43:42.534Z [INFO] server-ff812348-1eed-6cc1-a53a-cbff63b70ec3: aborting pipeline replication: peer="{Voter server-5dce1e35-2025-6191-dd51-6b854f02459f 5dce1e35-2025-6191-dd51-6b854f02459f}" === RUN TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true -2024-05-15T01:17:50.490Z [INFO] server-8de8a9bc-4d3b-4f1b-dd0a-71d1ac25f9d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8de8a9bc-4d3b-4f1b-dd0a-71d1ac25f9d0 Address:8de8a9bc-4d3b-4f1b-dd0a-71d1ac25f9d0} {Suffrage:Voter ID:server-b22053e3-1fb4-72b2-42e0-8768987746a7 Address:b22053e3-1fb4-72b2-42e0-8768987746a7} {Suffrage:Voter ID:server-1a0db606-4827-9421-4d8f-31f3da200b87 Address:1a0db606-4827-9421-4d8f-31f3da200b87}]" -2024-05-15T01:17:50.490Z [INFO] server-b22053e3-1fb4-72b2-42e0-8768987746a7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8de8a9bc-4d3b-4f1b-dd0a-71d1ac25f9d0 Address:8de8a9bc-4d3b-4f1b-dd0a-71d1ac25f9d0} {Suffrage:Voter ID:server-b22053e3-1fb4-72b2-42e0-8768987746a7 Address:b22053e3-1fb4-72b2-42e0-8768987746a7} {Suffrage:Voter ID:server-1a0db606-4827-9421-4d8f-31f3da200b87 Address:1a0db606-4827-9421-4d8f-31f3da200b87}]" -2024-05-15T01:17:50.491Z [INFO] server-1a0db606-4827-9421-4d8f-31f3da200b87: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8de8a9bc-4d3b-4f1b-dd0a-71d1ac25f9d0 Address:8de8a9bc-4d3b-4f1b-dd0a-71d1ac25f9d0} {Suffrage:Voter ID:server-b22053e3-1fb4-72b2-42e0-8768987746a7 Address:b22053e3-1fb4-72b2-42e0-8768987746a7} {Suffrage:Voter ID:server-1a0db606-4827-9421-4d8f-31f3da200b87 Address:1a0db606-4827-9421-4d8f-31f3da200b87}]" -2024-05-15T01:17:50.491Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:50.491Z [INFO] server-8de8a9bc-4d3b-4f1b-dd0a-71d1ac25f9d0: entering follower state: follower="Node at 8de8a9bc-4d3b-4f1b-dd0a-71d1ac25f9d0 [Follower]" leader-address= leader-id= -2024-05-15T01:17:50.491Z [INFO] server-b22053e3-1fb4-72b2-42e0-8768987746a7: entering follower state: follower="Node at b22053e3-1fb4-72b2-42e0-8768987746a7 [Follower]" leader-address= leader-id= -2024-05-15T01:17:50.491Z [INFO] server-1a0db606-4827-9421-4d8f-31f3da200b87: entering follower state: follower="Node at 1a0db606-4827-9421-4d8f-31f3da200b87 [Follower]" leader-address= leader-id= -2024-05-15T01:17:51.239Z [WARN] server-b22053e3-1fb4-72b2-42e0-8768987746a7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:51.239Z [INFO] server-b22053e3-1fb4-72b2-42e0-8768987746a7: entering candidate state: node="Node at b22053e3-1fb4-72b2-42e0-8768987746a7 [Candidate]" term=2 -2024-05-15T01:17:51.239Z [INFO] server-b22053e3-1fb4-72b2-42e0-8768987746a7: election won: term=2 tally=2 -2024-05-15T01:17:51.239Z [INFO] server-b22053e3-1fb4-72b2-42e0-8768987746a7: entering leader state: leader="Node at b22053e3-1fb4-72b2-42e0-8768987746a7 [Leader]" -2024-05-15T01:17:51.239Z [INFO] server-b22053e3-1fb4-72b2-42e0-8768987746a7: added peer, starting replication: peer=server-8de8a9bc-4d3b-4f1b-dd0a-71d1ac25f9d0 -2024-05-15T01:17:51.239Z [INFO] server-b22053e3-1fb4-72b2-42e0-8768987746a7: added peer, starting replication: peer=server-1a0db606-4827-9421-4d8f-31f3da200b87 -2024-05-15T01:17:51.239Z [INFO] server-b22053e3-1fb4-72b2-42e0-8768987746a7: pipelining replication: peer="{Voter server-1a0db606-4827-9421-4d8f-31f3da200b87 1a0db606-4827-9421-4d8f-31f3da200b87}" -2024-05-15T01:17:51.239Z [INFO] server-b22053e3-1fb4-72b2-42e0-8768987746a7: pipelining replication: peer="{Voter server-8de8a9bc-4d3b-4f1b-dd0a-71d1ac25f9d0 8de8a9bc-4d3b-4f1b-dd0a-71d1ac25f9d0}" -2024-05-15T01:17:52.245Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:51.240088574 +0000 UTC m=+51.781473342 (1 nodes), 748.975722ms from start of poll, 749.464683ms from cluster start. Timeout at 2024-05-15 01:17:52.245758007 +0000 UTC m=+52.787142764, 1.005669422s after stability -2024-05-15T01:17:52.247Z [INFO] server-b22053e3-1fb4-72b2-42e0-8768987746a7: starting snapshot up to: index=12 -2024-05-15T01:17:52.247Z [INFO] creating new snapshot: path=/tmp/raft3315113081/snapshots/2-12-1715735872247.tmp -2024-05-15T01:17:52.248Z [INFO] server-b22053e3-1fb4-72b2-42e0-8768987746a7: snapshot complete up to: index=12 -2024-05-15T01:17:52.251Z [INFO] server-4c234f58-6305-d9e2-99b6-853d389be5d3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4c234f58-6305-d9e2-99b6-853d389be5d3 Address:4c234f58-6305-d9e2-99b6-853d389be5d3} {Suffrage:Voter ID:server-315439b2-ab19-79e1-a2ef-4750f1225abc Address:315439b2-ab19-79e1-a2ef-4750f1225abc} {Suffrage:Voter ID:server-d48582a9-6062-4572-cc71-310c90920ad7 Address:d48582a9-6062-4572-cc71-310c90920ad7}]" -2024-05-15T01:17:52.252Z [INFO] server-315439b2-ab19-79e1-a2ef-4750f1225abc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4c234f58-6305-d9e2-99b6-853d389be5d3 Address:4c234f58-6305-d9e2-99b6-853d389be5d3} {Suffrage:Voter ID:server-315439b2-ab19-79e1-a2ef-4750f1225abc Address:315439b2-ab19-79e1-a2ef-4750f1225abc} {Suffrage:Voter ID:server-d48582a9-6062-4572-cc71-310c90920ad7 Address:d48582a9-6062-4572-cc71-310c90920ad7}]" -2024-05-15T01:17:52.252Z [INFO] server-d48582a9-6062-4572-cc71-310c90920ad7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4c234f58-6305-d9e2-99b6-853d389be5d3 Address:4c234f58-6305-d9e2-99b6-853d389be5d3} {Suffrage:Voter ID:server-315439b2-ab19-79e1-a2ef-4750f1225abc Address:315439b2-ab19-79e1-a2ef-4750f1225abc} {Suffrage:Voter ID:server-d48582a9-6062-4572-cc71-310c90920ad7 Address:d48582a9-6062-4572-cc71-310c90920ad7}]" -2024-05-15T01:17:52.252Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:52.252Z [INFO] server-4c234f58-6305-d9e2-99b6-853d389be5d3: entering follower state: follower="Node at 4c234f58-6305-d9e2-99b6-853d389be5d3 [Follower]" leader-address= leader-id= -2024-05-15T01:17:52.252Z [INFO] server-315439b2-ab19-79e1-a2ef-4750f1225abc: entering follower state: follower="Node at 315439b2-ab19-79e1-a2ef-4750f1225abc [Follower]" leader-address= leader-id= -2024-05-15T01:17:52.252Z [INFO] server-d48582a9-6062-4572-cc71-310c90920ad7: entering follower state: follower="Node at d48582a9-6062-4572-cc71-310c90920ad7 [Follower]" leader-address= leader-id= -2024-05-15T01:17:52.956Z [WARN] server-4c234f58-6305-d9e2-99b6-853d389be5d3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:52.956Z [INFO] server-4c234f58-6305-d9e2-99b6-853d389be5d3: entering candidate state: node="Node at 4c234f58-6305-d9e2-99b6-853d389be5d3 [Candidate]" term=2 -2024-05-15T01:17:52.956Z [INFO] server-4c234f58-6305-d9e2-99b6-853d389be5d3: election won: term=2 tally=2 -2024-05-15T01:17:52.956Z [INFO] server-4c234f58-6305-d9e2-99b6-853d389be5d3: entering leader state: leader="Node at 4c234f58-6305-d9e2-99b6-853d389be5d3 [Leader]" -2024-05-15T01:17:52.956Z [INFO] server-4c234f58-6305-d9e2-99b6-853d389be5d3: added peer, starting replication: peer=server-315439b2-ab19-79e1-a2ef-4750f1225abc -2024-05-15T01:17:52.956Z [INFO] server-4c234f58-6305-d9e2-99b6-853d389be5d3: added peer, starting replication: peer=server-d48582a9-6062-4572-cc71-310c90920ad7 -2024-05-15T01:17:52.956Z [INFO] server-4c234f58-6305-d9e2-99b6-853d389be5d3: pipelining replication: peer="{Voter server-d48582a9-6062-4572-cc71-310c90920ad7 d48582a9-6062-4572-cc71-310c90920ad7}" -2024-05-15T01:17:52.956Z [INFO] server-4c234f58-6305-d9e2-99b6-853d389be5d3: pipelining replication: peer="{Voter server-315439b2-ab19-79e1-a2ef-4750f1225abc 315439b2-ab19-79e1-a2ef-4750f1225abc}" -2024-05-15T01:17:53.962Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:52.957062071 +0000 UTC m=+53.498446832 (1 nodes), 704.768169ms from start of poll, 705.271021ms from cluster start. Timeout at 2024-05-15 01:17:53.962099927 +0000 UTC m=+54.503484663, 1.005037831s after stability -2024-05-15T01:17:53.962Z [INFO] creating new snapshot: path=/tmp/raft1506687337/snapshots/2-15-1715735873962.tmp -2024-05-15T01:17:53.963Z [INFO] server-4c234f58-6305-d9e2-99b6-853d389be5d3: copied to local snapshot: bytes=71 -2024-05-15T01:17:53.963Z [INFO] server-4c234f58-6305-d9e2-99b6-853d389be5d3: snapshot restore progress: id=2-15-1715735873962 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:53.963Z [INFO] server-4c234f58-6305-d9e2-99b6-853d389be5d3: removing all old logs from log store -2024-05-15T01:17:53.963Z [INFO] server-4c234f58-6305-d9e2-99b6-853d389be5d3: compacting logs: from=1 to=2 -2024-05-15T01:17:53.963Z [INFO] server-4c234f58-6305-d9e2-99b6-853d389be5d3: restored user snapshot: index=15 -2024-05-15T01:17:53.963Z [ERROR] server-4c234f58-6305-d9e2-99b6-853d389be5d3: failed to get log: index=2 error="log not found" -2024-05-15T01:17:53.963Z [INFO] server-4c234f58-6305-d9e2-99b6-853d389be5d3: aborting pipeline replication: peer="{Voter server-d48582a9-6062-4572-cc71-310c90920ad7 d48582a9-6062-4572-cc71-310c90920ad7}" -2024-05-15T01:17:53.963Z [ERROR] server-4c234f58-6305-d9e2-99b6-853d389be5d3: failed to get log: index=2 error="log not found" -2024-05-15T01:17:53.963Z [INFO] server-4c234f58-6305-d9e2-99b6-853d389be5d3: aborting pipeline replication: peer="{Voter server-315439b2-ab19-79e1-a2ef-4750f1225abc 315439b2-ab19-79e1-a2ef-4750f1225abc}" -2024-05-15T01:17:53.970Z [ERROR] server-4c234f58-6305-d9e2-99b6-853d389be5d3: failed to get log: index=2 error="log not found" -2024-05-15T01:17:53.970Z [INFO] creating new snapshot: path=/tmp/raft3649308421/snapshots/2-15-1715735873970.tmp -2024-05-15T01:17:53.971Z [INFO] server-315439b2-ab19-79e1-a2ef-4750f1225abc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:53.971Z [INFO] server-315439b2-ab19-79e1-a2ef-4750f1225abc: copied to local snapshot: bytes=71 -2024-05-15T01:17:53.971Z [INFO] server-315439b2-ab19-79e1-a2ef-4750f1225abc: snapshot restore progress: id=2-15-1715735873970 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:53.971Z [INFO] server-315439b2-ab19-79e1-a2ef-4750f1225abc: removing all old logs from log store -2024-05-15T01:17:53.971Z [INFO] server-315439b2-ab19-79e1-a2ef-4750f1225abc: compacting logs: from=1 to=2 -2024-05-15T01:17:53.971Z [INFO] server-315439b2-ab19-79e1-a2ef-4750f1225abc: Installed remote snapshot -2024-05-15T01:17:53.971Z [INFO] server-4c234f58-6305-d9e2-99b6-853d389be5d3: pipelining replication: peer="{Voter server-315439b2-ab19-79e1-a2ef-4750f1225abc 315439b2-ab19-79e1-a2ef-4750f1225abc}" -2024-05-15T01:17:53.971Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:53.972Z [ERROR] server-4c234f58-6305-d9e2-99b6-853d389be5d3: failed to get log: index=2 error="log not found" -2024-05-15T01:17:53.972Z [INFO] creating new snapshot: path=/tmp/raft164093596/snapshots/2-15-1715735873972.tmp -2024-05-15T01:17:53.979Z [INFO] server-d48582a9-6062-4572-cc71-310c90920ad7: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:53.979Z [INFO] server-d48582a9-6062-4572-cc71-310c90920ad7: copied to local snapshot: bytes=71 -2024-05-15T01:17:53.980Z [INFO] server-d48582a9-6062-4572-cc71-310c90920ad7: snapshot restore progress: id=2-15-1715735873972 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:53.980Z [INFO] server-d48582a9-6062-4572-cc71-310c90920ad7: removing all old logs from log store -2024-05-15T01:17:53.980Z [INFO] server-d48582a9-6062-4572-cc71-310c90920ad7: compacting logs: from=1 to=2 -2024-05-15T01:17:53.980Z [INFO] server-d48582a9-6062-4572-cc71-310c90920ad7: Installed remote snapshot -2024-05-15T01:17:53.980Z [INFO] server-4c234f58-6305-d9e2-99b6-853d389be5d3: pipelining replication: peer="{Voter server-d48582a9-6062-4572-cc71-310c90920ad7 d48582a9-6062-4572-cc71-310c90920ad7}" -2024-05-15T01:17:54.977Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:53.972003686 +0000 UTC m=+54.513388428 (2 nodes), 2.187µs from start of poll, 1.720212617s from cluster start. Timeout at 2024-05-15 01:17:54.977325194 +0000 UTC m=+55.518709941, 1.005321513s after stability -2024-05-15T01:17:54.990Z [INFO] server-b22053e3-1fb4-72b2-42e0-8768987746a7: aborting pipeline replication: peer="{Voter server-1a0db606-4827-9421-4d8f-31f3da200b87 1a0db606-4827-9421-4d8f-31f3da200b87}" -2024-05-15T01:17:54.990Z [INFO] server-b22053e3-1fb4-72b2-42e0-8768987746a7: aborting pipeline replication: peer="{Voter server-8de8a9bc-4d3b-4f1b-dd0a-71d1ac25f9d0 8de8a9bc-4d3b-4f1b-dd0a-71d1ac25f9d0}" +2025-06-17T07:43:42.535Z [INFO] server-86904887-c9d6-223a-e12c-8c39b4a4ed5a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-86904887-c9d6-223a-e12c-8c39b4a4ed5a Address:86904887-c9d6-223a-e12c-8c39b4a4ed5a} {Suffrage:Voter ID:server-21aeff56-48b6-0a6b-4715-b1ddf63d6f8f Address:21aeff56-48b6-0a6b-4715-b1ddf63d6f8f} {Suffrage:Voter ID:server-67bb91a6-5425-22a3-2191-bf20fee2e0fe Address:67bb91a6-5425-22a3-2191-bf20fee2e0fe}]" +2025-06-17T07:43:42.535Z [INFO] server-86904887-c9d6-223a-e12c-8c39b4a4ed5a: entering follower state: follower="Node at 86904887-c9d6-223a-e12c-8c39b4a4ed5a [Follower]" leader-address= leader-id= +2025-06-17T07:43:42.535Z [INFO] server-21aeff56-48b6-0a6b-4715-b1ddf63d6f8f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-86904887-c9d6-223a-e12c-8c39b4a4ed5a Address:86904887-c9d6-223a-e12c-8c39b4a4ed5a} {Suffrage:Voter ID:server-21aeff56-48b6-0a6b-4715-b1ddf63d6f8f Address:21aeff56-48b6-0a6b-4715-b1ddf63d6f8f} {Suffrage:Voter ID:server-67bb91a6-5425-22a3-2191-bf20fee2e0fe Address:67bb91a6-5425-22a3-2191-bf20fee2e0fe}]" +2025-06-17T07:43:42.535Z [INFO] server-21aeff56-48b6-0a6b-4715-b1ddf63d6f8f: entering follower state: follower="Node at 21aeff56-48b6-0a6b-4715-b1ddf63d6f8f [Follower]" leader-address= leader-id= +2025-06-17T07:43:42.535Z [INFO] server-67bb91a6-5425-22a3-2191-bf20fee2e0fe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-86904887-c9d6-223a-e12c-8c39b4a4ed5a Address:86904887-c9d6-223a-e12c-8c39b4a4ed5a} {Suffrage:Voter ID:server-21aeff56-48b6-0a6b-4715-b1ddf63d6f8f Address:21aeff56-48b6-0a6b-4715-b1ddf63d6f8f} {Suffrage:Voter ID:server-67bb91a6-5425-22a3-2191-bf20fee2e0fe Address:67bb91a6-5425-22a3-2191-bf20fee2e0fe}]" +2025-06-17T07:43:42.535Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:42.535Z [INFO] server-67bb91a6-5425-22a3-2191-bf20fee2e0fe: entering follower state: follower="Node at 67bb91a6-5425-22a3-2191-bf20fee2e0fe [Follower]" leader-address= leader-id= +2025-06-17T07:43:43.256Z [WARN] server-67bb91a6-5425-22a3-2191-bf20fee2e0fe: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:43.256Z [INFO] server-67bb91a6-5425-22a3-2191-bf20fee2e0fe: entering candidate state: node="Node at 67bb91a6-5425-22a3-2191-bf20fee2e0fe [Candidate]" term=2 +2025-06-17T07:43:43.256Z [INFO] server-67bb91a6-5425-22a3-2191-bf20fee2e0fe: election won: term=2 tally=2 +2025-06-17T07:43:43.256Z [INFO] server-67bb91a6-5425-22a3-2191-bf20fee2e0fe: entering leader state: leader="Node at 67bb91a6-5425-22a3-2191-bf20fee2e0fe [Leader]" +2025-06-17T07:43:43.256Z [INFO] server-67bb91a6-5425-22a3-2191-bf20fee2e0fe: added peer, starting replication: peer=server-86904887-c9d6-223a-e12c-8c39b4a4ed5a +2025-06-17T07:43:43.256Z [INFO] server-67bb91a6-5425-22a3-2191-bf20fee2e0fe: added peer, starting replication: peer=server-21aeff56-48b6-0a6b-4715-b1ddf63d6f8f +2025-06-17T07:43:43.256Z [INFO] server-67bb91a6-5425-22a3-2191-bf20fee2e0fe: pipelining replication: peer="{Voter server-86904887-c9d6-223a-e12c-8c39b4a4ed5a 86904887-c9d6-223a-e12c-8c39b4a4ed5a}" +2025-06-17T07:43:43.256Z [INFO] server-67bb91a6-5425-22a3-2191-bf20fee2e0fe: pipelining replication: peer="{Voter server-21aeff56-48b6-0a6b-4715-b1ddf63d6f8f 21aeff56-48b6-0a6b-4715-b1ddf63d6f8f}" +2025-06-17T07:43:44.261Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:43.256292785 +0000 UTC m=+51.859821510 (1 nodes), 720.612347ms from start of poll, 720.941155ms from cluster start. Timeout at 2025-06-17 07:43:44.26165961 +0000 UTC m=+52.865188316, 1.005366806s after stability +2025-06-17T07:43:44.262Z [INFO] server-67bb91a6-5425-22a3-2191-bf20fee2e0fe: starting snapshot up to: index=12 +2025-06-17T07:43:44.262Z [INFO] creating new snapshot: path=/tmp/raft592900708/snapshots/2-12-1750146224262.tmp +2025-06-17T07:43:44.335Z [INFO] server-67bb91a6-5425-22a3-2191-bf20fee2e0fe: snapshot complete up to: index=12 +2025-06-17T07:43:44.336Z [INFO] server-451fe878-cd49-b465-6a10-f8c136c2083f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-451fe878-cd49-b465-6a10-f8c136c2083f Address:451fe878-cd49-b465-6a10-f8c136c2083f} {Suffrage:Voter ID:server-b403cc71-b228-1469-2901-10f3dc867e88 Address:b403cc71-b228-1469-2901-10f3dc867e88} {Suffrage:Voter ID:server-c764f7d7-4b83-b138-86fa-05b6ec47af49 Address:c764f7d7-4b83-b138-86fa-05b6ec47af49}]" +2025-06-17T07:43:44.336Z [INFO] server-451fe878-cd49-b465-6a10-f8c136c2083f: entering follower state: follower="Node at 451fe878-cd49-b465-6a10-f8c136c2083f [Follower]" leader-address= leader-id= +2025-06-17T07:43:44.336Z [INFO] server-b403cc71-b228-1469-2901-10f3dc867e88: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-451fe878-cd49-b465-6a10-f8c136c2083f Address:451fe878-cd49-b465-6a10-f8c136c2083f} {Suffrage:Voter ID:server-b403cc71-b228-1469-2901-10f3dc867e88 Address:b403cc71-b228-1469-2901-10f3dc867e88} {Suffrage:Voter ID:server-c764f7d7-4b83-b138-86fa-05b6ec47af49 Address:c764f7d7-4b83-b138-86fa-05b6ec47af49}]" +2025-06-17T07:43:44.336Z [INFO] server-b403cc71-b228-1469-2901-10f3dc867e88: entering follower state: follower="Node at b403cc71-b228-1469-2901-10f3dc867e88 [Follower]" leader-address= leader-id= +2025-06-17T07:43:44.336Z [INFO] server-c764f7d7-4b83-b138-86fa-05b6ec47af49: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-451fe878-cd49-b465-6a10-f8c136c2083f Address:451fe878-cd49-b465-6a10-f8c136c2083f} {Suffrage:Voter ID:server-b403cc71-b228-1469-2901-10f3dc867e88 Address:b403cc71-b228-1469-2901-10f3dc867e88} {Suffrage:Voter ID:server-c764f7d7-4b83-b138-86fa-05b6ec47af49 Address:c764f7d7-4b83-b138-86fa-05b6ec47af49}]" +2025-06-17T07:43:44.336Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:44.336Z [INFO] server-c764f7d7-4b83-b138-86fa-05b6ec47af49: entering follower state: follower="Node at c764f7d7-4b83-b138-86fa-05b6ec47af49 [Follower]" leader-address= leader-id= +2025-06-17T07:43:45.116Z [WARN] server-b403cc71-b228-1469-2901-10f3dc867e88: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:45.116Z [INFO] server-b403cc71-b228-1469-2901-10f3dc867e88: entering candidate state: node="Node at b403cc71-b228-1469-2901-10f3dc867e88 [Candidate]" term=2 +2025-06-17T07:43:45.116Z [INFO] server-b403cc71-b228-1469-2901-10f3dc867e88: election won: term=2 tally=2 +2025-06-17T07:43:45.116Z [INFO] server-b403cc71-b228-1469-2901-10f3dc867e88: entering leader state: leader="Node at b403cc71-b228-1469-2901-10f3dc867e88 [Leader]" +2025-06-17T07:43:45.116Z [INFO] server-b403cc71-b228-1469-2901-10f3dc867e88: added peer, starting replication: peer=server-451fe878-cd49-b465-6a10-f8c136c2083f +2025-06-17T07:43:45.116Z [INFO] server-b403cc71-b228-1469-2901-10f3dc867e88: added peer, starting replication: peer=server-c764f7d7-4b83-b138-86fa-05b6ec47af49 +2025-06-17T07:43:45.116Z [INFO] server-b403cc71-b228-1469-2901-10f3dc867e88: pipelining replication: peer="{Voter server-451fe878-cd49-b465-6a10-f8c136c2083f 451fe878-cd49-b465-6a10-f8c136c2083f}" +2025-06-17T07:43:45.116Z [INFO] server-b403cc71-b228-1469-2901-10f3dc867e88: pipelining replication: peer="{Voter server-c764f7d7-4b83-b138-86fa-05b6ec47af49 c764f7d7-4b83-b138-86fa-05b6ec47af49}" +2025-06-17T07:43:46.122Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:45.116845182 +0000 UTC m=+53.720373881 (1 nodes), 780.278099ms from start of poll, 780.615667ms from cluster start. Timeout at 2025-06-17 07:43:46.122608366 +0000 UTC m=+54.726137070, 1.005763189s after stability +2025-06-17T07:43:46.122Z [INFO] creating new snapshot: path=/tmp/raft1437869356/snapshots/2-15-1750146226122.tmp +2025-06-17T07:43:46.123Z [INFO] server-b403cc71-b228-1469-2901-10f3dc867e88: copied to local snapshot: bytes=71 +2025-06-17T07:43:46.123Z [INFO] server-b403cc71-b228-1469-2901-10f3dc867e88: snapshot restore progress: id=2-15-1750146226122 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:46.123Z [INFO] server-b403cc71-b228-1469-2901-10f3dc867e88: removing all old logs from log store +2025-06-17T07:43:46.123Z [INFO] server-b403cc71-b228-1469-2901-10f3dc867e88: compacting logs: from=1 to=2 +2025-06-17T07:43:46.123Z [INFO] server-b403cc71-b228-1469-2901-10f3dc867e88: restored user snapshot: index=15 +2025-06-17T07:43:46.123Z [ERROR] server-b403cc71-b228-1469-2901-10f3dc867e88: failed to get log: index=2 error="log not found" +2025-06-17T07:43:46.123Z [ERROR] server-b403cc71-b228-1469-2901-10f3dc867e88: failed to get log: index=2 error="log not found" +2025-06-17T07:43:46.123Z [INFO] server-b403cc71-b228-1469-2901-10f3dc867e88: aborting pipeline replication: peer="{Voter server-c764f7d7-4b83-b138-86fa-05b6ec47af49 c764f7d7-4b83-b138-86fa-05b6ec47af49}" +2025-06-17T07:43:46.123Z [INFO] server-b403cc71-b228-1469-2901-10f3dc867e88: aborting pipeline replication: peer="{Voter server-451fe878-cd49-b465-6a10-f8c136c2083f 451fe878-cd49-b465-6a10-f8c136c2083f}" +2025-06-17T07:43:46.130Z [ERROR] server-b403cc71-b228-1469-2901-10f3dc867e88: failed to get log: index=2 error="log not found" +2025-06-17T07:43:46.131Z [INFO] creating new snapshot: path=/tmp/raft1351568750/snapshots/2-15-1750146226131.tmp +2025-06-17T07:43:46.131Z [INFO] server-451fe878-cd49-b465-6a10-f8c136c2083f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:46.131Z [INFO] server-451fe878-cd49-b465-6a10-f8c136c2083f: copied to local snapshot: bytes=71 +2025-06-17T07:43:46.131Z [ERROR] server-b403cc71-b228-1469-2901-10f3dc867e88: failed to get log: index=2 error="log not found" +2025-06-17T07:43:46.131Z [INFO] server-451fe878-cd49-b465-6a10-f8c136c2083f: snapshot restore progress: id=2-15-1750146226131 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:46.131Z [INFO] server-451fe878-cd49-b465-6a10-f8c136c2083f: removing all old logs from log store +2025-06-17T07:43:46.131Z [INFO] server-451fe878-cd49-b465-6a10-f8c136c2083f: compacting logs: from=1 to=2 +2025-06-17T07:43:46.131Z [INFO] server-451fe878-cd49-b465-6a10-f8c136c2083f: Installed remote snapshot +2025-06-17T07:43:46.132Z [INFO] server-b403cc71-b228-1469-2901-10f3dc867e88: pipelining replication: peer="{Voter server-451fe878-cd49-b465-6a10-f8c136c2083f 451fe878-cd49-b465-6a10-f8c136c2083f}" +2025-06-17T07:43:46.132Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:46.132Z [INFO] creating new snapshot: path=/tmp/raft1163709017/snapshots/2-15-1750146226132.tmp +2025-06-17T07:43:46.132Z [INFO] server-c764f7d7-4b83-b138-86fa-05b6ec47af49: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:46.132Z [INFO] server-c764f7d7-4b83-b138-86fa-05b6ec47af49: copied to local snapshot: bytes=71 +2025-06-17T07:43:46.132Z [INFO] server-c764f7d7-4b83-b138-86fa-05b6ec47af49: snapshot restore progress: id=2-15-1750146226132 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:46.132Z [INFO] server-c764f7d7-4b83-b138-86fa-05b6ec47af49: removing all old logs from log store +2025-06-17T07:43:46.132Z [INFO] server-c764f7d7-4b83-b138-86fa-05b6ec47af49: compacting logs: from=1 to=2 +2025-06-17T07:43:46.132Z [INFO] server-c764f7d7-4b83-b138-86fa-05b6ec47af49: Installed remote snapshot +2025-06-17T07:43:46.133Z [INFO] server-b403cc71-b228-1469-2901-10f3dc867e88: pipelining replication: peer="{Voter server-c764f7d7-4b83-b138-86fa-05b6ec47af49 c764f7d7-4b83-b138-86fa-05b6ec47af49}" +2025-06-17T07:43:47.137Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:46.132072442 +0000 UTC m=+54.735601131 (2 nodes), 706ns from start of poll, 1.795842917s from cluster start. Timeout at 2025-06-17 07:43:47.137930292 +0000 UTC m=+55.741458967, 1.005857836s after stability +2025-06-17T07:43:47.149Z [INFO] server-67bb91a6-5425-22a3-2191-bf20fee2e0fe: aborting pipeline replication: peer="{Voter server-21aeff56-48b6-0a6b-4715-b1ddf63d6f8f 21aeff56-48b6-0a6b-4715-b1ddf63d6f8f}" +2025-06-17T07:43:47.149Z [INFO] server-67bb91a6-5425-22a3-2191-bf20fee2e0fe: aborting pipeline replication: peer="{Voter server-86904887-c9d6-223a-e12c-8c39b4a4ed5a 86904887-c9d6-223a-e12c-8c39b4a4ed5a}" === RUN TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false -2024-05-15T01:17:54.992Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196 Address:d5501b73-6cdd-74f0-92a9-b2c0ec40c196} {Suffrage:Voter ID:server-acaa848a-ed2a-3f80-07d7-216e135785b6 Address:acaa848a-ed2a-3f80-07d7-216e135785b6} {Suffrage:Voter ID:server-5d188fe0-5d5e-e96c-686a-7a9f7c74a82a Address:5d188fe0-5d5e-e96c-686a-7a9f7c74a82a}]" -2024-05-15T01:17:54.992Z [INFO] server-acaa848a-ed2a-3f80-07d7-216e135785b6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196 Address:d5501b73-6cdd-74f0-92a9-b2c0ec40c196} {Suffrage:Voter ID:server-acaa848a-ed2a-3f80-07d7-216e135785b6 Address:acaa848a-ed2a-3f80-07d7-216e135785b6} {Suffrage:Voter ID:server-5d188fe0-5d5e-e96c-686a-7a9f7c74a82a Address:5d188fe0-5d5e-e96c-686a-7a9f7c74a82a}]" -2024-05-15T01:17:54.992Z [INFO] server-5d188fe0-5d5e-e96c-686a-7a9f7c74a82a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196 Address:d5501b73-6cdd-74f0-92a9-b2c0ec40c196} {Suffrage:Voter ID:server-acaa848a-ed2a-3f80-07d7-216e135785b6 Address:acaa848a-ed2a-3f80-07d7-216e135785b6} {Suffrage:Voter ID:server-5d188fe0-5d5e-e96c-686a-7a9f7c74a82a Address:5d188fe0-5d5e-e96c-686a-7a9f7c74a82a}]" -2024-05-15T01:17:54.992Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:54.992Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: entering follower state: follower="Node at d5501b73-6cdd-74f0-92a9-b2c0ec40c196 [Follower]" leader-address= leader-id= -2024-05-15T01:17:54.992Z [INFO] server-acaa848a-ed2a-3f80-07d7-216e135785b6: entering follower state: follower="Node at acaa848a-ed2a-3f80-07d7-216e135785b6 [Follower]" leader-address= leader-id= -2024-05-15T01:17:54.992Z [INFO] server-5d188fe0-5d5e-e96c-686a-7a9f7c74a82a: entering follower state: follower="Node at 5d188fe0-5d5e-e96c-686a-7a9f7c74a82a [Follower]" leader-address= leader-id= -2024-05-15T01:17:55.522Z [WARN] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:55.522Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: entering candidate state: node="Node at d5501b73-6cdd-74f0-92a9-b2c0ec40c196 [Candidate]" term=2 -2024-05-15T01:17:55.523Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: election won: term=2 tally=2 -2024-05-15T01:17:55.523Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: entering leader state: leader="Node at d5501b73-6cdd-74f0-92a9-b2c0ec40c196 [Leader]" -2024-05-15T01:17:55.523Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: added peer, starting replication: peer=server-acaa848a-ed2a-3f80-07d7-216e135785b6 -2024-05-15T01:17:55.523Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: added peer, starting replication: peer=server-5d188fe0-5d5e-e96c-686a-7a9f7c74a82a -2024-05-15T01:17:55.523Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: pipelining replication: peer="{Voter server-5d188fe0-5d5e-e96c-686a-7a9f7c74a82a 5d188fe0-5d5e-e96c-686a-7a9f7c74a82a}" -2024-05-15T01:17:55.523Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: pipelining replication: peer="{Voter server-acaa848a-ed2a-3f80-07d7-216e135785b6 acaa848a-ed2a-3f80-07d7-216e135785b6}" -2024-05-15T01:17:56.529Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:55.523854744 +0000 UTC m=+56.065239573 (1 nodes), 531.345852ms from start of poll, 531.833582ms from cluster start. Timeout at 2024-05-15 01:17:56.52950881 +0000 UTC m=+57.070893690, 1.005654117s after stability -2024-05-15T01:17:56.531Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: starting snapshot up to: index=12 -2024-05-15T01:17:56.531Z [INFO] creating new snapshot: path=/tmp/raft613694535/snapshots/2-12-1715735876531.tmp -2024-05-15T01:17:56.533Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: snapshot complete up to: index=12 -2024-05-15T01:17:56.542Z [INFO] creating new snapshot: path=/tmp/raft613694535/snapshots/2-113-1715735876542.tmp -2024-05-15T01:17:56.543Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: copied to local snapshot: bytes=71 -2024-05-15T01:17:56.544Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: snapshot restore progress: id=2-113-1715735876542 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:56.544Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: restored user snapshot: index=113 -2024-05-15T01:17:56.544Z [ERROR] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: failed to get log: index=23 error="log not found" -2024-05-15T01:17:56.544Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: aborting pipeline replication: peer="{Voter server-5d188fe0-5d5e-e96c-686a-7a9f7c74a82a 5d188fe0-5d5e-e96c-686a-7a9f7c74a82a}" -2024-05-15T01:17:56.544Z [ERROR] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: failed to get log: index=23 error="log not found" -2024-05-15T01:17:56.544Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: aborting pipeline replication: peer="{Voter server-acaa848a-ed2a-3f80-07d7-216e135785b6 acaa848a-ed2a-3f80-07d7-216e135785b6}" -2024-05-15T01:17:56.550Z [ERROR] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: failed to get log: index=23 error="log not found" -2024-05-15T01:17:56.550Z [INFO] creating new snapshot: path=/tmp/raft3755344093/snapshots/2-113-1715735876550.tmp -2024-05-15T01:17:56.551Z [INFO] server-5d188fe0-5d5e-e96c-686a-7a9f7c74a82a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:56.551Z [INFO] server-5d188fe0-5d5e-e96c-686a-7a9f7c74a82a: copied to local snapshot: bytes=71 -2024-05-15T01:17:56.551Z [ERROR] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: failed to get log: index=23 error="log not found" -2024-05-15T01:17:56.552Z [INFO] creating new snapshot: path=/tmp/raft3170570729/snapshots/2-113-1715735876552.tmp -2024-05-15T01:17:56.552Z [INFO] server-acaa848a-ed2a-3f80-07d7-216e135785b6: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:56.552Z [INFO] server-acaa848a-ed2a-3f80-07d7-216e135785b6: copied to local snapshot: bytes=71 -2024-05-15T01:17:56.553Z [INFO] server-5d188fe0-5d5e-e96c-686a-7a9f7c74a82a: snapshot restore progress: id=2-113-1715735876550 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:56.553Z [INFO] server-5d188fe0-5d5e-e96c-686a-7a9f7c74a82a: Installed remote snapshot -2024-05-15T01:17:56.553Z [INFO] server-acaa848a-ed2a-3f80-07d7-216e135785b6: snapshot restore progress: id=2-113-1715735876552 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:56.553Z [INFO] server-acaa848a-ed2a-3f80-07d7-216e135785b6: Installed remote snapshot -2024-05-15T01:17:56.553Z [WARN] server-5d188fe0-5d5e-e96c-686a-7a9f7c74a82a: skipping application of old log: index=22 -2024-05-15T01:17:56.553Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: pipelining replication: peer="{Voter server-5d188fe0-5d5e-e96c-686a-7a9f7c74a82a 5d188fe0-5d5e-e96c-686a-7a9f7c74a82a}" -2024-05-15T01:17:56.553Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:56.554Z [WARN] server-acaa848a-ed2a-3f80-07d7-216e135785b6: skipping application of old log: index=22 -2024-05-15T01:17:56.554Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: pipelining replication: peer="{Voter server-acaa848a-ed2a-3f80-07d7-216e135785b6 acaa848a-ed2a-3f80-07d7-216e135785b6}" -2024-05-15T01:17:57.559Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:56.554008461 +0000 UTC m=+57.095393197 (2 nodes), 1.952µs from start of poll, 1.561987206s from cluster start. Timeout at 2024-05-15 01:17:57.55951197 +0000 UTC m=+58.100896843, 1.005503646s after stability -2024-05-15T01:17:57.579Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: aborting pipeline replication: peer="{Voter server-5d188fe0-5d5e-e96c-686a-7a9f7c74a82a 5d188fe0-5d5e-e96c-686a-7a9f7c74a82a}" -2024-05-15T01:17:57.579Z [INFO] server-d5501b73-6cdd-74f0-92a9-b2c0ec40c196: aborting pipeline replication: peer="{Voter server-acaa848a-ed2a-3f80-07d7-216e135785b6 acaa848a-ed2a-3f80-07d7-216e135785b6}" +2025-06-17T07:43:47.150Z [INFO] server-888f1dd7-2580-37c6-ad69-cd52ebad5bca: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-888f1dd7-2580-37c6-ad69-cd52ebad5bca Address:888f1dd7-2580-37c6-ad69-cd52ebad5bca} {Suffrage:Voter ID:server-0242e812-03f3-143b-e373-d70f26b6bce3 Address:0242e812-03f3-143b-e373-d70f26b6bce3} {Suffrage:Voter ID:server-cad1c9f1-4278-03d5-2606-4892e5ea9dbd Address:cad1c9f1-4278-03d5-2606-4892e5ea9dbd}]" +2025-06-17T07:43:47.150Z [INFO] server-888f1dd7-2580-37c6-ad69-cd52ebad5bca: entering follower state: follower="Node at 888f1dd7-2580-37c6-ad69-cd52ebad5bca [Follower]" leader-address= leader-id= +2025-06-17T07:43:47.150Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-888f1dd7-2580-37c6-ad69-cd52ebad5bca Address:888f1dd7-2580-37c6-ad69-cd52ebad5bca} {Suffrage:Voter ID:server-0242e812-03f3-143b-e373-d70f26b6bce3 Address:0242e812-03f3-143b-e373-d70f26b6bce3} {Suffrage:Voter ID:server-cad1c9f1-4278-03d5-2606-4892e5ea9dbd Address:cad1c9f1-4278-03d5-2606-4892e5ea9dbd}]" +2025-06-17T07:43:47.150Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: entering follower state: follower="Node at 0242e812-03f3-143b-e373-d70f26b6bce3 [Follower]" leader-address= leader-id= +2025-06-17T07:43:47.150Z [INFO] server-cad1c9f1-4278-03d5-2606-4892e5ea9dbd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-888f1dd7-2580-37c6-ad69-cd52ebad5bca Address:888f1dd7-2580-37c6-ad69-cd52ebad5bca} {Suffrage:Voter ID:server-0242e812-03f3-143b-e373-d70f26b6bce3 Address:0242e812-03f3-143b-e373-d70f26b6bce3} {Suffrage:Voter ID:server-cad1c9f1-4278-03d5-2606-4892e5ea9dbd Address:cad1c9f1-4278-03d5-2606-4892e5ea9dbd}]" +2025-06-17T07:43:47.150Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:47.150Z [INFO] server-cad1c9f1-4278-03d5-2606-4892e5ea9dbd: entering follower state: follower="Node at cad1c9f1-4278-03d5-2606-4892e5ea9dbd [Follower]" leader-address= leader-id= +2025-06-17T07:43:47.778Z [WARN] server-0242e812-03f3-143b-e373-d70f26b6bce3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:47.778Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: entering candidate state: node="Node at 0242e812-03f3-143b-e373-d70f26b6bce3 [Candidate]" term=2 +2025-06-17T07:43:47.778Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: election won: term=2 tally=2 +2025-06-17T07:43:47.778Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: entering leader state: leader="Node at 0242e812-03f3-143b-e373-d70f26b6bce3 [Leader]" +2025-06-17T07:43:47.778Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: added peer, starting replication: peer=server-888f1dd7-2580-37c6-ad69-cd52ebad5bca +2025-06-17T07:43:47.778Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: added peer, starting replication: peer=server-cad1c9f1-4278-03d5-2606-4892e5ea9dbd +2025-06-17T07:43:47.778Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: pipelining replication: peer="{Voter server-cad1c9f1-4278-03d5-2606-4892e5ea9dbd cad1c9f1-4278-03d5-2606-4892e5ea9dbd}" +2025-06-17T07:43:47.778Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: pipelining replication: peer="{Voter server-888f1dd7-2580-37c6-ad69-cd52ebad5bca 888f1dd7-2580-37c6-ad69-cd52ebad5bca}" +2025-06-17T07:43:48.784Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:47.778652844 +0000 UTC m=+56.382181556 (1 nodes), 627.859939ms from start of poll, 628.243575ms from cluster start. Timeout at 2025-06-17 07:43:48.784657694 +0000 UTC m=+57.388186376, 1.00600482s after stability +2025-06-17T07:43:48.785Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: starting snapshot up to: index=12 +2025-06-17T07:43:48.785Z [INFO] creating new snapshot: path=/tmp/raft682565148/snapshots/2-12-1750146228785.tmp +2025-06-17T07:43:48.785Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: snapshot complete up to: index=12 +2025-06-17T07:43:48.786Z [INFO] creating new snapshot: path=/tmp/raft682565148/snapshots/2-113-1750146228786.tmp +2025-06-17T07:43:48.786Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: copied to local snapshot: bytes=71 +2025-06-17T07:43:48.786Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: snapshot restore progress: id=2-113-1750146228786 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:48.786Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: restored user snapshot: index=113 +2025-06-17T07:43:48.786Z [ERROR] server-0242e812-03f3-143b-e373-d70f26b6bce3: failed to get log: index=23 error="log not found" +2025-06-17T07:43:48.786Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: aborting pipeline replication: peer="{Voter server-cad1c9f1-4278-03d5-2606-4892e5ea9dbd cad1c9f1-4278-03d5-2606-4892e5ea9dbd}" +2025-06-17T07:43:48.786Z [ERROR] server-0242e812-03f3-143b-e373-d70f26b6bce3: failed to get log: index=23 error="log not found" +2025-06-17T07:43:48.786Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: aborting pipeline replication: peer="{Voter server-888f1dd7-2580-37c6-ad69-cd52ebad5bca 888f1dd7-2580-37c6-ad69-cd52ebad5bca}" +2025-06-17T07:43:48.795Z [ERROR] server-0242e812-03f3-143b-e373-d70f26b6bce3: failed to get log: index=23 error="log not found" +2025-06-17T07:43:48.795Z [INFO] creating new snapshot: path=/tmp/raft1525622868/snapshots/2-113-1750146228795.tmp +2025-06-17T07:43:48.795Z [ERROR] server-0242e812-03f3-143b-e373-d70f26b6bce3: failed to get log: index=23 error="log not found" +2025-06-17T07:43:48.795Z [INFO] server-cad1c9f1-4278-03d5-2606-4892e5ea9dbd: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:48.795Z [INFO] server-cad1c9f1-4278-03d5-2606-4892e5ea9dbd: copied to local snapshot: bytes=71 +2025-06-17T07:43:48.795Z [INFO] creating new snapshot: path=/tmp/raft1863778948/snapshots/2-113-1750146228795.tmp +2025-06-17T07:43:48.795Z [INFO] server-cad1c9f1-4278-03d5-2606-4892e5ea9dbd: snapshot restore progress: id=2-113-1750146228795 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:48.796Z [INFO] server-cad1c9f1-4278-03d5-2606-4892e5ea9dbd: Installed remote snapshot +2025-06-17T07:43:48.796Z [WARN] server-cad1c9f1-4278-03d5-2606-4892e5ea9dbd: skipping application of old log: index=22 +2025-06-17T07:43:48.796Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: pipelining replication: peer="{Voter server-cad1c9f1-4278-03d5-2606-4892e5ea9dbd cad1c9f1-4278-03d5-2606-4892e5ea9dbd}" +2025-06-17T07:43:48.796Z [INFO] server-888f1dd7-2580-37c6-ad69-cd52ebad5bca: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:48.796Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:48.796Z [INFO] server-888f1dd7-2580-37c6-ad69-cd52ebad5bca: copied to local snapshot: bytes=71 +2025-06-17T07:43:48.796Z [INFO] server-888f1dd7-2580-37c6-ad69-cd52ebad5bca: snapshot restore progress: id=2-113-1750146228795 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:48.796Z [INFO] server-888f1dd7-2580-37c6-ad69-cd52ebad5bca: Installed remote snapshot +2025-06-17T07:43:48.796Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: pipelining replication: peer="{Voter server-888f1dd7-2580-37c6-ad69-cd52ebad5bca 888f1dd7-2580-37c6-ad69-cd52ebad5bca}" +2025-06-17T07:43:49.802Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:48.796172931 +0000 UTC m=+57.399701616 (2 nodes), 637ns from start of poll, 1.645763635s from cluster start. Timeout at 2025-06-17 07:43:49.801989682 +0000 UTC m=+58.405518364, 1.005816748s after stability +2025-06-17T07:43:49.813Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: aborting pipeline replication: peer="{Voter server-888f1dd7-2580-37c6-ad69-cd52ebad5bca 888f1dd7-2580-37c6-ad69-cd52ebad5bca}" +2025-06-17T07:43:49.813Z [INFO] server-0242e812-03f3-143b-e373-d70f26b6bce3: aborting pipeline replication: peer="{Voter server-cad1c9f1-4278-03d5-2606-4892e5ea9dbd cad1c9f1-4278-03d5-2606-4892e5ea9dbd}" === RUN TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false -2024-05-15T01:17:57.581Z [INFO] server-11b4a468-6e2d-5c6b-bbd8-264e99c51202: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-11b4a468-6e2d-5c6b-bbd8-264e99c51202 Address:11b4a468-6e2d-5c6b-bbd8-264e99c51202} {Suffrage:Voter ID:server-f6d749a7-ba76-9d3b-b410-5a4143da3492 Address:f6d749a7-ba76-9d3b-b410-5a4143da3492} {Suffrage:Voter ID:server-e12560f5-d59e-4139-19ec-fb31af8d1d5e Address:e12560f5-d59e-4139-19ec-fb31af8d1d5e}]" -2024-05-15T01:17:57.581Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-11b4a468-6e2d-5c6b-bbd8-264e99c51202 Address:11b4a468-6e2d-5c6b-bbd8-264e99c51202} {Suffrage:Voter ID:server-f6d749a7-ba76-9d3b-b410-5a4143da3492 Address:f6d749a7-ba76-9d3b-b410-5a4143da3492} {Suffrage:Voter ID:server-e12560f5-d59e-4139-19ec-fb31af8d1d5e Address:e12560f5-d59e-4139-19ec-fb31af8d1d5e}]" -2024-05-15T01:17:57.582Z [INFO] server-e12560f5-d59e-4139-19ec-fb31af8d1d5e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-11b4a468-6e2d-5c6b-bbd8-264e99c51202 Address:11b4a468-6e2d-5c6b-bbd8-264e99c51202} {Suffrage:Voter ID:server-f6d749a7-ba76-9d3b-b410-5a4143da3492 Address:f6d749a7-ba76-9d3b-b410-5a4143da3492} {Suffrage:Voter ID:server-e12560f5-d59e-4139-19ec-fb31af8d1d5e Address:e12560f5-d59e-4139-19ec-fb31af8d1d5e}]" -2024-05-15T01:17:57.582Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:17:57.582Z [INFO] server-11b4a468-6e2d-5c6b-bbd8-264e99c51202: entering follower state: follower="Node at 11b4a468-6e2d-5c6b-bbd8-264e99c51202 [Follower]" leader-address= leader-id= -2024-05-15T01:17:57.582Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: entering follower state: follower="Node at f6d749a7-ba76-9d3b-b410-5a4143da3492 [Follower]" leader-address= leader-id= -2024-05-15T01:17:57.582Z [INFO] server-e12560f5-d59e-4139-19ec-fb31af8d1d5e: entering follower state: follower="Node at e12560f5-d59e-4139-19ec-fb31af8d1d5e [Follower]" leader-address= leader-id= -2024-05-15T01:17:58.143Z [WARN] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:17:58.143Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: entering candidate state: node="Node at f6d749a7-ba76-9d3b-b410-5a4143da3492 [Candidate]" term=2 -2024-05-15T01:17:58.143Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: election won: term=2 tally=2 -2024-05-15T01:17:58.143Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: entering leader state: leader="Node at f6d749a7-ba76-9d3b-b410-5a4143da3492 [Leader]" -2024-05-15T01:17:58.143Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: added peer, starting replication: peer=server-11b4a468-6e2d-5c6b-bbd8-264e99c51202 -2024-05-15T01:17:58.143Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: added peer, starting replication: peer=server-e12560f5-d59e-4139-19ec-fb31af8d1d5e -2024-05-15T01:17:58.143Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: pipelining replication: peer="{Voter server-e12560f5-d59e-4139-19ec-fb31af8d1d5e e12560f5-d59e-4139-19ec-fb31af8d1d5e}" -2024-05-15T01:17:58.143Z [WARN] server-e12560f5-d59e-4139-19ec-fb31af8d1d5e: rejecting vote request since our last term is greater: candidate=f6d749a7-ba76-9d3b-b410-5a4143da3492 last-term=2 last-candidate-term=1 -2024-05-15T01:17:58.144Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: pipelining replication: peer="{Voter server-11b4a468-6e2d-5c6b-bbd8-264e99c51202 11b4a468-6e2d-5c6b-bbd8-264e99c51202}" -2024-05-15T01:17:59.149Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:17:58.143945568 +0000 UTC m=+58.685330315 (1 nodes), 561.79555ms from start of poll, 562.280611ms from cluster start. Timeout at 2024-05-15 01:17:59.149883344 +0000 UTC m=+59.691268120, 1.005937805s after stability -2024-05-15T01:17:59.151Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: starting snapshot up to: index=12 -2024-05-15T01:17:59.151Z [INFO] creating new snapshot: path=/tmp/raft538066718/snapshots/2-12-1715735879151.tmp -2024-05-15T01:17:59.152Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: snapshot complete up to: index=12 -2024-05-15T01:17:59.154Z [INFO] creating new snapshot: path=/tmp/raft538066718/snapshots/2-113-1715735879154.tmp -2024-05-15T01:17:59.155Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: copied to local snapshot: bytes=71 -2024-05-15T01:17:59.155Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: snapshot restore progress: id=2-113-1715735879154 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:59.155Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: removing all old logs from log store -2024-05-15T01:17:59.155Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: compacting logs: from=1 to=22 -2024-05-15T01:17:59.155Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: restored user snapshot: index=113 -2024-05-15T01:17:59.155Z [ERROR] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: failed to get log: index=22 error="log not found" -2024-05-15T01:17:59.155Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: aborting pipeline replication: peer="{Voter server-11b4a468-6e2d-5c6b-bbd8-264e99c51202 11b4a468-6e2d-5c6b-bbd8-264e99c51202}" -2024-05-15T01:17:59.155Z [ERROR] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: failed to get log: index=22 error="log not found" -2024-05-15T01:17:59.155Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: aborting pipeline replication: peer="{Voter server-e12560f5-d59e-4139-19ec-fb31af8d1d5e e12560f5-d59e-4139-19ec-fb31af8d1d5e}" -2024-05-15T01:17:59.162Z [ERROR] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: failed to get log: index=22 error="log not found" -2024-05-15T01:17:59.164Z [INFO] creating new snapshot: path=/tmp/raft3846117297/snapshots/2-113-1715735879164.tmp -2024-05-15T01:17:59.164Z [ERROR] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: failed to get log: index=22 error="log not found" -2024-05-15T01:17:59.164Z [INFO] server-e12560f5-d59e-4139-19ec-fb31af8d1d5e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:59.164Z [INFO] server-e12560f5-d59e-4139-19ec-fb31af8d1d5e: copied to local snapshot: bytes=71 -2024-05-15T01:17:59.165Z [INFO] creating new snapshot: path=/tmp/raft4195619064/snapshots/2-113-1715735879165.tmp -2024-05-15T01:17:59.165Z [INFO] server-e12560f5-d59e-4139-19ec-fb31af8d1d5e: snapshot restore progress: id=2-113-1715735879164 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:59.165Z [INFO] server-e12560f5-d59e-4139-19ec-fb31af8d1d5e: removing all old logs from log store -2024-05-15T01:17:59.165Z [INFO] server-e12560f5-d59e-4139-19ec-fb31af8d1d5e: compacting logs: from=1 to=22 -2024-05-15T01:17:59.165Z [INFO] server-e12560f5-d59e-4139-19ec-fb31af8d1d5e: Installed remote snapshot -2024-05-15T01:17:59.165Z [WARN] server-e12560f5-d59e-4139-19ec-fb31af8d1d5e: skipping application of old log: index=22 -2024-05-15T01:17:59.165Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: pipelining replication: peer="{Voter server-e12560f5-d59e-4139-19ec-fb31af8d1d5e e12560f5-d59e-4139-19ec-fb31af8d1d5e}" -2024-05-15T01:17:59.165Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:17:59.166Z [INFO] server-11b4a468-6e2d-5c6b-bbd8-264e99c51202: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:59.166Z [INFO] server-11b4a468-6e2d-5c6b-bbd8-264e99c51202: copied to local snapshot: bytes=71 -2024-05-15T01:17:59.167Z [INFO] server-11b4a468-6e2d-5c6b-bbd8-264e99c51202: snapshot restore progress: id=2-113-1715735879165 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:17:59.167Z [INFO] server-11b4a468-6e2d-5c6b-bbd8-264e99c51202: removing all old logs from log store -2024-05-15T01:17:59.167Z [INFO] server-11b4a468-6e2d-5c6b-bbd8-264e99c51202: compacting logs: from=1 to=22 -2024-05-15T01:17:59.167Z [INFO] server-11b4a468-6e2d-5c6b-bbd8-264e99c51202: Installed remote snapshot -2024-05-15T01:17:59.167Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: pipelining replication: peer="{Voter server-11b4a468-6e2d-5c6b-bbd8-264e99c51202 11b4a468-6e2d-5c6b-bbd8-264e99c51202}" -2024-05-15T01:18:00.171Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:17:59.165698682 +0000 UTC m=+59.707083675 (2 nodes), 2.233µs from start of poll, 1.584033971s from cluster start. Timeout at 2024-05-15 01:18:00.171377563 +0000 UTC m=+60.712762303, 1.005678628s after stability -2024-05-15T01:18:00.185Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: aborting pipeline replication: peer="{Voter server-e12560f5-d59e-4139-19ec-fb31af8d1d5e e12560f5-d59e-4139-19ec-fb31af8d1d5e}" -2024-05-15T01:18:00.185Z [INFO] server-f6d749a7-ba76-9d3b-b410-5a4143da3492: aborting pipeline replication: peer="{Voter server-11b4a468-6e2d-5c6b-bbd8-264e99c51202 11b4a468-6e2d-5c6b-bbd8-264e99c51202}" +2025-06-17T07:43:49.814Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc Address:dbf22dd8-22fd-acfb-0b53-10d8be085bfc} {Suffrage:Voter ID:server-80d64ab0-8814-6888-8d22-ed6e595d47f8 Address:80d64ab0-8814-6888-8d22-ed6e595d47f8} {Suffrage:Voter ID:server-60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f Address:60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f}]" +2025-06-17T07:43:49.814Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: entering follower state: follower="Node at dbf22dd8-22fd-acfb-0b53-10d8be085bfc [Follower]" leader-address= leader-id= +2025-06-17T07:43:49.814Z [INFO] server-80d64ab0-8814-6888-8d22-ed6e595d47f8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc Address:dbf22dd8-22fd-acfb-0b53-10d8be085bfc} {Suffrage:Voter ID:server-80d64ab0-8814-6888-8d22-ed6e595d47f8 Address:80d64ab0-8814-6888-8d22-ed6e595d47f8} {Suffrage:Voter ID:server-60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f Address:60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f}]" +2025-06-17T07:43:49.814Z [INFO] server-80d64ab0-8814-6888-8d22-ed6e595d47f8: entering follower state: follower="Node at 80d64ab0-8814-6888-8d22-ed6e595d47f8 [Follower]" leader-address= leader-id= +2025-06-17T07:43:49.814Z [INFO] server-60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc Address:dbf22dd8-22fd-acfb-0b53-10d8be085bfc} {Suffrage:Voter ID:server-80d64ab0-8814-6888-8d22-ed6e595d47f8 Address:80d64ab0-8814-6888-8d22-ed6e595d47f8} {Suffrage:Voter ID:server-60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f Address:60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f}]" +2025-06-17T07:43:49.814Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:49.814Z [INFO] server-60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f: entering follower state: follower="Node at 60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f [Follower]" leader-address= leader-id= +2025-06-17T07:43:50.353Z [WARN] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:50.353Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: entering candidate state: node="Node at dbf22dd8-22fd-acfb-0b53-10d8be085bfc [Candidate]" term=2 +2025-06-17T07:43:50.353Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: election won: term=2 tally=2 +2025-06-17T07:43:50.353Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: entering leader state: leader="Node at dbf22dd8-22fd-acfb-0b53-10d8be085bfc [Leader]" +2025-06-17T07:43:50.353Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: added peer, starting replication: peer=server-80d64ab0-8814-6888-8d22-ed6e595d47f8 +2025-06-17T07:43:50.353Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: added peer, starting replication: peer=server-60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f +2025-06-17T07:43:50.353Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: pipelining replication: peer="{Voter server-60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f 60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f}" +2025-06-17T07:43:50.353Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: pipelining replication: peer="{Voter server-80d64ab0-8814-6888-8d22-ed6e595d47f8 80d64ab0-8814-6888-8d22-ed6e595d47f8}" +2025-06-17T07:43:51.359Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:50.353387681 +0000 UTC m=+58.956916371 (1 nodes), 538.474731ms from start of poll, 538.801483ms from cluster start. Timeout at 2025-06-17 07:43:51.359163299 +0000 UTC m=+59.962691989, 1.005775618s after stability +2025-06-17T07:43:51.359Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: starting snapshot up to: index=12 +2025-06-17T07:43:51.359Z [INFO] creating new snapshot: path=/tmp/raft214016913/snapshots/2-12-1750146231359.tmp +2025-06-17T07:43:51.360Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: snapshot complete up to: index=12 +2025-06-17T07:43:51.362Z [INFO] creating new snapshot: path=/tmp/raft214016913/snapshots/2-113-1750146231362.tmp +2025-06-17T07:43:51.362Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: copied to local snapshot: bytes=71 +2025-06-17T07:43:51.362Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: snapshot restore progress: id=2-113-1750146231362 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:51.362Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: removing all old logs from log store +2025-06-17T07:43:51.362Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: compacting logs: from=1 to=22 +2025-06-17T07:43:51.362Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: restored user snapshot: index=113 +2025-06-17T07:43:51.362Z [ERROR] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: failed to get log: index=22 error="log not found" +2025-06-17T07:43:51.362Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: aborting pipeline replication: peer="{Voter server-60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f 60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f}" +2025-06-17T07:43:51.362Z [ERROR] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: failed to get log: index=22 error="log not found" +2025-06-17T07:43:51.362Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: aborting pipeline replication: peer="{Voter server-80d64ab0-8814-6888-8d22-ed6e595d47f8 80d64ab0-8814-6888-8d22-ed6e595d47f8}" +2025-06-17T07:43:51.369Z [ERROR] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: failed to get log: index=22 error="log not found" +2025-06-17T07:43:51.370Z [INFO] creating new snapshot: path=/tmp/raft4087622971/snapshots/2-113-1750146231370.tmp +2025-06-17T07:43:51.370Z [INFO] server-60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:51.370Z [INFO] server-60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f: copied to local snapshot: bytes=71 +2025-06-17T07:43:51.370Z [ERROR] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: failed to get log: index=22 error="log not found" +2025-06-17T07:43:51.370Z [INFO] server-60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f: snapshot restore progress: id=2-113-1750146231370 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:51.370Z [INFO] server-60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f: removing all old logs from log store +2025-06-17T07:43:51.370Z [INFO] server-60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f: compacting logs: from=1 to=22 +2025-06-17T07:43:51.370Z [INFO] server-60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f: Installed remote snapshot +2025-06-17T07:43:51.370Z [WARN] server-60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f: skipping application of old log: index=22 +2025-06-17T07:43:51.370Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: pipelining replication: peer="{Voter server-60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f 60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f}" +2025-06-17T07:43:51.371Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:51.371Z [INFO] creating new snapshot: path=/tmp/raft3488587525/snapshots/2-113-1750146231371.tmp +2025-06-17T07:43:51.371Z [INFO] server-80d64ab0-8814-6888-8d22-ed6e595d47f8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:51.371Z [INFO] server-80d64ab0-8814-6888-8d22-ed6e595d47f8: copied to local snapshot: bytes=71 +2025-06-17T07:43:51.371Z [INFO] server-80d64ab0-8814-6888-8d22-ed6e595d47f8: snapshot restore progress: id=2-113-1750146231371 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:51.371Z [INFO] server-80d64ab0-8814-6888-8d22-ed6e595d47f8: removing all old logs from log store +2025-06-17T07:43:51.371Z [INFO] server-80d64ab0-8814-6888-8d22-ed6e595d47f8: compacting logs: from=1 to=22 +2025-06-17T07:43:51.371Z [INFO] server-80d64ab0-8814-6888-8d22-ed6e595d47f8: Installed remote snapshot +2025-06-17T07:43:51.372Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: pipelining replication: peer="{Voter server-80d64ab0-8814-6888-8d22-ed6e595d47f8 80d64ab0-8814-6888-8d22-ed6e595d47f8}" +2025-06-17T07:43:52.377Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:51.371028162 +0000 UTC m=+59.974556851 (2 nodes), 914ns from start of poll, 1.556441963s from cluster start. Timeout at 2025-06-17 07:43:52.377076855 +0000 UTC m=+60.980605571, 1.00604872s after stability +2025-06-17T07:43:52.389Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: aborting pipeline replication: peer="{Voter server-60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f 60f99dcb-3a5f-1f45-f34f-2ee3fbbf030f}" +2025-06-17T07:43:52.389Z [INFO] server-dbf22dd8-22fd-acfb-0b53-10d8be085bfc: aborting pipeline replication: peer="{Voter server-80d64ab0-8814-6888-8d22-ed6e595d47f8 80d64ab0-8814-6888-8d22-ed6e595d47f8}" === RUN TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true -2024-05-15T01:18:00.187Z [INFO] server-e7c38667-f13f-85b0-34c1-445d9d0969a2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e7c38667-f13f-85b0-34c1-445d9d0969a2 Address:e7c38667-f13f-85b0-34c1-445d9d0969a2} {Suffrage:Voter ID:server-d1898434-3740-a5ae-adbb-c2f3258472fe Address:d1898434-3740-a5ae-adbb-c2f3258472fe} {Suffrage:Voter ID:server-20e918b0-b9bb-5cf7-8922-8c494c5231d6 Address:20e918b0-b9bb-5cf7-8922-8c494c5231d6}]" -2024-05-15T01:18:00.187Z [INFO] server-e7c38667-f13f-85b0-34c1-445d9d0969a2: entering follower state: follower="Node at e7c38667-f13f-85b0-34c1-445d9d0969a2 [Follower]" leader-address= leader-id= -2024-05-15T01:18:00.187Z [INFO] server-d1898434-3740-a5ae-adbb-c2f3258472fe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e7c38667-f13f-85b0-34c1-445d9d0969a2 Address:e7c38667-f13f-85b0-34c1-445d9d0969a2} {Suffrage:Voter ID:server-d1898434-3740-a5ae-adbb-c2f3258472fe Address:d1898434-3740-a5ae-adbb-c2f3258472fe} {Suffrage:Voter ID:server-20e918b0-b9bb-5cf7-8922-8c494c5231d6 Address:20e918b0-b9bb-5cf7-8922-8c494c5231d6}]" -2024-05-15T01:18:00.187Z [INFO] server-d1898434-3740-a5ae-adbb-c2f3258472fe: entering follower state: follower="Node at d1898434-3740-a5ae-adbb-c2f3258472fe [Follower]" leader-address= leader-id= -2024-05-15T01:18:00.187Z [INFO] server-20e918b0-b9bb-5cf7-8922-8c494c5231d6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e7c38667-f13f-85b0-34c1-445d9d0969a2 Address:e7c38667-f13f-85b0-34c1-445d9d0969a2} {Suffrage:Voter ID:server-d1898434-3740-a5ae-adbb-c2f3258472fe Address:d1898434-3740-a5ae-adbb-c2f3258472fe} {Suffrage:Voter ID:server-20e918b0-b9bb-5cf7-8922-8c494c5231d6 Address:20e918b0-b9bb-5cf7-8922-8c494c5231d6}]" -2024-05-15T01:18:00.187Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:00.187Z [INFO] server-20e918b0-b9bb-5cf7-8922-8c494c5231d6: entering follower state: follower="Node at 20e918b0-b9bb-5cf7-8922-8c494c5231d6 [Follower]" leader-address= leader-id= -2024-05-15T01:18:00.743Z [WARN] server-20e918b0-b9bb-5cf7-8922-8c494c5231d6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:00.743Z [INFO] server-20e918b0-b9bb-5cf7-8922-8c494c5231d6: entering candidate state: node="Node at 20e918b0-b9bb-5cf7-8922-8c494c5231d6 [Candidate]" term=2 -2024-05-15T01:18:00.743Z [INFO] server-20e918b0-b9bb-5cf7-8922-8c494c5231d6: election won: term=2 tally=2 -2024-05-15T01:18:00.743Z [INFO] server-20e918b0-b9bb-5cf7-8922-8c494c5231d6: entering leader state: leader="Node at 20e918b0-b9bb-5cf7-8922-8c494c5231d6 [Leader]" -2024-05-15T01:18:00.743Z [INFO] server-20e918b0-b9bb-5cf7-8922-8c494c5231d6: added peer, starting replication: peer=server-e7c38667-f13f-85b0-34c1-445d9d0969a2 -2024-05-15T01:18:00.743Z [INFO] server-20e918b0-b9bb-5cf7-8922-8c494c5231d6: added peer, starting replication: peer=server-d1898434-3740-a5ae-adbb-c2f3258472fe -2024-05-15T01:18:00.743Z [INFO] server-20e918b0-b9bb-5cf7-8922-8c494c5231d6: pipelining replication: peer="{Voter server-d1898434-3740-a5ae-adbb-c2f3258472fe d1898434-3740-a5ae-adbb-c2f3258472fe}" -2024-05-15T01:18:00.743Z [INFO] server-20e918b0-b9bb-5cf7-8922-8c494c5231d6: pipelining replication: peer="{Voter server-e7c38667-f13f-85b0-34c1-445d9d0969a2 e7c38667-f13f-85b0-34c1-445d9d0969a2}" -2024-05-15T01:18:01.749Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:00.743795908 +0000 UTC m=+61.285187547 (1 nodes), 555.854968ms from start of poll, 556.442293ms from cluster start. Timeout at 2024-05-15 01:18:01.749882426 +0000 UTC m=+62.291267396, 1.006079849s after stability -2024-05-15T01:18:01.751Z [INFO] server-20e918b0-b9bb-5cf7-8922-8c494c5231d6: starting snapshot up to: index=12 -2024-05-15T01:18:01.751Z [INFO] creating new snapshot: path=/tmp/raft395912527/snapshots/2-12-1715735881751.tmp -2024-05-15T01:18:01.752Z [INFO] server-20e918b0-b9bb-5cf7-8922-8c494c5231d6: snapshot complete up to: index=12 -2024-05-15T01:18:01.754Z [INFO] server-70e7759b-4410-d190-a867-13e393de32ec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-70e7759b-4410-d190-a867-13e393de32ec Address:70e7759b-4410-d190-a867-13e393de32ec} {Suffrage:Voter ID:server-83f892d7-37c0-8def-1ff6-1f58f211f71b Address:83f892d7-37c0-8def-1ff6-1f58f211f71b} {Suffrage:Voter ID:server-17891503-d689-dd37-e6d0-b4a442fce372 Address:17891503-d689-dd37-e6d0-b4a442fce372}]" -2024-05-15T01:18:01.754Z [INFO] server-70e7759b-4410-d190-a867-13e393de32ec: entering follower state: follower="Node at 70e7759b-4410-d190-a867-13e393de32ec [Follower]" leader-address= leader-id= -2024-05-15T01:18:01.754Z [INFO] server-83f892d7-37c0-8def-1ff6-1f58f211f71b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-70e7759b-4410-d190-a867-13e393de32ec Address:70e7759b-4410-d190-a867-13e393de32ec} {Suffrage:Voter ID:server-83f892d7-37c0-8def-1ff6-1f58f211f71b Address:83f892d7-37c0-8def-1ff6-1f58f211f71b} {Suffrage:Voter ID:server-17891503-d689-dd37-e6d0-b4a442fce372 Address:17891503-d689-dd37-e6d0-b4a442fce372}]" -2024-05-15T01:18:01.754Z [INFO] server-17891503-d689-dd37-e6d0-b4a442fce372: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-70e7759b-4410-d190-a867-13e393de32ec Address:70e7759b-4410-d190-a867-13e393de32ec} {Suffrage:Voter ID:server-83f892d7-37c0-8def-1ff6-1f58f211f71b Address:83f892d7-37c0-8def-1ff6-1f58f211f71b} {Suffrage:Voter ID:server-17891503-d689-dd37-e6d0-b4a442fce372 Address:17891503-d689-dd37-e6d0-b4a442fce372}]" -2024-05-15T01:18:01.754Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:01.754Z [INFO] server-83f892d7-37c0-8def-1ff6-1f58f211f71b: entering follower state: follower="Node at 83f892d7-37c0-8def-1ff6-1f58f211f71b [Follower]" leader-address= leader-id= -2024-05-15T01:18:01.754Z [INFO] server-17891503-d689-dd37-e6d0-b4a442fce372: entering follower state: follower="Node at 17891503-d689-dd37-e6d0-b4a442fce372 [Follower]" leader-address= leader-id= -2024-05-15T01:18:02.312Z [WARN] server-70e7759b-4410-d190-a867-13e393de32ec: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:02.312Z [INFO] server-70e7759b-4410-d190-a867-13e393de32ec: entering candidate state: node="Node at 70e7759b-4410-d190-a867-13e393de32ec [Candidate]" term=2 -2024-05-15T01:18:02.312Z [INFO] server-70e7759b-4410-d190-a867-13e393de32ec: election won: term=2 tally=2 -2024-05-15T01:18:02.312Z [INFO] server-70e7759b-4410-d190-a867-13e393de32ec: entering leader state: leader="Node at 70e7759b-4410-d190-a867-13e393de32ec [Leader]" -2024-05-15T01:18:02.312Z [INFO] server-70e7759b-4410-d190-a867-13e393de32ec: added peer, starting replication: peer=server-83f892d7-37c0-8def-1ff6-1f58f211f71b -2024-05-15T01:18:02.312Z [INFO] server-70e7759b-4410-d190-a867-13e393de32ec: added peer, starting replication: peer=server-17891503-d689-dd37-e6d0-b4a442fce372 -2024-05-15T01:18:02.313Z [INFO] server-70e7759b-4410-d190-a867-13e393de32ec: pipelining replication: peer="{Voter server-17891503-d689-dd37-e6d0-b4a442fce372 17891503-d689-dd37-e6d0-b4a442fce372}" -2024-05-15T01:18:02.313Z [INFO] server-70e7759b-4410-d190-a867-13e393de32ec: pipelining replication: peer="{Voter server-83f892d7-37c0-8def-1ff6-1f58f211f71b 83f892d7-37c0-8def-1ff6-1f58f211f71b}" -2024-05-15T01:18:03.318Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:02.313001987 +0000 UTC m=+62.854386950 (1 nodes), 558.085727ms from start of poll, 558.664686ms from cluster start. Timeout at 2024-05-15 01:18:03.31835282 +0000 UTC m=+63.859737995, 1.005351045s after stability -2024-05-15T01:18:03.318Z [INFO] creating new snapshot: path=/tmp/raft2677009226/snapshots/2-113-1715735883318.tmp -2024-05-15T01:18:03.373Z [INFO] server-70e7759b-4410-d190-a867-13e393de32ec: copied to local snapshot: bytes=71 -2024-05-15T01:18:03.373Z [INFO] server-70e7759b-4410-d190-a867-13e393de32ec: snapshot restore progress: id=2-113-1715735883318 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:03.373Z [INFO] server-70e7759b-4410-d190-a867-13e393de32ec: restored user snapshot: index=113 -2024-05-15T01:18:03.373Z [ERROR] server-70e7759b-4410-d190-a867-13e393de32ec: failed to get log: index=3 error="log not found" -2024-05-15T01:18:03.373Z [INFO] server-70e7759b-4410-d190-a867-13e393de32ec: aborting pipeline replication: peer="{Voter server-17891503-d689-dd37-e6d0-b4a442fce372 17891503-d689-dd37-e6d0-b4a442fce372}" -2024-05-15T01:18:03.373Z [ERROR] server-70e7759b-4410-d190-a867-13e393de32ec: failed to get log: index=3 error="log not found" -2024-05-15T01:18:03.373Z [INFO] server-70e7759b-4410-d190-a867-13e393de32ec: aborting pipeline replication: peer="{Voter server-83f892d7-37c0-8def-1ff6-1f58f211f71b 83f892d7-37c0-8def-1ff6-1f58f211f71b}" -2024-05-15T01:18:03.379Z [ERROR] server-70e7759b-4410-d190-a867-13e393de32ec: failed to get log: index=3 error="log not found" -2024-05-15T01:18:03.379Z [INFO] creating new snapshot: path=/tmp/raft1405748442/snapshots/2-113-1715735883379.tmp -2024-05-15T01:18:03.380Z [INFO] server-17891503-d689-dd37-e6d0-b4a442fce372: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:03.381Z [ERROR] server-70e7759b-4410-d190-a867-13e393de32ec: failed to get log: index=3 error="log not found" -2024-05-15T01:18:03.381Z [INFO] creating new snapshot: path=/tmp/raft1484921444/snapshots/2-113-1715735883381.tmp -2024-05-15T01:18:03.382Z [INFO] server-83f892d7-37c0-8def-1ff6-1f58f211f71b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:03.506Z [INFO] server-17891503-d689-dd37-e6d0-b4a442fce372: copied to local snapshot: bytes=71 -2024-05-15T01:18:03.506Z [INFO] server-83f892d7-37c0-8def-1ff6-1f58f211f71b: copied to local snapshot: bytes=71 -2024-05-15T01:18:03.506Z [INFO] server-17891503-d689-dd37-e6d0-b4a442fce372: snapshot restore progress: id=2-113-1715735883379 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:03.506Z [INFO] server-17891503-d689-dd37-e6d0-b4a442fce372: Installed remote snapshot -2024-05-15T01:18:03.506Z [INFO] server-83f892d7-37c0-8def-1ff6-1f58f211f71b: snapshot restore progress: id=2-113-1715735883381 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:03.506Z [INFO] server-83f892d7-37c0-8def-1ff6-1f58f211f71b: Installed remote snapshot -2024-05-15T01:18:03.506Z [WARN] server-83f892d7-37c0-8def-1ff6-1f58f211f71b: skipping application of old log: index=113 -2024-05-15T01:18:03.506Z [INFO] server-70e7759b-4410-d190-a867-13e393de32ec: pipelining replication: peer="{Voter server-17891503-d689-dd37-e6d0-b4a442fce372 17891503-d689-dd37-e6d0-b4a442fce372}" -2024-05-15T01:18:03.507Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:03.507Z [INFO] server-70e7759b-4410-d190-a867-13e393de32ec: pipelining replication: peer="{Voter server-83f892d7-37c0-8def-1ff6-1f58f211f71b 83f892d7-37c0-8def-1ff6-1f58f211f71b}" -2024-05-15T01:18:04.512Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:03.507122196 +0000 UTC m=+64.048506937 (2 nodes), 2.014µs from start of poll, 1.752784673s from cluster start. Timeout at 2024-05-15 01:18:04.512685559 +0000 UTC m=+65.054070301, 1.005563364s after stability -2024-05-15T01:18:04.525Z [INFO] server-20e918b0-b9bb-5cf7-8922-8c494c5231d6: aborting pipeline replication: peer="{Voter server-d1898434-3740-a5ae-adbb-c2f3258472fe d1898434-3740-a5ae-adbb-c2f3258472fe}" -2024-05-15T01:18:04.525Z [INFO] server-20e918b0-b9bb-5cf7-8922-8c494c5231d6: aborting pipeline replication: peer="{Voter server-e7c38667-f13f-85b0-34c1-445d9d0969a2 e7c38667-f13f-85b0-34c1-445d9d0969a2}" +2025-06-17T07:43:52.390Z [INFO] server-ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e Address:ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e} {Suffrage:Voter ID:server-14c5870d-209f-b7d1-de5a-ef9c66d9eb56 Address:14c5870d-209f-b7d1-de5a-ef9c66d9eb56} {Suffrage:Voter ID:server-04030884-c9c5-d9c2-91be-bcce389b2251 Address:04030884-c9c5-d9c2-91be-bcce389b2251}]" +2025-06-17T07:43:52.391Z [INFO] server-ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e: entering follower state: follower="Node at ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e [Follower]" leader-address= leader-id= +2025-06-17T07:43:52.391Z [INFO] server-14c5870d-209f-b7d1-de5a-ef9c66d9eb56: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e Address:ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e} {Suffrage:Voter ID:server-14c5870d-209f-b7d1-de5a-ef9c66d9eb56 Address:14c5870d-209f-b7d1-de5a-ef9c66d9eb56} {Suffrage:Voter ID:server-04030884-c9c5-d9c2-91be-bcce389b2251 Address:04030884-c9c5-d9c2-91be-bcce389b2251}]" +2025-06-17T07:43:52.391Z [INFO] server-14c5870d-209f-b7d1-de5a-ef9c66d9eb56: entering follower state: follower="Node at 14c5870d-209f-b7d1-de5a-ef9c66d9eb56 [Follower]" leader-address= leader-id= +2025-06-17T07:43:52.391Z [INFO] server-04030884-c9c5-d9c2-91be-bcce389b2251: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e Address:ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e} {Suffrage:Voter ID:server-14c5870d-209f-b7d1-de5a-ef9c66d9eb56 Address:14c5870d-209f-b7d1-de5a-ef9c66d9eb56} {Suffrage:Voter ID:server-04030884-c9c5-d9c2-91be-bcce389b2251 Address:04030884-c9c5-d9c2-91be-bcce389b2251}]" +2025-06-17T07:43:52.391Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:52.391Z [INFO] server-04030884-c9c5-d9c2-91be-bcce389b2251: entering follower state: follower="Node at 04030884-c9c5-d9c2-91be-bcce389b2251 [Follower]" leader-address= leader-id= +2025-06-17T07:43:52.975Z [WARN] server-ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:52.975Z [INFO] server-ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e: entering candidate state: node="Node at ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e [Candidate]" term=2 +2025-06-17T07:43:52.975Z [INFO] server-ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e: election won: term=2 tally=2 +2025-06-17T07:43:52.975Z [INFO] server-ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e: entering leader state: leader="Node at ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e [Leader]" +2025-06-17T07:43:52.975Z [INFO] server-ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e: added peer, starting replication: peer=server-14c5870d-209f-b7d1-de5a-ef9c66d9eb56 +2025-06-17T07:43:52.975Z [INFO] server-ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e: added peer, starting replication: peer=server-04030884-c9c5-d9c2-91be-bcce389b2251 +2025-06-17T07:43:52.975Z [INFO] server-ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e: pipelining replication: peer="{Voter server-14c5870d-209f-b7d1-de5a-ef9c66d9eb56 14c5870d-209f-b7d1-de5a-ef9c66d9eb56}" +2025-06-17T07:43:52.975Z [INFO] server-ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e: pipelining replication: peer="{Voter server-04030884-c9c5-d9c2-91be-bcce389b2251 04030884-c9c5-d9c2-91be-bcce389b2251}" +2025-06-17T07:43:53.981Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:52.975508001 +0000 UTC m=+61.579036694 (1 nodes), 584.338587ms from start of poll, 584.698942ms from cluster start. Timeout at 2025-06-17 07:43:53.980981779 +0000 UTC m=+62.584510482, 1.005473788s after stability +2025-06-17T07:43:53.981Z [INFO] server-ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e: starting snapshot up to: index=12 +2025-06-17T07:43:53.981Z [INFO] creating new snapshot: path=/tmp/raft737019053/snapshots/2-12-1750146233981.tmp +2025-06-17T07:43:53.982Z [INFO] server-ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e: snapshot complete up to: index=12 +2025-06-17T07:43:53.983Z [INFO] server-a7fb4217-3cf5-f243-8806-29e3bb33f3b2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a7fb4217-3cf5-f243-8806-29e3bb33f3b2 Address:a7fb4217-3cf5-f243-8806-29e3bb33f3b2} {Suffrage:Voter ID:server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98 Address:d42993c1-96f4-11c3-3e1b-cfca7caeaf98} {Suffrage:Voter ID:server-674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4 Address:674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4}]" +2025-06-17T07:43:53.983Z [INFO] server-a7fb4217-3cf5-f243-8806-29e3bb33f3b2: entering follower state: follower="Node at a7fb4217-3cf5-f243-8806-29e3bb33f3b2 [Follower]" leader-address= leader-id= +2025-06-17T07:43:53.983Z [INFO] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a7fb4217-3cf5-f243-8806-29e3bb33f3b2 Address:a7fb4217-3cf5-f243-8806-29e3bb33f3b2} {Suffrage:Voter ID:server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98 Address:d42993c1-96f4-11c3-3e1b-cfca7caeaf98} {Suffrage:Voter ID:server-674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4 Address:674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4}]" +2025-06-17T07:43:53.983Z [INFO] server-674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a7fb4217-3cf5-f243-8806-29e3bb33f3b2 Address:a7fb4217-3cf5-f243-8806-29e3bb33f3b2} {Suffrage:Voter ID:server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98 Address:d42993c1-96f4-11c3-3e1b-cfca7caeaf98} {Suffrage:Voter ID:server-674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4 Address:674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4}]" +2025-06-17T07:43:53.983Z [INFO] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: entering follower state: follower="Node at d42993c1-96f4-11c3-3e1b-cfca7caeaf98 [Follower]" leader-address= leader-id= +2025-06-17T07:43:53.983Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:53.983Z [INFO] server-674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4: entering follower state: follower="Node at 674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4 [Follower]" leader-address= leader-id= +2025-06-17T07:43:54.520Z [WARN] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:54.520Z [INFO] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: entering candidate state: node="Node at d42993c1-96f4-11c3-3e1b-cfca7caeaf98 [Candidate]" term=2 +2025-06-17T07:43:54.520Z [INFO] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: election won: term=2 tally=2 +2025-06-17T07:43:54.520Z [INFO] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: entering leader state: leader="Node at d42993c1-96f4-11c3-3e1b-cfca7caeaf98 [Leader]" +2025-06-17T07:43:54.520Z [INFO] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: added peer, starting replication: peer=server-a7fb4217-3cf5-f243-8806-29e3bb33f3b2 +2025-06-17T07:43:54.520Z [INFO] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: added peer, starting replication: peer=server-674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4 +2025-06-17T07:43:54.520Z [INFO] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: pipelining replication: peer="{Voter server-a7fb4217-3cf5-f243-8806-29e3bb33f3b2 a7fb4217-3cf5-f243-8806-29e3bb33f3b2}" +2025-06-17T07:43:54.520Z [INFO] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: pipelining replication: peer="{Voter server-674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4 674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4}" +2025-06-17T07:43:55.525Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:54.520764227 +0000 UTC m=+63.124292938 (1 nodes), 537.290652ms from start of poll, 537.64253ms from cluster start. Timeout at 2025-06-17 07:43:55.525951799 +0000 UTC m=+64.129480511, 1.005187573s after stability +2025-06-17T07:43:55.526Z [INFO] creating new snapshot: path=/tmp/raft3684951248/snapshots/2-113-1750146235526.tmp +2025-06-17T07:43:55.526Z [INFO] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: copied to local snapshot: bytes=71 +2025-06-17T07:43:55.526Z [INFO] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: snapshot restore progress: id=2-113-1750146235526 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:55.526Z [INFO] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: restored user snapshot: index=113 +2025-06-17T07:43:55.526Z [ERROR] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: failed to get log: index=3 error="log not found" +2025-06-17T07:43:55.526Z [INFO] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: aborting pipeline replication: peer="{Voter server-674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4 674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4}" +2025-06-17T07:43:55.526Z [ERROR] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: failed to get log: index=3 error="log not found" +2025-06-17T07:43:55.526Z [INFO] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: aborting pipeline replication: peer="{Voter server-a7fb4217-3cf5-f243-8806-29e3bb33f3b2 a7fb4217-3cf5-f243-8806-29e3bb33f3b2}" +2025-06-17T07:43:55.537Z [ERROR] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: failed to get log: index=3 error="log not found" +2025-06-17T07:43:55.537Z [ERROR] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: failed to get log: index=3 error="log not found" +2025-06-17T07:43:55.537Z [INFO] creating new snapshot: path=/tmp/raft2156405463/snapshots/2-113-1750146235537.tmp +2025-06-17T07:43:55.537Z [INFO] creating new snapshot: path=/tmp/raft3368711093/snapshots/2-113-1750146235537.tmp +2025-06-17T07:43:55.537Z [INFO] server-a7fb4217-3cf5-f243-8806-29e3bb33f3b2: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:55.538Z [INFO] server-674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:55.538Z [INFO] server-a7fb4217-3cf5-f243-8806-29e3bb33f3b2: copied to local snapshot: bytes=71 +2025-06-17T07:43:55.538Z [INFO] server-674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4: copied to local snapshot: bytes=71 +2025-06-17T07:43:55.538Z [INFO] server-a7fb4217-3cf5-f243-8806-29e3bb33f3b2: snapshot restore progress: id=2-113-1750146235537 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:55.538Z [INFO] server-a7fb4217-3cf5-f243-8806-29e3bb33f3b2: Installed remote snapshot +2025-06-17T07:43:55.538Z [INFO] server-674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4: snapshot restore progress: id=2-113-1750146235537 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:55.538Z [INFO] server-674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4: Installed remote snapshot +2025-06-17T07:43:55.538Z [INFO] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: pipelining replication: peer="{Voter server-a7fb4217-3cf5-f243-8806-29e3bb33f3b2 a7fb4217-3cf5-f243-8806-29e3bb33f3b2}" +2025-06-17T07:43:55.538Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:43:55.538Z [WARN] server-674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4: skipping application of old log: index=113 +2025-06-17T07:43:55.538Z [INFO] server-d42993c1-96f4-11c3-3e1b-cfca7caeaf98: pipelining replication: peer="{Voter server-674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4 674a401e-c6f4-20bd-d6c2-57aa1c6fd4e4}" +2025-06-17T07:43:56.544Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:55.538749927 +0000 UTC m=+64.142278634 (2 nodes), 893ns from start of poll, 1.555628226s from cluster start. Timeout at 2025-06-17 07:43:56.544155298 +0000 UTC m=+65.147683979, 1.005405345s after stability +2025-06-17T07:43:56.555Z [INFO] server-ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e: aborting pipeline replication: peer="{Voter server-04030884-c9c5-d9c2-91be-bcce389b2251 04030884-c9c5-d9c2-91be-bcce389b2251}" +2025-06-17T07:43:56.555Z [INFO] server-ff4a7bc3-2ac4-9e46-4361-5d63fa486d2e: aborting pipeline replication: peer="{Voter server-14c5870d-209f-b7d1-de5a-ef9c66d9eb56 14c5870d-209f-b7d1-de5a-ef9c66d9eb56}" === RUN TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true -2024-05-15T01:18:04.527Z [INFO] server-2ba3484b-62c1-51e8-d6d7-95f0eb2ee5d4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2ba3484b-62c1-51e8-d6d7-95f0eb2ee5d4 Address:2ba3484b-62c1-51e8-d6d7-95f0eb2ee5d4} {Suffrage:Voter ID:server-e2914b73-208e-ad32-1f6f-016211883be9 Address:e2914b73-208e-ad32-1f6f-016211883be9} {Suffrage:Voter ID:server-153097e1-a75c-5f7d-ea39-d96929839427 Address:153097e1-a75c-5f7d-ea39-d96929839427}]" -2024-05-15T01:18:04.527Z [INFO] server-e2914b73-208e-ad32-1f6f-016211883be9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2ba3484b-62c1-51e8-d6d7-95f0eb2ee5d4 Address:2ba3484b-62c1-51e8-d6d7-95f0eb2ee5d4} {Suffrage:Voter ID:server-e2914b73-208e-ad32-1f6f-016211883be9 Address:e2914b73-208e-ad32-1f6f-016211883be9} {Suffrage:Voter ID:server-153097e1-a75c-5f7d-ea39-d96929839427 Address:153097e1-a75c-5f7d-ea39-d96929839427}]" -2024-05-15T01:18:04.527Z [INFO] server-153097e1-a75c-5f7d-ea39-d96929839427: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2ba3484b-62c1-51e8-d6d7-95f0eb2ee5d4 Address:2ba3484b-62c1-51e8-d6d7-95f0eb2ee5d4} {Suffrage:Voter ID:server-e2914b73-208e-ad32-1f6f-016211883be9 Address:e2914b73-208e-ad32-1f6f-016211883be9} {Suffrage:Voter ID:server-153097e1-a75c-5f7d-ea39-d96929839427 Address:153097e1-a75c-5f7d-ea39-d96929839427}]" -2024-05-15T01:18:04.528Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:04.528Z [INFO] server-2ba3484b-62c1-51e8-d6d7-95f0eb2ee5d4: entering follower state: follower="Node at 2ba3484b-62c1-51e8-d6d7-95f0eb2ee5d4 [Follower]" leader-address= leader-id= -2024-05-15T01:18:04.528Z [INFO] server-e2914b73-208e-ad32-1f6f-016211883be9: entering follower state: follower="Node at e2914b73-208e-ad32-1f6f-016211883be9 [Follower]" leader-address= leader-id= -2024-05-15T01:18:04.528Z [INFO] server-153097e1-a75c-5f7d-ea39-d96929839427: entering follower state: follower="Node at 153097e1-a75c-5f7d-ea39-d96929839427 [Follower]" leader-address= leader-id= -2024-05-15T01:18:05.109Z [WARN] server-e2914b73-208e-ad32-1f6f-016211883be9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:05.109Z [INFO] server-e2914b73-208e-ad32-1f6f-016211883be9: entering candidate state: node="Node at e2914b73-208e-ad32-1f6f-016211883be9 [Candidate]" term=2 -2024-05-15T01:18:05.109Z [INFO] server-e2914b73-208e-ad32-1f6f-016211883be9: election won: term=2 tally=2 -2024-05-15T01:18:05.109Z [INFO] server-e2914b73-208e-ad32-1f6f-016211883be9: entering leader state: leader="Node at e2914b73-208e-ad32-1f6f-016211883be9 [Leader]" -2024-05-15T01:18:05.109Z [INFO] server-e2914b73-208e-ad32-1f6f-016211883be9: added peer, starting replication: peer=server-2ba3484b-62c1-51e8-d6d7-95f0eb2ee5d4 -2024-05-15T01:18:05.109Z [INFO] server-e2914b73-208e-ad32-1f6f-016211883be9: added peer, starting replication: peer=server-153097e1-a75c-5f7d-ea39-d96929839427 -2024-05-15T01:18:05.110Z [INFO] server-e2914b73-208e-ad32-1f6f-016211883be9: pipelining replication: peer="{Voter server-153097e1-a75c-5f7d-ea39-d96929839427 153097e1-a75c-5f7d-ea39-d96929839427}" -2024-05-15T01:18:05.110Z [INFO] server-e2914b73-208e-ad32-1f6f-016211883be9: pipelining replication: peer="{Voter server-2ba3484b-62c1-51e8-d6d7-95f0eb2ee5d4 2ba3484b-62c1-51e8-d6d7-95f0eb2ee5d4}" -2024-05-15T01:18:06.115Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:05.109929979 +0000 UTC m=+65.651314217 (1 nodes), 581.909406ms from start of poll, 582.460347ms from cluster start. Timeout at 2024-05-15 01:18:06.115861939 +0000 UTC m=+66.657246866, 1.005932649s after stability -2024-05-15T01:18:06.117Z [INFO] server-e2914b73-208e-ad32-1f6f-016211883be9: starting snapshot up to: index=12 -2024-05-15T01:18:06.117Z [INFO] creating new snapshot: path=/tmp/raft2939014776/snapshots/2-12-1715735886117.tmp -2024-05-15T01:18:06.200Z [INFO] server-e2914b73-208e-ad32-1f6f-016211883be9: snapshot complete up to: index=12 -2024-05-15T01:18:06.205Z [INFO] server-734b69bd-2098-25ee-8317-5e0394343c11: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-734b69bd-2098-25ee-8317-5e0394343c11 Address:734b69bd-2098-25ee-8317-5e0394343c11} {Suffrage:Voter ID:server-6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6 Address:6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6} {Suffrage:Voter ID:server-d640c81b-8496-33a3-adf5-97c3d11233f1 Address:d640c81b-8496-33a3-adf5-97c3d11233f1}]" -2024-05-15T01:18:06.205Z [INFO] server-734b69bd-2098-25ee-8317-5e0394343c11: entering follower state: follower="Node at 734b69bd-2098-25ee-8317-5e0394343c11 [Follower]" leader-address= leader-id= -2024-05-15T01:18:06.205Z [INFO] server-6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-734b69bd-2098-25ee-8317-5e0394343c11 Address:734b69bd-2098-25ee-8317-5e0394343c11} {Suffrage:Voter ID:server-6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6 Address:6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6} {Suffrage:Voter ID:server-d640c81b-8496-33a3-adf5-97c3d11233f1 Address:d640c81b-8496-33a3-adf5-97c3d11233f1}]" -2024-05-15T01:18:06.205Z [INFO] server-6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6: entering follower state: follower="Node at 6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6 [Follower]" leader-address= leader-id= -2024-05-15T01:18:06.205Z [INFO] server-d640c81b-8496-33a3-adf5-97c3d11233f1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-734b69bd-2098-25ee-8317-5e0394343c11 Address:734b69bd-2098-25ee-8317-5e0394343c11} {Suffrage:Voter ID:server-6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6 Address:6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6} {Suffrage:Voter ID:server-d640c81b-8496-33a3-adf5-97c3d11233f1 Address:d640c81b-8496-33a3-adf5-97c3d11233f1}]" -2024-05-15T01:18:06.205Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:06.205Z [INFO] server-d640c81b-8496-33a3-adf5-97c3d11233f1: entering follower state: follower="Node at d640c81b-8496-33a3-adf5-97c3d11233f1 [Follower]" leader-address= leader-id= -2024-05-15T01:18:06.720Z [WARN] server-d640c81b-8496-33a3-adf5-97c3d11233f1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:06.720Z [INFO] server-d640c81b-8496-33a3-adf5-97c3d11233f1: entering candidate state: node="Node at d640c81b-8496-33a3-adf5-97c3d11233f1 [Candidate]" term=2 -2024-05-15T01:18:06.720Z [INFO] server-d640c81b-8496-33a3-adf5-97c3d11233f1: election won: term=2 tally=2 -2024-05-15T01:18:06.720Z [INFO] server-d640c81b-8496-33a3-adf5-97c3d11233f1: entering leader state: leader="Node at d640c81b-8496-33a3-adf5-97c3d11233f1 [Leader]" -2024-05-15T01:18:06.720Z [INFO] server-d640c81b-8496-33a3-adf5-97c3d11233f1: added peer, starting replication: peer=server-734b69bd-2098-25ee-8317-5e0394343c11 -2024-05-15T01:18:06.720Z [INFO] server-d640c81b-8496-33a3-adf5-97c3d11233f1: added peer, starting replication: peer=server-6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6 -2024-05-15T01:18:06.721Z [INFO] server-d640c81b-8496-33a3-adf5-97c3d11233f1: pipelining replication: peer="{Voter server-6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6 6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6}" -2024-05-15T01:18:06.721Z [INFO] server-d640c81b-8496-33a3-adf5-97c3d11233f1: pipelining replication: peer="{Voter server-734b69bd-2098-25ee-8317-5e0394343c11 734b69bd-2098-25ee-8317-5e0394343c11}" -2024-05-15T01:18:06.721Z [WARN] server-734b69bd-2098-25ee-8317-5e0394343c11: rejecting vote request since our last term is greater: candidate=d640c81b-8496-33a3-adf5-97c3d11233f1 last-term=2 last-candidate-term=1 -2024-05-15T01:18:07.727Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:06.721785999 +0000 UTC m=+67.263170777 (1 nodes), 515.808665ms from start of poll, 516.546665ms from cluster start. Timeout at 2024-05-15 01:18:07.727218196 +0000 UTC m=+68.268603024, 1.005432247s after stability -2024-05-15T01:18:07.727Z [INFO] creating new snapshot: path=/tmp/raft2943871739/snapshots/2-113-1715735887727.tmp -2024-05-15T01:18:07.728Z [INFO] server-d640c81b-8496-33a3-adf5-97c3d11233f1: copied to local snapshot: bytes=71 -2024-05-15T01:18:07.734Z [INFO] server-d640c81b-8496-33a3-adf5-97c3d11233f1: snapshot restore progress: id=2-113-1715735887727 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:07.734Z [INFO] server-d640c81b-8496-33a3-adf5-97c3d11233f1: removing all old logs from log store -2024-05-15T01:18:07.734Z [INFO] server-d640c81b-8496-33a3-adf5-97c3d11233f1: compacting logs: from=1 to=2 -2024-05-15T01:18:07.734Z [INFO] server-d640c81b-8496-33a3-adf5-97c3d11233f1: restored user snapshot: index=113 -2024-05-15T01:18:07.734Z [ERROR] server-d640c81b-8496-33a3-adf5-97c3d11233f1: failed to get log: index=2 error="log not found" -2024-05-15T01:18:07.734Z [INFO] server-d640c81b-8496-33a3-adf5-97c3d11233f1: aborting pipeline replication: peer="{Voter server-6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6 6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6}" -2024-05-15T01:18:07.734Z [ERROR] server-d640c81b-8496-33a3-adf5-97c3d11233f1: failed to get log: index=2 error="log not found" -2024-05-15T01:18:07.734Z [INFO] server-d640c81b-8496-33a3-adf5-97c3d11233f1: aborting pipeline replication: peer="{Voter server-734b69bd-2098-25ee-8317-5e0394343c11 734b69bd-2098-25ee-8317-5e0394343c11}" -2024-05-15T01:18:07.734Z [ERROR] server-d640c81b-8496-33a3-adf5-97c3d11233f1: failed to get log: index=2 error="log not found" -2024-05-15T01:18:07.735Z [INFO] creating new snapshot: path=/tmp/raft3713847592/snapshots/2-113-1715735887735.tmp -2024-05-15T01:18:07.735Z [INFO] server-734b69bd-2098-25ee-8317-5e0394343c11: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:07.736Z [INFO] server-734b69bd-2098-25ee-8317-5e0394343c11: copied to local snapshot: bytes=71 -2024-05-15T01:18:07.736Z [INFO] server-734b69bd-2098-25ee-8317-5e0394343c11: snapshot restore progress: id=2-113-1715735887735 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:07.736Z [INFO] server-734b69bd-2098-25ee-8317-5e0394343c11: removing all old logs from log store -2024-05-15T01:18:07.736Z [INFO] server-734b69bd-2098-25ee-8317-5e0394343c11: compacting logs: from=1 to=2 -2024-05-15T01:18:07.736Z [INFO] server-734b69bd-2098-25ee-8317-5e0394343c11: Installed remote snapshot -2024-05-15T01:18:07.736Z [INFO] server-d640c81b-8496-33a3-adf5-97c3d11233f1: pipelining replication: peer="{Voter server-734b69bd-2098-25ee-8317-5e0394343c11 734b69bd-2098-25ee-8317-5e0394343c11}" -2024-05-15T01:18:07.736Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:07.739Z [ERROR] server-d640c81b-8496-33a3-adf5-97c3d11233f1: failed to get log: index=2 error="log not found" -2024-05-15T01:18:07.740Z [INFO] creating new snapshot: path=/tmp/raft2024188475/snapshots/2-113-1715735887740.tmp -2024-05-15T01:18:07.740Z [INFO] server-6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:07.741Z [INFO] server-6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6: copied to local snapshot: bytes=71 -2024-05-15T01:18:07.741Z [INFO] server-6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6: snapshot restore progress: id=2-113-1715735887740 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:07.741Z [INFO] server-6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6: removing all old logs from log store -2024-05-15T01:18:07.741Z [INFO] server-6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6: compacting logs: from=1 to=2 -2024-05-15T01:18:07.741Z [INFO] server-6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6: Installed remote snapshot -2024-05-15T01:18:07.742Z [INFO] server-d640c81b-8496-33a3-adf5-97c3d11233f1: pipelining replication: peer="{Voter server-6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6 6062c89e-9d5e-9c0a-b1fc-e5379c4fa7d6}" -2024-05-15T01:18:08.742Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:07.736968024 +0000 UTC m=+68.278352748 (2 nodes), 1.78µs from start of poll, 1.531728636s from cluster start. Timeout at 2024-05-15 01:18:08.742404336 +0000 UTC m=+69.283789077, 1.005436329s after stability -2024-05-15T01:18:08.754Z [INFO] server-e2914b73-208e-ad32-1f6f-016211883be9: aborting pipeline replication: peer="{Voter server-153097e1-a75c-5f7d-ea39-d96929839427 153097e1-a75c-5f7d-ea39-d96929839427}" -2024-05-15T01:18:08.754Z [INFO] server-e2914b73-208e-ad32-1f6f-016211883be9: aborting pipeline replication: peer="{Voter server-2ba3484b-62c1-51e8-d6d7-95f0eb2ee5d4 2ba3484b-62c1-51e8-d6d7-95f0eb2ee5d4}" +2025-06-17T07:43:56.556Z [INFO] server-3647a439-572d-54ce-7c20-cc9c0f4241d2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3647a439-572d-54ce-7c20-cc9c0f4241d2 Address:3647a439-572d-54ce-7c20-cc9c0f4241d2} {Suffrage:Voter ID:server-74dd7b93-c14b-9501-a5e4-19629cd4a179 Address:74dd7b93-c14b-9501-a5e4-19629cd4a179} {Suffrage:Voter ID:server-94c9ed9b-a1ca-823b-43e2-0074182f6b93 Address:94c9ed9b-a1ca-823b-43e2-0074182f6b93}]" +2025-06-17T07:43:56.556Z [INFO] server-3647a439-572d-54ce-7c20-cc9c0f4241d2: entering follower state: follower="Node at 3647a439-572d-54ce-7c20-cc9c0f4241d2 [Follower]" leader-address= leader-id= +2025-06-17T07:43:56.556Z [INFO] server-74dd7b93-c14b-9501-a5e4-19629cd4a179: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3647a439-572d-54ce-7c20-cc9c0f4241d2 Address:3647a439-572d-54ce-7c20-cc9c0f4241d2} {Suffrage:Voter ID:server-74dd7b93-c14b-9501-a5e4-19629cd4a179 Address:74dd7b93-c14b-9501-a5e4-19629cd4a179} {Suffrage:Voter ID:server-94c9ed9b-a1ca-823b-43e2-0074182f6b93 Address:94c9ed9b-a1ca-823b-43e2-0074182f6b93}]" +2025-06-17T07:43:56.556Z [INFO] server-74dd7b93-c14b-9501-a5e4-19629cd4a179: entering follower state: follower="Node at 74dd7b93-c14b-9501-a5e4-19629cd4a179 [Follower]" leader-address= leader-id= +2025-06-17T07:43:56.556Z [INFO] server-94c9ed9b-a1ca-823b-43e2-0074182f6b93: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3647a439-572d-54ce-7c20-cc9c0f4241d2 Address:3647a439-572d-54ce-7c20-cc9c0f4241d2} {Suffrage:Voter ID:server-74dd7b93-c14b-9501-a5e4-19629cd4a179 Address:74dd7b93-c14b-9501-a5e4-19629cd4a179} {Suffrage:Voter ID:server-94c9ed9b-a1ca-823b-43e2-0074182f6b93 Address:94c9ed9b-a1ca-823b-43e2-0074182f6b93}]" +2025-06-17T07:43:56.556Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:56.556Z [INFO] server-94c9ed9b-a1ca-823b-43e2-0074182f6b93: entering follower state: follower="Node at 94c9ed9b-a1ca-823b-43e2-0074182f6b93 [Follower]" leader-address= leader-id= +2025-06-17T07:43:57.135Z [WARN] server-74dd7b93-c14b-9501-a5e4-19629cd4a179: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:57.135Z [INFO] server-74dd7b93-c14b-9501-a5e4-19629cd4a179: entering candidate state: node="Node at 74dd7b93-c14b-9501-a5e4-19629cd4a179 [Candidate]" term=2 +2025-06-17T07:43:57.135Z [INFO] server-74dd7b93-c14b-9501-a5e4-19629cd4a179: election won: term=2 tally=2 +2025-06-17T07:43:57.135Z [INFO] server-74dd7b93-c14b-9501-a5e4-19629cd4a179: entering leader state: leader="Node at 74dd7b93-c14b-9501-a5e4-19629cd4a179 [Leader]" +2025-06-17T07:43:57.135Z [INFO] server-74dd7b93-c14b-9501-a5e4-19629cd4a179: added peer, starting replication: peer=server-3647a439-572d-54ce-7c20-cc9c0f4241d2 +2025-06-17T07:43:57.135Z [INFO] server-74dd7b93-c14b-9501-a5e4-19629cd4a179: added peer, starting replication: peer=server-94c9ed9b-a1ca-823b-43e2-0074182f6b93 +2025-06-17T07:43:57.135Z [INFO] server-74dd7b93-c14b-9501-a5e4-19629cd4a179: pipelining replication: peer="{Voter server-94c9ed9b-a1ca-823b-43e2-0074182f6b93 94c9ed9b-a1ca-823b-43e2-0074182f6b93}" +2025-06-17T07:43:57.135Z [INFO] server-74dd7b93-c14b-9501-a5e4-19629cd4a179: pipelining replication: peer="{Voter server-3647a439-572d-54ce-7c20-cc9c0f4241d2 3647a439-572d-54ce-7c20-cc9c0f4241d2}" +2025-06-17T07:43:58.141Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:57.135405705 +0000 UTC m=+65.738934408 (1 nodes), 578.91588ms from start of poll, 579.208188ms from cluster start. Timeout at 2025-06-17 07:43:58.141014683 +0000 UTC m=+66.744543363, 1.005608955s after stability +2025-06-17T07:43:58.141Z [INFO] server-74dd7b93-c14b-9501-a5e4-19629cd4a179: starting snapshot up to: index=12 +2025-06-17T07:43:58.141Z [INFO] creating new snapshot: path=/tmp/raft1897519598/snapshots/2-12-1750146238141.tmp +2025-06-17T07:43:58.167Z [INFO] server-74dd7b93-c14b-9501-a5e4-19629cd4a179: snapshot complete up to: index=12 +2025-06-17T07:43:58.168Z [INFO] server-795f8545-d805-69b7-40a8-0b8ea9050856: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-795f8545-d805-69b7-40a8-0b8ea9050856 Address:795f8545-d805-69b7-40a8-0b8ea9050856} {Suffrage:Voter ID:server-e18dd508-bdad-60e1-868a-56e1cff97987 Address:e18dd508-bdad-60e1-868a-56e1cff97987} {Suffrage:Voter ID:server-52a35aba-b6d4-6a46-f402-716c7a42611a Address:52a35aba-b6d4-6a46-f402-716c7a42611a}]" +2025-06-17T07:43:58.168Z [INFO] server-795f8545-d805-69b7-40a8-0b8ea9050856: entering follower state: follower="Node at 795f8545-d805-69b7-40a8-0b8ea9050856 [Follower]" leader-address= leader-id= +2025-06-17T07:43:58.168Z [INFO] server-e18dd508-bdad-60e1-868a-56e1cff97987: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-795f8545-d805-69b7-40a8-0b8ea9050856 Address:795f8545-d805-69b7-40a8-0b8ea9050856} {Suffrage:Voter ID:server-e18dd508-bdad-60e1-868a-56e1cff97987 Address:e18dd508-bdad-60e1-868a-56e1cff97987} {Suffrage:Voter ID:server-52a35aba-b6d4-6a46-f402-716c7a42611a Address:52a35aba-b6d4-6a46-f402-716c7a42611a}]" +2025-06-17T07:43:58.168Z [INFO] server-e18dd508-bdad-60e1-868a-56e1cff97987: entering follower state: follower="Node at e18dd508-bdad-60e1-868a-56e1cff97987 [Follower]" leader-address= leader-id= +2025-06-17T07:43:58.168Z [INFO] server-52a35aba-b6d4-6a46-f402-716c7a42611a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-795f8545-d805-69b7-40a8-0b8ea9050856 Address:795f8545-d805-69b7-40a8-0b8ea9050856} {Suffrage:Voter ID:server-e18dd508-bdad-60e1-868a-56e1cff97987 Address:e18dd508-bdad-60e1-868a-56e1cff97987} {Suffrage:Voter ID:server-52a35aba-b6d4-6a46-f402-716c7a42611a Address:52a35aba-b6d4-6a46-f402-716c7a42611a}]" +2025-06-17T07:43:58.168Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:43:58.168Z [INFO] server-52a35aba-b6d4-6a46-f402-716c7a42611a: entering follower state: follower="Node at 52a35aba-b6d4-6a46-f402-716c7a42611a [Follower]" leader-address= leader-id= +2025-06-17T07:43:58.875Z [WARN] server-795f8545-d805-69b7-40a8-0b8ea9050856: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:43:58.876Z [INFO] server-795f8545-d805-69b7-40a8-0b8ea9050856: entering candidate state: node="Node at 795f8545-d805-69b7-40a8-0b8ea9050856 [Candidate]" term=2 +2025-06-17T07:43:58.876Z [INFO] server-795f8545-d805-69b7-40a8-0b8ea9050856: election won: term=2 tally=2 +2025-06-17T07:43:58.876Z [INFO] server-795f8545-d805-69b7-40a8-0b8ea9050856: entering leader state: leader="Node at 795f8545-d805-69b7-40a8-0b8ea9050856 [Leader]" +2025-06-17T07:43:58.876Z [INFO] server-795f8545-d805-69b7-40a8-0b8ea9050856: added peer, starting replication: peer=server-e18dd508-bdad-60e1-868a-56e1cff97987 +2025-06-17T07:43:58.877Z [INFO] server-795f8545-d805-69b7-40a8-0b8ea9050856: added peer, starting replication: peer=server-52a35aba-b6d4-6a46-f402-716c7a42611a +2025-06-17T07:43:58.877Z [INFO] server-795f8545-d805-69b7-40a8-0b8ea9050856: pipelining replication: peer="{Voter server-e18dd508-bdad-60e1-868a-56e1cff97987 e18dd508-bdad-60e1-868a-56e1cff97987}" +2025-06-17T07:43:58.877Z [INFO] server-795f8545-d805-69b7-40a8-0b8ea9050856: pipelining replication: peer="{Voter server-52a35aba-b6d4-6a46-f402-716c7a42611a 52a35aba-b6d4-6a46-f402-716c7a42611a}" +2025-06-17T07:43:59.882Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:43:58.877107013 +0000 UTC m=+67.480635713 (1 nodes), 708.418141ms from start of poll, 708.832369ms from cluster start. Timeout at 2025-06-17 07:43:59.88263745 +0000 UTC m=+68.486166167, 1.005530454s after stability +2025-06-17T07:43:59.882Z [INFO] creating new snapshot: path=/tmp/raft933553308/snapshots/2-113-1750146239882.tmp +2025-06-17T07:43:59.883Z [INFO] server-795f8545-d805-69b7-40a8-0b8ea9050856: copied to local snapshot: bytes=71 +2025-06-17T07:43:59.883Z [INFO] server-795f8545-d805-69b7-40a8-0b8ea9050856: snapshot restore progress: id=2-113-1750146239882 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:59.883Z [INFO] server-795f8545-d805-69b7-40a8-0b8ea9050856: removing all old logs from log store +2025-06-17T07:43:59.883Z [INFO] server-795f8545-d805-69b7-40a8-0b8ea9050856: compacting logs: from=1 to=2 +2025-06-17T07:43:59.883Z [INFO] server-795f8545-d805-69b7-40a8-0b8ea9050856: restored user snapshot: index=113 +2025-06-17T07:43:59.883Z [ERROR] server-795f8545-d805-69b7-40a8-0b8ea9050856: failed to get log: index=2 error="log not found" +2025-06-17T07:43:59.883Z [ERROR] server-795f8545-d805-69b7-40a8-0b8ea9050856: failed to get log: index=2 error="log not found" +2025-06-17T07:43:59.883Z [INFO] server-795f8545-d805-69b7-40a8-0b8ea9050856: aborting pipeline replication: peer="{Voter server-52a35aba-b6d4-6a46-f402-716c7a42611a 52a35aba-b6d4-6a46-f402-716c7a42611a}" +2025-06-17T07:43:59.883Z [INFO] server-795f8545-d805-69b7-40a8-0b8ea9050856: aborting pipeline replication: peer="{Voter server-e18dd508-bdad-60e1-868a-56e1cff97987 e18dd508-bdad-60e1-868a-56e1cff97987}" +2025-06-17T07:43:59.893Z [ERROR] server-795f8545-d805-69b7-40a8-0b8ea9050856: failed to get log: index=2 error="log not found" +2025-06-17T07:43:59.893Z [ERROR] server-795f8545-d805-69b7-40a8-0b8ea9050856: failed to get log: index=2 error="log not found" +2025-06-17T07:43:59.893Z [INFO] creating new snapshot: path=/tmp/raft3200608380/snapshots/2-113-1750146239893.tmp +2025-06-17T07:43:59.893Z [INFO] creating new snapshot: path=/tmp/raft542879330/snapshots/2-113-1750146239893.tmp +2025-06-17T07:43:59.893Z [INFO] server-e18dd508-bdad-60e1-868a-56e1cff97987: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:59.893Z [INFO] server-52a35aba-b6d4-6a46-f402-716c7a42611a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:59.894Z [INFO] server-e18dd508-bdad-60e1-868a-56e1cff97987: copied to local snapshot: bytes=71 +2025-06-17T07:43:59.894Z [INFO] server-52a35aba-b6d4-6a46-f402-716c7a42611a: copied to local snapshot: bytes=71 +2025-06-17T07:43:59.894Z [INFO] server-52a35aba-b6d4-6a46-f402-716c7a42611a: snapshot restore progress: id=2-113-1750146239893 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:59.894Z [INFO] server-52a35aba-b6d4-6a46-f402-716c7a42611a: removing all old logs from log store +2025-06-17T07:43:59.894Z [INFO] server-52a35aba-b6d4-6a46-f402-716c7a42611a: compacting logs: from=1 to=2 +2025-06-17T07:43:59.894Z [INFO] server-e18dd508-bdad-60e1-868a-56e1cff97987: snapshot restore progress: id=2-113-1750146239893 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:43:59.894Z [INFO] server-52a35aba-b6d4-6a46-f402-716c7a42611a: Installed remote snapshot +2025-06-17T07:43:59.894Z [INFO] server-e18dd508-bdad-60e1-868a-56e1cff97987: removing all old logs from log store +2025-06-17T07:43:59.894Z [INFO] server-e18dd508-bdad-60e1-868a-56e1cff97987: compacting logs: from=1 to=2 +2025-06-17T07:43:59.894Z [INFO] server-e18dd508-bdad-60e1-868a-56e1cff97987: Installed remote snapshot +2025-06-17T07:43:59.894Z [INFO] server-795f8545-d805-69b7-40a8-0b8ea9050856: pipelining replication: peer="{Voter server-52a35aba-b6d4-6a46-f402-716c7a42611a 52a35aba-b6d4-6a46-f402-716c7a42611a}" +2025-06-17T07:43:59.894Z [INFO] server-795f8545-d805-69b7-40a8-0b8ea9050856: pipelining replication: peer="{Voter server-e18dd508-bdad-60e1-868a-56e1cff97987 e18dd508-bdad-60e1-868a-56e1cff97987}" +2025-06-17T07:43:59.894Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:00.900Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:43:59.894553524 +0000 UTC m=+68.498082230 (2 nodes), 900ns from start of poll, 1.726278886s from cluster start. Timeout at 2025-06-17 07:44:00.900128167 +0000 UTC m=+69.503656847, 1.005574617s after stability +2025-06-17T07:44:00.911Z [INFO] server-74dd7b93-c14b-9501-a5e4-19629cd4a179: aborting pipeline replication: peer="{Voter server-94c9ed9b-a1ca-823b-43e2-0074182f6b93 94c9ed9b-a1ca-823b-43e2-0074182f6b93}" +2025-06-17T07:44:00.911Z [INFO] server-74dd7b93-c14b-9501-a5e4-19629cd4a179: aborting pipeline replication: peer="{Voter server-3647a439-572d-54ce-7c20-cc9c0f4241d2 3647a439-572d-54ce-7c20-cc9c0f4241d2}" === RUN TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false -2024-05-15T01:18:08.756Z [INFO] server-4862a0c2-87a4-a214-bd7c-e36caee4a969: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4862a0c2-87a4-a214-bd7c-e36caee4a969 Address:4862a0c2-87a4-a214-bd7c-e36caee4a969} {Suffrage:Voter ID:server-8d872e7b-3d73-1d60-ec20-7299fd9d2ab8 Address:8d872e7b-3d73-1d60-ec20-7299fd9d2ab8} {Suffrage:Voter ID:server-3887ada9-4375-41d4-ac4b-a4337427e245 Address:3887ada9-4375-41d4-ac4b-a4337427e245}]" -2024-05-15T01:18:08.756Z [INFO] server-8d872e7b-3d73-1d60-ec20-7299fd9d2ab8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4862a0c2-87a4-a214-bd7c-e36caee4a969 Address:4862a0c2-87a4-a214-bd7c-e36caee4a969} {Suffrage:Voter ID:server-8d872e7b-3d73-1d60-ec20-7299fd9d2ab8 Address:8d872e7b-3d73-1d60-ec20-7299fd9d2ab8} {Suffrage:Voter ID:server-3887ada9-4375-41d4-ac4b-a4337427e245 Address:3887ada9-4375-41d4-ac4b-a4337427e245}]" -2024-05-15T01:18:08.757Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4862a0c2-87a4-a214-bd7c-e36caee4a969 Address:4862a0c2-87a4-a214-bd7c-e36caee4a969} {Suffrage:Voter ID:server-8d872e7b-3d73-1d60-ec20-7299fd9d2ab8 Address:8d872e7b-3d73-1d60-ec20-7299fd9d2ab8} {Suffrage:Voter ID:server-3887ada9-4375-41d4-ac4b-a4337427e245 Address:3887ada9-4375-41d4-ac4b-a4337427e245}]" -2024-05-15T01:18:08.757Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:08.757Z [INFO] server-4862a0c2-87a4-a214-bd7c-e36caee4a969: entering follower state: follower="Node at 4862a0c2-87a4-a214-bd7c-e36caee4a969 [Follower]" leader-address= leader-id= -2024-05-15T01:18:08.757Z [INFO] server-8d872e7b-3d73-1d60-ec20-7299fd9d2ab8: entering follower state: follower="Node at 8d872e7b-3d73-1d60-ec20-7299fd9d2ab8 [Follower]" leader-address= leader-id= -2024-05-15T01:18:08.757Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: entering follower state: follower="Node at 3887ada9-4375-41d4-ac4b-a4337427e245 [Follower]" leader-address= leader-id= -2024-05-15T01:18:09.419Z [WARN] server-3887ada9-4375-41d4-ac4b-a4337427e245: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:09.419Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: entering candidate state: node="Node at 3887ada9-4375-41d4-ac4b-a4337427e245 [Candidate]" term=2 -2024-05-15T01:18:09.419Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: election won: term=2 tally=2 -2024-05-15T01:18:09.419Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: entering leader state: leader="Node at 3887ada9-4375-41d4-ac4b-a4337427e245 [Leader]" -2024-05-15T01:18:09.419Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: added peer, starting replication: peer=server-4862a0c2-87a4-a214-bd7c-e36caee4a969 -2024-05-15T01:18:09.419Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: added peer, starting replication: peer=server-8d872e7b-3d73-1d60-ec20-7299fd9d2ab8 -2024-05-15T01:18:09.419Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: pipelining replication: peer="{Voter server-8d872e7b-3d73-1d60-ec20-7299fd9d2ab8 8d872e7b-3d73-1d60-ec20-7299fd9d2ab8}" -2024-05-15T01:18:09.420Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: pipelining replication: peer="{Voter server-4862a0c2-87a4-a214-bd7c-e36caee4a969 4862a0c2-87a4-a214-bd7c-e36caee4a969}" -2024-05-15T01:18:10.425Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:09.420309655 +0000 UTC m=+69.961694422 (1 nodes), 663.113841ms from start of poll, 663.649434ms from cluster start. Timeout at 2024-05-15 01:18:10.425896024 +0000 UTC m=+70.967282008, 1.005587586s after stability -2024-05-15T01:18:10.427Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: starting snapshot up to: index=12 -2024-05-15T01:18:10.427Z [INFO] creating new snapshot: path=/tmp/raft2493460995/snapshots/2-12-1715735890427.tmp -2024-05-15T01:18:10.428Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: snapshot complete up to: index=12 -2024-05-15T01:18:10.430Z [INFO] creating new snapshot: path=/tmp/raft2493460995/snapshots/2-1013-1715735890430.tmp -2024-05-15T01:18:10.431Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: copied to local snapshot: bytes=71 -2024-05-15T01:18:10.431Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: snapshot restore progress: id=2-1013-1715735890430 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:10.431Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: restored user snapshot: index=1013 -2024-05-15T01:18:10.432Z [ERROR] server-3887ada9-4375-41d4-ac4b-a4337427e245: failed to get log: index=23 error="log not found" -2024-05-15T01:18:10.432Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: aborting pipeline replication: peer="{Voter server-8d872e7b-3d73-1d60-ec20-7299fd9d2ab8 8d872e7b-3d73-1d60-ec20-7299fd9d2ab8}" -2024-05-15T01:18:10.432Z [ERROR] server-3887ada9-4375-41d4-ac4b-a4337427e245: failed to get log: index=23 error="log not found" -2024-05-15T01:18:10.432Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: aborting pipeline replication: peer="{Voter server-4862a0c2-87a4-a214-bd7c-e36caee4a969 4862a0c2-87a4-a214-bd7c-e36caee4a969}" -2024-05-15T01:18:10.439Z [ERROR] server-3887ada9-4375-41d4-ac4b-a4337427e245: failed to get log: index=23 error="log not found" -2024-05-15T01:18:10.440Z [INFO] creating new snapshot: path=/tmp/raft2799332080/snapshots/2-1013-1715735890440.tmp -2024-05-15T01:18:10.440Z [INFO] server-4862a0c2-87a4-a214-bd7c-e36caee4a969: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:10.440Z [INFO] server-4862a0c2-87a4-a214-bd7c-e36caee4a969: copied to local snapshot: bytes=71 -2024-05-15T01:18:10.441Z [INFO] server-4862a0c2-87a4-a214-bd7c-e36caee4a969: snapshot restore progress: id=2-1013-1715735890440 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:10.441Z [INFO] server-4862a0c2-87a4-a214-bd7c-e36caee4a969: Installed remote snapshot -2024-05-15T01:18:10.441Z [WARN] server-4862a0c2-87a4-a214-bd7c-e36caee4a969: skipping application of old log: index=22 -2024-05-15T01:18:10.441Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: pipelining replication: peer="{Voter server-4862a0c2-87a4-a214-bd7c-e36caee4a969 4862a0c2-87a4-a214-bd7c-e36caee4a969}" -2024-05-15T01:18:10.441Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:10.441Z [ERROR] server-3887ada9-4375-41d4-ac4b-a4337427e245: failed to get log: index=23 error="log not found" -2024-05-15T01:18:10.448Z [INFO] creating new snapshot: path=/tmp/raft3056439134/snapshots/2-1013-1715735890448.tmp -2024-05-15T01:18:10.448Z [INFO] server-8d872e7b-3d73-1d60-ec20-7299fd9d2ab8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:10.449Z [INFO] server-8d872e7b-3d73-1d60-ec20-7299fd9d2ab8: copied to local snapshot: bytes=71 -2024-05-15T01:18:10.451Z [INFO] server-8d872e7b-3d73-1d60-ec20-7299fd9d2ab8: snapshot restore progress: id=2-1013-1715735890448 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:10.451Z [INFO] server-8d872e7b-3d73-1d60-ec20-7299fd9d2ab8: Installed remote snapshot -2024-05-15T01:18:10.451Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: pipelining replication: peer="{Voter server-8d872e7b-3d73-1d60-ec20-7299fd9d2ab8 8d872e7b-3d73-1d60-ec20-7299fd9d2ab8}" -2024-05-15T01:18:11.447Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:10.441561157 +0000 UTC m=+70.982945907 (2 nodes), 1.935µs from start of poll, 1.684900919s from cluster start. Timeout at 2024-05-15 01:18:11.447356096 +0000 UTC m=+71.988740842, 1.005794935s after stability -2024-05-15T01:18:11.460Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: aborting pipeline replication: peer="{Voter server-4862a0c2-87a4-a214-bd7c-e36caee4a969 4862a0c2-87a4-a214-bd7c-e36caee4a969}" -2024-05-15T01:18:11.460Z [INFO] server-3887ada9-4375-41d4-ac4b-a4337427e245: aborting pipeline replication: peer="{Voter server-8d872e7b-3d73-1d60-ec20-7299fd9d2ab8 8d872e7b-3d73-1d60-ec20-7299fd9d2ab8}" +2025-06-17T07:44:00.912Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f02409a4-3f2c-78aa-efc9-47d288b11812 Address:f02409a4-3f2c-78aa-efc9-47d288b11812} {Suffrage:Voter ID:server-c8843568-32df-bb3c-241b-d415f15cd8e7 Address:c8843568-32df-bb3c-241b-d415f15cd8e7} {Suffrage:Voter ID:server-9f3de548-9e76-0fd0-f094-5a0217c6cbe8 Address:9f3de548-9e76-0fd0-f094-5a0217c6cbe8}]" +2025-06-17T07:44:00.913Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: entering follower state: follower="Node at f02409a4-3f2c-78aa-efc9-47d288b11812 [Follower]" leader-address= leader-id= +2025-06-17T07:44:00.913Z [INFO] server-c8843568-32df-bb3c-241b-d415f15cd8e7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f02409a4-3f2c-78aa-efc9-47d288b11812 Address:f02409a4-3f2c-78aa-efc9-47d288b11812} {Suffrage:Voter ID:server-c8843568-32df-bb3c-241b-d415f15cd8e7 Address:c8843568-32df-bb3c-241b-d415f15cd8e7} {Suffrage:Voter ID:server-9f3de548-9e76-0fd0-f094-5a0217c6cbe8 Address:9f3de548-9e76-0fd0-f094-5a0217c6cbe8}]" +2025-06-17T07:44:00.913Z [INFO] server-c8843568-32df-bb3c-241b-d415f15cd8e7: entering follower state: follower="Node at c8843568-32df-bb3c-241b-d415f15cd8e7 [Follower]" leader-address= leader-id= +2025-06-17T07:44:00.913Z [INFO] server-9f3de548-9e76-0fd0-f094-5a0217c6cbe8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f02409a4-3f2c-78aa-efc9-47d288b11812 Address:f02409a4-3f2c-78aa-efc9-47d288b11812} {Suffrage:Voter ID:server-c8843568-32df-bb3c-241b-d415f15cd8e7 Address:c8843568-32df-bb3c-241b-d415f15cd8e7} {Suffrage:Voter ID:server-9f3de548-9e76-0fd0-f094-5a0217c6cbe8 Address:9f3de548-9e76-0fd0-f094-5a0217c6cbe8}]" +2025-06-17T07:44:00.913Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:00.913Z [INFO] server-9f3de548-9e76-0fd0-f094-5a0217c6cbe8: entering follower state: follower="Node at 9f3de548-9e76-0fd0-f094-5a0217c6cbe8 [Follower]" leader-address= leader-id= +2025-06-17T07:44:01.688Z [WARN] server-f02409a4-3f2c-78aa-efc9-47d288b11812: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:01.688Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: entering candidate state: node="Node at f02409a4-3f2c-78aa-efc9-47d288b11812 [Candidate]" term=2 +2025-06-17T07:44:01.689Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: election won: term=2 tally=2 +2025-06-17T07:44:01.689Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: entering leader state: leader="Node at f02409a4-3f2c-78aa-efc9-47d288b11812 [Leader]" +2025-06-17T07:44:01.689Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: added peer, starting replication: peer=server-c8843568-32df-bb3c-241b-d415f15cd8e7 +2025-06-17T07:44:01.689Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: added peer, starting replication: peer=server-9f3de548-9e76-0fd0-f094-5a0217c6cbe8 +2025-06-17T07:44:01.689Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: pipelining replication: peer="{Voter server-9f3de548-9e76-0fd0-f094-5a0217c6cbe8 9f3de548-9e76-0fd0-f094-5a0217c6cbe8}" +2025-06-17T07:44:01.689Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: pipelining replication: peer="{Voter server-c8843568-32df-bb3c-241b-d415f15cd8e7 c8843568-32df-bb3c-241b-d415f15cd8e7}" +2025-06-17T07:44:02.694Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:01.689117354 +0000 UTC m=+70.292646065 (1 nodes), 775.794495ms from start of poll, 776.213917ms from cluster start. Timeout at 2025-06-17 07:44:02.694226862 +0000 UTC m=+71.297755580, 1.005109515s after stability +2025-06-17T07:44:02.694Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: starting snapshot up to: index=12 +2025-06-17T07:44:02.694Z [INFO] creating new snapshot: path=/tmp/raft3395250291/snapshots/2-12-1750146242694.tmp +2025-06-17T07:44:02.709Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: snapshot complete up to: index=12 +2025-06-17T07:44:02.710Z [INFO] creating new snapshot: path=/tmp/raft3395250291/snapshots/2-1013-1750146242710.tmp +2025-06-17T07:44:02.710Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: copied to local snapshot: bytes=71 +2025-06-17T07:44:02.710Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: snapshot restore progress: id=2-1013-1750146242710 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:02.710Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: restored user snapshot: index=1013 +2025-06-17T07:44:02.710Z [ERROR] server-f02409a4-3f2c-78aa-efc9-47d288b11812: failed to get log: index=23 error="log not found" +2025-06-17T07:44:02.710Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: aborting pipeline replication: peer="{Voter server-c8843568-32df-bb3c-241b-d415f15cd8e7 c8843568-32df-bb3c-241b-d415f15cd8e7}" +2025-06-17T07:44:02.710Z [ERROR] server-f02409a4-3f2c-78aa-efc9-47d288b11812: failed to get log: index=23 error="log not found" +2025-06-17T07:44:02.710Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: aborting pipeline replication: peer="{Voter server-9f3de548-9e76-0fd0-f094-5a0217c6cbe8 9f3de548-9e76-0fd0-f094-5a0217c6cbe8}" +2025-06-17T07:44:02.719Z [ERROR] server-f02409a4-3f2c-78aa-efc9-47d288b11812: failed to get log: index=23 error="log not found" +2025-06-17T07:44:02.719Z [ERROR] server-f02409a4-3f2c-78aa-efc9-47d288b11812: failed to get log: index=23 error="log not found" +2025-06-17T07:44:02.719Z [INFO] creating new snapshot: path=/tmp/raft194460342/snapshots/2-1013-1750146242719.tmp +2025-06-17T07:44:02.719Z [INFO] creating new snapshot: path=/tmp/raft3327028908/snapshots/2-1013-1750146242719.tmp +2025-06-17T07:44:02.720Z [INFO] server-c8843568-32df-bb3c-241b-d415f15cd8e7: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:02.720Z [INFO] server-9f3de548-9e76-0fd0-f094-5a0217c6cbe8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:02.860Z [INFO] server-9f3de548-9e76-0fd0-f094-5a0217c6cbe8: copied to local snapshot: bytes=71 +2025-06-17T07:44:02.860Z [INFO] server-c8843568-32df-bb3c-241b-d415f15cd8e7: copied to local snapshot: bytes=71 +2025-06-17T07:44:02.860Z [INFO] server-9f3de548-9e76-0fd0-f094-5a0217c6cbe8: snapshot restore progress: id=2-1013-1750146242719 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:02.860Z [INFO] server-c8843568-32df-bb3c-241b-d415f15cd8e7: snapshot restore progress: id=2-1013-1750146242719 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:02.860Z [INFO] server-9f3de548-9e76-0fd0-f094-5a0217c6cbe8: Installed remote snapshot +2025-06-17T07:44:02.860Z [INFO] server-c8843568-32df-bb3c-241b-d415f15cd8e7: Installed remote snapshot +2025-06-17T07:44:02.860Z [WARN] server-9f3de548-9e76-0fd0-f094-5a0217c6cbe8: skipping application of old log: index=22 +2025-06-17T07:44:02.860Z [WARN] server-c8843568-32df-bb3c-241b-d415f15cd8e7: skipping application of old log: index=22 +2025-06-17T07:44:02.860Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: pipelining replication: peer="{Voter server-9f3de548-9e76-0fd0-f094-5a0217c6cbe8 9f3de548-9e76-0fd0-f094-5a0217c6cbe8}" +2025-06-17T07:44:02.860Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: pipelining replication: peer="{Voter server-c8843568-32df-bb3c-241b-d415f15cd8e7 c8843568-32df-bb3c-241b-d415f15cd8e7}" +2025-06-17T07:44:02.860Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:03.865Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:02.860663428 +0000 UTC m=+71.464192134 (2 nodes), 1.052µs from start of poll, 1.947759986s from cluster start. Timeout at 2025-06-17 07:44:03.86571674 +0000 UTC m=+72.469245432, 1.005053298s after stability +2025-06-17T07:44:03.877Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: aborting pipeline replication: peer="{Voter server-c8843568-32df-bb3c-241b-d415f15cd8e7 c8843568-32df-bb3c-241b-d415f15cd8e7}" +2025-06-17T07:44:03.877Z [INFO] server-f02409a4-3f2c-78aa-efc9-47d288b11812: aborting pipeline replication: peer="{Voter server-9f3de548-9e76-0fd0-f094-5a0217c6cbe8 9f3de548-9e76-0fd0-f094-5a0217c6cbe8}" === RUN TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false -2024-05-15T01:18:11.462Z [INFO] server-3315b305-dcc1-0ab7-303e-786ee84562a2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3315b305-dcc1-0ab7-303e-786ee84562a2 Address:3315b305-dcc1-0ab7-303e-786ee84562a2} {Suffrage:Voter ID:server-b3202361-d5eb-333c-41da-e156388988e4 Address:b3202361-d5eb-333c-41da-e156388988e4} {Suffrage:Voter ID:server-2341d6ba-ed4d-de11-9c4f-029937d799d4 Address:2341d6ba-ed4d-de11-9c4f-029937d799d4}]" -2024-05-15T01:18:11.462Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3315b305-dcc1-0ab7-303e-786ee84562a2 Address:3315b305-dcc1-0ab7-303e-786ee84562a2} {Suffrage:Voter ID:server-b3202361-d5eb-333c-41da-e156388988e4 Address:b3202361-d5eb-333c-41da-e156388988e4} {Suffrage:Voter ID:server-2341d6ba-ed4d-de11-9c4f-029937d799d4 Address:2341d6ba-ed4d-de11-9c4f-029937d799d4}]" -2024-05-15T01:18:11.462Z [INFO] server-2341d6ba-ed4d-de11-9c4f-029937d799d4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3315b305-dcc1-0ab7-303e-786ee84562a2 Address:3315b305-dcc1-0ab7-303e-786ee84562a2} {Suffrage:Voter ID:server-b3202361-d5eb-333c-41da-e156388988e4 Address:b3202361-d5eb-333c-41da-e156388988e4} {Suffrage:Voter ID:server-2341d6ba-ed4d-de11-9c4f-029937d799d4 Address:2341d6ba-ed4d-de11-9c4f-029937d799d4}]" -2024-05-15T01:18:11.462Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:11.463Z [INFO] server-3315b305-dcc1-0ab7-303e-786ee84562a2: entering follower state: follower="Node at 3315b305-dcc1-0ab7-303e-786ee84562a2 [Follower]" leader-address= leader-id= -2024-05-15T01:18:11.463Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: entering follower state: follower="Node at b3202361-d5eb-333c-41da-e156388988e4 [Follower]" leader-address= leader-id= -2024-05-15T01:18:11.463Z [INFO] server-2341d6ba-ed4d-de11-9c4f-029937d799d4: entering follower state: follower="Node at 2341d6ba-ed4d-de11-9c4f-029937d799d4 [Follower]" leader-address= leader-id= -2024-05-15T01:18:11.988Z [WARN] server-b3202361-d5eb-333c-41da-e156388988e4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:11.988Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: entering candidate state: node="Node at b3202361-d5eb-333c-41da-e156388988e4 [Candidate]" term=2 -2024-05-15T01:18:11.989Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: election won: term=2 tally=2 -2024-05-15T01:18:11.989Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: entering leader state: leader="Node at b3202361-d5eb-333c-41da-e156388988e4 [Leader]" -2024-05-15T01:18:11.989Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: added peer, starting replication: peer=server-3315b305-dcc1-0ab7-303e-786ee84562a2 -2024-05-15T01:18:11.989Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: added peer, starting replication: peer=server-2341d6ba-ed4d-de11-9c4f-029937d799d4 -2024-05-15T01:18:11.989Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: pipelining replication: peer="{Voter server-2341d6ba-ed4d-de11-9c4f-029937d799d4 2341d6ba-ed4d-de11-9c4f-029937d799d4}" -2024-05-15T01:18:11.989Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: pipelining replication: peer="{Voter server-3315b305-dcc1-0ab7-303e-786ee84562a2 3315b305-dcc1-0ab7-303e-786ee84562a2}" -2024-05-15T01:18:12.995Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:11.989548601 +0000 UTC m=+72.530933393 (1 nodes), 526.572914ms from start of poll, 527.062322ms from cluster start. Timeout at 2024-05-15 01:18:12.995590449 +0000 UTC m=+73.536975112, 1.006041719s after stability -2024-05-15T01:18:12.997Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: starting snapshot up to: index=12 -2024-05-15T01:18:12.997Z [INFO] creating new snapshot: path=/tmp/raft2712354853/snapshots/2-12-1715735892997.tmp -2024-05-15T01:18:13.004Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: snapshot complete up to: index=12 -2024-05-15T01:18:13.005Z [INFO] creating new snapshot: path=/tmp/raft2712354853/snapshots/2-1013-1715735893005.tmp -2024-05-15T01:18:13.006Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: copied to local snapshot: bytes=71 -2024-05-15T01:18:13.006Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: snapshot restore progress: id=2-1013-1715735893005 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:13.006Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: removing all old logs from log store -2024-05-15T01:18:13.006Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: compacting logs: from=1 to=22 -2024-05-15T01:18:13.007Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: restored user snapshot: index=1013 -2024-05-15T01:18:13.007Z [ERROR] server-b3202361-d5eb-333c-41da-e156388988e4: failed to get log: index=22 error="log not found" -2024-05-15T01:18:13.007Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: aborting pipeline replication: peer="{Voter server-2341d6ba-ed4d-de11-9c4f-029937d799d4 2341d6ba-ed4d-de11-9c4f-029937d799d4}" -2024-05-15T01:18:13.007Z [ERROR] server-b3202361-d5eb-333c-41da-e156388988e4: failed to get log: index=22 error="log not found" -2024-05-15T01:18:13.007Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: aborting pipeline replication: peer="{Voter server-3315b305-dcc1-0ab7-303e-786ee84562a2 3315b305-dcc1-0ab7-303e-786ee84562a2}" -2024-05-15T01:18:13.014Z [ERROR] server-b3202361-d5eb-333c-41da-e156388988e4: failed to get log: index=22 error="log not found" -2024-05-15T01:18:13.015Z [INFO] creating new snapshot: path=/tmp/raft653772889/snapshots/2-1013-1715735893015.tmp -2024-05-15T01:18:13.015Z [INFO] server-3315b305-dcc1-0ab7-303e-786ee84562a2: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:13.017Z [ERROR] server-b3202361-d5eb-333c-41da-e156388988e4: failed to get log: index=22 error="log not found" -2024-05-15T01:18:13.017Z [INFO] creating new snapshot: path=/tmp/raft3783831461/snapshots/2-1013-1715735893017.tmp -2024-05-15T01:18:13.018Z [INFO] server-2341d6ba-ed4d-de11-9c4f-029937d799d4: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:13.062Z [INFO] server-3315b305-dcc1-0ab7-303e-786ee84562a2: copied to local snapshot: bytes=71 -2024-05-15T01:18:13.062Z [INFO] server-2341d6ba-ed4d-de11-9c4f-029937d799d4: copied to local snapshot: bytes=71 -2024-05-15T01:18:13.062Z [INFO] server-3315b305-dcc1-0ab7-303e-786ee84562a2: snapshot restore progress: id=2-1013-1715735893015 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:13.062Z [INFO] server-3315b305-dcc1-0ab7-303e-786ee84562a2: removing all old logs from log store -2024-05-15T01:18:13.062Z [INFO] server-3315b305-dcc1-0ab7-303e-786ee84562a2: compacting logs: from=1 to=22 -2024-05-15T01:18:13.062Z [INFO] server-3315b305-dcc1-0ab7-303e-786ee84562a2: Installed remote snapshot -2024-05-15T01:18:13.062Z [WARN] server-3315b305-dcc1-0ab7-303e-786ee84562a2: skipping application of old log: index=22 -2024-05-15T01:18:13.062Z [INFO] server-2341d6ba-ed4d-de11-9c4f-029937d799d4: snapshot restore progress: id=2-1013-1715735893017 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:13.062Z [INFO] server-2341d6ba-ed4d-de11-9c4f-029937d799d4: removing all old logs from log store -2024-05-15T01:18:13.062Z [INFO] server-2341d6ba-ed4d-de11-9c4f-029937d799d4: compacting logs: from=1 to=22 -2024-05-15T01:18:13.062Z [INFO] server-2341d6ba-ed4d-de11-9c4f-029937d799d4: Installed remote snapshot -2024-05-15T01:18:13.062Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: pipelining replication: peer="{Voter server-3315b305-dcc1-0ab7-303e-786ee84562a2 3315b305-dcc1-0ab7-303e-786ee84562a2}" -2024-05-15T01:18:13.062Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:13.063Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: pipelining replication: peer="{Voter server-2341d6ba-ed4d-de11-9c4f-029937d799d4 2341d6ba-ed4d-de11-9c4f-029937d799d4}" -2024-05-15T01:18:14.068Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:13.063146549 +0000 UTC m=+73.604531504 (2 nodes), 2.65µs from start of poll, 1.600660433s from cluster start. Timeout at 2024-05-15 01:18:14.068181475 +0000 UTC m=+74.609566454, 1.00503495s after stability -2024-05-15T01:18:14.081Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: aborting pipeline replication: peer="{Voter server-2341d6ba-ed4d-de11-9c4f-029937d799d4 2341d6ba-ed4d-de11-9c4f-029937d799d4}" -2024-05-15T01:18:14.081Z [INFO] server-b3202361-d5eb-333c-41da-e156388988e4: aborting pipeline replication: peer="{Voter server-3315b305-dcc1-0ab7-303e-786ee84562a2 3315b305-dcc1-0ab7-303e-786ee84562a2}" +2025-06-17T07:44:03.878Z [INFO] server-48d0b77a-f3e4-2b82-b579-d38217eb2fe4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-48d0b77a-f3e4-2b82-b579-d38217eb2fe4 Address:48d0b77a-f3e4-2b82-b579-d38217eb2fe4} {Suffrage:Voter ID:server-907c90ec-d0ce-d48e-d52c-e63a0283d9e4 Address:907c90ec-d0ce-d48e-d52c-e63a0283d9e4} {Suffrage:Voter ID:server-c5764222-c010-ab8b-4951-31b343a02749 Address:c5764222-c010-ab8b-4951-31b343a02749}]" +2025-06-17T07:44:03.878Z [INFO] server-48d0b77a-f3e4-2b82-b579-d38217eb2fe4: entering follower state: follower="Node at 48d0b77a-f3e4-2b82-b579-d38217eb2fe4 [Follower]" leader-address= leader-id= +2025-06-17T07:44:03.878Z [INFO] server-907c90ec-d0ce-d48e-d52c-e63a0283d9e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-48d0b77a-f3e4-2b82-b579-d38217eb2fe4 Address:48d0b77a-f3e4-2b82-b579-d38217eb2fe4} {Suffrage:Voter ID:server-907c90ec-d0ce-d48e-d52c-e63a0283d9e4 Address:907c90ec-d0ce-d48e-d52c-e63a0283d9e4} {Suffrage:Voter ID:server-c5764222-c010-ab8b-4951-31b343a02749 Address:c5764222-c010-ab8b-4951-31b343a02749}]" +2025-06-17T07:44:03.878Z [INFO] server-907c90ec-d0ce-d48e-d52c-e63a0283d9e4: entering follower state: follower="Node at 907c90ec-d0ce-d48e-d52c-e63a0283d9e4 [Follower]" leader-address= leader-id= +2025-06-17T07:44:03.878Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-48d0b77a-f3e4-2b82-b579-d38217eb2fe4 Address:48d0b77a-f3e4-2b82-b579-d38217eb2fe4} {Suffrage:Voter ID:server-907c90ec-d0ce-d48e-d52c-e63a0283d9e4 Address:907c90ec-d0ce-d48e-d52c-e63a0283d9e4} {Suffrage:Voter ID:server-c5764222-c010-ab8b-4951-31b343a02749 Address:c5764222-c010-ab8b-4951-31b343a02749}]" +2025-06-17T07:44:03.879Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:03.879Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: entering follower state: follower="Node at c5764222-c010-ab8b-4951-31b343a02749 [Follower]" leader-address= leader-id= +2025-06-17T07:44:04.724Z [WARN] server-c5764222-c010-ab8b-4951-31b343a02749: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:04.724Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: entering candidate state: node="Node at c5764222-c010-ab8b-4951-31b343a02749 [Candidate]" term=2 +2025-06-17T07:44:04.724Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: election won: term=2 tally=2 +2025-06-17T07:44:04.724Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: entering leader state: leader="Node at c5764222-c010-ab8b-4951-31b343a02749 [Leader]" +2025-06-17T07:44:04.724Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: added peer, starting replication: peer=server-48d0b77a-f3e4-2b82-b579-d38217eb2fe4 +2025-06-17T07:44:04.724Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: added peer, starting replication: peer=server-907c90ec-d0ce-d48e-d52c-e63a0283d9e4 +2025-06-17T07:44:04.724Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: pipelining replication: peer="{Voter server-48d0b77a-f3e4-2b82-b579-d38217eb2fe4 48d0b77a-f3e4-2b82-b579-d38217eb2fe4}" +2025-06-17T07:44:04.724Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: pipelining replication: peer="{Voter server-907c90ec-d0ce-d48e-d52c-e63a0283d9e4 907c90ec-d0ce-d48e-d52c-e63a0283d9e4}" +2025-06-17T07:44:05.730Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:04.724693441 +0000 UTC m=+73.328222153 (1 nodes), 845.661157ms from start of poll, 846.048835ms from cluster start. Timeout at 2025-06-17 07:44:05.730357509 +0000 UTC m=+74.333886226, 1.005664073s after stability +2025-06-17T07:44:05.730Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: starting snapshot up to: index=12 +2025-06-17T07:44:05.730Z [INFO] creating new snapshot: path=/tmp/raft3525528768/snapshots/2-12-1750146245730.tmp +2025-06-17T07:44:05.731Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: snapshot complete up to: index=12 +2025-06-17T07:44:05.732Z [INFO] creating new snapshot: path=/tmp/raft3525528768/snapshots/2-1013-1750146245732.tmp +2025-06-17T07:44:05.732Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: copied to local snapshot: bytes=71 +2025-06-17T07:44:05.733Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: snapshot restore progress: id=2-1013-1750146245732 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:05.733Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: removing all old logs from log store +2025-06-17T07:44:05.733Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: compacting logs: from=1 to=22 +2025-06-17T07:44:05.733Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: restored user snapshot: index=1013 +2025-06-17T07:44:05.733Z [ERROR] server-c5764222-c010-ab8b-4951-31b343a02749: failed to get log: index=22 error="log not found" +2025-06-17T07:44:05.733Z [ERROR] server-c5764222-c010-ab8b-4951-31b343a02749: failed to get log: index=22 error="log not found" +2025-06-17T07:44:05.733Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: aborting pipeline replication: peer="{Voter server-48d0b77a-f3e4-2b82-b579-d38217eb2fe4 48d0b77a-f3e4-2b82-b579-d38217eb2fe4}" +2025-06-17T07:44:05.733Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: aborting pipeline replication: peer="{Voter server-907c90ec-d0ce-d48e-d52c-e63a0283d9e4 907c90ec-d0ce-d48e-d52c-e63a0283d9e4}" +2025-06-17T07:44:05.740Z [ERROR] server-c5764222-c010-ab8b-4951-31b343a02749: failed to get log: index=22 error="log not found" +2025-06-17T07:44:05.742Z [ERROR] server-c5764222-c010-ab8b-4951-31b343a02749: failed to get log: index=22 error="log not found" +2025-06-17T07:44:05.742Z [INFO] creating new snapshot: path=/tmp/raft1875846621/snapshots/2-1013-1750146245742.tmp +2025-06-17T07:44:05.743Z [INFO] server-907c90ec-d0ce-d48e-d52c-e63a0283d9e4: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:05.743Z [INFO] creating new snapshot: path=/tmp/raft2639563811/snapshots/2-1013-1750146245743.tmp +2025-06-17T07:44:05.743Z [INFO] server-907c90ec-d0ce-d48e-d52c-e63a0283d9e4: copied to local snapshot: bytes=71 +2025-06-17T07:44:05.744Z [INFO] server-48d0b77a-f3e4-2b82-b579-d38217eb2fe4: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:05.744Z [INFO] server-907c90ec-d0ce-d48e-d52c-e63a0283d9e4: snapshot restore progress: id=2-1013-1750146245742 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:05.744Z [INFO] server-907c90ec-d0ce-d48e-d52c-e63a0283d9e4: removing all old logs from log store +2025-06-17T07:44:05.744Z [INFO] server-907c90ec-d0ce-d48e-d52c-e63a0283d9e4: compacting logs: from=1 to=22 +2025-06-17T07:44:05.744Z [INFO] server-907c90ec-d0ce-d48e-d52c-e63a0283d9e4: Installed remote snapshot +2025-06-17T07:44:05.744Z [INFO] server-48d0b77a-f3e4-2b82-b579-d38217eb2fe4: copied to local snapshot: bytes=71 +2025-06-17T07:44:05.744Z [WARN] server-907c90ec-d0ce-d48e-d52c-e63a0283d9e4: skipping application of old log: index=22 +2025-06-17T07:44:05.745Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: pipelining replication: peer="{Voter server-907c90ec-d0ce-d48e-d52c-e63a0283d9e4 907c90ec-d0ce-d48e-d52c-e63a0283d9e4}" +2025-06-17T07:44:05.745Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:05.745Z [INFO] server-48d0b77a-f3e4-2b82-b579-d38217eb2fe4: snapshot restore progress: id=2-1013-1750146245743 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:05.745Z [INFO] server-48d0b77a-f3e4-2b82-b579-d38217eb2fe4: removing all old logs from log store +2025-06-17T07:44:05.745Z [INFO] server-48d0b77a-f3e4-2b82-b579-d38217eb2fe4: compacting logs: from=1 to=22 +2025-06-17T07:44:05.745Z [INFO] server-48d0b77a-f3e4-2b82-b579-d38217eb2fe4: Installed remote snapshot +2025-06-17T07:44:05.745Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: pipelining replication: peer="{Voter server-48d0b77a-f3e4-2b82-b579-d38217eb2fe4 48d0b77a-f3e4-2b82-b579-d38217eb2fe4}" +2025-06-17T07:44:06.751Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:05.74505017 +0000 UTC m=+74.348578857 (2 nodes), 671ns from start of poll, 1.866405539s from cluster start. Timeout at 2025-06-17 07:44:06.751068051 +0000 UTC m=+75.354596750, 1.006017893s after stability +2025-06-17T07:44:06.762Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: aborting pipeline replication: peer="{Voter server-907c90ec-d0ce-d48e-d52c-e63a0283d9e4 907c90ec-d0ce-d48e-d52c-e63a0283d9e4}" +2025-06-17T07:44:06.762Z [INFO] server-c5764222-c010-ab8b-4951-31b343a02749: aborting pipeline replication: peer="{Voter server-48d0b77a-f3e4-2b82-b579-d38217eb2fe4 48d0b77a-f3e4-2b82-b579-d38217eb2fe4}" === RUN TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true -2024-05-15T01:18:14.084Z [INFO] server-8c903c1e-31eb-c505-5a1f-b9ce4ff48270: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8c903c1e-31eb-c505-5a1f-b9ce4ff48270 Address:8c903c1e-31eb-c505-5a1f-b9ce4ff48270} {Suffrage:Voter ID:server-4b48da76-7a00-9bb4-09ef-b257667b6efd Address:4b48da76-7a00-9bb4-09ef-b257667b6efd} {Suffrage:Voter ID:server-108aaf4e-9ad6-2bfe-1126-035c137b2942 Address:108aaf4e-9ad6-2bfe-1126-035c137b2942}]" -2024-05-15T01:18:14.084Z [INFO] server-8c903c1e-31eb-c505-5a1f-b9ce4ff48270: entering follower state: follower="Node at 8c903c1e-31eb-c505-5a1f-b9ce4ff48270 [Follower]" leader-address= leader-id= -2024-05-15T01:18:14.084Z [INFO] server-4b48da76-7a00-9bb4-09ef-b257667b6efd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8c903c1e-31eb-c505-5a1f-b9ce4ff48270 Address:8c903c1e-31eb-c505-5a1f-b9ce4ff48270} {Suffrage:Voter ID:server-4b48da76-7a00-9bb4-09ef-b257667b6efd Address:4b48da76-7a00-9bb4-09ef-b257667b6efd} {Suffrage:Voter ID:server-108aaf4e-9ad6-2bfe-1126-035c137b2942 Address:108aaf4e-9ad6-2bfe-1126-035c137b2942}]" -2024-05-15T01:18:14.084Z [INFO] server-108aaf4e-9ad6-2bfe-1126-035c137b2942: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8c903c1e-31eb-c505-5a1f-b9ce4ff48270 Address:8c903c1e-31eb-c505-5a1f-b9ce4ff48270} {Suffrage:Voter ID:server-4b48da76-7a00-9bb4-09ef-b257667b6efd Address:4b48da76-7a00-9bb4-09ef-b257667b6efd} {Suffrage:Voter ID:server-108aaf4e-9ad6-2bfe-1126-035c137b2942 Address:108aaf4e-9ad6-2bfe-1126-035c137b2942}]" -2024-05-15T01:18:14.084Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:14.084Z [INFO] server-108aaf4e-9ad6-2bfe-1126-035c137b2942: entering follower state: follower="Node at 108aaf4e-9ad6-2bfe-1126-035c137b2942 [Follower]" leader-address= leader-id= -2024-05-15T01:18:14.084Z [INFO] server-4b48da76-7a00-9bb4-09ef-b257667b6efd: entering follower state: follower="Node at 4b48da76-7a00-9bb4-09ef-b257667b6efd [Follower]" leader-address= leader-id= -2024-05-15T01:18:14.772Z [WARN] server-4b48da76-7a00-9bb4-09ef-b257667b6efd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:14.772Z [INFO] server-4b48da76-7a00-9bb4-09ef-b257667b6efd: entering candidate state: node="Node at 4b48da76-7a00-9bb4-09ef-b257667b6efd [Candidate]" term=2 -2024-05-15T01:18:14.772Z [INFO] server-4b48da76-7a00-9bb4-09ef-b257667b6efd: election won: term=2 tally=2 -2024-05-15T01:18:14.772Z [INFO] server-4b48da76-7a00-9bb4-09ef-b257667b6efd: entering leader state: leader="Node at 4b48da76-7a00-9bb4-09ef-b257667b6efd [Leader]" -2024-05-15T01:18:14.772Z [INFO] server-4b48da76-7a00-9bb4-09ef-b257667b6efd: added peer, starting replication: peer=server-8c903c1e-31eb-c505-5a1f-b9ce4ff48270 -2024-05-15T01:18:14.772Z [INFO] server-4b48da76-7a00-9bb4-09ef-b257667b6efd: added peer, starting replication: peer=server-108aaf4e-9ad6-2bfe-1126-035c137b2942 -2024-05-15T01:18:14.773Z [INFO] server-4b48da76-7a00-9bb4-09ef-b257667b6efd: pipelining replication: peer="{Voter server-108aaf4e-9ad6-2bfe-1126-035c137b2942 108aaf4e-9ad6-2bfe-1126-035c137b2942}" -2024-05-15T01:18:14.773Z [INFO] server-4b48da76-7a00-9bb4-09ef-b257667b6efd: pipelining replication: peer="{Voter server-8c903c1e-31eb-c505-5a1f-b9ce4ff48270 8c903c1e-31eb-c505-5a1f-b9ce4ff48270}" -2024-05-15T01:18:15.778Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:14.772839207 +0000 UTC m=+75.314224180 (1 nodes), 687.861092ms from start of poll, 688.459712ms from cluster start. Timeout at 2024-05-15 01:18:15.778166904 +0000 UTC m=+76.319551858, 1.005327678s after stability -2024-05-15T01:18:15.780Z [INFO] server-4b48da76-7a00-9bb4-09ef-b257667b6efd: starting snapshot up to: index=12 -2024-05-15T01:18:15.780Z [INFO] creating new snapshot: path=/tmp/raft1849775906/snapshots/2-12-1715735895780.tmp -2024-05-15T01:18:15.781Z [INFO] server-4b48da76-7a00-9bb4-09ef-b257667b6efd: snapshot complete up to: index=12 -2024-05-15T01:18:15.783Z [INFO] server-8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4 Address:8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4} {Suffrage:Voter ID:server-e81fc3cb-2d2b-1ff2-7335-97d52cb18b81 Address:e81fc3cb-2d2b-1ff2-7335-97d52cb18b81} {Suffrage:Voter ID:server-7980e2cc-1b76-8a58-967f-73fe182e7083 Address:7980e2cc-1b76-8a58-967f-73fe182e7083}]" -2024-05-15T01:18:15.783Z [INFO] server-8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4: entering follower state: follower="Node at 8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4 [Follower]" leader-address= leader-id= -2024-05-15T01:18:15.783Z [INFO] server-e81fc3cb-2d2b-1ff2-7335-97d52cb18b81: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4 Address:8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4} {Suffrage:Voter ID:server-e81fc3cb-2d2b-1ff2-7335-97d52cb18b81 Address:e81fc3cb-2d2b-1ff2-7335-97d52cb18b81} {Suffrage:Voter ID:server-7980e2cc-1b76-8a58-967f-73fe182e7083 Address:7980e2cc-1b76-8a58-967f-73fe182e7083}]" -2024-05-15T01:18:15.783Z [INFO] server-e81fc3cb-2d2b-1ff2-7335-97d52cb18b81: entering follower state: follower="Node at e81fc3cb-2d2b-1ff2-7335-97d52cb18b81 [Follower]" leader-address= leader-id= -2024-05-15T01:18:15.783Z [INFO] server-7980e2cc-1b76-8a58-967f-73fe182e7083: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4 Address:8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4} {Suffrage:Voter ID:server-e81fc3cb-2d2b-1ff2-7335-97d52cb18b81 Address:e81fc3cb-2d2b-1ff2-7335-97d52cb18b81} {Suffrage:Voter ID:server-7980e2cc-1b76-8a58-967f-73fe182e7083 Address:7980e2cc-1b76-8a58-967f-73fe182e7083}]" -2024-05-15T01:18:15.783Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:15.783Z [INFO] server-7980e2cc-1b76-8a58-967f-73fe182e7083: entering follower state: follower="Node at 7980e2cc-1b76-8a58-967f-73fe182e7083 [Follower]" leader-address= leader-id= -2024-05-15T01:18:16.295Z [WARN] server-7980e2cc-1b76-8a58-967f-73fe182e7083: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:16.295Z [INFO] server-7980e2cc-1b76-8a58-967f-73fe182e7083: entering candidate state: node="Node at 7980e2cc-1b76-8a58-967f-73fe182e7083 [Candidate]" term=2 -2024-05-15T01:18:16.296Z [INFO] server-7980e2cc-1b76-8a58-967f-73fe182e7083: election won: term=2 tally=2 -2024-05-15T01:18:16.296Z [INFO] server-7980e2cc-1b76-8a58-967f-73fe182e7083: entering leader state: leader="Node at 7980e2cc-1b76-8a58-967f-73fe182e7083 [Leader]" -2024-05-15T01:18:16.296Z [INFO] server-7980e2cc-1b76-8a58-967f-73fe182e7083: added peer, starting replication: peer=server-8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4 -2024-05-15T01:18:16.296Z [INFO] server-7980e2cc-1b76-8a58-967f-73fe182e7083: added peer, starting replication: peer=server-e81fc3cb-2d2b-1ff2-7335-97d52cb18b81 -2024-05-15T01:18:16.296Z [INFO] server-7980e2cc-1b76-8a58-967f-73fe182e7083: pipelining replication: peer="{Voter server-e81fc3cb-2d2b-1ff2-7335-97d52cb18b81 e81fc3cb-2d2b-1ff2-7335-97d52cb18b81}" -2024-05-15T01:18:16.296Z [INFO] server-7980e2cc-1b76-8a58-967f-73fe182e7083: pipelining replication: peer="{Voter server-8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4 8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4}" -2024-05-15T01:18:17.301Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:16.296140985 +0000 UTC m=+76.837525926 (1 nodes), 512.206949ms from start of poll, 512.833295ms from cluster start. Timeout at 2024-05-15 01:18:17.301353003 +0000 UTC m=+77.842737762, 1.005211836s after stability -2024-05-15T01:18:17.301Z [INFO] creating new snapshot: path=/tmp/raft901219573/snapshots/2-1013-1715735897301.tmp -2024-05-15T01:18:17.302Z [INFO] server-7980e2cc-1b76-8a58-967f-73fe182e7083: copied to local snapshot: bytes=71 -2024-05-15T01:18:17.302Z [INFO] server-7980e2cc-1b76-8a58-967f-73fe182e7083: snapshot restore progress: id=2-1013-1715735897301 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:17.303Z [INFO] server-7980e2cc-1b76-8a58-967f-73fe182e7083: restored user snapshot: index=1013 -2024-05-15T01:18:17.303Z [ERROR] server-7980e2cc-1b76-8a58-967f-73fe182e7083: failed to get log: index=3 error="log not found" -2024-05-15T01:18:17.303Z [INFO] server-7980e2cc-1b76-8a58-967f-73fe182e7083: aborting pipeline replication: peer="{Voter server-e81fc3cb-2d2b-1ff2-7335-97d52cb18b81 e81fc3cb-2d2b-1ff2-7335-97d52cb18b81}" -2024-05-15T01:18:17.303Z [ERROR] server-7980e2cc-1b76-8a58-967f-73fe182e7083: failed to get log: index=3 error="log not found" -2024-05-15T01:18:17.303Z [INFO] server-7980e2cc-1b76-8a58-967f-73fe182e7083: aborting pipeline replication: peer="{Voter server-8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4 8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4}" -2024-05-15T01:18:17.309Z [ERROR] server-7980e2cc-1b76-8a58-967f-73fe182e7083: failed to get log: index=3 error="log not found" -2024-05-15T01:18:17.310Z [INFO] creating new snapshot: path=/tmp/raft782631713/snapshots/2-1013-1715735897310.tmp -2024-05-15T01:18:17.310Z [INFO] server-e81fc3cb-2d2b-1ff2-7335-97d52cb18b81: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:17.311Z [INFO] server-e81fc3cb-2d2b-1ff2-7335-97d52cb18b81: copied to local snapshot: bytes=71 -2024-05-15T01:18:17.311Z [INFO] server-e81fc3cb-2d2b-1ff2-7335-97d52cb18b81: snapshot restore progress: id=2-1013-1715735897310 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:17.311Z [INFO] server-e81fc3cb-2d2b-1ff2-7335-97d52cb18b81: Installed remote snapshot -2024-05-15T01:18:17.311Z [INFO] server-7980e2cc-1b76-8a58-967f-73fe182e7083: pipelining replication: peer="{Voter server-e81fc3cb-2d2b-1ff2-7335-97d52cb18b81 e81fc3cb-2d2b-1ff2-7335-97d52cb18b81}" -2024-05-15T01:18:17.312Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:17.312Z [ERROR] server-7980e2cc-1b76-8a58-967f-73fe182e7083: failed to get log: index=3 error="log not found" -2024-05-15T01:18:17.313Z [INFO] creating new snapshot: path=/tmp/raft2527685181/snapshots/2-1013-1715735897313.tmp -2024-05-15T01:18:17.313Z [INFO] server-8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:17.314Z [INFO] server-8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4: copied to local snapshot: bytes=71 -2024-05-15T01:18:17.314Z [INFO] server-8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4: snapshot restore progress: id=2-1013-1715735897313 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:17.314Z [INFO] server-8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4: Installed remote snapshot -2024-05-15T01:18:17.314Z [INFO] server-7980e2cc-1b76-8a58-967f-73fe182e7083: pipelining replication: peer="{Voter server-8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4 8fdf06a8-4c93-2dc6-ef2f-6442298a0fe4}" -2024-05-15T01:18:18.317Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:17.312045291 +0000 UTC m=+77.853430166 (2 nodes), 2.263µs from start of poll, 1.528737535s from cluster start. Timeout at 2024-05-15 01:18:18.31729841 +0000 UTC m=+78.858683305, 1.005253139s after stability -2024-05-15T01:18:18.333Z [INFO] server-4b48da76-7a00-9bb4-09ef-b257667b6efd: aborting pipeline replication: peer="{Voter server-108aaf4e-9ad6-2bfe-1126-035c137b2942 108aaf4e-9ad6-2bfe-1126-035c137b2942}" -2024-05-15T01:18:18.333Z [INFO] server-4b48da76-7a00-9bb4-09ef-b257667b6efd: aborting pipeline replication: peer="{Voter server-8c903c1e-31eb-c505-5a1f-b9ce4ff48270 8c903c1e-31eb-c505-5a1f-b9ce4ff48270}" +2025-06-17T07:44:06.763Z [INFO] server-1d934dc2-0391-39b8-2190-dedb324f418f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1d934dc2-0391-39b8-2190-dedb324f418f Address:1d934dc2-0391-39b8-2190-dedb324f418f} {Suffrage:Voter ID:server-7395f46e-c980-3b25-addd-8dbba536e0d9 Address:7395f46e-c980-3b25-addd-8dbba536e0d9} {Suffrage:Voter ID:server-c2a3c778-f52d-11ec-d1e8-ab9198fe8b7d Address:c2a3c778-f52d-11ec-d1e8-ab9198fe8b7d}]" +2025-06-17T07:44:06.763Z [INFO] server-1d934dc2-0391-39b8-2190-dedb324f418f: entering follower state: follower="Node at 1d934dc2-0391-39b8-2190-dedb324f418f [Follower]" leader-address= leader-id= +2025-06-17T07:44:06.764Z [INFO] server-7395f46e-c980-3b25-addd-8dbba536e0d9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1d934dc2-0391-39b8-2190-dedb324f418f Address:1d934dc2-0391-39b8-2190-dedb324f418f} {Suffrage:Voter ID:server-7395f46e-c980-3b25-addd-8dbba536e0d9 Address:7395f46e-c980-3b25-addd-8dbba536e0d9} {Suffrage:Voter ID:server-c2a3c778-f52d-11ec-d1e8-ab9198fe8b7d Address:c2a3c778-f52d-11ec-d1e8-ab9198fe8b7d}]" +2025-06-17T07:44:06.764Z [INFO] server-7395f46e-c980-3b25-addd-8dbba536e0d9: entering follower state: follower="Node at 7395f46e-c980-3b25-addd-8dbba536e0d9 [Follower]" leader-address= leader-id= +2025-06-17T07:44:06.764Z [INFO] server-c2a3c778-f52d-11ec-d1e8-ab9198fe8b7d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1d934dc2-0391-39b8-2190-dedb324f418f Address:1d934dc2-0391-39b8-2190-dedb324f418f} {Suffrage:Voter ID:server-7395f46e-c980-3b25-addd-8dbba536e0d9 Address:7395f46e-c980-3b25-addd-8dbba536e0d9} {Suffrage:Voter ID:server-c2a3c778-f52d-11ec-d1e8-ab9198fe8b7d Address:c2a3c778-f52d-11ec-d1e8-ab9198fe8b7d}]" +2025-06-17T07:44:06.764Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:06.764Z [INFO] server-c2a3c778-f52d-11ec-d1e8-ab9198fe8b7d: entering follower state: follower="Node at c2a3c778-f52d-11ec-d1e8-ab9198fe8b7d [Follower]" leader-address= leader-id= +2025-06-17T07:44:07.387Z [WARN] server-7395f46e-c980-3b25-addd-8dbba536e0d9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:07.387Z [INFO] server-7395f46e-c980-3b25-addd-8dbba536e0d9: entering candidate state: node="Node at 7395f46e-c980-3b25-addd-8dbba536e0d9 [Candidate]" term=2 +2025-06-17T07:44:07.387Z [INFO] server-7395f46e-c980-3b25-addd-8dbba536e0d9: election won: term=2 tally=2 +2025-06-17T07:44:07.387Z [INFO] server-7395f46e-c980-3b25-addd-8dbba536e0d9: entering leader state: leader="Node at 7395f46e-c980-3b25-addd-8dbba536e0d9 [Leader]" +2025-06-17T07:44:07.387Z [INFO] server-7395f46e-c980-3b25-addd-8dbba536e0d9: added peer, starting replication: peer=server-1d934dc2-0391-39b8-2190-dedb324f418f +2025-06-17T07:44:07.387Z [INFO] server-7395f46e-c980-3b25-addd-8dbba536e0d9: added peer, starting replication: peer=server-c2a3c778-f52d-11ec-d1e8-ab9198fe8b7d +2025-06-17T07:44:07.388Z [INFO] server-7395f46e-c980-3b25-addd-8dbba536e0d9: pipelining replication: peer="{Voter server-1d934dc2-0391-39b8-2190-dedb324f418f 1d934dc2-0391-39b8-2190-dedb324f418f}" +2025-06-17T07:44:07.388Z [INFO] server-7395f46e-c980-3b25-addd-8dbba536e0d9: pipelining replication: peer="{Voter server-c2a3c778-f52d-11ec-d1e8-ab9198fe8b7d c2a3c778-f52d-11ec-d1e8-ab9198fe8b7d}" +2025-06-17T07:44:08.393Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:07.388001619 +0000 UTC m=+75.991530339 (1 nodes), 623.876535ms from start of poll, 624.233966ms from cluster start. Timeout at 2025-06-17 07:44:08.393801794 +0000 UTC m=+76.997330476, 1.005800137s after stability +2025-06-17T07:44:08.394Z [INFO] server-7395f46e-c980-3b25-addd-8dbba536e0d9: starting snapshot up to: index=12 +2025-06-17T07:44:08.394Z [INFO] creating new snapshot: path=/tmp/raft2115614277/snapshots/2-12-1750146248394.tmp +2025-06-17T07:44:08.394Z [INFO] server-7395f46e-c980-3b25-addd-8dbba536e0d9: snapshot complete up to: index=12 +2025-06-17T07:44:08.395Z [INFO] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d Address:b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d} {Suffrage:Voter ID:server-bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d Address:bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d} {Suffrage:Voter ID:server-b872008f-009f-04a2-ff84-9ab0b5f33c30 Address:b872008f-009f-04a2-ff84-9ab0b5f33c30}]" +2025-06-17T07:44:08.395Z [INFO] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: entering follower state: follower="Node at b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d [Follower]" leader-address= leader-id= +2025-06-17T07:44:08.396Z [INFO] server-bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d Address:b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d} {Suffrage:Voter ID:server-bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d Address:bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d} {Suffrage:Voter ID:server-b872008f-009f-04a2-ff84-9ab0b5f33c30 Address:b872008f-009f-04a2-ff84-9ab0b5f33c30}]" +2025-06-17T07:44:08.396Z [INFO] server-bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d: entering follower state: follower="Node at bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d [Follower]" leader-address= leader-id= +2025-06-17T07:44:08.396Z [INFO] server-b872008f-009f-04a2-ff84-9ab0b5f33c30: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d Address:b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d} {Suffrage:Voter ID:server-bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d Address:bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d} {Suffrage:Voter ID:server-b872008f-009f-04a2-ff84-9ab0b5f33c30 Address:b872008f-009f-04a2-ff84-9ab0b5f33c30}]" +2025-06-17T07:44:08.396Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:08.396Z [INFO] server-b872008f-009f-04a2-ff84-9ab0b5f33c30: entering follower state: follower="Node at b872008f-009f-04a2-ff84-9ab0b5f33c30 [Follower]" leader-address= leader-id= +2025-06-17T07:44:08.924Z [WARN] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:08.924Z [INFO] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: entering candidate state: node="Node at b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d [Candidate]" term=2 +2025-06-17T07:44:08.924Z [INFO] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: election won: term=2 tally=2 +2025-06-17T07:44:08.924Z [INFO] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: entering leader state: leader="Node at b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d [Leader]" +2025-06-17T07:44:08.924Z [INFO] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: added peer, starting replication: peer=server-bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d +2025-06-17T07:44:08.924Z [INFO] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: added peer, starting replication: peer=server-b872008f-009f-04a2-ff84-9ab0b5f33c30 +2025-06-17T07:44:08.924Z [INFO] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: pipelining replication: peer="{Voter server-bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d}" +2025-06-17T07:44:08.924Z [INFO] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: pipelining replication: peer="{Voter server-b872008f-009f-04a2-ff84-9ab0b5f33c30 b872008f-009f-04a2-ff84-9ab0b5f33c30}" +2025-06-17T07:44:09.930Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:08.924499438 +0000 UTC m=+77.528028160 (1 nodes), 528.362145ms from start of poll, 528.66951ms from cluster start. Timeout at 2025-06-17 07:44:09.930349117 +0000 UTC m=+78.533877806, 1.005849646s after stability +2025-06-17T07:44:09.931Z [INFO] creating new snapshot: path=/tmp/raft1225195171/snapshots/2-1013-1750146249931.tmp +2025-06-17T07:44:09.931Z [INFO] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: copied to local snapshot: bytes=71 +2025-06-17T07:44:09.932Z [INFO] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: snapshot restore progress: id=2-1013-1750146249931 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:09.932Z [INFO] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: restored user snapshot: index=1013 +2025-06-17T07:44:09.932Z [ERROR] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: failed to get log: index=3 error="log not found" +2025-06-17T07:44:09.932Z [INFO] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: aborting pipeline replication: peer="{Voter server-b872008f-009f-04a2-ff84-9ab0b5f33c30 b872008f-009f-04a2-ff84-9ab0b5f33c30}" +2025-06-17T07:44:09.932Z [ERROR] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: failed to get log: index=3 error="log not found" +2025-06-17T07:44:09.932Z [INFO] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: aborting pipeline replication: peer="{Voter server-bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d}" +2025-06-17T07:44:09.937Z [ERROR] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: failed to get log: index=3 error="log not found" +2025-06-17T07:44:09.937Z [INFO] creating new snapshot: path=/tmp/raft1785512878/snapshots/2-1013-1750146249937.tmp +2025-06-17T07:44:09.937Z [INFO] server-b872008f-009f-04a2-ff84-9ab0b5f33c30: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:09.938Z [INFO] server-b872008f-009f-04a2-ff84-9ab0b5f33c30: copied to local snapshot: bytes=71 +2025-06-17T07:44:09.938Z [INFO] server-b872008f-009f-04a2-ff84-9ab0b5f33c30: snapshot restore progress: id=2-1013-1750146249937 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:09.938Z [INFO] server-b872008f-009f-04a2-ff84-9ab0b5f33c30: Installed remote snapshot +2025-06-17T07:44:09.938Z [INFO] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: pipelining replication: peer="{Voter server-b872008f-009f-04a2-ff84-9ab0b5f33c30 b872008f-009f-04a2-ff84-9ab0b5f33c30}" +2025-06-17T07:44:09.938Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:09.940Z [ERROR] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: failed to get log: index=3 error="log not found" +2025-06-17T07:44:09.941Z [INFO] creating new snapshot: path=/tmp/raft1300568881/snapshots/2-1013-1750146249941.tmp +2025-06-17T07:44:09.941Z [INFO] server-bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:09.941Z [INFO] server-bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d: copied to local snapshot: bytes=71 +2025-06-17T07:44:09.941Z [INFO] server-bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d: snapshot restore progress: id=2-1013-1750146249941 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:09.941Z [INFO] server-bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d: Installed remote snapshot +2025-06-17T07:44:09.941Z [INFO] server-b55b7fd6-9f6f-f2d0-d8c1-f9713d0d188d: pipelining replication: peer="{Voter server-bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d bd12a7d2-1955-ac0d-d9f1-ff1f0cbf943d}" +2025-06-17T07:44:10.944Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:09.938446114 +0000 UTC m=+78.541974800 (2 nodes), 721ns from start of poll, 1.54261615s from cluster start. Timeout at 2025-06-17 07:44:10.944173832 +0000 UTC m=+79.547702515, 1.005727715s after stability +2025-06-17T07:44:10.956Z [INFO] server-7395f46e-c980-3b25-addd-8dbba536e0d9: aborting pipeline replication: peer="{Voter server-c2a3c778-f52d-11ec-d1e8-ab9198fe8b7d c2a3c778-f52d-11ec-d1e8-ab9198fe8b7d}" +2025-06-17T07:44:10.956Z [INFO] server-7395f46e-c980-3b25-addd-8dbba536e0d9: aborting pipeline replication: peer="{Voter server-1d934dc2-0391-39b8-2190-dedb324f418f 1d934dc2-0391-39b8-2190-dedb324f418f}" === RUN TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true -2024-05-15T01:18:18.335Z [INFO] server-19298f41-10e1-4453-ad21-70473fc3d1ba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-19298f41-10e1-4453-ad21-70473fc3d1ba Address:19298f41-10e1-4453-ad21-70473fc3d1ba} {Suffrage:Voter ID:server-71fa0367-ef7a-26b3-4306-64823a64e91e Address:71fa0367-ef7a-26b3-4306-64823a64e91e} {Suffrage:Voter ID:server-ab914151-958e-8486-f81a-478ffd27a2e1 Address:ab914151-958e-8486-f81a-478ffd27a2e1}]" -2024-05-15T01:18:18.335Z [INFO] server-71fa0367-ef7a-26b3-4306-64823a64e91e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-19298f41-10e1-4453-ad21-70473fc3d1ba Address:19298f41-10e1-4453-ad21-70473fc3d1ba} {Suffrage:Voter ID:server-71fa0367-ef7a-26b3-4306-64823a64e91e Address:71fa0367-ef7a-26b3-4306-64823a64e91e} {Suffrage:Voter ID:server-ab914151-958e-8486-f81a-478ffd27a2e1 Address:ab914151-958e-8486-f81a-478ffd27a2e1}]" -2024-05-15T01:18:18.336Z [INFO] server-ab914151-958e-8486-f81a-478ffd27a2e1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-19298f41-10e1-4453-ad21-70473fc3d1ba Address:19298f41-10e1-4453-ad21-70473fc3d1ba} {Suffrage:Voter ID:server-71fa0367-ef7a-26b3-4306-64823a64e91e Address:71fa0367-ef7a-26b3-4306-64823a64e91e} {Suffrage:Voter ID:server-ab914151-958e-8486-f81a-478ffd27a2e1 Address:ab914151-958e-8486-f81a-478ffd27a2e1}]" -2024-05-15T01:18:18.336Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:18.336Z [INFO] server-19298f41-10e1-4453-ad21-70473fc3d1ba: entering follower state: follower="Node at 19298f41-10e1-4453-ad21-70473fc3d1ba [Follower]" leader-address= leader-id= -2024-05-15T01:18:18.336Z [INFO] server-71fa0367-ef7a-26b3-4306-64823a64e91e: entering follower state: follower="Node at 71fa0367-ef7a-26b3-4306-64823a64e91e [Follower]" leader-address= leader-id= -2024-05-15T01:18:18.336Z [INFO] server-ab914151-958e-8486-f81a-478ffd27a2e1: entering follower state: follower="Node at ab914151-958e-8486-f81a-478ffd27a2e1 [Follower]" leader-address= leader-id= -2024-05-15T01:18:19.007Z [WARN] server-ab914151-958e-8486-f81a-478ffd27a2e1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:19.007Z [INFO] server-ab914151-958e-8486-f81a-478ffd27a2e1: entering candidate state: node="Node at ab914151-958e-8486-f81a-478ffd27a2e1 [Candidate]" term=2 -2024-05-15T01:18:19.007Z [INFO] server-ab914151-958e-8486-f81a-478ffd27a2e1: election won: term=2 tally=2 -2024-05-15T01:18:19.007Z [INFO] server-ab914151-958e-8486-f81a-478ffd27a2e1: entering leader state: leader="Node at ab914151-958e-8486-f81a-478ffd27a2e1 [Leader]" -2024-05-15T01:18:19.007Z [INFO] server-ab914151-958e-8486-f81a-478ffd27a2e1: added peer, starting replication: peer=server-19298f41-10e1-4453-ad21-70473fc3d1ba -2024-05-15T01:18:19.007Z [INFO] server-ab914151-958e-8486-f81a-478ffd27a2e1: added peer, starting replication: peer=server-71fa0367-ef7a-26b3-4306-64823a64e91e -2024-05-15T01:18:19.007Z [INFO] server-ab914151-958e-8486-f81a-478ffd27a2e1: pipelining replication: peer="{Voter server-71fa0367-ef7a-26b3-4306-64823a64e91e 71fa0367-ef7a-26b3-4306-64823a64e91e}" -2024-05-15T01:18:19.007Z [INFO] server-ab914151-958e-8486-f81a-478ffd27a2e1: pipelining replication: peer="{Voter server-19298f41-10e1-4453-ad21-70473fc3d1ba 19298f41-10e1-4453-ad21-70473fc3d1ba}" -2024-05-15T01:18:20.014Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:19.00809156 +0000 UTC m=+79.549476453 (1 nodes), 671.902115ms from start of poll, 672.449901ms from cluster start. Timeout at 2024-05-15 01:18:20.014081153 +0000 UTC m=+80.555465936, 1.005989483s after stability -2024-05-15T01:18:20.015Z [INFO] server-ab914151-958e-8486-f81a-478ffd27a2e1: starting snapshot up to: index=12 -2024-05-15T01:18:20.015Z [INFO] creating new snapshot: path=/tmp/raft129529892/snapshots/2-12-1715735900015.tmp -2024-05-15T01:18:20.058Z [INFO] server-ab914151-958e-8486-f81a-478ffd27a2e1: snapshot complete up to: index=12 -2024-05-15T01:18:20.061Z [INFO] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0 Address:0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0} {Suffrage:Voter ID:server-c0f8db6a-3fae-7cf0-432c-4024f8afb68a Address:c0f8db6a-3fae-7cf0-432c-4024f8afb68a} {Suffrage:Voter ID:server-3f944a66-0ae4-c116-9666-2393763d1069 Address:3f944a66-0ae4-c116-9666-2393763d1069}]" -2024-05-15T01:18:20.061Z [INFO] server-c0f8db6a-3fae-7cf0-432c-4024f8afb68a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0 Address:0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0} {Suffrage:Voter ID:server-c0f8db6a-3fae-7cf0-432c-4024f8afb68a Address:c0f8db6a-3fae-7cf0-432c-4024f8afb68a} {Suffrage:Voter ID:server-3f944a66-0ae4-c116-9666-2393763d1069 Address:3f944a66-0ae4-c116-9666-2393763d1069}]" -2024-05-15T01:18:20.061Z [INFO] server-3f944a66-0ae4-c116-9666-2393763d1069: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0 Address:0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0} {Suffrage:Voter ID:server-c0f8db6a-3fae-7cf0-432c-4024f8afb68a Address:c0f8db6a-3fae-7cf0-432c-4024f8afb68a} {Suffrage:Voter ID:server-3f944a66-0ae4-c116-9666-2393763d1069 Address:3f944a66-0ae4-c116-9666-2393763d1069}]" -2024-05-15T01:18:20.061Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:20.061Z [INFO] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: entering follower state: follower="Node at 0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0 [Follower]" leader-address= leader-id= -2024-05-15T01:18:20.061Z [INFO] server-c0f8db6a-3fae-7cf0-432c-4024f8afb68a: entering follower state: follower="Node at c0f8db6a-3fae-7cf0-432c-4024f8afb68a [Follower]" leader-address= leader-id= -2024-05-15T01:18:20.061Z [INFO] server-3f944a66-0ae4-c116-9666-2393763d1069: entering follower state: follower="Node at 3f944a66-0ae4-c116-9666-2393763d1069 [Follower]" leader-address= leader-id= -2024-05-15T01:18:20.658Z [WARN] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:20.658Z [INFO] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: entering candidate state: node="Node at 0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0 [Candidate]" term=2 -2024-05-15T01:18:20.663Z [INFO] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: election won: term=2 tally=2 -2024-05-15T01:18:20.663Z [INFO] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: entering leader state: leader="Node at 0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0 [Leader]" -2024-05-15T01:18:20.663Z [INFO] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: added peer, starting replication: peer=server-c0f8db6a-3fae-7cf0-432c-4024f8afb68a -2024-05-15T01:18:20.663Z [INFO] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: added peer, starting replication: peer=server-3f944a66-0ae4-c116-9666-2393763d1069 -2024-05-15T01:18:20.664Z [WARN] server-c0f8db6a-3fae-7cf0-432c-4024f8afb68a: rejecting vote request since our last term is greater: candidate=0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0 last-term=2 last-candidate-term=1 -2024-05-15T01:18:20.664Z [INFO] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: pipelining replication: peer="{Voter server-3f944a66-0ae4-c116-9666-2393763d1069 3f944a66-0ae4-c116-9666-2393763d1069}" -2024-05-15T01:18:20.664Z [INFO] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: pipelining replication: peer="{Voter server-c0f8db6a-3fae-7cf0-432c-4024f8afb68a c0f8db6a-3fae-7cf0-432c-4024f8afb68a}" -2024-05-15T01:18:21.670Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:20.664378141 +0000 UTC m=+81.205763028 (1 nodes), 602.82453ms from start of poll, 603.378628ms from cluster start. Timeout at 2024-05-15 01:18:21.670801527 +0000 UTC m=+82.212186289, 1.006423261s after stability -2024-05-15T01:18:21.671Z [INFO] creating new snapshot: path=/tmp/raft1617380047/snapshots/2-1013-1715735901671.tmp -2024-05-15T01:18:21.672Z [INFO] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: copied to local snapshot: bytes=71 -2024-05-15T01:18:21.672Z [INFO] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: snapshot restore progress: id=2-1013-1715735901671 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:21.672Z [INFO] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: removing all old logs from log store -2024-05-15T01:18:21.672Z [INFO] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: compacting logs: from=1 to=2 -2024-05-15T01:18:21.672Z [INFO] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: restored user snapshot: index=1013 -2024-05-15T01:18:21.672Z [ERROR] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: failed to get log: index=2 error="log not found" -2024-05-15T01:18:21.672Z [INFO] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: aborting pipeline replication: peer="{Voter server-3f944a66-0ae4-c116-9666-2393763d1069 3f944a66-0ae4-c116-9666-2393763d1069}" -2024-05-15T01:18:21.672Z [ERROR] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: failed to get log: index=2 error="log not found" -2024-05-15T01:18:21.672Z [INFO] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: aborting pipeline replication: peer="{Voter server-c0f8db6a-3fae-7cf0-432c-4024f8afb68a c0f8db6a-3fae-7cf0-432c-4024f8afb68a}" -2024-05-15T01:18:21.678Z [ERROR] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: failed to get log: index=2 error="log not found" -2024-05-15T01:18:21.679Z [INFO] creating new snapshot: path=/tmp/raft1882661514/snapshots/2-1013-1715735901679.tmp -2024-05-15T01:18:21.679Z [INFO] server-3f944a66-0ae4-c116-9666-2393763d1069: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:21.680Z [INFO] server-3f944a66-0ae4-c116-9666-2393763d1069: copied to local snapshot: bytes=71 -2024-05-15T01:18:21.680Z [INFO] server-3f944a66-0ae4-c116-9666-2393763d1069: snapshot restore progress: id=2-1013-1715735901679 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:21.680Z [INFO] server-3f944a66-0ae4-c116-9666-2393763d1069: removing all old logs from log store -2024-05-15T01:18:21.680Z [INFO] server-3f944a66-0ae4-c116-9666-2393763d1069: compacting logs: from=1 to=2 -2024-05-15T01:18:21.680Z [INFO] server-3f944a66-0ae4-c116-9666-2393763d1069: Installed remote snapshot -2024-05-15T01:18:21.680Z [INFO] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: pipelining replication: peer="{Voter server-3f944a66-0ae4-c116-9666-2393763d1069 3f944a66-0ae4-c116-9666-2393763d1069}" -2024-05-15T01:18:21.680Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:21.682Z [ERROR] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: failed to get log: index=2 error="log not found" -2024-05-15T01:18:21.683Z [INFO] creating new snapshot: path=/tmp/raft1623073777/snapshots/2-1013-1715735901683.tmp -2024-05-15T01:18:21.683Z [INFO] server-c0f8db6a-3fae-7cf0-432c-4024f8afb68a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:21.683Z [INFO] server-c0f8db6a-3fae-7cf0-432c-4024f8afb68a: copied to local snapshot: bytes=71 -2024-05-15T01:18:21.684Z [INFO] server-c0f8db6a-3fae-7cf0-432c-4024f8afb68a: snapshot restore progress: id=2-1013-1715735901683 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:21.684Z [INFO] server-c0f8db6a-3fae-7cf0-432c-4024f8afb68a: removing all old logs from log store -2024-05-15T01:18:21.684Z [INFO] server-c0f8db6a-3fae-7cf0-432c-4024f8afb68a: compacting logs: from=1 to=2 -2024-05-15T01:18:21.684Z [INFO] server-c0f8db6a-3fae-7cf0-432c-4024f8afb68a: Installed remote snapshot -2024-05-15T01:18:21.684Z [INFO] server-0745a5e6-5a76-a03e-cd9d-5b4fe8c841f0: pipelining replication: peer="{Voter server-c0f8db6a-3fae-7cf0-432c-4024f8afb68a c0f8db6a-3fae-7cf0-432c-4024f8afb68a}" -2024-05-15T01:18:22.687Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:21.680877233 +0000 UTC m=+82.222261808 (2 nodes), 2.186µs from start of poll, 1.619877408s from cluster start. Timeout at 2024-05-15 01:18:22.686989185 +0000 UTC m=+83.228374089, 1.006112281s after stability -2024-05-15T01:18:22.700Z [INFO] server-ab914151-958e-8486-f81a-478ffd27a2e1: aborting pipeline replication: peer="{Voter server-71fa0367-ef7a-26b3-4306-64823a64e91e 71fa0367-ef7a-26b3-4306-64823a64e91e}" -2024-05-15T01:18:22.700Z [INFO] server-ab914151-958e-8486-f81a-478ffd27a2e1: aborting pipeline replication: peer="{Voter server-19298f41-10e1-4453-ad21-70473fc3d1ba 19298f41-10e1-4453-ad21-70473fc3d1ba}" +2025-06-17T07:44:10.957Z [INFO] server-1e327592-8f65-20c7-426c-9052b78b9a96: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e327592-8f65-20c7-426c-9052b78b9a96 Address:1e327592-8f65-20c7-426c-9052b78b9a96} {Suffrage:Voter ID:server-509c4420-5097-d888-1293-85be3d010efd Address:509c4420-5097-d888-1293-85be3d010efd} {Suffrage:Voter ID:server-0cc58d09-b1b7-4616-4dc5-4fb3eeaa36fd Address:0cc58d09-b1b7-4616-4dc5-4fb3eeaa36fd}]" +2025-06-17T07:44:10.957Z [INFO] server-1e327592-8f65-20c7-426c-9052b78b9a96: entering follower state: follower="Node at 1e327592-8f65-20c7-426c-9052b78b9a96 [Follower]" leader-address= leader-id= +2025-06-17T07:44:10.957Z [INFO] server-509c4420-5097-d888-1293-85be3d010efd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e327592-8f65-20c7-426c-9052b78b9a96 Address:1e327592-8f65-20c7-426c-9052b78b9a96} {Suffrage:Voter ID:server-509c4420-5097-d888-1293-85be3d010efd Address:509c4420-5097-d888-1293-85be3d010efd} {Suffrage:Voter ID:server-0cc58d09-b1b7-4616-4dc5-4fb3eeaa36fd Address:0cc58d09-b1b7-4616-4dc5-4fb3eeaa36fd}]" +2025-06-17T07:44:10.957Z [INFO] server-509c4420-5097-d888-1293-85be3d010efd: entering follower state: follower="Node at 509c4420-5097-d888-1293-85be3d010efd [Follower]" leader-address= leader-id= +2025-06-17T07:44:10.957Z [INFO] server-0cc58d09-b1b7-4616-4dc5-4fb3eeaa36fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e327592-8f65-20c7-426c-9052b78b9a96 Address:1e327592-8f65-20c7-426c-9052b78b9a96} {Suffrage:Voter ID:server-509c4420-5097-d888-1293-85be3d010efd Address:509c4420-5097-d888-1293-85be3d010efd} {Suffrage:Voter ID:server-0cc58d09-b1b7-4616-4dc5-4fb3eeaa36fd Address:0cc58d09-b1b7-4616-4dc5-4fb3eeaa36fd}]" +2025-06-17T07:44:10.957Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:10.957Z [INFO] server-0cc58d09-b1b7-4616-4dc5-4fb3eeaa36fd: entering follower state: follower="Node at 0cc58d09-b1b7-4616-4dc5-4fb3eeaa36fd [Follower]" leader-address= leader-id= +2025-06-17T07:44:11.482Z [WARN] server-509c4420-5097-d888-1293-85be3d010efd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:11.482Z [INFO] server-509c4420-5097-d888-1293-85be3d010efd: entering candidate state: node="Node at 509c4420-5097-d888-1293-85be3d010efd [Candidate]" term=2 +2025-06-17T07:44:11.482Z [INFO] server-509c4420-5097-d888-1293-85be3d010efd: election won: term=2 tally=2 +2025-06-17T07:44:11.482Z [INFO] server-509c4420-5097-d888-1293-85be3d010efd: entering leader state: leader="Node at 509c4420-5097-d888-1293-85be3d010efd [Leader]" +2025-06-17T07:44:11.482Z [INFO] server-509c4420-5097-d888-1293-85be3d010efd: added peer, starting replication: peer=server-1e327592-8f65-20c7-426c-9052b78b9a96 +2025-06-17T07:44:11.482Z [INFO] server-509c4420-5097-d888-1293-85be3d010efd: added peer, starting replication: peer=server-0cc58d09-b1b7-4616-4dc5-4fb3eeaa36fd +2025-06-17T07:44:11.482Z [INFO] server-509c4420-5097-d888-1293-85be3d010efd: pipelining replication: peer="{Voter server-1e327592-8f65-20c7-426c-9052b78b9a96 1e327592-8f65-20c7-426c-9052b78b9a96}" +2025-06-17T07:44:11.482Z [INFO] server-509c4420-5097-d888-1293-85be3d010efd: pipelining replication: peer="{Voter server-0cc58d09-b1b7-4616-4dc5-4fb3eeaa36fd 0cc58d09-b1b7-4616-4dc5-4fb3eeaa36fd}" +2025-06-17T07:44:12.488Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:11.482827712 +0000 UTC m=+80.086356400 (1 nodes), 525.408825ms from start of poll, 525.721799ms from cluster start. Timeout at 2025-06-17 07:44:12.488708294 +0000 UTC m=+81.092237007, 1.005880607s after stability +2025-06-17T07:44:12.489Z [INFO] server-509c4420-5097-d888-1293-85be3d010efd: starting snapshot up to: index=12 +2025-06-17T07:44:12.489Z [INFO] creating new snapshot: path=/tmp/raft2163706915/snapshots/2-12-1750146252489.tmp +2025-06-17T07:44:12.490Z [INFO] server-509c4420-5097-d888-1293-85be3d010efd: snapshot complete up to: index=12 +2025-06-17T07:44:12.492Z [INFO] server-9d547ce7-b1f4-8bfc-5ec8-a342b46e3619: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d547ce7-b1f4-8bfc-5ec8-a342b46e3619 Address:9d547ce7-b1f4-8bfc-5ec8-a342b46e3619} {Suffrage:Voter ID:server-df172244-0dca-08f4-f57a-c3b4c71be7d3 Address:df172244-0dca-08f4-f57a-c3b4c71be7d3} {Suffrage:Voter ID:server-6d6ab7a5-efac-139e-eb89-d90c2f922ba6 Address:6d6ab7a5-efac-139e-eb89-d90c2f922ba6}]" +2025-06-17T07:44:12.492Z [INFO] server-9d547ce7-b1f4-8bfc-5ec8-a342b46e3619: entering follower state: follower="Node at 9d547ce7-b1f4-8bfc-5ec8-a342b46e3619 [Follower]" leader-address= leader-id= +2025-06-17T07:44:12.493Z [INFO] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d547ce7-b1f4-8bfc-5ec8-a342b46e3619 Address:9d547ce7-b1f4-8bfc-5ec8-a342b46e3619} {Suffrage:Voter ID:server-df172244-0dca-08f4-f57a-c3b4c71be7d3 Address:df172244-0dca-08f4-f57a-c3b4c71be7d3} {Suffrage:Voter ID:server-6d6ab7a5-efac-139e-eb89-d90c2f922ba6 Address:6d6ab7a5-efac-139e-eb89-d90c2f922ba6}]" +2025-06-17T07:44:12.493Z [INFO] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: entering follower state: follower="Node at df172244-0dca-08f4-f57a-c3b4c71be7d3 [Follower]" leader-address= leader-id= +2025-06-17T07:44:12.493Z [INFO] server-6d6ab7a5-efac-139e-eb89-d90c2f922ba6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d547ce7-b1f4-8bfc-5ec8-a342b46e3619 Address:9d547ce7-b1f4-8bfc-5ec8-a342b46e3619} {Suffrage:Voter ID:server-df172244-0dca-08f4-f57a-c3b4c71be7d3 Address:df172244-0dca-08f4-f57a-c3b4c71be7d3} {Suffrage:Voter ID:server-6d6ab7a5-efac-139e-eb89-d90c2f922ba6 Address:6d6ab7a5-efac-139e-eb89-d90c2f922ba6}]" +2025-06-17T07:44:12.493Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:12.493Z [INFO] server-6d6ab7a5-efac-139e-eb89-d90c2f922ba6: entering follower state: follower="Node at 6d6ab7a5-efac-139e-eb89-d90c2f922ba6 [Follower]" leader-address= leader-id= +2025-06-17T07:44:13.052Z [WARN] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:13.052Z [INFO] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: entering candidate state: node="Node at df172244-0dca-08f4-f57a-c3b4c71be7d3 [Candidate]" term=2 +2025-06-17T07:44:13.052Z [INFO] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: election won: term=2 tally=2 +2025-06-17T07:44:13.052Z [INFO] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: entering leader state: leader="Node at df172244-0dca-08f4-f57a-c3b4c71be7d3 [Leader]" +2025-06-17T07:44:13.052Z [INFO] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: added peer, starting replication: peer=server-9d547ce7-b1f4-8bfc-5ec8-a342b46e3619 +2025-06-17T07:44:13.052Z [INFO] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: added peer, starting replication: peer=server-6d6ab7a5-efac-139e-eb89-d90c2f922ba6 +2025-06-17T07:44:13.052Z [INFO] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: pipelining replication: peer="{Voter server-9d547ce7-b1f4-8bfc-5ec8-a342b46e3619 9d547ce7-b1f4-8bfc-5ec8-a342b46e3619}" +2025-06-17T07:44:13.052Z [INFO] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: pipelining replication: peer="{Voter server-6d6ab7a5-efac-139e-eb89-d90c2f922ba6 6d6ab7a5-efac-139e-eb89-d90c2f922ba6}" +2025-06-17T07:44:14.057Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:13.052105914 +0000 UTC m=+81.655634620 (1 nodes), 558.882391ms from start of poll, 559.298192ms from cluster start. Timeout at 2025-06-17 07:44:14.057493993 +0000 UTC m=+82.661022675, 1.005388055s after stability +2025-06-17T07:44:14.057Z [INFO] creating new snapshot: path=/tmp/raft3817555975/snapshots/2-1013-1750146254057.tmp +2025-06-17T07:44:14.058Z [INFO] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: copied to local snapshot: bytes=71 +2025-06-17T07:44:14.058Z [INFO] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: snapshot restore progress: id=2-1013-1750146254057 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:14.058Z [INFO] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: removing all old logs from log store +2025-06-17T07:44:14.058Z [INFO] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: compacting logs: from=1 to=2 +2025-06-17T07:44:14.058Z [INFO] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: restored user snapshot: index=1013 +2025-06-17T07:44:14.058Z [ERROR] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: failed to get log: index=2 error="log not found" +2025-06-17T07:44:14.058Z [INFO] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: aborting pipeline replication: peer="{Voter server-6d6ab7a5-efac-139e-eb89-d90c2f922ba6 6d6ab7a5-efac-139e-eb89-d90c2f922ba6}" +2025-06-17T07:44:14.058Z [ERROR] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: failed to get log: index=2 error="log not found" +2025-06-17T07:44:14.058Z [INFO] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: aborting pipeline replication: peer="{Voter server-9d547ce7-b1f4-8bfc-5ec8-a342b46e3619 9d547ce7-b1f4-8bfc-5ec8-a342b46e3619}" +2025-06-17T07:44:14.064Z [ERROR] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: failed to get log: index=2 error="log not found" +2025-06-17T07:44:14.064Z [INFO] creating new snapshot: path=/tmp/raft2420819214/snapshots/2-1013-1750146254064.tmp +2025-06-17T07:44:14.065Z [INFO] server-6d6ab7a5-efac-139e-eb89-d90c2f922ba6: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:14.065Z [INFO] server-6d6ab7a5-efac-139e-eb89-d90c2f922ba6: copied to local snapshot: bytes=71 +2025-06-17T07:44:14.065Z [INFO] server-6d6ab7a5-efac-139e-eb89-d90c2f922ba6: snapshot restore progress: id=2-1013-1750146254064 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:14.065Z [INFO] server-6d6ab7a5-efac-139e-eb89-d90c2f922ba6: removing all old logs from log store +2025-06-17T07:44:14.065Z [INFO] server-6d6ab7a5-efac-139e-eb89-d90c2f922ba6: compacting logs: from=1 to=2 +2025-06-17T07:44:14.065Z [INFO] server-6d6ab7a5-efac-139e-eb89-d90c2f922ba6: Installed remote snapshot +2025-06-17T07:44:14.065Z [INFO] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: pipelining replication: peer="{Voter server-6d6ab7a5-efac-139e-eb89-d90c2f922ba6 6d6ab7a5-efac-139e-eb89-d90c2f922ba6}" +2025-06-17T07:44:14.065Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:14.067Z [ERROR] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: failed to get log: index=2 error="log not found" +2025-06-17T07:44:14.068Z [INFO] creating new snapshot: path=/tmp/raft3328607785/snapshots/2-1013-1750146254068.tmp +2025-06-17T07:44:14.068Z [INFO] server-9d547ce7-b1f4-8bfc-5ec8-a342b46e3619: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:14.068Z [INFO] server-9d547ce7-b1f4-8bfc-5ec8-a342b46e3619: copied to local snapshot: bytes=71 +2025-06-17T07:44:14.068Z [INFO] server-9d547ce7-b1f4-8bfc-5ec8-a342b46e3619: snapshot restore progress: id=2-1013-1750146254068 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:14.068Z [INFO] server-9d547ce7-b1f4-8bfc-5ec8-a342b46e3619: removing all old logs from log store +2025-06-17T07:44:14.068Z [INFO] server-9d547ce7-b1f4-8bfc-5ec8-a342b46e3619: compacting logs: from=1 to=2 +2025-06-17T07:44:14.068Z [INFO] server-9d547ce7-b1f4-8bfc-5ec8-a342b46e3619: Installed remote snapshot +2025-06-17T07:44:14.068Z [INFO] server-df172244-0dca-08f4-f57a-c3b4c71be7d3: pipelining replication: peer="{Voter server-9d547ce7-b1f4-8bfc-5ec8-a342b46e3619 9d547ce7-b1f4-8bfc-5ec8-a342b46e3619}" +2025-06-17T07:44:15.071Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:14.065625751 +0000 UTC m=+82.669154455 (2 nodes), 679ns from start of poll, 1.572818027s from cluster start. Timeout at 2025-06-17 07:44:15.071018483 +0000 UTC m=+83.674547206, 1.005392751s after stability +2025-06-17T07:44:15.083Z [INFO] server-509c4420-5097-d888-1293-85be3d010efd: aborting pipeline replication: peer="{Voter server-0cc58d09-b1b7-4616-4dc5-4fb3eeaa36fd 0cc58d09-b1b7-4616-4dc5-4fb3eeaa36fd}" +2025-06-17T07:44:15.083Z [INFO] server-509c4420-5097-d888-1293-85be3d010efd: aborting pipeline replication: peer="{Voter server-1e327592-8f65-20c7-426c-9052b78b9a96 1e327592-8f65-20c7-426c-9052b78b9a96}" === RUN TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false -2024-05-15T01:18:22.702Z [INFO] server-d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb Address:d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb} {Suffrage:Voter ID:server-e83a9335-e03a-afa3-cee7-b389b17cf86c Address:e83a9335-e03a-afa3-cee7-b389b17cf86c} {Suffrage:Voter ID:server-ecce60db-e20e-6ba6-d782-281b51387512 Address:ecce60db-e20e-6ba6-d782-281b51387512}]" -2024-05-15T01:18:22.702Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb Address:d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb} {Suffrage:Voter ID:server-e83a9335-e03a-afa3-cee7-b389b17cf86c Address:e83a9335-e03a-afa3-cee7-b389b17cf86c} {Suffrage:Voter ID:server-ecce60db-e20e-6ba6-d782-281b51387512 Address:ecce60db-e20e-6ba6-d782-281b51387512}]" -2024-05-15T01:18:22.702Z [INFO] server-ecce60db-e20e-6ba6-d782-281b51387512: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb Address:d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb} {Suffrage:Voter ID:server-e83a9335-e03a-afa3-cee7-b389b17cf86c Address:e83a9335-e03a-afa3-cee7-b389b17cf86c} {Suffrage:Voter ID:server-ecce60db-e20e-6ba6-d782-281b51387512 Address:ecce60db-e20e-6ba6-d782-281b51387512}]" -2024-05-15T01:18:22.702Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:22.702Z [INFO] server-d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb: entering follower state: follower="Node at d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb [Follower]" leader-address= leader-id= -2024-05-15T01:18:22.702Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: entering follower state: follower="Node at e83a9335-e03a-afa3-cee7-b389b17cf86c [Follower]" leader-address= leader-id= -2024-05-15T01:18:22.702Z [INFO] server-ecce60db-e20e-6ba6-d782-281b51387512: entering follower state: follower="Node at ecce60db-e20e-6ba6-d782-281b51387512 [Follower]" leader-address= leader-id= -2024-05-15T01:18:23.208Z [WARN] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:23.208Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: entering candidate state: node="Node at e83a9335-e03a-afa3-cee7-b389b17cf86c [Candidate]" term=2 -2024-05-15T01:18:23.208Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: election won: term=2 tally=2 -2024-05-15T01:18:23.208Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: entering leader state: leader="Node at e83a9335-e03a-afa3-cee7-b389b17cf86c [Leader]" -2024-05-15T01:18:23.208Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: added peer, starting replication: peer=server-d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb -2024-05-15T01:18:23.208Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: added peer, starting replication: peer=server-ecce60db-e20e-6ba6-d782-281b51387512 -2024-05-15T01:18:23.208Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: pipelining replication: peer="{Voter server-ecce60db-e20e-6ba6-d782-281b51387512 ecce60db-e20e-6ba6-d782-281b51387512}" -2024-05-15T01:18:23.209Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: pipelining replication: peer="{Voter server-d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb}" -2024-05-15T01:18:24.214Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:23.209348816 +0000 UTC m=+83.750733688 (1 nodes), 506.72063ms from start of poll, 507.28526ms from cluster start. Timeout at 2024-05-15 01:18:24.214362892 +0000 UTC m=+84.755747637, 1.005013949s after stability -2024-05-15T01:18:24.216Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: starting snapshot up to: index=12 -2024-05-15T01:18:24.216Z [INFO] creating new snapshot: path=/tmp/raft3717183701/snapshots/2-12-1715735904216.tmp -2024-05-15T01:18:24.216Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: snapshot complete up to: index=12 -2024-05-15T01:18:24.225Z [INFO] creating new snapshot: path=/tmp/raft3717183701/snapshots/2-10013-1715735904225.tmp -2024-05-15T01:18:24.226Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: copied to local snapshot: bytes=71 -2024-05-15T01:18:24.227Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: snapshot restore progress: id=2-10013-1715735904225 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:24.227Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: restored user snapshot: index=10013 -2024-05-15T01:18:24.227Z [ERROR] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: failed to get log: index=23 error="log not found" -2024-05-15T01:18:24.227Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: aborting pipeline replication: peer="{Voter server-ecce60db-e20e-6ba6-d782-281b51387512 ecce60db-e20e-6ba6-d782-281b51387512}" -2024-05-15T01:18:24.227Z [ERROR] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: failed to get log: index=23 error="log not found" -2024-05-15T01:18:24.227Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: aborting pipeline replication: peer="{Voter server-d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb}" -2024-05-15T01:18:24.232Z [ERROR] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: failed to get log: index=23 error="log not found" -2024-05-15T01:18:24.233Z [INFO] creating new snapshot: path=/tmp/raft2334881953/snapshots/2-10013-1715735904233.tmp -2024-05-15T01:18:24.233Z [INFO] server-ecce60db-e20e-6ba6-d782-281b51387512: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:24.234Z [INFO] server-ecce60db-e20e-6ba6-d782-281b51387512: copied to local snapshot: bytes=71 -2024-05-15T01:18:24.234Z [INFO] server-ecce60db-e20e-6ba6-d782-281b51387512: snapshot restore progress: id=2-10013-1715735904233 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:24.234Z [INFO] server-ecce60db-e20e-6ba6-d782-281b51387512: Installed remote snapshot -2024-05-15T01:18:24.234Z [WARN] server-ecce60db-e20e-6ba6-d782-281b51387512: skipping application of old log: index=22 -2024-05-15T01:18:24.234Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: pipelining replication: peer="{Voter server-ecce60db-e20e-6ba6-d782-281b51387512 ecce60db-e20e-6ba6-d782-281b51387512}" -2024-05-15T01:18:24.235Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:24.236Z [ERROR] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: failed to get log: index=23 error="log not found" -2024-05-15T01:18:24.236Z [INFO] creating new snapshot: path=/tmp/raft2973695378/snapshots/2-10013-1715735904236.tmp -2024-05-15T01:18:24.237Z [INFO] server-d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:24.237Z [INFO] server-d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb: copied to local snapshot: bytes=71 -2024-05-15T01:18:24.237Z [INFO] server-d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb: snapshot restore progress: id=2-10013-1715735904236 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:24.237Z [INFO] server-d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb: Installed remote snapshot -2024-05-15T01:18:24.238Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: pipelining replication: peer="{Voter server-d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb}" -2024-05-15T01:18:25.241Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:24.2350475 +0000 UTC m=+84.776432330 (2 nodes), 2.232µs from start of poll, 1.532983902s from cluster start. Timeout at 2024-05-15 01:18:25.240929034 +0000 UTC m=+85.782313898, 1.005881568s after stability -2024-05-15T01:18:25.257Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: aborting pipeline replication: peer="{Voter server-ecce60db-e20e-6ba6-d782-281b51387512 ecce60db-e20e-6ba6-d782-281b51387512}" -2024-05-15T01:18:25.257Z [INFO] server-e83a9335-e03a-afa3-cee7-b389b17cf86c: aborting pipeline replication: peer="{Voter server-d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb d6c1b173-4df0-aaba-0b73-e1c4cf7c83fb}" +2025-06-17T07:44:15.084Z [INFO] server-bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d Address:bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d} {Suffrage:Voter ID:server-46b78c5a-b98b-3891-a32b-98151f959491 Address:46b78c5a-b98b-3891-a32b-98151f959491} {Suffrage:Voter ID:server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50 Address:44c803b6-aa4d-7ce3-6fe1-45986ffc9a50}]" +2025-06-17T07:44:15.084Z [INFO] server-bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d: entering follower state: follower="Node at bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d [Follower]" leader-address= leader-id= +2025-06-17T07:44:15.084Z [INFO] server-46b78c5a-b98b-3891-a32b-98151f959491: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d Address:bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d} {Suffrage:Voter ID:server-46b78c5a-b98b-3891-a32b-98151f959491 Address:46b78c5a-b98b-3891-a32b-98151f959491} {Suffrage:Voter ID:server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50 Address:44c803b6-aa4d-7ce3-6fe1-45986ffc9a50}]" +2025-06-17T07:44:15.085Z [INFO] server-46b78c5a-b98b-3891-a32b-98151f959491: entering follower state: follower="Node at 46b78c5a-b98b-3891-a32b-98151f959491 [Follower]" leader-address= leader-id= +2025-06-17T07:44:15.085Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d Address:bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d} {Suffrage:Voter ID:server-46b78c5a-b98b-3891-a32b-98151f959491 Address:46b78c5a-b98b-3891-a32b-98151f959491} {Suffrage:Voter ID:server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50 Address:44c803b6-aa4d-7ce3-6fe1-45986ffc9a50}]" +2025-06-17T07:44:15.085Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:15.085Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: entering follower state: follower="Node at 44c803b6-aa4d-7ce3-6fe1-45986ffc9a50 [Follower]" leader-address= leader-id= +2025-06-17T07:44:15.645Z [WARN] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:15.645Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: entering candidate state: node="Node at 44c803b6-aa4d-7ce3-6fe1-45986ffc9a50 [Candidate]" term=2 +2025-06-17T07:44:15.645Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: election won: term=2 tally=2 +2025-06-17T07:44:15.645Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: entering leader state: leader="Node at 44c803b6-aa4d-7ce3-6fe1-45986ffc9a50 [Leader]" +2025-06-17T07:44:15.645Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: added peer, starting replication: peer=server-bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d +2025-06-17T07:44:15.645Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: added peer, starting replication: peer=server-46b78c5a-b98b-3891-a32b-98151f959491 +2025-06-17T07:44:15.646Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: pipelining replication: peer="{Voter server-bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d}" +2025-06-17T07:44:15.646Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: pipelining replication: peer="{Voter server-46b78c5a-b98b-3891-a32b-98151f959491 46b78c5a-b98b-3891-a32b-98151f959491}" +2025-06-17T07:44:16.650Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:15.645908772 +0000 UTC m=+84.249437472 (1 nodes), 560.821148ms from start of poll, 561.143856ms from cluster start. Timeout at 2025-06-17 07:44:16.650953936 +0000 UTC m=+85.254482618, 1.005045146s after stability +2025-06-17T07:44:16.651Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: starting snapshot up to: index=12 +2025-06-17T07:44:16.651Z [INFO] creating new snapshot: path=/tmp/raft2242795068/snapshots/2-12-1750146256651.tmp +2025-06-17T07:44:16.651Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: snapshot complete up to: index=12 +2025-06-17T07:44:16.652Z [INFO] creating new snapshot: path=/tmp/raft2242795068/snapshots/2-10013-1750146256652.tmp +2025-06-17T07:44:16.653Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: copied to local snapshot: bytes=71 +2025-06-17T07:44:16.653Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: snapshot restore progress: id=2-10013-1750146256652 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:16.653Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: restored user snapshot: index=10013 +2025-06-17T07:44:16.653Z [ERROR] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: failed to get log: index=23 error="log not found" +2025-06-17T07:44:16.653Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: aborting pipeline replication: peer="{Voter server-bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d}" +2025-06-17T07:44:16.653Z [ERROR] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: failed to get log: index=23 error="log not found" +2025-06-17T07:44:16.653Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: aborting pipeline replication: peer="{Voter server-46b78c5a-b98b-3891-a32b-98151f959491 46b78c5a-b98b-3891-a32b-98151f959491}" +2025-06-17T07:44:16.659Z [ERROR] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: failed to get log: index=23 error="log not found" +2025-06-17T07:44:16.659Z [INFO] creating new snapshot: path=/tmp/raft2353938283/snapshots/2-10013-1750146256659.tmp +2025-06-17T07:44:16.659Z [INFO] server-bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:16.659Z [INFO] server-bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d: copied to local snapshot: bytes=71 +2025-06-17T07:44:16.660Z [INFO] server-bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d: snapshot restore progress: id=2-10013-1750146256659 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:16.660Z [INFO] server-bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d: Installed remote snapshot +2025-06-17T07:44:16.660Z [WARN] server-bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d: skipping application of old log: index=22 +2025-06-17T07:44:16.660Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: pipelining replication: peer="{Voter server-bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d}" +2025-06-17T07:44:16.660Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:16.662Z [ERROR] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: failed to get log: index=23 error="log not found" +2025-06-17T07:44:16.662Z [INFO] creating new snapshot: path=/tmp/raft714370951/snapshots/2-10013-1750146256662.tmp +2025-06-17T07:44:16.663Z [INFO] server-46b78c5a-b98b-3891-a32b-98151f959491: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:16.663Z [INFO] server-46b78c5a-b98b-3891-a32b-98151f959491: copied to local snapshot: bytes=71 +2025-06-17T07:44:16.663Z [INFO] server-46b78c5a-b98b-3891-a32b-98151f959491: snapshot restore progress: id=2-10013-1750146256662 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:16.663Z [INFO] server-46b78c5a-b98b-3891-a32b-98151f959491: Installed remote snapshot +2025-06-17T07:44:16.663Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: pipelining replication: peer="{Voter server-46b78c5a-b98b-3891-a32b-98151f959491 46b78c5a-b98b-3891-a32b-98151f959491}" +2025-06-17T07:44:17.665Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:16.660220092 +0000 UTC m=+85.263748802 (2 nodes), 902ns from start of poll, 1.575455186s from cluster start. Timeout at 2025-06-17 07:44:17.665751932 +0000 UTC m=+86.269280647, 1.005531845s after stability +2025-06-17T07:44:17.677Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: aborting pipeline replication: peer="{Voter server-46b78c5a-b98b-3891-a32b-98151f959491 46b78c5a-b98b-3891-a32b-98151f959491}" +2025-06-17T07:44:17.677Z [INFO] server-44c803b6-aa4d-7ce3-6fe1-45986ffc9a50: aborting pipeline replication: peer="{Voter server-bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d bc4d7ae5-a931-2fcd-0072-6cd5c02dcb7d}" === RUN TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false -2024-05-15T01:18:25.261Z [INFO] server-d286d66f-a465-b0fd-d65e-ee0ca865ea6d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d286d66f-a465-b0fd-d65e-ee0ca865ea6d Address:d286d66f-a465-b0fd-d65e-ee0ca865ea6d} {Suffrage:Voter ID:server-66655477-a249-5b79-f2b4-8ae6fc4bb348 Address:66655477-a249-5b79-f2b4-8ae6fc4bb348} {Suffrage:Voter ID:server-34b9990c-7836-7c0c-f351-02792cc79a03 Address:34b9990c-7836-7c0c-f351-02792cc79a03}]" -2024-05-15T01:18:25.261Z [INFO] server-d286d66f-a465-b0fd-d65e-ee0ca865ea6d: entering follower state: follower="Node at d286d66f-a465-b0fd-d65e-ee0ca865ea6d [Follower]" leader-address= leader-id= -2024-05-15T01:18:25.261Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d286d66f-a465-b0fd-d65e-ee0ca865ea6d Address:d286d66f-a465-b0fd-d65e-ee0ca865ea6d} {Suffrage:Voter ID:server-66655477-a249-5b79-f2b4-8ae6fc4bb348 Address:66655477-a249-5b79-f2b4-8ae6fc4bb348} {Suffrage:Voter ID:server-34b9990c-7836-7c0c-f351-02792cc79a03 Address:34b9990c-7836-7c0c-f351-02792cc79a03}]" -2024-05-15T01:18:25.261Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: entering follower state: follower="Node at 66655477-a249-5b79-f2b4-8ae6fc4bb348 [Follower]" leader-address= leader-id= -2024-05-15T01:18:25.262Z [INFO] server-34b9990c-7836-7c0c-f351-02792cc79a03: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d286d66f-a465-b0fd-d65e-ee0ca865ea6d Address:d286d66f-a465-b0fd-d65e-ee0ca865ea6d} {Suffrage:Voter ID:server-66655477-a249-5b79-f2b4-8ae6fc4bb348 Address:66655477-a249-5b79-f2b4-8ae6fc4bb348} {Suffrage:Voter ID:server-34b9990c-7836-7c0c-f351-02792cc79a03 Address:34b9990c-7836-7c0c-f351-02792cc79a03}]" -2024-05-15T01:18:25.262Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:25.262Z [INFO] server-34b9990c-7836-7c0c-f351-02792cc79a03: entering follower state: follower="Node at 34b9990c-7836-7c0c-f351-02792cc79a03 [Follower]" leader-address= leader-id= -2024-05-15T01:18:26.040Z [WARN] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:26.041Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: entering candidate state: node="Node at 66655477-a249-5b79-f2b4-8ae6fc4bb348 [Candidate]" term=2 -2024-05-15T01:18:26.041Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: election won: term=2 tally=2 -2024-05-15T01:18:26.041Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: entering leader state: leader="Node at 66655477-a249-5b79-f2b4-8ae6fc4bb348 [Leader]" -2024-05-15T01:18:26.041Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: added peer, starting replication: peer=server-d286d66f-a465-b0fd-d65e-ee0ca865ea6d -2024-05-15T01:18:26.041Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: added peer, starting replication: peer=server-34b9990c-7836-7c0c-f351-02792cc79a03 -2024-05-15T01:18:26.041Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: pipelining replication: peer="{Voter server-34b9990c-7836-7c0c-f351-02792cc79a03 34b9990c-7836-7c0c-f351-02792cc79a03}" -2024-05-15T01:18:26.041Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: pipelining replication: peer="{Voter server-d286d66f-a465-b0fd-d65e-ee0ca865ea6d d286d66f-a465-b0fd-d65e-ee0ca865ea6d}" -2024-05-15T01:18:27.047Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:26.041769875 +0000 UTC m=+86.583154635 (1 nodes), 779.578287ms from start of poll, 780.202234ms from cluster start. Timeout at 2024-05-15 01:18:27.047050931 +0000 UTC m=+87.588435670, 1.005281035s after stability -2024-05-15T01:18:27.048Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: starting snapshot up to: index=12 -2024-05-15T01:18:27.048Z [INFO] creating new snapshot: path=/tmp/raft3011448899/snapshots/2-12-1715735907048.tmp -2024-05-15T01:18:27.050Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: snapshot complete up to: index=12 -2024-05-15T01:18:27.052Z [INFO] creating new snapshot: path=/tmp/raft3011448899/snapshots/2-10013-1715735907052.tmp -2024-05-15T01:18:27.053Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: copied to local snapshot: bytes=71 -2024-05-15T01:18:27.053Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: snapshot restore progress: id=2-10013-1715735907052 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:27.053Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: removing all old logs from log store -2024-05-15T01:18:27.053Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: compacting logs: from=1 to=22 -2024-05-15T01:18:27.053Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: restored user snapshot: index=10013 -2024-05-15T01:18:27.053Z [ERROR] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: failed to get log: index=22 error="log not found" -2024-05-15T01:18:27.053Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: aborting pipeline replication: peer="{Voter server-34b9990c-7836-7c0c-f351-02792cc79a03 34b9990c-7836-7c0c-f351-02792cc79a03}" -2024-05-15T01:18:27.053Z [ERROR] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: failed to get log: index=22 error="log not found" -2024-05-15T01:18:27.053Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: aborting pipeline replication: peer="{Voter server-d286d66f-a465-b0fd-d65e-ee0ca865ea6d d286d66f-a465-b0fd-d65e-ee0ca865ea6d}" -2024-05-15T01:18:27.061Z [ERROR] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: failed to get log: index=22 error="log not found" -2024-05-15T01:18:27.061Z [INFO] creating new snapshot: path=/tmp/raft1066019701/snapshots/2-10013-1715735907061.tmp -2024-05-15T01:18:27.062Z [INFO] server-34b9990c-7836-7c0c-f351-02792cc79a03: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:27.062Z [INFO] server-34b9990c-7836-7c0c-f351-02792cc79a03: copied to local snapshot: bytes=71 -2024-05-15T01:18:27.062Z [INFO] server-34b9990c-7836-7c0c-f351-02792cc79a03: snapshot restore progress: id=2-10013-1715735907061 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:27.062Z [INFO] server-34b9990c-7836-7c0c-f351-02792cc79a03: removing all old logs from log store -2024-05-15T01:18:27.062Z [INFO] server-34b9990c-7836-7c0c-f351-02792cc79a03: compacting logs: from=1 to=22 -2024-05-15T01:18:27.063Z [INFO] server-34b9990c-7836-7c0c-f351-02792cc79a03: Installed remote snapshot -2024-05-15T01:18:27.063Z [ERROR] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: failed to get log: index=22 error="log not found" -2024-05-15T01:18:27.063Z [WARN] server-34b9990c-7836-7c0c-f351-02792cc79a03: skipping application of old log: index=22 -2024-05-15T01:18:27.063Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: pipelining replication: peer="{Voter server-34b9990c-7836-7c0c-f351-02792cc79a03 34b9990c-7836-7c0c-f351-02792cc79a03}" -2024-05-15T01:18:27.063Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:27.063Z [INFO] creating new snapshot: path=/tmp/raft523463487/snapshots/2-10013-1715735907063.tmp -2024-05-15T01:18:27.063Z [INFO] server-d286d66f-a465-b0fd-d65e-ee0ca865ea6d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:27.064Z [INFO] server-d286d66f-a465-b0fd-d65e-ee0ca865ea6d: copied to local snapshot: bytes=71 -2024-05-15T01:18:27.064Z [INFO] server-d286d66f-a465-b0fd-d65e-ee0ca865ea6d: snapshot restore progress: id=2-10013-1715735907063 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:27.064Z [INFO] server-d286d66f-a465-b0fd-d65e-ee0ca865ea6d: removing all old logs from log store -2024-05-15T01:18:27.064Z [INFO] server-d286d66f-a465-b0fd-d65e-ee0ca865ea6d: compacting logs: from=1 to=22 -2024-05-15T01:18:27.064Z [INFO] server-d286d66f-a465-b0fd-d65e-ee0ca865ea6d: Installed remote snapshot -2024-05-15T01:18:27.064Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: pipelining replication: peer="{Voter server-d286d66f-a465-b0fd-d65e-ee0ca865ea6d d286d66f-a465-b0fd-d65e-ee0ca865ea6d}" -2024-05-15T01:18:28.068Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:27.063297939 +0000 UTC m=+87.604682663 (2 nodes), 1.982µs from start of poll, 1.801730262s from cluster start. Timeout at 2024-05-15 01:18:28.068547773 +0000 UTC m=+88.609932628, 1.005249965s after stability -2024-05-15T01:18:28.081Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: aborting pipeline replication: peer="{Voter server-d286d66f-a465-b0fd-d65e-ee0ca865ea6d d286d66f-a465-b0fd-d65e-ee0ca865ea6d}" -2024-05-15T01:18:28.081Z [INFO] server-66655477-a249-5b79-f2b4-8ae6fc4bb348: aborting pipeline replication: peer="{Voter server-34b9990c-7836-7c0c-f351-02792cc79a03 34b9990c-7836-7c0c-f351-02792cc79a03}" +2025-06-17T07:44:17.678Z [INFO] server-878c913a-dfa3-88d1-0441-1569f7830610: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-878c913a-dfa3-88d1-0441-1569f7830610 Address:878c913a-dfa3-88d1-0441-1569f7830610} {Suffrage:Voter ID:server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0 Address:58be48c0-4de6-eb2b-d0a4-5c8cab8549b0} {Suffrage:Voter ID:server-dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4 Address:dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4}]" +2025-06-17T07:44:17.678Z [INFO] server-878c913a-dfa3-88d1-0441-1569f7830610: entering follower state: follower="Node at 878c913a-dfa3-88d1-0441-1569f7830610 [Follower]" leader-address= leader-id= +2025-06-17T07:44:17.678Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-878c913a-dfa3-88d1-0441-1569f7830610 Address:878c913a-dfa3-88d1-0441-1569f7830610} {Suffrage:Voter ID:server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0 Address:58be48c0-4de6-eb2b-d0a4-5c8cab8549b0} {Suffrage:Voter ID:server-dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4 Address:dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4}]" +2025-06-17T07:44:17.678Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: entering follower state: follower="Node at 58be48c0-4de6-eb2b-d0a4-5c8cab8549b0 [Follower]" leader-address= leader-id= +2025-06-17T07:44:17.678Z [INFO] server-dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-878c913a-dfa3-88d1-0441-1569f7830610 Address:878c913a-dfa3-88d1-0441-1569f7830610} {Suffrage:Voter ID:server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0 Address:58be48c0-4de6-eb2b-d0a4-5c8cab8549b0} {Suffrage:Voter ID:server-dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4 Address:dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4}]" +2025-06-17T07:44:17.678Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:17.678Z [INFO] server-dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4: entering follower state: follower="Node at dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4 [Follower]" leader-address= leader-id= +2025-06-17T07:44:18.247Z [WARN] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:18.247Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: entering candidate state: node="Node at 58be48c0-4de6-eb2b-d0a4-5c8cab8549b0 [Candidate]" term=2 +2025-06-17T07:44:18.247Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: election won: term=2 tally=2 +2025-06-17T07:44:18.247Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: entering leader state: leader="Node at 58be48c0-4de6-eb2b-d0a4-5c8cab8549b0 [Leader]" +2025-06-17T07:44:18.247Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: added peer, starting replication: peer=server-878c913a-dfa3-88d1-0441-1569f7830610 +2025-06-17T07:44:18.247Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: added peer, starting replication: peer=server-dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4 +2025-06-17T07:44:18.247Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: pipelining replication: peer="{Voter server-878c913a-dfa3-88d1-0441-1569f7830610 878c913a-dfa3-88d1-0441-1569f7830610}" +2025-06-17T07:44:18.247Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: pipelining replication: peer="{Voter server-dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4 dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4}" +2025-06-17T07:44:19.252Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:18.24724158 +0000 UTC m=+86.850770281 (1 nodes), 568.613702ms from start of poll, 568.919224ms from cluster start. Timeout at 2025-06-17 07:44:19.252613398 +0000 UTC m=+87.856142082, 1.005371801s after stability +2025-06-17T07:44:19.253Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: starting snapshot up to: index=12 +2025-06-17T07:44:19.253Z [INFO] creating new snapshot: path=/tmp/raft3474258787/snapshots/2-12-1750146259253.tmp +2025-06-17T07:44:19.254Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: snapshot complete up to: index=12 +2025-06-17T07:44:19.255Z [INFO] creating new snapshot: path=/tmp/raft3474258787/snapshots/2-10013-1750146259255.tmp +2025-06-17T07:44:19.256Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: copied to local snapshot: bytes=71 +2025-06-17T07:44:19.256Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: snapshot restore progress: id=2-10013-1750146259255 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:19.256Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: removing all old logs from log store +2025-06-17T07:44:19.256Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: compacting logs: from=1 to=22 +2025-06-17T07:44:19.256Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: restored user snapshot: index=10013 +2025-06-17T07:44:19.256Z [ERROR] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: failed to get log: index=22 error="log not found" +2025-06-17T07:44:19.256Z [ERROR] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: failed to get log: index=22 error="log not found" +2025-06-17T07:44:19.256Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: aborting pipeline replication: peer="{Voter server-878c913a-dfa3-88d1-0441-1569f7830610 878c913a-dfa3-88d1-0441-1569f7830610}" +2025-06-17T07:44:19.256Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: aborting pipeline replication: peer="{Voter server-dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4 dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4}" +2025-06-17T07:44:19.262Z [ERROR] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: failed to get log: index=22 error="log not found" +2025-06-17T07:44:19.263Z [INFO] creating new snapshot: path=/tmp/raft575825605/snapshots/2-10013-1750146259263.tmp +2025-06-17T07:44:19.263Z [ERROR] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: failed to get log: index=22 error="log not found" +2025-06-17T07:44:19.263Z [INFO] server-dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:19.263Z [INFO] creating new snapshot: path=/tmp/raft1714457958/snapshots/2-10013-1750146259263.tmp +2025-06-17T07:44:19.263Z [INFO] server-dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4: copied to local snapshot: bytes=71 +2025-06-17T07:44:19.263Z [INFO] server-878c913a-dfa3-88d1-0441-1569f7830610: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:19.264Z [INFO] server-dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4: snapshot restore progress: id=2-10013-1750146259263 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:19.264Z [INFO] server-dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4: removing all old logs from log store +2025-06-17T07:44:19.264Z [INFO] server-dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4: compacting logs: from=1 to=22 +2025-06-17T07:44:19.264Z [INFO] server-dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4: Installed remote snapshot +2025-06-17T07:44:19.264Z [INFO] server-878c913a-dfa3-88d1-0441-1569f7830610: copied to local snapshot: bytes=71 +2025-06-17T07:44:19.264Z [WARN] server-dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4: skipping application of old log: index=22 +2025-06-17T07:44:19.264Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: pipelining replication: peer="{Voter server-dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4 dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4}" +2025-06-17T07:44:19.264Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:19.264Z [INFO] server-878c913a-dfa3-88d1-0441-1569f7830610: snapshot restore progress: id=2-10013-1750146259263 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:19.264Z [INFO] server-878c913a-dfa3-88d1-0441-1569f7830610: removing all old logs from log store +2025-06-17T07:44:19.264Z [INFO] server-878c913a-dfa3-88d1-0441-1569f7830610: compacting logs: from=1 to=22 +2025-06-17T07:44:19.264Z [INFO] server-878c913a-dfa3-88d1-0441-1569f7830610: Installed remote snapshot +2025-06-17T07:44:19.264Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: pipelining replication: peer="{Voter server-878c913a-dfa3-88d1-0441-1569f7830610 878c913a-dfa3-88d1-0441-1569f7830610}" +2025-06-17T07:44:20.270Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:19.264396258 +0000 UTC m=+87.867924972 (2 nodes), 1.044µs from start of poll, 1.586073915s from cluster start. Timeout at 2025-06-17 07:44:20.270180635 +0000 UTC m=+88.873709317, 1.005784345s after stability +2025-06-17T07:44:20.282Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: aborting pipeline replication: peer="{Voter server-dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4 dbf4d11f-ae6f-0597-8ccb-9c660fd5adc4}" +2025-06-17T07:44:20.282Z [INFO] server-58be48c0-4de6-eb2b-d0a4-5c8cab8549b0: aborting pipeline replication: peer="{Voter server-878c913a-dfa3-88d1-0441-1569f7830610 878c913a-dfa3-88d1-0441-1569f7830610}" === RUN TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true -2024-05-15T01:18:28.084Z [INFO] server-64fb5fcc-8aee-72d9-31a2-abec770f814e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-64fb5fcc-8aee-72d9-31a2-abec770f814e Address:64fb5fcc-8aee-72d9-31a2-abec770f814e} {Suffrage:Voter ID:server-665b0344-6127-0468-e3c8-8b9ea8d8c432 Address:665b0344-6127-0468-e3c8-8b9ea8d8c432} {Suffrage:Voter ID:server-2d2c6007-f6e9-91b7-47f4-54317f25a40d Address:2d2c6007-f6e9-91b7-47f4-54317f25a40d}]" -2024-05-15T01:18:28.084Z [INFO] server-665b0344-6127-0468-e3c8-8b9ea8d8c432: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-64fb5fcc-8aee-72d9-31a2-abec770f814e Address:64fb5fcc-8aee-72d9-31a2-abec770f814e} {Suffrage:Voter ID:server-665b0344-6127-0468-e3c8-8b9ea8d8c432 Address:665b0344-6127-0468-e3c8-8b9ea8d8c432} {Suffrage:Voter ID:server-2d2c6007-f6e9-91b7-47f4-54317f25a40d Address:2d2c6007-f6e9-91b7-47f4-54317f25a40d}]" -2024-05-15T01:18:28.084Z [INFO] server-2d2c6007-f6e9-91b7-47f4-54317f25a40d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-64fb5fcc-8aee-72d9-31a2-abec770f814e Address:64fb5fcc-8aee-72d9-31a2-abec770f814e} {Suffrage:Voter ID:server-665b0344-6127-0468-e3c8-8b9ea8d8c432 Address:665b0344-6127-0468-e3c8-8b9ea8d8c432} {Suffrage:Voter ID:server-2d2c6007-f6e9-91b7-47f4-54317f25a40d Address:2d2c6007-f6e9-91b7-47f4-54317f25a40d}]" -2024-05-15T01:18:28.084Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:28.084Z [INFO] server-665b0344-6127-0468-e3c8-8b9ea8d8c432: entering follower state: follower="Node at 665b0344-6127-0468-e3c8-8b9ea8d8c432 [Follower]" leader-address= leader-id= -2024-05-15T01:18:28.084Z [INFO] server-64fb5fcc-8aee-72d9-31a2-abec770f814e: entering follower state: follower="Node at 64fb5fcc-8aee-72d9-31a2-abec770f814e [Follower]" leader-address= leader-id= -2024-05-15T01:18:28.084Z [INFO] server-2d2c6007-f6e9-91b7-47f4-54317f25a40d: entering follower state: follower="Node at 2d2c6007-f6e9-91b7-47f4-54317f25a40d [Follower]" leader-address= leader-id= -2024-05-15T01:18:28.677Z [WARN] server-64fb5fcc-8aee-72d9-31a2-abec770f814e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:28.677Z [INFO] server-64fb5fcc-8aee-72d9-31a2-abec770f814e: entering candidate state: node="Node at 64fb5fcc-8aee-72d9-31a2-abec770f814e [Candidate]" term=2 -2024-05-15T01:18:28.677Z [INFO] server-64fb5fcc-8aee-72d9-31a2-abec770f814e: election won: term=2 tally=2 -2024-05-15T01:18:28.677Z [INFO] server-64fb5fcc-8aee-72d9-31a2-abec770f814e: entering leader state: leader="Node at 64fb5fcc-8aee-72d9-31a2-abec770f814e [Leader]" -2024-05-15T01:18:28.677Z [INFO] server-64fb5fcc-8aee-72d9-31a2-abec770f814e: added peer, starting replication: peer=server-665b0344-6127-0468-e3c8-8b9ea8d8c432 -2024-05-15T01:18:28.677Z [INFO] server-64fb5fcc-8aee-72d9-31a2-abec770f814e: added peer, starting replication: peer=server-2d2c6007-f6e9-91b7-47f4-54317f25a40d -2024-05-15T01:18:28.677Z [INFO] server-64fb5fcc-8aee-72d9-31a2-abec770f814e: pipelining replication: peer="{Voter server-2d2c6007-f6e9-91b7-47f4-54317f25a40d 2d2c6007-f6e9-91b7-47f4-54317f25a40d}" -2024-05-15T01:18:28.677Z [INFO] server-64fb5fcc-8aee-72d9-31a2-abec770f814e: pipelining replication: peer="{Voter server-665b0344-6127-0468-e3c8-8b9ea8d8c432 665b0344-6127-0468-e3c8-8b9ea8d8c432}" -2024-05-15T01:18:29.683Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:28.677966683 +0000 UTC m=+89.219351448 (1 nodes), 593.309264ms from start of poll, 593.85165ms from cluster start. Timeout at 2024-05-15 01:18:29.6835547 +0000 UTC m=+90.224939588, 1.00558814s after stability -2024-05-15T01:18:29.696Z [INFO] server-64fb5fcc-8aee-72d9-31a2-abec770f814e: starting snapshot up to: index=12 -2024-05-15T01:18:29.696Z [INFO] creating new snapshot: path=/tmp/raft1738969169/snapshots/2-12-1715735909696.tmp -2024-05-15T01:18:29.697Z [INFO] server-64fb5fcc-8aee-72d9-31a2-abec770f814e: snapshot complete up to: index=12 -2024-05-15T01:18:29.700Z [INFO] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6f34380-2810-120d-e9ba-61d8d7951ccd Address:b6f34380-2810-120d-e9ba-61d8d7951ccd} {Suffrage:Voter ID:server-096a1352-632e-8e5b-fa30-8fa75e426f3e Address:096a1352-632e-8e5b-fa30-8fa75e426f3e} {Suffrage:Voter ID:server-66f691cf-4af3-844a-4f5d-9f9a7eb70c10 Address:66f691cf-4af3-844a-4f5d-9f9a7eb70c10}]" -2024-05-15T01:18:29.700Z [INFO] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: entering follower state: follower="Node at b6f34380-2810-120d-e9ba-61d8d7951ccd [Follower]" leader-address= leader-id= -2024-05-15T01:18:29.700Z [INFO] server-096a1352-632e-8e5b-fa30-8fa75e426f3e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6f34380-2810-120d-e9ba-61d8d7951ccd Address:b6f34380-2810-120d-e9ba-61d8d7951ccd} {Suffrage:Voter ID:server-096a1352-632e-8e5b-fa30-8fa75e426f3e Address:096a1352-632e-8e5b-fa30-8fa75e426f3e} {Suffrage:Voter ID:server-66f691cf-4af3-844a-4f5d-9f9a7eb70c10 Address:66f691cf-4af3-844a-4f5d-9f9a7eb70c10}]" -2024-05-15T01:18:29.700Z [INFO] server-096a1352-632e-8e5b-fa30-8fa75e426f3e: entering follower state: follower="Node at 096a1352-632e-8e5b-fa30-8fa75e426f3e [Follower]" leader-address= leader-id= -2024-05-15T01:18:29.700Z [INFO] server-66f691cf-4af3-844a-4f5d-9f9a7eb70c10: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6f34380-2810-120d-e9ba-61d8d7951ccd Address:b6f34380-2810-120d-e9ba-61d8d7951ccd} {Suffrage:Voter ID:server-096a1352-632e-8e5b-fa30-8fa75e426f3e Address:096a1352-632e-8e5b-fa30-8fa75e426f3e} {Suffrage:Voter ID:server-66f691cf-4af3-844a-4f5d-9f9a7eb70c10 Address:66f691cf-4af3-844a-4f5d-9f9a7eb70c10}]" -2024-05-15T01:18:29.700Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:29.700Z [INFO] server-66f691cf-4af3-844a-4f5d-9f9a7eb70c10: entering follower state: follower="Node at 66f691cf-4af3-844a-4f5d-9f9a7eb70c10 [Follower]" leader-address= leader-id= -2024-05-15T01:18:30.421Z [WARN] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:30.421Z [INFO] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: entering candidate state: node="Node at b6f34380-2810-120d-e9ba-61d8d7951ccd [Candidate]" term=2 -2024-05-15T01:18:30.421Z [INFO] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: election won: term=2 tally=2 -2024-05-15T01:18:30.421Z [INFO] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: entering leader state: leader="Node at b6f34380-2810-120d-e9ba-61d8d7951ccd [Leader]" -2024-05-15T01:18:30.421Z [INFO] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: added peer, starting replication: peer=server-096a1352-632e-8e5b-fa30-8fa75e426f3e -2024-05-15T01:18:30.421Z [INFO] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: added peer, starting replication: peer=server-66f691cf-4af3-844a-4f5d-9f9a7eb70c10 -2024-05-15T01:18:30.421Z [INFO] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: pipelining replication: peer="{Voter server-096a1352-632e-8e5b-fa30-8fa75e426f3e 096a1352-632e-8e5b-fa30-8fa75e426f3e}" -2024-05-15T01:18:30.421Z [INFO] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: pipelining replication: peer="{Voter server-66f691cf-4af3-844a-4f5d-9f9a7eb70c10 66f691cf-4af3-844a-4f5d-9f9a7eb70c10}" -2024-05-15T01:18:31.427Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:30.421698934 +0000 UTC m=+90.963083937 (1 nodes), 720.721796ms from start of poll, 721.519452ms from cluster start. Timeout at 2024-05-15 01:18:31.427135547 +0000 UTC m=+91.968520484, 1.005436547s after stability -2024-05-15T01:18:31.427Z [INFO] creating new snapshot: path=/tmp/raft852591037/snapshots/2-10013-1715735911427.tmp -2024-05-15T01:18:31.428Z [INFO] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: copied to local snapshot: bytes=71 -2024-05-15T01:18:31.429Z [INFO] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: snapshot restore progress: id=2-10013-1715735911427 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:31.429Z [INFO] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: restored user snapshot: index=10013 -2024-05-15T01:18:31.429Z [ERROR] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: failed to get log: index=3 error="log not found" -2024-05-15T01:18:31.429Z [INFO] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: aborting pipeline replication: peer="{Voter server-66f691cf-4af3-844a-4f5d-9f9a7eb70c10 66f691cf-4af3-844a-4f5d-9f9a7eb70c10}" -2024-05-15T01:18:31.429Z [ERROR] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: failed to get log: index=3 error="log not found" -2024-05-15T01:18:31.429Z [INFO] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: aborting pipeline replication: peer="{Voter server-096a1352-632e-8e5b-fa30-8fa75e426f3e 096a1352-632e-8e5b-fa30-8fa75e426f3e}" -2024-05-15T01:18:31.436Z [ERROR] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: failed to get log: index=3 error="log not found" -2024-05-15T01:18:31.436Z [INFO] creating new snapshot: path=/tmp/raft1922888992/snapshots/2-10013-1715735911436.tmp -2024-05-15T01:18:31.437Z [INFO] server-096a1352-632e-8e5b-fa30-8fa75e426f3e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:31.437Z [INFO] server-096a1352-632e-8e5b-fa30-8fa75e426f3e: copied to local snapshot: bytes=71 -2024-05-15T01:18:31.438Z [INFO] server-096a1352-632e-8e5b-fa30-8fa75e426f3e: snapshot restore progress: id=2-10013-1715735911436 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:31.438Z [INFO] server-096a1352-632e-8e5b-fa30-8fa75e426f3e: Installed remote snapshot -2024-05-15T01:18:31.438Z [INFO] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: pipelining replication: peer="{Voter server-096a1352-632e-8e5b-fa30-8fa75e426f3e 096a1352-632e-8e5b-fa30-8fa75e426f3e}" -2024-05-15T01:18:31.438Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:31.439Z [ERROR] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: failed to get log: index=3 error="log not found" -2024-05-15T01:18:31.440Z [INFO] creating new snapshot: path=/tmp/raft2216638681/snapshots/2-10013-1715735911440.tmp -2024-05-15T01:18:31.440Z [INFO] server-66f691cf-4af3-844a-4f5d-9f9a7eb70c10: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:31.441Z [INFO] server-66f691cf-4af3-844a-4f5d-9f9a7eb70c10: copied to local snapshot: bytes=71 -2024-05-15T01:18:31.441Z [INFO] server-66f691cf-4af3-844a-4f5d-9f9a7eb70c10: snapshot restore progress: id=2-10013-1715735911440 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:31.441Z [INFO] server-66f691cf-4af3-844a-4f5d-9f9a7eb70c10: Installed remote snapshot -2024-05-15T01:18:31.441Z [INFO] server-b6f34380-2810-120d-e9ba-61d8d7951ccd: pipelining replication: peer="{Voter server-66f691cf-4af3-844a-4f5d-9f9a7eb70c10 66f691cf-4af3-844a-4f5d-9f9a7eb70c10}" -2024-05-15T01:18:32.444Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:31.438981591 +0000 UTC m=+91.980366392 (2 nodes), 2.497µs from start of poll, 1.738801907s from cluster start. Timeout at 2024-05-15 01:18:32.44478118 +0000 UTC m=+92.986170419, 1.005804027s after stability -2024-05-15T01:18:32.457Z [INFO] server-64fb5fcc-8aee-72d9-31a2-abec770f814e: aborting pipeline replication: peer="{Voter server-2d2c6007-f6e9-91b7-47f4-54317f25a40d 2d2c6007-f6e9-91b7-47f4-54317f25a40d}" -2024-05-15T01:18:32.457Z [INFO] server-64fb5fcc-8aee-72d9-31a2-abec770f814e: aborting pipeline replication: peer="{Voter server-665b0344-6127-0468-e3c8-8b9ea8d8c432 665b0344-6127-0468-e3c8-8b9ea8d8c432}" +2025-06-17T07:44:20.283Z [INFO] server-e6a19ae0-6afd-ea06-de83-abb619191039: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e6a19ae0-6afd-ea06-de83-abb619191039 Address:e6a19ae0-6afd-ea06-de83-abb619191039} {Suffrage:Voter ID:server-a9e2aa83-8bac-b1f3-6e86-514995917b4a Address:a9e2aa83-8bac-b1f3-6e86-514995917b4a} {Suffrage:Voter ID:server-b5def66f-30f8-0208-811d-f12e7245faa3 Address:b5def66f-30f8-0208-811d-f12e7245faa3}]" +2025-06-17T07:44:20.283Z [INFO] server-e6a19ae0-6afd-ea06-de83-abb619191039: entering follower state: follower="Node at e6a19ae0-6afd-ea06-de83-abb619191039 [Follower]" leader-address= leader-id= +2025-06-17T07:44:20.283Z [INFO] server-a9e2aa83-8bac-b1f3-6e86-514995917b4a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e6a19ae0-6afd-ea06-de83-abb619191039 Address:e6a19ae0-6afd-ea06-de83-abb619191039} {Suffrage:Voter ID:server-a9e2aa83-8bac-b1f3-6e86-514995917b4a Address:a9e2aa83-8bac-b1f3-6e86-514995917b4a} {Suffrage:Voter ID:server-b5def66f-30f8-0208-811d-f12e7245faa3 Address:b5def66f-30f8-0208-811d-f12e7245faa3}]" +2025-06-17T07:44:20.283Z [INFO] server-a9e2aa83-8bac-b1f3-6e86-514995917b4a: entering follower state: follower="Node at a9e2aa83-8bac-b1f3-6e86-514995917b4a [Follower]" leader-address= leader-id= +2025-06-17T07:44:20.283Z [INFO] server-b5def66f-30f8-0208-811d-f12e7245faa3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e6a19ae0-6afd-ea06-de83-abb619191039 Address:e6a19ae0-6afd-ea06-de83-abb619191039} {Suffrage:Voter ID:server-a9e2aa83-8bac-b1f3-6e86-514995917b4a Address:a9e2aa83-8bac-b1f3-6e86-514995917b4a} {Suffrage:Voter ID:server-b5def66f-30f8-0208-811d-f12e7245faa3 Address:b5def66f-30f8-0208-811d-f12e7245faa3}]" +2025-06-17T07:44:20.283Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:20.283Z [INFO] server-b5def66f-30f8-0208-811d-f12e7245faa3: entering follower state: follower="Node at b5def66f-30f8-0208-811d-f12e7245faa3 [Follower]" leader-address= leader-id= +2025-06-17T07:44:20.820Z [WARN] server-a9e2aa83-8bac-b1f3-6e86-514995917b4a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:20.820Z [INFO] server-a9e2aa83-8bac-b1f3-6e86-514995917b4a: entering candidate state: node="Node at a9e2aa83-8bac-b1f3-6e86-514995917b4a [Candidate]" term=2 +2025-06-17T07:44:20.820Z [INFO] server-a9e2aa83-8bac-b1f3-6e86-514995917b4a: election won: term=2 tally=2 +2025-06-17T07:44:20.820Z [INFO] server-a9e2aa83-8bac-b1f3-6e86-514995917b4a: entering leader state: leader="Node at a9e2aa83-8bac-b1f3-6e86-514995917b4a [Leader]" +2025-06-17T07:44:20.820Z [INFO] server-a9e2aa83-8bac-b1f3-6e86-514995917b4a: added peer, starting replication: peer=server-e6a19ae0-6afd-ea06-de83-abb619191039 +2025-06-17T07:44:20.820Z [INFO] server-a9e2aa83-8bac-b1f3-6e86-514995917b4a: added peer, starting replication: peer=server-b5def66f-30f8-0208-811d-f12e7245faa3 +2025-06-17T07:44:20.820Z [INFO] server-a9e2aa83-8bac-b1f3-6e86-514995917b4a: pipelining replication: peer="{Voter server-b5def66f-30f8-0208-811d-f12e7245faa3 b5def66f-30f8-0208-811d-f12e7245faa3}" +2025-06-17T07:44:20.820Z [INFO] server-a9e2aa83-8bac-b1f3-6e86-514995917b4a: pipelining replication: peer="{Voter server-e6a19ae0-6afd-ea06-de83-abb619191039 e6a19ae0-6afd-ea06-de83-abb619191039}" +2025-06-17T07:44:21.825Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:20.820668207 +0000 UTC m=+89.424196896 (1 nodes), 537.039432ms from start of poll, 537.385028ms from cluster start. Timeout at 2025-06-17 07:44:21.825942149 +0000 UTC m=+90.429470831, 1.005273935s after stability +2025-06-17T07:44:21.826Z [INFO] server-a9e2aa83-8bac-b1f3-6e86-514995917b4a: starting snapshot up to: index=12 +2025-06-17T07:44:21.826Z [INFO] creating new snapshot: path=/tmp/raft940792909/snapshots/2-12-1750146261826.tmp +2025-06-17T07:44:21.826Z [INFO] server-a9e2aa83-8bac-b1f3-6e86-514995917b4a: snapshot complete up to: index=12 +2025-06-17T07:44:21.827Z [INFO] server-8d3bd332-36f6-00d1-cd44-84e48a5a21fe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8d3bd332-36f6-00d1-cd44-84e48a5a21fe Address:8d3bd332-36f6-00d1-cd44-84e48a5a21fe} {Suffrage:Voter ID:server-18b14da1-a05b-237b-b7b3-56847c64633f Address:18b14da1-a05b-237b-b7b3-56847c64633f} {Suffrage:Voter ID:server-4d397d6b-5475-054f-493f-432b139914b2 Address:4d397d6b-5475-054f-493f-432b139914b2}]" +2025-06-17T07:44:21.828Z [INFO] server-18b14da1-a05b-237b-b7b3-56847c64633f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8d3bd332-36f6-00d1-cd44-84e48a5a21fe Address:8d3bd332-36f6-00d1-cd44-84e48a5a21fe} {Suffrage:Voter ID:server-18b14da1-a05b-237b-b7b3-56847c64633f Address:18b14da1-a05b-237b-b7b3-56847c64633f} {Suffrage:Voter ID:server-4d397d6b-5475-054f-493f-432b139914b2 Address:4d397d6b-5475-054f-493f-432b139914b2}]" +2025-06-17T07:44:21.828Z [INFO] server-8d3bd332-36f6-00d1-cd44-84e48a5a21fe: entering follower state: follower="Node at 8d3bd332-36f6-00d1-cd44-84e48a5a21fe [Follower]" leader-address= leader-id= +2025-06-17T07:44:21.828Z [INFO] server-18b14da1-a05b-237b-b7b3-56847c64633f: entering follower state: follower="Node at 18b14da1-a05b-237b-b7b3-56847c64633f [Follower]" leader-address= leader-id= +2025-06-17T07:44:21.828Z [INFO] server-4d397d6b-5475-054f-493f-432b139914b2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8d3bd332-36f6-00d1-cd44-84e48a5a21fe Address:8d3bd332-36f6-00d1-cd44-84e48a5a21fe} {Suffrage:Voter ID:server-18b14da1-a05b-237b-b7b3-56847c64633f Address:18b14da1-a05b-237b-b7b3-56847c64633f} {Suffrage:Voter ID:server-4d397d6b-5475-054f-493f-432b139914b2 Address:4d397d6b-5475-054f-493f-432b139914b2}]" +2025-06-17T07:44:21.828Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:21.828Z [INFO] server-4d397d6b-5475-054f-493f-432b139914b2: entering follower state: follower="Node at 4d397d6b-5475-054f-493f-432b139914b2 [Follower]" leader-address= leader-id= +2025-06-17T07:44:22.592Z [WARN] server-18b14da1-a05b-237b-b7b3-56847c64633f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:22.592Z [INFO] server-18b14da1-a05b-237b-b7b3-56847c64633f: entering candidate state: node="Node at 18b14da1-a05b-237b-b7b3-56847c64633f [Candidate]" term=2 +2025-06-17T07:44:22.593Z [INFO] server-18b14da1-a05b-237b-b7b3-56847c64633f: election won: term=2 tally=2 +2025-06-17T07:44:22.593Z [INFO] server-18b14da1-a05b-237b-b7b3-56847c64633f: entering leader state: leader="Node at 18b14da1-a05b-237b-b7b3-56847c64633f [Leader]" +2025-06-17T07:44:22.593Z [INFO] server-18b14da1-a05b-237b-b7b3-56847c64633f: added peer, starting replication: peer=server-8d3bd332-36f6-00d1-cd44-84e48a5a21fe +2025-06-17T07:44:22.593Z [INFO] server-18b14da1-a05b-237b-b7b3-56847c64633f: added peer, starting replication: peer=server-4d397d6b-5475-054f-493f-432b139914b2 +2025-06-17T07:44:22.593Z [INFO] server-18b14da1-a05b-237b-b7b3-56847c64633f: pipelining replication: peer="{Voter server-8d3bd332-36f6-00d1-cd44-84e48a5a21fe 8d3bd332-36f6-00d1-cd44-84e48a5a21fe}" +2025-06-17T07:44:22.593Z [INFO] server-18b14da1-a05b-237b-b7b3-56847c64633f: pipelining replication: peer="{Voter server-4d397d6b-5475-054f-493f-432b139914b2 4d397d6b-5475-054f-493f-432b139914b2}" +2025-06-17T07:44:23.598Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:22.593170834 +0000 UTC m=+91.196699554 (1 nodes), 764.998203ms from start of poll, 765.362587ms from cluster start. Timeout at 2025-06-17 07:44:23.598803638 +0000 UTC m=+92.202332355, 1.005632801s after stability +2025-06-17T07:44:23.599Z [INFO] creating new snapshot: path=/tmp/raft3448753949/snapshots/2-10013-1750146263599.tmp +2025-06-17T07:44:23.599Z [INFO] server-18b14da1-a05b-237b-b7b3-56847c64633f: copied to local snapshot: bytes=71 +2025-06-17T07:44:23.600Z [INFO] server-18b14da1-a05b-237b-b7b3-56847c64633f: snapshot restore progress: id=2-10013-1750146263599 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:23.600Z [INFO] server-18b14da1-a05b-237b-b7b3-56847c64633f: restored user snapshot: index=10013 +2025-06-17T07:44:23.600Z [ERROR] server-18b14da1-a05b-237b-b7b3-56847c64633f: failed to get log: index=3 error="log not found" +2025-06-17T07:44:23.600Z [ERROR] server-18b14da1-a05b-237b-b7b3-56847c64633f: failed to get log: index=3 error="log not found" +2025-06-17T07:44:23.600Z [INFO] server-18b14da1-a05b-237b-b7b3-56847c64633f: aborting pipeline replication: peer="{Voter server-8d3bd332-36f6-00d1-cd44-84e48a5a21fe 8d3bd332-36f6-00d1-cd44-84e48a5a21fe}" +2025-06-17T07:44:23.600Z [INFO] server-18b14da1-a05b-237b-b7b3-56847c64633f: aborting pipeline replication: peer="{Voter server-4d397d6b-5475-054f-493f-432b139914b2 4d397d6b-5475-054f-493f-432b139914b2}" +2025-06-17T07:44:23.606Z [ERROR] server-18b14da1-a05b-237b-b7b3-56847c64633f: failed to get log: index=3 error="log not found" +2025-06-17T07:44:23.606Z [INFO] creating new snapshot: path=/tmp/raft2859884471/snapshots/2-10013-1750146263606.tmp +2025-06-17T07:44:23.606Z [INFO] server-8d3bd332-36f6-00d1-cd44-84e48a5a21fe: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:23.606Z [INFO] server-8d3bd332-36f6-00d1-cd44-84e48a5a21fe: copied to local snapshot: bytes=71 +2025-06-17T07:44:23.607Z [INFO] server-8d3bd332-36f6-00d1-cd44-84e48a5a21fe: snapshot restore progress: id=2-10013-1750146263606 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:23.607Z [INFO] server-8d3bd332-36f6-00d1-cd44-84e48a5a21fe: Installed remote snapshot +2025-06-17T07:44:23.607Z [INFO] server-18b14da1-a05b-237b-b7b3-56847c64633f: pipelining replication: peer="{Voter server-8d3bd332-36f6-00d1-cd44-84e48a5a21fe 8d3bd332-36f6-00d1-cd44-84e48a5a21fe}" +2025-06-17T07:44:23.607Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:23.608Z [ERROR] server-18b14da1-a05b-237b-b7b3-56847c64633f: failed to get log: index=3 error="log not found" +2025-06-17T07:44:23.608Z [INFO] creating new snapshot: path=/tmp/raft258660824/snapshots/2-10013-1750146263608.tmp +2025-06-17T07:44:23.609Z [INFO] server-4d397d6b-5475-054f-493f-432b139914b2: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:23.609Z [INFO] server-4d397d6b-5475-054f-493f-432b139914b2: copied to local snapshot: bytes=71 +2025-06-17T07:44:23.609Z [INFO] server-4d397d6b-5475-054f-493f-432b139914b2: snapshot restore progress: id=2-10013-1750146263608 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:23.609Z [INFO] server-4d397d6b-5475-054f-493f-432b139914b2: Installed remote snapshot +2025-06-17T07:44:23.609Z [INFO] server-18b14da1-a05b-237b-b7b3-56847c64633f: pipelining replication: peer="{Voter server-4d397d6b-5475-054f-493f-432b139914b2 4d397d6b-5475-054f-493f-432b139914b2}" +2025-06-17T07:44:24.612Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:23.607330533 +0000 UTC m=+92.210859220 (2 nodes), 697ns from start of poll, 1.779522253s from cluster start. Timeout at 2025-06-17 07:44:24.61268557 +0000 UTC m=+93.216214291, 1.005355071s after stability +2025-06-17T07:44:24.624Z [INFO] server-a9e2aa83-8bac-b1f3-6e86-514995917b4a: aborting pipeline replication: peer="{Voter server-b5def66f-30f8-0208-811d-f12e7245faa3 b5def66f-30f8-0208-811d-f12e7245faa3}" +2025-06-17T07:44:24.624Z [INFO] server-a9e2aa83-8bac-b1f3-6e86-514995917b4a: aborting pipeline replication: peer="{Voter server-e6a19ae0-6afd-ea06-de83-abb619191039 e6a19ae0-6afd-ea06-de83-abb619191039}" === RUN TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true -2024-05-15T01:18:32.459Z [INFO] server-5e18b38b-c45c-a002-024a-f12ac8a599c5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5e18b38b-c45c-a002-024a-f12ac8a599c5 Address:5e18b38b-c45c-a002-024a-f12ac8a599c5} {Suffrage:Voter ID:server-4e908e50-89f7-df48-f80c-85bc53a39c93 Address:4e908e50-89f7-df48-f80c-85bc53a39c93} {Suffrage:Voter ID:server-1b317626-0298-0dfe-974f-c4707498327a Address:1b317626-0298-0dfe-974f-c4707498327a}]" -2024-05-15T01:18:32.459Z [INFO] server-5e18b38b-c45c-a002-024a-f12ac8a599c5: entering follower state: follower="Node at 5e18b38b-c45c-a002-024a-f12ac8a599c5 [Follower]" leader-address= leader-id= -2024-05-15T01:18:32.459Z [INFO] server-4e908e50-89f7-df48-f80c-85bc53a39c93: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5e18b38b-c45c-a002-024a-f12ac8a599c5 Address:5e18b38b-c45c-a002-024a-f12ac8a599c5} {Suffrage:Voter ID:server-4e908e50-89f7-df48-f80c-85bc53a39c93 Address:4e908e50-89f7-df48-f80c-85bc53a39c93} {Suffrage:Voter ID:server-1b317626-0298-0dfe-974f-c4707498327a Address:1b317626-0298-0dfe-974f-c4707498327a}]" -2024-05-15T01:18:32.459Z [INFO] server-4e908e50-89f7-df48-f80c-85bc53a39c93: entering follower state: follower="Node at 4e908e50-89f7-df48-f80c-85bc53a39c93 [Follower]" leader-address= leader-id= -2024-05-15T01:18:32.459Z [INFO] server-1b317626-0298-0dfe-974f-c4707498327a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5e18b38b-c45c-a002-024a-f12ac8a599c5 Address:5e18b38b-c45c-a002-024a-f12ac8a599c5} {Suffrage:Voter ID:server-4e908e50-89f7-df48-f80c-85bc53a39c93 Address:4e908e50-89f7-df48-f80c-85bc53a39c93} {Suffrage:Voter ID:server-1b317626-0298-0dfe-974f-c4707498327a Address:1b317626-0298-0dfe-974f-c4707498327a}]" -2024-05-15T01:18:32.459Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:32.459Z [INFO] server-1b317626-0298-0dfe-974f-c4707498327a: entering follower state: follower="Node at 1b317626-0298-0dfe-974f-c4707498327a [Follower]" leader-address= leader-id= -2024-05-15T01:18:33.130Z [WARN] server-5e18b38b-c45c-a002-024a-f12ac8a599c5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:33.130Z [INFO] server-5e18b38b-c45c-a002-024a-f12ac8a599c5: entering candidate state: node="Node at 5e18b38b-c45c-a002-024a-f12ac8a599c5 [Candidate]" term=2 -2024-05-15T01:18:33.130Z [INFO] server-5e18b38b-c45c-a002-024a-f12ac8a599c5: election won: term=2 tally=2 -2024-05-15T01:18:33.131Z [INFO] server-5e18b38b-c45c-a002-024a-f12ac8a599c5: entering leader state: leader="Node at 5e18b38b-c45c-a002-024a-f12ac8a599c5 [Leader]" -2024-05-15T01:18:33.131Z [INFO] server-5e18b38b-c45c-a002-024a-f12ac8a599c5: added peer, starting replication: peer=server-4e908e50-89f7-df48-f80c-85bc53a39c93 -2024-05-15T01:18:33.131Z [INFO] server-5e18b38b-c45c-a002-024a-f12ac8a599c5: added peer, starting replication: peer=server-1b317626-0298-0dfe-974f-c4707498327a -2024-05-15T01:18:33.131Z [INFO] server-5e18b38b-c45c-a002-024a-f12ac8a599c5: pipelining replication: peer="{Voter server-1b317626-0298-0dfe-974f-c4707498327a 1b317626-0298-0dfe-974f-c4707498327a}" -2024-05-15T01:18:33.131Z [INFO] server-5e18b38b-c45c-a002-024a-f12ac8a599c5: pipelining replication: peer="{Voter server-4e908e50-89f7-df48-f80c-85bc53a39c93 4e908e50-89f7-df48-f80c-85bc53a39c93}" -2024-05-15T01:18:34.136Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:33.131566856 +0000 UTC m=+93.672951629 (1 nodes), 671.681516ms from start of poll, 672.245099ms from cluster start. Timeout at 2024-05-15 01:18:34.13665219 +0000 UTC m=+94.678036951, 1.005085322s after stability -2024-05-15T01:18:34.138Z [INFO] server-5e18b38b-c45c-a002-024a-f12ac8a599c5: starting snapshot up to: index=12 -2024-05-15T01:18:34.138Z [INFO] creating new snapshot: path=/tmp/raft537060735/snapshots/2-12-1715735914138.tmp -2024-05-15T01:18:34.139Z [INFO] server-5e18b38b-c45c-a002-024a-f12ac8a599c5: snapshot complete up to: index=12 -2024-05-15T01:18:34.142Z [INFO] server-3f34e894-6fa3-c9bc-660d-9aa4427106e7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f34e894-6fa3-c9bc-660d-9aa4427106e7 Address:3f34e894-6fa3-c9bc-660d-9aa4427106e7} {Suffrage:Voter ID:server-b7e70869-b993-ddcc-4de6-9b659063b41f Address:b7e70869-b993-ddcc-4de6-9b659063b41f} {Suffrage:Voter ID:server-9794a32d-9270-958c-8a30-9a198d1f4538 Address:9794a32d-9270-958c-8a30-9a198d1f4538}]" -2024-05-15T01:18:34.142Z [INFO] server-3f34e894-6fa3-c9bc-660d-9aa4427106e7: entering follower state: follower="Node at 3f34e894-6fa3-c9bc-660d-9aa4427106e7 [Follower]" leader-address= leader-id= -2024-05-15T01:18:34.142Z [INFO] server-b7e70869-b993-ddcc-4de6-9b659063b41f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f34e894-6fa3-c9bc-660d-9aa4427106e7 Address:3f34e894-6fa3-c9bc-660d-9aa4427106e7} {Suffrage:Voter ID:server-b7e70869-b993-ddcc-4de6-9b659063b41f Address:b7e70869-b993-ddcc-4de6-9b659063b41f} {Suffrage:Voter ID:server-9794a32d-9270-958c-8a30-9a198d1f4538 Address:9794a32d-9270-958c-8a30-9a198d1f4538}]" -2024-05-15T01:18:34.142Z [INFO] server-b7e70869-b993-ddcc-4de6-9b659063b41f: entering follower state: follower="Node at b7e70869-b993-ddcc-4de6-9b659063b41f [Follower]" leader-address= leader-id= -2024-05-15T01:18:34.142Z [INFO] server-9794a32d-9270-958c-8a30-9a198d1f4538: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f34e894-6fa3-c9bc-660d-9aa4427106e7 Address:3f34e894-6fa3-c9bc-660d-9aa4427106e7} {Suffrage:Voter ID:server-b7e70869-b993-ddcc-4de6-9b659063b41f Address:b7e70869-b993-ddcc-4de6-9b659063b41f} {Suffrage:Voter ID:server-9794a32d-9270-958c-8a30-9a198d1f4538 Address:9794a32d-9270-958c-8a30-9a198d1f4538}]" -2024-05-15T01:18:34.142Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:34.142Z [INFO] server-9794a32d-9270-958c-8a30-9a198d1f4538: entering follower state: follower="Node at 9794a32d-9270-958c-8a30-9a198d1f4538 [Follower]" leader-address= leader-id= -2024-05-15T01:18:34.664Z [WARN] server-b7e70869-b993-ddcc-4de6-9b659063b41f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:34.664Z [INFO] server-b7e70869-b993-ddcc-4de6-9b659063b41f: entering candidate state: node="Node at b7e70869-b993-ddcc-4de6-9b659063b41f [Candidate]" term=2 -2024-05-15T01:18:34.665Z [INFO] server-b7e70869-b993-ddcc-4de6-9b659063b41f: election won: term=2 tally=2 -2024-05-15T01:18:34.665Z [INFO] server-b7e70869-b993-ddcc-4de6-9b659063b41f: entering leader state: leader="Node at b7e70869-b993-ddcc-4de6-9b659063b41f [Leader]" -2024-05-15T01:18:34.665Z [INFO] server-b7e70869-b993-ddcc-4de6-9b659063b41f: added peer, starting replication: peer=server-3f34e894-6fa3-c9bc-660d-9aa4427106e7 -2024-05-15T01:18:34.665Z [INFO] server-b7e70869-b993-ddcc-4de6-9b659063b41f: added peer, starting replication: peer=server-9794a32d-9270-958c-8a30-9a198d1f4538 -2024-05-15T01:18:34.665Z [INFO] server-b7e70869-b993-ddcc-4de6-9b659063b41f: pipelining replication: peer="{Voter server-9794a32d-9270-958c-8a30-9a198d1f4538 9794a32d-9270-958c-8a30-9a198d1f4538}" -2024-05-15T01:18:34.665Z [INFO] server-b7e70869-b993-ddcc-4de6-9b659063b41f: pipelining replication: peer="{Voter server-3f34e894-6fa3-c9bc-660d-9aa4427106e7 3f34e894-6fa3-c9bc-660d-9aa4427106e7}" -2024-05-15T01:18:35.671Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:34.666019434 +0000 UTC m=+95.207404370 (1 nodes), 523.48977ms from start of poll, 524.007046ms from cluster start. Timeout at 2024-05-15 01:18:35.671144275 +0000 UTC m=+96.212529015, 1.005124645s after stability -2024-05-15T01:18:35.671Z [INFO] creating new snapshot: path=/tmp/raft2551169507/snapshots/2-10013-1715735915671.tmp -2024-05-15T01:18:35.676Z [INFO] server-b7e70869-b993-ddcc-4de6-9b659063b41f: copied to local snapshot: bytes=71 -2024-05-15T01:18:35.690Z [INFO] server-b7e70869-b993-ddcc-4de6-9b659063b41f: snapshot restore progress: id=2-10013-1715735915671 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:35.691Z [INFO] server-b7e70869-b993-ddcc-4de6-9b659063b41f: removing all old logs from log store -2024-05-15T01:18:35.691Z [INFO] server-b7e70869-b993-ddcc-4de6-9b659063b41f: compacting logs: from=1 to=2 -2024-05-15T01:18:35.691Z [INFO] server-b7e70869-b993-ddcc-4de6-9b659063b41f: restored user snapshot: index=10013 -2024-05-15T01:18:35.691Z [ERROR] server-b7e70869-b993-ddcc-4de6-9b659063b41f: failed to get log: index=2 error="log not found" -2024-05-15T01:18:35.691Z [INFO] server-b7e70869-b993-ddcc-4de6-9b659063b41f: aborting pipeline replication: peer="{Voter server-9794a32d-9270-958c-8a30-9a198d1f4538 9794a32d-9270-958c-8a30-9a198d1f4538}" -2024-05-15T01:18:35.691Z [ERROR] server-b7e70869-b993-ddcc-4de6-9b659063b41f: failed to get log: index=2 error="log not found" -2024-05-15T01:18:35.691Z [INFO] server-b7e70869-b993-ddcc-4de6-9b659063b41f: aborting pipeline replication: peer="{Voter server-3f34e894-6fa3-c9bc-660d-9aa4427106e7 3f34e894-6fa3-c9bc-660d-9aa4427106e7}" -2024-05-15T01:18:35.697Z [ERROR] server-b7e70869-b993-ddcc-4de6-9b659063b41f: failed to get log: index=2 error="log not found" -2024-05-15T01:18:35.697Z [INFO] creating new snapshot: path=/tmp/raft3082318135/snapshots/2-10013-1715735915697.tmp -2024-05-15T01:18:35.698Z [ERROR] server-b7e70869-b993-ddcc-4de6-9b659063b41f: failed to get log: index=2 error="log not found" -2024-05-15T01:18:35.698Z [INFO] server-9794a32d-9270-958c-8a30-9a198d1f4538: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:35.698Z [INFO] server-9794a32d-9270-958c-8a30-9a198d1f4538: copied to local snapshot: bytes=71 -2024-05-15T01:18:35.698Z [INFO] creating new snapshot: path=/tmp/raft741680732/snapshots/2-10013-1715735915698.tmp -2024-05-15T01:18:35.699Z [INFO] server-9794a32d-9270-958c-8a30-9a198d1f4538: snapshot restore progress: id=2-10013-1715735915697 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:35.699Z [INFO] server-3f34e894-6fa3-c9bc-660d-9aa4427106e7: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:35.699Z [INFO] server-9794a32d-9270-958c-8a30-9a198d1f4538: removing all old logs from log store -2024-05-15T01:18:35.699Z [INFO] server-9794a32d-9270-958c-8a30-9a198d1f4538: compacting logs: from=1 to=2 -2024-05-15T01:18:35.699Z [INFO] server-9794a32d-9270-958c-8a30-9a198d1f4538: Installed remote snapshot -2024-05-15T01:18:35.699Z [INFO] server-3f34e894-6fa3-c9bc-660d-9aa4427106e7: copied to local snapshot: bytes=71 -2024-05-15T01:18:35.699Z [INFO] server-b7e70869-b993-ddcc-4de6-9b659063b41f: pipelining replication: peer="{Voter server-9794a32d-9270-958c-8a30-9a198d1f4538 9794a32d-9270-958c-8a30-9a198d1f4538}" -2024-05-15T01:18:35.699Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:35.699Z [INFO] server-3f34e894-6fa3-c9bc-660d-9aa4427106e7: snapshot restore progress: id=2-10013-1715735915698 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-15T01:18:35.699Z [INFO] server-3f34e894-6fa3-c9bc-660d-9aa4427106e7: removing all old logs from log store -2024-05-15T01:18:35.699Z [INFO] server-3f34e894-6fa3-c9bc-660d-9aa4427106e7: compacting logs: from=1 to=2 -2024-05-15T01:18:35.699Z [INFO] server-3f34e894-6fa3-c9bc-660d-9aa4427106e7: Installed remote snapshot -2024-05-15T01:18:35.700Z [INFO] server-b7e70869-b993-ddcc-4de6-9b659063b41f: pipelining replication: peer="{Voter server-3f34e894-6fa3-c9bc-660d-9aa4427106e7 3f34e894-6fa3-c9bc-660d-9aa4427106e7}" -2024-05-15T01:18:36.705Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:35.699799421 +0000 UTC m=+96.241184512 (2 nodes), 2.573µs from start of poll, 1.557787188s from cluster start. Timeout at 2024-05-15 01:18:36.705578804 +0000 UTC m=+97.246963543, 1.005779031s after stability -2024-05-15T01:18:36.718Z [INFO] server-5e18b38b-c45c-a002-024a-f12ac8a599c5: aborting pipeline replication: peer="{Voter server-1b317626-0298-0dfe-974f-c4707498327a 1b317626-0298-0dfe-974f-c4707498327a}" -2024-05-15T01:18:36.718Z [INFO] server-5e18b38b-c45c-a002-024a-f12ac8a599c5: aborting pipeline replication: peer="{Voter server-4e908e50-89f7-df48-f80c-85bc53a39c93 4e908e50-89f7-df48-f80c-85bc53a39c93}" ---- PASS: TestRaft_UserRestore (84.07s) - --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false (2.63s) - --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false (2.66s) - --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true (4.43s) - --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true (4.16s) - --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false (2.76s) - --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false (2.60s) - --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true (4.47s) - --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true (4.25s) - --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false (2.80s) - --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false (2.80s) - --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true (4.28s) - --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true (4.50s) - --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false (2.59s) - --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false (2.61s) - --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true (4.34s) - --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true (4.23s) - --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false (2.71s) - --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false (2.62s) - --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true (4.25s) - --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true (4.37s) - --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false (2.56s) - --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false (2.82s) - --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true (4.38s) - --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true (4.26s) +2025-06-17T07:44:24.625Z [INFO] server-d2f3482c-3c3f-8933-06d9-b8250b210f7e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d2f3482c-3c3f-8933-06d9-b8250b210f7e Address:d2f3482c-3c3f-8933-06d9-b8250b210f7e} {Suffrage:Voter ID:server-c62ca0f2-e284-4fc6-7b7d-2e6e20fd6b3b Address:c62ca0f2-e284-4fc6-7b7d-2e6e20fd6b3b} {Suffrage:Voter ID:server-6726ddd8-e6da-f043-dd14-b2044bca1d86 Address:6726ddd8-e6da-f043-dd14-b2044bca1d86}]" +2025-06-17T07:44:24.625Z [INFO] server-d2f3482c-3c3f-8933-06d9-b8250b210f7e: entering follower state: follower="Node at d2f3482c-3c3f-8933-06d9-b8250b210f7e [Follower]" leader-address= leader-id= +2025-06-17T07:44:24.625Z [INFO] server-c62ca0f2-e284-4fc6-7b7d-2e6e20fd6b3b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d2f3482c-3c3f-8933-06d9-b8250b210f7e Address:d2f3482c-3c3f-8933-06d9-b8250b210f7e} {Suffrage:Voter ID:server-c62ca0f2-e284-4fc6-7b7d-2e6e20fd6b3b Address:c62ca0f2-e284-4fc6-7b7d-2e6e20fd6b3b} {Suffrage:Voter ID:server-6726ddd8-e6da-f043-dd14-b2044bca1d86 Address:6726ddd8-e6da-f043-dd14-b2044bca1d86}]" +2025-06-17T07:44:24.625Z [INFO] server-c62ca0f2-e284-4fc6-7b7d-2e6e20fd6b3b: entering follower state: follower="Node at c62ca0f2-e284-4fc6-7b7d-2e6e20fd6b3b [Follower]" leader-address= leader-id= +2025-06-17T07:44:24.625Z [INFO] server-6726ddd8-e6da-f043-dd14-b2044bca1d86: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d2f3482c-3c3f-8933-06d9-b8250b210f7e Address:d2f3482c-3c3f-8933-06d9-b8250b210f7e} {Suffrage:Voter ID:server-c62ca0f2-e284-4fc6-7b7d-2e6e20fd6b3b Address:c62ca0f2-e284-4fc6-7b7d-2e6e20fd6b3b} {Suffrage:Voter ID:server-6726ddd8-e6da-f043-dd14-b2044bca1d86 Address:6726ddd8-e6da-f043-dd14-b2044bca1d86}]" +2025-06-17T07:44:24.625Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:24.625Z [INFO] server-6726ddd8-e6da-f043-dd14-b2044bca1d86: entering follower state: follower="Node at 6726ddd8-e6da-f043-dd14-b2044bca1d86 [Follower]" leader-address= leader-id= +2025-06-17T07:44:25.236Z [WARN] server-d2f3482c-3c3f-8933-06d9-b8250b210f7e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:25.236Z [INFO] server-d2f3482c-3c3f-8933-06d9-b8250b210f7e: entering candidate state: node="Node at d2f3482c-3c3f-8933-06d9-b8250b210f7e [Candidate]" term=2 +2025-06-17T07:44:25.236Z [INFO] server-d2f3482c-3c3f-8933-06d9-b8250b210f7e: election won: term=2 tally=2 +2025-06-17T07:44:25.236Z [INFO] server-d2f3482c-3c3f-8933-06d9-b8250b210f7e: entering leader state: leader="Node at d2f3482c-3c3f-8933-06d9-b8250b210f7e [Leader]" +2025-06-17T07:44:25.236Z [INFO] server-d2f3482c-3c3f-8933-06d9-b8250b210f7e: added peer, starting replication: peer=server-c62ca0f2-e284-4fc6-7b7d-2e6e20fd6b3b +2025-06-17T07:44:25.236Z [INFO] server-d2f3482c-3c3f-8933-06d9-b8250b210f7e: added peer, starting replication: peer=server-6726ddd8-e6da-f043-dd14-b2044bca1d86 +2025-06-17T07:44:25.236Z [INFO] server-d2f3482c-3c3f-8933-06d9-b8250b210f7e: pipelining replication: peer="{Voter server-6726ddd8-e6da-f043-dd14-b2044bca1d86 6726ddd8-e6da-f043-dd14-b2044bca1d86}" +2025-06-17T07:44:25.236Z [INFO] server-d2f3482c-3c3f-8933-06d9-b8250b210f7e: pipelining replication: peer="{Voter server-c62ca0f2-e284-4fc6-7b7d-2e6e20fd6b3b c62ca0f2-e284-4fc6-7b7d-2e6e20fd6b3b}" +2025-06-17T07:44:26.242Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:25.236627557 +0000 UTC m=+93.840156263 (1 nodes), 611.163862ms from start of poll, 611.491854ms from cluster start. Timeout at 2025-06-17 07:44:26.242290818 +0000 UTC m=+94.845819509, 1.005663246s after stability +2025-06-17T07:44:26.242Z [INFO] server-d2f3482c-3c3f-8933-06d9-b8250b210f7e: starting snapshot up to: index=12 +2025-06-17T07:44:26.242Z [INFO] creating new snapshot: path=/tmp/raft861448722/snapshots/2-12-1750146266242.tmp +2025-06-17T07:44:26.243Z [INFO] server-d2f3482c-3c3f-8933-06d9-b8250b210f7e: snapshot complete up to: index=12 +2025-06-17T07:44:26.244Z [INFO] server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29 Address:f954bf62-2081-9eb4-5979-fd4aaf9f5d29} {Suffrage:Voter ID:server-f04117d8-ea56-2998-e9ec-634bcc6cf8ae Address:f04117d8-ea56-2998-e9ec-634bcc6cf8ae} {Suffrage:Voter ID:server-469c96ac-51fe-e8b3-9513-ebd33b57bf36 Address:469c96ac-51fe-e8b3-9513-ebd33b57bf36}]" +2025-06-17T07:44:26.244Z [INFO] server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29: entering follower state: follower="Node at f954bf62-2081-9eb4-5979-fd4aaf9f5d29 [Follower]" leader-address= leader-id= +2025-06-17T07:44:26.244Z [INFO] server-f04117d8-ea56-2998-e9ec-634bcc6cf8ae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29 Address:f954bf62-2081-9eb4-5979-fd4aaf9f5d29} {Suffrage:Voter ID:server-f04117d8-ea56-2998-e9ec-634bcc6cf8ae Address:f04117d8-ea56-2998-e9ec-634bcc6cf8ae} {Suffrage:Voter ID:server-469c96ac-51fe-e8b3-9513-ebd33b57bf36 Address:469c96ac-51fe-e8b3-9513-ebd33b57bf36}]" +2025-06-17T07:44:26.244Z [INFO] server-f04117d8-ea56-2998-e9ec-634bcc6cf8ae: entering follower state: follower="Node at f04117d8-ea56-2998-e9ec-634bcc6cf8ae [Follower]" leader-address= leader-id= +2025-06-17T07:44:26.244Z [INFO] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29 Address:f954bf62-2081-9eb4-5979-fd4aaf9f5d29} {Suffrage:Voter ID:server-f04117d8-ea56-2998-e9ec-634bcc6cf8ae Address:f04117d8-ea56-2998-e9ec-634bcc6cf8ae} {Suffrage:Voter ID:server-469c96ac-51fe-e8b3-9513-ebd33b57bf36 Address:469c96ac-51fe-e8b3-9513-ebd33b57bf36}]" +2025-06-17T07:44:26.244Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:26.244Z [INFO] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: entering follower state: follower="Node at 469c96ac-51fe-e8b3-9513-ebd33b57bf36 [Follower]" leader-address= leader-id= +2025-06-17T07:44:26.750Z [WARN] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:26.750Z [INFO] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: entering candidate state: node="Node at 469c96ac-51fe-e8b3-9513-ebd33b57bf36 [Candidate]" term=2 +2025-06-17T07:44:26.750Z [WARN] server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:26.750Z [INFO] server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29: entering candidate state: node="Node at f954bf62-2081-9eb4-5979-fd4aaf9f5d29 [Candidate]" term=2 +2025-06-17T07:44:26.750Z [INFO] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: election won: term=2 tally=2 +2025-06-17T07:44:26.750Z [INFO] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: entering leader state: leader="Node at 469c96ac-51fe-e8b3-9513-ebd33b57bf36 [Leader]" +2025-06-17T07:44:26.750Z [INFO] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: added peer, starting replication: peer=server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29 +2025-06-17T07:44:26.750Z [INFO] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: added peer, starting replication: peer=server-f04117d8-ea56-2998-e9ec-634bcc6cf8ae +2025-06-17T07:44:26.750Z [INFO] server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29: duplicate requestVote for same term: term=2 +2025-06-17T07:44:26.750Z [INFO] server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29: entering follower state: follower="Node at f954bf62-2081-9eb4-5979-fd4aaf9f5d29 [Follower]" leader-address=469c96ac-51fe-e8b3-9513-ebd33b57bf36 leader-id=server-469c96ac-51fe-e8b3-9513-ebd33b57bf36 +2025-06-17T07:44:26.750Z [INFO] server-f04117d8-ea56-2998-e9ec-634bcc6cf8ae: duplicate requestVote for same term: term=2 +2025-06-17T07:44:26.750Z [WARN] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: rejecting vote request since we have a leader: from=f954bf62-2081-9eb4-5979-fd4aaf9f5d29 leader=469c96ac-51fe-e8b3-9513-ebd33b57bf36 leader-id=server-469c96ac-51fe-e8b3-9513-ebd33b57bf36 +2025-06-17T07:44:26.750Z [INFO] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: pipelining replication: peer="{Voter server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29 f954bf62-2081-9eb4-5979-fd4aaf9f5d29}" +2025-06-17T07:44:26.750Z [INFO] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: pipelining replication: peer="{Voter server-f04117d8-ea56-2998-e9ec-634bcc6cf8ae f04117d8-ea56-2998-e9ec-634bcc6cf8ae}" +2025-06-17T07:44:27.756Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:26.750595184 +0000 UTC m=+95.354123883 (1 nodes), 506.152774ms from start of poll, 506.459334ms from cluster start. Timeout at 2025-06-17 07:44:27.756119967 +0000 UTC m=+96.359648661, 1.005524778s after stability +2025-06-17T07:44:27.756Z [INFO] creating new snapshot: path=/tmp/raft1945861591/snapshots/2-10013-1750146267756.tmp +2025-06-17T07:44:27.756Z [INFO] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: copied to local snapshot: bytes=71 +2025-06-17T07:44:27.757Z [INFO] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: snapshot restore progress: id=2-10013-1750146267756 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:27.757Z [INFO] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: removing all old logs from log store +2025-06-17T07:44:27.757Z [INFO] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: compacting logs: from=1 to=2 +2025-06-17T07:44:27.757Z [INFO] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: restored user snapshot: index=10013 +2025-06-17T07:44:27.757Z [ERROR] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: failed to get log: index=2 error="log not found" +2025-06-17T07:44:27.757Z [ERROR] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: failed to get log: index=2 error="log not found" +2025-06-17T07:44:27.757Z [INFO] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: aborting pipeline replication: peer="{Voter server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29 f954bf62-2081-9eb4-5979-fd4aaf9f5d29}" +2025-06-17T07:44:27.757Z [INFO] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: aborting pipeline replication: peer="{Voter server-f04117d8-ea56-2998-e9ec-634bcc6cf8ae f04117d8-ea56-2998-e9ec-634bcc6cf8ae}" +2025-06-17T07:44:27.764Z [ERROR] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: failed to get log: index=2 error="log not found" +2025-06-17T07:44:27.765Z [INFO] creating new snapshot: path=/tmp/raft3231717100/snapshots/2-10013-1750146267765.tmp +2025-06-17T07:44:27.765Z [INFO] server-f04117d8-ea56-2998-e9ec-634bcc6cf8ae: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:27.766Z [INFO] server-f04117d8-ea56-2998-e9ec-634bcc6cf8ae: copied to local snapshot: bytes=71 +2025-06-17T07:44:27.766Z [INFO] server-f04117d8-ea56-2998-e9ec-634bcc6cf8ae: snapshot restore progress: id=2-10013-1750146267765 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:27.766Z [INFO] server-f04117d8-ea56-2998-e9ec-634bcc6cf8ae: removing all old logs from log store +2025-06-17T07:44:27.766Z [INFO] server-f04117d8-ea56-2998-e9ec-634bcc6cf8ae: compacting logs: from=1 to=2 +2025-06-17T07:44:27.766Z [INFO] server-f04117d8-ea56-2998-e9ec-634bcc6cf8ae: Installed remote snapshot +2025-06-17T07:44:27.766Z [INFO] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: pipelining replication: peer="{Voter server-f04117d8-ea56-2998-e9ec-634bcc6cf8ae f04117d8-ea56-2998-e9ec-634bcc6cf8ae}" +2025-06-17T07:44:27.766Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:27.767Z [ERROR] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: failed to get log: index=2 error="log not found" +2025-06-17T07:44:27.767Z [INFO] creating new snapshot: path=/tmp/raft1311820264/snapshots/2-10013-1750146267767.tmp +2025-06-17T07:44:27.767Z [INFO] server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:27.767Z [INFO] server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29: copied to local snapshot: bytes=71 +2025-06-17T07:44:27.767Z [INFO] server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29: snapshot restore progress: id=2-10013-1750146267767 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-17T07:44:27.768Z [INFO] server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29: removing all old logs from log store +2025-06-17T07:44:27.768Z [INFO] server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29: compacting logs: from=1 to=2 +2025-06-17T07:44:27.768Z [INFO] server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29: Installed remote snapshot +2025-06-17T07:44:27.768Z [INFO] server-469c96ac-51fe-e8b3-9513-ebd33b57bf36: pipelining replication: peer="{Voter server-f954bf62-2081-9eb4-5979-fd4aaf9f5d29 f954bf62-2081-9eb4-5979-fd4aaf9f5d29}" +2025-06-17T07:44:28.771Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:27.766808715 +0000 UTC m=+96.370337415 (2 nodes), 1.076µs from start of poll, 1.522672866s from cluster start. Timeout at 2025-06-17 07:44:28.771944695 +0000 UTC m=+97.375473392, 1.005135977s after stability +2025-06-17T07:44:28.783Z [INFO] server-d2f3482c-3c3f-8933-06d9-b8250b210f7e: aborting pipeline replication: peer="{Voter server-6726ddd8-e6da-f043-dd14-b2044bca1d86 6726ddd8-e6da-f043-dd14-b2044bca1d86}" +2025-06-17T07:44:28.783Z [INFO] server-d2f3482c-3c3f-8933-06d9-b8250b210f7e: aborting pipeline replication: peer="{Voter server-c62ca0f2-e284-4fc6-7b7d-2e6e20fd6b3b c62ca0f2-e284-4fc6-7b7d-2e6e20fd6b3b}" +--- PASS: TestRaft_UserRestore (83.99s) + --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false (2.74s) + --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false (2.83s) + --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true (4.21s) + --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true (4.56s) + --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false (2.75s) + --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false (2.57s) + --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true (4.36s) + --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true (4.14s) + --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false (2.63s) + --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false (2.73s) + --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true (4.22s) + --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true (4.62s) + --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false (2.66s) + --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false (2.58s) + --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true (4.17s) + --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true (4.36s) + --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false (2.97s) + --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false (2.89s) + --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true (4.19s) + --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true (4.13s) + --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false (2.59s) + --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false (2.60s) + --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true (4.34s) + --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true (4.16s) === RUN TestRaft_SendSnapshotFollower -2024-05-15T01:18:36.720Z [INFO] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6 Address:6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6} {Suffrage:Voter ID:server-2ce003a4-191e-b329-7e76-f8f6256dd091 Address:2ce003a4-191e-b329-7e76-f8f6256dd091} {Suffrage:Voter ID:server-7c765286-5ad1-3c3d-731e-a6b4e94e7ac4 Address:7c765286-5ad1-3c3d-731e-a6b4e94e7ac4}]" -2024-05-15T01:18:36.720Z [INFO] server-2ce003a4-191e-b329-7e76-f8f6256dd091: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6 Address:6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6} {Suffrage:Voter ID:server-2ce003a4-191e-b329-7e76-f8f6256dd091 Address:2ce003a4-191e-b329-7e76-f8f6256dd091} {Suffrage:Voter ID:server-7c765286-5ad1-3c3d-731e-a6b4e94e7ac4 Address:7c765286-5ad1-3c3d-731e-a6b4e94e7ac4}]" -2024-05-15T01:18:36.720Z [INFO] server-7c765286-5ad1-3c3d-731e-a6b4e94e7ac4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6 Address:6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6} {Suffrage:Voter ID:server-2ce003a4-191e-b329-7e76-f8f6256dd091 Address:2ce003a4-191e-b329-7e76-f8f6256dd091} {Suffrage:Voter ID:server-7c765286-5ad1-3c3d-731e-a6b4e94e7ac4 Address:7c765286-5ad1-3c3d-731e-a6b4e94e7ac4}]" -2024-05-15T01:18:36.720Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:36.720Z [INFO] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: entering follower state: follower="Node at 6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6 [Follower]" leader-address= leader-id= -2024-05-15T01:18:36.720Z [INFO] server-2ce003a4-191e-b329-7e76-f8f6256dd091: entering follower state: follower="Node at 2ce003a4-191e-b329-7e76-f8f6256dd091 [Follower]" leader-address= leader-id= -2024-05-15T01:18:36.720Z [INFO] server-7c765286-5ad1-3c3d-731e-a6b4e94e7ac4: entering follower state: follower="Node at 7c765286-5ad1-3c3d-731e-a6b4e94e7ac4 [Follower]" leader-address= leader-id= -2024-05-15T01:18:36.778Z [WARN] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:36.778Z [INFO] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: entering candidate state: node="Node at 6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6 [Candidate]" term=2 -2024-05-15T01:18:36.778Z [INFO] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: election won: term=2 tally=2 -2024-05-15T01:18:36.778Z [INFO] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: entering leader state: leader="Node at 6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6 [Leader]" -2024-05-15T01:18:36.778Z [INFO] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: added peer, starting replication: peer=server-2ce003a4-191e-b329-7e76-f8f6256dd091 -2024-05-15T01:18:36.778Z [INFO] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: added peer, starting replication: peer=server-7c765286-5ad1-3c3d-731e-a6b4e94e7ac4 -2024-05-15T01:18:36.779Z [INFO] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: pipelining replication: peer="{Voter server-7c765286-5ad1-3c3d-731e-a6b4e94e7ac4 7c765286-5ad1-3c3d-731e-a6b4e94e7ac4}" -2024-05-15T01:18:36.779Z [INFO] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: pipelining replication: peer="{Voter server-2ce003a4-191e-b329-7e76-f8f6256dd091 2ce003a4-191e-b329-7e76-f8f6256dd091}" -2024-05-15T01:18:36.894Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:36.77881676 +0000 UTC m=+97.320201643 (2 nodes), 58.442479ms from start of poll, 58.920753ms from cluster start. Timeout at 2024-05-15 01:18:36.893911182 +0000 UTC m=+97.435296060, 115.094417ms after stability -2024-05-15T01:18:36.894Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:36.999Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:36.894217006 +0000 UTC m=+97.435601759 (1 nodes), 1.693µs from start of poll, 174.320869ms from cluster start. Timeout at 2024-05-15 01:18:36.999798484 +0000 UTC m=+97.541183418, 105.581659ms after stability -2024-05-15T01:18:37.000Z [ERROR] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: failed to pipeline appendEntries: peer="{Voter server-2ce003a4-191e-b329-7e76-f8f6256dd091 2ce003a4-191e-b329-7e76-f8f6256dd091}" error="append pipeline closed" -2024-05-15T01:18:37.000Z [INFO] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: aborting pipeline replication: peer="{Voter server-2ce003a4-191e-b329-7e76-f8f6256dd091 2ce003a4-191e-b329-7e76-f8f6256dd091}" -2024-05-15T01:18:37.000Z [ERROR] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: failed to appendEntries to: peer="{Voter server-2ce003a4-191e-b329-7e76-f8f6256dd091 2ce003a4-191e-b329-7e76-f8f6256dd091}" error="failed to connect to peer: 2ce003a4-191e-b329-7e76-f8f6256dd091" -2024-05-15T01:18:37.005Z [ERROR] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: failed to heartbeat to: peer=2ce003a4-191e-b329-7e76-f8f6256dd091 backoff time=10ms error="failed to connect to peer: 2ce003a4-191e-b329-7e76-f8f6256dd091" -2024-05-15T01:18:37.012Z [ERROR] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: failed to appendEntries to: peer="{Voter server-2ce003a4-191e-b329-7e76-f8f6256dd091 2ce003a4-191e-b329-7e76-f8f6256dd091}" error="failed to connect to peer: 2ce003a4-191e-b329-7e76-f8f6256dd091" +2025-06-17T07:44:28.784Z [INFO] server-c3f67058-ad63-2295-4793-187625a281b4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3f67058-ad63-2295-4793-187625a281b4 Address:c3f67058-ad63-2295-4793-187625a281b4} {Suffrage:Voter ID:server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857 Address:6d479a80-ce9c-3ce3-03ac-6329f2aa0857} {Suffrage:Voter ID:server-fff9898a-4b08-3955-908f-7c0b6768c198 Address:fff9898a-4b08-3955-908f-7c0b6768c198}]" +2025-06-17T07:44:28.785Z [INFO] server-c3f67058-ad63-2295-4793-187625a281b4: entering follower state: follower="Node at c3f67058-ad63-2295-4793-187625a281b4 [Follower]" leader-address= leader-id= +2025-06-17T07:44:28.785Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3f67058-ad63-2295-4793-187625a281b4 Address:c3f67058-ad63-2295-4793-187625a281b4} {Suffrage:Voter ID:server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857 Address:6d479a80-ce9c-3ce3-03ac-6329f2aa0857} {Suffrage:Voter ID:server-fff9898a-4b08-3955-908f-7c0b6768c198 Address:fff9898a-4b08-3955-908f-7c0b6768c198}]" +2025-06-17T07:44:28.785Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: entering follower state: follower="Node at 6d479a80-ce9c-3ce3-03ac-6329f2aa0857 [Follower]" leader-address= leader-id= +2025-06-17T07:44:28.785Z [INFO] server-fff9898a-4b08-3955-908f-7c0b6768c198: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3f67058-ad63-2295-4793-187625a281b4 Address:c3f67058-ad63-2295-4793-187625a281b4} {Suffrage:Voter ID:server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857 Address:6d479a80-ce9c-3ce3-03ac-6329f2aa0857} {Suffrage:Voter ID:server-fff9898a-4b08-3955-908f-7c0b6768c198 Address:fff9898a-4b08-3955-908f-7c0b6768c198}]" +2025-06-17T07:44:28.785Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:28.785Z [INFO] server-fff9898a-4b08-3955-908f-7c0b6768c198: entering follower state: follower="Node at fff9898a-4b08-3955-908f-7c0b6768c198 [Follower]" leader-address= leader-id= +2025-06-17T07:44:28.863Z [WARN] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:28.863Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: entering candidate state: node="Node at 6d479a80-ce9c-3ce3-03ac-6329f2aa0857 [Candidate]" term=2 +2025-06-17T07:44:28.863Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: election won: term=2 tally=2 +2025-06-17T07:44:28.863Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: entering leader state: leader="Node at 6d479a80-ce9c-3ce3-03ac-6329f2aa0857 [Leader]" +2025-06-17T07:44:28.863Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: added peer, starting replication: peer=server-c3f67058-ad63-2295-4793-187625a281b4 +2025-06-17T07:44:28.863Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: added peer, starting replication: peer=server-fff9898a-4b08-3955-908f-7c0b6768c198 +2025-06-17T07:44:28.863Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: pipelining replication: peer="{Voter server-c3f67058-ad63-2295-4793-187625a281b4 c3f67058-ad63-2295-4793-187625a281b4}" +2025-06-17T07:44:28.863Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: pipelining replication: peer="{Voter server-fff9898a-4b08-3955-908f-7c0b6768c198 fff9898a-4b08-3955-908f-7c0b6768c198}" +2025-06-17T07:44:28.968Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:28.863549133 +0000 UTC m=+97.467077819 (2 nodes), 78.386232ms from start of poll, 78.707419ms from cluster start. Timeout at 2025-06-17 07:44:28.968673426 +0000 UTC m=+97.572202118, 105.124299ms after stability +2025-06-17T07:44:28.968Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:29.073Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:28.968761815 +0000 UTC m=+97.572290502 (1 nodes), 682ns from start of poll, 183.920102ms from cluster start. Timeout at 2025-06-17 07:44:29.073768505 +0000 UTC m=+97.677297179, 105.006677ms after stability +2025-06-17T07:44:29.073Z [ERROR] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to pipeline appendEntries: peer="{Voter server-c3f67058-ad63-2295-4793-187625a281b4 c3f67058-ad63-2295-4793-187625a281b4}" error="append pipeline closed" +2025-06-17T07:44:29.073Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: aborting pipeline replication: peer="{Voter server-c3f67058-ad63-2295-4793-187625a281b4 c3f67058-ad63-2295-4793-187625a281b4}" +2025-06-17T07:44:29.073Z [ERROR] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to appendEntries to: peer="{Voter server-c3f67058-ad63-2295-4793-187625a281b4 c3f67058-ad63-2295-4793-187625a281b4}" error="failed to connect to peer: c3f67058-ad63-2295-4793-187625a281b4" +2025-06-17T07:44:29.074Z [ERROR] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to heartbeat to: peer=c3f67058-ad63-2295-4793-187625a281b4 backoff time=10ms error="failed to connect to peer: c3f67058-ad63-2295-4793-187625a281b4" raft_test.go:1627: [INFO] Finished apply without behind follower -2024-05-15T01:18:37.014Z [INFO] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: starting snapshot up to: index=102 -2024-05-15T01:18:37.014Z [INFO] creating new snapshot: path=/tmp/raft2538542701/snapshots/2-102-1715735917014.tmp -2024-05-15T01:18:37.022Z [ERROR] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: failed to appendEntries to: peer="{Voter server-2ce003a4-191e-b329-7e76-f8f6256dd091 2ce003a4-191e-b329-7e76-f8f6256dd091}" error="failed to connect to peer: 2ce003a4-191e-b329-7e76-f8f6256dd091" -2024-05-15T01:18:37.025Z [ERROR] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: failed to heartbeat to: peer=2ce003a4-191e-b329-7e76-f8f6256dd091 backoff time=10ms error="failed to connect to peer: 2ce003a4-191e-b329-7e76-f8f6256dd091" -2024-05-15T01:18:37.041Z [INFO] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: compacting logs: from=1 to=92 -2024-05-15T01:18:37.041Z [INFO] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: snapshot complete up to: index=102 -2024-05-15T01:18:37.041Z [ERROR] server-2ce003a4-191e-b329-7e76-f8f6256dd091: failed to take snapshot: error="nothing new to snapshot" -2024-05-15T01:18:37.041Z [INFO] server-7c765286-5ad1-3c3d-731e-a6b4e94e7ac4: starting snapshot up to: index=102 -2024-05-15T01:18:37.041Z [INFO] creating new snapshot: path=/tmp/raft1804942860/snapshots/2-102-1715735917041.tmp -2024-05-15T01:18:37.042Z [INFO] server-7c765286-5ad1-3c3d-731e-a6b4e94e7ac4: compacting logs: from=1 to=92 -2024-05-15T01:18:37.042Z [INFO] server-7c765286-5ad1-3c3d-731e-a6b4e94e7ac4: snapshot complete up to: index=102 -2024-05-15T01:18:37.042Z [ERROR] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: failed to get log: index=2 error="log not found" -2024-05-15T01:18:37.043Z [INFO] creating new snapshot: path=/tmp/raft1526043674/snapshots/2-102-1715735917043.tmp -2024-05-15T01:18:37.043Z [INFO] server-2ce003a4-191e-b329-7e76-f8f6256dd091: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% -2024-05-15T01:18:37.043Z [INFO] server-2ce003a4-191e-b329-7e76-f8f6256dd091: copied to local snapshot: bytes=693 -2024-05-15T01:18:37.044Z [INFO] server-2ce003a4-191e-b329-7e76-f8f6256dd091: snapshot restore progress: id=2-102-1715735917043 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2024-05-15T01:18:37.044Z [INFO] server-2ce003a4-191e-b329-7e76-f8f6256dd091: Installed remote snapshot -2024-05-15T01:18:37.044Z [INFO] server-6dbe9b2b-de05-2bc7-c04b-b23e93e2ccb6: aborting pipeline replication: peer="{Voter server-7c765286-5ad1-3c3d-731e-a6b4e94e7ac4 7c765286-5ad1-3c3d-731e-a6b4e94e7ac4}" ---- PASS: TestRaft_SendSnapshotFollower (0.34s) +2025-06-17T07:44:29.077Z [ERROR] server-c3f67058-ad63-2295-4793-187625a281b4: failed to take snapshot: error="nothing new to snapshot" +2025-06-17T07:44:29.077Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: starting snapshot up to: index=102 +2025-06-17T07:44:29.077Z [INFO] creating new snapshot: path=/tmp/raft643208886/snapshots/2-102-1750146269077.tmp +2025-06-17T07:44:29.084Z [ERROR] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to appendEntries to: peer="{Voter server-c3f67058-ad63-2295-4793-187625a281b4 c3f67058-ad63-2295-4793-187625a281b4}" error="failed to connect to peer: c3f67058-ad63-2295-4793-187625a281b4" +2025-06-17T07:44:29.095Z [ERROR] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to appendEntries to: peer="{Voter server-c3f67058-ad63-2295-4793-187625a281b4 c3f67058-ad63-2295-4793-187625a281b4}" error="failed to connect to peer: c3f67058-ad63-2295-4793-187625a281b4" +2025-06-17T07:44:29.095Z [ERROR] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to heartbeat to: peer=c3f67058-ad63-2295-4793-187625a281b4 backoff time=10ms error="failed to connect to peer: c3f67058-ad63-2295-4793-187625a281b4" +2025-06-17T07:44:29.114Z [ERROR] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to heartbeat to: peer=c3f67058-ad63-2295-4793-187625a281b4 backoff time=10ms error="failed to connect to peer: c3f67058-ad63-2295-4793-187625a281b4" +2025-06-17T07:44:29.116Z [WARN] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to contact: server-id=server-c3f67058-ad63-2295-4793-187625a281b4 time=50.981506ms +2025-06-17T07:44:29.125Z [ERROR] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to appendEntries to: peer="{Voter server-c3f67058-ad63-2295-4793-187625a281b4 c3f67058-ad63-2295-4793-187625a281b4}" error="failed to connect to peer: c3f67058-ad63-2295-4793-187625a281b4" +2025-06-17T07:44:29.134Z [ERROR] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to heartbeat to: peer=c3f67058-ad63-2295-4793-187625a281b4 backoff time=20ms error="failed to connect to peer: c3f67058-ad63-2295-4793-187625a281b4" +2025-06-17T07:44:29.158Z [WARN] server-c3f67058-ad63-2295-4793-187625a281b4: heartbeat timeout reached, starting election: last-leader-addr=6d479a80-ce9c-3ce3-03ac-6329f2aa0857 last-leader-id=server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857 +2025-06-17T07:44:29.158Z [INFO] server-c3f67058-ad63-2295-4793-187625a281b4: entering candidate state: node="Node at c3f67058-ad63-2295-4793-187625a281b4 [Candidate]" term=3 +2025-06-17T07:44:29.158Z [ERROR] server-c3f67058-ad63-2295-4793-187625a281b4: failed to make requestVote RPC: target="{Voter server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857 6d479a80-ce9c-3ce3-03ac-6329f2aa0857}" error="failed to connect to peer: 6d479a80-ce9c-3ce3-03ac-6329f2aa0857" term=3 +2025-06-17T07:44:29.158Z [ERROR] server-c3f67058-ad63-2295-4793-187625a281b4: failed to make requestVote RPC: target="{Voter server-fff9898a-4b08-3955-908f-7c0b6768c198 fff9898a-4b08-3955-908f-7c0b6768c198}" error="failed to connect to peer: fff9898a-4b08-3955-908f-7c0b6768c198" term=3 +2025-06-17T07:44:29.163Z [ERROR] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to heartbeat to: peer=c3f67058-ad63-2295-4793-187625a281b4 backoff time=25ms error="failed to connect to peer: c3f67058-ad63-2295-4793-187625a281b4" +2025-06-17T07:44:29.164Z [WARN] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to contact: server-id=server-c3f67058-ad63-2295-4793-187625a281b4 time=99.147677ms +2025-06-17T07:44:29.174Z [ERROR] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to appendEntries to: peer="{Voter server-c3f67058-ad63-2295-4793-187625a281b4 c3f67058-ad63-2295-4793-187625a281b4}" error="failed to connect to peer: c3f67058-ad63-2295-4793-187625a281b4" +2025-06-17T07:44:29.195Z [ERROR] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to heartbeat to: peer=c3f67058-ad63-2295-4793-187625a281b4 backoff time=25ms error="failed to connect to peer: c3f67058-ad63-2295-4793-187625a281b4" +2025-06-17T07:44:29.215Z [WARN] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to contact: server-id=server-c3f67058-ad63-2295-4793-187625a281b4 time=149.425078ms +2025-06-17T07:44:29.224Z [WARN] server-c3f67058-ad63-2295-4793-187625a281b4: Election timeout reached, restarting election +2025-06-17T07:44:29.224Z [INFO] server-c3f67058-ad63-2295-4793-187625a281b4: entering candidate state: node="Node at c3f67058-ad63-2295-4793-187625a281b4 [Candidate]" term=4 +2025-06-17T07:44:29.224Z [ERROR] server-c3f67058-ad63-2295-4793-187625a281b4: failed to make requestVote RPC: target="{Voter server-fff9898a-4b08-3955-908f-7c0b6768c198 fff9898a-4b08-3955-908f-7c0b6768c198}" error="failed to connect to peer: fff9898a-4b08-3955-908f-7c0b6768c198" term=4 +2025-06-17T07:44:29.224Z [ERROR] server-c3f67058-ad63-2295-4793-187625a281b4: failed to make requestVote RPC: target="{Voter server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857 6d479a80-ce9c-3ce3-03ac-6329f2aa0857}" error="failed to connect to peer: 6d479a80-ce9c-3ce3-03ac-6329f2aa0857" term=4 +2025-06-17T07:44:29.230Z [ERROR] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to heartbeat to: peer=c3f67058-ad63-2295-4793-187625a281b4 backoff time=25ms error="failed to connect to peer: c3f67058-ad63-2295-4793-187625a281b4" +2025-06-17T07:44:29.248Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: compacting logs: from=1 to=92 +2025-06-17T07:44:29.248Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: snapshot complete up to: index=102 +2025-06-17T07:44:29.248Z [INFO] server-fff9898a-4b08-3955-908f-7c0b6768c198: starting snapshot up to: index=102 +2025-06-17T07:44:29.248Z [INFO] creating new snapshot: path=/tmp/raft137762179/snapshots/2-102-1750146269248.tmp +2025-06-17T07:44:29.249Z [INFO] server-fff9898a-4b08-3955-908f-7c0b6768c198: compacting logs: from=1 to=92 +2025-06-17T07:44:29.249Z [INFO] server-fff9898a-4b08-3955-908f-7c0b6768c198: snapshot complete up to: index=102 +2025-06-17T07:44:29.263Z [ERROR] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to get log: index=2 error="log not found" +2025-06-17T07:44:29.263Z [INFO] server-c3f67058-ad63-2295-4793-187625a281b4: ignoring installSnapshot request with older term than current term: request-term=2 current-term=4 +2025-06-17T07:44:29.263Z [ERROR] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: peer has newer term, stopping replication: peer="{Voter server-c3f67058-ad63-2295-4793-187625a281b4 c3f67058-ad63-2295-4793-187625a281b4}" +2025-06-17T07:44:29.263Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: entering follower state: follower="Node at 6d479a80-ce9c-3ce3-03ac-6329f2aa0857 [Follower]" leader-address= leader-id= +2025-06-17T07:44:29.263Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: aborting pipeline replication: peer="{Voter server-fff9898a-4b08-3955-908f-7c0b6768c198 fff9898a-4b08-3955-908f-7c0b6768c198}" +2025-06-17T07:44:29.284Z [WARN] server-c3f67058-ad63-2295-4793-187625a281b4: Election timeout reached, restarting election +2025-06-17T07:44:29.284Z [INFO] server-c3f67058-ad63-2295-4793-187625a281b4: entering candidate state: node="Node at c3f67058-ad63-2295-4793-187625a281b4 [Candidate]" term=5 +2025-06-17T07:44:29.284Z [WARN] server-fff9898a-4b08-3955-908f-7c0b6768c198: rejecting vote request since we have a leader: from=c3f67058-ad63-2295-4793-187625a281b4 leader=6d479a80-ce9c-3ce3-03ac-6329f2aa0857 leader-id=server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857 +2025-06-17T07:44:29.284Z [WARN] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: rejecting vote request since our last index is greater: candidate=c3f67058-ad63-2295-4793-187625a281b4 last-index=102 last-candidate-index=2 +2025-06-17T07:44:29.314Z [WARN] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:29.314Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: entering candidate state: node="Node at 6d479a80-ce9c-3ce3-03ac-6329f2aa0857 [Candidate]" term=6 +2025-06-17T07:44:29.314Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: election won: term=6 tally=2 +2025-06-17T07:44:29.314Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: entering leader state: leader="Node at 6d479a80-ce9c-3ce3-03ac-6329f2aa0857 [Leader]" +2025-06-17T07:44:29.314Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: added peer, starting replication: peer=server-c3f67058-ad63-2295-4793-187625a281b4 +2025-06-17T07:44:29.314Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: added peer, starting replication: peer=server-fff9898a-4b08-3955-908f-7c0b6768c198 +2025-06-17T07:44:29.314Z [INFO] server-c3f67058-ad63-2295-4793-187625a281b4: entering follower state: follower="Node at c3f67058-ad63-2295-4793-187625a281b4 [Follower]" leader-address= leader-id= +2025-06-17T07:44:29.315Z [WARN] server-c3f67058-ad63-2295-4793-187625a281b4: failed to get previous log: previous-index=102 last-index=2 error="log not found" +2025-06-17T07:44:29.315Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: pipelining replication: peer="{Voter server-fff9898a-4b08-3955-908f-7c0b6768c198 fff9898a-4b08-3955-908f-7c0b6768c198}" +2025-06-17T07:44:29.315Z [WARN] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: appendEntries rejected, sending older logs: peer="{Voter server-c3f67058-ad63-2295-4793-187625a281b4 c3f67058-ad63-2295-4793-187625a281b4}" next=3 +2025-06-17T07:44:29.315Z [ERROR] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to get log: index=2 error="log not found" +2025-06-17T07:44:29.315Z [INFO] creating new snapshot: path=/tmp/raft3969889319/snapshots/2-102-1750146269315.tmp +2025-06-17T07:44:29.315Z [INFO] server-c3f67058-ad63-2295-4793-187625a281b4: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% +2025-06-17T07:44:29.315Z [INFO] server-c3f67058-ad63-2295-4793-187625a281b4: copied to local snapshot: bytes=693 +2025-06-17T07:44:29.316Z [INFO] server-c3f67058-ad63-2295-4793-187625a281b4: snapshot restore progress: id=2-102-1750146269315 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-06-17T07:44:29.316Z [INFO] server-c3f67058-ad63-2295-4793-187625a281b4: Installed remote snapshot +2025-06-17T07:44:29.316Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: pipelining replication: peer="{Voter server-c3f67058-ad63-2295-4793-187625a281b4 c3f67058-ad63-2295-4793-187625a281b4}" +2025-06-17T07:44:29.320Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: aborting pipeline replication: peer="{Voter server-fff9898a-4b08-3955-908f-7c0b6768c198 fff9898a-4b08-3955-908f-7c0b6768c198}" +2025-06-17T07:44:29.320Z [INFO] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: aborting pipeline replication: peer="{Voter server-c3f67058-ad63-2295-4793-187625a281b4 c3f67058-ad63-2295-4793-187625a281b4}" +2025-06-17T07:44:29.821Z [ERROR] server-6d479a80-ce9c-3ce3-03ac-6329f2aa0857: failed to heartbeat to: peer=c3f67058-ad63-2295-4793-187625a281b4 backoff time=10ms error="command timed out" +--- PASS: TestRaft_SendSnapshotFollower (1.04s) === RUN TestRaft_SendSnapshotAndLogsFollower -2024-05-15T01:18:37.062Z [INFO] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd Address:e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd} {Suffrage:Voter ID:server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6 Address:ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6} {Suffrage:Voter ID:server-08c69e94-8c7e-006a-095c-0164ff0ddebf Address:08c69e94-8c7e-006a-095c-0164ff0ddebf}]" -2024-05-15T01:18:37.062Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd Address:e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd} {Suffrage:Voter ID:server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6 Address:ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6} {Suffrage:Voter ID:server-08c69e94-8c7e-006a-095c-0164ff0ddebf Address:08c69e94-8c7e-006a-095c-0164ff0ddebf}]" -2024-05-15T01:18:37.063Z [INFO] server-08c69e94-8c7e-006a-095c-0164ff0ddebf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd Address:e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd} {Suffrage:Voter ID:server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6 Address:ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6} {Suffrage:Voter ID:server-08c69e94-8c7e-006a-095c-0164ff0ddebf Address:08c69e94-8c7e-006a-095c-0164ff0ddebf}]" -2024-05-15T01:18:37.063Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:37.063Z [INFO] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: entering follower state: follower="Node at e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd [Follower]" leader-address= leader-id= -2024-05-15T01:18:37.063Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: entering follower state: follower="Node at ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6 [Follower]" leader-address= leader-id= -2024-05-15T01:18:37.063Z [INFO] server-08c69e94-8c7e-006a-095c-0164ff0ddebf: entering follower state: follower="Node at 08c69e94-8c7e-006a-095c-0164ff0ddebf [Follower]" leader-address= leader-id= -2024-05-15T01:18:37.135Z [WARN] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:37.135Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: entering candidate state: node="Node at ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6 [Candidate]" term=2 -2024-05-15T01:18:37.135Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: election won: term=2 tally=2 -2024-05-15T01:18:37.135Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: entering leader state: leader="Node at ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6 [Leader]" -2024-05-15T01:18:37.135Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: added peer, starting replication: peer=server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd -2024-05-15T01:18:37.135Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: added peer, starting replication: peer=server-08c69e94-8c7e-006a-095c-0164ff0ddebf -2024-05-15T01:18:37.135Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: pipelining replication: peer="{Voter server-08c69e94-8c7e-006a-095c-0164ff0ddebf 08c69e94-8c7e-006a-095c-0164ff0ddebf}" -2024-05-15T01:18:37.135Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: pipelining replication: peer="{Voter server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd}" -2024-05-15T01:18:37.242Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:37.135421196 +0000 UTC m=+97.676806204 (2 nodes), 72.237708ms from start of poll, 72.971342ms from cluster start. Timeout at 2024-05-15 01:18:37.242571546 +0000 UTC m=+97.783956408, 107.150204ms after stability -2024-05-15T01:18:37.242Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:37.348Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:37.242691433 +0000 UTC m=+97.784076207 (1 nodes), 1.897µs from start of poll, 180.241345ms from cluster start. Timeout at 2024-05-15 01:18:37.348869881 +0000 UTC m=+97.890254874, 106.178667ms after stability -2024-05-15T01:18:37.349Z [ERROR] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: failed to pipeline appendEntries: peer="{Voter server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd}" error="append pipeline closed" -2024-05-15T01:18:37.349Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: aborting pipeline replication: peer="{Voter server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd}" -2024-05-15T01:18:37.349Z [ERROR] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: failed to appendEntries to: peer="{Voter server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd}" error="failed to connect to peer: e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd" -2024-05-15T01:18:37.352Z [ERROR] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: failed to heartbeat to: peer=e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd backoff time=10ms error="failed to connect to peer: e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd" -2024-05-15T01:18:37.363Z [ERROR] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: failed to appendEntries to: peer="{Voter server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd}" error="failed to connect to peer: e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd" +2025-06-17T07:44:29.822Z [INFO] server-eac9d437-396d-0a38-89d0-3a8f293835a5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eac9d437-396d-0a38-89d0-3a8f293835a5 Address:eac9d437-396d-0a38-89d0-3a8f293835a5} {Suffrage:Voter ID:server-1b400bc6-db25-4a67-1ae5-9be885f2ee34 Address:1b400bc6-db25-4a67-1ae5-9be885f2ee34} {Suffrage:Voter ID:server-6e3c6e88-89c6-a9b1-bb0c-8db445a25e93 Address:6e3c6e88-89c6-a9b1-bb0c-8db445a25e93}]" +2025-06-17T07:44:29.822Z [INFO] server-eac9d437-396d-0a38-89d0-3a8f293835a5: entering follower state: follower="Node at eac9d437-396d-0a38-89d0-3a8f293835a5 [Follower]" leader-address= leader-id= +2025-06-17T07:44:29.822Z [INFO] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eac9d437-396d-0a38-89d0-3a8f293835a5 Address:eac9d437-396d-0a38-89d0-3a8f293835a5} {Suffrage:Voter ID:server-1b400bc6-db25-4a67-1ae5-9be885f2ee34 Address:1b400bc6-db25-4a67-1ae5-9be885f2ee34} {Suffrage:Voter ID:server-6e3c6e88-89c6-a9b1-bb0c-8db445a25e93 Address:6e3c6e88-89c6-a9b1-bb0c-8db445a25e93}]" +2025-06-17T07:44:29.822Z [INFO] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: entering follower state: follower="Node at 1b400bc6-db25-4a67-1ae5-9be885f2ee34 [Follower]" leader-address= leader-id= +2025-06-17T07:44:29.822Z [INFO] server-6e3c6e88-89c6-a9b1-bb0c-8db445a25e93: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eac9d437-396d-0a38-89d0-3a8f293835a5 Address:eac9d437-396d-0a38-89d0-3a8f293835a5} {Suffrage:Voter ID:server-1b400bc6-db25-4a67-1ae5-9be885f2ee34 Address:1b400bc6-db25-4a67-1ae5-9be885f2ee34} {Suffrage:Voter ID:server-6e3c6e88-89c6-a9b1-bb0c-8db445a25e93 Address:6e3c6e88-89c6-a9b1-bb0c-8db445a25e93}]" +2025-06-17T07:44:29.822Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:29.822Z [INFO] server-6e3c6e88-89c6-a9b1-bb0c-8db445a25e93: entering follower state: follower="Node at 6e3c6e88-89c6-a9b1-bb0c-8db445a25e93 [Follower]" leader-address= leader-id= +2025-06-17T07:44:29.875Z [WARN] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:29.875Z [INFO] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: entering candidate state: node="Node at 1b400bc6-db25-4a67-1ae5-9be885f2ee34 [Candidate]" term=2 +2025-06-17T07:44:29.875Z [INFO] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: election won: term=2 tally=2 +2025-06-17T07:44:29.875Z [INFO] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: entering leader state: leader="Node at 1b400bc6-db25-4a67-1ae5-9be885f2ee34 [Leader]" +2025-06-17T07:44:29.875Z [INFO] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: added peer, starting replication: peer=server-eac9d437-396d-0a38-89d0-3a8f293835a5 +2025-06-17T07:44:29.875Z [INFO] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: added peer, starting replication: peer=server-6e3c6e88-89c6-a9b1-bb0c-8db445a25e93 +2025-06-17T07:44:29.875Z [INFO] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: pipelining replication: peer="{Voter server-eac9d437-396d-0a38-89d0-3a8f293835a5 eac9d437-396d-0a38-89d0-3a8f293835a5}" +2025-06-17T07:44:29.875Z [INFO] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: pipelining replication: peer="{Voter server-6e3c6e88-89c6-a9b1-bb0c-8db445a25e93 6e3c6e88-89c6-a9b1-bb0c-8db445a25e93}" +2025-06-17T07:44:29.981Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:29.875785663 +0000 UTC m=+98.479314373 (2 nodes), 53.053077ms from start of poll, 53.498675ms from cluster start. Timeout at 2025-06-17 07:44:29.981674438 +0000 UTC m=+98.585203130, 105.888757ms after stability +2025-06-17T07:44:29.981Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:30.087Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:29.981732591 +0000 UTC m=+98.585261282 (1 nodes), 687ns from start of poll, 159.445584ms from cluster start. Timeout at 2025-06-17 07:44:30.087382432 +0000 UTC m=+98.690911150, 105.649868ms after stability +2025-06-17T07:44:30.087Z [ERROR] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: failed to pipeline appendEntries: peer="{Voter server-eac9d437-396d-0a38-89d0-3a8f293835a5 eac9d437-396d-0a38-89d0-3a8f293835a5}" error="append pipeline closed" +2025-06-17T07:44:30.087Z [INFO] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: aborting pipeline replication: peer="{Voter server-eac9d437-396d-0a38-89d0-3a8f293835a5 eac9d437-396d-0a38-89d0-3a8f293835a5}" +2025-06-17T07:44:30.087Z [ERROR] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: failed to appendEntries to: peer="{Voter server-eac9d437-396d-0a38-89d0-3a8f293835a5 eac9d437-396d-0a38-89d0-3a8f293835a5}" error="failed to connect to peer: eac9d437-396d-0a38-89d0-3a8f293835a5" +2025-06-17T07:44:30.090Z [ERROR] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: failed to heartbeat to: peer=eac9d437-396d-0a38-89d0-3a8f293835a5 backoff time=10ms error="failed to connect to peer: eac9d437-396d-0a38-89d0-3a8f293835a5" raft_test.go:1669: [INFO] Finished apply without behind follower -2024-05-15T01:18:37.366Z [ERROR] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: failed to take snapshot: error="nothing new to snapshot" -2024-05-15T01:18:37.366Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: starting snapshot up to: index=102 -2024-05-15T01:18:37.366Z [INFO] creating new snapshot: path=/tmp/raft113164515/snapshots/2-102-1715735917366.tmp -2024-05-15T01:18:37.369Z [ERROR] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: failed to heartbeat to: peer=e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd backoff time=10ms error="failed to connect to peer: e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd" -2024-05-15T01:18:37.373Z [ERROR] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: failed to appendEntries to: peer="{Voter server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd}" error="failed to connect to peer: e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd" -2024-05-15T01:18:37.389Z [ERROR] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: failed to heartbeat to: peer=e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd backoff time=10ms error="failed to connect to peer: e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd" -2024-05-15T01:18:37.403Z [WARN] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: failed to contact: server-id=server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd time=59.703024ms -2024-05-15T01:18:37.405Z [ERROR] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: failed to heartbeat to: peer=e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd backoff time=20ms error="failed to connect to peer: e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd" -2024-05-15T01:18:37.410Z [ERROR] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: failed to appendEntries to: peer="{Voter server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd}" error="failed to connect to peer: e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd" -2024-05-15T01:18:37.417Z [WARN] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: heartbeat timeout reached, starting election: last-leader-addr=ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6 last-leader-id=server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6 -2024-05-15T01:18:37.417Z [INFO] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: entering candidate state: node="Node at e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd [Candidate]" term=3 -2024-05-15T01:18:37.417Z [ERROR] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: failed to make requestVote RPC: target="{Voter server-08c69e94-8c7e-006a-095c-0164ff0ddebf 08c69e94-8c7e-006a-095c-0164ff0ddebf}" error="failed to connect to peer: 08c69e94-8c7e-006a-095c-0164ff0ddebf" term=3 -2024-05-15T01:18:37.417Z [ERROR] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: failed to make requestVote RPC: target="{Voter server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6 ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6}" error="failed to connect to peer: ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6" term=3 -2024-05-15T01:18:37.435Z [ERROR] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: failed to heartbeat to: peer=e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd backoff time=25ms error="failed to connect to peer: e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd" -2024-05-15T01:18:37.451Z [WARN] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: failed to contact: server-id=server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd time=107.700251ms -2024-05-15T01:18:37.455Z [ERROR] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: failed to appendEntries to: peer="{Voter server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd}" error="failed to connect to peer: e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd" -2024-05-15T01:18:37.470Z [ERROR] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: failed to heartbeat to: peer=e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd backoff time=25ms error="failed to connect to peer: e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd" -2024-05-15T01:18:37.500Z [WARN] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: Election timeout reached, restarting election -2024-05-15T01:18:37.501Z [INFO] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: entering candidate state: node="Node at e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd [Candidate]" term=4 -2024-05-15T01:18:37.501Z [ERROR] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: failed to make requestVote RPC: target="{Voter server-08c69e94-8c7e-006a-095c-0164ff0ddebf 08c69e94-8c7e-006a-095c-0164ff0ddebf}" error="failed to connect to peer: 08c69e94-8c7e-006a-095c-0164ff0ddebf" term=4 -2024-05-15T01:18:37.501Z [ERROR] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: failed to make requestVote RPC: target="{Voter server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6 ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6}" error="failed to connect to peer: ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6" term=4 -2024-05-15T01:18:37.501Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: compacting logs: from=1 to=92 -2024-05-15T01:18:37.501Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: snapshot complete up to: index=102 -2024-05-15T01:18:37.502Z [INFO] server-08c69e94-8c7e-006a-095c-0164ff0ddebf: starting snapshot up to: index=102 -2024-05-15T01:18:37.502Z [INFO] creating new snapshot: path=/tmp/raft534130947/snapshots/2-102-1715735917502.tmp -2024-05-15T01:18:37.503Z [INFO] server-08c69e94-8c7e-006a-095c-0164ff0ddebf: compacting logs: from=1 to=92 -2024-05-15T01:18:37.503Z [INFO] server-08c69e94-8c7e-006a-095c-0164ff0ddebf: snapshot complete up to: index=102 -2024-05-15T01:18:37.505Z [ERROR] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: failed to heartbeat to: peer=e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd backoff time=25ms error="failed to connect to peer: e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd" +2025-06-17T07:44:30.090Z [ERROR] server-eac9d437-396d-0a38-89d0-3a8f293835a5: failed to take snapshot: error="nothing new to snapshot" +2025-06-17T07:44:30.090Z [INFO] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: starting snapshot up to: index=102 +2025-06-17T07:44:30.090Z [INFO] creating new snapshot: path=/tmp/raft4065855880/snapshots/2-102-1750146270090.tmp +2025-06-17T07:44:30.090Z [INFO] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: compacting logs: from=1 to=92 +2025-06-17T07:44:30.091Z [INFO] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: snapshot complete up to: index=102 +2025-06-17T07:44:30.091Z [INFO] server-6e3c6e88-89c6-a9b1-bb0c-8db445a25e93: starting snapshot up to: index=99 +2025-06-17T07:44:30.091Z [INFO] creating new snapshot: path=/tmp/raft565361252/snapshots/2-99-1750146270091.tmp +2025-06-17T07:44:30.091Z [INFO] server-6e3c6e88-89c6-a9b1-bb0c-8db445a25e93: compacting logs: from=1 to=92 +2025-06-17T07:44:30.091Z [INFO] server-6e3c6e88-89c6-a9b1-bb0c-8db445a25e93: snapshot complete up to: index=99 raft_test.go:1690: [INFO] Finished apply without behind follower -2024-05-15T01:18:37.560Z [ERROR] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: failed to get log: index=2 error="log not found" -2024-05-15T01:18:37.563Z [INFO] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: ignoring installSnapshot request with older term than current term: request-term=2 current-term=4 -2024-05-15T01:18:37.564Z [ERROR] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: peer has newer term, stopping replication: peer="{Voter server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd}" -2024-05-15T01:18:37.565Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: entering follower state: follower="Node at ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6 [Follower]" leader-address= leader-id= -2024-05-15T01:18:37.565Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: aborting pipeline replication: peer="{Voter server-08c69e94-8c7e-006a-095c-0164ff0ddebf 08c69e94-8c7e-006a-095c-0164ff0ddebf}" -2024-05-15T01:18:37.574Z [WARN] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: Election timeout reached, restarting election -2024-05-15T01:18:37.574Z [INFO] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: entering candidate state: node="Node at e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd [Candidate]" term=5 -2024-05-15T01:18:37.575Z [WARN] server-08c69e94-8c7e-006a-095c-0164ff0ddebf: rejecting vote request since we have a leader: from=e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd leader=ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6 leader-id=server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6 -2024-05-15T01:18:37.575Z [WARN] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: rejecting vote request since our last index is greater: candidate=e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd last-index=202 last-candidate-index=2 -2024-05-15T01:18:37.635Z [WARN] server-08c69e94-8c7e-006a-095c-0164ff0ddebf: heartbeat timeout reached, starting election: last-leader-addr=ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6 last-leader-id=server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6 -2024-05-15T01:18:37.636Z [INFO] server-08c69e94-8c7e-006a-095c-0164ff0ddebf: entering candidate state: node="Node at 08c69e94-8c7e-006a-095c-0164ff0ddebf [Candidate]" term=3 -2024-05-15T01:18:37.661Z [INFO] server-08c69e94-8c7e-006a-095c-0164ff0ddebf: entering follower state: follower="Node at 08c69e94-8c7e-006a-095c-0164ff0ddebf [Follower]" leader-address= leader-id= -2024-05-15T01:18:37.663Z [WARN] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: Election timeout reached, restarting election -2024-05-15T01:18:37.664Z [INFO] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: entering candidate state: node="Node at e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd [Candidate]" term=6 -2024-05-15T01:18:37.664Z [WARN] server-08c69e94-8c7e-006a-095c-0164ff0ddebf: rejecting vote request since our last index is greater: candidate=e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd last-index=202 last-candidate-index=2 -2024-05-15T01:18:37.673Z [WARN] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: rejecting vote request since our last index is greater: candidate=e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd last-index=202 last-candidate-index=2 -2024-05-15T01:18:37.673Z [WARN] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:37.673Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: entering candidate state: node="Node at ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6 [Candidate]" term=7 -2024-05-15T01:18:37.673Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: election won: term=7 tally=2 -2024-05-15T01:18:37.673Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: entering leader state: leader="Node at ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6 [Leader]" -2024-05-15T01:18:37.673Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: added peer, starting replication: peer=server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd -2024-05-15T01:18:37.673Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: added peer, starting replication: peer=server-08c69e94-8c7e-006a-095c-0164ff0ddebf -2024-05-15T01:18:37.674Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: pipelining replication: peer="{Voter server-08c69e94-8c7e-006a-095c-0164ff0ddebf 08c69e94-8c7e-006a-095c-0164ff0ddebf}" -2024-05-15T01:18:37.674Z [INFO] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: entering follower state: follower="Node at e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd [Follower]" leader-address= leader-id= -2024-05-15T01:18:37.674Z [WARN] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: failed to get previous log: previous-index=202 last-index=2 error="log not found" -2024-05-15T01:18:37.674Z [WARN] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: appendEntries rejected, sending older logs: peer="{Voter server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd}" next=3 -2024-05-15T01:18:37.674Z [ERROR] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: failed to get log: index=2 error="log not found" -2024-05-15T01:18:37.675Z [INFO] creating new snapshot: path=/tmp/raft1721275900/snapshots/2-102-1715735917675.tmp -2024-05-15T01:18:37.675Z [INFO] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% -2024-05-15T01:18:37.685Z [INFO] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: copied to local snapshot: bytes=693 -2024-05-15T01:18:37.686Z [INFO] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: snapshot restore progress: id=2-102-1715735917675 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2024-05-15T01:18:37.686Z [INFO] server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd: Installed remote snapshot -2024-05-15T01:18:37.686Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: pipelining replication: peer="{Voter server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd}" -2024-05-15T01:18:37.693Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: aborting pipeline replication: peer="{Voter server-08c69e94-8c7e-006a-095c-0164ff0ddebf 08c69e94-8c7e-006a-095c-0164ff0ddebf}" -2024-05-15T01:18:37.693Z [INFO] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: aborting pipeline replication: peer="{Voter server-e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd e0f8b8c1-0a06-d564-1f82-f8cb2e944ecd}" -2024-05-15T01:18:38.214Z [ERROR] server-ddf37cb6-dcfa-9cc4-2596-9c0cb720fad6: failed to heartbeat to: peer=08c69e94-8c7e-006a-095c-0164ff0ddebf backoff time=10ms error="command timed out" ---- PASS: TestRaft_SendSnapshotAndLogsFollower (1.15s) +2025-06-17T07:44:30.098Z [ERROR] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: failed to get log: index=2 error="log not found" +2025-06-17T07:44:30.098Z [INFO] creating new snapshot: path=/tmp/raft3439209563/snapshots/2-102-1750146270098.tmp +2025-06-17T07:44:30.099Z [INFO] server-eac9d437-396d-0a38-89d0-3a8f293835a5: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% +2025-06-17T07:44:30.099Z [INFO] server-eac9d437-396d-0a38-89d0-3a8f293835a5: copied to local snapshot: bytes=693 +2025-06-17T07:44:30.099Z [INFO] server-eac9d437-396d-0a38-89d0-3a8f293835a5: snapshot restore progress: id=2-102-1750146270098 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-06-17T07:44:30.099Z [INFO] server-eac9d437-396d-0a38-89d0-3a8f293835a5: Installed remote snapshot +2025-06-17T07:44:30.099Z [INFO] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: pipelining replication: peer="{Voter server-eac9d437-396d-0a38-89d0-3a8f293835a5 eac9d437-396d-0a38-89d0-3a8f293835a5}" +2025-06-17T07:44:30.107Z [INFO] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: aborting pipeline replication: peer="{Voter server-6e3c6e88-89c6-a9b1-bb0c-8db445a25e93 6e3c6e88-89c6-a9b1-bb0c-8db445a25e93}" +2025-06-17T07:44:30.107Z [INFO] server-1b400bc6-db25-4a67-1ae5-9be885f2ee34: aborting pipeline replication: peer="{Voter server-eac9d437-396d-0a38-89d0-3a8f293835a5 eac9d437-396d-0a38-89d0-3a8f293835a5}" +--- PASS: TestRaft_SendSnapshotAndLogsFollower (0.29s) === RUN TestRaft_ReJoinFollower -2024-05-15T01:18:38.221Z [INFO] server-d37b1d34-822e-d540-49e2-2c1c999881db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d37b1d34-822e-d540-49e2-2c1c999881db Address:d37b1d34-822e-d540-49e2-2c1c999881db} {Suffrage:Voter ID:server-2fcae558-1d4d-24da-8068-d722fcb14b3d Address:2fcae558-1d4d-24da-8068-d722fcb14b3d} {Suffrage:Voter ID:server-d07aff94-4157-5498-b096-28a4065697f3 Address:d07aff94-4157-5498-b096-28a4065697f3}]" -2024-05-15T01:18:38.221Z [INFO] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d37b1d34-822e-d540-49e2-2c1c999881db Address:d37b1d34-822e-d540-49e2-2c1c999881db} {Suffrage:Voter ID:server-2fcae558-1d4d-24da-8068-d722fcb14b3d Address:2fcae558-1d4d-24da-8068-d722fcb14b3d} {Suffrage:Voter ID:server-d07aff94-4157-5498-b096-28a4065697f3 Address:d07aff94-4157-5498-b096-28a4065697f3}]" -2024-05-15T01:18:38.222Z [INFO] server-d07aff94-4157-5498-b096-28a4065697f3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d37b1d34-822e-d540-49e2-2c1c999881db Address:d37b1d34-822e-d540-49e2-2c1c999881db} {Suffrage:Voter ID:server-2fcae558-1d4d-24da-8068-d722fcb14b3d Address:2fcae558-1d4d-24da-8068-d722fcb14b3d} {Suffrage:Voter ID:server-d07aff94-4157-5498-b096-28a4065697f3 Address:d07aff94-4157-5498-b096-28a4065697f3}]" -2024-05-15T01:18:38.222Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:38.222Z [INFO] server-d37b1d34-822e-d540-49e2-2c1c999881db: entering follower state: follower="Node at d37b1d34-822e-d540-49e2-2c1c999881db [Follower]" leader-address= leader-id= -2024-05-15T01:18:38.222Z [INFO] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: entering follower state: follower="Node at 2fcae558-1d4d-24da-8068-d722fcb14b3d [Follower]" leader-address= leader-id= -2024-05-15T01:18:38.222Z [INFO] server-d07aff94-4157-5498-b096-28a4065697f3: entering follower state: follower="Node at d07aff94-4157-5498-b096-28a4065697f3 [Follower]" leader-address= leader-id= -2024-05-15T01:18:38.289Z [WARN] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:38.289Z [INFO] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: entering candidate state: node="Node at 2fcae558-1d4d-24da-8068-d722fcb14b3d [Candidate]" term=2 -2024-05-15T01:18:38.289Z [INFO] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: election won: term=2 tally=2 -2024-05-15T01:18:38.289Z [INFO] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: entering leader state: leader="Node at 2fcae558-1d4d-24da-8068-d722fcb14b3d [Leader]" -2024-05-15T01:18:38.289Z [INFO] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: added peer, starting replication: peer=server-d37b1d34-822e-d540-49e2-2c1c999881db -2024-05-15T01:18:38.289Z [INFO] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: added peer, starting replication: peer=server-d07aff94-4157-5498-b096-28a4065697f3 -2024-05-15T01:18:38.290Z [INFO] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: pipelining replication: peer="{Voter server-d07aff94-4157-5498-b096-28a4065697f3 d07aff94-4157-5498-b096-28a4065697f3}" -2024-05-15T01:18:38.290Z [INFO] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: pipelining replication: peer="{Voter server-d37b1d34-822e-d540-49e2-2c1c999881db d37b1d34-822e-d540-49e2-2c1c999881db}" -2024-05-15T01:18:38.396Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:38.290679794 +0000 UTC m=+98.832064549 (1 nodes), 68.557325ms from start of poll, 69.267373ms from cluster start. Timeout at 2024-05-15 01:18:38.39598398 +0000 UTC m=+98.937368742, 105.304193ms after stability -2024-05-15T01:18:38.401Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:38.507Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:38.401508449 +0000 UTC m=+98.942893190 (2 nodes), 2.393µs from start of poll, 180.096014ms from cluster start. Timeout at 2024-05-15 01:18:38.507524369 +0000 UTC m=+99.048909192, 106.016002ms after stability -2024-05-15T01:18:38.507Z [INFO] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: updating configuration: command=RemoveServer server-id=server-d37b1d34-822e-d540-49e2-2c1c999881db server-addr= servers="[{Suffrage:Voter ID:server-2fcae558-1d4d-24da-8068-d722fcb14b3d Address:2fcae558-1d4d-24da-8068-d722fcb14b3d} {Suffrage:Voter ID:server-d07aff94-4157-5498-b096-28a4065697f3 Address:d07aff94-4157-5498-b096-28a4065697f3}]" -2024-05-15T01:18:38.507Z [INFO] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: removed peer, stopping replication: peer=server-d37b1d34-822e-d540-49e2-2c1c999881db last-index=3 -2024-05-15T01:18:38.508Z [INFO] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: aborting pipeline replication: peer="{Voter server-d37b1d34-822e-d540-49e2-2c1c999881db d37b1d34-822e-d540-49e2-2c1c999881db}" -2024-05-15T01:18:38.558Z [WARN] server-d37b1d34-822e-d540-49e2-2c1c999881db: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2024-05-15T01:18:38.618Z [INFO] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: updating configuration: command=AddVoter server-id=server-d37b1d34-822e-d540-49e2-2c1c999881db server-addr=d37b1d34-822e-d540-49e2-2c1c999881db servers="[{Suffrage:Voter ID:server-2fcae558-1d4d-24da-8068-d722fcb14b3d Address:2fcae558-1d4d-24da-8068-d722fcb14b3d} {Suffrage:Voter ID:server-d07aff94-4157-5498-b096-28a4065697f3 Address:d07aff94-4157-5498-b096-28a4065697f3} {Suffrage:Voter ID:server-d37b1d34-822e-d540-49e2-2c1c999881db Address:d37b1d34-822e-d540-49e2-2c1c999881db}]" -2024-05-15T01:18:38.618Z [INFO] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: added peer, starting replication: peer=server-d37b1d34-822e-d540-49e2-2c1c999881db -2024-05-15T01:18:38.618Z [WARN] server-d37b1d34-822e-d540-49e2-2c1c999881db: failed to get previous log: previous-index=4 last-index=3 error="log not found" -2024-05-15T01:18:38.618Z [WARN] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: appendEntries rejected, sending older logs: peer="{Voter server-d37b1d34-822e-d540-49e2-2c1c999881db d37b1d34-822e-d540-49e2-2c1c999881db}" next=4 -2024-05-15T01:18:38.618Z [INFO] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: pipelining replication: peer="{Voter server-d37b1d34-822e-d540-49e2-2c1c999881db d37b1d34-822e-d540-49e2-2c1c999881db}" -2024-05-15T01:18:38.618Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:38.736Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:38.618838711 +0000 UTC m=+99.160223542 (1 nodes), 1.774µs from start of poll, 397.426366ms from cluster start. Timeout at 2024-05-15 01:18:38.736195706 +0000 UTC m=+99.277580692, 117.35715ms after stability -2024-05-15T01:18:38.736Z [INFO] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: aborting pipeline replication: peer="{Voter server-d37b1d34-822e-d540-49e2-2c1c999881db d37b1d34-822e-d540-49e2-2c1c999881db}" -2024-05-15T01:18:38.736Z [INFO] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: aborting pipeline replication: peer="{Voter server-d07aff94-4157-5498-b096-28a4065697f3 d07aff94-4157-5498-b096-28a4065697f3}" -2024-05-15T01:18:39.236Z [ERROR] server-2fcae558-1d4d-24da-8068-d722fcb14b3d: failed to heartbeat to: peer=d37b1d34-822e-d540-49e2-2c1c999881db backoff time=10ms error="command timed out" ---- PASS: TestRaft_ReJoinFollower (1.02s) +2025-06-17T07:44:30.108Z [INFO] server-5784e5ee-42bf-5baa-dde9-fc201576a3c9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5784e5ee-42bf-5baa-dde9-fc201576a3c9 Address:5784e5ee-42bf-5baa-dde9-fc201576a3c9} {Suffrage:Voter ID:server-2d56241f-5520-a34e-3f12-d4b9a91d0524 Address:2d56241f-5520-a34e-3f12-d4b9a91d0524} {Suffrage:Voter ID:server-f303438a-5f08-f4a1-35ba-f3c280d39bcc Address:f303438a-5f08-f4a1-35ba-f3c280d39bcc}]" +2025-06-17T07:44:30.108Z [INFO] server-5784e5ee-42bf-5baa-dde9-fc201576a3c9: entering follower state: follower="Node at 5784e5ee-42bf-5baa-dde9-fc201576a3c9 [Follower]" leader-address= leader-id= +2025-06-17T07:44:30.108Z [INFO] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5784e5ee-42bf-5baa-dde9-fc201576a3c9 Address:5784e5ee-42bf-5baa-dde9-fc201576a3c9} {Suffrage:Voter ID:server-2d56241f-5520-a34e-3f12-d4b9a91d0524 Address:2d56241f-5520-a34e-3f12-d4b9a91d0524} {Suffrage:Voter ID:server-f303438a-5f08-f4a1-35ba-f3c280d39bcc Address:f303438a-5f08-f4a1-35ba-f3c280d39bcc}]" +2025-06-17T07:44:30.108Z [INFO] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: entering follower state: follower="Node at 2d56241f-5520-a34e-3f12-d4b9a91d0524 [Follower]" leader-address= leader-id= +2025-06-17T07:44:30.108Z [INFO] server-f303438a-5f08-f4a1-35ba-f3c280d39bcc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5784e5ee-42bf-5baa-dde9-fc201576a3c9 Address:5784e5ee-42bf-5baa-dde9-fc201576a3c9} {Suffrage:Voter ID:server-2d56241f-5520-a34e-3f12-d4b9a91d0524 Address:2d56241f-5520-a34e-3f12-d4b9a91d0524} {Suffrage:Voter ID:server-f303438a-5f08-f4a1-35ba-f3c280d39bcc Address:f303438a-5f08-f4a1-35ba-f3c280d39bcc}]" +2025-06-17T07:44:30.108Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:30.108Z [INFO] server-f303438a-5f08-f4a1-35ba-f3c280d39bcc: entering follower state: follower="Node at f303438a-5f08-f4a1-35ba-f3c280d39bcc [Follower]" leader-address= leader-id= +2025-06-17T07:44:30.175Z [WARN] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:30.175Z [INFO] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: entering candidate state: node="Node at 2d56241f-5520-a34e-3f12-d4b9a91d0524 [Candidate]" term=2 +2025-06-17T07:44:30.175Z [INFO] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: election won: term=2 tally=2 +2025-06-17T07:44:30.175Z [INFO] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: entering leader state: leader="Node at 2d56241f-5520-a34e-3f12-d4b9a91d0524 [Leader]" +2025-06-17T07:44:30.175Z [INFO] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: added peer, starting replication: peer=server-5784e5ee-42bf-5baa-dde9-fc201576a3c9 +2025-06-17T07:44:30.175Z [INFO] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: added peer, starting replication: peer=server-f303438a-5f08-f4a1-35ba-f3c280d39bcc +2025-06-17T07:44:30.175Z [INFO] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: pipelining replication: peer="{Voter server-f303438a-5f08-f4a1-35ba-f3c280d39bcc f303438a-5f08-f4a1-35ba-f3c280d39bcc}" +2025-06-17T07:44:30.175Z [INFO] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: pipelining replication: peer="{Voter server-5784e5ee-42bf-5baa-dde9-fc201576a3c9 5784e5ee-42bf-5baa-dde9-fc201576a3c9}" +2025-06-17T07:44:30.281Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:30.175540649 +0000 UTC m=+98.779069362 (1 nodes), 66.993201ms from start of poll, 67.358023ms from cluster start. Timeout at 2025-06-17 07:44:30.281143476 +0000 UTC m=+98.884672173, 105.602811ms after stability +2025-06-17T07:44:30.287Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:30.393Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:30.287108214 +0000 UTC m=+98.890636909 (2 nodes), 1.318µs from start of poll, 178.92557ms from cluster start. Timeout at 2025-06-17 07:44:30.392994889 +0000 UTC m=+98.996523574, 105.886665ms after stability +2025-06-17T07:44:30.393Z [INFO] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: updating configuration: command=RemoveServer server-id=server-5784e5ee-42bf-5baa-dde9-fc201576a3c9 server-addr= servers="[{Suffrage:Voter ID:server-2d56241f-5520-a34e-3f12-d4b9a91d0524 Address:2d56241f-5520-a34e-3f12-d4b9a91d0524} {Suffrage:Voter ID:server-f303438a-5f08-f4a1-35ba-f3c280d39bcc Address:f303438a-5f08-f4a1-35ba-f3c280d39bcc}]" +2025-06-17T07:44:30.393Z [INFO] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: removed peer, stopping replication: peer=server-5784e5ee-42bf-5baa-dde9-fc201576a3c9 last-index=3 +2025-06-17T07:44:30.393Z [INFO] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: aborting pipeline replication: peer="{Voter server-5784e5ee-42bf-5baa-dde9-fc201576a3c9 5784e5ee-42bf-5baa-dde9-fc201576a3c9}" +2025-06-17T07:44:30.498Z [INFO] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: updating configuration: command=AddVoter server-id=server-5784e5ee-42bf-5baa-dde9-fc201576a3c9 server-addr=5784e5ee-42bf-5baa-dde9-fc201576a3c9 servers="[{Suffrage:Voter ID:server-2d56241f-5520-a34e-3f12-d4b9a91d0524 Address:2d56241f-5520-a34e-3f12-d4b9a91d0524} {Suffrage:Voter ID:server-f303438a-5f08-f4a1-35ba-f3c280d39bcc Address:f303438a-5f08-f4a1-35ba-f3c280d39bcc} {Suffrage:Voter ID:server-5784e5ee-42bf-5baa-dde9-fc201576a3c9 Address:5784e5ee-42bf-5baa-dde9-fc201576a3c9}]" +2025-06-17T07:44:30.498Z [INFO] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: added peer, starting replication: peer=server-5784e5ee-42bf-5baa-dde9-fc201576a3c9 +2025-06-17T07:44:30.498Z [WARN] server-5784e5ee-42bf-5baa-dde9-fc201576a3c9: failed to get previous log: previous-index=4 last-index=3 error="log not found" +2025-06-17T07:44:30.498Z [WARN] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: appendEntries rejected, sending older logs: peer="{Voter server-5784e5ee-42bf-5baa-dde9-fc201576a3c9 5784e5ee-42bf-5baa-dde9-fc201576a3c9}" next=4 +2025-06-17T07:44:30.498Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:30.498Z [INFO] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: pipelining replication: peer="{Voter server-5784e5ee-42bf-5baa-dde9-fc201576a3c9 5784e5ee-42bf-5baa-dde9-fc201576a3c9}" +2025-06-17T07:44:30.603Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:30.498564559 +0000 UTC m=+99.102093243 (1 nodes), 528ns from start of poll, 390.381904ms from cluster start. Timeout at 2025-06-17 07:44:30.603901661 +0000 UTC m=+99.207430373, 105.33713ms after stability +2025-06-17T07:44:30.603Z [INFO] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: aborting pipeline replication: peer="{Voter server-f303438a-5f08-f4a1-35ba-f3c280d39bcc f303438a-5f08-f4a1-35ba-f3c280d39bcc}" +2025-06-17T07:44:30.604Z [INFO] server-2d56241f-5520-a34e-3f12-d4b9a91d0524: aborting pipeline replication: peer="{Voter server-5784e5ee-42bf-5baa-dde9-fc201576a3c9 5784e5ee-42bf-5baa-dde9-fc201576a3c9}" +--- PASS: TestRaft_ReJoinFollower (0.50s) === RUN TestRaft_LeaderLeaseExpire -2024-05-15T01:18:39.238Z [INFO] server-afe9b552-3874-0998-63ae-76b4e0dc2753: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-afe9b552-3874-0998-63ae-76b4e0dc2753 Address:afe9b552-3874-0998-63ae-76b4e0dc2753} {Suffrage:Voter ID:server-6bc70401-4028-ead2-1656-dc039599612a Address:6bc70401-4028-ead2-1656-dc039599612a}]" -2024-05-15T01:18:39.239Z [INFO] server-afe9b552-3874-0998-63ae-76b4e0dc2753: entering follower state: follower="Node at afe9b552-3874-0998-63ae-76b4e0dc2753 [Follower]" leader-address= leader-id= -2024-05-15T01:18:39.239Z [INFO] server-6bc70401-4028-ead2-1656-dc039599612a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-afe9b552-3874-0998-63ae-76b4e0dc2753 Address:afe9b552-3874-0998-63ae-76b4e0dc2753} {Suffrage:Voter ID:server-6bc70401-4028-ead2-1656-dc039599612a Address:6bc70401-4028-ead2-1656-dc039599612a}]" -2024-05-15T01:18:39.239Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:39.239Z [INFO] server-6bc70401-4028-ead2-1656-dc039599612a: entering follower state: follower="Node at 6bc70401-4028-ead2-1656-dc039599612a [Follower]" leader-address= leader-id= -2024-05-15T01:18:39.313Z [WARN] server-6bc70401-4028-ead2-1656-dc039599612a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:39.313Z [INFO] server-6bc70401-4028-ead2-1656-dc039599612a: entering candidate state: node="Node at 6bc70401-4028-ead2-1656-dc039599612a [Candidate]" term=2 -2024-05-15T01:18:39.313Z [INFO] server-6bc70401-4028-ead2-1656-dc039599612a: election won: term=2 tally=2 -2024-05-15T01:18:39.313Z [INFO] server-6bc70401-4028-ead2-1656-dc039599612a: entering leader state: leader="Node at 6bc70401-4028-ead2-1656-dc039599612a [Leader]" -2024-05-15T01:18:39.313Z [INFO] server-6bc70401-4028-ead2-1656-dc039599612a: added peer, starting replication: peer=server-afe9b552-3874-0998-63ae-76b4e0dc2753 -2024-05-15T01:18:39.314Z [INFO] server-6bc70401-4028-ead2-1656-dc039599612a: pipelining replication: peer="{Voter server-afe9b552-3874-0998-63ae-76b4e0dc2753 afe9b552-3874-0998-63ae-76b4e0dc2753}" -2024-05-15T01:18:39.419Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:39.313879655 +0000 UTC m=+99.855263893 (1 nodes), 74.661518ms from start of poll, 75.091799ms from cluster start. Timeout at 2024-05-15 01:18:39.41895522 +0000 UTC m=+99.960340052, 105.076159ms after stability -2024-05-15T01:18:39.424Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:39.529Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:39.424698999 +0000 UTC m=+99.966084170 (1 nodes), 3.32µs from start of poll, 185.912076ms from cluster start. Timeout at 2024-05-15 01:18:39.529744249 +0000 UTC m=+100.071129297, 105.045127ms after stability - raft_test.go:1799: [INFO] Disconnecting Node at afe9b552-3874-0998-63ae-76b4e0dc2753 [Follower] -2024-05-15T01:18:39.534Z [ERROR] server-6bc70401-4028-ead2-1656-dc039599612a: failed to heartbeat to: peer=afe9b552-3874-0998-63ae-76b4e0dc2753 backoff time=10ms error="failed to connect to peer: afe9b552-3874-0998-63ae-76b4e0dc2753" -2024-05-15T01:18:39.534Z [ERROR] server-6bc70401-4028-ead2-1656-dc039599612a: failed to pipeline appendEntries: peer="{Voter server-afe9b552-3874-0998-63ae-76b4e0dc2753 afe9b552-3874-0998-63ae-76b4e0dc2753}" error="append pipeline closed" -2024-05-15T01:18:39.534Z [INFO] server-6bc70401-4028-ead2-1656-dc039599612a: aborting pipeline replication: peer="{Voter server-afe9b552-3874-0998-63ae-76b4e0dc2753 afe9b552-3874-0998-63ae-76b4e0dc2753}" -2024-05-15T01:18:39.543Z [ERROR] server-6bc70401-4028-ead2-1656-dc039599612a: failed to appendEntries to: peer="{Voter server-afe9b552-3874-0998-63ae-76b4e0dc2753 afe9b552-3874-0998-63ae-76b4e0dc2753}" error="failed to connect to peer: afe9b552-3874-0998-63ae-76b4e0dc2753" -2024-05-15T01:18:39.550Z [ERROR] server-6bc70401-4028-ead2-1656-dc039599612a: failed to heartbeat to: peer=afe9b552-3874-0998-63ae-76b4e0dc2753 backoff time=10ms error="failed to connect to peer: afe9b552-3874-0998-63ae-76b4e0dc2753" -2024-05-15T01:18:39.563Z [ERROR] server-6bc70401-4028-ead2-1656-dc039599612a: failed to appendEntries to: peer="{Voter server-afe9b552-3874-0998-63ae-76b4e0dc2753 afe9b552-3874-0998-63ae-76b4e0dc2753}" error="failed to connect to peer: afe9b552-3874-0998-63ae-76b4e0dc2753" -2024-05-15T01:18:39.570Z [ERROR] server-6bc70401-4028-ead2-1656-dc039599612a: failed to heartbeat to: peer=afe9b552-3874-0998-63ae-76b4e0dc2753 backoff time=10ms error="failed to connect to peer: afe9b552-3874-0998-63ae-76b4e0dc2753" -2024-05-15T01:18:39.579Z [WARN] server-6bc70401-4028-ead2-1656-dc039599612a: failed to contact: server-id=server-afe9b552-3874-0998-63ae-76b4e0dc2753 time=50.721638ms -2024-05-15T01:18:39.579Z [WARN] server-6bc70401-4028-ead2-1656-dc039599612a: failed to contact quorum of nodes, stepping down -2024-05-15T01:18:39.579Z [INFO] server-6bc70401-4028-ead2-1656-dc039599612a: entering follower state: follower="Node at 6bc70401-4028-ead2-1656-dc039599612a [Follower]" leader-address= leader-id= -2024-05-15T01:18:39.579Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:39.582Z [ERROR] server-6bc70401-4028-ead2-1656-dc039599612a: failed to appendEntries to: peer="{Voter server-afe9b552-3874-0998-63ae-76b4e0dc2753 afe9b552-3874-0998-63ae-76b4e0dc2753}" error="failed to connect to peer: afe9b552-3874-0998-63ae-76b4e0dc2753" -2024-05-15T01:18:39.595Z [WARN] server-afe9b552-3874-0998-63ae-76b4e0dc2753: heartbeat timeout reached, starting election: last-leader-addr=6bc70401-4028-ead2-1656-dc039599612a last-leader-id=server-6bc70401-4028-ead2-1656-dc039599612a -2024-05-15T01:18:39.595Z [INFO] server-afe9b552-3874-0998-63ae-76b4e0dc2753: entering candidate state: node="Node at afe9b552-3874-0998-63ae-76b4e0dc2753 [Candidate]" term=3 -2024-05-15T01:18:39.595Z [ERROR] server-afe9b552-3874-0998-63ae-76b4e0dc2753: failed to make requestVote RPC: target="{Voter server-6bc70401-4028-ead2-1656-dc039599612a 6bc70401-4028-ead2-1656-dc039599612a}" error="failed to connect to peer: 6bc70401-4028-ead2-1656-dc039599612a" term=3 -2024-05-15T01:18:39.670Z [WARN] server-6bc70401-4028-ead2-1656-dc039599612a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:39.670Z [INFO] server-6bc70401-4028-ead2-1656-dc039599612a: entering candidate state: node="Node at 6bc70401-4028-ead2-1656-dc039599612a [Candidate]" term=3 -2024-05-15T01:18:39.670Z [ERROR] server-6bc70401-4028-ead2-1656-dc039599612a: failed to make requestVote RPC: target="{Voter server-afe9b552-3874-0998-63ae-76b4e0dc2753 afe9b552-3874-0998-63ae-76b4e0dc2753}" error="failed to connect to peer: afe9b552-3874-0998-63ae-76b4e0dc2753" term=3 -2024-05-15T01:18:39.688Z [WARN] server-afe9b552-3874-0998-63ae-76b4e0dc2753: Election timeout reached, restarting election -2024-05-15T01:18:39.688Z [INFO] server-afe9b552-3874-0998-63ae-76b4e0dc2753: entering candidate state: node="Node at afe9b552-3874-0998-63ae-76b4e0dc2753 [Candidate]" term=4 -2024-05-15T01:18:39.688Z [ERROR] server-afe9b552-3874-0998-63ae-76b4e0dc2753: failed to make requestVote RPC: target="{Voter server-6bc70401-4028-ead2-1656-dc039599612a 6bc70401-4028-ead2-1656-dc039599612a}" error="failed to connect to peer: 6bc70401-4028-ead2-1656-dc039599612a" term=4 -2024-05-15T01:18:39.750Z [WARN] server-6bc70401-4028-ead2-1656-dc039599612a: Election timeout reached, restarting election -2024-05-15T01:18:39.750Z [INFO] server-6bc70401-4028-ead2-1656-dc039599612a: entering candidate state: node="Node at 6bc70401-4028-ead2-1656-dc039599612a [Candidate]" term=4 -2024-05-15T01:18:39.750Z [ERROR] server-6bc70401-4028-ead2-1656-dc039599612a: failed to make requestVote RPC: target="{Voter server-afe9b552-3874-0998-63ae-76b4e0dc2753 afe9b552-3874-0998-63ae-76b4e0dc2753}" error="failed to connect to peer: afe9b552-3874-0998-63ae-76b4e0dc2753" term=4 -2024-05-15T01:18:39.776Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:39.670799139 +0000 UTC m=+100.212184130 (0 nodes), 91.343671ms from start of poll, 432.012036ms from cluster start. Timeout at 2024-05-15 01:18:39.776690803 +0000 UTC m=+100.318075864, 105.891734ms after stability -2024-05-15T01:18:39.780Z [WARN] server-afe9b552-3874-0998-63ae-76b4e0dc2753: Election timeout reached, restarting election -2024-05-15T01:18:39.780Z [INFO] server-afe9b552-3874-0998-63ae-76b4e0dc2753: entering candidate state: node="Node at afe9b552-3874-0998-63ae-76b4e0dc2753 [Candidate]" term=5 -2024-05-15T01:18:39.780Z [ERROR] server-afe9b552-3874-0998-63ae-76b4e0dc2753: failed to make requestVote RPC: target="{Voter server-6bc70401-4028-ead2-1656-dc039599612a 6bc70401-4028-ead2-1656-dc039599612a}" error="failed to connect to peer: 6bc70401-4028-ead2-1656-dc039599612a" term=5 -2024-05-15T01:18:39.831Z [WARN] server-afe9b552-3874-0998-63ae-76b4e0dc2753: Election timeout reached, restarting election -2024-05-15T01:18:39.831Z [INFO] server-afe9b552-3874-0998-63ae-76b4e0dc2753: entering candidate state: node="Node at afe9b552-3874-0998-63ae-76b4e0dc2753 [Candidate]" term=6 -2024-05-15T01:18:39.831Z [ERROR] server-afe9b552-3874-0998-63ae-76b4e0dc2753: failed to make requestVote RPC: target="{Voter server-6bc70401-4028-ead2-1656-dc039599612a 6bc70401-4028-ead2-1656-dc039599612a}" error="failed to connect to peer: 6bc70401-4028-ead2-1656-dc039599612a" term=6 -2024-05-15T01:18:39.832Z [WARN] server-6bc70401-4028-ead2-1656-dc039599612a: Election timeout reached, restarting election -2024-05-15T01:18:39.832Z [INFO] server-6bc70401-4028-ead2-1656-dc039599612a: entering candidate state: node="Node at 6bc70401-4028-ead2-1656-dc039599612a [Candidate]" term=5 -2024-05-15T01:18:39.832Z [ERROR] server-6bc70401-4028-ead2-1656-dc039599612a: failed to make requestVote RPC: target="{Voter server-afe9b552-3874-0998-63ae-76b4e0dc2753 afe9b552-3874-0998-63ae-76b4e0dc2753}" error="failed to connect to peer: afe9b552-3874-0998-63ae-76b4e0dc2753" term=5 ---- PASS: TestRaft_LeaderLeaseExpire (0.65s) +2025-06-17T07:44:30.604Z [INFO] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b Address:fff3e787-a2c0-09cb-ec6a-9aba038a1c1b} {Suffrage:Voter ID:server-e93fd42e-281e-588d-2746-4fc29ba5463e Address:e93fd42e-281e-588d-2746-4fc29ba5463e}]" +2025-06-17T07:44:30.604Z [INFO] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: entering follower state: follower="Node at fff3e787-a2c0-09cb-ec6a-9aba038a1c1b [Follower]" leader-address= leader-id= +2025-06-17T07:44:30.604Z [INFO] server-e93fd42e-281e-588d-2746-4fc29ba5463e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b Address:fff3e787-a2c0-09cb-ec6a-9aba038a1c1b} {Suffrage:Voter ID:server-e93fd42e-281e-588d-2746-4fc29ba5463e Address:e93fd42e-281e-588d-2746-4fc29ba5463e}]" +2025-06-17T07:44:30.604Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:30.604Z [INFO] server-e93fd42e-281e-588d-2746-4fc29ba5463e: entering follower state: follower="Node at e93fd42e-281e-588d-2746-4fc29ba5463e [Follower]" leader-address= leader-id= +2025-06-17T07:44:30.665Z [WARN] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:30.665Z [INFO] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: entering candidate state: node="Node at fff3e787-a2c0-09cb-ec6a-9aba038a1c1b [Candidate]" term=2 +2025-06-17T07:44:30.665Z [INFO] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: election won: term=2 tally=2 +2025-06-17T07:44:30.665Z [INFO] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: entering leader state: leader="Node at fff3e787-a2c0-09cb-ec6a-9aba038a1c1b [Leader]" +2025-06-17T07:44:30.665Z [INFO] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: added peer, starting replication: peer=server-e93fd42e-281e-588d-2746-4fc29ba5463e +2025-06-17T07:44:30.665Z [INFO] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: pipelining replication: peer="{Voter server-e93fd42e-281e-588d-2746-4fc29ba5463e e93fd42e-281e-588d-2746-4fc29ba5463e}" +2025-06-17T07:44:30.771Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:30.665735902 +0000 UTC m=+99.269264589 (1 nodes), 60.788238ms from start of poll, 60.989056ms from cluster start. Timeout at 2025-06-17 07:44:30.770967047 +0000 UTC m=+99.374495762, 105.231173ms after stability +2025-06-17T07:44:30.776Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:30.882Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:30.776616364 +0000 UTC m=+99.380145059 (1 nodes), 759ns from start of poll, 171.869526ms from cluster start. Timeout at 2025-06-17 07:44:30.882094416 +0000 UTC m=+99.485623107, 105.478048ms after stability + raft_test.go:1799: [INFO] Disconnecting Node at e93fd42e-281e-588d-2746-4fc29ba5463e [Follower] +2025-06-17T07:44:30.884Z [ERROR] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: failed to pipeline appendEntries: peer="{Voter server-e93fd42e-281e-588d-2746-4fc29ba5463e e93fd42e-281e-588d-2746-4fc29ba5463e}" error="append pipeline closed" +2025-06-17T07:44:30.884Z [INFO] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: aborting pipeline replication: peer="{Voter server-e93fd42e-281e-588d-2746-4fc29ba5463e e93fd42e-281e-588d-2746-4fc29ba5463e}" +2025-06-17T07:44:30.886Z [ERROR] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: failed to heartbeat to: peer=e93fd42e-281e-588d-2746-4fc29ba5463e backoff time=10ms error="failed to connect to peer: e93fd42e-281e-588d-2746-4fc29ba5463e" +2025-06-17T07:44:30.891Z [ERROR] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: failed to appendEntries to: peer="{Voter server-e93fd42e-281e-588d-2746-4fc29ba5463e e93fd42e-281e-588d-2746-4fc29ba5463e}" error="failed to connect to peer: e93fd42e-281e-588d-2746-4fc29ba5463e" +2025-06-17T07:44:30.905Z [ERROR] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: failed to heartbeat to: peer=e93fd42e-281e-588d-2746-4fc29ba5463e backoff time=10ms error="failed to connect to peer: e93fd42e-281e-588d-2746-4fc29ba5463e" +2025-06-17T07:44:30.910Z [ERROR] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: failed to appendEntries to: peer="{Voter server-e93fd42e-281e-588d-2746-4fc29ba5463e e93fd42e-281e-588d-2746-4fc29ba5463e}" error="failed to connect to peer: e93fd42e-281e-588d-2746-4fc29ba5463e" +2025-06-17T07:44:30.922Z [ERROR] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: failed to heartbeat to: peer=e93fd42e-281e-588d-2746-4fc29ba5463e backoff time=10ms error="failed to connect to peer: e93fd42e-281e-588d-2746-4fc29ba5463e" +2025-06-17T07:44:30.927Z [WARN] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: failed to contact: server-id=server-e93fd42e-281e-588d-2746-4fc29ba5463e time=50.766342ms +2025-06-17T07:44:30.927Z [WARN] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: failed to contact quorum of nodes, stepping down +2025-06-17T07:44:30.927Z [INFO] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: entering follower state: follower="Node at fff3e787-a2c0-09cb-ec6a-9aba038a1c1b [Follower]" leader-address= leader-id= +2025-06-17T07:44:30.927Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:30.929Z [ERROR] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: failed to appendEntries to: peer="{Voter server-e93fd42e-281e-588d-2746-4fc29ba5463e e93fd42e-281e-588d-2746-4fc29ba5463e}" error="failed to connect to peer: e93fd42e-281e-588d-2746-4fc29ba5463e" +2025-06-17T07:44:30.967Z [WARN] server-e93fd42e-281e-588d-2746-4fc29ba5463e: heartbeat timeout reached, starting election: last-leader-addr=fff3e787-a2c0-09cb-ec6a-9aba038a1c1b last-leader-id=server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b +2025-06-17T07:44:30.967Z [INFO] server-e93fd42e-281e-588d-2746-4fc29ba5463e: entering candidate state: node="Node at e93fd42e-281e-588d-2746-4fc29ba5463e [Candidate]" term=3 +2025-06-17T07:44:30.967Z [ERROR] server-e93fd42e-281e-588d-2746-4fc29ba5463e: failed to make requestVote RPC: target="{Voter server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b fff3e787-a2c0-09cb-ec6a-9aba038a1c1b}" error="failed to connect to peer: fff3e787-a2c0-09cb-ec6a-9aba038a1c1b" term=3 +2025-06-17T07:44:31.000Z [WARN] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:31.000Z [INFO] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: entering candidate state: node="Node at fff3e787-a2c0-09cb-ec6a-9aba038a1c1b [Candidate]" term=3 +2025-06-17T07:44:31.000Z [ERROR] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: failed to make requestVote RPC: target="{Voter server-e93fd42e-281e-588d-2746-4fc29ba5463e e93fd42e-281e-588d-2746-4fc29ba5463e}" error="failed to connect to peer: e93fd42e-281e-588d-2746-4fc29ba5463e" term=3 +2025-06-17T07:44:31.063Z [WARN] server-e93fd42e-281e-588d-2746-4fc29ba5463e: Election timeout reached, restarting election +2025-06-17T07:44:31.063Z [INFO] server-e93fd42e-281e-588d-2746-4fc29ba5463e: entering candidate state: node="Node at e93fd42e-281e-588d-2746-4fc29ba5463e [Candidate]" term=4 +2025-06-17T07:44:31.063Z [ERROR] server-e93fd42e-281e-588d-2746-4fc29ba5463e: failed to make requestVote RPC: target="{Voter server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b fff3e787-a2c0-09cb-ec6a-9aba038a1c1b}" error="failed to connect to peer: fff3e787-a2c0-09cb-ec6a-9aba038a1c1b" term=4 +2025-06-17T07:44:31.073Z [WARN] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: Election timeout reached, restarting election +2025-06-17T07:44:31.073Z [INFO] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: entering candidate state: node="Node at fff3e787-a2c0-09cb-ec6a-9aba038a1c1b [Candidate]" term=4 +2025-06-17T07:44:31.073Z [ERROR] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: failed to make requestVote RPC: target="{Voter server-e93fd42e-281e-588d-2746-4fc29ba5463e e93fd42e-281e-588d-2746-4fc29ba5463e}" error="failed to connect to peer: e93fd42e-281e-588d-2746-4fc29ba5463e" term=4 +2025-06-17T07:44:31.106Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:31.000891311 +0000 UTC m=+99.604420018 (0 nodes), 73.751326ms from start of poll, 396.144485ms from cluster start. Timeout at 2025-06-17 07:44:31.106816559 +0000 UTC m=+99.710345261, 105.925243ms after stability +2025-06-17T07:44:31.136Z [WARN] server-e93fd42e-281e-588d-2746-4fc29ba5463e: Election timeout reached, restarting election +2025-06-17T07:44:31.136Z [INFO] server-e93fd42e-281e-588d-2746-4fc29ba5463e: entering candidate state: node="Node at e93fd42e-281e-588d-2746-4fc29ba5463e [Candidate]" term=5 +2025-06-17T07:44:31.136Z [ERROR] server-e93fd42e-281e-588d-2746-4fc29ba5463e: failed to make requestVote RPC: target="{Voter server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b fff3e787-a2c0-09cb-ec6a-9aba038a1c1b}" error="failed to connect to peer: fff3e787-a2c0-09cb-ec6a-9aba038a1c1b" term=5 +2025-06-17T07:44:31.162Z [WARN] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: Election timeout reached, restarting election +2025-06-17T07:44:31.162Z [INFO] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: entering candidate state: node="Node at fff3e787-a2c0-09cb-ec6a-9aba038a1c1b [Candidate]" term=5 +2025-06-17T07:44:31.162Z [ERROR] server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b: failed to make requestVote RPC: target="{Voter server-e93fd42e-281e-588d-2746-4fc29ba5463e e93fd42e-281e-588d-2746-4fc29ba5463e}" error="failed to connect to peer: e93fd42e-281e-588d-2746-4fc29ba5463e" term=5 +2025-06-17T07:44:31.203Z [WARN] server-e93fd42e-281e-588d-2746-4fc29ba5463e: Election timeout reached, restarting election +2025-06-17T07:44:31.203Z [INFO] server-e93fd42e-281e-588d-2746-4fc29ba5463e: entering candidate state: node="Node at e93fd42e-281e-588d-2746-4fc29ba5463e [Candidate]" term=6 +2025-06-17T07:44:31.203Z [ERROR] server-e93fd42e-281e-588d-2746-4fc29ba5463e: failed to make requestVote RPC: target="{Voter server-fff3e787-a2c0-09cb-ec6a-9aba038a1c1b fff3e787-a2c0-09cb-ec6a-9aba038a1c1b}" error="failed to connect to peer: fff3e787-a2c0-09cb-ec6a-9aba038a1c1b" term=6 +--- PASS: TestRaft_LeaderLeaseExpire (0.61s) === RUN TestRaft_Barrier -2024-05-15T01:18:39.884Z [INFO] server-69df75b3-65e4-2ea1-3e44-66c63ff4ea37: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-69df75b3-65e4-2ea1-3e44-66c63ff4ea37 Address:69df75b3-65e4-2ea1-3e44-66c63ff4ea37} {Suffrage:Voter ID:server-211ffbd5-1c29-e79d-d8dd-e3e930c5c619 Address:211ffbd5-1c29-e79d-d8dd-e3e930c5c619} {Suffrage:Voter ID:server-51312770-fd0b-be6b-093b-464d5179f0f1 Address:51312770-fd0b-be6b-093b-464d5179f0f1}]" -2024-05-15T01:18:39.884Z [INFO] server-211ffbd5-1c29-e79d-d8dd-e3e930c5c619: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-69df75b3-65e4-2ea1-3e44-66c63ff4ea37 Address:69df75b3-65e4-2ea1-3e44-66c63ff4ea37} {Suffrage:Voter ID:server-211ffbd5-1c29-e79d-d8dd-e3e930c5c619 Address:211ffbd5-1c29-e79d-d8dd-e3e930c5c619} {Suffrage:Voter ID:server-51312770-fd0b-be6b-093b-464d5179f0f1 Address:51312770-fd0b-be6b-093b-464d5179f0f1}]" -2024-05-15T01:18:39.884Z [INFO] server-51312770-fd0b-be6b-093b-464d5179f0f1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-69df75b3-65e4-2ea1-3e44-66c63ff4ea37 Address:69df75b3-65e4-2ea1-3e44-66c63ff4ea37} {Suffrage:Voter ID:server-211ffbd5-1c29-e79d-d8dd-e3e930c5c619 Address:211ffbd5-1c29-e79d-d8dd-e3e930c5c619} {Suffrage:Voter ID:server-51312770-fd0b-be6b-093b-464d5179f0f1 Address:51312770-fd0b-be6b-093b-464d5179f0f1}]" -2024-05-15T01:18:39.884Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:39.885Z [INFO] server-69df75b3-65e4-2ea1-3e44-66c63ff4ea37: entering follower state: follower="Node at 69df75b3-65e4-2ea1-3e44-66c63ff4ea37 [Follower]" leader-address= leader-id= -2024-05-15T01:18:39.885Z [INFO] server-211ffbd5-1c29-e79d-d8dd-e3e930c5c619: entering follower state: follower="Node at 211ffbd5-1c29-e79d-d8dd-e3e930c5c619 [Follower]" leader-address= leader-id= -2024-05-15T01:18:39.885Z [INFO] server-51312770-fd0b-be6b-093b-464d5179f0f1: entering follower state: follower="Node at 51312770-fd0b-be6b-093b-464d5179f0f1 [Follower]" leader-address= leader-id= -2024-05-15T01:18:39.942Z [WARN] server-69df75b3-65e4-2ea1-3e44-66c63ff4ea37: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:39.942Z [INFO] server-69df75b3-65e4-2ea1-3e44-66c63ff4ea37: entering candidate state: node="Node at 69df75b3-65e4-2ea1-3e44-66c63ff4ea37 [Candidate]" term=2 -2024-05-15T01:18:39.942Z [INFO] server-69df75b3-65e4-2ea1-3e44-66c63ff4ea37: election won: term=2 tally=2 -2024-05-15T01:18:39.942Z [INFO] server-69df75b3-65e4-2ea1-3e44-66c63ff4ea37: entering leader state: leader="Node at 69df75b3-65e4-2ea1-3e44-66c63ff4ea37 [Leader]" -2024-05-15T01:18:39.942Z [INFO] server-69df75b3-65e4-2ea1-3e44-66c63ff4ea37: added peer, starting replication: peer=server-211ffbd5-1c29-e79d-d8dd-e3e930c5c619 -2024-05-15T01:18:39.942Z [INFO] server-69df75b3-65e4-2ea1-3e44-66c63ff4ea37: added peer, starting replication: peer=server-51312770-fd0b-be6b-093b-464d5179f0f1 -2024-05-15T01:18:39.942Z [INFO] server-69df75b3-65e4-2ea1-3e44-66c63ff4ea37: pipelining replication: peer="{Voter server-51312770-fd0b-be6b-093b-464d5179f0f1 51312770-fd0b-be6b-093b-464d5179f0f1}" -2024-05-15T01:18:39.942Z [INFO] server-69df75b3-65e4-2ea1-3e44-66c63ff4ea37: pipelining replication: peer="{Voter server-211ffbd5-1c29-e79d-d8dd-e3e930c5c619 211ffbd5-1c29-e79d-d8dd-e3e930c5c619}" -2024-05-15T01:18:40.049Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:39.942922156 +0000 UTC m=+100.484306923 (1 nodes), 57.937405ms from start of poll, 58.475862ms from cluster start. Timeout at 2024-05-15 01:18:40.049315942 +0000 UTC m=+100.590700678, 106.393755ms after stability -2024-05-15T01:18:40.077Z [INFO] server-69df75b3-65e4-2ea1-3e44-66c63ff4ea37: aborting pipeline replication: peer="{Voter server-51312770-fd0b-be6b-093b-464d5179f0f1 51312770-fd0b-be6b-093b-464d5179f0f1}" -2024-05-15T01:18:40.077Z [INFO] server-69df75b3-65e4-2ea1-3e44-66c63ff4ea37: aborting pipeline replication: peer="{Voter server-211ffbd5-1c29-e79d-d8dd-e3e930c5c619 211ffbd5-1c29-e79d-d8dd-e3e930c5c619}" ---- PASS: TestRaft_Barrier (0.19s) +2025-06-17T07:44:31.212Z [INFO] server-e378d6a7-6f73-5d77-a022-72b62cb979f5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e378d6a7-6f73-5d77-a022-72b62cb979f5 Address:e378d6a7-6f73-5d77-a022-72b62cb979f5} {Suffrage:Voter ID:server-2727a435-6c3d-cc3b-d29f-aa38bdd0de20 Address:2727a435-6c3d-cc3b-d29f-aa38bdd0de20} {Suffrage:Voter ID:server-5191e81d-9b83-c202-506d-a468805feb15 Address:5191e81d-9b83-c202-506d-a468805feb15}]" +2025-06-17T07:44:31.212Z [INFO] server-e378d6a7-6f73-5d77-a022-72b62cb979f5: entering follower state: follower="Node at e378d6a7-6f73-5d77-a022-72b62cb979f5 [Follower]" leader-address= leader-id= +2025-06-17T07:44:31.212Z [INFO] server-2727a435-6c3d-cc3b-d29f-aa38bdd0de20: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e378d6a7-6f73-5d77-a022-72b62cb979f5 Address:e378d6a7-6f73-5d77-a022-72b62cb979f5} {Suffrage:Voter ID:server-2727a435-6c3d-cc3b-d29f-aa38bdd0de20 Address:2727a435-6c3d-cc3b-d29f-aa38bdd0de20} {Suffrage:Voter ID:server-5191e81d-9b83-c202-506d-a468805feb15 Address:5191e81d-9b83-c202-506d-a468805feb15}]" +2025-06-17T07:44:31.213Z [INFO] server-2727a435-6c3d-cc3b-d29f-aa38bdd0de20: entering follower state: follower="Node at 2727a435-6c3d-cc3b-d29f-aa38bdd0de20 [Follower]" leader-address= leader-id= +2025-06-17T07:44:31.213Z [INFO] server-5191e81d-9b83-c202-506d-a468805feb15: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e378d6a7-6f73-5d77-a022-72b62cb979f5 Address:e378d6a7-6f73-5d77-a022-72b62cb979f5} {Suffrage:Voter ID:server-2727a435-6c3d-cc3b-d29f-aa38bdd0de20 Address:2727a435-6c3d-cc3b-d29f-aa38bdd0de20} {Suffrage:Voter ID:server-5191e81d-9b83-c202-506d-a468805feb15 Address:5191e81d-9b83-c202-506d-a468805feb15}]" +2025-06-17T07:44:31.213Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:31.213Z [INFO] server-5191e81d-9b83-c202-506d-a468805feb15: entering follower state: follower="Node at 5191e81d-9b83-c202-506d-a468805feb15 [Follower]" leader-address= leader-id= +2025-06-17T07:44:31.266Z [WARN] server-2727a435-6c3d-cc3b-d29f-aa38bdd0de20: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:31.266Z [INFO] server-2727a435-6c3d-cc3b-d29f-aa38bdd0de20: entering candidate state: node="Node at 2727a435-6c3d-cc3b-d29f-aa38bdd0de20 [Candidate]" term=2 +2025-06-17T07:44:31.266Z [INFO] server-2727a435-6c3d-cc3b-d29f-aa38bdd0de20: election won: term=2 tally=2 +2025-06-17T07:44:31.266Z [INFO] server-2727a435-6c3d-cc3b-d29f-aa38bdd0de20: entering leader state: leader="Node at 2727a435-6c3d-cc3b-d29f-aa38bdd0de20 [Leader]" +2025-06-17T07:44:31.266Z [INFO] server-2727a435-6c3d-cc3b-d29f-aa38bdd0de20: added peer, starting replication: peer=server-e378d6a7-6f73-5d77-a022-72b62cb979f5 +2025-06-17T07:44:31.266Z [INFO] server-2727a435-6c3d-cc3b-d29f-aa38bdd0de20: added peer, starting replication: peer=server-5191e81d-9b83-c202-506d-a468805feb15 +2025-06-17T07:44:31.266Z [INFO] server-2727a435-6c3d-cc3b-d29f-aa38bdd0de20: pipelining replication: peer="{Voter server-e378d6a7-6f73-5d77-a022-72b62cb979f5 e378d6a7-6f73-5d77-a022-72b62cb979f5}" +2025-06-17T07:44:31.266Z [INFO] server-2727a435-6c3d-cc3b-d29f-aa38bdd0de20: pipelining replication: peer="{Voter server-5191e81d-9b83-c202-506d-a468805feb15 5191e81d-9b83-c202-506d-a468805feb15}" +2025-06-17T07:44:31.371Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:31.266390827 +0000 UTC m=+99.869919512 (1 nodes), 53.293974ms from start of poll, 53.594395ms from cluster start. Timeout at 2025-06-17 07:44:31.371397467 +0000 UTC m=+99.974926148, 105.006636ms after stability +2025-06-17T07:44:31.388Z [INFO] server-2727a435-6c3d-cc3b-d29f-aa38bdd0de20: aborting pipeline replication: peer="{Voter server-5191e81d-9b83-c202-506d-a468805feb15 5191e81d-9b83-c202-506d-a468805feb15}" +2025-06-17T07:44:31.388Z [INFO] server-2727a435-6c3d-cc3b-d29f-aa38bdd0de20: aborting pipeline replication: peer="{Voter server-e378d6a7-6f73-5d77-a022-72b62cb979f5 e378d6a7-6f73-5d77-a022-72b62cb979f5}" +--- PASS: TestRaft_Barrier (0.18s) === RUN TestRaft_VerifyLeader -2024-05-15T01:18:40.079Z [INFO] server-622af952-3669-f45f-1a07-e252aba9ab9f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-622af952-3669-f45f-1a07-e252aba9ab9f Address:622af952-3669-f45f-1a07-e252aba9ab9f} {Suffrage:Voter ID:server-170b44b2-a451-ff0e-218e-4ff2fa90f445 Address:170b44b2-a451-ff0e-218e-4ff2fa90f445} {Suffrage:Voter ID:server-1b92270a-95b5-f50a-f165-66c603184285 Address:1b92270a-95b5-f50a-f165-66c603184285}]" -2024-05-15T01:18:40.079Z [INFO] server-622af952-3669-f45f-1a07-e252aba9ab9f: entering follower state: follower="Node at 622af952-3669-f45f-1a07-e252aba9ab9f [Follower]" leader-address= leader-id= -2024-05-15T01:18:40.079Z [INFO] server-170b44b2-a451-ff0e-218e-4ff2fa90f445: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-622af952-3669-f45f-1a07-e252aba9ab9f Address:622af952-3669-f45f-1a07-e252aba9ab9f} {Suffrage:Voter ID:server-170b44b2-a451-ff0e-218e-4ff2fa90f445 Address:170b44b2-a451-ff0e-218e-4ff2fa90f445} {Suffrage:Voter ID:server-1b92270a-95b5-f50a-f165-66c603184285 Address:1b92270a-95b5-f50a-f165-66c603184285}]" -2024-05-15T01:18:40.079Z [INFO] server-170b44b2-a451-ff0e-218e-4ff2fa90f445: entering follower state: follower="Node at 170b44b2-a451-ff0e-218e-4ff2fa90f445 [Follower]" leader-address= leader-id= -2024-05-15T01:18:40.079Z [INFO] server-1b92270a-95b5-f50a-f165-66c603184285: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-622af952-3669-f45f-1a07-e252aba9ab9f Address:622af952-3669-f45f-1a07-e252aba9ab9f} {Suffrage:Voter ID:server-170b44b2-a451-ff0e-218e-4ff2fa90f445 Address:170b44b2-a451-ff0e-218e-4ff2fa90f445} {Suffrage:Voter ID:server-1b92270a-95b5-f50a-f165-66c603184285 Address:1b92270a-95b5-f50a-f165-66c603184285}]" -2024-05-15T01:18:40.079Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:40.079Z [INFO] server-1b92270a-95b5-f50a-f165-66c603184285: entering follower state: follower="Node at 1b92270a-95b5-f50a-f165-66c603184285 [Follower]" leader-address= leader-id= -2024-05-15T01:18:40.144Z [WARN] server-622af952-3669-f45f-1a07-e252aba9ab9f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:40.144Z [INFO] server-622af952-3669-f45f-1a07-e252aba9ab9f: entering candidate state: node="Node at 622af952-3669-f45f-1a07-e252aba9ab9f [Candidate]" term=2 -2024-05-15T01:18:40.144Z [INFO] server-622af952-3669-f45f-1a07-e252aba9ab9f: election won: term=2 tally=2 -2024-05-15T01:18:40.144Z [INFO] server-622af952-3669-f45f-1a07-e252aba9ab9f: entering leader state: leader="Node at 622af952-3669-f45f-1a07-e252aba9ab9f [Leader]" -2024-05-15T01:18:40.144Z [INFO] server-622af952-3669-f45f-1a07-e252aba9ab9f: added peer, starting replication: peer=server-170b44b2-a451-ff0e-218e-4ff2fa90f445 -2024-05-15T01:18:40.144Z [INFO] server-622af952-3669-f45f-1a07-e252aba9ab9f: added peer, starting replication: peer=server-1b92270a-95b5-f50a-f165-66c603184285 -2024-05-15T01:18:40.144Z [INFO] server-622af952-3669-f45f-1a07-e252aba9ab9f: pipelining replication: peer="{Voter server-1b92270a-95b5-f50a-f165-66c603184285 1b92270a-95b5-f50a-f165-66c603184285}" -2024-05-15T01:18:40.144Z [INFO] server-622af952-3669-f45f-1a07-e252aba9ab9f: pipelining replication: peer="{Voter server-170b44b2-a451-ff0e-218e-4ff2fa90f445 170b44b2-a451-ff0e-218e-4ff2fa90f445}" -2024-05-15T01:18:40.263Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:40.144748286 +0000 UTC m=+100.686133049 (1 nodes), 65.05562ms from start of poll, 65.739093ms from cluster start. Timeout at 2024-05-15 01:18:40.263141536 +0000 UTC m=+100.804526773, 118.393724ms after stability -2024-05-15T01:18:40.263Z [INFO] server-622af952-3669-f45f-1a07-e252aba9ab9f: aborting pipeline replication: peer="{Voter server-1b92270a-95b5-f50a-f165-66c603184285 1b92270a-95b5-f50a-f165-66c603184285}" -2024-05-15T01:18:40.263Z [INFO] server-622af952-3669-f45f-1a07-e252aba9ab9f: aborting pipeline replication: peer="{Voter server-170b44b2-a451-ff0e-218e-4ff2fa90f445 170b44b2-a451-ff0e-218e-4ff2fa90f445}" ---- PASS: TestRaft_VerifyLeader (0.19s) +2025-06-17T07:44:31.389Z [INFO] server-921ede8b-0809-aea1-038b-ab1f6af5eb46: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-921ede8b-0809-aea1-038b-ab1f6af5eb46 Address:921ede8b-0809-aea1-038b-ab1f6af5eb46} {Suffrage:Voter ID:server-48efb41d-3d76-0e92-265d-60e180bddd44 Address:48efb41d-3d76-0e92-265d-60e180bddd44} {Suffrage:Voter ID:server-c3817c50-7dc6-d156-2bbb-679d6b62e66c Address:c3817c50-7dc6-d156-2bbb-679d6b62e66c}]" +2025-06-17T07:44:31.389Z [INFO] server-921ede8b-0809-aea1-038b-ab1f6af5eb46: entering follower state: follower="Node at 921ede8b-0809-aea1-038b-ab1f6af5eb46 [Follower]" leader-address= leader-id= +2025-06-17T07:44:31.389Z [INFO] server-48efb41d-3d76-0e92-265d-60e180bddd44: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-921ede8b-0809-aea1-038b-ab1f6af5eb46 Address:921ede8b-0809-aea1-038b-ab1f6af5eb46} {Suffrage:Voter ID:server-48efb41d-3d76-0e92-265d-60e180bddd44 Address:48efb41d-3d76-0e92-265d-60e180bddd44} {Suffrage:Voter ID:server-c3817c50-7dc6-d156-2bbb-679d6b62e66c Address:c3817c50-7dc6-d156-2bbb-679d6b62e66c}]" +2025-06-17T07:44:31.389Z [INFO] server-48efb41d-3d76-0e92-265d-60e180bddd44: entering follower state: follower="Node at 48efb41d-3d76-0e92-265d-60e180bddd44 [Follower]" leader-address= leader-id= +2025-06-17T07:44:31.389Z [INFO] server-c3817c50-7dc6-d156-2bbb-679d6b62e66c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-921ede8b-0809-aea1-038b-ab1f6af5eb46 Address:921ede8b-0809-aea1-038b-ab1f6af5eb46} {Suffrage:Voter ID:server-48efb41d-3d76-0e92-265d-60e180bddd44 Address:48efb41d-3d76-0e92-265d-60e180bddd44} {Suffrage:Voter ID:server-c3817c50-7dc6-d156-2bbb-679d6b62e66c Address:c3817c50-7dc6-d156-2bbb-679d6b62e66c}]" +2025-06-17T07:44:31.389Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:31.389Z [INFO] server-c3817c50-7dc6-d156-2bbb-679d6b62e66c: entering follower state: follower="Node at c3817c50-7dc6-d156-2bbb-679d6b62e66c [Follower]" leader-address= leader-id= +2025-06-17T07:44:31.454Z [WARN] server-c3817c50-7dc6-d156-2bbb-679d6b62e66c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:31.454Z [INFO] server-c3817c50-7dc6-d156-2bbb-679d6b62e66c: entering candidate state: node="Node at c3817c50-7dc6-d156-2bbb-679d6b62e66c [Candidate]" term=2 +2025-06-17T07:44:31.454Z [INFO] server-c3817c50-7dc6-d156-2bbb-679d6b62e66c: election won: term=2 tally=2 +2025-06-17T07:44:31.454Z [INFO] server-c3817c50-7dc6-d156-2bbb-679d6b62e66c: entering leader state: leader="Node at c3817c50-7dc6-d156-2bbb-679d6b62e66c [Leader]" +2025-06-17T07:44:31.454Z [INFO] server-c3817c50-7dc6-d156-2bbb-679d6b62e66c: added peer, starting replication: peer=server-921ede8b-0809-aea1-038b-ab1f6af5eb46 +2025-06-17T07:44:31.454Z [INFO] server-c3817c50-7dc6-d156-2bbb-679d6b62e66c: added peer, starting replication: peer=server-48efb41d-3d76-0e92-265d-60e180bddd44 +2025-06-17T07:44:31.454Z [INFO] server-c3817c50-7dc6-d156-2bbb-679d6b62e66c: pipelining replication: peer="{Voter server-921ede8b-0809-aea1-038b-ab1f6af5eb46 921ede8b-0809-aea1-038b-ab1f6af5eb46}" +2025-06-17T07:44:31.454Z [INFO] server-c3817c50-7dc6-d156-2bbb-679d6b62e66c: pipelining replication: peer="{Voter server-48efb41d-3d76-0e92-265d-60e180bddd44 48efb41d-3d76-0e92-265d-60e180bddd44}" +2025-06-17T07:44:31.560Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:31.45457948 +0000 UTC m=+100.058108178 (1 nodes), 64.867688ms from start of poll, 65.314372ms from cluster start. Timeout at 2025-06-17 07:44:31.560081512 +0000 UTC m=+100.163610205, 105.502027ms after stability +2025-06-17T07:44:31.560Z [INFO] server-c3817c50-7dc6-d156-2bbb-679d6b62e66c: aborting pipeline replication: peer="{Voter server-921ede8b-0809-aea1-038b-ab1f6af5eb46 921ede8b-0809-aea1-038b-ab1f6af5eb46}" +2025-06-17T07:44:31.560Z [INFO] server-c3817c50-7dc6-d156-2bbb-679d6b62e66c: aborting pipeline replication: peer="{Voter server-48efb41d-3d76-0e92-265d-60e180bddd44 48efb41d-3d76-0e92-265d-60e180bddd44}" +2025-06-17T07:44:32.060Z [ERROR] server-c3817c50-7dc6-d156-2bbb-679d6b62e66c: failed to heartbeat to: peer=921ede8b-0809-aea1-038b-ab1f6af5eb46 backoff time=10ms error="command timed out" +--- PASS: TestRaft_VerifyLeader (0.67s) === RUN TestRaft_VerifyLeader_Single -2024-05-15T01:18:40.264Z [INFO] server-660407af-57d3-72d4-0db7-42a953c26d34: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-660407af-57d3-72d4-0db7-42a953c26d34 Address:660407af-57d3-72d4-0db7-42a953c26d34}]" -2024-05-15T01:18:40.264Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:40.264Z [INFO] server-660407af-57d3-72d4-0db7-42a953c26d34: entering follower state: follower="Node at 660407af-57d3-72d4-0db7-42a953c26d34 [Follower]" leader-address= leader-id= -2024-05-15T01:18:40.350Z [WARN] server-660407af-57d3-72d4-0db7-42a953c26d34: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:40.350Z [INFO] server-660407af-57d3-72d4-0db7-42a953c26d34: entering candidate state: node="Node at 660407af-57d3-72d4-0db7-42a953c26d34 [Candidate]" term=2 -2024-05-15T01:18:40.350Z [INFO] server-660407af-57d3-72d4-0db7-42a953c26d34: election won: term=2 tally=1 -2024-05-15T01:18:40.350Z [INFO] server-660407af-57d3-72d4-0db7-42a953c26d34: entering leader state: leader="Node at 660407af-57d3-72d4-0db7-42a953c26d34 [Leader]" -2024-05-15T01:18:40.456Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:40.350919326 +0000 UTC m=+100.892306600 (1 nodes), 85.99781ms from start of poll, 86.143484ms from cluster start. Timeout at 2024-05-15 01:18:40.456435201 +0000 UTC m=+100.997826507, 105.519907ms after stability ---- PASS: TestRaft_VerifyLeader_Single (0.19s) +2025-06-17T07:44:32.061Z [INFO] server-d48d5235-4cc0-cbb6-e333-dacb23c4d973: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d48d5235-4cc0-cbb6-e333-dacb23c4d973 Address:d48d5235-4cc0-cbb6-e333-dacb23c4d973}]" +2025-06-17T07:44:32.061Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:32.061Z [INFO] server-d48d5235-4cc0-cbb6-e333-dacb23c4d973: entering follower state: follower="Node at d48d5235-4cc0-cbb6-e333-dacb23c4d973 [Follower]" leader-address= leader-id= +2025-06-17T07:44:32.121Z [WARN] server-d48d5235-4cc0-cbb6-e333-dacb23c4d973: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:32.121Z [INFO] server-d48d5235-4cc0-cbb6-e333-dacb23c4d973: entering candidate state: node="Node at d48d5235-4cc0-cbb6-e333-dacb23c4d973 [Candidate]" term=2 +2025-06-17T07:44:32.121Z [INFO] server-d48d5235-4cc0-cbb6-e333-dacb23c4d973: election won: term=2 tally=1 +2025-06-17T07:44:32.121Z [INFO] server-d48d5235-4cc0-cbb6-e333-dacb23c4d973: entering leader state: leader="Node at d48d5235-4cc0-cbb6-e333-dacb23c4d973 [Leader]" +2025-06-17T07:44:32.227Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:32.121919397 +0000 UTC m=+100.725448104 (1 nodes), 60.361169ms from start of poll, 60.486448ms from cluster start. Timeout at 2025-06-17 07:44:32.22726732 +0000 UTC m=+100.830795997, 105.347893ms after stability +--- PASS: TestRaft_VerifyLeader_Single (0.17s) === RUN TestRaft_VerifyLeader_Fail -2024-05-15T01:18:40.459Z [INFO] server-3808974f-a79e-c3b4-e538-af16bd9cbcc3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3808974f-a79e-c3b4-e538-af16bd9cbcc3 Address:3808974f-a79e-c3b4-e538-af16bd9cbcc3} {Suffrage:Voter ID:server-69e80e90-ee92-6476-f07e-ec266814bf1d Address:69e80e90-ee92-6476-f07e-ec266814bf1d}]" -2024-05-15T01:18:40.459Z [INFO] server-69e80e90-ee92-6476-f07e-ec266814bf1d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3808974f-a79e-c3b4-e538-af16bd9cbcc3 Address:3808974f-a79e-c3b4-e538-af16bd9cbcc3} {Suffrage:Voter ID:server-69e80e90-ee92-6476-f07e-ec266814bf1d Address:69e80e90-ee92-6476-f07e-ec266814bf1d}]" -2024-05-15T01:18:40.459Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:40.459Z [INFO] server-3808974f-a79e-c3b4-e538-af16bd9cbcc3: entering follower state: follower="Node at 3808974f-a79e-c3b4-e538-af16bd9cbcc3 [Follower]" leader-address= leader-id= -2024-05-15T01:18:40.460Z [INFO] server-69e80e90-ee92-6476-f07e-ec266814bf1d: entering follower state: follower="Node at 69e80e90-ee92-6476-f07e-ec266814bf1d [Follower]" leader-address= leader-id= -2024-05-15T01:18:40.531Z [WARN] server-3808974f-a79e-c3b4-e538-af16bd9cbcc3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:40.531Z [INFO] server-3808974f-a79e-c3b4-e538-af16bd9cbcc3: entering candidate state: node="Node at 3808974f-a79e-c3b4-e538-af16bd9cbcc3 [Candidate]" term=2 -2024-05-15T01:18:40.531Z [INFO] server-3808974f-a79e-c3b4-e538-af16bd9cbcc3: election won: term=2 tally=2 -2024-05-15T01:18:40.531Z [INFO] server-3808974f-a79e-c3b4-e538-af16bd9cbcc3: entering leader state: leader="Node at 3808974f-a79e-c3b4-e538-af16bd9cbcc3 [Leader]" -2024-05-15T01:18:40.531Z [INFO] server-3808974f-a79e-c3b4-e538-af16bd9cbcc3: added peer, starting replication: peer=server-69e80e90-ee92-6476-f07e-ec266814bf1d -2024-05-15T01:18:40.531Z [INFO] server-3808974f-a79e-c3b4-e538-af16bd9cbcc3: pipelining replication: peer="{Voter server-69e80e90-ee92-6476-f07e-ec266814bf1d 69e80e90-ee92-6476-f07e-ec266814bf1d}" -2024-05-15T01:18:40.637Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:40.531579292 +0000 UTC m=+101.072964407 (1 nodes), 71.634835ms from start of poll, 72.000192ms from cluster start. Timeout at 2024-05-15 01:18:40.637605229 +0000 UTC m=+101.178992392, 106.027985ms after stability -2024-05-15T01:18:40.637Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:40.743Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:40.637726058 +0000 UTC m=+101.179110296 (1 nodes), 4.753µs from start of poll, 178.146081ms from cluster start. Timeout at 2024-05-15 01:18:40.743201126 +0000 UTC m=+101.284586117, 105.475821ms after stability -2024-05-15T01:18:40.743Z [ERROR] server-3808974f-a79e-c3b4-e538-af16bd9cbcc3: peer has newer term, stopping replication: peer="{Voter server-69e80e90-ee92-6476-f07e-ec266814bf1d 69e80e90-ee92-6476-f07e-ec266814bf1d}" -2024-05-15T01:18:40.743Z [INFO] server-3808974f-a79e-c3b4-e538-af16bd9cbcc3: aborting pipeline replication: peer="{Voter server-69e80e90-ee92-6476-f07e-ec266814bf1d 69e80e90-ee92-6476-f07e-ec266814bf1d}" -2024-05-15T01:18:40.743Z [INFO] server-3808974f-a79e-c3b4-e538-af16bd9cbcc3: entering follower state: follower="Node at 3808974f-a79e-c3b4-e538-af16bd9cbcc3 [Follower]" leader-address= leader-id= ---- PASS: TestRaft_VerifyLeader_Fail (0.29s) +2025-06-17T07:44:32.227Z [INFO] server-e1bd64f3-3c25-109b-e5e5-8316f5a8a9a2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e1bd64f3-3c25-109b-e5e5-8316f5a8a9a2 Address:e1bd64f3-3c25-109b-e5e5-8316f5a8a9a2} {Suffrage:Voter ID:server-8c77513d-6e59-220e-cd28-0744a22b9dda Address:8c77513d-6e59-220e-cd28-0744a22b9dda}]" +2025-06-17T07:44:32.228Z [INFO] server-e1bd64f3-3c25-109b-e5e5-8316f5a8a9a2: entering follower state: follower="Node at e1bd64f3-3c25-109b-e5e5-8316f5a8a9a2 [Follower]" leader-address= leader-id= +2025-06-17T07:44:32.228Z [INFO] server-8c77513d-6e59-220e-cd28-0744a22b9dda: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e1bd64f3-3c25-109b-e5e5-8316f5a8a9a2 Address:e1bd64f3-3c25-109b-e5e5-8316f5a8a9a2} {Suffrage:Voter ID:server-8c77513d-6e59-220e-cd28-0744a22b9dda Address:8c77513d-6e59-220e-cd28-0744a22b9dda}]" +2025-06-17T07:44:32.228Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:32.228Z [INFO] server-8c77513d-6e59-220e-cd28-0744a22b9dda: entering follower state: follower="Node at 8c77513d-6e59-220e-cd28-0744a22b9dda [Follower]" leader-address= leader-id= +2025-06-17T07:44:32.313Z [WARN] server-8c77513d-6e59-220e-cd28-0744a22b9dda: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:32.313Z [INFO] server-8c77513d-6e59-220e-cd28-0744a22b9dda: entering candidate state: node="Node at 8c77513d-6e59-220e-cd28-0744a22b9dda [Candidate]" term=2 +2025-06-17T07:44:32.313Z [INFO] server-8c77513d-6e59-220e-cd28-0744a22b9dda: election won: term=2 tally=2 +2025-06-17T07:44:32.313Z [INFO] server-8c77513d-6e59-220e-cd28-0744a22b9dda: entering leader state: leader="Node at 8c77513d-6e59-220e-cd28-0744a22b9dda [Leader]" +2025-06-17T07:44:32.313Z [INFO] server-8c77513d-6e59-220e-cd28-0744a22b9dda: added peer, starting replication: peer=server-e1bd64f3-3c25-109b-e5e5-8316f5a8a9a2 +2025-06-17T07:44:32.314Z [INFO] server-8c77513d-6e59-220e-cd28-0744a22b9dda: pipelining replication: peer="{Voter server-e1bd64f3-3c25-109b-e5e5-8316f5a8a9a2 e1bd64f3-3c25-109b-e5e5-8316f5a8a9a2}" +2025-06-17T07:44:32.419Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:32.313891482 +0000 UTC m=+100.917420182 (1 nodes), 85.741925ms from start of poll, 85.987055ms from cluster start. Timeout at 2025-06-17 07:44:32.41900461 +0000 UTC m=+101.022533292, 105.11311ms after stability +2025-06-17T07:44:32.419Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:32.524Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:32.419064472 +0000 UTC m=+101.022593166 (1 nodes), 454ns from start of poll, 191.160039ms from cluster start. Timeout at 2025-06-17 07:44:32.524961562 +0000 UTC m=+101.128490245, 105.897079ms after stability +2025-06-17T07:44:32.526Z [ERROR] server-8c77513d-6e59-220e-cd28-0744a22b9dda: peer has newer term, stopping replication: peer="{Voter server-e1bd64f3-3c25-109b-e5e5-8316f5a8a9a2 e1bd64f3-3c25-109b-e5e5-8316f5a8a9a2}" +2025-06-17T07:44:32.526Z [INFO] server-8c77513d-6e59-220e-cd28-0744a22b9dda: aborting pipeline replication: peer="{Voter server-e1bd64f3-3c25-109b-e5e5-8316f5a8a9a2 e1bd64f3-3c25-109b-e5e5-8316f5a8a9a2}" +2025-06-17T07:44:32.526Z [INFO] server-8c77513d-6e59-220e-cd28-0744a22b9dda: entering follower state: follower="Node at 8c77513d-6e59-220e-cd28-0744a22b9dda [Follower]" leader-address= leader-id= +--- PASS: TestRaft_VerifyLeader_Fail (0.30s) === RUN TestRaft_VerifyLeader_PartialConnect -2024-05-15T01:18:40.745Z [INFO] server-5e2c74e1-7c5d-309d-2329-faa588c43418: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5e2c74e1-7c5d-309d-2329-faa588c43418 Address:5e2c74e1-7c5d-309d-2329-faa588c43418} {Suffrage:Voter ID:server-5963bcf4-4cf6-8602-adeb-133568e4b346 Address:5963bcf4-4cf6-8602-adeb-133568e4b346} {Suffrage:Voter ID:server-a25c2858-6cc0-c6a8-631a-662786ed8105 Address:a25c2858-6cc0-c6a8-631a-662786ed8105}]" -2024-05-15T01:18:40.745Z [INFO] server-5963bcf4-4cf6-8602-adeb-133568e4b346: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5e2c74e1-7c5d-309d-2329-faa588c43418 Address:5e2c74e1-7c5d-309d-2329-faa588c43418} {Suffrage:Voter ID:server-5963bcf4-4cf6-8602-adeb-133568e4b346 Address:5963bcf4-4cf6-8602-adeb-133568e4b346} {Suffrage:Voter ID:server-a25c2858-6cc0-c6a8-631a-662786ed8105 Address:a25c2858-6cc0-c6a8-631a-662786ed8105}]" -2024-05-15T01:18:40.745Z [INFO] server-a25c2858-6cc0-c6a8-631a-662786ed8105: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5e2c74e1-7c5d-309d-2329-faa588c43418 Address:5e2c74e1-7c5d-309d-2329-faa588c43418} {Suffrage:Voter ID:server-5963bcf4-4cf6-8602-adeb-133568e4b346 Address:5963bcf4-4cf6-8602-adeb-133568e4b346} {Suffrage:Voter ID:server-a25c2858-6cc0-c6a8-631a-662786ed8105 Address:a25c2858-6cc0-c6a8-631a-662786ed8105}]" -2024-05-15T01:18:40.745Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:40.745Z [INFO] server-5e2c74e1-7c5d-309d-2329-faa588c43418: entering follower state: follower="Node at 5e2c74e1-7c5d-309d-2329-faa588c43418 [Follower]" leader-address= leader-id= -2024-05-15T01:18:40.745Z [INFO] server-5963bcf4-4cf6-8602-adeb-133568e4b346: entering follower state: follower="Node at 5963bcf4-4cf6-8602-adeb-133568e4b346 [Follower]" leader-address= leader-id= -2024-05-15T01:18:40.745Z [INFO] server-a25c2858-6cc0-c6a8-631a-662786ed8105: entering follower state: follower="Node at a25c2858-6cc0-c6a8-631a-662786ed8105 [Follower]" leader-address= leader-id= -2024-05-15T01:18:40.819Z [WARN] server-a25c2858-6cc0-c6a8-631a-662786ed8105: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:40.819Z [INFO] server-a25c2858-6cc0-c6a8-631a-662786ed8105: entering candidate state: node="Node at a25c2858-6cc0-c6a8-631a-662786ed8105 [Candidate]" term=2 -2024-05-15T01:18:40.819Z [INFO] server-a25c2858-6cc0-c6a8-631a-662786ed8105: election won: term=2 tally=2 -2024-05-15T01:18:40.819Z [INFO] server-a25c2858-6cc0-c6a8-631a-662786ed8105: entering leader state: leader="Node at a25c2858-6cc0-c6a8-631a-662786ed8105 [Leader]" -2024-05-15T01:18:40.819Z [INFO] server-a25c2858-6cc0-c6a8-631a-662786ed8105: added peer, starting replication: peer=server-5e2c74e1-7c5d-309d-2329-faa588c43418 -2024-05-15T01:18:40.819Z [INFO] server-a25c2858-6cc0-c6a8-631a-662786ed8105: added peer, starting replication: peer=server-5963bcf4-4cf6-8602-adeb-133568e4b346 -2024-05-15T01:18:40.819Z [INFO] server-a25c2858-6cc0-c6a8-631a-662786ed8105: pipelining replication: peer="{Voter server-5e2c74e1-7c5d-309d-2329-faa588c43418 5e2c74e1-7c5d-309d-2329-faa588c43418}" -2024-05-15T01:18:40.820Z [INFO] server-a25c2858-6cc0-c6a8-631a-662786ed8105: pipelining replication: peer="{Voter server-5963bcf4-4cf6-8602-adeb-133568e4b346 5963bcf4-4cf6-8602-adeb-133568e4b346}" -2024-05-15T01:18:40.925Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:40.820069192 +0000 UTC m=+101.361453430 (1 nodes), 74.449009ms from start of poll, 75.030394ms from cluster start. Timeout at 2024-05-15 01:18:40.925457218 +0000 UTC m=+101.466841973, 105.388543ms after stability -2024-05-15T01:18:40.930Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:41.037Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:40.930796699 +0000 UTC m=+101.472183793 (2 nodes), 2.856µs from start of poll, 185.760757ms from cluster start. Timeout at 2024-05-15 01:18:41.037254845 +0000 UTC m=+101.578639837, 106.456044ms after stability - raft_test.go:1968: [INFO] Disconnecting Node at 5e2c74e1-7c5d-309d-2329-faa588c43418 [Follower] -2024-05-15T01:18:41.037Z [ERROR] server-a25c2858-6cc0-c6a8-631a-662786ed8105: failed to heartbeat to: peer=5e2c74e1-7c5d-309d-2329-faa588c43418 backoff time=10ms error="failed to connect to peer: 5e2c74e1-7c5d-309d-2329-faa588c43418" -2024-05-15T01:18:41.037Z [INFO] server-a25c2858-6cc0-c6a8-631a-662786ed8105: aborting pipeline replication: peer="{Voter server-5963bcf4-4cf6-8602-adeb-133568e4b346 5963bcf4-4cf6-8602-adeb-133568e4b346}" -2024-05-15T01:18:41.037Z [INFO] server-a25c2858-6cc0-c6a8-631a-662786ed8105: aborting pipeline replication: peer="{Voter server-5e2c74e1-7c5d-309d-2329-faa588c43418 5e2c74e1-7c5d-309d-2329-faa588c43418}" ---- PASS: TestRaft_VerifyLeader_PartialConnect (0.29s) +2025-06-17T07:44:32.527Z [INFO] server-f03ed76b-09a0-f263-6ab4-bc86fbf667a5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f03ed76b-09a0-f263-6ab4-bc86fbf667a5 Address:f03ed76b-09a0-f263-6ab4-bc86fbf667a5} {Suffrage:Voter ID:server-9af3b792-571e-908c-e6ef-eeb0e8e3b759 Address:9af3b792-571e-908c-e6ef-eeb0e8e3b759} {Suffrage:Voter ID:server-450bc437-3814-e9e3-5d2f-39aefc5f4f46 Address:450bc437-3814-e9e3-5d2f-39aefc5f4f46}]" +2025-06-17T07:44:32.527Z [INFO] server-f03ed76b-09a0-f263-6ab4-bc86fbf667a5: entering follower state: follower="Node at f03ed76b-09a0-f263-6ab4-bc86fbf667a5 [Follower]" leader-address= leader-id= +2025-06-17T07:44:32.528Z [INFO] server-9af3b792-571e-908c-e6ef-eeb0e8e3b759: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f03ed76b-09a0-f263-6ab4-bc86fbf667a5 Address:f03ed76b-09a0-f263-6ab4-bc86fbf667a5} {Suffrage:Voter ID:server-9af3b792-571e-908c-e6ef-eeb0e8e3b759 Address:9af3b792-571e-908c-e6ef-eeb0e8e3b759} {Suffrage:Voter ID:server-450bc437-3814-e9e3-5d2f-39aefc5f4f46 Address:450bc437-3814-e9e3-5d2f-39aefc5f4f46}]" +2025-06-17T07:44:32.528Z [INFO] server-9af3b792-571e-908c-e6ef-eeb0e8e3b759: entering follower state: follower="Node at 9af3b792-571e-908c-e6ef-eeb0e8e3b759 [Follower]" leader-address= leader-id= +2025-06-17T07:44:32.528Z [INFO] server-450bc437-3814-e9e3-5d2f-39aefc5f4f46: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f03ed76b-09a0-f263-6ab4-bc86fbf667a5 Address:f03ed76b-09a0-f263-6ab4-bc86fbf667a5} {Suffrage:Voter ID:server-9af3b792-571e-908c-e6ef-eeb0e8e3b759 Address:9af3b792-571e-908c-e6ef-eeb0e8e3b759} {Suffrage:Voter ID:server-450bc437-3814-e9e3-5d2f-39aefc5f4f46 Address:450bc437-3814-e9e3-5d2f-39aefc5f4f46}]" +2025-06-17T07:44:32.528Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:32.528Z [INFO] server-450bc437-3814-e9e3-5d2f-39aefc5f4f46: entering follower state: follower="Node at 450bc437-3814-e9e3-5d2f-39aefc5f4f46 [Follower]" leader-address= leader-id= +2025-06-17T07:44:32.579Z [WARN] server-f03ed76b-09a0-f263-6ab4-bc86fbf667a5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:32.579Z [INFO] server-f03ed76b-09a0-f263-6ab4-bc86fbf667a5: entering candidate state: node="Node at f03ed76b-09a0-f263-6ab4-bc86fbf667a5 [Candidate]" term=2 +2025-06-17T07:44:32.579Z [INFO] server-f03ed76b-09a0-f263-6ab4-bc86fbf667a5: election won: term=2 tally=2 +2025-06-17T07:44:32.579Z [INFO] server-f03ed76b-09a0-f263-6ab4-bc86fbf667a5: entering leader state: leader="Node at f03ed76b-09a0-f263-6ab4-bc86fbf667a5 [Leader]" +2025-06-17T07:44:32.579Z [INFO] server-f03ed76b-09a0-f263-6ab4-bc86fbf667a5: added peer, starting replication: peer=server-9af3b792-571e-908c-e6ef-eeb0e8e3b759 +2025-06-17T07:44:32.579Z [INFO] server-f03ed76b-09a0-f263-6ab4-bc86fbf667a5: added peer, starting replication: peer=server-450bc437-3814-e9e3-5d2f-39aefc5f4f46 +2025-06-17T07:44:32.579Z [INFO] server-f03ed76b-09a0-f263-6ab4-bc86fbf667a5: pipelining replication: peer="{Voter server-9af3b792-571e-908c-e6ef-eeb0e8e3b759 9af3b792-571e-908c-e6ef-eeb0e8e3b759}" +2025-06-17T07:44:32.579Z [INFO] server-f03ed76b-09a0-f263-6ab4-bc86fbf667a5: pipelining replication: peer="{Voter server-450bc437-3814-e9e3-5d2f-39aefc5f4f46 450bc437-3814-e9e3-5d2f-39aefc5f4f46}" +2025-06-17T07:44:32.685Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:32.579209969 +0000 UTC m=+101.182738671 (1 nodes), 50.638861ms from start of poll, 51.666959ms from cluster start. Timeout at 2025-06-17 07:44:32.684978223 +0000 UTC m=+101.288506905, 105.768234ms after stability +2025-06-17T07:44:32.691Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:32.796Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:32.691104873 +0000 UTC m=+101.294633570 (2 nodes), 837ns from start of poll, 163.561858ms from cluster start. Timeout at 2025-06-17 07:44:32.796554179 +0000 UTC m=+101.400082862, 105.449292ms after stability + raft_test.go:1968: [INFO] Disconnecting Node at 9af3b792-571e-908c-e6ef-eeb0e8e3b759 [Follower] +2025-06-17T07:44:32.796Z [ERROR] server-f03ed76b-09a0-f263-6ab4-bc86fbf667a5: failed to heartbeat to: peer=9af3b792-571e-908c-e6ef-eeb0e8e3b759 backoff time=10ms error="failed to connect to peer: 9af3b792-571e-908c-e6ef-eeb0e8e3b759" +2025-06-17T07:44:32.796Z [INFO] server-f03ed76b-09a0-f263-6ab4-bc86fbf667a5: aborting pipeline replication: peer="{Voter server-450bc437-3814-e9e3-5d2f-39aefc5f4f46 450bc437-3814-e9e3-5d2f-39aefc5f4f46}" +2025-06-17T07:44:32.796Z [INFO] server-f03ed76b-09a0-f263-6ab4-bc86fbf667a5: aborting pipeline replication: peer="{Voter server-9af3b792-571e-908c-e6ef-eeb0e8e3b759 9af3b792-571e-908c-e6ef-eeb0e8e3b759}" +--- PASS: TestRaft_VerifyLeader_PartialConnect (0.27s) === RUN TestRaft_NotifyCh -2024-05-15T01:18:41.038Z [INFO] server-73e77fab-ba5d-8494-ec8a-0bf230682929: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-73e77fab-ba5d-8494-ec8a-0bf230682929 Address:73e77fab-ba5d-8494-ec8a-0bf230682929}]" -2024-05-15T01:18:41.038Z [INFO] server-73e77fab-ba5d-8494-ec8a-0bf230682929: entering follower state: follower="Node at 73e77fab-ba5d-8494-ec8a-0bf230682929 [Follower]" leader-address= leader-id= -2024-05-15T01:18:41.130Z [WARN] server-73e77fab-ba5d-8494-ec8a-0bf230682929: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:41.130Z [INFO] server-73e77fab-ba5d-8494-ec8a-0bf230682929: entering candidate state: node="Node at 73e77fab-ba5d-8494-ec8a-0bf230682929 [Candidate]" term=2 -2024-05-15T01:18:41.130Z [INFO] server-73e77fab-ba5d-8494-ec8a-0bf230682929: election won: term=2 tally=1 -2024-05-15T01:18:41.130Z [INFO] server-73e77fab-ba5d-8494-ec8a-0bf230682929: entering leader state: leader="Node at 73e77fab-ba5d-8494-ec8a-0bf230682929 [Leader]" ---- PASS: TestRaft_NotifyCh (0.09s) +2025-06-17T07:44:32.797Z [INFO] server-4180d1fe-5f83-e60c-fc2e-ad404801ab2c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4180d1fe-5f83-e60c-fc2e-ad404801ab2c Address:4180d1fe-5f83-e60c-fc2e-ad404801ab2c}]" +2025-06-17T07:44:32.797Z [INFO] server-4180d1fe-5f83-e60c-fc2e-ad404801ab2c: entering follower state: follower="Node at 4180d1fe-5f83-e60c-fc2e-ad404801ab2c [Follower]" leader-address= leader-id= +2025-06-17T07:44:32.865Z [WARN] server-4180d1fe-5f83-e60c-fc2e-ad404801ab2c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:32.865Z [INFO] server-4180d1fe-5f83-e60c-fc2e-ad404801ab2c: entering candidate state: node="Node at 4180d1fe-5f83-e60c-fc2e-ad404801ab2c [Candidate]" term=2 +2025-06-17T07:44:32.865Z [INFO] server-4180d1fe-5f83-e60c-fc2e-ad404801ab2c: election won: term=2 tally=1 +2025-06-17T07:44:32.865Z [INFO] server-4180d1fe-5f83-e60c-fc2e-ad404801ab2c: entering leader state: leader="Node at 4180d1fe-5f83-e60c-fc2e-ad404801ab2c [Leader]" +--- PASS: TestRaft_NotifyCh (0.07s) === RUN TestRaft_AppendEntry -2024-05-15T01:18:41.131Z [INFO] server-23cab09a-e9b8-f33b-d795-c4abc93d83df: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-23cab09a-e9b8-f33b-d795-c4abc93d83df Address:23cab09a-e9b8-f33b-d795-c4abc93d83df} {Suffrage:Voter ID:server-47e31f80-98ff-189f-9608-8d8c5c9c7910 Address:47e31f80-98ff-189f-9608-8d8c5c9c7910} {Suffrage:Voter ID:server-456b1536-1f03-c01a-ef14-20e077d308ba Address:456b1536-1f03-c01a-ef14-20e077d308ba}]" -2024-05-15T01:18:41.131Z [INFO] server-47e31f80-98ff-189f-9608-8d8c5c9c7910: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-23cab09a-e9b8-f33b-d795-c4abc93d83df Address:23cab09a-e9b8-f33b-d795-c4abc93d83df} {Suffrage:Voter ID:server-47e31f80-98ff-189f-9608-8d8c5c9c7910 Address:47e31f80-98ff-189f-9608-8d8c5c9c7910} {Suffrage:Voter ID:server-456b1536-1f03-c01a-ef14-20e077d308ba Address:456b1536-1f03-c01a-ef14-20e077d308ba}]" -2024-05-15T01:18:41.131Z [INFO] server-23cab09a-e9b8-f33b-d795-c4abc93d83df: entering follower state: follower="Node at 23cab09a-e9b8-f33b-d795-c4abc93d83df [Follower]" leader-address= leader-id= -2024-05-15T01:18:41.131Z [INFO] server-47e31f80-98ff-189f-9608-8d8c5c9c7910: entering follower state: follower="Node at 47e31f80-98ff-189f-9608-8d8c5c9c7910 [Follower]" leader-address= leader-id= -2024-05-15T01:18:41.131Z [INFO] server-456b1536-1f03-c01a-ef14-20e077d308ba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-23cab09a-e9b8-f33b-d795-c4abc93d83df Address:23cab09a-e9b8-f33b-d795-c4abc93d83df} {Suffrage:Voter ID:server-47e31f80-98ff-189f-9608-8d8c5c9c7910 Address:47e31f80-98ff-189f-9608-8d8c5c9c7910} {Suffrage:Voter ID:server-456b1536-1f03-c01a-ef14-20e077d308ba Address:456b1536-1f03-c01a-ef14-20e077d308ba}]" -2024-05-15T01:18:41.132Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:41.132Z [INFO] server-456b1536-1f03-c01a-ef14-20e077d308ba: entering follower state: follower="Node at 456b1536-1f03-c01a-ef14-20e077d308ba [Follower]" leader-address= leader-id= -2024-05-15T01:18:41.183Z [WARN] server-47e31f80-98ff-189f-9608-8d8c5c9c7910: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:41.183Z [INFO] server-47e31f80-98ff-189f-9608-8d8c5c9c7910: entering candidate state: node="Node at 47e31f80-98ff-189f-9608-8d8c5c9c7910 [Candidate]" term=2 -2024-05-15T01:18:41.184Z [INFO] server-47e31f80-98ff-189f-9608-8d8c5c9c7910: election won: term=2 tally=2 -2024-05-15T01:18:41.184Z [INFO] server-47e31f80-98ff-189f-9608-8d8c5c9c7910: entering leader state: leader="Node at 47e31f80-98ff-189f-9608-8d8c5c9c7910 [Leader]" -2024-05-15T01:18:41.184Z [INFO] server-47e31f80-98ff-189f-9608-8d8c5c9c7910: added peer, starting replication: peer=server-23cab09a-e9b8-f33b-d795-c4abc93d83df -2024-05-15T01:18:41.184Z [INFO] server-47e31f80-98ff-189f-9608-8d8c5c9c7910: added peer, starting replication: peer=server-456b1536-1f03-c01a-ef14-20e077d308ba -2024-05-15T01:18:41.184Z [INFO] server-47e31f80-98ff-189f-9608-8d8c5c9c7910: pipelining replication: peer="{Voter server-456b1536-1f03-c01a-ef14-20e077d308ba 456b1536-1f03-c01a-ef14-20e077d308ba}" -2024-05-15T01:18:41.184Z [INFO] server-47e31f80-98ff-189f-9608-8d8c5c9c7910: pipelining replication: peer="{Voter server-23cab09a-e9b8-f33b-d795-c4abc93d83df 23cab09a-e9b8-f33b-d795-c4abc93d83df}" -2024-05-15T01:18:41.289Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:41.184113939 +0000 UTC m=+101.725498177 (2 nodes), 52.086623ms from start of poll, 52.638624ms from cluster start. Timeout at 2024-05-15 01:18:41.289839796 +0000 UTC m=+101.831224538, 105.726361ms after stability -2024-05-15T01:18:41.289Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:41.395Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:41.289995094 +0000 UTC m=+101.831380994 (1 nodes), 7.062µs from start of poll, 158.521441ms from cluster start. Timeout at 2024-05-15 01:18:41.395434006 +0000 UTC m=+101.936819065, 105.438071ms after stability -2024-05-15T01:18:41.395Z [WARN] server-23cab09a-e9b8-f33b-d795-c4abc93d83df: clearing log suffix: from=1 to=2 -2024-05-15T01:18:41.395Z [WARN] server-23cab09a-e9b8-f33b-d795-c4abc93d83df: skipping application of old log: index=1 -2024-05-15T01:18:41.395Z [WARN] server-23cab09a-e9b8-f33b-d795-c4abc93d83df: skipping application of old log: index=1 -2024-05-15T01:18:41.395Z [INFO] server-47e31f80-98ff-189f-9608-8d8c5c9c7910: aborting pipeline replication: peer="{Voter server-456b1536-1f03-c01a-ef14-20e077d308ba 456b1536-1f03-c01a-ef14-20e077d308ba}" -2024-05-15T01:18:41.395Z [INFO] server-47e31f80-98ff-189f-9608-8d8c5c9c7910: aborting pipeline replication: peer="{Voter server-23cab09a-e9b8-f33b-d795-c4abc93d83df 23cab09a-e9b8-f33b-d795-c4abc93d83df}" ---- PASS: TestRaft_AppendEntry (0.27s) +2025-06-17T07:44:32.866Z [INFO] server-6ba39708-1953-733a-b2ac-8ae3d2e04a64: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ba39708-1953-733a-b2ac-8ae3d2e04a64 Address:6ba39708-1953-733a-b2ac-8ae3d2e04a64} {Suffrage:Voter ID:server-1601dd54-c44e-c1cf-8fa6-897c4804bcec Address:1601dd54-c44e-c1cf-8fa6-897c4804bcec} {Suffrage:Voter ID:server-98e20ff9-59d9-8e31-b007-bf695d94a9f6 Address:98e20ff9-59d9-8e31-b007-bf695d94a9f6}]" +2025-06-17T07:44:32.866Z [INFO] server-6ba39708-1953-733a-b2ac-8ae3d2e04a64: entering follower state: follower="Node at 6ba39708-1953-733a-b2ac-8ae3d2e04a64 [Follower]" leader-address= leader-id= +2025-06-17T07:44:32.866Z [INFO] server-1601dd54-c44e-c1cf-8fa6-897c4804bcec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ba39708-1953-733a-b2ac-8ae3d2e04a64 Address:6ba39708-1953-733a-b2ac-8ae3d2e04a64} {Suffrage:Voter ID:server-1601dd54-c44e-c1cf-8fa6-897c4804bcec Address:1601dd54-c44e-c1cf-8fa6-897c4804bcec} {Suffrage:Voter ID:server-98e20ff9-59d9-8e31-b007-bf695d94a9f6 Address:98e20ff9-59d9-8e31-b007-bf695d94a9f6}]" +2025-06-17T07:44:32.866Z [INFO] server-1601dd54-c44e-c1cf-8fa6-897c4804bcec: entering follower state: follower="Node at 1601dd54-c44e-c1cf-8fa6-897c4804bcec [Follower]" leader-address= leader-id= +2025-06-17T07:44:32.866Z [INFO] server-98e20ff9-59d9-8e31-b007-bf695d94a9f6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ba39708-1953-733a-b2ac-8ae3d2e04a64 Address:6ba39708-1953-733a-b2ac-8ae3d2e04a64} {Suffrage:Voter ID:server-1601dd54-c44e-c1cf-8fa6-897c4804bcec Address:1601dd54-c44e-c1cf-8fa6-897c4804bcec} {Suffrage:Voter ID:server-98e20ff9-59d9-8e31-b007-bf695d94a9f6 Address:98e20ff9-59d9-8e31-b007-bf695d94a9f6}]" +2025-06-17T07:44:32.866Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:32.866Z [INFO] server-98e20ff9-59d9-8e31-b007-bf695d94a9f6: entering follower state: follower="Node at 98e20ff9-59d9-8e31-b007-bf695d94a9f6 [Follower]" leader-address= leader-id= +2025-06-17T07:44:32.919Z [WARN] server-6ba39708-1953-733a-b2ac-8ae3d2e04a64: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:32.919Z [INFO] server-6ba39708-1953-733a-b2ac-8ae3d2e04a64: entering candidate state: node="Node at 6ba39708-1953-733a-b2ac-8ae3d2e04a64 [Candidate]" term=2 +2025-06-17T07:44:32.919Z [INFO] server-6ba39708-1953-733a-b2ac-8ae3d2e04a64: election won: term=2 tally=2 +2025-06-17T07:44:32.919Z [INFO] server-6ba39708-1953-733a-b2ac-8ae3d2e04a64: entering leader state: leader="Node at 6ba39708-1953-733a-b2ac-8ae3d2e04a64 [Leader]" +2025-06-17T07:44:32.919Z [INFO] server-6ba39708-1953-733a-b2ac-8ae3d2e04a64: added peer, starting replication: peer=server-1601dd54-c44e-c1cf-8fa6-897c4804bcec +2025-06-17T07:44:32.919Z [INFO] server-6ba39708-1953-733a-b2ac-8ae3d2e04a64: added peer, starting replication: peer=server-98e20ff9-59d9-8e31-b007-bf695d94a9f6 +2025-06-17T07:44:32.919Z [INFO] server-6ba39708-1953-733a-b2ac-8ae3d2e04a64: pipelining replication: peer="{Voter server-1601dd54-c44e-c1cf-8fa6-897c4804bcec 1601dd54-c44e-c1cf-8fa6-897c4804bcec}" +2025-06-17T07:44:32.919Z [INFO] server-6ba39708-1953-733a-b2ac-8ae3d2e04a64: pipelining replication: peer="{Voter server-98e20ff9-59d9-8e31-b007-bf695d94a9f6 98e20ff9-59d9-8e31-b007-bf695d94a9f6}" +2025-06-17T07:44:33.024Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:32.919243128 +0000 UTC m=+101.522771844 (2 nodes), 52.831369ms from start of poll, 53.175462ms from cluster start. Timeout at 2025-06-17 07:44:33.024318257 +0000 UTC m=+101.627846950, 105.075106ms after stability +2025-06-17T07:44:33.024Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:33.129Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:33.024388367 +0000 UTC m=+101.627917054 (1 nodes), 617ns from start of poll, 158.320672ms from cluster start. Timeout at 2025-06-17 07:44:33.129602414 +0000 UTC m=+101.733131097, 105.214043ms after stability +2025-06-17T07:44:33.129Z [WARN] server-1601dd54-c44e-c1cf-8fa6-897c4804bcec: clearing log suffix: from=1 to=2 +2025-06-17T07:44:33.129Z [WARN] server-1601dd54-c44e-c1cf-8fa6-897c4804bcec: skipping application of old log: index=1 +2025-06-17T07:44:33.129Z [WARN] server-1601dd54-c44e-c1cf-8fa6-897c4804bcec: skipping application of old log: index=1 +2025-06-17T07:44:33.129Z [INFO] server-6ba39708-1953-733a-b2ac-8ae3d2e04a64: aborting pipeline replication: peer="{Voter server-98e20ff9-59d9-8e31-b007-bf695d94a9f6 98e20ff9-59d9-8e31-b007-bf695d94a9f6}" +2025-06-17T07:44:33.129Z [INFO] server-6ba39708-1953-733a-b2ac-8ae3d2e04a64: aborting pipeline replication: peer="{Voter server-1601dd54-c44e-c1cf-8fa6-897c4804bcec 1601dd54-c44e-c1cf-8fa6-897c4804bcec}" +--- PASS: TestRaft_AppendEntry (0.26s) === RUN TestRaft_VotingGrant_WhenLeaderAvailable -2024-05-15T01:18:41.397Z [INFO] server-703e3ae9-e0e3-4c14-b32e-5032a3785236: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-703e3ae9-e0e3-4c14-b32e-5032a3785236 Address:703e3ae9-e0e3-4c14-b32e-5032a3785236} {Suffrage:Voter ID:server-c774f9b9-f92b-9fed-8251-3af8aaacf317 Address:c774f9b9-f92b-9fed-8251-3af8aaacf317} {Suffrage:Voter ID:server-872a7319-59bc-885f-d0f7-aa872d946c78 Address:872a7319-59bc-885f-d0f7-aa872d946c78}]" -2024-05-15T01:18:41.397Z [INFO] server-703e3ae9-e0e3-4c14-b32e-5032a3785236: entering follower state: follower="Node at 703e3ae9-e0e3-4c14-b32e-5032a3785236 [Follower]" leader-address= leader-id= -2024-05-15T01:18:41.397Z [INFO] server-c774f9b9-f92b-9fed-8251-3af8aaacf317: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-703e3ae9-e0e3-4c14-b32e-5032a3785236 Address:703e3ae9-e0e3-4c14-b32e-5032a3785236} {Suffrage:Voter ID:server-c774f9b9-f92b-9fed-8251-3af8aaacf317 Address:c774f9b9-f92b-9fed-8251-3af8aaacf317} {Suffrage:Voter ID:server-872a7319-59bc-885f-d0f7-aa872d946c78 Address:872a7319-59bc-885f-d0f7-aa872d946c78}]" -2024-05-15T01:18:41.398Z [INFO] server-c774f9b9-f92b-9fed-8251-3af8aaacf317: entering follower state: follower="Node at c774f9b9-f92b-9fed-8251-3af8aaacf317 [Follower]" leader-address= leader-id= -2024-05-15T01:18:41.398Z [INFO] server-872a7319-59bc-885f-d0f7-aa872d946c78: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-703e3ae9-e0e3-4c14-b32e-5032a3785236 Address:703e3ae9-e0e3-4c14-b32e-5032a3785236} {Suffrage:Voter ID:server-c774f9b9-f92b-9fed-8251-3af8aaacf317 Address:c774f9b9-f92b-9fed-8251-3af8aaacf317} {Suffrage:Voter ID:server-872a7319-59bc-885f-d0f7-aa872d946c78 Address:872a7319-59bc-885f-d0f7-aa872d946c78}]" -2024-05-15T01:18:41.398Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:41.398Z [INFO] server-872a7319-59bc-885f-d0f7-aa872d946c78: entering follower state: follower="Node at 872a7319-59bc-885f-d0f7-aa872d946c78 [Follower]" leader-address= leader-id= -2024-05-15T01:18:41.460Z [WARN] server-c774f9b9-f92b-9fed-8251-3af8aaacf317: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:41.460Z [INFO] server-c774f9b9-f92b-9fed-8251-3af8aaacf317: entering candidate state: node="Node at c774f9b9-f92b-9fed-8251-3af8aaacf317 [Candidate]" term=2 -2024-05-15T01:18:41.461Z [INFO] server-c774f9b9-f92b-9fed-8251-3af8aaacf317: election won: term=2 tally=2 -2024-05-15T01:18:41.461Z [INFO] server-c774f9b9-f92b-9fed-8251-3af8aaacf317: entering leader state: leader="Node at c774f9b9-f92b-9fed-8251-3af8aaacf317 [Leader]" -2024-05-15T01:18:41.461Z [INFO] server-c774f9b9-f92b-9fed-8251-3af8aaacf317: added peer, starting replication: peer=server-703e3ae9-e0e3-4c14-b32e-5032a3785236 -2024-05-15T01:18:41.461Z [INFO] server-c774f9b9-f92b-9fed-8251-3af8aaacf317: added peer, starting replication: peer=server-872a7319-59bc-885f-d0f7-aa872d946c78 -2024-05-15T01:18:41.461Z [INFO] server-c774f9b9-f92b-9fed-8251-3af8aaacf317: pipelining replication: peer="{Voter server-872a7319-59bc-885f-d0f7-aa872d946c78 872a7319-59bc-885f-d0f7-aa872d946c78}" -2024-05-15T01:18:41.461Z [INFO] server-c774f9b9-f92b-9fed-8251-3af8aaacf317: pipelining replication: peer="{Voter server-703e3ae9-e0e3-4c14-b32e-5032a3785236 703e3ae9-e0e3-4c14-b32e-5032a3785236}" -2024-05-15T01:18:41.566Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:41.461193117 +0000 UTC m=+102.002578297 (2 nodes), 63.054376ms from start of poll, 63.578802ms from cluster start. Timeout at 2024-05-15 01:18:41.566312946 +0000 UTC m=+102.107697713, 105.119416ms after stability -2024-05-15T01:18:41.566Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:41.673Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:41.566508807 +0000 UTC m=+102.107893045 (1 nodes), 0s from start of poll, 168.89355ms from cluster start. Timeout at 2024-05-15 01:18:41.673385608 +0000 UTC m=+102.214770541, 106.877496ms after stability -2024-05-15T01:18:41.673Z [WARN] server-872a7319-59bc-885f-d0f7-aa872d946c78: rejecting vote request since we have a leader: from=703e3ae9-e0e3-4c14-b32e-5032a3785236 leader=c774f9b9-f92b-9fed-8251-3af8aaacf317 leader-id=server-c774f9b9-f92b-9fed-8251-3af8aaacf317 -2024-05-15T01:18:41.673Z [INFO] server-c774f9b9-f92b-9fed-8251-3af8aaacf317: aborting pipeline replication: peer="{Voter server-703e3ae9-e0e3-4c14-b32e-5032a3785236 703e3ae9-e0e3-4c14-b32e-5032a3785236}" -2024-05-15T01:18:41.673Z [INFO] server-c774f9b9-f92b-9fed-8251-3af8aaacf317: aborting pipeline replication: peer="{Voter server-872a7319-59bc-885f-d0f7-aa872d946c78 872a7319-59bc-885f-d0f7-aa872d946c78}" +2025-06-17T07:44:33.130Z [INFO] server-995e1e2b-560c-b22d-4d4a-f298fe4cde08: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-995e1e2b-560c-b22d-4d4a-f298fe4cde08 Address:995e1e2b-560c-b22d-4d4a-f298fe4cde08} {Suffrage:Voter ID:server-179dcfd9-d214-828d-0586-493c57139763 Address:179dcfd9-d214-828d-0586-493c57139763} {Suffrage:Voter ID:server-6ea0a45e-792e-67af-02b5-dff05298c185 Address:6ea0a45e-792e-67af-02b5-dff05298c185}]" +2025-06-17T07:44:33.130Z [INFO] server-995e1e2b-560c-b22d-4d4a-f298fe4cde08: entering follower state: follower="Node at 995e1e2b-560c-b22d-4d4a-f298fe4cde08 [Follower]" leader-address= leader-id= +2025-06-17T07:44:33.130Z [INFO] server-179dcfd9-d214-828d-0586-493c57139763: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-995e1e2b-560c-b22d-4d4a-f298fe4cde08 Address:995e1e2b-560c-b22d-4d4a-f298fe4cde08} {Suffrage:Voter ID:server-179dcfd9-d214-828d-0586-493c57139763 Address:179dcfd9-d214-828d-0586-493c57139763} {Suffrage:Voter ID:server-6ea0a45e-792e-67af-02b5-dff05298c185 Address:6ea0a45e-792e-67af-02b5-dff05298c185}]" +2025-06-17T07:44:33.130Z [INFO] server-179dcfd9-d214-828d-0586-493c57139763: entering follower state: follower="Node at 179dcfd9-d214-828d-0586-493c57139763 [Follower]" leader-address= leader-id= +2025-06-17T07:44:33.130Z [INFO] server-6ea0a45e-792e-67af-02b5-dff05298c185: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-995e1e2b-560c-b22d-4d4a-f298fe4cde08 Address:995e1e2b-560c-b22d-4d4a-f298fe4cde08} {Suffrage:Voter ID:server-179dcfd9-d214-828d-0586-493c57139763 Address:179dcfd9-d214-828d-0586-493c57139763} {Suffrage:Voter ID:server-6ea0a45e-792e-67af-02b5-dff05298c185 Address:6ea0a45e-792e-67af-02b5-dff05298c185}]" +2025-06-17T07:44:33.130Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:33.130Z [INFO] server-6ea0a45e-792e-67af-02b5-dff05298c185: entering follower state: follower="Node at 6ea0a45e-792e-67af-02b5-dff05298c185 [Follower]" leader-address= leader-id= +2025-06-17T07:44:33.195Z [WARN] server-179dcfd9-d214-828d-0586-493c57139763: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:33.195Z [INFO] server-179dcfd9-d214-828d-0586-493c57139763: entering candidate state: node="Node at 179dcfd9-d214-828d-0586-493c57139763 [Candidate]" term=2 +2025-06-17T07:44:33.195Z [INFO] server-179dcfd9-d214-828d-0586-493c57139763: election won: term=2 tally=2 +2025-06-17T07:44:33.195Z [INFO] server-179dcfd9-d214-828d-0586-493c57139763: entering leader state: leader="Node at 179dcfd9-d214-828d-0586-493c57139763 [Leader]" +2025-06-17T07:44:33.195Z [INFO] server-179dcfd9-d214-828d-0586-493c57139763: added peer, starting replication: peer=server-995e1e2b-560c-b22d-4d4a-f298fe4cde08 +2025-06-17T07:44:33.195Z [INFO] server-179dcfd9-d214-828d-0586-493c57139763: added peer, starting replication: peer=server-6ea0a45e-792e-67af-02b5-dff05298c185 +2025-06-17T07:44:33.195Z [INFO] server-179dcfd9-d214-828d-0586-493c57139763: pipelining replication: peer="{Voter server-6ea0a45e-792e-67af-02b5-dff05298c185 6ea0a45e-792e-67af-02b5-dff05298c185}" +2025-06-17T07:44:33.195Z [INFO] server-179dcfd9-d214-828d-0586-493c57139763: pipelining replication: peer="{Voter server-995e1e2b-560c-b22d-4d4a-f298fe4cde08 995e1e2b-560c-b22d-4d4a-f298fe4cde08}" +2025-06-17T07:44:33.300Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:33.195804071 +0000 UTC m=+101.799332758 (2 nodes), 64.848354ms from start of poll, 65.147728ms from cluster start. Timeout at 2025-06-17 07:44:33.300830721 +0000 UTC m=+101.904359411, 105.026653ms after stability +2025-06-17T07:44:33.300Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:33.406Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:33.300908969 +0000 UTC m=+101.904437657 (1 nodes), 865ns from start of poll, 170.252627ms from cluster start. Timeout at 2025-06-17 07:44:33.40632134 +0000 UTC m=+102.009850033, 105.412376ms after stability +2025-06-17T07:44:33.406Z [WARN] server-6ea0a45e-792e-67af-02b5-dff05298c185: rejecting vote request since we have a leader: from=995e1e2b-560c-b22d-4d4a-f298fe4cde08 leader=179dcfd9-d214-828d-0586-493c57139763 leader-id=server-179dcfd9-d214-828d-0586-493c57139763 +2025-06-17T07:44:33.406Z [INFO] server-179dcfd9-d214-828d-0586-493c57139763: aborting pipeline replication: peer="{Voter server-6ea0a45e-792e-67af-02b5-dff05298c185 6ea0a45e-792e-67af-02b5-dff05298c185}" +2025-06-17T07:44:33.406Z [INFO] server-179dcfd9-d214-828d-0586-493c57139763: aborting pipeline replication: peer="{Voter server-995e1e2b-560c-b22d-4d4a-f298fe4cde08 995e1e2b-560c-b22d-4d4a-f298fe4cde08}" --- PASS: TestRaft_VotingGrant_WhenLeaderAvailable (0.28s) === RUN TestRaft_ProtocolVersion_RejectRPC -2024-05-15T01:18:41.675Z [INFO] server-a01d50cf-252d-e6c4-7e5d-a14ee2a5aab0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a01d50cf-252d-e6c4-7e5d-a14ee2a5aab0 Address:a01d50cf-252d-e6c4-7e5d-a14ee2a5aab0} {Suffrage:Voter ID:server-f9d1d8da-8742-6a76-6820-80bb772360b5 Address:f9d1d8da-8742-6a76-6820-80bb772360b5} {Suffrage:Voter ID:server-2ece8409-8817-a179-6dff-02af2a40f285 Address:2ece8409-8817-a179-6dff-02af2a40f285}]" -2024-05-15T01:18:41.675Z [INFO] server-a01d50cf-252d-e6c4-7e5d-a14ee2a5aab0: entering follower state: follower="Node at a01d50cf-252d-e6c4-7e5d-a14ee2a5aab0 [Follower]" leader-address= leader-id= -2024-05-15T01:18:41.675Z [INFO] server-f9d1d8da-8742-6a76-6820-80bb772360b5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a01d50cf-252d-e6c4-7e5d-a14ee2a5aab0 Address:a01d50cf-252d-e6c4-7e5d-a14ee2a5aab0} {Suffrage:Voter ID:server-f9d1d8da-8742-6a76-6820-80bb772360b5 Address:f9d1d8da-8742-6a76-6820-80bb772360b5} {Suffrage:Voter ID:server-2ece8409-8817-a179-6dff-02af2a40f285 Address:2ece8409-8817-a179-6dff-02af2a40f285}]" -2024-05-15T01:18:41.675Z [INFO] server-f9d1d8da-8742-6a76-6820-80bb772360b5: entering follower state: follower="Node at f9d1d8da-8742-6a76-6820-80bb772360b5 [Follower]" leader-address= leader-id= -2024-05-15T01:18:41.676Z [INFO] server-2ece8409-8817-a179-6dff-02af2a40f285: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a01d50cf-252d-e6c4-7e5d-a14ee2a5aab0 Address:a01d50cf-252d-e6c4-7e5d-a14ee2a5aab0} {Suffrage:Voter ID:server-f9d1d8da-8742-6a76-6820-80bb772360b5 Address:f9d1d8da-8742-6a76-6820-80bb772360b5} {Suffrage:Voter ID:server-2ece8409-8817-a179-6dff-02af2a40f285 Address:2ece8409-8817-a179-6dff-02af2a40f285}]" -2024-05-15T01:18:41.676Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:41.676Z [INFO] server-2ece8409-8817-a179-6dff-02af2a40f285: entering follower state: follower="Node at 2ece8409-8817-a179-6dff-02af2a40f285 [Follower]" leader-address= leader-id= -2024-05-15T01:18:41.749Z [WARN] server-f9d1d8da-8742-6a76-6820-80bb772360b5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:41.749Z [INFO] server-f9d1d8da-8742-6a76-6820-80bb772360b5: entering candidate state: node="Node at f9d1d8da-8742-6a76-6820-80bb772360b5 [Candidate]" term=2 -2024-05-15T01:18:41.750Z [INFO] server-f9d1d8da-8742-6a76-6820-80bb772360b5: election won: term=2 tally=2 -2024-05-15T01:18:41.750Z [INFO] server-f9d1d8da-8742-6a76-6820-80bb772360b5: entering leader state: leader="Node at f9d1d8da-8742-6a76-6820-80bb772360b5 [Leader]" -2024-05-15T01:18:41.750Z [INFO] server-f9d1d8da-8742-6a76-6820-80bb772360b5: added peer, starting replication: peer=server-a01d50cf-252d-e6c4-7e5d-a14ee2a5aab0 -2024-05-15T01:18:41.750Z [INFO] server-f9d1d8da-8742-6a76-6820-80bb772360b5: added peer, starting replication: peer=server-2ece8409-8817-a179-6dff-02af2a40f285 -2024-05-15T01:18:41.750Z [INFO] server-f9d1d8da-8742-6a76-6820-80bb772360b5: pipelining replication: peer="{Voter server-a01d50cf-252d-e6c4-7e5d-a14ee2a5aab0 a01d50cf-252d-e6c4-7e5d-a14ee2a5aab0}" -2024-05-15T01:18:41.750Z [INFO] server-f9d1d8da-8742-6a76-6820-80bb772360b5: pipelining replication: peer="{Voter server-2ece8409-8817-a179-6dff-02af2a40f285 2ece8409-8817-a179-6dff-02af2a40f285}" -2024-05-15T01:18:41.858Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:41.750157325 +0000 UTC m=+102.291542195 (2 nodes), 73.808955ms from start of poll, 74.752779ms from cluster start. Timeout at 2024-05-15 01:18:41.858283985 +0000 UTC m=+102.399668223, 108.126028ms after stability -2024-05-15T01:18:41.859Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:41.966Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:41.860049718 +0000 UTC m=+102.401434716 (1 nodes), 3.773µs from start of poll, 184.6453ms from cluster start. Timeout at 2024-05-15 01:18:41.966065222 +0000 UTC m=+102.507449460, 106.014744ms after stability -2024-05-15T01:18:41.966Z [INFO] server-f9d1d8da-8742-6a76-6820-80bb772360b5: aborting pipeline replication: peer="{Voter server-2ece8409-8817-a179-6dff-02af2a40f285 2ece8409-8817-a179-6dff-02af2a40f285}" -2024-05-15T01:18:41.966Z [INFO] server-f9d1d8da-8742-6a76-6820-80bb772360b5: aborting pipeline replication: peer="{Voter server-a01d50cf-252d-e6c4-7e5d-a14ee2a5aab0 a01d50cf-252d-e6c4-7e5d-a14ee2a5aab0}" ---- PASS: TestRaft_ProtocolVersion_RejectRPC (0.29s) +2025-06-17T07:44:33.407Z [INFO] server-8bf5b205-ce5a-476c-c088-fabe7c534bc2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8bf5b205-ce5a-476c-c088-fabe7c534bc2 Address:8bf5b205-ce5a-476c-c088-fabe7c534bc2} {Suffrage:Voter ID:server-7d6494a7-da8f-641d-7ca2-d936fd579223 Address:7d6494a7-da8f-641d-7ca2-d936fd579223} {Suffrage:Voter ID:server-310b5aae-aad9-8db6-ac63-8ba49907461d Address:310b5aae-aad9-8db6-ac63-8ba49907461d}]" +2025-06-17T07:44:33.407Z [INFO] server-8bf5b205-ce5a-476c-c088-fabe7c534bc2: entering follower state: follower="Node at 8bf5b205-ce5a-476c-c088-fabe7c534bc2 [Follower]" leader-address= leader-id= +2025-06-17T07:44:33.407Z [INFO] server-7d6494a7-da8f-641d-7ca2-d936fd579223: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8bf5b205-ce5a-476c-c088-fabe7c534bc2 Address:8bf5b205-ce5a-476c-c088-fabe7c534bc2} {Suffrage:Voter ID:server-7d6494a7-da8f-641d-7ca2-d936fd579223 Address:7d6494a7-da8f-641d-7ca2-d936fd579223} {Suffrage:Voter ID:server-310b5aae-aad9-8db6-ac63-8ba49907461d Address:310b5aae-aad9-8db6-ac63-8ba49907461d}]" +2025-06-17T07:44:33.407Z [INFO] server-7d6494a7-da8f-641d-7ca2-d936fd579223: entering follower state: follower="Node at 7d6494a7-da8f-641d-7ca2-d936fd579223 [Follower]" leader-address= leader-id= +2025-06-17T07:44:33.407Z [INFO] server-310b5aae-aad9-8db6-ac63-8ba49907461d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8bf5b205-ce5a-476c-c088-fabe7c534bc2 Address:8bf5b205-ce5a-476c-c088-fabe7c534bc2} {Suffrage:Voter ID:server-7d6494a7-da8f-641d-7ca2-d936fd579223 Address:7d6494a7-da8f-641d-7ca2-d936fd579223} {Suffrage:Voter ID:server-310b5aae-aad9-8db6-ac63-8ba49907461d Address:310b5aae-aad9-8db6-ac63-8ba49907461d}]" +2025-06-17T07:44:33.407Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:33.407Z [INFO] server-310b5aae-aad9-8db6-ac63-8ba49907461d: entering follower state: follower="Node at 310b5aae-aad9-8db6-ac63-8ba49907461d [Follower]" leader-address= leader-id= +2025-06-17T07:44:33.466Z [WARN] server-8bf5b205-ce5a-476c-c088-fabe7c534bc2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:33.466Z [WARN] server-310b5aae-aad9-8db6-ac63-8ba49907461d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:33.466Z [INFO] server-310b5aae-aad9-8db6-ac63-8ba49907461d: entering candidate state: node="Node at 310b5aae-aad9-8db6-ac63-8ba49907461d [Candidate]" term=2 +2025-06-17T07:44:33.466Z [INFO] server-8bf5b205-ce5a-476c-c088-fabe7c534bc2: entering candidate state: node="Node at 8bf5b205-ce5a-476c-c088-fabe7c534bc2 [Candidate]" term=2 +2025-06-17T07:44:33.466Z [INFO] server-310b5aae-aad9-8db6-ac63-8ba49907461d: election won: term=2 tally=2 +2025-06-17T07:44:33.466Z [INFO] server-310b5aae-aad9-8db6-ac63-8ba49907461d: entering leader state: leader="Node at 310b5aae-aad9-8db6-ac63-8ba49907461d [Leader]" +2025-06-17T07:44:33.466Z [INFO] server-310b5aae-aad9-8db6-ac63-8ba49907461d: added peer, starting replication: peer=server-8bf5b205-ce5a-476c-c088-fabe7c534bc2 +2025-06-17T07:44:33.466Z [INFO] server-310b5aae-aad9-8db6-ac63-8ba49907461d: added peer, starting replication: peer=server-7d6494a7-da8f-641d-7ca2-d936fd579223 +2025-06-17T07:44:33.466Z [INFO] server-8bf5b205-ce5a-476c-c088-fabe7c534bc2: duplicate requestVote for same term: term=2 +2025-06-17T07:44:33.466Z [INFO] server-7d6494a7-da8f-641d-7ca2-d936fd579223: duplicate requestVote for same term: term=2 +2025-06-17T07:44:33.466Z [INFO] server-8bf5b205-ce5a-476c-c088-fabe7c534bc2: entering follower state: follower="Node at 8bf5b205-ce5a-476c-c088-fabe7c534bc2 [Follower]" leader-address=310b5aae-aad9-8db6-ac63-8ba49907461d leader-id=server-310b5aae-aad9-8db6-ac63-8ba49907461d +2025-06-17T07:44:33.466Z [WARN] server-310b5aae-aad9-8db6-ac63-8ba49907461d: rejecting vote request since we have a leader: from=8bf5b205-ce5a-476c-c088-fabe7c534bc2 leader=310b5aae-aad9-8db6-ac63-8ba49907461d leader-id=server-310b5aae-aad9-8db6-ac63-8ba49907461d +2025-06-17T07:44:33.466Z [INFO] server-310b5aae-aad9-8db6-ac63-8ba49907461d: pipelining replication: peer="{Voter server-8bf5b205-ce5a-476c-c088-fabe7c534bc2 8bf5b205-ce5a-476c-c088-fabe7c534bc2}" +2025-06-17T07:44:33.466Z [INFO] server-310b5aae-aad9-8db6-ac63-8ba49907461d: pipelining replication: peer="{Voter server-7d6494a7-da8f-641d-7ca2-d936fd579223 7d6494a7-da8f-641d-7ca2-d936fd579223}" +2025-06-17T07:44:33.571Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:33.466256102 +0000 UTC m=+102.069784789 (2 nodes), 58.59908ms from start of poll, 58.886211ms from cluster start. Timeout at 2025-06-17 07:44:33.571715946 +0000 UTC m=+102.175244628, 105.459839ms after stability +2025-06-17T07:44:33.571Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:33.677Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:33.57177839 +0000 UTC m=+102.175307077 (1 nodes), 660ns from start of poll, 164.408499ms from cluster start. Timeout at 2025-06-17 07:44:33.67763909 +0000 UTC m=+102.281167770, 105.860693ms after stability +2025-06-17T07:44:33.677Z [INFO] server-310b5aae-aad9-8db6-ac63-8ba49907461d: aborting pipeline replication: peer="{Voter server-7d6494a7-da8f-641d-7ca2-d936fd579223 7d6494a7-da8f-641d-7ca2-d936fd579223}" +2025-06-17T07:44:33.677Z [INFO] server-310b5aae-aad9-8db6-ac63-8ba49907461d: aborting pipeline replication: peer="{Voter server-8bf5b205-ce5a-476c-c088-fabe7c534bc2 8bf5b205-ce5a-476c-c088-fabe7c534bc2}" +--- PASS: TestRaft_ProtocolVersion_RejectRPC (0.27s) === RUN TestRaft_ProtocolVersion_Upgrade_1_2 raft_test.go:2151: Skipping unreliable test --- SKIP: TestRaft_ProtocolVersion_Upgrade_1_2 (0.00s) @@ -3586,267 +3623,268 @@ raft_test.go:2196: Skipping unreliable test --- SKIP: TestRaft_ProtocolVersion_Upgrade_2_3 (0.00s) === RUN TestRaft_LeaderID_Propagated -2024-05-15T01:18:41.968Z [INFO] server-9b014381-cba1-5d57-dc00-c242f1388317: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b014381-cba1-5d57-dc00-c242f1388317 Address:9b014381-cba1-5d57-dc00-c242f1388317} {Suffrage:Voter ID:server-fdf73056-19c5-aa62-4785-19b2ffd9ab3e Address:fdf73056-19c5-aa62-4785-19b2ffd9ab3e} {Suffrage:Voter ID:server-5376fca1-d378-ead5-393c-0833e41b1d00 Address:5376fca1-d378-ead5-393c-0833e41b1d00}]" -2024-05-15T01:18:41.968Z [INFO] server-fdf73056-19c5-aa62-4785-19b2ffd9ab3e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b014381-cba1-5d57-dc00-c242f1388317 Address:9b014381-cba1-5d57-dc00-c242f1388317} {Suffrage:Voter ID:server-fdf73056-19c5-aa62-4785-19b2ffd9ab3e Address:fdf73056-19c5-aa62-4785-19b2ffd9ab3e} {Suffrage:Voter ID:server-5376fca1-d378-ead5-393c-0833e41b1d00 Address:5376fca1-d378-ead5-393c-0833e41b1d00}]" -2024-05-15T01:18:41.968Z [INFO] server-5376fca1-d378-ead5-393c-0833e41b1d00: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b014381-cba1-5d57-dc00-c242f1388317 Address:9b014381-cba1-5d57-dc00-c242f1388317} {Suffrage:Voter ID:server-fdf73056-19c5-aa62-4785-19b2ffd9ab3e Address:fdf73056-19c5-aa62-4785-19b2ffd9ab3e} {Suffrage:Voter ID:server-5376fca1-d378-ead5-393c-0833e41b1d00 Address:5376fca1-d378-ead5-393c-0833e41b1d00}]" -2024-05-15T01:18:41.968Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:41.968Z [INFO] server-9b014381-cba1-5d57-dc00-c242f1388317: entering follower state: follower="Node at 9b014381-cba1-5d57-dc00-c242f1388317 [Follower]" leader-address= leader-id= -2024-05-15T01:18:41.968Z [INFO] server-fdf73056-19c5-aa62-4785-19b2ffd9ab3e: entering follower state: follower="Node at fdf73056-19c5-aa62-4785-19b2ffd9ab3e [Follower]" leader-address= leader-id= -2024-05-15T01:18:41.968Z [INFO] server-5376fca1-d378-ead5-393c-0833e41b1d00: entering follower state: follower="Node at 5376fca1-d378-ead5-393c-0833e41b1d00 [Follower]" leader-address= leader-id= -2024-05-15T01:18:42.030Z [WARN] server-fdf73056-19c5-aa62-4785-19b2ffd9ab3e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:42.030Z [INFO] server-fdf73056-19c5-aa62-4785-19b2ffd9ab3e: entering candidate state: node="Node at fdf73056-19c5-aa62-4785-19b2ffd9ab3e [Candidate]" term=2 -2024-05-15T01:18:42.031Z [INFO] server-fdf73056-19c5-aa62-4785-19b2ffd9ab3e: election won: term=2 tally=2 -2024-05-15T01:18:42.031Z [INFO] server-fdf73056-19c5-aa62-4785-19b2ffd9ab3e: entering leader state: leader="Node at fdf73056-19c5-aa62-4785-19b2ffd9ab3e [Leader]" -2024-05-15T01:18:42.031Z [INFO] server-fdf73056-19c5-aa62-4785-19b2ffd9ab3e: added peer, starting replication: peer=server-9b014381-cba1-5d57-dc00-c242f1388317 -2024-05-15T01:18:42.031Z [INFO] server-fdf73056-19c5-aa62-4785-19b2ffd9ab3e: added peer, starting replication: peer=server-5376fca1-d378-ead5-393c-0833e41b1d00 -2024-05-15T01:18:42.031Z [INFO] server-fdf73056-19c5-aa62-4785-19b2ffd9ab3e: pipelining replication: peer="{Voter server-9b014381-cba1-5d57-dc00-c242f1388317 9b014381-cba1-5d57-dc00-c242f1388317}" -2024-05-15T01:18:42.031Z [INFO] server-fdf73056-19c5-aa62-4785-19b2ffd9ab3e: pipelining replication: peer="{Voter server-5376fca1-d378-ead5-393c-0833e41b1d00 5376fca1-d378-ead5-393c-0833e41b1d00}" -2024-05-15T01:18:42.136Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:42.03116045 +0000 UTC m=+102.572545207 (1 nodes), 62.33584ms from start of poll, 62.830501ms from cluster start. Timeout at 2024-05-15 01:18:42.136499245 +0000 UTC m=+102.677884137, 105.33893ms after stability -2024-05-15T01:18:42.136Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:42.241Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:42.136719369 +0000 UTC m=+102.678104112 (1 nodes), 1.69µs from start of poll, 168.389406ms from cluster start. Timeout at 2024-05-15 01:18:42.241791148 +0000 UTC m=+102.783175909, 105.071797ms after stability -2024-05-15T01:18:42.242Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:42.347Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:42.242345527 +0000 UTC m=+102.783730500 (1 nodes), 1.979µs from start of poll, 274.015794ms from cluster start. Timeout at 2024-05-15 01:18:42.34738609 +0000 UTC m=+102.888770328, 105.039828ms after stability -2024-05-15T01:18:42.347Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:42.453Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:42.347725635 +0000 UTC m=+102.889109873 (1 nodes), 0s from start of poll, 379.395167ms from cluster start. Timeout at 2024-05-15 01:18:42.453408691 +0000 UTC m=+102.994793772, 105.683899ms after stability -2024-05-15T01:18:42.453Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:42.560Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:42.45387502 +0000 UTC m=+102.995259258 (1 nodes), 0s from start of poll, 485.544552ms from cluster start. Timeout at 2024-05-15 01:18:42.560036879 +0000 UTC m=+103.101421725, 106.162467ms after stability -2024-05-15T01:18:42.560Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:42.666Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:42.560338541 +0000 UTC m=+103.101722779 (1 nodes), 0s from start of poll, 592.008073ms from cluster start. Timeout at 2024-05-15 01:18:42.6663017 +0000 UTC m=+103.207686701, 105.963922ms after stability -2024-05-15T01:18:42.771Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:42.883Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:42.77181696 +0000 UTC m=+103.313201198 (1 nodes), 7.858µs from start of poll, 803.486492ms from cluster start. Timeout at 2024-05-15 01:18:42.877191032 +0000 UTC m=+103.418575270, 105.374072ms after stability -2024-05-15T01:18:42.883Z [INFO] server-fdf73056-19c5-aa62-4785-19b2ffd9ab3e: aborting pipeline replication: peer="{Voter server-5376fca1-d378-ead5-393c-0833e41b1d00 5376fca1-d378-ead5-393c-0833e41b1d00}" -2024-05-15T01:18:42.883Z [INFO] server-fdf73056-19c5-aa62-4785-19b2ffd9ab3e: aborting pipeline replication: peer="{Voter server-9b014381-cba1-5d57-dc00-c242f1388317 9b014381-cba1-5d57-dc00-c242f1388317}" ---- PASS: TestRaft_LeaderID_Propagated (0.92s) +2025-06-17T07:44:33.678Z [INFO] server-b3c9af5d-859a-c42b-84ee-1979bbf3dc83: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b3c9af5d-859a-c42b-84ee-1979bbf3dc83 Address:b3c9af5d-859a-c42b-84ee-1979bbf3dc83} {Suffrage:Voter ID:server-8cc6cce2-66a0-3a9a-d70e-06da38d50063 Address:8cc6cce2-66a0-3a9a-d70e-06da38d50063} {Suffrage:Voter ID:server-5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec Address:5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec}]" +2025-06-17T07:44:33.678Z [INFO] server-b3c9af5d-859a-c42b-84ee-1979bbf3dc83: entering follower state: follower="Node at b3c9af5d-859a-c42b-84ee-1979bbf3dc83 [Follower]" leader-address= leader-id= +2025-06-17T07:44:33.679Z [INFO] server-8cc6cce2-66a0-3a9a-d70e-06da38d50063: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b3c9af5d-859a-c42b-84ee-1979bbf3dc83 Address:b3c9af5d-859a-c42b-84ee-1979bbf3dc83} {Suffrage:Voter ID:server-8cc6cce2-66a0-3a9a-d70e-06da38d50063 Address:8cc6cce2-66a0-3a9a-d70e-06da38d50063} {Suffrage:Voter ID:server-5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec Address:5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec}]" +2025-06-17T07:44:33.679Z [INFO] server-8cc6cce2-66a0-3a9a-d70e-06da38d50063: entering follower state: follower="Node at 8cc6cce2-66a0-3a9a-d70e-06da38d50063 [Follower]" leader-address= leader-id= +2025-06-17T07:44:33.679Z [INFO] server-5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b3c9af5d-859a-c42b-84ee-1979bbf3dc83 Address:b3c9af5d-859a-c42b-84ee-1979bbf3dc83} {Suffrage:Voter ID:server-8cc6cce2-66a0-3a9a-d70e-06da38d50063 Address:8cc6cce2-66a0-3a9a-d70e-06da38d50063} {Suffrage:Voter ID:server-5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec Address:5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec}]" +2025-06-17T07:44:33.679Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:33.679Z [INFO] server-5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec: entering follower state: follower="Node at 5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec [Follower]" leader-address= leader-id= +2025-06-17T07:44:33.734Z [WARN] server-5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:33.734Z [INFO] server-5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec: entering candidate state: node="Node at 5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec [Candidate]" term=2 +2025-06-17T07:44:33.735Z [INFO] server-5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec: election won: term=2 tally=2 +2025-06-17T07:44:33.735Z [INFO] server-5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec: entering leader state: leader="Node at 5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec [Leader]" +2025-06-17T07:44:33.735Z [INFO] server-5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec: added peer, starting replication: peer=server-b3c9af5d-859a-c42b-84ee-1979bbf3dc83 +2025-06-17T07:44:33.735Z [INFO] server-5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec: added peer, starting replication: peer=server-8cc6cce2-66a0-3a9a-d70e-06da38d50063 +2025-06-17T07:44:33.735Z [INFO] server-5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec: pipelining replication: peer="{Voter server-8cc6cce2-66a0-3a9a-d70e-06da38d50063 8cc6cce2-66a0-3a9a-d70e-06da38d50063}" +2025-06-17T07:44:33.735Z [INFO] server-5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec: pipelining replication: peer="{Voter server-b3c9af5d-859a-c42b-84ee-1979bbf3dc83 b3c9af5d-859a-c42b-84ee-1979bbf3dc83}" +2025-06-17T07:44:33.840Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:33.735091863 +0000 UTC m=+102.338620570 (1 nodes), 55.958219ms from start of poll, 56.351012ms from cluster start. Timeout at 2025-06-17 07:44:33.840917997 +0000 UTC m=+102.444446677, 105.826107ms after stability +2025-06-17T07:44:33.841Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:33.946Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:33.841017525 +0000 UTC m=+102.444546238 (1 nodes), 698ns from start of poll, 162.27668ms from cluster start. Timeout at 2025-06-17 07:44:33.946199855 +0000 UTC m=+102.549728555, 105.182317ms after stability +2025-06-17T07:44:33.946Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:34.051Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:33.94641412 +0000 UTC m=+102.549942817 (1 nodes), 831ns from start of poll, 267.673259ms from cluster start. Timeout at 2025-06-17 07:44:34.051949742 +0000 UTC m=+102.655478424, 105.535607ms after stability +2025-06-17T07:44:34.052Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:34.157Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:34.052141241 +0000 UTC m=+102.655669926 (1 nodes), 774ns from start of poll, 373.400368ms from cluster start. Timeout at 2025-06-17 07:44:34.157835601 +0000 UTC m=+102.761364286, 105.69436ms after stability +2025-06-17T07:44:34.158Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:34.263Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:34.158023826 +0000 UTC m=+102.761552514 (1 nodes), 769ns from start of poll, 479.282956ms from cluster start. Timeout at 2025-06-17 07:44:34.263411053 +0000 UTC m=+102.866939743, 105.387229ms after stability +2025-06-17T07:44:34.263Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:34.368Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:34.263608504 +0000 UTC m=+102.867137229 (1 nodes), 2.003µs from start of poll, 584.867671ms from cluster start. Timeout at 2025-06-17 07:44:34.368730582 +0000 UTC m=+102.972259265, 105.122036ms after stability +2025-06-17T07:44:34.474Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:34.579Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:34.474155215 +0000 UTC m=+103.077683919 (1 nodes), 920ns from start of poll, 795.414361ms from cluster start. Timeout at 2025-06-17 07:44:34.579681629 +0000 UTC m=+103.183210321, 105.526402ms after stability +2025-06-17T07:44:34.579Z [INFO] server-5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec: aborting pipeline replication: peer="{Voter server-b3c9af5d-859a-c42b-84ee-1979bbf3dc83 b3c9af5d-859a-c42b-84ee-1979bbf3dc83}" +2025-06-17T07:44:34.579Z [INFO] server-5f21c8f5-84a4-7083-ecf2-c0ab4c6939ec: aborting pipeline replication: peer="{Voter server-8cc6cce2-66a0-3a9a-d70e-06da38d50063 8cc6cce2-66a0-3a9a-d70e-06da38d50063}" +--- PASS: TestRaft_LeaderID_Propagated (0.93s) === RUN TestRaft_LeadershipTransferInProgress --- PASS: TestRaft_LeadershipTransferInProgress (0.00s) === RUN TestRaft_LeadershipTransferPickServer --- PASS: TestRaft_LeadershipTransferPickServer (0.00s) === RUN TestRaft_LeadershipTransfer -2024-05-15T01:18:42.885Z [INFO] server-047112f4-e110-22ee-23f6-7f369252427b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-047112f4-e110-22ee-23f6-7f369252427b Address:047112f4-e110-22ee-23f6-7f369252427b} {Suffrage:Voter ID:server-901da87d-8918-a6e6-50ac-9e22a535b411 Address:901da87d-8918-a6e6-50ac-9e22a535b411} {Suffrage:Voter ID:server-f3ad97af-854c-5094-8d69-006dc051e630 Address:f3ad97af-854c-5094-8d69-006dc051e630}]" -2024-05-15T01:18:42.885Z [INFO] server-901da87d-8918-a6e6-50ac-9e22a535b411: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-047112f4-e110-22ee-23f6-7f369252427b Address:047112f4-e110-22ee-23f6-7f369252427b} {Suffrage:Voter ID:server-901da87d-8918-a6e6-50ac-9e22a535b411 Address:901da87d-8918-a6e6-50ac-9e22a535b411} {Suffrage:Voter ID:server-f3ad97af-854c-5094-8d69-006dc051e630 Address:f3ad97af-854c-5094-8d69-006dc051e630}]" -2024-05-15T01:18:42.885Z [INFO] server-f3ad97af-854c-5094-8d69-006dc051e630: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-047112f4-e110-22ee-23f6-7f369252427b Address:047112f4-e110-22ee-23f6-7f369252427b} {Suffrage:Voter ID:server-901da87d-8918-a6e6-50ac-9e22a535b411 Address:901da87d-8918-a6e6-50ac-9e22a535b411} {Suffrage:Voter ID:server-f3ad97af-854c-5094-8d69-006dc051e630 Address:f3ad97af-854c-5094-8d69-006dc051e630}]" -2024-05-15T01:18:42.885Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:42.886Z [INFO] server-047112f4-e110-22ee-23f6-7f369252427b: entering follower state: follower="Node at 047112f4-e110-22ee-23f6-7f369252427b [Follower]" leader-address= leader-id= -2024-05-15T01:18:42.886Z [INFO] server-901da87d-8918-a6e6-50ac-9e22a535b411: entering follower state: follower="Node at 901da87d-8918-a6e6-50ac-9e22a535b411 [Follower]" leader-address= leader-id= -2024-05-15T01:18:42.886Z [INFO] server-f3ad97af-854c-5094-8d69-006dc051e630: entering follower state: follower="Node at f3ad97af-854c-5094-8d69-006dc051e630 [Follower]" leader-address= leader-id= -2024-05-15T01:18:42.960Z [WARN] server-047112f4-e110-22ee-23f6-7f369252427b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:42.960Z [INFO] server-047112f4-e110-22ee-23f6-7f369252427b: entering candidate state: node="Node at 047112f4-e110-22ee-23f6-7f369252427b [Candidate]" term=2 -2024-05-15T01:18:42.960Z [INFO] server-047112f4-e110-22ee-23f6-7f369252427b: election won: term=2 tally=2 -2024-05-15T01:18:42.960Z [INFO] server-047112f4-e110-22ee-23f6-7f369252427b: entering leader state: leader="Node at 047112f4-e110-22ee-23f6-7f369252427b [Leader]" -2024-05-15T01:18:42.960Z [INFO] server-047112f4-e110-22ee-23f6-7f369252427b: added peer, starting replication: peer=server-901da87d-8918-a6e6-50ac-9e22a535b411 -2024-05-15T01:18:42.960Z [INFO] server-047112f4-e110-22ee-23f6-7f369252427b: added peer, starting replication: peer=server-f3ad97af-854c-5094-8d69-006dc051e630 -2024-05-15T01:18:42.961Z [INFO] server-047112f4-e110-22ee-23f6-7f369252427b: pipelining replication: peer="{Voter server-f3ad97af-854c-5094-8d69-006dc051e630 f3ad97af-854c-5094-8d69-006dc051e630}" -2024-05-15T01:18:42.961Z [INFO] server-047112f4-e110-22ee-23f6-7f369252427b: pipelining replication: peer="{Voter server-901da87d-8918-a6e6-50ac-9e22a535b411 901da87d-8918-a6e6-50ac-9e22a535b411}" -2024-05-15T01:18:43.073Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:42.961587289 +0000 UTC m=+103.502972149 (1 nodes), 75.594454ms from start of poll, 76.23761ms from cluster start. Timeout at 2024-05-15 01:18:43.070041756 +0000 UTC m=+103.611425994, 108.453845ms after stability -2024-05-15T01:18:43.073Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:43.179Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:43.073980986 +0000 UTC m=+103.615366110 (1 nodes), 2.599µs from start of poll, 188.631571ms from cluster start. Timeout at 2024-05-15 01:18:43.179388108 +0000 UTC m=+103.720772859, 105.406749ms after stability -2024-05-15T01:18:43.179Z [INFO] server-901da87d-8918-a6e6-50ac-9e22a535b411: entering candidate state: node="Node at 901da87d-8918-a6e6-50ac-9e22a535b411 [Candidate]" term=3 -2024-05-15T01:18:43.179Z [INFO] server-901da87d-8918-a6e6-50ac-9e22a535b411: election won: term=3 tally=2 -2024-05-15T01:18:43.179Z [INFO] server-901da87d-8918-a6e6-50ac-9e22a535b411: entering leader state: leader="Node at 901da87d-8918-a6e6-50ac-9e22a535b411 [Leader]" -2024-05-15T01:18:43.179Z [INFO] server-901da87d-8918-a6e6-50ac-9e22a535b411: added peer, starting replication: peer=server-047112f4-e110-22ee-23f6-7f369252427b -2024-05-15T01:18:43.179Z [INFO] server-901da87d-8918-a6e6-50ac-9e22a535b411: added peer, starting replication: peer=server-f3ad97af-854c-5094-8d69-006dc051e630 -2024-05-15T01:18:43.179Z [INFO] server-901da87d-8918-a6e6-50ac-9e22a535b411: pipelining replication: peer="{Voter server-f3ad97af-854c-5094-8d69-006dc051e630 f3ad97af-854c-5094-8d69-006dc051e630}" -2024-05-15T01:18:43.179Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:43.180Z [INFO] server-047112f4-e110-22ee-23f6-7f369252427b: entering follower state: follower="Node at 047112f4-e110-22ee-23f6-7f369252427b [Follower]" leader-address= leader-id= -2024-05-15T01:18:43.180Z [INFO] server-901da87d-8918-a6e6-50ac-9e22a535b411: pipelining replication: peer="{Voter server-047112f4-e110-22ee-23f6-7f369252427b 047112f4-e110-22ee-23f6-7f369252427b}" -2024-05-15T01:18:43.180Z [INFO] server-047112f4-e110-22ee-23f6-7f369252427b: aborting pipeline replication: peer="{Voter server-901da87d-8918-a6e6-50ac-9e22a535b411 901da87d-8918-a6e6-50ac-9e22a535b411}" -2024-05-15T01:18:43.180Z [INFO] server-047112f4-e110-22ee-23f6-7f369252427b: aborting pipeline replication: peer="{Voter server-f3ad97af-854c-5094-8d69-006dc051e630 f3ad97af-854c-5094-8d69-006dc051e630}" -2024-05-15T01:18:43.285Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:43.18032975 +0000 UTC m=+103.721714484 (1 nodes), 340.882µs from start of poll, 294.979945ms from cluster start. Timeout at 2024-05-15 01:18:43.285876614 +0000 UTC m=+103.827261441, 105.546957ms after stability -2024-05-15T01:18:43.286Z [INFO] server-901da87d-8918-a6e6-50ac-9e22a535b411: aborting pipeline replication: peer="{Voter server-f3ad97af-854c-5094-8d69-006dc051e630 f3ad97af-854c-5094-8d69-006dc051e630}" -2024-05-15T01:18:43.286Z [INFO] server-901da87d-8918-a6e6-50ac-9e22a535b411: aborting pipeline replication: peer="{Voter server-047112f4-e110-22ee-23f6-7f369252427b 047112f4-e110-22ee-23f6-7f369252427b}" +2025-06-17T07:44:34.605Z [INFO] server-de02bfba-af4b-3546-076c-97cf044e1634: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-de02bfba-af4b-3546-076c-97cf044e1634 Address:de02bfba-af4b-3546-076c-97cf044e1634} {Suffrage:Voter ID:server-763ec60f-86c9-7b23-4496-fef8394d4472 Address:763ec60f-86c9-7b23-4496-fef8394d4472} {Suffrage:Voter ID:server-31366042-df2c-b312-886b-92a73697ebb7 Address:31366042-df2c-b312-886b-92a73697ebb7}]" +2025-06-17T07:44:34.605Z [INFO] server-de02bfba-af4b-3546-076c-97cf044e1634: entering follower state: follower="Node at de02bfba-af4b-3546-076c-97cf044e1634 [Follower]" leader-address= leader-id= +2025-06-17T07:44:34.606Z [INFO] server-763ec60f-86c9-7b23-4496-fef8394d4472: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-de02bfba-af4b-3546-076c-97cf044e1634 Address:de02bfba-af4b-3546-076c-97cf044e1634} {Suffrage:Voter ID:server-763ec60f-86c9-7b23-4496-fef8394d4472 Address:763ec60f-86c9-7b23-4496-fef8394d4472} {Suffrage:Voter ID:server-31366042-df2c-b312-886b-92a73697ebb7 Address:31366042-df2c-b312-886b-92a73697ebb7}]" +2025-06-17T07:44:34.606Z [INFO] server-763ec60f-86c9-7b23-4496-fef8394d4472: entering follower state: follower="Node at 763ec60f-86c9-7b23-4496-fef8394d4472 [Follower]" leader-address= leader-id= +2025-06-17T07:44:34.606Z [INFO] server-31366042-df2c-b312-886b-92a73697ebb7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-de02bfba-af4b-3546-076c-97cf044e1634 Address:de02bfba-af4b-3546-076c-97cf044e1634} {Suffrage:Voter ID:server-763ec60f-86c9-7b23-4496-fef8394d4472 Address:763ec60f-86c9-7b23-4496-fef8394d4472} {Suffrage:Voter ID:server-31366042-df2c-b312-886b-92a73697ebb7 Address:31366042-df2c-b312-886b-92a73697ebb7}]" +2025-06-17T07:44:34.606Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:34.606Z [INFO] server-31366042-df2c-b312-886b-92a73697ebb7: entering follower state: follower="Node at 31366042-df2c-b312-886b-92a73697ebb7 [Follower]" leader-address= leader-id= +2025-06-17T07:44:34.691Z [WARN] server-31366042-df2c-b312-886b-92a73697ebb7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:34.691Z [INFO] server-31366042-df2c-b312-886b-92a73697ebb7: entering candidate state: node="Node at 31366042-df2c-b312-886b-92a73697ebb7 [Candidate]" term=2 +2025-06-17T07:44:34.691Z [INFO] server-31366042-df2c-b312-886b-92a73697ebb7: election won: term=2 tally=2 +2025-06-17T07:44:34.691Z [INFO] server-31366042-df2c-b312-886b-92a73697ebb7: entering leader state: leader="Node at 31366042-df2c-b312-886b-92a73697ebb7 [Leader]" +2025-06-17T07:44:34.691Z [INFO] server-31366042-df2c-b312-886b-92a73697ebb7: added peer, starting replication: peer=server-de02bfba-af4b-3546-076c-97cf044e1634 +2025-06-17T07:44:34.691Z [INFO] server-31366042-df2c-b312-886b-92a73697ebb7: added peer, starting replication: peer=server-763ec60f-86c9-7b23-4496-fef8394d4472 +2025-06-17T07:44:34.691Z [INFO] server-31366042-df2c-b312-886b-92a73697ebb7: pipelining replication: peer="{Voter server-de02bfba-af4b-3546-076c-97cf044e1634 de02bfba-af4b-3546-076c-97cf044e1634}" +2025-06-17T07:44:34.691Z [INFO] server-31366042-df2c-b312-886b-92a73697ebb7: pipelining replication: peer="{Voter server-763ec60f-86c9-7b23-4496-fef8394d4472 763ec60f-86c9-7b23-4496-fef8394d4472}" +2025-06-17T07:44:34.797Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:34.69155208 +0000 UTC m=+103.295080778 (1 nodes), 85.420847ms from start of poll, 85.724258ms from cluster start. Timeout at 2025-06-17 07:44:34.797428876 +0000 UTC m=+103.400957559, 105.876781ms after stability +2025-06-17T07:44:34.797Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:34.903Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:34.79749004 +0000 UTC m=+103.401018730 (1 nodes), 670ns from start of poll, 191.66221ms from cluster start. Timeout at 2025-06-17 07:44:34.903539366 +0000 UTC m=+103.507068055, 106.049325ms after stability +2025-06-17T07:44:34.903Z [INFO] server-de02bfba-af4b-3546-076c-97cf044e1634: entering candidate state: node="Node at de02bfba-af4b-3546-076c-97cf044e1634 [Candidate]" term=3 +2025-06-17T07:44:34.903Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:34.903Z [INFO] server-31366042-df2c-b312-886b-92a73697ebb7: entering follower state: follower="Node at 31366042-df2c-b312-886b-92a73697ebb7 [Follower]" leader-address= leader-id= +2025-06-17T07:44:34.903Z [INFO] server-de02bfba-af4b-3546-076c-97cf044e1634: election won: term=3 tally=2 +2025-06-17T07:44:34.903Z [INFO] server-31366042-df2c-b312-886b-92a73697ebb7: aborting pipeline replication: peer="{Voter server-de02bfba-af4b-3546-076c-97cf044e1634 de02bfba-af4b-3546-076c-97cf044e1634}" +2025-06-17T07:44:34.903Z [INFO] server-de02bfba-af4b-3546-076c-97cf044e1634: entering leader state: leader="Node at de02bfba-af4b-3546-076c-97cf044e1634 [Leader]" +2025-06-17T07:44:34.903Z [INFO] server-31366042-df2c-b312-886b-92a73697ebb7: aborting pipeline replication: peer="{Voter server-763ec60f-86c9-7b23-4496-fef8394d4472 763ec60f-86c9-7b23-4496-fef8394d4472}" +2025-06-17T07:44:34.903Z [INFO] server-de02bfba-af4b-3546-076c-97cf044e1634: added peer, starting replication: peer=server-763ec60f-86c9-7b23-4496-fef8394d4472 +2025-06-17T07:44:34.903Z [INFO] server-de02bfba-af4b-3546-076c-97cf044e1634: added peer, starting replication: peer=server-31366042-df2c-b312-886b-92a73697ebb7 +2025-06-17T07:44:34.903Z [INFO] server-de02bfba-af4b-3546-076c-97cf044e1634: pipelining replication: peer="{Voter server-31366042-df2c-b312-886b-92a73697ebb7 31366042-df2c-b312-886b-92a73697ebb7}" +2025-06-17T07:44:34.904Z [INFO] server-de02bfba-af4b-3546-076c-97cf044e1634: pipelining replication: peer="{Voter server-763ec60f-86c9-7b23-4496-fef8394d4472 763ec60f-86c9-7b23-4496-fef8394d4472}" +2025-06-17T07:44:35.009Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:34.903841248 +0000 UTC m=+103.507369963 (1 nodes), 87.384µs from start of poll, 298.013443ms from cluster start. Timeout at 2025-06-17 07:44:35.00947815 +0000 UTC m=+103.613006843, 105.63688ms after stability +2025-06-17T07:44:35.009Z [INFO] server-de02bfba-af4b-3546-076c-97cf044e1634: aborting pipeline replication: peer="{Voter server-31366042-df2c-b312-886b-92a73697ebb7 31366042-df2c-b312-886b-92a73697ebb7}" +2025-06-17T07:44:35.009Z [INFO] server-de02bfba-af4b-3546-076c-97cf044e1634: aborting pipeline replication: peer="{Voter server-763ec60f-86c9-7b23-4496-fef8394d4472 763ec60f-86c9-7b23-4496-fef8394d4472}" --- PASS: TestRaft_LeadershipTransfer (0.40s) === RUN TestRaft_LeadershipTransferWithOneNode -2024-05-15T01:18:43.287Z [INFO] server-f4033239-bff7-ec56-e1cc-e931ad67cca8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f4033239-bff7-ec56-e1cc-e931ad67cca8 Address:f4033239-bff7-ec56-e1cc-e931ad67cca8}]" -2024-05-15T01:18:43.287Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:43.287Z [INFO] server-f4033239-bff7-ec56-e1cc-e931ad67cca8: entering follower state: follower="Node at f4033239-bff7-ec56-e1cc-e931ad67cca8 [Follower]" leader-address= leader-id= -2024-05-15T01:18:43.344Z [WARN] server-f4033239-bff7-ec56-e1cc-e931ad67cca8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:43.344Z [INFO] server-f4033239-bff7-ec56-e1cc-e931ad67cca8: entering candidate state: node="Node at f4033239-bff7-ec56-e1cc-e931ad67cca8 [Candidate]" term=2 -2024-05-15T01:18:43.344Z [INFO] server-f4033239-bff7-ec56-e1cc-e931ad67cca8: election won: term=2 tally=1 -2024-05-15T01:18:43.344Z [INFO] server-f4033239-bff7-ec56-e1cc-e931ad67cca8: entering leader state: leader="Node at f4033239-bff7-ec56-e1cc-e931ad67cca8 [Leader]" -2024-05-15T01:18:43.450Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:43.345211091 +0000 UTC m=+103.886595329 (1 nodes), 57.749248ms from start of poll, 57.928688ms from cluster start. Timeout at 2024-05-15 01:18:43.45023342 +0000 UTC m=+103.991618173, 105.022844ms after stability ---- PASS: TestRaft_LeadershipTransferWithOneNode (0.16s) +2025-06-17T07:44:35.010Z [INFO] server-6c868ab9-4ffb-ba11-f198-4d269b3133a2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6c868ab9-4ffb-ba11-f198-4d269b3133a2 Address:6c868ab9-4ffb-ba11-f198-4d269b3133a2}]" +2025-06-17T07:44:35.010Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:35.010Z [INFO] server-6c868ab9-4ffb-ba11-f198-4d269b3133a2: entering follower state: follower="Node at 6c868ab9-4ffb-ba11-f198-4d269b3133a2 [Follower]" leader-address= leader-id= +2025-06-17T07:44:35.093Z [WARN] server-6c868ab9-4ffb-ba11-f198-4d269b3133a2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:35.093Z [INFO] server-6c868ab9-4ffb-ba11-f198-4d269b3133a2: entering candidate state: node="Node at 6c868ab9-4ffb-ba11-f198-4d269b3133a2 [Candidate]" term=2 +2025-06-17T07:44:35.093Z [INFO] server-6c868ab9-4ffb-ba11-f198-4d269b3133a2: election won: term=2 tally=1 +2025-06-17T07:44:35.093Z [INFO] server-6c868ab9-4ffb-ba11-f198-4d269b3133a2: entering leader state: leader="Node at 6c868ab9-4ffb-ba11-f198-4d269b3133a2 [Leader]" +2025-06-17T07:44:35.199Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:35.09331511 +0000 UTC m=+103.696843794 (1 nodes), 82.933273ms from start of poll, 83.061882ms from cluster start. Timeout at 2025-06-17 07:44:35.199037708 +0000 UTC m=+103.802566394, 105.7226ms after stability +--- PASS: TestRaft_LeadershipTransferWithOneNode (0.19s) === RUN TestRaft_LeadershipTransferWithSevenNodes -2024-05-15T01:18:43.452Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55ed4422-338f-1f8b-5715-d47cdfcf7a30 Address:55ed4422-338f-1f8b-5715-d47cdfcf7a30} {Suffrage:Voter ID:server-23e5cf23-7fb5-622a-1506-511ab3dd20bc Address:23e5cf23-7fb5-622a-1506-511ab3dd20bc} {Suffrage:Voter ID:server-4bdd63d2-d807-3627-bb88-04ea222cc19e Address:4bdd63d2-d807-3627-bb88-04ea222cc19e} {Suffrage:Voter ID:server-6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7 Address:6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7} {Suffrage:Voter ID:server-8bb00598-0887-985a-5146-ad2226e75116 Address:8bb00598-0887-985a-5146-ad2226e75116} {Suffrage:Voter ID:server-61b0bc0d-7f42-6028-ace9-4d5d0a521262 Address:61b0bc0d-7f42-6028-ace9-4d5d0a521262} {Suffrage:Voter ID:server-7fc7ace5-9311-a6a1-18bf-7b32693aa05b Address:7fc7ace5-9311-a6a1-18bf-7b32693aa05b}]" -2024-05-15T01:18:43.452Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55ed4422-338f-1f8b-5715-d47cdfcf7a30 Address:55ed4422-338f-1f8b-5715-d47cdfcf7a30} {Suffrage:Voter ID:server-23e5cf23-7fb5-622a-1506-511ab3dd20bc Address:23e5cf23-7fb5-622a-1506-511ab3dd20bc} {Suffrage:Voter ID:server-4bdd63d2-d807-3627-bb88-04ea222cc19e Address:4bdd63d2-d807-3627-bb88-04ea222cc19e} {Suffrage:Voter ID:server-6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7 Address:6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7} {Suffrage:Voter ID:server-8bb00598-0887-985a-5146-ad2226e75116 Address:8bb00598-0887-985a-5146-ad2226e75116} {Suffrage:Voter ID:server-61b0bc0d-7f42-6028-ace9-4d5d0a521262 Address:61b0bc0d-7f42-6028-ace9-4d5d0a521262} {Suffrage:Voter ID:server-7fc7ace5-9311-a6a1-18bf-7b32693aa05b Address:7fc7ace5-9311-a6a1-18bf-7b32693aa05b}]" -2024-05-15T01:18:43.452Z [INFO] server-4bdd63d2-d807-3627-bb88-04ea222cc19e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55ed4422-338f-1f8b-5715-d47cdfcf7a30 Address:55ed4422-338f-1f8b-5715-d47cdfcf7a30} {Suffrage:Voter ID:server-23e5cf23-7fb5-622a-1506-511ab3dd20bc Address:23e5cf23-7fb5-622a-1506-511ab3dd20bc} {Suffrage:Voter ID:server-4bdd63d2-d807-3627-bb88-04ea222cc19e Address:4bdd63d2-d807-3627-bb88-04ea222cc19e} {Suffrage:Voter ID:server-6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7 Address:6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7} {Suffrage:Voter ID:server-8bb00598-0887-985a-5146-ad2226e75116 Address:8bb00598-0887-985a-5146-ad2226e75116} {Suffrage:Voter ID:server-61b0bc0d-7f42-6028-ace9-4d5d0a521262 Address:61b0bc0d-7f42-6028-ace9-4d5d0a521262} {Suffrage:Voter ID:server-7fc7ace5-9311-a6a1-18bf-7b32693aa05b Address:7fc7ace5-9311-a6a1-18bf-7b32693aa05b}]" -2024-05-15T01:18:43.452Z [INFO] server-6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55ed4422-338f-1f8b-5715-d47cdfcf7a30 Address:55ed4422-338f-1f8b-5715-d47cdfcf7a30} {Suffrage:Voter ID:server-23e5cf23-7fb5-622a-1506-511ab3dd20bc Address:23e5cf23-7fb5-622a-1506-511ab3dd20bc} {Suffrage:Voter ID:server-4bdd63d2-d807-3627-bb88-04ea222cc19e Address:4bdd63d2-d807-3627-bb88-04ea222cc19e} {Suffrage:Voter ID:server-6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7 Address:6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7} {Suffrage:Voter ID:server-8bb00598-0887-985a-5146-ad2226e75116 Address:8bb00598-0887-985a-5146-ad2226e75116} {Suffrage:Voter ID:server-61b0bc0d-7f42-6028-ace9-4d5d0a521262 Address:61b0bc0d-7f42-6028-ace9-4d5d0a521262} {Suffrage:Voter ID:server-7fc7ace5-9311-a6a1-18bf-7b32693aa05b Address:7fc7ace5-9311-a6a1-18bf-7b32693aa05b}]" -2024-05-15T01:18:43.453Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: entering follower state: follower="Node at 55ed4422-338f-1f8b-5715-d47cdfcf7a30 [Follower]" leader-address= leader-id= -2024-05-15T01:18:43.453Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: entering follower state: follower="Node at 23e5cf23-7fb5-622a-1506-511ab3dd20bc [Follower]" leader-address= leader-id= -2024-05-15T01:18:43.453Z [INFO] server-4bdd63d2-d807-3627-bb88-04ea222cc19e: entering follower state: follower="Node at 4bdd63d2-d807-3627-bb88-04ea222cc19e [Follower]" leader-address= leader-id= -2024-05-15T01:18:43.453Z [INFO] server-6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7: entering follower state: follower="Node at 6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7 [Follower]" leader-address= leader-id= -2024-05-15T01:18:43.454Z [INFO] server-8bb00598-0887-985a-5146-ad2226e75116: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55ed4422-338f-1f8b-5715-d47cdfcf7a30 Address:55ed4422-338f-1f8b-5715-d47cdfcf7a30} {Suffrage:Voter ID:server-23e5cf23-7fb5-622a-1506-511ab3dd20bc Address:23e5cf23-7fb5-622a-1506-511ab3dd20bc} {Suffrage:Voter ID:server-4bdd63d2-d807-3627-bb88-04ea222cc19e Address:4bdd63d2-d807-3627-bb88-04ea222cc19e} {Suffrage:Voter ID:server-6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7 Address:6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7} {Suffrage:Voter ID:server-8bb00598-0887-985a-5146-ad2226e75116 Address:8bb00598-0887-985a-5146-ad2226e75116} {Suffrage:Voter ID:server-61b0bc0d-7f42-6028-ace9-4d5d0a521262 Address:61b0bc0d-7f42-6028-ace9-4d5d0a521262} {Suffrage:Voter ID:server-7fc7ace5-9311-a6a1-18bf-7b32693aa05b Address:7fc7ace5-9311-a6a1-18bf-7b32693aa05b}]" -2024-05-15T01:18:43.455Z [INFO] server-61b0bc0d-7f42-6028-ace9-4d5d0a521262: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55ed4422-338f-1f8b-5715-d47cdfcf7a30 Address:55ed4422-338f-1f8b-5715-d47cdfcf7a30} {Suffrage:Voter ID:server-23e5cf23-7fb5-622a-1506-511ab3dd20bc Address:23e5cf23-7fb5-622a-1506-511ab3dd20bc} {Suffrage:Voter ID:server-4bdd63d2-d807-3627-bb88-04ea222cc19e Address:4bdd63d2-d807-3627-bb88-04ea222cc19e} {Suffrage:Voter ID:server-6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7 Address:6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7} {Suffrage:Voter ID:server-8bb00598-0887-985a-5146-ad2226e75116 Address:8bb00598-0887-985a-5146-ad2226e75116} {Suffrage:Voter ID:server-61b0bc0d-7f42-6028-ace9-4d5d0a521262 Address:61b0bc0d-7f42-6028-ace9-4d5d0a521262} {Suffrage:Voter ID:server-7fc7ace5-9311-a6a1-18bf-7b32693aa05b Address:7fc7ace5-9311-a6a1-18bf-7b32693aa05b}]" -2024-05-15T01:18:43.455Z [INFO] server-7fc7ace5-9311-a6a1-18bf-7b32693aa05b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55ed4422-338f-1f8b-5715-d47cdfcf7a30 Address:55ed4422-338f-1f8b-5715-d47cdfcf7a30} {Suffrage:Voter ID:server-23e5cf23-7fb5-622a-1506-511ab3dd20bc Address:23e5cf23-7fb5-622a-1506-511ab3dd20bc} {Suffrage:Voter ID:server-4bdd63d2-d807-3627-bb88-04ea222cc19e Address:4bdd63d2-d807-3627-bb88-04ea222cc19e} {Suffrage:Voter ID:server-6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7 Address:6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7} {Suffrage:Voter ID:server-8bb00598-0887-985a-5146-ad2226e75116 Address:8bb00598-0887-985a-5146-ad2226e75116} {Suffrage:Voter ID:server-61b0bc0d-7f42-6028-ace9-4d5d0a521262 Address:61b0bc0d-7f42-6028-ace9-4d5d0a521262} {Suffrage:Voter ID:server-7fc7ace5-9311-a6a1-18bf-7b32693aa05b Address:7fc7ace5-9311-a6a1-18bf-7b32693aa05b}]" -2024-05-15T01:18:43.455Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:43.455Z [INFO] server-8bb00598-0887-985a-5146-ad2226e75116: entering follower state: follower="Node at 8bb00598-0887-985a-5146-ad2226e75116 [Follower]" leader-address= leader-id= -2024-05-15T01:18:43.455Z [INFO] server-61b0bc0d-7f42-6028-ace9-4d5d0a521262: entering follower state: follower="Node at 61b0bc0d-7f42-6028-ace9-4d5d0a521262 [Follower]" leader-address= leader-id= -2024-05-15T01:18:43.455Z [INFO] server-7fc7ace5-9311-a6a1-18bf-7b32693aa05b: entering follower state: follower="Node at 7fc7ace5-9311-a6a1-18bf-7b32693aa05b [Follower]" leader-address= leader-id= -2024-05-15T01:18:43.516Z [WARN] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:43.516Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: entering candidate state: node="Node at 55ed4422-338f-1f8b-5715-d47cdfcf7a30 [Candidate]" term=2 -2024-05-15T01:18:43.516Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: election won: term=2 tally=4 -2024-05-15T01:18:43.516Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: entering leader state: leader="Node at 55ed4422-338f-1f8b-5715-d47cdfcf7a30 [Leader]" -2024-05-15T01:18:43.517Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: added peer, starting replication: peer=server-23e5cf23-7fb5-622a-1506-511ab3dd20bc -2024-05-15T01:18:43.517Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: added peer, starting replication: peer=server-4bdd63d2-d807-3627-bb88-04ea222cc19e -2024-05-15T01:18:43.517Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: added peer, starting replication: peer=server-6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7 -2024-05-15T01:18:43.517Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: added peer, starting replication: peer=server-8bb00598-0887-985a-5146-ad2226e75116 -2024-05-15T01:18:43.517Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: added peer, starting replication: peer=server-61b0bc0d-7f42-6028-ace9-4d5d0a521262 -2024-05-15T01:18:43.517Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: added peer, starting replication: peer=server-7fc7ace5-9311-a6a1-18bf-7b32693aa05b -2024-05-15T01:18:43.517Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: pipelining replication: peer="{Voter server-23e5cf23-7fb5-622a-1506-511ab3dd20bc 23e5cf23-7fb5-622a-1506-511ab3dd20bc}" -2024-05-15T01:18:43.517Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: pipelining replication: peer="{Voter server-4bdd63d2-d807-3627-bb88-04ea222cc19e 4bdd63d2-d807-3627-bb88-04ea222cc19e}" -2024-05-15T01:18:43.517Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: pipelining replication: peer="{Voter server-8bb00598-0887-985a-5146-ad2226e75116 8bb00598-0887-985a-5146-ad2226e75116}" -2024-05-15T01:18:43.517Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: pipelining replication: peer="{Voter server-7fc7ace5-9311-a6a1-18bf-7b32693aa05b 7fc7ace5-9311-a6a1-18bf-7b32693aa05b}" -2024-05-15T01:18:43.518Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: pipelining replication: peer="{Voter server-61b0bc0d-7f42-6028-ace9-4d5d0a521262 61b0bc0d-7f42-6028-ace9-4d5d0a521262}" -2024-05-15T01:18:43.518Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: pipelining replication: peer="{Voter server-6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7 6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7}" -2024-05-15T01:18:43.623Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:43.517460051 +0000 UTC m=+104.058844289 (1 nodes), 62.195343ms from start of poll, 65.138151ms from cluster start. Timeout at 2024-05-15 01:18:43.623580156 +0000 UTC m=+104.164964394, 106.120105ms after stability -2024-05-15T01:18:43.623Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:43.728Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:43.62368549 +0000 UTC m=+104.165069728 (6 nodes), 0s from start of poll, 171.36359ms from cluster start. Timeout at 2024-05-15 01:18:43.728703475 +0000 UTC m=+104.270088610, 105.018882ms after stability -2024-05-15T01:18:43.728Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:43.833Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:43.728812975 +0000 UTC m=+104.270197816 (1 nodes), 2.588µs from start of poll, 276.491678ms from cluster start. Timeout at 2024-05-15 01:18:43.833870582 +0000 UTC m=+104.375255701, 105.057885ms after stability -2024-05-15T01:18:43.834Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: entering candidate state: node="Node at 23e5cf23-7fb5-622a-1506-511ab3dd20bc [Candidate]" term=3 -2024-05-15T01:18:43.834Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:43.834Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: entering follower state: follower="Node at 55ed4422-338f-1f8b-5715-d47cdfcf7a30 [Follower]" leader-address= leader-id= -2024-05-15T01:18:43.834Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: aborting pipeline replication: peer="{Voter server-61b0bc0d-7f42-6028-ace9-4d5d0a521262 61b0bc0d-7f42-6028-ace9-4d5d0a521262}" -2024-05-15T01:18:43.834Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: election won: term=3 tally=4 -2024-05-15T01:18:43.834Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: aborting pipeline replication: peer="{Voter server-7fc7ace5-9311-a6a1-18bf-7b32693aa05b 7fc7ace5-9311-a6a1-18bf-7b32693aa05b}" -2024-05-15T01:18:43.834Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: entering leader state: leader="Node at 23e5cf23-7fb5-622a-1506-511ab3dd20bc [Leader]" -2024-05-15T01:18:43.834Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: aborting pipeline replication: peer="{Voter server-23e5cf23-7fb5-622a-1506-511ab3dd20bc 23e5cf23-7fb5-622a-1506-511ab3dd20bc}" -2024-05-15T01:18:43.834Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: added peer, starting replication: peer=server-55ed4422-338f-1f8b-5715-d47cdfcf7a30 -2024-05-15T01:18:43.834Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: aborting pipeline replication: peer="{Voter server-4bdd63d2-d807-3627-bb88-04ea222cc19e 4bdd63d2-d807-3627-bb88-04ea222cc19e}" -2024-05-15T01:18:43.834Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: added peer, starting replication: peer=server-4bdd63d2-d807-3627-bb88-04ea222cc19e -2024-05-15T01:18:43.834Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: aborting pipeline replication: peer="{Voter server-6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7 6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7}" -2024-05-15T01:18:43.834Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: added peer, starting replication: peer=server-6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7 -2024-05-15T01:18:43.834Z [INFO] server-55ed4422-338f-1f8b-5715-d47cdfcf7a30: aborting pipeline replication: peer="{Voter server-8bb00598-0887-985a-5146-ad2226e75116 8bb00598-0887-985a-5146-ad2226e75116}" -2024-05-15T01:18:43.834Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: added peer, starting replication: peer=server-8bb00598-0887-985a-5146-ad2226e75116 -2024-05-15T01:18:43.834Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: added peer, starting replication: peer=server-61b0bc0d-7f42-6028-ace9-4d5d0a521262 -2024-05-15T01:18:43.834Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: added peer, starting replication: peer=server-7fc7ace5-9311-a6a1-18bf-7b32693aa05b -2024-05-15T01:18:43.834Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: pipelining replication: peer="{Voter server-7fc7ace5-9311-a6a1-18bf-7b32693aa05b 7fc7ace5-9311-a6a1-18bf-7b32693aa05b}" -2024-05-15T01:18:43.835Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: pipelining replication: peer="{Voter server-8bb00598-0887-985a-5146-ad2226e75116 8bb00598-0887-985a-5146-ad2226e75116}" -2024-05-15T01:18:43.835Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: pipelining replication: peer="{Voter server-55ed4422-338f-1f8b-5715-d47cdfcf7a30 55ed4422-338f-1f8b-5715-d47cdfcf7a30}" -2024-05-15T01:18:43.835Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: pipelining replication: peer="{Voter server-4bdd63d2-d807-3627-bb88-04ea222cc19e 4bdd63d2-d807-3627-bb88-04ea222cc19e}" -2024-05-15T01:18:43.835Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: pipelining replication: peer="{Voter server-61b0bc0d-7f42-6028-ace9-4d5d0a521262 61b0bc0d-7f42-6028-ace9-4d5d0a521262}" -2024-05-15T01:18:43.835Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: pipelining replication: peer="{Voter server-6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7 6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7}" -2024-05-15T01:18:43.940Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:43.834805724 +0000 UTC m=+104.376194537 (1 nodes), 579.423µs from start of poll, 382.488399ms from cluster start. Timeout at 2024-05-15 01:18:43.940664001 +0000 UTC m=+104.482049102, 105.854565ms after stability -2024-05-15T01:18:43.941Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: aborting pipeline replication: peer="{Voter server-7fc7ace5-9311-a6a1-18bf-7b32693aa05b 7fc7ace5-9311-a6a1-18bf-7b32693aa05b}" -2024-05-15T01:18:43.941Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: aborting pipeline replication: peer="{Voter server-55ed4422-338f-1f8b-5715-d47cdfcf7a30 55ed4422-338f-1f8b-5715-d47cdfcf7a30}" -2024-05-15T01:18:43.941Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: aborting pipeline replication: peer="{Voter server-4bdd63d2-d807-3627-bb88-04ea222cc19e 4bdd63d2-d807-3627-bb88-04ea222cc19e}" -2024-05-15T01:18:43.941Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: aborting pipeline replication: peer="{Voter server-6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7 6d63e76b-b5bd-9d81-a32f-3e1b6e30aca7}" -2024-05-15T01:18:43.941Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: aborting pipeline replication: peer="{Voter server-8bb00598-0887-985a-5146-ad2226e75116 8bb00598-0887-985a-5146-ad2226e75116}" -2024-05-15T01:18:43.941Z [INFO] server-23e5cf23-7fb5-622a-1506-511ab3dd20bc: aborting pipeline replication: peer="{Voter server-61b0bc0d-7f42-6028-ace9-4d5d0a521262 61b0bc0d-7f42-6028-ace9-4d5d0a521262}" ---- PASS: TestRaft_LeadershipTransferWithSevenNodes (0.49s) +2025-06-17T07:44:35.200Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90e67331-62c6-35e6-39c4-4e068a807855 Address:90e67331-62c6-35e6-39c4-4e068a807855} {Suffrage:Voter ID:server-d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de Address:d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de} {Suffrage:Voter ID:server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f Address:e8c9cacb-8daa-f3c3-92a9-053c38db1e6f} {Suffrage:Voter ID:server-ebe0d69f-8172-c5eb-4226-cfe7a105d675 Address:ebe0d69f-8172-c5eb-4226-cfe7a105d675} {Suffrage:Voter ID:server-ff8a90de-b961-4bd6-0159-c3422192deb2 Address:ff8a90de-b961-4bd6-0159-c3422192deb2} {Suffrage:Voter ID:server-a1b9d969-bf2d-7195-e814-9899a5ce02de Address:a1b9d969-bf2d-7195-e814-9899a5ce02de} {Suffrage:Voter ID:server-5e5ab324-cb6a-6865-0ad6-cb99f9637dd5 Address:5e5ab324-cb6a-6865-0ad6-cb99f9637dd5}]" +2025-06-17T07:44:35.200Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: entering follower state: follower="Node at 90e67331-62c6-35e6-39c4-4e068a807855 [Follower]" leader-address= leader-id= +2025-06-17T07:44:35.200Z [INFO] server-d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90e67331-62c6-35e6-39c4-4e068a807855 Address:90e67331-62c6-35e6-39c4-4e068a807855} {Suffrage:Voter ID:server-d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de Address:d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de} {Suffrage:Voter ID:server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f Address:e8c9cacb-8daa-f3c3-92a9-053c38db1e6f} {Suffrage:Voter ID:server-ebe0d69f-8172-c5eb-4226-cfe7a105d675 Address:ebe0d69f-8172-c5eb-4226-cfe7a105d675} {Suffrage:Voter ID:server-ff8a90de-b961-4bd6-0159-c3422192deb2 Address:ff8a90de-b961-4bd6-0159-c3422192deb2} {Suffrage:Voter ID:server-a1b9d969-bf2d-7195-e814-9899a5ce02de Address:a1b9d969-bf2d-7195-e814-9899a5ce02de} {Suffrage:Voter ID:server-5e5ab324-cb6a-6865-0ad6-cb99f9637dd5 Address:5e5ab324-cb6a-6865-0ad6-cb99f9637dd5}]" +2025-06-17T07:44:35.200Z [INFO] server-d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de: entering follower state: follower="Node at d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de [Follower]" leader-address= leader-id= +2025-06-17T07:44:35.200Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90e67331-62c6-35e6-39c4-4e068a807855 Address:90e67331-62c6-35e6-39c4-4e068a807855} {Suffrage:Voter ID:server-d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de Address:d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de} {Suffrage:Voter ID:server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f Address:e8c9cacb-8daa-f3c3-92a9-053c38db1e6f} {Suffrage:Voter ID:server-ebe0d69f-8172-c5eb-4226-cfe7a105d675 Address:ebe0d69f-8172-c5eb-4226-cfe7a105d675} {Suffrage:Voter ID:server-ff8a90de-b961-4bd6-0159-c3422192deb2 Address:ff8a90de-b961-4bd6-0159-c3422192deb2} {Suffrage:Voter ID:server-a1b9d969-bf2d-7195-e814-9899a5ce02de Address:a1b9d969-bf2d-7195-e814-9899a5ce02de} {Suffrage:Voter ID:server-5e5ab324-cb6a-6865-0ad6-cb99f9637dd5 Address:5e5ab324-cb6a-6865-0ad6-cb99f9637dd5}]" +2025-06-17T07:44:35.200Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: entering follower state: follower="Node at e8c9cacb-8daa-f3c3-92a9-053c38db1e6f [Follower]" leader-address= leader-id= +2025-06-17T07:44:35.200Z [INFO] server-ebe0d69f-8172-c5eb-4226-cfe7a105d675: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90e67331-62c6-35e6-39c4-4e068a807855 Address:90e67331-62c6-35e6-39c4-4e068a807855} {Suffrage:Voter ID:server-d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de Address:d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de} {Suffrage:Voter ID:server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f Address:e8c9cacb-8daa-f3c3-92a9-053c38db1e6f} {Suffrage:Voter ID:server-ebe0d69f-8172-c5eb-4226-cfe7a105d675 Address:ebe0d69f-8172-c5eb-4226-cfe7a105d675} {Suffrage:Voter ID:server-ff8a90de-b961-4bd6-0159-c3422192deb2 Address:ff8a90de-b961-4bd6-0159-c3422192deb2} {Suffrage:Voter ID:server-a1b9d969-bf2d-7195-e814-9899a5ce02de Address:a1b9d969-bf2d-7195-e814-9899a5ce02de} {Suffrage:Voter ID:server-5e5ab324-cb6a-6865-0ad6-cb99f9637dd5 Address:5e5ab324-cb6a-6865-0ad6-cb99f9637dd5}]" +2025-06-17T07:44:35.200Z [INFO] server-ebe0d69f-8172-c5eb-4226-cfe7a105d675: entering follower state: follower="Node at ebe0d69f-8172-c5eb-4226-cfe7a105d675 [Follower]" leader-address= leader-id= +2025-06-17T07:44:35.200Z [INFO] server-ff8a90de-b961-4bd6-0159-c3422192deb2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90e67331-62c6-35e6-39c4-4e068a807855 Address:90e67331-62c6-35e6-39c4-4e068a807855} {Suffrage:Voter ID:server-d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de Address:d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de} {Suffrage:Voter ID:server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f Address:e8c9cacb-8daa-f3c3-92a9-053c38db1e6f} {Suffrage:Voter ID:server-ebe0d69f-8172-c5eb-4226-cfe7a105d675 Address:ebe0d69f-8172-c5eb-4226-cfe7a105d675} {Suffrage:Voter ID:server-ff8a90de-b961-4bd6-0159-c3422192deb2 Address:ff8a90de-b961-4bd6-0159-c3422192deb2} {Suffrage:Voter ID:server-a1b9d969-bf2d-7195-e814-9899a5ce02de Address:a1b9d969-bf2d-7195-e814-9899a5ce02de} {Suffrage:Voter ID:server-5e5ab324-cb6a-6865-0ad6-cb99f9637dd5 Address:5e5ab324-cb6a-6865-0ad6-cb99f9637dd5}]" +2025-06-17T07:44:35.200Z [INFO] server-ff8a90de-b961-4bd6-0159-c3422192deb2: entering follower state: follower="Node at ff8a90de-b961-4bd6-0159-c3422192deb2 [Follower]" leader-address= leader-id= +2025-06-17T07:44:35.200Z [INFO] server-a1b9d969-bf2d-7195-e814-9899a5ce02de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90e67331-62c6-35e6-39c4-4e068a807855 Address:90e67331-62c6-35e6-39c4-4e068a807855} {Suffrage:Voter ID:server-d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de Address:d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de} {Suffrage:Voter ID:server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f Address:e8c9cacb-8daa-f3c3-92a9-053c38db1e6f} {Suffrage:Voter ID:server-ebe0d69f-8172-c5eb-4226-cfe7a105d675 Address:ebe0d69f-8172-c5eb-4226-cfe7a105d675} {Suffrage:Voter ID:server-ff8a90de-b961-4bd6-0159-c3422192deb2 Address:ff8a90de-b961-4bd6-0159-c3422192deb2} {Suffrage:Voter ID:server-a1b9d969-bf2d-7195-e814-9899a5ce02de Address:a1b9d969-bf2d-7195-e814-9899a5ce02de} {Suffrage:Voter ID:server-5e5ab324-cb6a-6865-0ad6-cb99f9637dd5 Address:5e5ab324-cb6a-6865-0ad6-cb99f9637dd5}]" +2025-06-17T07:44:35.200Z [INFO] server-a1b9d969-bf2d-7195-e814-9899a5ce02de: entering follower state: follower="Node at a1b9d969-bf2d-7195-e814-9899a5ce02de [Follower]" leader-address= leader-id= +2025-06-17T07:44:35.200Z [INFO] server-5e5ab324-cb6a-6865-0ad6-cb99f9637dd5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90e67331-62c6-35e6-39c4-4e068a807855 Address:90e67331-62c6-35e6-39c4-4e068a807855} {Suffrage:Voter ID:server-d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de Address:d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de} {Suffrage:Voter ID:server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f Address:e8c9cacb-8daa-f3c3-92a9-053c38db1e6f} {Suffrage:Voter ID:server-ebe0d69f-8172-c5eb-4226-cfe7a105d675 Address:ebe0d69f-8172-c5eb-4226-cfe7a105d675} {Suffrage:Voter ID:server-ff8a90de-b961-4bd6-0159-c3422192deb2 Address:ff8a90de-b961-4bd6-0159-c3422192deb2} {Suffrage:Voter ID:server-a1b9d969-bf2d-7195-e814-9899a5ce02de Address:a1b9d969-bf2d-7195-e814-9899a5ce02de} {Suffrage:Voter ID:server-5e5ab324-cb6a-6865-0ad6-cb99f9637dd5 Address:5e5ab324-cb6a-6865-0ad6-cb99f9637dd5}]" +2025-06-17T07:44:35.201Z [INFO] server-5e5ab324-cb6a-6865-0ad6-cb99f9637dd5: entering follower state: follower="Node at 5e5ab324-cb6a-6865-0ad6-cb99f9637dd5 [Follower]" leader-address= leader-id= +2025-06-17T07:44:35.201Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:35.268Z [WARN] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:35.268Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: entering candidate state: node="Node at e8c9cacb-8daa-f3c3-92a9-053c38db1e6f [Candidate]" term=2 +2025-06-17T07:44:35.268Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: election won: term=2 tally=4 +2025-06-17T07:44:35.268Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: entering leader state: leader="Node at e8c9cacb-8daa-f3c3-92a9-053c38db1e6f [Leader]" +2025-06-17T07:44:35.268Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: added peer, starting replication: peer=server-90e67331-62c6-35e6-39c4-4e068a807855 +2025-06-17T07:44:35.268Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: added peer, starting replication: peer=server-d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de +2025-06-17T07:44:35.268Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: added peer, starting replication: peer=server-ebe0d69f-8172-c5eb-4226-cfe7a105d675 +2025-06-17T07:44:35.268Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: added peer, starting replication: peer=server-ff8a90de-b961-4bd6-0159-c3422192deb2 +2025-06-17T07:44:35.268Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: added peer, starting replication: peer=server-a1b9d969-bf2d-7195-e814-9899a5ce02de +2025-06-17T07:44:35.268Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: added peer, starting replication: peer=server-5e5ab324-cb6a-6865-0ad6-cb99f9637dd5 +2025-06-17T07:44:35.268Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: pipelining replication: peer="{Voter server-90e67331-62c6-35e6-39c4-4e068a807855 90e67331-62c6-35e6-39c4-4e068a807855}" +2025-06-17T07:44:35.268Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: pipelining replication: peer="{Voter server-5e5ab324-cb6a-6865-0ad6-cb99f9637dd5 5e5ab324-cb6a-6865-0ad6-cb99f9637dd5}" +2025-06-17T07:44:35.268Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: pipelining replication: peer="{Voter server-a1b9d969-bf2d-7195-e814-9899a5ce02de a1b9d969-bf2d-7195-e814-9899a5ce02de}" +2025-06-17T07:44:35.268Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: pipelining replication: peer="{Voter server-ff8a90de-b961-4bd6-0159-c3422192deb2 ff8a90de-b961-4bd6-0159-c3422192deb2}" +2025-06-17T07:44:35.268Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: pipelining replication: peer="{Voter server-d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de}" +2025-06-17T07:44:35.268Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: pipelining replication: peer="{Voter server-ebe0d69f-8172-c5eb-4226-cfe7a105d675 ebe0d69f-8172-c5eb-4226-cfe7a105d675}" +2025-06-17T07:44:35.374Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:35.268509864 +0000 UTC m=+103.872038549 (1 nodes), 67.42628ms from start of poll, 68.303467ms from cluster start. Timeout at 2025-06-17 07:44:35.374530537 +0000 UTC m=+103.978059231, 106.020682ms after stability +2025-06-17T07:44:35.374Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:35.480Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:35.374583977 +0000 UTC m=+103.978112664 (6 nodes), 1.052µs from start of poll, 174.377582ms from cluster start. Timeout at 2025-06-17 07:44:35.480488781 +0000 UTC m=+104.084017462, 105.904798ms after stability +2025-06-17T07:44:35.480Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:35.586Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:35.480540761 +0000 UTC m=+104.084069449 (1 nodes), 722ns from start of poll, 280.334367ms from cluster start. Timeout at 2025-06-17 07:44:35.586309645 +0000 UTC m=+104.189838336, 105.768887ms after stability +2025-06-17T07:44:35.586Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: entering candidate state: node="Node at 90e67331-62c6-35e6-39c4-4e068a807855 [Candidate]" term=3 +2025-06-17T07:44:35.586Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:35.586Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: election won: term=3 tally=4 +2025-06-17T07:44:35.586Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: entering follower state: follower="Node at e8c9cacb-8daa-f3c3-92a9-053c38db1e6f [Follower]" leader-address= leader-id= +2025-06-17T07:44:35.586Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: entering leader state: leader="Node at 90e67331-62c6-35e6-39c4-4e068a807855 [Leader]" +2025-06-17T07:44:35.586Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: aborting pipeline replication: peer="{Voter server-ebe0d69f-8172-c5eb-4226-cfe7a105d675 ebe0d69f-8172-c5eb-4226-cfe7a105d675}" +2025-06-17T07:44:35.586Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: aborting pipeline replication: peer="{Voter server-a1b9d969-bf2d-7195-e814-9899a5ce02de a1b9d969-bf2d-7195-e814-9899a5ce02de}" +2025-06-17T07:44:35.586Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: added peer, starting replication: peer=server-d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de +2025-06-17T07:44:35.586Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: aborting pipeline replication: peer="{Voter server-5e5ab324-cb6a-6865-0ad6-cb99f9637dd5 5e5ab324-cb6a-6865-0ad6-cb99f9637dd5}" +2025-06-17T07:44:35.586Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: added peer, starting replication: peer=server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f +2025-06-17T07:44:35.586Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: added peer, starting replication: peer=server-ebe0d69f-8172-c5eb-4226-cfe7a105d675 +2025-06-17T07:44:35.586Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: added peer, starting replication: peer=server-ff8a90de-b961-4bd6-0159-c3422192deb2 +2025-06-17T07:44:35.586Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: aborting pipeline replication: peer="{Voter server-ff8a90de-b961-4bd6-0159-c3422192deb2 ff8a90de-b961-4bd6-0159-c3422192deb2}" +2025-06-17T07:44:35.586Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: added peer, starting replication: peer=server-a1b9d969-bf2d-7195-e814-9899a5ce02de +2025-06-17T07:44:35.586Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: aborting pipeline replication: peer="{Voter server-d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de}" +2025-06-17T07:44:35.586Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: pipelining replication: peer="{Voter server-d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de}" +2025-06-17T07:44:35.586Z [INFO] server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f: aborting pipeline replication: peer="{Voter server-90e67331-62c6-35e6-39c4-4e068a807855 90e67331-62c6-35e6-39c4-4e068a807855}" +2025-06-17T07:44:35.586Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: added peer, starting replication: peer=server-5e5ab324-cb6a-6865-0ad6-cb99f9637dd5 +2025-06-17T07:44:35.586Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: pipelining replication: peer="{Voter server-ff8a90de-b961-4bd6-0159-c3422192deb2 ff8a90de-b961-4bd6-0159-c3422192deb2}" +2025-06-17T07:44:35.586Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: pipelining replication: peer="{Voter server-ebe0d69f-8172-c5eb-4226-cfe7a105d675 ebe0d69f-8172-c5eb-4226-cfe7a105d675}" +2025-06-17T07:44:35.586Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: pipelining replication: peer="{Voter server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f e8c9cacb-8daa-f3c3-92a9-053c38db1e6f}" +2025-06-17T07:44:35.586Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: pipelining replication: peer="{Voter server-a1b9d969-bf2d-7195-e814-9899a5ce02de a1b9d969-bf2d-7195-e814-9899a5ce02de}" +2025-06-17T07:44:35.586Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: pipelining replication: peer="{Voter server-5e5ab324-cb6a-6865-0ad6-cb99f9637dd5 5e5ab324-cb6a-6865-0ad6-cb99f9637dd5}" +2025-06-17T07:44:35.692Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:35.586503012 +0000 UTC m=+104.190031728 (1 nodes), 77.774µs from start of poll, 386.296646ms from cluster start. Timeout at 2025-06-17 07:44:35.692245616 +0000 UTC m=+104.295774335, 105.742607ms after stability +2025-06-17T07:44:35.692Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: aborting pipeline replication: peer="{Voter server-5e5ab324-cb6a-6865-0ad6-cb99f9637dd5 5e5ab324-cb6a-6865-0ad6-cb99f9637dd5}" +2025-06-17T07:44:35.692Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: aborting pipeline replication: peer="{Voter server-d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de}" +2025-06-17T07:44:35.692Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: aborting pipeline replication: peer="{Voter server-ff8a90de-b961-4bd6-0159-c3422192deb2 ff8a90de-b961-4bd6-0159-c3422192deb2}" +2025-06-17T07:44:35.692Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: aborting pipeline replication: peer="{Voter server-a1b9d969-bf2d-7195-e814-9899a5ce02de a1b9d969-bf2d-7195-e814-9899a5ce02de}" +2025-06-17T07:44:35.692Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: aborting pipeline replication: peer="{Voter server-ebe0d69f-8172-c5eb-4226-cfe7a105d675 ebe0d69f-8172-c5eb-4226-cfe7a105d675}" +2025-06-17T07:44:35.692Z [INFO] server-90e67331-62c6-35e6-39c4-4e068a807855: aborting pipeline replication: peer="{Voter server-e8c9cacb-8daa-f3c3-92a9-053c38db1e6f e8c9cacb-8daa-f3c3-92a9-053c38db1e6f}" +2025-06-17T07:44:36.192Z [ERROR] server-90e67331-62c6-35e6-39c4-4e068a807855: failed to heartbeat to: peer=d0ed2ca2-9f1f-9ab9-22de-1bd27a7324de backoff time=10ms error="command timed out" +--- PASS: TestRaft_LeadershipTransferWithSevenNodes (0.99s) === RUN TestRaft_LeadershipTransferToInvalidID -2024-05-15T01:18:43.944Z [INFO] server-14ee3755-4066-d868-7b4a-776597f7f9f1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-14ee3755-4066-d868-7b4a-776597f7f9f1 Address:14ee3755-4066-d868-7b4a-776597f7f9f1} {Suffrage:Voter ID:server-3fe605df-0d80-ce17-8557-23249a6263d2 Address:3fe605df-0d80-ce17-8557-23249a6263d2} {Suffrage:Voter ID:server-b4ecf291-86ca-bdbb-89e4-18ae8558fd27 Address:b4ecf291-86ca-bdbb-89e4-18ae8558fd27}]" -2024-05-15T01:18:43.944Z [INFO] server-3fe605df-0d80-ce17-8557-23249a6263d2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-14ee3755-4066-d868-7b4a-776597f7f9f1 Address:14ee3755-4066-d868-7b4a-776597f7f9f1} {Suffrage:Voter ID:server-3fe605df-0d80-ce17-8557-23249a6263d2 Address:3fe605df-0d80-ce17-8557-23249a6263d2} {Suffrage:Voter ID:server-b4ecf291-86ca-bdbb-89e4-18ae8558fd27 Address:b4ecf291-86ca-bdbb-89e4-18ae8558fd27}]" -2024-05-15T01:18:43.944Z [INFO] server-b4ecf291-86ca-bdbb-89e4-18ae8558fd27: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-14ee3755-4066-d868-7b4a-776597f7f9f1 Address:14ee3755-4066-d868-7b4a-776597f7f9f1} {Suffrage:Voter ID:server-3fe605df-0d80-ce17-8557-23249a6263d2 Address:3fe605df-0d80-ce17-8557-23249a6263d2} {Suffrage:Voter ID:server-b4ecf291-86ca-bdbb-89e4-18ae8558fd27 Address:b4ecf291-86ca-bdbb-89e4-18ae8558fd27}]" -2024-05-15T01:18:43.944Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:43.944Z [INFO] server-14ee3755-4066-d868-7b4a-776597f7f9f1: entering follower state: follower="Node at 14ee3755-4066-d868-7b4a-776597f7f9f1 [Follower]" leader-address= leader-id= -2024-05-15T01:18:43.945Z [INFO] server-3fe605df-0d80-ce17-8557-23249a6263d2: entering follower state: follower="Node at 3fe605df-0d80-ce17-8557-23249a6263d2 [Follower]" leader-address= leader-id= -2024-05-15T01:18:43.945Z [INFO] server-b4ecf291-86ca-bdbb-89e4-18ae8558fd27: entering follower state: follower="Node at b4ecf291-86ca-bdbb-89e4-18ae8558fd27 [Follower]" leader-address= leader-id= -2024-05-15T01:18:44.005Z [WARN] server-14ee3755-4066-d868-7b4a-776597f7f9f1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:44.005Z [INFO] server-14ee3755-4066-d868-7b4a-776597f7f9f1: entering candidate state: node="Node at 14ee3755-4066-d868-7b4a-776597f7f9f1 [Candidate]" term=2 -2024-05-15T01:18:44.006Z [INFO] server-14ee3755-4066-d868-7b4a-776597f7f9f1: election won: term=2 tally=2 -2024-05-15T01:18:44.006Z [INFO] server-14ee3755-4066-d868-7b4a-776597f7f9f1: entering leader state: leader="Node at 14ee3755-4066-d868-7b4a-776597f7f9f1 [Leader]" -2024-05-15T01:18:44.006Z [INFO] server-14ee3755-4066-d868-7b4a-776597f7f9f1: added peer, starting replication: peer=server-3fe605df-0d80-ce17-8557-23249a6263d2 -2024-05-15T01:18:44.006Z [INFO] server-14ee3755-4066-d868-7b4a-776597f7f9f1: added peer, starting replication: peer=server-b4ecf291-86ca-bdbb-89e4-18ae8558fd27 -2024-05-15T01:18:44.006Z [INFO] server-14ee3755-4066-d868-7b4a-776597f7f9f1: pipelining replication: peer="{Voter server-b4ecf291-86ca-bdbb-89e4-18ae8558fd27 b4ecf291-86ca-bdbb-89e4-18ae8558fd27}" -2024-05-15T01:18:44.006Z [INFO] server-14ee3755-4066-d868-7b4a-776597f7f9f1: pipelining replication: peer="{Voter server-3fe605df-0d80-ce17-8557-23249a6263d2 3fe605df-0d80-ce17-8557-23249a6263d2}" -2024-05-15T01:18:44.111Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:44.006623326 +0000 UTC m=+104.548007564 (1 nodes), 61.649255ms from start of poll, 62.337868ms from cluster start. Timeout at 2024-05-15 01:18:44.111639954 +0000 UTC m=+104.653024192, 105.016628ms after stability -2024-05-15T01:18:44.111Z [INFO] server-14ee3755-4066-d868-7b4a-776597f7f9f1: aborting pipeline replication: peer="{Voter server-b4ecf291-86ca-bdbb-89e4-18ae8558fd27 b4ecf291-86ca-bdbb-89e4-18ae8558fd27}" -2024-05-15T01:18:44.111Z [INFO] server-14ee3755-4066-d868-7b4a-776597f7f9f1: aborting pipeline replication: peer="{Voter server-3fe605df-0d80-ce17-8557-23249a6263d2 3fe605df-0d80-ce17-8557-23249a6263d2}" ---- PASS: TestRaft_LeadershipTransferToInvalidID (0.17s) +2025-06-17T07:44:36.194Z [INFO] server-077b0bc5-6c13-abfe-d5b8-a5eed6f80712: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-077b0bc5-6c13-abfe-d5b8-a5eed6f80712 Address:077b0bc5-6c13-abfe-d5b8-a5eed6f80712} {Suffrage:Voter ID:server-d27837a8-68d4-6927-6f4c-657a34231e4e Address:d27837a8-68d4-6927-6f4c-657a34231e4e} {Suffrage:Voter ID:server-785f5339-69df-f0bf-9adb-82712dac497f Address:785f5339-69df-f0bf-9adb-82712dac497f}]" +2025-06-17T07:44:36.194Z [INFO] server-077b0bc5-6c13-abfe-d5b8-a5eed6f80712: entering follower state: follower="Node at 077b0bc5-6c13-abfe-d5b8-a5eed6f80712 [Follower]" leader-address= leader-id= +2025-06-17T07:44:36.194Z [INFO] server-d27837a8-68d4-6927-6f4c-657a34231e4e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-077b0bc5-6c13-abfe-d5b8-a5eed6f80712 Address:077b0bc5-6c13-abfe-d5b8-a5eed6f80712} {Suffrage:Voter ID:server-d27837a8-68d4-6927-6f4c-657a34231e4e Address:d27837a8-68d4-6927-6f4c-657a34231e4e} {Suffrage:Voter ID:server-785f5339-69df-f0bf-9adb-82712dac497f Address:785f5339-69df-f0bf-9adb-82712dac497f}]" +2025-06-17T07:44:36.194Z [INFO] server-d27837a8-68d4-6927-6f4c-657a34231e4e: entering follower state: follower="Node at d27837a8-68d4-6927-6f4c-657a34231e4e [Follower]" leader-address= leader-id= +2025-06-17T07:44:36.194Z [INFO] server-785f5339-69df-f0bf-9adb-82712dac497f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-077b0bc5-6c13-abfe-d5b8-a5eed6f80712 Address:077b0bc5-6c13-abfe-d5b8-a5eed6f80712} {Suffrage:Voter ID:server-d27837a8-68d4-6927-6f4c-657a34231e4e Address:d27837a8-68d4-6927-6f4c-657a34231e4e} {Suffrage:Voter ID:server-785f5339-69df-f0bf-9adb-82712dac497f Address:785f5339-69df-f0bf-9adb-82712dac497f}]" +2025-06-17T07:44:36.194Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:36.194Z [INFO] server-785f5339-69df-f0bf-9adb-82712dac497f: entering follower state: follower="Node at 785f5339-69df-f0bf-9adb-82712dac497f [Follower]" leader-address= leader-id= +2025-06-17T07:44:36.248Z [WARN] server-077b0bc5-6c13-abfe-d5b8-a5eed6f80712: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:36.248Z [INFO] server-077b0bc5-6c13-abfe-d5b8-a5eed6f80712: entering candidate state: node="Node at 077b0bc5-6c13-abfe-d5b8-a5eed6f80712 [Candidate]" term=2 +2025-06-17T07:44:36.248Z [INFO] server-077b0bc5-6c13-abfe-d5b8-a5eed6f80712: election won: term=2 tally=2 +2025-06-17T07:44:36.248Z [INFO] server-077b0bc5-6c13-abfe-d5b8-a5eed6f80712: entering leader state: leader="Node at 077b0bc5-6c13-abfe-d5b8-a5eed6f80712 [Leader]" +2025-06-17T07:44:36.248Z [INFO] server-077b0bc5-6c13-abfe-d5b8-a5eed6f80712: added peer, starting replication: peer=server-d27837a8-68d4-6927-6f4c-657a34231e4e +2025-06-17T07:44:36.248Z [INFO] server-077b0bc5-6c13-abfe-d5b8-a5eed6f80712: added peer, starting replication: peer=server-785f5339-69df-f0bf-9adb-82712dac497f +2025-06-17T07:44:36.249Z [INFO] server-077b0bc5-6c13-abfe-d5b8-a5eed6f80712: pipelining replication: peer="{Voter server-d27837a8-68d4-6927-6f4c-657a34231e4e d27837a8-68d4-6927-6f4c-657a34231e4e}" +2025-06-17T07:44:36.249Z [INFO] server-077b0bc5-6c13-abfe-d5b8-a5eed6f80712: pipelining replication: peer="{Voter server-785f5339-69df-f0bf-9adb-82712dac497f 785f5339-69df-f0bf-9adb-82712dac497f}" +2025-06-17T07:44:36.354Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:36.24892113 +0000 UTC m=+104.852449820 (1 nodes), 54.459353ms from start of poll, 54.861897ms from cluster start. Timeout at 2025-06-17 07:44:36.354647582 +0000 UTC m=+104.958176264, 105.726444ms after stability +2025-06-17T07:44:36.354Z [INFO] server-077b0bc5-6c13-abfe-d5b8-a5eed6f80712: aborting pipeline replication: peer="{Voter server-785f5339-69df-f0bf-9adb-82712dac497f 785f5339-69df-f0bf-9adb-82712dac497f}" +2025-06-17T07:44:36.354Z [INFO] server-077b0bc5-6c13-abfe-d5b8-a5eed6f80712: aborting pipeline replication: peer="{Voter server-d27837a8-68d4-6927-6f4c-657a34231e4e d27837a8-68d4-6927-6f4c-657a34231e4e}" +--- PASS: TestRaft_LeadershipTransferToInvalidID (0.16s) === RUN TestRaft_LeadershipTransferToInvalidAddress -2024-05-15T01:18:44.113Z [INFO] server-7823bcc9-42b7-4b19-fb75-4067ef2c9f16: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7823bcc9-42b7-4b19-fb75-4067ef2c9f16 Address:7823bcc9-42b7-4b19-fb75-4067ef2c9f16} {Suffrage:Voter ID:server-a4d11307-5d0f-0843-e204-7de21762c57e Address:a4d11307-5d0f-0843-e204-7de21762c57e} {Suffrage:Voter ID:server-9a1f0b0e-19de-fb7f-4a60-8cafff456264 Address:9a1f0b0e-19de-fb7f-4a60-8cafff456264}]" -2024-05-15T01:18:44.114Z [INFO] server-a4d11307-5d0f-0843-e204-7de21762c57e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7823bcc9-42b7-4b19-fb75-4067ef2c9f16 Address:7823bcc9-42b7-4b19-fb75-4067ef2c9f16} {Suffrage:Voter ID:server-a4d11307-5d0f-0843-e204-7de21762c57e Address:a4d11307-5d0f-0843-e204-7de21762c57e} {Suffrage:Voter ID:server-9a1f0b0e-19de-fb7f-4a60-8cafff456264 Address:9a1f0b0e-19de-fb7f-4a60-8cafff456264}]" -2024-05-15T01:18:44.114Z [INFO] server-9a1f0b0e-19de-fb7f-4a60-8cafff456264: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7823bcc9-42b7-4b19-fb75-4067ef2c9f16 Address:7823bcc9-42b7-4b19-fb75-4067ef2c9f16} {Suffrage:Voter ID:server-a4d11307-5d0f-0843-e204-7de21762c57e Address:a4d11307-5d0f-0843-e204-7de21762c57e} {Suffrage:Voter ID:server-9a1f0b0e-19de-fb7f-4a60-8cafff456264 Address:9a1f0b0e-19de-fb7f-4a60-8cafff456264}]" -2024-05-15T01:18:44.114Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:44.114Z [INFO] server-7823bcc9-42b7-4b19-fb75-4067ef2c9f16: entering follower state: follower="Node at 7823bcc9-42b7-4b19-fb75-4067ef2c9f16 [Follower]" leader-address= leader-id= -2024-05-15T01:18:44.114Z [INFO] server-a4d11307-5d0f-0843-e204-7de21762c57e: entering follower state: follower="Node at a4d11307-5d0f-0843-e204-7de21762c57e [Follower]" leader-address= leader-id= -2024-05-15T01:18:44.114Z [INFO] server-9a1f0b0e-19de-fb7f-4a60-8cafff456264: entering follower state: follower="Node at 9a1f0b0e-19de-fb7f-4a60-8cafff456264 [Follower]" leader-address= leader-id= -2024-05-15T01:18:44.173Z [WARN] server-9a1f0b0e-19de-fb7f-4a60-8cafff456264: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:44.173Z [INFO] server-9a1f0b0e-19de-fb7f-4a60-8cafff456264: entering candidate state: node="Node at 9a1f0b0e-19de-fb7f-4a60-8cafff456264 [Candidate]" term=2 -2024-05-15T01:18:44.173Z [INFO] server-9a1f0b0e-19de-fb7f-4a60-8cafff456264: election won: term=2 tally=2 -2024-05-15T01:18:44.173Z [INFO] server-9a1f0b0e-19de-fb7f-4a60-8cafff456264: entering leader state: leader="Node at 9a1f0b0e-19de-fb7f-4a60-8cafff456264 [Leader]" -2024-05-15T01:18:44.173Z [INFO] server-9a1f0b0e-19de-fb7f-4a60-8cafff456264: added peer, starting replication: peer=server-7823bcc9-42b7-4b19-fb75-4067ef2c9f16 -2024-05-15T01:18:44.173Z [INFO] server-9a1f0b0e-19de-fb7f-4a60-8cafff456264: added peer, starting replication: peer=server-a4d11307-5d0f-0843-e204-7de21762c57e -2024-05-15T01:18:44.173Z [INFO] server-9a1f0b0e-19de-fb7f-4a60-8cafff456264: pipelining replication: peer="{Voter server-7823bcc9-42b7-4b19-fb75-4067ef2c9f16 7823bcc9-42b7-4b19-fb75-4067ef2c9f16}" -2024-05-15T01:18:44.174Z [INFO] server-9a1f0b0e-19de-fb7f-4a60-8cafff456264: pipelining replication: peer="{Voter server-a4d11307-5d0f-0843-e204-7de21762c57e a4d11307-5d0f-0843-e204-7de21762c57e}" -2024-05-15T01:18:44.279Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:44.173841707 +0000 UTC m=+104.715225945 (2 nodes), 59.404965ms from start of poll, 60.02662ms from cluster start. Timeout at 2024-05-15 01:18:44.279021757 +0000 UTC m=+104.820406485, 105.18054ms after stability -2024-05-15T01:18:44.279Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:44.384Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:44.279215198 +0000 UTC m=+104.820603050 (1 nodes), 3.614µs from start of poll, 165.403725ms from cluster start. Timeout at 2024-05-15 01:18:44.38455883 +0000 UTC m=+104.925944042, 105.340992ms after stability -2024-05-15T01:18:44.384Z [INFO] server-9a1f0b0e-19de-fb7f-4a60-8cafff456264: aborting pipeline replication: peer="{Voter server-a4d11307-5d0f-0843-e204-7de21762c57e a4d11307-5d0f-0843-e204-7de21762c57e}" -2024-05-15T01:18:44.384Z [INFO] server-9a1f0b0e-19de-fb7f-4a60-8cafff456264: aborting pipeline replication: peer="{Voter server-7823bcc9-42b7-4b19-fb75-4067ef2c9f16 7823bcc9-42b7-4b19-fb75-4067ef2c9f16}" ---- PASS: TestRaft_LeadershipTransferToInvalidAddress (0.27s) +2025-06-17T07:44:36.355Z [INFO] server-3686bfa0-1d95-fea5-8ae3-d87e8c503ccd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3686bfa0-1d95-fea5-8ae3-d87e8c503ccd Address:3686bfa0-1d95-fea5-8ae3-d87e8c503ccd} {Suffrage:Voter ID:server-a41f9e94-a96a-1a6b-a0b7-52d56857cd8e Address:a41f9e94-a96a-1a6b-a0b7-52d56857cd8e} {Suffrage:Voter ID:server-a0bf8ac3-9cca-83da-1c43-728360b48e63 Address:a0bf8ac3-9cca-83da-1c43-728360b48e63}]" +2025-06-17T07:44:36.355Z [INFO] server-3686bfa0-1d95-fea5-8ae3-d87e8c503ccd: entering follower state: follower="Node at 3686bfa0-1d95-fea5-8ae3-d87e8c503ccd [Follower]" leader-address= leader-id= +2025-06-17T07:44:36.355Z [INFO] server-a41f9e94-a96a-1a6b-a0b7-52d56857cd8e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3686bfa0-1d95-fea5-8ae3-d87e8c503ccd Address:3686bfa0-1d95-fea5-8ae3-d87e8c503ccd} {Suffrage:Voter ID:server-a41f9e94-a96a-1a6b-a0b7-52d56857cd8e Address:a41f9e94-a96a-1a6b-a0b7-52d56857cd8e} {Suffrage:Voter ID:server-a0bf8ac3-9cca-83da-1c43-728360b48e63 Address:a0bf8ac3-9cca-83da-1c43-728360b48e63}]" +2025-06-17T07:44:36.355Z [INFO] server-a41f9e94-a96a-1a6b-a0b7-52d56857cd8e: entering follower state: follower="Node at a41f9e94-a96a-1a6b-a0b7-52d56857cd8e [Follower]" leader-address= leader-id= +2025-06-17T07:44:36.355Z [INFO] server-a0bf8ac3-9cca-83da-1c43-728360b48e63: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3686bfa0-1d95-fea5-8ae3-d87e8c503ccd Address:3686bfa0-1d95-fea5-8ae3-d87e8c503ccd} {Suffrage:Voter ID:server-a41f9e94-a96a-1a6b-a0b7-52d56857cd8e Address:a41f9e94-a96a-1a6b-a0b7-52d56857cd8e} {Suffrage:Voter ID:server-a0bf8ac3-9cca-83da-1c43-728360b48e63 Address:a0bf8ac3-9cca-83da-1c43-728360b48e63}]" +2025-06-17T07:44:36.355Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:36.355Z [INFO] server-a0bf8ac3-9cca-83da-1c43-728360b48e63: entering follower state: follower="Node at a0bf8ac3-9cca-83da-1c43-728360b48e63 [Follower]" leader-address= leader-id= +2025-06-17T07:44:36.425Z [WARN] server-a0bf8ac3-9cca-83da-1c43-728360b48e63: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:36.425Z [INFO] server-a0bf8ac3-9cca-83da-1c43-728360b48e63: entering candidate state: node="Node at a0bf8ac3-9cca-83da-1c43-728360b48e63 [Candidate]" term=2 +2025-06-17T07:44:36.425Z [INFO] server-a0bf8ac3-9cca-83da-1c43-728360b48e63: election won: term=2 tally=2 +2025-06-17T07:44:36.425Z [INFO] server-a0bf8ac3-9cca-83da-1c43-728360b48e63: entering leader state: leader="Node at a0bf8ac3-9cca-83da-1c43-728360b48e63 [Leader]" +2025-06-17T07:44:36.425Z [INFO] server-a0bf8ac3-9cca-83da-1c43-728360b48e63: added peer, starting replication: peer=server-3686bfa0-1d95-fea5-8ae3-d87e8c503ccd +2025-06-17T07:44:36.425Z [INFO] server-a0bf8ac3-9cca-83da-1c43-728360b48e63: added peer, starting replication: peer=server-a41f9e94-a96a-1a6b-a0b7-52d56857cd8e +2025-06-17T07:44:36.425Z [INFO] server-a0bf8ac3-9cca-83da-1c43-728360b48e63: pipelining replication: peer="{Voter server-a41f9e94-a96a-1a6b-a0b7-52d56857cd8e a41f9e94-a96a-1a6b-a0b7-52d56857cd8e}" +2025-06-17T07:44:36.425Z [INFO] server-a0bf8ac3-9cca-83da-1c43-728360b48e63: pipelining replication: peer="{Voter server-3686bfa0-1d95-fea5-8ae3-d87e8c503ccd 3686bfa0-1d95-fea5-8ae3-d87e8c503ccd}" +2025-06-17T07:44:36.530Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:36.425407052 +0000 UTC m=+105.028935761 (2 nodes), 69.559359ms from start of poll, 69.896252ms from cluster start. Timeout at 2025-06-17 07:44:36.53078888 +0000 UTC m=+105.134317577, 105.381816ms after stability +2025-06-17T07:44:36.530Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:36.636Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:36.53084197 +0000 UTC m=+105.134370657 (1 nodes), 522ns from start of poll, 175.331148ms from cluster start. Timeout at 2025-06-17 07:44:36.636577756 +0000 UTC m=+105.240106438, 105.735781ms after stability +2025-06-17T07:44:36.636Z [INFO] server-a0bf8ac3-9cca-83da-1c43-728360b48e63: aborting pipeline replication: peer="{Voter server-a41f9e94-a96a-1a6b-a0b7-52d56857cd8e a41f9e94-a96a-1a6b-a0b7-52d56857cd8e}" +2025-06-17T07:44:36.636Z [INFO] server-a0bf8ac3-9cca-83da-1c43-728360b48e63: aborting pipeline replication: peer="{Voter server-3686bfa0-1d95-fea5-8ae3-d87e8c503ccd 3686bfa0-1d95-fea5-8ae3-d87e8c503ccd}" +--- PASS: TestRaft_LeadershipTransferToInvalidAddress (0.28s) === RUN TestRaft_LeadershipTransferToBehindServer -2024-05-15T01:18:44.386Z [INFO] server-5de4a366-bf58-a5fe-b6b1-cdec756d9180: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5de4a366-bf58-a5fe-b6b1-cdec756d9180 Address:5de4a366-bf58-a5fe-b6b1-cdec756d9180} {Suffrage:Voter ID:server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd Address:64002b77-dc72-9eb6-b1ae-b0e9b6463ebd} {Suffrage:Voter ID:server-23e88483-6532-fbba-663e-41f0cf492552 Address:23e88483-6532-fbba-663e-41f0cf492552}]" -2024-05-15T01:18:44.386Z [INFO] server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5de4a366-bf58-a5fe-b6b1-cdec756d9180 Address:5de4a366-bf58-a5fe-b6b1-cdec756d9180} {Suffrage:Voter ID:server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd Address:64002b77-dc72-9eb6-b1ae-b0e9b6463ebd} {Suffrage:Voter ID:server-23e88483-6532-fbba-663e-41f0cf492552 Address:23e88483-6532-fbba-663e-41f0cf492552}]" -2024-05-15T01:18:44.386Z [INFO] server-23e88483-6532-fbba-663e-41f0cf492552: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5de4a366-bf58-a5fe-b6b1-cdec756d9180 Address:5de4a366-bf58-a5fe-b6b1-cdec756d9180} {Suffrage:Voter ID:server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd Address:64002b77-dc72-9eb6-b1ae-b0e9b6463ebd} {Suffrage:Voter ID:server-23e88483-6532-fbba-663e-41f0cf492552 Address:23e88483-6532-fbba-663e-41f0cf492552}]" -2024-05-15T01:18:44.386Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:44.387Z [INFO] server-5de4a366-bf58-a5fe-b6b1-cdec756d9180: entering follower state: follower="Node at 5de4a366-bf58-a5fe-b6b1-cdec756d9180 [Follower]" leader-address= leader-id= -2024-05-15T01:18:44.387Z [INFO] server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd: entering follower state: follower="Node at 64002b77-dc72-9eb6-b1ae-b0e9b6463ebd [Follower]" leader-address= leader-id= -2024-05-15T01:18:44.387Z [INFO] server-23e88483-6532-fbba-663e-41f0cf492552: entering follower state: follower="Node at 23e88483-6532-fbba-663e-41f0cf492552 [Follower]" leader-address= leader-id= -2024-05-15T01:18:44.455Z [WARN] server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:44.455Z [INFO] server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd: entering candidate state: node="Node at 64002b77-dc72-9eb6-b1ae-b0e9b6463ebd [Candidate]" term=2 -2024-05-15T01:18:44.455Z [INFO] server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd: election won: term=2 tally=2 -2024-05-15T01:18:44.455Z [INFO] server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd: entering leader state: leader="Node at 64002b77-dc72-9eb6-b1ae-b0e9b6463ebd [Leader]" -2024-05-15T01:18:44.455Z [INFO] server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd: added peer, starting replication: peer=server-5de4a366-bf58-a5fe-b6b1-cdec756d9180 -2024-05-15T01:18:44.455Z [INFO] server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd: added peer, starting replication: peer=server-23e88483-6532-fbba-663e-41f0cf492552 -2024-05-15T01:18:44.455Z [INFO] server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd: pipelining replication: peer="{Voter server-5de4a366-bf58-a5fe-b6b1-cdec756d9180 5de4a366-bf58-a5fe-b6b1-cdec756d9180}" -2024-05-15T01:18:44.456Z [INFO] server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd: pipelining replication: peer="{Voter server-23e88483-6532-fbba-663e-41f0cf492552 23e88483-6532-fbba-663e-41f0cf492552}" -2024-05-15T01:18:44.560Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:44.455606011 +0000 UTC m=+104.996990249 (1 nodes), 68.614574ms from start of poll, 69.109057ms from cluster start. Timeout at 2024-05-15 01:18:44.560667084 +0000 UTC m=+105.102051322, 105.061073ms after stability -2024-05-15T01:18:44.560Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:44.666Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:44.560771745 +0000 UTC m=+105.102155983 (2 nodes), 0s from start of poll, 174.274791ms from cluster start. Timeout at 2024-05-15 01:18:44.666793102 +0000 UTC m=+105.208178169, 106.022186ms after stability -2024-05-15T01:18:44.811Z [INFO] server-5de4a366-bf58-a5fe-b6b1-cdec756d9180: entering candidate state: node="Node at 5de4a366-bf58-a5fe-b6b1-cdec756d9180 [Candidate]" term=3 -2024-05-15T01:18:44.811Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:44.811Z [INFO] server-5de4a366-bf58-a5fe-b6b1-cdec756d9180: election won: term=3 tally=2 -2024-05-15T01:18:44.811Z [INFO] server-5de4a366-bf58-a5fe-b6b1-cdec756d9180: entering leader state: leader="Node at 5de4a366-bf58-a5fe-b6b1-cdec756d9180 [Leader]" -2024-05-15T01:18:44.811Z [INFO] server-5de4a366-bf58-a5fe-b6b1-cdec756d9180: added peer, starting replication: peer=server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd -2024-05-15T01:18:44.811Z [INFO] server-5de4a366-bf58-a5fe-b6b1-cdec756d9180: added peer, starting replication: peer=server-23e88483-6532-fbba-663e-41f0cf492552 -2024-05-15T01:18:44.811Z [INFO] server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd: entering follower state: follower="Node at 64002b77-dc72-9eb6-b1ae-b0e9b6463ebd [Follower]" leader-address= leader-id= -2024-05-15T01:18:44.811Z [INFO] server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd: aborting pipeline replication: peer="{Voter server-5de4a366-bf58-a5fe-b6b1-cdec756d9180 5de4a366-bf58-a5fe-b6b1-cdec756d9180}" -2024-05-15T01:18:44.811Z [INFO] server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd: aborting pipeline replication: peer="{Voter server-23e88483-6532-fbba-663e-41f0cf492552 23e88483-6532-fbba-663e-41f0cf492552}" -2024-05-15T01:18:44.811Z [INFO] server-5de4a366-bf58-a5fe-b6b1-cdec756d9180: pipelining replication: peer="{Voter server-23e88483-6532-fbba-663e-41f0cf492552 23e88483-6532-fbba-663e-41f0cf492552}" -2024-05-15T01:18:44.812Z [INFO] server-5de4a366-bf58-a5fe-b6b1-cdec756d9180: pipelining replication: peer="{Voter server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd 64002b77-dc72-9eb6-b1ae-b0e9b6463ebd}" -2024-05-15T01:18:44.918Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:44.81181422 +0000 UTC m=+105.353198458 (1 nodes), 196.014µs from start of poll, 425.317266ms from cluster start. Timeout at 2024-05-15 01:18:44.917991219 +0000 UTC m=+105.459376831, 106.178373ms after stability -2024-05-15T01:18:44.918Z [INFO] server-5de4a366-bf58-a5fe-b6b1-cdec756d9180: aborting pipeline replication: peer="{Voter server-23e88483-6532-fbba-663e-41f0cf492552 23e88483-6532-fbba-663e-41f0cf492552}" -2024-05-15T01:18:44.918Z [INFO] server-5de4a366-bf58-a5fe-b6b1-cdec756d9180: aborting pipeline replication: peer="{Voter server-64002b77-dc72-9eb6-b1ae-b0e9b6463ebd 64002b77-dc72-9eb6-b1ae-b0e9b6463ebd}" ---- PASS: TestRaft_LeadershipTransferToBehindServer (0.53s) +2025-06-17T07:44:36.637Z [INFO] server-213538bb-f04d-c01f-327e-4db468c572b4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-213538bb-f04d-c01f-327e-4db468c572b4 Address:213538bb-f04d-c01f-327e-4db468c572b4} {Suffrage:Voter ID:server-ab72e3e2-4bb3-99e6-0953-33a244fc20e2 Address:ab72e3e2-4bb3-99e6-0953-33a244fc20e2} {Suffrage:Voter ID:server-4a198e9c-5ef9-69ef-7a31-6f45611ff769 Address:4a198e9c-5ef9-69ef-7a31-6f45611ff769}]" +2025-06-17T07:44:36.637Z [INFO] server-213538bb-f04d-c01f-327e-4db468c572b4: entering follower state: follower="Node at 213538bb-f04d-c01f-327e-4db468c572b4 [Follower]" leader-address= leader-id= +2025-06-17T07:44:36.637Z [INFO] server-ab72e3e2-4bb3-99e6-0953-33a244fc20e2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-213538bb-f04d-c01f-327e-4db468c572b4 Address:213538bb-f04d-c01f-327e-4db468c572b4} {Suffrage:Voter ID:server-ab72e3e2-4bb3-99e6-0953-33a244fc20e2 Address:ab72e3e2-4bb3-99e6-0953-33a244fc20e2} {Suffrage:Voter ID:server-4a198e9c-5ef9-69ef-7a31-6f45611ff769 Address:4a198e9c-5ef9-69ef-7a31-6f45611ff769}]" +2025-06-17T07:44:36.637Z [INFO] server-ab72e3e2-4bb3-99e6-0953-33a244fc20e2: entering follower state: follower="Node at ab72e3e2-4bb3-99e6-0953-33a244fc20e2 [Follower]" leader-address= leader-id= +2025-06-17T07:44:36.637Z [INFO] server-4a198e9c-5ef9-69ef-7a31-6f45611ff769: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-213538bb-f04d-c01f-327e-4db468c572b4 Address:213538bb-f04d-c01f-327e-4db468c572b4} {Suffrage:Voter ID:server-ab72e3e2-4bb3-99e6-0953-33a244fc20e2 Address:ab72e3e2-4bb3-99e6-0953-33a244fc20e2} {Suffrage:Voter ID:server-4a198e9c-5ef9-69ef-7a31-6f45611ff769 Address:4a198e9c-5ef9-69ef-7a31-6f45611ff769}]" +2025-06-17T07:44:36.637Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:36.637Z [INFO] server-4a198e9c-5ef9-69ef-7a31-6f45611ff769: entering follower state: follower="Node at 4a198e9c-5ef9-69ef-7a31-6f45611ff769 [Follower]" leader-address= leader-id= +2025-06-17T07:44:36.694Z [WARN] server-4a198e9c-5ef9-69ef-7a31-6f45611ff769: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:36.694Z [INFO] server-4a198e9c-5ef9-69ef-7a31-6f45611ff769: entering candidate state: node="Node at 4a198e9c-5ef9-69ef-7a31-6f45611ff769 [Candidate]" term=2 +2025-06-17T07:44:36.694Z [INFO] server-4a198e9c-5ef9-69ef-7a31-6f45611ff769: election won: term=2 tally=2 +2025-06-17T07:44:36.694Z [INFO] server-4a198e9c-5ef9-69ef-7a31-6f45611ff769: entering leader state: leader="Node at 4a198e9c-5ef9-69ef-7a31-6f45611ff769 [Leader]" +2025-06-17T07:44:36.694Z [INFO] server-4a198e9c-5ef9-69ef-7a31-6f45611ff769: added peer, starting replication: peer=server-213538bb-f04d-c01f-327e-4db468c572b4 +2025-06-17T07:44:36.694Z [INFO] server-4a198e9c-5ef9-69ef-7a31-6f45611ff769: added peer, starting replication: peer=server-ab72e3e2-4bb3-99e6-0953-33a244fc20e2 +2025-06-17T07:44:36.694Z [INFO] server-4a198e9c-5ef9-69ef-7a31-6f45611ff769: pipelining replication: peer="{Voter server-ab72e3e2-4bb3-99e6-0953-33a244fc20e2 ab72e3e2-4bb3-99e6-0953-33a244fc20e2}" +2025-06-17T07:44:36.694Z [INFO] server-4a198e9c-5ef9-69ef-7a31-6f45611ff769: pipelining replication: peer="{Voter server-213538bb-f04d-c01f-327e-4db468c572b4 213538bb-f04d-c01f-327e-4db468c572b4}" +2025-06-17T07:44:36.800Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:36.69479832 +0000 UTC m=+105.298327022 (1 nodes), 56.960117ms from start of poll, 57.260975ms from cluster start. Timeout at 2025-06-17 07:44:36.80078454 +0000 UTC m=+105.404313222, 105.9862ms after stability +2025-06-17T07:44:36.800Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:36.906Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:36.800835239 +0000 UTC m=+105.404363931 (2 nodes), 773ns from start of poll, 163.297884ms from cluster start. Timeout at 2025-06-17 07:44:36.906793781 +0000 UTC m=+105.510322473, 105.958542ms after stability +2025-06-17T07:44:36.953Z [INFO] server-213538bb-f04d-c01f-327e-4db468c572b4: entering candidate state: node="Node at 213538bb-f04d-c01f-327e-4db468c572b4 [Candidate]" term=3 +2025-06-17T07:44:36.954Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:36.954Z [INFO] server-213538bb-f04d-c01f-327e-4db468c572b4: election won: term=3 tally=2 +2025-06-17T07:44:36.954Z [INFO] server-213538bb-f04d-c01f-327e-4db468c572b4: entering leader state: leader="Node at 213538bb-f04d-c01f-327e-4db468c572b4 [Leader]" +2025-06-17T07:44:36.954Z [INFO] server-213538bb-f04d-c01f-327e-4db468c572b4: added peer, starting replication: peer=server-ab72e3e2-4bb3-99e6-0953-33a244fc20e2 +2025-06-17T07:44:36.954Z [INFO] server-213538bb-f04d-c01f-327e-4db468c572b4: added peer, starting replication: peer=server-4a198e9c-5ef9-69ef-7a31-6f45611ff769 +2025-06-17T07:44:36.954Z [INFO] server-4a198e9c-5ef9-69ef-7a31-6f45611ff769: entering follower state: follower="Node at 4a198e9c-5ef9-69ef-7a31-6f45611ff769 [Follower]" leader-address= leader-id= +2025-06-17T07:44:36.954Z [INFO] server-4a198e9c-5ef9-69ef-7a31-6f45611ff769: aborting pipeline replication: peer="{Voter server-ab72e3e2-4bb3-99e6-0953-33a244fc20e2 ab72e3e2-4bb3-99e6-0953-33a244fc20e2}" +2025-06-17T07:44:36.954Z [INFO] server-4a198e9c-5ef9-69ef-7a31-6f45611ff769: aborting pipeline replication: peer="{Voter server-213538bb-f04d-c01f-327e-4db468c572b4 213538bb-f04d-c01f-327e-4db468c572b4}" +2025-06-17T07:44:36.954Z [INFO] server-213538bb-f04d-c01f-327e-4db468c572b4: pipelining replication: peer="{Voter server-4a198e9c-5ef9-69ef-7a31-6f45611ff769 4a198e9c-5ef9-69ef-7a31-6f45611ff769}" +2025-06-17T07:44:36.954Z [INFO] server-213538bb-f04d-c01f-327e-4db468c572b4: pipelining replication: peer="{Voter server-ab72e3e2-4bb3-99e6-0953-33a244fc20e2 ab72e3e2-4bb3-99e6-0953-33a244fc20e2}" +2025-06-17T07:44:37.059Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:36.954104156 +0000 UTC m=+105.557632852 (1 nodes), 27.84µs from start of poll, 316.566805ms from cluster start. Timeout at 2025-06-17 07:44:37.059360262 +0000 UTC m=+105.662888955, 105.256103ms after stability +2025-06-17T07:44:37.059Z [INFO] server-213538bb-f04d-c01f-327e-4db468c572b4: aborting pipeline replication: peer="{Voter server-4a198e9c-5ef9-69ef-7a31-6f45611ff769 4a198e9c-5ef9-69ef-7a31-6f45611ff769}" +2025-06-17T07:44:37.059Z [INFO] server-213538bb-f04d-c01f-327e-4db468c572b4: aborting pipeline replication: peer="{Voter server-ab72e3e2-4bb3-99e6-0953-33a244fc20e2 ab72e3e2-4bb3-99e6-0953-33a244fc20e2}" +--- PASS: TestRaft_LeadershipTransferToBehindServer (0.42s) === RUN TestRaft_LeadershipTransferToItself -2024-05-15T01:18:44.920Z [INFO] server-4365dfec-c237-fad7-2ec7-bd645992728d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4365dfec-c237-fad7-2ec7-bd645992728d Address:4365dfec-c237-fad7-2ec7-bd645992728d} {Suffrage:Voter ID:server-8a124a63-7a55-7dcd-e916-7da84346bf4e Address:8a124a63-7a55-7dcd-e916-7da84346bf4e} {Suffrage:Voter ID:server-82a78c0f-6bb1-2355-b79b-721ecbd13c7c Address:82a78c0f-6bb1-2355-b79b-721ecbd13c7c}]" -2024-05-15T01:18:44.920Z [INFO] server-8a124a63-7a55-7dcd-e916-7da84346bf4e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4365dfec-c237-fad7-2ec7-bd645992728d Address:4365dfec-c237-fad7-2ec7-bd645992728d} {Suffrage:Voter ID:server-8a124a63-7a55-7dcd-e916-7da84346bf4e Address:8a124a63-7a55-7dcd-e916-7da84346bf4e} {Suffrage:Voter ID:server-82a78c0f-6bb1-2355-b79b-721ecbd13c7c Address:82a78c0f-6bb1-2355-b79b-721ecbd13c7c}]" -2024-05-15T01:18:44.920Z [INFO] server-82a78c0f-6bb1-2355-b79b-721ecbd13c7c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4365dfec-c237-fad7-2ec7-bd645992728d Address:4365dfec-c237-fad7-2ec7-bd645992728d} {Suffrage:Voter ID:server-8a124a63-7a55-7dcd-e916-7da84346bf4e Address:8a124a63-7a55-7dcd-e916-7da84346bf4e} {Suffrage:Voter ID:server-82a78c0f-6bb1-2355-b79b-721ecbd13c7c Address:82a78c0f-6bb1-2355-b79b-721ecbd13c7c}]" -2024-05-15T01:18:44.920Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:44.920Z [INFO] server-4365dfec-c237-fad7-2ec7-bd645992728d: entering follower state: follower="Node at 4365dfec-c237-fad7-2ec7-bd645992728d [Follower]" leader-address= leader-id= -2024-05-15T01:18:44.921Z [INFO] server-8a124a63-7a55-7dcd-e916-7da84346bf4e: entering follower state: follower="Node at 8a124a63-7a55-7dcd-e916-7da84346bf4e [Follower]" leader-address= leader-id= -2024-05-15T01:18:44.921Z [INFO] server-82a78c0f-6bb1-2355-b79b-721ecbd13c7c: entering follower state: follower="Node at 82a78c0f-6bb1-2355-b79b-721ecbd13c7c [Follower]" leader-address= leader-id= -2024-05-15T01:18:44.989Z [WARN] server-8a124a63-7a55-7dcd-e916-7da84346bf4e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:44.989Z [INFO] server-8a124a63-7a55-7dcd-e916-7da84346bf4e: entering candidate state: node="Node at 8a124a63-7a55-7dcd-e916-7da84346bf4e [Candidate]" term=2 -2024-05-15T01:18:44.989Z [INFO] server-8a124a63-7a55-7dcd-e916-7da84346bf4e: election won: term=2 tally=2 -2024-05-15T01:18:44.989Z [INFO] server-8a124a63-7a55-7dcd-e916-7da84346bf4e: entering leader state: leader="Node at 8a124a63-7a55-7dcd-e916-7da84346bf4e [Leader]" -2024-05-15T01:18:44.989Z [INFO] server-8a124a63-7a55-7dcd-e916-7da84346bf4e: added peer, starting replication: peer=server-4365dfec-c237-fad7-2ec7-bd645992728d -2024-05-15T01:18:44.989Z [INFO] server-8a124a63-7a55-7dcd-e916-7da84346bf4e: added peer, starting replication: peer=server-82a78c0f-6bb1-2355-b79b-721ecbd13c7c -2024-05-15T01:18:44.989Z [INFO] server-8a124a63-7a55-7dcd-e916-7da84346bf4e: pipelining replication: peer="{Voter server-4365dfec-c237-fad7-2ec7-bd645992728d 4365dfec-c237-fad7-2ec7-bd645992728d}" -2024-05-15T01:18:44.989Z [INFO] server-8a124a63-7a55-7dcd-e916-7da84346bf4e: pipelining replication: peer="{Voter server-82a78c0f-6bb1-2355-b79b-721ecbd13c7c 82a78c0f-6bb1-2355-b79b-721ecbd13c7c}" -2024-05-15T01:18:45.095Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:44.989604845 +0000 UTC m=+105.530989812 (1 nodes), 68.70251ms from start of poll, 69.409235ms from cluster start. Timeout at 2024-05-15 01:18:45.095343437 +0000 UTC m=+105.636728411, 105.738599ms after stability -2024-05-15T01:18:45.095Z [INFO] server-8a124a63-7a55-7dcd-e916-7da84346bf4e: cannot transfer leadership to itself -2024-05-15T01:18:45.095Z [INFO] server-8a124a63-7a55-7dcd-e916-7da84346bf4e: aborting pipeline replication: peer="{Voter server-82a78c0f-6bb1-2355-b79b-721ecbd13c7c 82a78c0f-6bb1-2355-b79b-721ecbd13c7c}" -2024-05-15T01:18:45.095Z [INFO] server-8a124a63-7a55-7dcd-e916-7da84346bf4e: aborting pipeline replication: peer="{Voter server-4365dfec-c237-fad7-2ec7-bd645992728d 4365dfec-c237-fad7-2ec7-bd645992728d}" ---- PASS: TestRaft_LeadershipTransferToItself (0.18s) +2025-06-17T07:44:37.060Z [INFO] server-3e936b14-5de3-f998-bae2-dc5cfe03e706: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3e936b14-5de3-f998-bae2-dc5cfe03e706 Address:3e936b14-5de3-f998-bae2-dc5cfe03e706} {Suffrage:Voter ID:server-e92976a3-9fdf-9d41-e9bf-ef67afcb4925 Address:e92976a3-9fdf-9d41-e9bf-ef67afcb4925} {Suffrage:Voter ID:server-01a8892d-64fe-0409-af8f-a168c4b87919 Address:01a8892d-64fe-0409-af8f-a168c4b87919}]" +2025-06-17T07:44:37.060Z [INFO] server-3e936b14-5de3-f998-bae2-dc5cfe03e706: entering follower state: follower="Node at 3e936b14-5de3-f998-bae2-dc5cfe03e706 [Follower]" leader-address= leader-id= +2025-06-17T07:44:37.060Z [INFO] server-e92976a3-9fdf-9d41-e9bf-ef67afcb4925: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3e936b14-5de3-f998-bae2-dc5cfe03e706 Address:3e936b14-5de3-f998-bae2-dc5cfe03e706} {Suffrage:Voter ID:server-e92976a3-9fdf-9d41-e9bf-ef67afcb4925 Address:e92976a3-9fdf-9d41-e9bf-ef67afcb4925} {Suffrage:Voter ID:server-01a8892d-64fe-0409-af8f-a168c4b87919 Address:01a8892d-64fe-0409-af8f-a168c4b87919}]" +2025-06-17T07:44:37.060Z [INFO] server-e92976a3-9fdf-9d41-e9bf-ef67afcb4925: entering follower state: follower="Node at e92976a3-9fdf-9d41-e9bf-ef67afcb4925 [Follower]" leader-address= leader-id= +2025-06-17T07:44:37.060Z [INFO] server-01a8892d-64fe-0409-af8f-a168c4b87919: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3e936b14-5de3-f998-bae2-dc5cfe03e706 Address:3e936b14-5de3-f998-bae2-dc5cfe03e706} {Suffrage:Voter ID:server-e92976a3-9fdf-9d41-e9bf-ef67afcb4925 Address:e92976a3-9fdf-9d41-e9bf-ef67afcb4925} {Suffrage:Voter ID:server-01a8892d-64fe-0409-af8f-a168c4b87919 Address:01a8892d-64fe-0409-af8f-a168c4b87919}]" +2025-06-17T07:44:37.060Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:37.060Z [INFO] server-01a8892d-64fe-0409-af8f-a168c4b87919: entering follower state: follower="Node at 01a8892d-64fe-0409-af8f-a168c4b87919 [Follower]" leader-address= leader-id= +2025-06-17T07:44:37.122Z [WARN] server-e92976a3-9fdf-9d41-e9bf-ef67afcb4925: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:37.122Z [INFO] server-e92976a3-9fdf-9d41-e9bf-ef67afcb4925: entering candidate state: node="Node at e92976a3-9fdf-9d41-e9bf-ef67afcb4925 [Candidate]" term=2 +2025-06-17T07:44:37.122Z [INFO] server-e92976a3-9fdf-9d41-e9bf-ef67afcb4925: election won: term=2 tally=2 +2025-06-17T07:44:37.122Z [INFO] server-e92976a3-9fdf-9d41-e9bf-ef67afcb4925: entering leader state: leader="Node at e92976a3-9fdf-9d41-e9bf-ef67afcb4925 [Leader]" +2025-06-17T07:44:37.122Z [INFO] server-e92976a3-9fdf-9d41-e9bf-ef67afcb4925: added peer, starting replication: peer=server-3e936b14-5de3-f998-bae2-dc5cfe03e706 +2025-06-17T07:44:37.122Z [INFO] server-e92976a3-9fdf-9d41-e9bf-ef67afcb4925: added peer, starting replication: peer=server-01a8892d-64fe-0409-af8f-a168c4b87919 +2025-06-17T07:44:37.122Z [INFO] server-e92976a3-9fdf-9d41-e9bf-ef67afcb4925: pipelining replication: peer="{Voter server-3e936b14-5de3-f998-bae2-dc5cfe03e706 3e936b14-5de3-f998-bae2-dc5cfe03e706}" +2025-06-17T07:44:37.122Z [INFO] server-e92976a3-9fdf-9d41-e9bf-ef67afcb4925: pipelining replication: peer="{Voter server-01a8892d-64fe-0409-af8f-a168c4b87919 01a8892d-64fe-0409-af8f-a168c4b87919}" +2025-06-17T07:44:37.228Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:37.122604846 +0000 UTC m=+105.726133533 (1 nodes), 61.928348ms from start of poll, 62.283849ms from cluster start. Timeout at 2025-06-17 07:44:37.228450354 +0000 UTC m=+105.831979038, 105.845505ms after stability +2025-06-17T07:44:37.228Z [INFO] server-e92976a3-9fdf-9d41-e9bf-ef67afcb4925: cannot transfer leadership to itself +2025-06-17T07:44:37.228Z [INFO] server-e92976a3-9fdf-9d41-e9bf-ef67afcb4925: aborting pipeline replication: peer="{Voter server-01a8892d-64fe-0409-af8f-a168c4b87919 01a8892d-64fe-0409-af8f-a168c4b87919}" +2025-06-17T07:44:37.228Z [INFO] server-e92976a3-9fdf-9d41-e9bf-ef67afcb4925: aborting pipeline replication: peer="{Voter server-3e936b14-5de3-f998-bae2-dc5cfe03e706 3e936b14-5de3-f998-bae2-dc5cfe03e706}" +--- PASS: TestRaft_LeadershipTransferToItself (0.17s) === RUN TestRaft_LeadershipTransferLeaderRejectsClientRequests -2024-05-15T01:18:45.097Z [INFO] server-367df221-b6c3-dd14-05cd-3966fa131d28: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-367df221-b6c3-dd14-05cd-3966fa131d28 Address:367df221-b6c3-dd14-05cd-3966fa131d28} {Suffrage:Voter ID:server-a64ef9ad-ceab-6435-b146-252f91133686 Address:a64ef9ad-ceab-6435-b146-252f91133686} {Suffrage:Voter ID:server-48be50a8-7e9a-ae39-9ad6-a6fc4d6d5fac Address:48be50a8-7e9a-ae39-9ad6-a6fc4d6d5fac}]" -2024-05-15T01:18:45.097Z [INFO] server-a64ef9ad-ceab-6435-b146-252f91133686: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-367df221-b6c3-dd14-05cd-3966fa131d28 Address:367df221-b6c3-dd14-05cd-3966fa131d28} {Suffrage:Voter ID:server-a64ef9ad-ceab-6435-b146-252f91133686 Address:a64ef9ad-ceab-6435-b146-252f91133686} {Suffrage:Voter ID:server-48be50a8-7e9a-ae39-9ad6-a6fc4d6d5fac Address:48be50a8-7e9a-ae39-9ad6-a6fc4d6d5fac}]" -2024-05-15T01:18:45.098Z [INFO] server-48be50a8-7e9a-ae39-9ad6-a6fc4d6d5fac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-367df221-b6c3-dd14-05cd-3966fa131d28 Address:367df221-b6c3-dd14-05cd-3966fa131d28} {Suffrage:Voter ID:server-a64ef9ad-ceab-6435-b146-252f91133686 Address:a64ef9ad-ceab-6435-b146-252f91133686} {Suffrage:Voter ID:server-48be50a8-7e9a-ae39-9ad6-a6fc4d6d5fac Address:48be50a8-7e9a-ae39-9ad6-a6fc4d6d5fac}]" -2024-05-15T01:18:45.098Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:45.098Z [INFO] server-a64ef9ad-ceab-6435-b146-252f91133686: entering follower state: follower="Node at a64ef9ad-ceab-6435-b146-252f91133686 [Follower]" leader-address= leader-id= -2024-05-15T01:18:45.098Z [INFO] server-367df221-b6c3-dd14-05cd-3966fa131d28: entering follower state: follower="Node at 367df221-b6c3-dd14-05cd-3966fa131d28 [Follower]" leader-address= leader-id= -2024-05-15T01:18:45.098Z [INFO] server-48be50a8-7e9a-ae39-9ad6-a6fc4d6d5fac: entering follower state: follower="Node at 48be50a8-7e9a-ae39-9ad6-a6fc4d6d5fac [Follower]" leader-address= leader-id= -2024-05-15T01:18:45.154Z [WARN] server-367df221-b6c3-dd14-05cd-3966fa131d28: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:45.154Z [INFO] server-367df221-b6c3-dd14-05cd-3966fa131d28: entering candidate state: node="Node at 367df221-b6c3-dd14-05cd-3966fa131d28 [Candidate]" term=2 -2024-05-15T01:18:45.154Z [INFO] server-367df221-b6c3-dd14-05cd-3966fa131d28: election won: term=2 tally=2 -2024-05-15T01:18:45.154Z [INFO] server-367df221-b6c3-dd14-05cd-3966fa131d28: entering leader state: leader="Node at 367df221-b6c3-dd14-05cd-3966fa131d28 [Leader]" -2024-05-15T01:18:45.154Z [INFO] server-367df221-b6c3-dd14-05cd-3966fa131d28: added peer, starting replication: peer=server-a64ef9ad-ceab-6435-b146-252f91133686 -2024-05-15T01:18:45.154Z [INFO] server-367df221-b6c3-dd14-05cd-3966fa131d28: added peer, starting replication: peer=server-48be50a8-7e9a-ae39-9ad6-a6fc4d6d5fac -2024-05-15T01:18:45.155Z [INFO] server-367df221-b6c3-dd14-05cd-3966fa131d28: pipelining replication: peer="{Voter server-a64ef9ad-ceab-6435-b146-252f91133686 a64ef9ad-ceab-6435-b146-252f91133686}" -2024-05-15T01:18:45.155Z [INFO] server-367df221-b6c3-dd14-05cd-3966fa131d28: pipelining replication: peer="{Voter server-48be50a8-7e9a-ae39-9ad6-a6fc4d6d5fac 48be50a8-7e9a-ae39-9ad6-a6fc4d6d5fac}" -2024-05-15T01:18:45.260Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:45.154904894 +0000 UTC m=+105.696289889 (1 nodes), 56.776341ms from start of poll, 57.393156ms from cluster start. Timeout at 2024-05-15 01:18:45.26007285 +0000 UTC m=+105.801457691, 105.167802ms after stability +2025-06-17T07:44:37.229Z [INFO] server-f6750410-d121-284f-d422-936011650bca: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f6750410-d121-284f-d422-936011650bca Address:f6750410-d121-284f-d422-936011650bca} {Suffrage:Voter ID:server-05c0b393-5bba-704d-92e8-a97895f8958e Address:05c0b393-5bba-704d-92e8-a97895f8958e} {Suffrage:Voter ID:server-1016f8d9-2a42-748a-87c7-4b3da951c1e2 Address:1016f8d9-2a42-748a-87c7-4b3da951c1e2}]" +2025-06-17T07:44:37.229Z [INFO] server-f6750410-d121-284f-d422-936011650bca: entering follower state: follower="Node at f6750410-d121-284f-d422-936011650bca [Follower]" leader-address= leader-id= +2025-06-17T07:44:37.229Z [INFO] server-05c0b393-5bba-704d-92e8-a97895f8958e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f6750410-d121-284f-d422-936011650bca Address:f6750410-d121-284f-d422-936011650bca} {Suffrage:Voter ID:server-05c0b393-5bba-704d-92e8-a97895f8958e Address:05c0b393-5bba-704d-92e8-a97895f8958e} {Suffrage:Voter ID:server-1016f8d9-2a42-748a-87c7-4b3da951c1e2 Address:1016f8d9-2a42-748a-87c7-4b3da951c1e2}]" +2025-06-17T07:44:37.229Z [INFO] server-05c0b393-5bba-704d-92e8-a97895f8958e: entering follower state: follower="Node at 05c0b393-5bba-704d-92e8-a97895f8958e [Follower]" leader-address= leader-id= +2025-06-17T07:44:37.229Z [INFO] server-1016f8d9-2a42-748a-87c7-4b3da951c1e2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f6750410-d121-284f-d422-936011650bca Address:f6750410-d121-284f-d422-936011650bca} {Suffrage:Voter ID:server-05c0b393-5bba-704d-92e8-a97895f8958e Address:05c0b393-5bba-704d-92e8-a97895f8958e} {Suffrage:Voter ID:server-1016f8d9-2a42-748a-87c7-4b3da951c1e2 Address:1016f8d9-2a42-748a-87c7-4b3da951c1e2}]" +2025-06-17T07:44:37.229Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:37.229Z [INFO] server-1016f8d9-2a42-748a-87c7-4b3da951c1e2: entering follower state: follower="Node at 1016f8d9-2a42-748a-87c7-4b3da951c1e2 [Follower]" leader-address= leader-id= +2025-06-17T07:44:37.282Z [WARN] server-05c0b393-5bba-704d-92e8-a97895f8958e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:37.282Z [INFO] server-05c0b393-5bba-704d-92e8-a97895f8958e: entering candidate state: node="Node at 05c0b393-5bba-704d-92e8-a97895f8958e [Candidate]" term=2 +2025-06-17T07:44:37.282Z [INFO] server-05c0b393-5bba-704d-92e8-a97895f8958e: election won: term=2 tally=2 +2025-06-17T07:44:37.282Z [INFO] server-05c0b393-5bba-704d-92e8-a97895f8958e: entering leader state: leader="Node at 05c0b393-5bba-704d-92e8-a97895f8958e [Leader]" +2025-06-17T07:44:37.282Z [INFO] server-05c0b393-5bba-704d-92e8-a97895f8958e: added peer, starting replication: peer=server-f6750410-d121-284f-d422-936011650bca +2025-06-17T07:44:37.282Z [INFO] server-05c0b393-5bba-704d-92e8-a97895f8958e: added peer, starting replication: peer=server-1016f8d9-2a42-748a-87c7-4b3da951c1e2 +2025-06-17T07:44:37.282Z [INFO] server-05c0b393-5bba-704d-92e8-a97895f8958e: pipelining replication: peer="{Voter server-1016f8d9-2a42-748a-87c7-4b3da951c1e2 1016f8d9-2a42-748a-87c7-4b3da951c1e2}" +2025-06-17T07:44:37.282Z [INFO] server-05c0b393-5bba-704d-92e8-a97895f8958e: pipelining replication: peer="{Voter server-f6750410-d121-284f-d422-936011650bca f6750410-d121-284f-d422-936011650bca}" +2025-06-17T07:44:37.388Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:37.282408114 +0000 UTC m=+105.885936818 (1 nodes), 52.747899ms from start of poll, 53.070252ms from cluster start. Timeout at 2025-06-17 07:44:37.387971988 +0000 UTC m=+105.991500692, 105.563874ms after stability raft_test.go:2457: waiting on future 0 raft_test.go:2457: waiting on future 1 raft_test.go:2457: waiting on future 2 @@ -3854,338 +3892,343 @@ raft_test.go:2457: waiting on future 4 raft_test.go:2457: waiting on future 5 raft_test.go:2457: waiting on future 6 -2024-05-15T01:18:45.260Z [INFO] server-367df221-b6c3-dd14-05cd-3966fa131d28: aborting pipeline replication: peer="{Voter server-48be50a8-7e9a-ae39-9ad6-a6fc4d6d5fac 48be50a8-7e9a-ae39-9ad6-a6fc4d6d5fac}" -2024-05-15T01:18:45.260Z [INFO] server-367df221-b6c3-dd14-05cd-3966fa131d28: aborting pipeline replication: peer="{Voter server-a64ef9ad-ceab-6435-b146-252f91133686 a64ef9ad-ceab-6435-b146-252f91133686}" +2025-06-17T07:44:37.388Z [INFO] server-05c0b393-5bba-704d-92e8-a97895f8958e: aborting pipeline replication: peer="{Voter server-1016f8d9-2a42-748a-87c7-4b3da951c1e2 1016f8d9-2a42-748a-87c7-4b3da951c1e2}" +2025-06-17T07:44:37.388Z [INFO] server-05c0b393-5bba-704d-92e8-a97895f8958e: aborting pipeline replication: peer="{Voter server-f6750410-d121-284f-d422-936011650bca f6750410-d121-284f-d422-936011650bca}" --- PASS: TestRaft_LeadershipTransferLeaderRejectsClientRequests (0.16s) === RUN TestRaft_LeadershipTransferLeaderReplicationTimeout -2024-05-15T01:18:45.262Z [INFO] server-137dc089-fb16-9066-0eb8-557780b20ba0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-137dc089-fb16-9066-0eb8-557780b20ba0 Address:137dc089-fb16-9066-0eb8-557780b20ba0} {Suffrage:Voter ID:server-01267b32-4d0d-fb2d-36fa-7786f2ffdb34 Address:01267b32-4d0d-fb2d-36fa-7786f2ffdb34} {Suffrage:Voter ID:server-822a9a70-d2b5-af4c-ad1e-f8413d2600cd Address:822a9a70-d2b5-af4c-ad1e-f8413d2600cd}]" -2024-05-15T01:18:45.262Z [INFO] server-137dc089-fb16-9066-0eb8-557780b20ba0: entering follower state: follower="Node at 137dc089-fb16-9066-0eb8-557780b20ba0 [Follower]" leader-address= leader-id= -2024-05-15T01:18:45.262Z [INFO] server-01267b32-4d0d-fb2d-36fa-7786f2ffdb34: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-137dc089-fb16-9066-0eb8-557780b20ba0 Address:137dc089-fb16-9066-0eb8-557780b20ba0} {Suffrage:Voter ID:server-01267b32-4d0d-fb2d-36fa-7786f2ffdb34 Address:01267b32-4d0d-fb2d-36fa-7786f2ffdb34} {Suffrage:Voter ID:server-822a9a70-d2b5-af4c-ad1e-f8413d2600cd Address:822a9a70-d2b5-af4c-ad1e-f8413d2600cd}]" -2024-05-15T01:18:45.263Z [INFO] server-01267b32-4d0d-fb2d-36fa-7786f2ffdb34: entering follower state: follower="Node at 01267b32-4d0d-fb2d-36fa-7786f2ffdb34 [Follower]" leader-address= leader-id= -2024-05-15T01:18:45.263Z [INFO] server-822a9a70-d2b5-af4c-ad1e-f8413d2600cd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-137dc089-fb16-9066-0eb8-557780b20ba0 Address:137dc089-fb16-9066-0eb8-557780b20ba0} {Suffrage:Voter ID:server-01267b32-4d0d-fb2d-36fa-7786f2ffdb34 Address:01267b32-4d0d-fb2d-36fa-7786f2ffdb34} {Suffrage:Voter ID:server-822a9a70-d2b5-af4c-ad1e-f8413d2600cd Address:822a9a70-d2b5-af4c-ad1e-f8413d2600cd}]" -2024-05-15T01:18:45.263Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:45.263Z [INFO] server-822a9a70-d2b5-af4c-ad1e-f8413d2600cd: entering follower state: follower="Node at 822a9a70-d2b5-af4c-ad1e-f8413d2600cd [Follower]" leader-address= leader-id= -2024-05-15T01:18:45.318Z [WARN] server-01267b32-4d0d-fb2d-36fa-7786f2ffdb34: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:45.318Z [INFO] server-01267b32-4d0d-fb2d-36fa-7786f2ffdb34: entering candidate state: node="Node at 01267b32-4d0d-fb2d-36fa-7786f2ffdb34 [Candidate]" term=2 -2024-05-15T01:18:45.318Z [INFO] server-01267b32-4d0d-fb2d-36fa-7786f2ffdb34: election won: term=2 tally=2 -2024-05-15T01:18:45.318Z [INFO] server-01267b32-4d0d-fb2d-36fa-7786f2ffdb34: entering leader state: leader="Node at 01267b32-4d0d-fb2d-36fa-7786f2ffdb34 [Leader]" -2024-05-15T01:18:45.318Z [INFO] server-01267b32-4d0d-fb2d-36fa-7786f2ffdb34: added peer, starting replication: peer=server-137dc089-fb16-9066-0eb8-557780b20ba0 -2024-05-15T01:18:45.318Z [INFO] server-01267b32-4d0d-fb2d-36fa-7786f2ffdb34: added peer, starting replication: peer=server-822a9a70-d2b5-af4c-ad1e-f8413d2600cd -2024-05-15T01:18:45.318Z [INFO] server-01267b32-4d0d-fb2d-36fa-7786f2ffdb34: pipelining replication: peer="{Voter server-822a9a70-d2b5-af4c-ad1e-f8413d2600cd 822a9a70-d2b5-af4c-ad1e-f8413d2600cd}" -2024-05-15T01:18:45.319Z [INFO] server-01267b32-4d0d-fb2d-36fa-7786f2ffdb34: pipelining replication: peer="{Voter server-137dc089-fb16-9066-0eb8-557780b20ba0 137dc089-fb16-9066-0eb8-557780b20ba0}" -2024-05-15T01:18:45.424Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:45.319489684 +0000 UTC m=+105.860874545 (1 nodes), 56.275749ms from start of poll, 57.010846ms from cluster start. Timeout at 2024-05-15 01:18:45.424497627 +0000 UTC m=+105.965882755, 105.00821ms after stability -2024-05-15T01:18:45.424Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:45.529Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:45.424609688 +0000 UTC m=+105.965994948 (2 nodes), 2.948µs from start of poll, 162.131249ms from cluster start. Timeout at 2024-05-15 01:18:45.529635035 +0000 UTC m=+106.071019773, 105.024825ms after stability -2024-05-15T01:18:46.921Z [INFO] server-01267b32-4d0d-fb2d-36fa-7786f2ffdb34: aborting pipeline replication: peer="{Voter server-822a9a70-d2b5-af4c-ad1e-f8413d2600cd 822a9a70-d2b5-af4c-ad1e-f8413d2600cd}" -2024-05-15T01:18:46.921Z [INFO] server-01267b32-4d0d-fb2d-36fa-7786f2ffdb34: aborting pipeline replication: peer="{Voter server-137dc089-fb16-9066-0eb8-557780b20ba0 137dc089-fb16-9066-0eb8-557780b20ba0}" ---- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (1.66s) +2025-06-17T07:44:37.388Z [INFO] server-c063db9d-9695-0726-2c9c-ee8e9388217e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c063db9d-9695-0726-2c9c-ee8e9388217e Address:c063db9d-9695-0726-2c9c-ee8e9388217e} {Suffrage:Voter ID:server-4d992598-cbb9-3b1f-5823-cc6c0759aa4f Address:4d992598-cbb9-3b1f-5823-cc6c0759aa4f} {Suffrage:Voter ID:server-45bf6923-a4e4-33a7-b9db-c0de9fff9de2 Address:45bf6923-a4e4-33a7-b9db-c0de9fff9de2}]" +2025-06-17T07:44:37.389Z [INFO] server-c063db9d-9695-0726-2c9c-ee8e9388217e: entering follower state: follower="Node at c063db9d-9695-0726-2c9c-ee8e9388217e [Follower]" leader-address= leader-id= +2025-06-17T07:44:37.389Z [INFO] server-4d992598-cbb9-3b1f-5823-cc6c0759aa4f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c063db9d-9695-0726-2c9c-ee8e9388217e Address:c063db9d-9695-0726-2c9c-ee8e9388217e} {Suffrage:Voter ID:server-4d992598-cbb9-3b1f-5823-cc6c0759aa4f Address:4d992598-cbb9-3b1f-5823-cc6c0759aa4f} {Suffrage:Voter ID:server-45bf6923-a4e4-33a7-b9db-c0de9fff9de2 Address:45bf6923-a4e4-33a7-b9db-c0de9fff9de2}]" +2025-06-17T07:44:37.389Z [INFO] server-4d992598-cbb9-3b1f-5823-cc6c0759aa4f: entering follower state: follower="Node at 4d992598-cbb9-3b1f-5823-cc6c0759aa4f [Follower]" leader-address= leader-id= +2025-06-17T07:44:37.389Z [INFO] server-45bf6923-a4e4-33a7-b9db-c0de9fff9de2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c063db9d-9695-0726-2c9c-ee8e9388217e Address:c063db9d-9695-0726-2c9c-ee8e9388217e} {Suffrage:Voter ID:server-4d992598-cbb9-3b1f-5823-cc6c0759aa4f Address:4d992598-cbb9-3b1f-5823-cc6c0759aa4f} {Suffrage:Voter ID:server-45bf6923-a4e4-33a7-b9db-c0de9fff9de2 Address:45bf6923-a4e4-33a7-b9db-c0de9fff9de2}]" +2025-06-17T07:44:37.389Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:37.389Z [INFO] server-45bf6923-a4e4-33a7-b9db-c0de9fff9de2: entering follower state: follower="Node at 45bf6923-a4e4-33a7-b9db-c0de9fff9de2 [Follower]" leader-address= leader-id= +2025-06-17T07:44:37.471Z [WARN] server-45bf6923-a4e4-33a7-b9db-c0de9fff9de2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:37.471Z [INFO] server-45bf6923-a4e4-33a7-b9db-c0de9fff9de2: entering candidate state: node="Node at 45bf6923-a4e4-33a7-b9db-c0de9fff9de2 [Candidate]" term=2 +2025-06-17T07:44:37.471Z [INFO] server-45bf6923-a4e4-33a7-b9db-c0de9fff9de2: election won: term=2 tally=2 +2025-06-17T07:44:37.471Z [INFO] server-45bf6923-a4e4-33a7-b9db-c0de9fff9de2: entering leader state: leader="Node at 45bf6923-a4e4-33a7-b9db-c0de9fff9de2 [Leader]" +2025-06-17T07:44:37.471Z [INFO] server-45bf6923-a4e4-33a7-b9db-c0de9fff9de2: added peer, starting replication: peer=server-c063db9d-9695-0726-2c9c-ee8e9388217e +2025-06-17T07:44:37.471Z [INFO] server-45bf6923-a4e4-33a7-b9db-c0de9fff9de2: added peer, starting replication: peer=server-4d992598-cbb9-3b1f-5823-cc6c0759aa4f +2025-06-17T07:44:37.471Z [INFO] server-45bf6923-a4e4-33a7-b9db-c0de9fff9de2: pipelining replication: peer="{Voter server-c063db9d-9695-0726-2c9c-ee8e9388217e c063db9d-9695-0726-2c9c-ee8e9388217e}" +2025-06-17T07:44:37.471Z [INFO] server-45bf6923-a4e4-33a7-b9db-c0de9fff9de2: pipelining replication: peer="{Voter server-4d992598-cbb9-3b1f-5823-cc6c0759aa4f 4d992598-cbb9-3b1f-5823-cc6c0759aa4f}" +2025-06-17T07:44:37.577Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:37.471534735 +0000 UTC m=+106.075063431 (1 nodes), 82.358604ms from start of poll, 82.620553ms from cluster start. Timeout at 2025-06-17 07:44:37.577214878 +0000 UTC m=+106.180743585, 105.680154ms after stability +2025-06-17T07:44:37.577Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:37.682Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:37.577264364 +0000 UTC m=+106.180793049 (2 nodes), 991ns from start of poll, 188.350171ms from cluster start. Timeout at 2025-06-17 07:44:37.682271078 +0000 UTC m=+106.285799788, 105.006739ms after stability +2025-06-17T07:44:38.132Z [INFO] server-45bf6923-a4e4-33a7-b9db-c0de9fff9de2: aborting pipeline replication: peer="{Voter server-4d992598-cbb9-3b1f-5823-cc6c0759aa4f 4d992598-cbb9-3b1f-5823-cc6c0759aa4f}" +2025-06-17T07:44:38.132Z [INFO] server-45bf6923-a4e4-33a7-b9db-c0de9fff9de2: aborting pipeline replication: peer="{Voter server-c063db9d-9695-0726-2c9c-ee8e9388217e c063db9d-9695-0726-2c9c-ee8e9388217e}" +--- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (0.74s) === RUN TestRaft_LeadershipTransferIgnoresNonvoters -2024-05-15T01:18:46.923Z [INFO] server-79557b80-7133-87ff-d87b-0e122ceba503: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-79557b80-7133-87ff-d87b-0e122ceba503 Address:79557b80-7133-87ff-d87b-0e122ceba503} {Suffrage:Voter ID:server-78a0831b-4870-317e-e92a-f30d14656d22 Address:78a0831b-4870-317e-e92a-f30d14656d22}]" -2024-05-15T01:18:46.923Z [INFO] server-79557b80-7133-87ff-d87b-0e122ceba503: entering follower state: follower="Node at 79557b80-7133-87ff-d87b-0e122ceba503 [Follower]" leader-address= leader-id= -2024-05-15T01:18:46.923Z [INFO] server-78a0831b-4870-317e-e92a-f30d14656d22: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-79557b80-7133-87ff-d87b-0e122ceba503 Address:79557b80-7133-87ff-d87b-0e122ceba503} {Suffrage:Voter ID:server-78a0831b-4870-317e-e92a-f30d14656d22 Address:78a0831b-4870-317e-e92a-f30d14656d22}]" -2024-05-15T01:18:46.923Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:46.923Z [INFO] server-78a0831b-4870-317e-e92a-f30d14656d22: entering follower state: follower="Node at 78a0831b-4870-317e-e92a-f30d14656d22 [Follower]" leader-address= leader-id= -2024-05-15T01:18:46.981Z [WARN] server-78a0831b-4870-317e-e92a-f30d14656d22: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:46.981Z [INFO] server-78a0831b-4870-317e-e92a-f30d14656d22: entering candidate state: node="Node at 78a0831b-4870-317e-e92a-f30d14656d22 [Candidate]" term=2 -2024-05-15T01:18:46.981Z [INFO] server-78a0831b-4870-317e-e92a-f30d14656d22: election won: term=2 tally=2 -2024-05-15T01:18:46.981Z [INFO] server-78a0831b-4870-317e-e92a-f30d14656d22: entering leader state: leader="Node at 78a0831b-4870-317e-e92a-f30d14656d22 [Leader]" -2024-05-15T01:18:46.981Z [INFO] server-78a0831b-4870-317e-e92a-f30d14656d22: added peer, starting replication: peer=server-79557b80-7133-87ff-d87b-0e122ceba503 -2024-05-15T01:18:46.982Z [INFO] server-78a0831b-4870-317e-e92a-f30d14656d22: pipelining replication: peer="{Voter server-79557b80-7133-87ff-d87b-0e122ceba503 79557b80-7133-87ff-d87b-0e122ceba503}" -2024-05-15T01:18:47.087Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:46.981852922 +0000 UTC m=+107.523237794 (1 nodes), 58.063853ms from start of poll, 58.473451ms from cluster start. Timeout at 2024-05-15 01:18:47.087137929 +0000 UTC m=+107.628522888, 105.285094ms after stability -2024-05-15T01:18:47.087Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:47.193Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:47.087265718 +0000 UTC m=+107.628650621 (1 nodes), 2.163µs from start of poll, 163.886278ms from cluster start. Timeout at 2024-05-15 01:18:47.193203305 +0000 UTC m=+107.734588215, 105.937594ms after stability -2024-05-15T01:18:47.193Z [INFO] server-78a0831b-4870-317e-e92a-f30d14656d22: updating configuration: command=DemoteVoter server-id=server-79557b80-7133-87ff-d87b-0e122ceba503 server-addr= servers="[{Suffrage:Nonvoter ID:server-79557b80-7133-87ff-d87b-0e122ceba503 Address:79557b80-7133-87ff-d87b-0e122ceba503} {Suffrage:Voter ID:server-78a0831b-4870-317e-e92a-f30d14656d22 Address:78a0831b-4870-317e-e92a-f30d14656d22}]" -2024-05-15T01:18:47.193Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:47.298Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:47.193580238 +0000 UTC m=+107.734965203 (1 nodes), 2.501µs from start of poll, 270.20086ms from cluster start. Timeout at 2024-05-15 01:18:47.298596767 +0000 UTC m=+107.839981879, 105.016676ms after stability -2024-05-15T01:18:47.298Z [INFO] server-78a0831b-4870-317e-e92a-f30d14656d22: aborting pipeline replication: peer="{Voter server-79557b80-7133-87ff-d87b-0e122ceba503 79557b80-7133-87ff-d87b-0e122ceba503}" ---- PASS: TestRaft_LeadershipTransferIgnoresNonvoters (0.38s) +2025-06-17T07:44:38.132Z [INFO] server-339497c0-70cf-6217-9580-662c89d85119: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-339497c0-70cf-6217-9580-662c89d85119 Address:339497c0-70cf-6217-9580-662c89d85119} {Suffrage:Voter ID:server-94700534-c8a2-2d93-9eda-9bae3e03dadf Address:94700534-c8a2-2d93-9eda-9bae3e03dadf}]" +2025-06-17T07:44:38.133Z [INFO] server-339497c0-70cf-6217-9580-662c89d85119: entering follower state: follower="Node at 339497c0-70cf-6217-9580-662c89d85119 [Follower]" leader-address= leader-id= +2025-06-17T07:44:38.133Z [INFO] server-94700534-c8a2-2d93-9eda-9bae3e03dadf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-339497c0-70cf-6217-9580-662c89d85119 Address:339497c0-70cf-6217-9580-662c89d85119} {Suffrage:Voter ID:server-94700534-c8a2-2d93-9eda-9bae3e03dadf Address:94700534-c8a2-2d93-9eda-9bae3e03dadf}]" +2025-06-17T07:44:38.133Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:38.133Z [INFO] server-94700534-c8a2-2d93-9eda-9bae3e03dadf: entering follower state: follower="Node at 94700534-c8a2-2d93-9eda-9bae3e03dadf [Follower]" leader-address= leader-id= +2025-06-17T07:44:38.222Z [WARN] server-94700534-c8a2-2d93-9eda-9bae3e03dadf: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:38.222Z [INFO] server-94700534-c8a2-2d93-9eda-9bae3e03dadf: entering candidate state: node="Node at 94700534-c8a2-2d93-9eda-9bae3e03dadf [Candidate]" term=2 +2025-06-17T07:44:38.222Z [INFO] server-94700534-c8a2-2d93-9eda-9bae3e03dadf: election won: term=2 tally=2 +2025-06-17T07:44:38.222Z [INFO] server-94700534-c8a2-2d93-9eda-9bae3e03dadf: entering leader state: leader="Node at 94700534-c8a2-2d93-9eda-9bae3e03dadf [Leader]" +2025-06-17T07:44:38.222Z [INFO] server-94700534-c8a2-2d93-9eda-9bae3e03dadf: added peer, starting replication: peer=server-339497c0-70cf-6217-9580-662c89d85119 +2025-06-17T07:44:38.222Z [INFO] server-94700534-c8a2-2d93-9eda-9bae3e03dadf: pipelining replication: peer="{Voter server-339497c0-70cf-6217-9580-662c89d85119 339497c0-70cf-6217-9580-662c89d85119}" +2025-06-17T07:44:38.327Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:38.222777812 +0000 UTC m=+106.826306522 (1 nodes), 89.673502ms from start of poll, 89.862346ms from cluster start. Timeout at 2025-06-17 07:44:38.327794183 +0000 UTC m=+106.931322866, 105.016344ms after stability +2025-06-17T07:44:38.327Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:38.433Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:38.32784066 +0000 UTC m=+106.931369348 (1 nodes), 625ns from start of poll, 194.925172ms from cluster start. Timeout at 2025-06-17 07:44:38.433608861 +0000 UTC m=+107.037137543, 105.768195ms after stability +2025-06-17T07:44:38.433Z [INFO] server-94700534-c8a2-2d93-9eda-9bae3e03dadf: updating configuration: command=DemoteVoter server-id=server-339497c0-70cf-6217-9580-662c89d85119 server-addr= servers="[{Suffrage:Nonvoter ID:server-339497c0-70cf-6217-9580-662c89d85119 Address:339497c0-70cf-6217-9580-662c89d85119} {Suffrage:Voter ID:server-94700534-c8a2-2d93-9eda-9bae3e03dadf Address:94700534-c8a2-2d93-9eda-9bae3e03dadf}]" +2025-06-17T07:44:38.433Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:38.539Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:38.433757438 +0000 UTC m=+107.037286122 (1 nodes), 423ns from start of poll, 300.841946ms from cluster start. Timeout at 2025-06-17 07:44:38.539521426 +0000 UTC m=+107.143050115, 105.763993ms after stability +2025-06-17T07:44:38.539Z [INFO] server-94700534-c8a2-2d93-9eda-9bae3e03dadf: aborting pipeline replication: peer="{Voter server-339497c0-70cf-6217-9580-662c89d85119 339497c0-70cf-6217-9580-662c89d85119}" +--- PASS: TestRaft_LeadershipTransferIgnoresNonvoters (0.41s) === RUN TestRaft_LeadershipTransferStopRightAway --- PASS: TestRaft_LeadershipTransferStopRightAway (0.00s) === RUN TestRaft_GetConfigurationNoBootstrap -2024-05-15T01:18:47.300Z [INFO] server-df8655bc-1ecd-c306-8d3f-8c56e79e1295: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-df8655bc-1ecd-c306-8d3f-8c56e79e1295 Address:df8655bc-1ecd-c306-8d3f-8c56e79e1295} {Suffrage:Voter ID:server-4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533 Address:4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533}]" -2024-05-15T01:18:47.300Z [INFO] server-4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-df8655bc-1ecd-c306-8d3f-8c56e79e1295 Address:df8655bc-1ecd-c306-8d3f-8c56e79e1295} {Suffrage:Voter ID:server-4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533 Address:4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533}]" -2024-05-15T01:18:47.300Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:47.300Z [INFO] server-df8655bc-1ecd-c306-8d3f-8c56e79e1295: entering follower state: follower="Node at df8655bc-1ecd-c306-8d3f-8c56e79e1295 [Follower]" leader-address= leader-id= -2024-05-15T01:18:47.300Z [INFO] server-4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533: entering follower state: follower="Node at 4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533 [Follower]" leader-address= leader-id= -2024-05-15T01:18:47.363Z [WARN] server-4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:47.363Z [INFO] server-4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533: entering candidate state: node="Node at 4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533 [Candidate]" term=2 -2024-05-15T01:18:47.364Z [INFO] server-4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533: election won: term=2 tally=2 -2024-05-15T01:18:47.364Z [INFO] server-4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533: entering leader state: leader="Node at 4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533 [Leader]" -2024-05-15T01:18:47.364Z [INFO] server-4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533: added peer, starting replication: peer=server-df8655bc-1ecd-c306-8d3f-8c56e79e1295 -2024-05-15T01:18:47.364Z [INFO] server-4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533: pipelining replication: peer="{Voter server-df8655bc-1ecd-c306-8d3f-8c56e79e1295 df8655bc-1ecd-c306-8d3f-8c56e79e1295}" -2024-05-15T01:18:47.469Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:47.364076772 +0000 UTC m=+107.905461788 (1 nodes), 63.387395ms from start of poll, 63.774654ms from cluster start. Timeout at 2024-05-15 01:18:47.469704432 +0000 UTC m=+108.011089463, 105.627675ms after stability -2024-05-15T01:18:47.469Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:47.575Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:47.469852885 +0000 UTC m=+108.011237831 (1 nodes), 2.002µs from start of poll, 169.550697ms from cluster start. Timeout at 2024-05-15 01:18:47.575458583 +0000 UTC m=+108.116843534, 105.605703ms after stability -2024-05-15T01:18:47.587Z [INFO] server-4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-df8655bc-1ecd-c306-8d3f-8c56e79e1295 Address:df8655bc-1ecd-c306-8d3f-8c56e79e1295} {Suffrage:Voter ID:server-4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533 Address:4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533}]" -2024-05-15T01:18:47.587Z [INFO] server-4c6dff61-f0d3-306a-ac8f-3dd2d3bfe533: aborting pipeline replication: peer="{Voter server-df8655bc-1ecd-c306-8d3f-8c56e79e1295 df8655bc-1ecd-c306-8d3f-8c56e79e1295}" ---- PASS: TestRaft_GetConfigurationNoBootstrap (0.29s) +2025-06-17T07:44:38.540Z [INFO] server-c13d1fea-4b1d-31f7-21b3-14665e39faba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c13d1fea-4b1d-31f7-21b3-14665e39faba Address:c13d1fea-4b1d-31f7-21b3-14665e39faba} {Suffrage:Voter ID:server-ebea25e0-6b13-4168-5740-beb19d5e3776 Address:ebea25e0-6b13-4168-5740-beb19d5e3776}]" +2025-06-17T07:44:38.540Z [INFO] server-c13d1fea-4b1d-31f7-21b3-14665e39faba: entering follower state: follower="Node at c13d1fea-4b1d-31f7-21b3-14665e39faba [Follower]" leader-address= leader-id= +2025-06-17T07:44:38.540Z [INFO] server-ebea25e0-6b13-4168-5740-beb19d5e3776: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c13d1fea-4b1d-31f7-21b3-14665e39faba Address:c13d1fea-4b1d-31f7-21b3-14665e39faba} {Suffrage:Voter ID:server-ebea25e0-6b13-4168-5740-beb19d5e3776 Address:ebea25e0-6b13-4168-5740-beb19d5e3776}]" +2025-06-17T07:44:38.540Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:38.540Z [INFO] server-ebea25e0-6b13-4168-5740-beb19d5e3776: entering follower state: follower="Node at ebea25e0-6b13-4168-5740-beb19d5e3776 [Follower]" leader-address= leader-id= +2025-06-17T07:44:38.621Z [WARN] server-c13d1fea-4b1d-31f7-21b3-14665e39faba: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:38.621Z [INFO] server-c13d1fea-4b1d-31f7-21b3-14665e39faba: entering candidate state: node="Node at c13d1fea-4b1d-31f7-21b3-14665e39faba [Candidate]" term=2 +2025-06-17T07:44:38.621Z [INFO] server-c13d1fea-4b1d-31f7-21b3-14665e39faba: election won: term=2 tally=2 +2025-06-17T07:44:38.621Z [INFO] server-c13d1fea-4b1d-31f7-21b3-14665e39faba: entering leader state: leader="Node at c13d1fea-4b1d-31f7-21b3-14665e39faba [Leader]" +2025-06-17T07:44:38.621Z [INFO] server-c13d1fea-4b1d-31f7-21b3-14665e39faba: added peer, starting replication: peer=server-ebea25e0-6b13-4168-5740-beb19d5e3776 +2025-06-17T07:44:38.621Z [INFO] server-c13d1fea-4b1d-31f7-21b3-14665e39faba: pipelining replication: peer="{Voter server-ebea25e0-6b13-4168-5740-beb19d5e3776 ebea25e0-6b13-4168-5740-beb19d5e3776}" +2025-06-17T07:44:38.726Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:38.621545554 +0000 UTC m=+107.225074273 (1 nodes), 81.016085ms from start of poll, 81.216208ms from cluster start. Timeout at 2025-06-17 07:44:38.726575208 +0000 UTC m=+107.330103923, 105.02965ms after stability +2025-06-17T07:44:38.726Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:38.831Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:38.726648616 +0000 UTC m=+107.330177305 (1 nodes), 664ns from start of poll, 186.31924ms from cluster start. Timeout at 2025-06-17 07:44:38.831816044 +0000 UTC m=+107.435344730, 105.167425ms after stability +2025-06-17T07:44:38.843Z [INFO] server-ebea25e0-6b13-4168-5740-beb19d5e3776: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c13d1fea-4b1d-31f7-21b3-14665e39faba Address:c13d1fea-4b1d-31f7-21b3-14665e39faba} {Suffrage:Voter ID:server-ebea25e0-6b13-4168-5740-beb19d5e3776 Address:ebea25e0-6b13-4168-5740-beb19d5e3776}]" +2025-06-17T07:44:38.843Z [INFO] server-c13d1fea-4b1d-31f7-21b3-14665e39faba: aborting pipeline replication: peer="{Voter server-ebea25e0-6b13-4168-5740-beb19d5e3776 ebea25e0-6b13-4168-5740-beb19d5e3776}" +--- PASS: TestRaft_GetConfigurationNoBootstrap (0.30s) === RUN TestRaft_LogStoreIsMonotonic -2024-05-15T01:18:47.588Z [INFO] server-16a095cb-4233-fdbd-a6e0-e4fbde90820b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16a095cb-4233-fdbd-a6e0-e4fbde90820b Address:16a095cb-4233-fdbd-a6e0-e4fbde90820b}]" -2024-05-15T01:18:47.588Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:47.588Z [INFO] server-16a095cb-4233-fdbd-a6e0-e4fbde90820b: entering follower state: follower="Node at 16a095cb-4233-fdbd-a6e0-e4fbde90820b [Follower]" leader-address= leader-id= -2024-05-15T01:18:47.672Z [WARN] server-16a095cb-4233-fdbd-a6e0-e4fbde90820b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:47.672Z [INFO] server-16a095cb-4233-fdbd-a6e0-e4fbde90820b: entering candidate state: node="Node at 16a095cb-4233-fdbd-a6e0-e4fbde90820b [Candidate]" term=2 -2024-05-15T01:18:47.672Z [INFO] server-16a095cb-4233-fdbd-a6e0-e4fbde90820b: election won: term=2 tally=1 -2024-05-15T01:18:47.672Z [INFO] server-16a095cb-4233-fdbd-a6e0-e4fbde90820b: entering leader state: leader="Node at 16a095cb-4233-fdbd-a6e0-e4fbde90820b [Leader]" -2024-05-15T01:18:47.779Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:47.672855848 +0000 UTC m=+108.214240971 (1 nodes), 84.270229ms from start of poll, 84.489502ms from cluster start. Timeout at 2024-05-15 01:18:47.778971167 +0000 UTC m=+108.320356158, 106.115187ms after stability ---- PASS: TestRaft_LogStoreIsMonotonic (0.19s) +2025-06-17T07:44:38.844Z [INFO] server-0704c5f9-a7bf-6a00-61d1-3b331e76db23: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0704c5f9-a7bf-6a00-61d1-3b331e76db23 Address:0704c5f9-a7bf-6a00-61d1-3b331e76db23}]" +2025-06-17T07:44:38.844Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:38.844Z [INFO] server-0704c5f9-a7bf-6a00-61d1-3b331e76db23: entering follower state: follower="Node at 0704c5f9-a7bf-6a00-61d1-3b331e76db23 [Follower]" leader-address= leader-id= +2025-06-17T07:44:38.898Z [WARN] server-0704c5f9-a7bf-6a00-61d1-3b331e76db23: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:38.898Z [INFO] server-0704c5f9-a7bf-6a00-61d1-3b331e76db23: entering candidate state: node="Node at 0704c5f9-a7bf-6a00-61d1-3b331e76db23 [Candidate]" term=2 +2025-06-17T07:44:38.898Z [INFO] server-0704c5f9-a7bf-6a00-61d1-3b331e76db23: election won: term=2 tally=1 +2025-06-17T07:44:38.898Z [INFO] server-0704c5f9-a7bf-6a00-61d1-3b331e76db23: entering leader state: leader="Node at 0704c5f9-a7bf-6a00-61d1-3b331e76db23 [Leader]" +2025-06-17T07:44:39.004Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:38.89899529 +0000 UTC m=+107.502523976 (1 nodes), 54.506551ms from start of poll, 54.614984ms from cluster start. Timeout at 2025-06-17 07:44:39.00477928 +0000 UTC m=+107.608307965, 105.783989ms after stability +--- PASS: TestRaft_LogStoreIsMonotonic (0.16s) === RUN TestRaft_CacheLogWithStoreError -2024-05-15T01:18:47.780Z [INFO] server-1cde338d-df08-c57c-f978-87ae5256fc6a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1cde338d-df08-c57c-f978-87ae5256fc6a Address:1cde338d-df08-c57c-f978-87ae5256fc6a} {Suffrage:Voter ID:server-4b8509b2-83a8-deaf-4752-41ec926093c3 Address:4b8509b2-83a8-deaf-4752-41ec926093c3}]" -2024-05-15T01:18:47.780Z [INFO] server-4b8509b2-83a8-deaf-4752-41ec926093c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1cde338d-df08-c57c-f978-87ae5256fc6a Address:1cde338d-df08-c57c-f978-87ae5256fc6a} {Suffrage:Voter ID:server-4b8509b2-83a8-deaf-4752-41ec926093c3 Address:4b8509b2-83a8-deaf-4752-41ec926093c3}]" -2024-05-15T01:18:47.780Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:47.780Z [INFO] server-1cde338d-df08-c57c-f978-87ae5256fc6a: entering follower state: follower="Node at 1cde338d-df08-c57c-f978-87ae5256fc6a [Follower]" leader-address= leader-id= -2024-05-15T01:18:47.780Z [INFO] server-4b8509b2-83a8-deaf-4752-41ec926093c3: entering follower state: follower="Node at 4b8509b2-83a8-deaf-4752-41ec926093c3 [Follower]" leader-address= leader-id= -2024-05-15T01:18:47.859Z [WARN] server-1cde338d-df08-c57c-f978-87ae5256fc6a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:47.859Z [INFO] server-1cde338d-df08-c57c-f978-87ae5256fc6a: entering candidate state: node="Node at 1cde338d-df08-c57c-f978-87ae5256fc6a [Candidate]" term=2 -2024-05-15T01:18:47.860Z [INFO] server-1cde338d-df08-c57c-f978-87ae5256fc6a: election won: term=2 tally=2 -2024-05-15T01:18:47.860Z [INFO] server-1cde338d-df08-c57c-f978-87ae5256fc6a: entering leader state: leader="Node at 1cde338d-df08-c57c-f978-87ae5256fc6a [Leader]" -2024-05-15T01:18:47.860Z [INFO] server-1cde338d-df08-c57c-f978-87ae5256fc6a: added peer, starting replication: peer=server-4b8509b2-83a8-deaf-4752-41ec926093c3 -2024-05-15T01:18:47.860Z [INFO] server-1cde338d-df08-c57c-f978-87ae5256fc6a: pipelining replication: peer="{Voter server-4b8509b2-83a8-deaf-4752-41ec926093c3 4b8509b2-83a8-deaf-4752-41ec926093c3}" -2024-05-15T01:18:47.966Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:47.860915757 +0000 UTC m=+108.402300715 (1 nodes), 80.153487ms from start of poll, 80.5978ms from cluster start. Timeout at 2024-05-15 01:18:47.96596207 +0000 UTC m=+108.507347039, 105.046324ms after stability -2024-05-15T01:18:47.966Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:48.073Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:47.966072806 +0000 UTC m=+108.507457634 (1 nodes), 1.954µs from start of poll, 185.754719ms from cluster start. Timeout at 2024-05-15 01:18:48.071594228 +0000 UTC m=+108.612978983, 105.521349ms after stability -2024-05-15T01:18:48.074Z [ERROR] server-4b8509b2-83a8-deaf-4752-41ec926093c3: failed to append to logs: error="unable to store logs within log store, err: "some error"" -2024-05-15T01:18:48.074Z [WARN] server-4b8509b2-83a8-deaf-4752-41ec926093c3: failed to get previous log: previous-index=9 last-index=7 error="log not found" -2024-05-15T01:18:48.074Z [INFO] server-1cde338d-df08-c57c-f978-87ae5256fc6a: aborting pipeline replication: peer="{Voter server-4b8509b2-83a8-deaf-4752-41ec926093c3 4b8509b2-83a8-deaf-4752-41ec926093c3}" -2024-05-15T01:18:48.081Z [INFO] server-1cde338d-df08-c57c-f978-87ae5256fc6a: pipelining replication: peer="{Voter server-4b8509b2-83a8-deaf-4752-41ec926093c3 4b8509b2-83a8-deaf-4752-41ec926093c3}" -2024-05-15T01:18:48.081Z [INFO] server-4b8509b2-83a8-deaf-4752-41ec926093c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1cde338d-df08-c57c-f978-87ae5256fc6a Address:1cde338d-df08-c57c-f978-87ae5256fc6a} {Suffrage:Voter ID:server-4b8509b2-83a8-deaf-4752-41ec926093c3 Address:4b8509b2-83a8-deaf-4752-41ec926093c3}]" -2024-05-15T01:18:48.082Z [INFO] server-1cde338d-df08-c57c-f978-87ae5256fc6a: aborting pipeline replication: peer="{Voter server-4b8509b2-83a8-deaf-4752-41ec926093c3 4b8509b2-83a8-deaf-4752-41ec926093c3}" ---- PASS: TestRaft_CacheLogWithStoreError (0.30s) +2025-06-17T07:44:39.005Z [INFO] server-96ed30bb-00ab-edb8-23d2-a950e1f1787a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-96ed30bb-00ab-edb8-23d2-a950e1f1787a Address:96ed30bb-00ab-edb8-23d2-a950e1f1787a} {Suffrage:Voter ID:server-b0c39ccf-2415-93a7-4ad3-45f8adfd554d Address:b0c39ccf-2415-93a7-4ad3-45f8adfd554d}]" +2025-06-17T07:44:39.005Z [INFO] server-96ed30bb-00ab-edb8-23d2-a950e1f1787a: entering follower state: follower="Node at 96ed30bb-00ab-edb8-23d2-a950e1f1787a [Follower]" leader-address= leader-id= +2025-06-17T07:44:39.005Z [INFO] server-b0c39ccf-2415-93a7-4ad3-45f8adfd554d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-96ed30bb-00ab-edb8-23d2-a950e1f1787a Address:96ed30bb-00ab-edb8-23d2-a950e1f1787a} {Suffrage:Voter ID:server-b0c39ccf-2415-93a7-4ad3-45f8adfd554d Address:b0c39ccf-2415-93a7-4ad3-45f8adfd554d}]" +2025-06-17T07:44:39.005Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:39.005Z [INFO] server-b0c39ccf-2415-93a7-4ad3-45f8adfd554d: entering follower state: follower="Node at b0c39ccf-2415-93a7-4ad3-45f8adfd554d [Follower]" leader-address= leader-id= +2025-06-17T07:44:39.080Z [WARN] server-b0c39ccf-2415-93a7-4ad3-45f8adfd554d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:39.080Z [INFO] server-b0c39ccf-2415-93a7-4ad3-45f8adfd554d: entering candidate state: node="Node at b0c39ccf-2415-93a7-4ad3-45f8adfd554d [Candidate]" term=2 +2025-06-17T07:44:39.080Z [INFO] server-b0c39ccf-2415-93a7-4ad3-45f8adfd554d: election won: term=2 tally=2 +2025-06-17T07:44:39.080Z [INFO] server-b0c39ccf-2415-93a7-4ad3-45f8adfd554d: entering leader state: leader="Node at b0c39ccf-2415-93a7-4ad3-45f8adfd554d [Leader]" +2025-06-17T07:44:39.080Z [INFO] server-b0c39ccf-2415-93a7-4ad3-45f8adfd554d: added peer, starting replication: peer=server-96ed30bb-00ab-edb8-23d2-a950e1f1787a +2025-06-17T07:44:39.080Z [INFO] server-b0c39ccf-2415-93a7-4ad3-45f8adfd554d: pipelining replication: peer="{Voter server-96ed30bb-00ab-edb8-23d2-a950e1f1787a 96ed30bb-00ab-edb8-23d2-a950e1f1787a}" +2025-06-17T07:44:39.186Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:39.080873518 +0000 UTC m=+107.684402218 (1 nodes), 75.207301ms from start of poll, 75.433443ms from cluster start. Timeout at 2025-06-17 07:44:39.186006396 +0000 UTC m=+107.789535088, 105.13287ms after stability +2025-06-17T07:44:39.186Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:39.291Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:39.186091305 +0000 UTC m=+107.789620009 (1 nodes), 875ns from start of poll, 180.651234ms from cluster start. Timeout at 2025-06-17 07:44:39.291664292 +0000 UTC m=+107.895192989, 105.57298ms after stability +2025-06-17T07:44:39.292Z [ERROR] server-96ed30bb-00ab-edb8-23d2-a950e1f1787a: failed to append to logs: error="unable to store logs within log store, err: "some error"" +2025-06-17T07:44:39.292Z [WARN] server-96ed30bb-00ab-edb8-23d2-a950e1f1787a: failed to get previous log: previous-index=9 last-index=7 error="log not found" +2025-06-17T07:44:39.292Z [INFO] server-b0c39ccf-2415-93a7-4ad3-45f8adfd554d: aborting pipeline replication: peer="{Voter server-96ed30bb-00ab-edb8-23d2-a950e1f1787a 96ed30bb-00ab-edb8-23d2-a950e1f1787a}" +2025-06-17T07:44:39.299Z [INFO] server-b0c39ccf-2415-93a7-4ad3-45f8adfd554d: pipelining replication: peer="{Voter server-96ed30bb-00ab-edb8-23d2-a950e1f1787a 96ed30bb-00ab-edb8-23d2-a950e1f1787a}" +2025-06-17T07:44:39.299Z [INFO] server-96ed30bb-00ab-edb8-23d2-a950e1f1787a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-96ed30bb-00ab-edb8-23d2-a950e1f1787a Address:96ed30bb-00ab-edb8-23d2-a950e1f1787a} {Suffrage:Voter ID:server-b0c39ccf-2415-93a7-4ad3-45f8adfd554d Address:b0c39ccf-2415-93a7-4ad3-45f8adfd554d}]" +2025-06-17T07:44:39.299Z [INFO] server-b0c39ccf-2415-93a7-4ad3-45f8adfd554d: aborting pipeline replication: peer="{Voter server-96ed30bb-00ab-edb8-23d2-a950e1f1787a 96ed30bb-00ab-edb8-23d2-a950e1f1787a}" +--- PASS: TestRaft_CacheLogWithStoreError (0.29s) === RUN TestRaft_ReloadConfig -2024-05-15T01:18:48.083Z [INFO] server-3d63b820-ca09-5064-cd93-6c616ca013bc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3d63b820-ca09-5064-cd93-6c616ca013bc Address:3d63b820-ca09-5064-cd93-6c616ca013bc}]" +2025-06-17T07:44:39.299Z [INFO] server-acc12bc9-e1bf-119f-6037-d4e8ae6d698e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-acc12bc9-e1bf-119f-6037-d4e8ae6d698e Address:acc12bc9-e1bf-119f-6037-d4e8ae6d698e}]" +2025-06-17T07:44:39.300Z [INFO] server-acc12bc9-e1bf-119f-6037-d4e8ae6d698e: entering follower state: follower="Node at acc12bc9-e1bf-119f-6037-d4e8ae6d698e [Shutdown]" leader-address= leader-id= --- PASS: TestRaft_ReloadConfig (0.00s) === RUN TestRaft_ReloadConfigValidates -2024-05-15T01:18:48.083Z [INFO] server-8c79631c-c9aa-64c2-2f24-b751482456d8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8c79631c-c9aa-64c2-2f24-b751482456d8 Address:8c79631c-c9aa-64c2-2f24-b751482456d8}]" +2025-06-17T07:44:39.300Z [INFO] server-08d4af34-91d5-1bde-0985-7a7a0d83618c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08d4af34-91d5-1bde-0985-7a7a0d83618c Address:08d4af34-91d5-1bde-0985-7a7a0d83618c}]" +2025-06-17T07:44:39.300Z [INFO] server-08d4af34-91d5-1bde-0985-7a7a0d83618c: entering follower state: follower="Node at 08d4af34-91d5-1bde-0985-7a7a0d83618c [Shutdown]" leader-address= leader-id= --- PASS: TestRaft_ReloadConfigValidates (0.00s) === RUN TestRaft_InstallSnapshot_InvalidPeers -2024-05-15T01:18:48.084Z [ERROR] failed to install snapshot: error="failed to decode peers: codec.decoder: Only encoded map or array can be decoded into a slice. (valueType: 2)" +2025-06-17T07:44:39.300Z [ERROR] failed to install snapshot: error="failed to decode peers: codec.decoder: Only encoded map or array can be decoded into a slice. (valueType: 2)" --- PASS: TestRaft_InstallSnapshot_InvalidPeers (0.00s) === RUN TestRaft_VoteNotGranted_WhenNodeNotInCluster -2024-05-15T01:18:48.085Z [INFO] server-4d7cf6c9-efc9-37cd-141c-6a10b596d7d4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d7cf6c9-efc9-37cd-141c-6a10b596d7d4 Address:4d7cf6c9-efc9-37cd-141c-6a10b596d7d4} {Suffrage:Voter ID:server-b66db6a6-7a3c-9ab2-535b-f1edcae087d5 Address:b66db6a6-7a3c-9ab2-535b-f1edcae087d5} {Suffrage:Voter ID:server-37af9e2b-a617-cab8-1fef-3a1727b79e30 Address:37af9e2b-a617-cab8-1fef-3a1727b79e30}]" -2024-05-15T01:18:48.086Z [INFO] server-b66db6a6-7a3c-9ab2-535b-f1edcae087d5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d7cf6c9-efc9-37cd-141c-6a10b596d7d4 Address:4d7cf6c9-efc9-37cd-141c-6a10b596d7d4} {Suffrage:Voter ID:server-b66db6a6-7a3c-9ab2-535b-f1edcae087d5 Address:b66db6a6-7a3c-9ab2-535b-f1edcae087d5} {Suffrage:Voter ID:server-37af9e2b-a617-cab8-1fef-3a1727b79e30 Address:37af9e2b-a617-cab8-1fef-3a1727b79e30}]" -2024-05-15T01:18:48.086Z [INFO] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d7cf6c9-efc9-37cd-141c-6a10b596d7d4 Address:4d7cf6c9-efc9-37cd-141c-6a10b596d7d4} {Suffrage:Voter ID:server-b66db6a6-7a3c-9ab2-535b-f1edcae087d5 Address:b66db6a6-7a3c-9ab2-535b-f1edcae087d5} {Suffrage:Voter ID:server-37af9e2b-a617-cab8-1fef-3a1727b79e30 Address:37af9e2b-a617-cab8-1fef-3a1727b79e30}]" -2024-05-15T01:18:48.086Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:48.086Z [INFO] server-4d7cf6c9-efc9-37cd-141c-6a10b596d7d4: entering follower state: follower="Node at 4d7cf6c9-efc9-37cd-141c-6a10b596d7d4 [Follower]" leader-address= leader-id= -2024-05-15T01:18:48.086Z [INFO] server-b66db6a6-7a3c-9ab2-535b-f1edcae087d5: entering follower state: follower="Node at b66db6a6-7a3c-9ab2-535b-f1edcae087d5 [Follower]" leader-address= leader-id= -2024-05-15T01:18:48.086Z [INFO] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: entering follower state: follower="Node at 37af9e2b-a617-cab8-1fef-3a1727b79e30 [Follower]" leader-address= leader-id= -2024-05-15T01:18:48.136Z [WARN] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:48.136Z [INFO] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: entering candidate state: node="Node at 37af9e2b-a617-cab8-1fef-3a1727b79e30 [Candidate]" term=2 -2024-05-15T01:18:48.136Z [INFO] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: election won: term=2 tally=2 -2024-05-15T01:18:48.136Z [INFO] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: entering leader state: leader="Node at 37af9e2b-a617-cab8-1fef-3a1727b79e30 [Leader]" -2024-05-15T01:18:48.137Z [INFO] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: added peer, starting replication: peer=server-4d7cf6c9-efc9-37cd-141c-6a10b596d7d4 -2024-05-15T01:18:48.137Z [INFO] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: added peer, starting replication: peer=server-b66db6a6-7a3c-9ab2-535b-f1edcae087d5 -2024-05-15T01:18:48.137Z [INFO] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: pipelining replication: peer="{Voter server-b66db6a6-7a3c-9ab2-535b-f1edcae087d5 b66db6a6-7a3c-9ab2-535b-f1edcae087d5}" -2024-05-15T01:18:48.137Z [INFO] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: pipelining replication: peer="{Voter server-4d7cf6c9-efc9-37cd-141c-6a10b596d7d4 4d7cf6c9-efc9-37cd-141c-6a10b596d7d4}" -2024-05-15T01:18:48.243Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:48.137545029 +0000 UTC m=+108.678929783 (1 nodes), 51.181627ms from start of poll, 52.527842ms from cluster start. Timeout at 2024-05-15 01:18:48.243512305 +0000 UTC m=+108.784897057, 105.967274ms after stability -2024-05-15T01:18:48.254Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:48.364Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:48.257886917 +0000 UTC m=+108.799271659 (2 nodes), 2.09µs from start of poll, 172.869718ms from cluster start. Timeout at 2024-05-15 01:18:48.364846714 +0000 UTC m=+108.906231457, 106.959798ms after stability -2024-05-15T01:18:48.364Z [INFO] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: updating configuration: command=RemoveServer server-id=server-4d7cf6c9-efc9-37cd-141c-6a10b596d7d4 server-addr= servers="[{Suffrage:Voter ID:server-b66db6a6-7a3c-9ab2-535b-f1edcae087d5 Address:b66db6a6-7a3c-9ab2-535b-f1edcae087d5} {Suffrage:Voter ID:server-37af9e2b-a617-cab8-1fef-3a1727b79e30 Address:37af9e2b-a617-cab8-1fef-3a1727b79e30}]" -2024-05-15T01:18:48.365Z [INFO] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: removed peer, stopping replication: peer=server-4d7cf6c9-efc9-37cd-141c-6a10b596d7d4 last-index=3 -2024-05-15T01:18:48.365Z [INFO] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: aborting pipeline replication: peer="{Voter server-4d7cf6c9-efc9-37cd-141c-6a10b596d7d4 4d7cf6c9-efc9-37cd-141c-6a10b596d7d4}" -2024-05-15T01:18:48.418Z [WARN] server-4d7cf6c9-efc9-37cd-141c-6a10b596d7d4: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2024-05-15T01:18:48.475Z [ERROR] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: failed to pipeline appendEntries: peer="{Voter server-b66db6a6-7a3c-9ab2-535b-f1edcae087d5 b66db6a6-7a3c-9ab2-535b-f1edcae087d5}" error="append pipeline closed" -2024-05-15T01:18:48.475Z [INFO] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: aborting pipeline replication: peer="{Voter server-b66db6a6-7a3c-9ab2-535b-f1edcae087d5 b66db6a6-7a3c-9ab2-535b-f1edcae087d5}" -2024-05-15T01:18:48.476Z [ERROR] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: failed to heartbeat to: peer=b66db6a6-7a3c-9ab2-535b-f1edcae087d5 backoff time=10ms error="failed to connect to peer: b66db6a6-7a3c-9ab2-535b-f1edcae087d5" -2024-05-15T01:18:48.485Z [ERROR] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: failed to appendEntries to: peer="{Voter server-b66db6a6-7a3c-9ab2-535b-f1edcae087d5 b66db6a6-7a3c-9ab2-535b-f1edcae087d5}" error="failed to connect to peer: b66db6a6-7a3c-9ab2-535b-f1edcae087d5" -2024-05-15T01:18:48.495Z [ERROR] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: failed to heartbeat to: peer=b66db6a6-7a3c-9ab2-535b-f1edcae087d5 backoff time=10ms error="failed to connect to peer: b66db6a6-7a3c-9ab2-535b-f1edcae087d5" -2024-05-15T01:18:48.506Z [ERROR] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: failed to appendEntries to: peer="{Voter server-b66db6a6-7a3c-9ab2-535b-f1edcae087d5 b66db6a6-7a3c-9ab2-535b-f1edcae087d5}" error="failed to connect to peer: b66db6a6-7a3c-9ab2-535b-f1edcae087d5" -2024-05-15T01:18:48.513Z [ERROR] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: failed to heartbeat to: peer=b66db6a6-7a3c-9ab2-535b-f1edcae087d5 backoff time=10ms error="failed to connect to peer: b66db6a6-7a3c-9ab2-535b-f1edcae087d5" -2024-05-15T01:18:48.520Z [WARN] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: failed to contact: server-id=server-b66db6a6-7a3c-9ab2-535b-f1edcae087d5 time=50.183384ms -2024-05-15T01:18:48.520Z [WARN] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: failed to contact quorum of nodes, stepping down -2024-05-15T01:18:48.520Z [INFO] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: entering follower state: follower="Node at 37af9e2b-a617-cab8-1fef-3a1727b79e30 [Follower]" leader-address= leader-id= -2024-05-15T01:18:48.525Z [ERROR] server-37af9e2b-a617-cab8-1fef-3a1727b79e30: failed to appendEntries to: peer="{Voter server-b66db6a6-7a3c-9ab2-535b-f1edcae087d5 b66db6a6-7a3c-9ab2-535b-f1edcae087d5}" error="failed to connect to peer: b66db6a6-7a3c-9ab2-535b-f1edcae087d5" -2024-05-15T01:18:48.532Z [WARN] server-b66db6a6-7a3c-9ab2-535b-f1edcae087d5: heartbeat timeout reached, starting election: last-leader-addr=37af9e2b-a617-cab8-1fef-3a1727b79e30 last-leader-id=server-37af9e2b-a617-cab8-1fef-3a1727b79e30 -2024-05-15T01:18:48.532Z [INFO] server-b66db6a6-7a3c-9ab2-535b-f1edcae087d5: entering candidate state: node="Node at b66db6a6-7a3c-9ab2-535b-f1edcae087d5 [Candidate]" term=3 -2024-05-15T01:18:48.532Z [ERROR] server-b66db6a6-7a3c-9ab2-535b-f1edcae087d5: failed to make requestVote RPC: target="{Voter server-37af9e2b-a617-cab8-1fef-3a1727b79e30 37af9e2b-a617-cab8-1fef-3a1727b79e30}" error="failed to connect to peer: 37af9e2b-a617-cab8-1fef-3a1727b79e30" term=3 -2024-05-15T01:18:48.575Z [WARN] server-b66db6a6-7a3c-9ab2-535b-f1edcae087d5: rejecting vote request since node is not in configuration: from=4d7cf6c9-efc9-37cd-141c-6a10b596d7d4 ---- PASS: TestRaft_VoteNotGranted_WhenNodeNotInCluster (0.49s) +2025-06-17T07:44:39.301Z [INFO] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6eeeaa70-c54a-91e2-c093-0267275f65dd Address:6eeeaa70-c54a-91e2-c093-0267275f65dd} {Suffrage:Voter ID:server-e39f35b7-014d-6030-b6d2-f22a7e584551 Address:e39f35b7-014d-6030-b6d2-f22a7e584551} {Suffrage:Voter ID:server-5b70c6c8-0cfc-00ce-3bde-78827381f20d Address:5b70c6c8-0cfc-00ce-3bde-78827381f20d}]" +2025-06-17T07:44:39.301Z [INFO] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: entering follower state: follower="Node at 6eeeaa70-c54a-91e2-c093-0267275f65dd [Follower]" leader-address= leader-id= +2025-06-17T07:44:39.301Z [INFO] server-e39f35b7-014d-6030-b6d2-f22a7e584551: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6eeeaa70-c54a-91e2-c093-0267275f65dd Address:6eeeaa70-c54a-91e2-c093-0267275f65dd} {Suffrage:Voter ID:server-e39f35b7-014d-6030-b6d2-f22a7e584551 Address:e39f35b7-014d-6030-b6d2-f22a7e584551} {Suffrage:Voter ID:server-5b70c6c8-0cfc-00ce-3bde-78827381f20d Address:5b70c6c8-0cfc-00ce-3bde-78827381f20d}]" +2025-06-17T07:44:39.301Z [INFO] server-e39f35b7-014d-6030-b6d2-f22a7e584551: entering follower state: follower="Node at e39f35b7-014d-6030-b6d2-f22a7e584551 [Follower]" leader-address= leader-id= +2025-06-17T07:44:39.301Z [INFO] server-5b70c6c8-0cfc-00ce-3bde-78827381f20d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6eeeaa70-c54a-91e2-c093-0267275f65dd Address:6eeeaa70-c54a-91e2-c093-0267275f65dd} {Suffrage:Voter ID:server-e39f35b7-014d-6030-b6d2-f22a7e584551 Address:e39f35b7-014d-6030-b6d2-f22a7e584551} {Suffrage:Voter ID:server-5b70c6c8-0cfc-00ce-3bde-78827381f20d Address:5b70c6c8-0cfc-00ce-3bde-78827381f20d}]" +2025-06-17T07:44:39.301Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:39.301Z [INFO] server-5b70c6c8-0cfc-00ce-3bde-78827381f20d: entering follower state: follower="Node at 5b70c6c8-0cfc-00ce-3bde-78827381f20d [Follower]" leader-address= leader-id= +2025-06-17T07:44:39.376Z [WARN] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:39.376Z [INFO] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: entering candidate state: node="Node at 6eeeaa70-c54a-91e2-c093-0267275f65dd [Candidate]" term=2 +2025-06-17T07:44:39.377Z [INFO] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: election won: term=2 tally=2 +2025-06-17T07:44:39.377Z [INFO] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: entering leader state: leader="Node at 6eeeaa70-c54a-91e2-c093-0267275f65dd [Leader]" +2025-06-17T07:44:39.377Z [INFO] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: added peer, starting replication: peer=server-e39f35b7-014d-6030-b6d2-f22a7e584551 +2025-06-17T07:44:39.377Z [INFO] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: added peer, starting replication: peer=server-5b70c6c8-0cfc-00ce-3bde-78827381f20d +2025-06-17T07:44:39.377Z [INFO] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: pipelining replication: peer="{Voter server-5b70c6c8-0cfc-00ce-3bde-78827381f20d 5b70c6c8-0cfc-00ce-3bde-78827381f20d}" +2025-06-17T07:44:39.377Z [INFO] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: pipelining replication: peer="{Voter server-e39f35b7-014d-6030-b6d2-f22a7e584551 e39f35b7-014d-6030-b6d2-f22a7e584551}" +2025-06-17T07:44:39.483Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:39.377094043 +0000 UTC m=+107.980622748 (1 nodes), 75.480198ms from start of poll, 75.782427ms from cluster start. Timeout at 2025-06-17 07:44:39.482960719 +0000 UTC m=+108.086489409, 105.866661ms after stability +2025-06-17T07:44:39.488Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:39.594Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:39.489000167 +0000 UTC m=+108.092528854 (2 nodes), 1.075µs from start of poll, 187.688533ms from cluster start. Timeout at 2025-06-17 07:44:39.594509899 +0000 UTC m=+108.198038581, 105.509727ms after stability +2025-06-17T07:44:39.594Z [INFO] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: updating configuration: command=RemoveServer server-id=server-e39f35b7-014d-6030-b6d2-f22a7e584551 server-addr= servers="[{Suffrage:Voter ID:server-6eeeaa70-c54a-91e2-c093-0267275f65dd Address:6eeeaa70-c54a-91e2-c093-0267275f65dd} {Suffrage:Voter ID:server-5b70c6c8-0cfc-00ce-3bde-78827381f20d Address:5b70c6c8-0cfc-00ce-3bde-78827381f20d}]" +2025-06-17T07:44:39.594Z [INFO] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: removed peer, stopping replication: peer=server-e39f35b7-014d-6030-b6d2-f22a7e584551 last-index=3 +2025-06-17T07:44:39.594Z [INFO] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: aborting pipeline replication: peer="{Voter server-e39f35b7-014d-6030-b6d2-f22a7e584551 e39f35b7-014d-6030-b6d2-f22a7e584551}" +2025-06-17T07:44:39.650Z [WARN] server-e39f35b7-014d-6030-b6d2-f22a7e584551: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-06-17T07:44:39.705Z [ERROR] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: failed to heartbeat to: peer=5b70c6c8-0cfc-00ce-3bde-78827381f20d backoff time=10ms error="failed to connect to peer: 5b70c6c8-0cfc-00ce-3bde-78827381f20d" +2025-06-17T07:44:39.705Z [ERROR] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: failed to pipeline appendEntries: peer="{Voter server-5b70c6c8-0cfc-00ce-3bde-78827381f20d 5b70c6c8-0cfc-00ce-3bde-78827381f20d}" error="append pipeline closed" +2025-06-17T07:44:39.705Z [INFO] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: aborting pipeline replication: peer="{Voter server-5b70c6c8-0cfc-00ce-3bde-78827381f20d 5b70c6c8-0cfc-00ce-3bde-78827381f20d}" +2025-06-17T07:44:39.712Z [ERROR] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: failed to appendEntries to: peer="{Voter server-5b70c6c8-0cfc-00ce-3bde-78827381f20d 5b70c6c8-0cfc-00ce-3bde-78827381f20d}" error="failed to connect to peer: 5b70c6c8-0cfc-00ce-3bde-78827381f20d" +2025-06-17T07:44:39.722Z [ERROR] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: failed to heartbeat to: peer=5b70c6c8-0cfc-00ce-3bde-78827381f20d backoff time=10ms error="failed to connect to peer: 5b70c6c8-0cfc-00ce-3bde-78827381f20d" +2025-06-17T07:44:39.729Z [ERROR] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: failed to appendEntries to: peer="{Voter server-5b70c6c8-0cfc-00ce-3bde-78827381f20d 5b70c6c8-0cfc-00ce-3bde-78827381f20d}" error="failed to connect to peer: 5b70c6c8-0cfc-00ce-3bde-78827381f20d" +2025-06-17T07:44:39.742Z [ERROR] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: failed to heartbeat to: peer=5b70c6c8-0cfc-00ce-3bde-78827381f20d backoff time=10ms error="failed to connect to peer: 5b70c6c8-0cfc-00ce-3bde-78827381f20d" +2025-06-17T07:44:39.746Z [ERROR] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: failed to appendEntries to: peer="{Voter server-5b70c6c8-0cfc-00ce-3bde-78827381f20d 5b70c6c8-0cfc-00ce-3bde-78827381f20d}" error="failed to connect to peer: 5b70c6c8-0cfc-00ce-3bde-78827381f20d" +2025-06-17T07:44:39.748Z [WARN] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: failed to contact: server-id=server-5b70c6c8-0cfc-00ce-3bde-78827381f20d time=50.030727ms +2025-06-17T07:44:39.748Z [WARN] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: failed to contact quorum of nodes, stepping down +2025-06-17T07:44:39.748Z [INFO] server-6eeeaa70-c54a-91e2-c093-0267275f65dd: entering follower state: follower="Node at 6eeeaa70-c54a-91e2-c093-0267275f65dd [Follower]" leader-address= leader-id= +2025-06-17T07:44:39.807Z [WARN] server-5b70c6c8-0cfc-00ce-3bde-78827381f20d: heartbeat timeout reached, starting election: last-leader-addr=6eeeaa70-c54a-91e2-c093-0267275f65dd last-leader-id=server-6eeeaa70-c54a-91e2-c093-0267275f65dd +2025-06-17T07:44:39.807Z [INFO] server-5b70c6c8-0cfc-00ce-3bde-78827381f20d: entering candidate state: node="Node at 5b70c6c8-0cfc-00ce-3bde-78827381f20d [Candidate]" term=3 +2025-06-17T07:44:39.807Z [ERROR] server-5b70c6c8-0cfc-00ce-3bde-78827381f20d: failed to make requestVote RPC: target="{Voter server-6eeeaa70-c54a-91e2-c093-0267275f65dd 6eeeaa70-c54a-91e2-c093-0267275f65dd}" error="failed to connect to peer: 6eeeaa70-c54a-91e2-c093-0267275f65dd" term=3 +2025-06-17T07:44:39.808Z [WARN] server-5b70c6c8-0cfc-00ce-3bde-78827381f20d: rejecting vote request since node is not in configuration: from=e39f35b7-014d-6030-b6d2-f22a7e584551 +--- PASS: TestRaft_VoteNotGranted_WhenNodeNotInCluster (0.51s) === RUN TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin -2024-05-15T01:18:48.577Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff Address:caefbd7e-cffb-dc6a-3609-7f695a17b3ff} {Suffrage:Voter ID:server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec Address:42e2a768-bf9b-af1d-5c79-a793d0aa35ec} {Suffrage:Voter ID:server-00d70eb9-6c15-0748-5dc5-b88babcb7070 Address:00d70eb9-6c15-0748-5dc5-b88babcb7070}]" -2024-05-15T01:18:48.578Z [INFO] server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff Address:caefbd7e-cffb-dc6a-3609-7f695a17b3ff} {Suffrage:Voter ID:server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec Address:42e2a768-bf9b-af1d-5c79-a793d0aa35ec} {Suffrage:Voter ID:server-00d70eb9-6c15-0748-5dc5-b88babcb7070 Address:00d70eb9-6c15-0748-5dc5-b88babcb7070}]" -2024-05-15T01:18:48.578Z [INFO] server-00d70eb9-6c15-0748-5dc5-b88babcb7070: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff Address:caefbd7e-cffb-dc6a-3609-7f695a17b3ff} {Suffrage:Voter ID:server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec Address:42e2a768-bf9b-af1d-5c79-a793d0aa35ec} {Suffrage:Voter ID:server-00d70eb9-6c15-0748-5dc5-b88babcb7070 Address:00d70eb9-6c15-0748-5dc5-b88babcb7070}]" -2024-05-15T01:18:48.578Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:48.578Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: entering follower state: follower="Node at caefbd7e-cffb-dc6a-3609-7f695a17b3ff [Follower]" leader-address= leader-id= -2024-05-15T01:18:48.578Z [INFO] server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec: entering follower state: follower="Node at 42e2a768-bf9b-af1d-5c79-a793d0aa35ec [Follower]" leader-address= leader-id= -2024-05-15T01:18:48.578Z [INFO] server-00d70eb9-6c15-0748-5dc5-b88babcb7070: entering follower state: follower="Node at 00d70eb9-6c15-0748-5dc5-b88babcb7070 [Follower]" leader-address= leader-id= -2024-05-15T01:18:48.657Z [WARN] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:48.657Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: entering candidate state: node="Node at caefbd7e-cffb-dc6a-3609-7f695a17b3ff [Candidate]" term=2 -2024-05-15T01:18:48.657Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: election won: term=2 tally=2 -2024-05-15T01:18:48.657Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: entering leader state: leader="Node at caefbd7e-cffb-dc6a-3609-7f695a17b3ff [Leader]" -2024-05-15T01:18:48.657Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: added peer, starting replication: peer=server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec -2024-05-15T01:18:48.657Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: added peer, starting replication: peer=server-00d70eb9-6c15-0748-5dc5-b88babcb7070 -2024-05-15T01:18:48.657Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: pipelining replication: peer="{Voter server-00d70eb9-6c15-0748-5dc5-b88babcb7070 00d70eb9-6c15-0748-5dc5-b88babcb7070}" -2024-05-15T01:18:48.658Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: pipelining replication: peer="{Voter server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec 42e2a768-bf9b-af1d-5c79-a793d0aa35ec}" -2024-05-15T01:18:48.763Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:48.657966824 +0000 UTC m=+109.199351563 (1 nodes), 79.749904ms from start of poll, 80.298283ms from cluster start. Timeout at 2024-05-15 01:18:48.763175139 +0000 UTC m=+109.304559911, 105.208348ms after stability -2024-05-15T01:18:48.769Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:48.874Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:48.769095077 +0000 UTC m=+109.310479947 (2 nodes), 2.244µs from start of poll, 191.426667ms from cluster start. Timeout at 2024-05-15 01:18:48.874442032 +0000 UTC m=+109.415826780, 105.346833ms after stability -2024-05-15T01:18:48.880Z [ERROR] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: failed to pipeline appendEntries: peer="{Voter server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec 42e2a768-bf9b-af1d-5c79-a793d0aa35ec}" error="append pipeline closed" -2024-05-15T01:18:48.880Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: aborting pipeline replication: peer="{Voter server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec 42e2a768-bf9b-af1d-5c79-a793d0aa35ec}" -2024-05-15T01:18:48.880Z [ERROR] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: failed to heartbeat to: peer=42e2a768-bf9b-af1d-5c79-a793d0aa35ec backoff time=10ms error="failed to connect to peer: 42e2a768-bf9b-af1d-5c79-a793d0aa35ec" -2024-05-15T01:18:48.891Z [ERROR] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: failed to appendEntries to: peer="{Voter server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec 42e2a768-bf9b-af1d-5c79-a793d0aa35ec}" error="failed to connect to peer: 42e2a768-bf9b-af1d-5c79-a793d0aa35ec" -2024-05-15T01:18:48.902Z [ERROR] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: failed to heartbeat to: peer=42e2a768-bf9b-af1d-5c79-a793d0aa35ec backoff time=10ms error="failed to connect to peer: 42e2a768-bf9b-af1d-5c79-a793d0aa35ec" -2024-05-15T01:18:48.909Z [ERROR] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: failed to appendEntries to: peer="{Voter server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec 42e2a768-bf9b-af1d-5c79-a793d0aa35ec}" error="failed to connect to peer: 42e2a768-bf9b-af1d-5c79-a793d0aa35ec" -2024-05-15T01:18:48.920Z [ERROR] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: failed to heartbeat to: peer=42e2a768-bf9b-af1d-5c79-a793d0aa35ec backoff time=10ms error="failed to connect to peer: 42e2a768-bf9b-af1d-5c79-a793d0aa35ec" -2024-05-15T01:18:48.923Z [WARN] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: failed to contact: server-id=server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec time=50.163232ms -2024-05-15T01:18:48.927Z [ERROR] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: failed to appendEntries to: peer="{Voter server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec 42e2a768-bf9b-af1d-5c79-a793d0aa35ec}" error="failed to connect to peer: 42e2a768-bf9b-af1d-5c79-a793d0aa35ec" -2024-05-15T01:18:48.938Z [WARN] server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec: heartbeat timeout reached, starting election: last-leader-addr=caefbd7e-cffb-dc6a-3609-7f695a17b3ff last-leader-id=server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff -2024-05-15T01:18:48.938Z [INFO] server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec: entering candidate state: node="Node at 42e2a768-bf9b-af1d-5c79-a793d0aa35ec [Candidate]" term=3 -2024-05-15T01:18:48.938Z [ERROR] server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec: failed to make requestVote RPC: target="{Voter server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff caefbd7e-cffb-dc6a-3609-7f695a17b3ff}" error="failed to connect to peer: caefbd7e-cffb-dc6a-3609-7f695a17b3ff" term=3 -2024-05-15T01:18:48.938Z [ERROR] server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec: failed to make requestVote RPC: target="{Voter server-00d70eb9-6c15-0748-5dc5-b88babcb7070 00d70eb9-6c15-0748-5dc5-b88babcb7070}" error="failed to connect to peer: 00d70eb9-6c15-0748-5dc5-b88babcb7070" term=3 -2024-05-15T01:18:48.941Z [ERROR] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: failed to heartbeat to: peer=42e2a768-bf9b-af1d-5c79-a793d0aa35ec backoff time=20ms error="failed to connect to peer: 42e2a768-bf9b-af1d-5c79-a793d0aa35ec" -2024-05-15T01:18:48.955Z [ERROR] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: failed to appendEntries to: peer="{Voter server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec 42e2a768-bf9b-af1d-5c79-a793d0aa35ec}" error="failed to connect to peer: 42e2a768-bf9b-af1d-5c79-a793d0aa35ec" -2024-05-15T01:18:48.967Z [ERROR] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: failed to heartbeat to: peer=42e2a768-bf9b-af1d-5c79-a793d0aa35ec backoff time=25ms error="failed to connect to peer: 42e2a768-bf9b-af1d-5c79-a793d0aa35ec" -2024-05-15T01:18:48.970Z [WARN] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: failed to contact: server-id=server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec time=96.872717ms -2024-05-15T01:18:48.979Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: updating configuration: command=RemoveServer server-id=server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec server-addr= servers="[{Suffrage:Voter ID:server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff Address:caefbd7e-cffb-dc6a-3609-7f695a17b3ff} {Suffrage:Voter ID:server-00d70eb9-6c15-0748-5dc5-b88babcb7070 Address:00d70eb9-6c15-0748-5dc5-b88babcb7070}]" -2024-05-15T01:18:48.979Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: removed peer, stopping replication: peer=server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec last-index=3 -2024-05-15T01:18:48.980Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: updating configuration: command=AddNonvoter server-id=server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec server-addr=42e2a768-bf9b-af1d-5c79-a793d0aa35ec servers="[{Suffrage:Voter ID:server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff Address:caefbd7e-cffb-dc6a-3609-7f695a17b3ff} {Suffrage:Voter ID:server-00d70eb9-6c15-0748-5dc5-b88babcb7070 Address:00d70eb9-6c15-0748-5dc5-b88babcb7070} {Suffrage:Nonvoter ID:server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec Address:42e2a768-bf9b-af1d-5c79-a793d0aa35ec}]" -2024-05-15T01:18:48.980Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: added peer, starting replication: peer=server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec -2024-05-15T01:18:48.980Z [ERROR] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: failed to appendEntries to: peer="{Nonvoter server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec 42e2a768-bf9b-af1d-5c79-a793d0aa35ec}" error="failed to connect to peer: 42e2a768-bf9b-af1d-5c79-a793d0aa35ec" -2024-05-15T01:18:48.999Z [ERROR] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: peer has newer term, stopping replication: peer="{Nonvoter server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec 42e2a768-bf9b-af1d-5c79-a793d0aa35ec}" -2024-05-15T01:18:48.999Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: entering follower state: follower="Node at caefbd7e-cffb-dc6a-3609-7f695a17b3ff [Follower]" leader-address= leader-id= -2024-05-15T01:18:48.999Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: aborting pipeline replication: peer="{Voter server-00d70eb9-6c15-0748-5dc5-b88babcb7070 00d70eb9-6c15-0748-5dc5-b88babcb7070}" -2024-05-15T01:18:49.005Z [ERROR] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: peer has newer term, stopping replication: peer="{Voter server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec 42e2a768-bf9b-af1d-5c79-a793d0aa35ec}" -2024-05-15T01:18:49.029Z [WARN] server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec: Election timeout reached, restarting election -2024-05-15T01:18:49.029Z [INFO] server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec: entering candidate state: node="Node at 42e2a768-bf9b-af1d-5c79-a793d0aa35ec [Candidate]" term=23 -2024-05-15T01:18:49.029Z [WARN] server-00d70eb9-6c15-0748-5dc5-b88babcb7070: rejecting vote request since we have a leader: from=42e2a768-bf9b-af1d-5c79-a793d0aa35ec leader=caefbd7e-cffb-dc6a-3609-7f695a17b3ff leader-id=server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff -2024-05-15T01:18:49.029Z [WARN] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: rejecting vote request since node is not a voter: from=42e2a768-bf9b-af1d-5c79-a793d0aa35ec -2024-05-15T01:18:49.058Z [WARN] server-00d70eb9-6c15-0748-5dc5-b88babcb7070: heartbeat timeout reached, starting election: last-leader-addr=caefbd7e-cffb-dc6a-3609-7f695a17b3ff last-leader-id=server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff -2024-05-15T01:18:49.058Z [INFO] server-00d70eb9-6c15-0748-5dc5-b88babcb7070: entering candidate state: node="Node at 00d70eb9-6c15-0748-5dc5-b88babcb7070 [Candidate]" term=3 -2024-05-15T01:18:49.059Z [INFO] server-00d70eb9-6c15-0748-5dc5-b88babcb7070: entering follower state: follower="Node at 00d70eb9-6c15-0748-5dc5-b88babcb7070 [Follower]" leader-address= leader-id= -2024-05-15T01:18:49.084Z [WARN] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:49.084Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: entering candidate state: node="Node at caefbd7e-cffb-dc6a-3609-7f695a17b3ff [Candidate]" term=24 -2024-05-15T01:18:49.084Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: election won: term=24 tally=2 -2024-05-15T01:18:49.084Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: entering leader state: leader="Node at caefbd7e-cffb-dc6a-3609-7f695a17b3ff [Leader]" -2024-05-15T01:18:49.084Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: added peer, starting replication: peer=server-00d70eb9-6c15-0748-5dc5-b88babcb7070 -2024-05-15T01:18:49.084Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: added peer, starting replication: peer=server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec -2024-05-15T01:18:49.084Z [WARN] server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec: failed to get previous log: previous-index=4 last-index=2 error="log not found" -2024-05-15T01:18:49.084Z [INFO] server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec: entering follower state: follower="Node at 42e2a768-bf9b-af1d-5c79-a793d0aa35ec [Follower]" leader-address=caefbd7e-cffb-dc6a-3609-7f695a17b3ff leader-id=server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff -2024-05-15T01:18:49.084Z [WARN] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: appendEntries rejected, sending older logs: peer="{Nonvoter server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec 42e2a768-bf9b-af1d-5c79-a793d0aa35ec}" next=3 -2024-05-15T01:18:49.084Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: pipelining replication: peer="{Voter server-00d70eb9-6c15-0748-5dc5-b88babcb7070 00d70eb9-6c15-0748-5dc5-b88babcb7070}" -2024-05-15T01:18:49.084Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: pipelining replication: peer="{Nonvoter server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec 42e2a768-bf9b-af1d-5c79-a793d0aa35ec}" -2024-05-15T01:18:49.085Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:49.190Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:49.085506835 +0000 UTC m=+109.626891574 (1 nodes), 96.432µs from start of poll, 507.838294ms from cluster start. Timeout at 2024-05-15 01:18:49.190511977 +0000 UTC m=+109.731896872, 105.005298ms after stability -2024-05-15T01:18:49.238Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: updating configuration: command=RemoveServer server-id=server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec server-addr= servers="[{Suffrage:Voter ID:server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff Address:caefbd7e-cffb-dc6a-3609-7f695a17b3ff} {Suffrage:Voter ID:server-00d70eb9-6c15-0748-5dc5-b88babcb7070 Address:00d70eb9-6c15-0748-5dc5-b88babcb7070}]" -2024-05-15T01:18:49.238Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: removed peer, stopping replication: peer=server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec last-index=106 -2024-05-15T01:18:49.238Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: aborting pipeline replication: peer="{Nonvoter server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec 42e2a768-bf9b-af1d-5c79-a793d0aa35ec}" -2024-05-15T01:18:49.238Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: updating configuration: command=AddVoter server-id=server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec server-addr=42e2a768-bf9b-af1d-5c79-a793d0aa35ec servers="[{Suffrage:Voter ID:server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff Address:caefbd7e-cffb-dc6a-3609-7f695a17b3ff} {Suffrage:Voter ID:server-00d70eb9-6c15-0748-5dc5-b88babcb7070 Address:00d70eb9-6c15-0748-5dc5-b88babcb7070} {Suffrage:Voter ID:server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec Address:42e2a768-bf9b-af1d-5c79-a793d0aa35ec}]" -2024-05-15T01:18:49.238Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: added peer, starting replication: peer=server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec -2024-05-15T01:18:49.238Z [WARN] server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec: failed to get previous log: previous-index=107 last-index=106 error="log not found" -2024-05-15T01:18:49.238Z [WARN] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: appendEntries rejected, sending older logs: peer="{Voter server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec 42e2a768-bf9b-af1d-5c79-a793d0aa35ec}" next=107 -2024-05-15T01:18:49.238Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: pipelining replication: peer="{Voter server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec 42e2a768-bf9b-af1d-5c79-a793d0aa35ec}" -2024-05-15T01:18:50.334Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:50.439Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:50.334096818 +0000 UTC m=+110.875481631 (1 nodes), 1.875µs from start of poll, 1.756428351s from cluster start. Timeout at 2024-05-15 01:18:50.439611242 +0000 UTC m=+110.980996009, 105.514378ms after stability -2024-05-15T01:18:50.439Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: aborting pipeline replication: peer="{Voter server-42e2a768-bf9b-af1d-5c79-a793d0aa35ec 42e2a768-bf9b-af1d-5c79-a793d0aa35ec}" -2024-05-15T01:18:50.439Z [INFO] server-caefbd7e-cffb-dc6a-3609-7f695a17b3ff: aborting pipeline replication: peer="{Voter server-00d70eb9-6c15-0748-5dc5-b88babcb7070 00d70eb9-6c15-0748-5dc5-b88babcb7070}" ---- PASS: TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin (1.86s) +2025-06-17T07:44:39.809Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-370f4164-e543-05c3-f2e7-c9c93c63e16b Address:370f4164-e543-05c3-f2e7-c9c93c63e16b} {Suffrage:Voter ID:server-6ab3ab76-8902-dcf5-b621-a2d82b152eca Address:6ab3ab76-8902-dcf5-b621-a2d82b152eca} {Suffrage:Voter ID:server-48627f38-e759-e157-7454-e2050218072b Address:48627f38-e759-e157-7454-e2050218072b}]" +2025-06-17T07:44:39.809Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: entering follower state: follower="Node at 370f4164-e543-05c3-f2e7-c9c93c63e16b [Follower]" leader-address= leader-id= +2025-06-17T07:44:39.809Z [INFO] server-6ab3ab76-8902-dcf5-b621-a2d82b152eca: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-370f4164-e543-05c3-f2e7-c9c93c63e16b Address:370f4164-e543-05c3-f2e7-c9c93c63e16b} {Suffrage:Voter ID:server-6ab3ab76-8902-dcf5-b621-a2d82b152eca Address:6ab3ab76-8902-dcf5-b621-a2d82b152eca} {Suffrage:Voter ID:server-48627f38-e759-e157-7454-e2050218072b Address:48627f38-e759-e157-7454-e2050218072b}]" +2025-06-17T07:44:39.809Z [INFO] server-6ab3ab76-8902-dcf5-b621-a2d82b152eca: entering follower state: follower="Node at 6ab3ab76-8902-dcf5-b621-a2d82b152eca [Follower]" leader-address= leader-id= +2025-06-17T07:44:39.809Z [INFO] server-48627f38-e759-e157-7454-e2050218072b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-370f4164-e543-05c3-f2e7-c9c93c63e16b Address:370f4164-e543-05c3-f2e7-c9c93c63e16b} {Suffrage:Voter ID:server-6ab3ab76-8902-dcf5-b621-a2d82b152eca Address:6ab3ab76-8902-dcf5-b621-a2d82b152eca} {Suffrage:Voter ID:server-48627f38-e759-e157-7454-e2050218072b Address:48627f38-e759-e157-7454-e2050218072b}]" +2025-06-17T07:44:39.809Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:39.809Z [INFO] server-48627f38-e759-e157-7454-e2050218072b: entering follower state: follower="Node at 48627f38-e759-e157-7454-e2050218072b [Follower]" leader-address= leader-id= +2025-06-17T07:44:39.860Z [WARN] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:39.860Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: entering candidate state: node="Node at 370f4164-e543-05c3-f2e7-c9c93c63e16b [Candidate]" term=2 +2025-06-17T07:44:39.860Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: election won: term=2 tally=2 +2025-06-17T07:44:39.860Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: entering leader state: leader="Node at 370f4164-e543-05c3-f2e7-c9c93c63e16b [Leader]" +2025-06-17T07:44:39.860Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: added peer, starting replication: peer=server-6ab3ab76-8902-dcf5-b621-a2d82b152eca +2025-06-17T07:44:39.860Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: added peer, starting replication: peer=server-48627f38-e759-e157-7454-e2050218072b +2025-06-17T07:44:39.860Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: pipelining replication: peer="{Voter server-6ab3ab76-8902-dcf5-b621-a2d82b152eca 6ab3ab76-8902-dcf5-b621-a2d82b152eca}" +2025-06-17T07:44:39.860Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: pipelining replication: peer="{Voter server-48627f38-e759-e157-7454-e2050218072b 48627f38-e759-e157-7454-e2050218072b}" +2025-06-17T07:44:39.966Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:39.860766566 +0000 UTC m=+108.464295253 (1 nodes), 51.247872ms from start of poll, 51.545787ms from cluster start. Timeout at 2025-06-17 07:44:39.966710145 +0000 UTC m=+108.570238837, 105.943584ms after stability +2025-06-17T07:44:39.972Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:40.077Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:39.972714442 +0000 UTC m=+108.576243132 (2 nodes), 758ns from start of poll, 163.493666ms from cluster start. Timeout at 2025-06-17 07:44:40.077765415 +0000 UTC m=+108.681294098, 105.050966ms after stability +2025-06-17T07:44:40.079Z [ERROR] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: failed to heartbeat to: peer=6ab3ab76-8902-dcf5-b621-a2d82b152eca backoff time=10ms error="failed to connect to peer: 6ab3ab76-8902-dcf5-b621-a2d82b152eca" +2025-06-17T07:44:40.081Z [ERROR] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: failed to pipeline appendEntries: peer="{Voter server-6ab3ab76-8902-dcf5-b621-a2d82b152eca 6ab3ab76-8902-dcf5-b621-a2d82b152eca}" error="append pipeline closed" +2025-06-17T07:44:40.081Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: aborting pipeline replication: peer="{Voter server-6ab3ab76-8902-dcf5-b621-a2d82b152eca 6ab3ab76-8902-dcf5-b621-a2d82b152eca}" +2025-06-17T07:44:40.089Z [ERROR] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: failed to appendEntries to: peer="{Voter server-6ab3ab76-8902-dcf5-b621-a2d82b152eca 6ab3ab76-8902-dcf5-b621-a2d82b152eca}" error="failed to connect to peer: 6ab3ab76-8902-dcf5-b621-a2d82b152eca" +2025-06-17T07:44:40.098Z [ERROR] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: failed to heartbeat to: peer=6ab3ab76-8902-dcf5-b621-a2d82b152eca backoff time=10ms error="failed to connect to peer: 6ab3ab76-8902-dcf5-b621-a2d82b152eca" +2025-06-17T07:44:40.107Z [ERROR] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: failed to appendEntries to: peer="{Voter server-6ab3ab76-8902-dcf5-b621-a2d82b152eca 6ab3ab76-8902-dcf5-b621-a2d82b152eca}" error="failed to connect to peer: 6ab3ab76-8902-dcf5-b621-a2d82b152eca" +2025-06-17T07:44:40.118Z [ERROR] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: failed to heartbeat to: peer=6ab3ab76-8902-dcf5-b621-a2d82b152eca backoff time=10ms error="failed to connect to peer: 6ab3ab76-8902-dcf5-b621-a2d82b152eca" +2025-06-17T07:44:40.121Z [WARN] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: failed to contact: server-id=server-6ab3ab76-8902-dcf5-b621-a2d82b152eca time=50.279494ms +2025-06-17T07:44:40.123Z [ERROR] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: failed to appendEntries to: peer="{Voter server-6ab3ab76-8902-dcf5-b621-a2d82b152eca 6ab3ab76-8902-dcf5-b621-a2d82b152eca}" error="failed to connect to peer: 6ab3ab76-8902-dcf5-b621-a2d82b152eca" +2025-06-17T07:44:40.139Z [ERROR] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: failed to heartbeat to: peer=6ab3ab76-8902-dcf5-b621-a2d82b152eca backoff time=20ms error="failed to connect to peer: 6ab3ab76-8902-dcf5-b621-a2d82b152eca" +2025-06-17T07:44:40.142Z [WARN] server-6ab3ab76-8902-dcf5-b621-a2d82b152eca: heartbeat timeout reached, starting election: last-leader-addr=370f4164-e543-05c3-f2e7-c9c93c63e16b last-leader-id=server-370f4164-e543-05c3-f2e7-c9c93c63e16b +2025-06-17T07:44:40.142Z [INFO] server-6ab3ab76-8902-dcf5-b621-a2d82b152eca: entering candidate state: node="Node at 6ab3ab76-8902-dcf5-b621-a2d82b152eca [Candidate]" term=3 +2025-06-17T07:44:40.142Z [ERROR] server-6ab3ab76-8902-dcf5-b621-a2d82b152eca: failed to make requestVote RPC: target="{Voter server-48627f38-e759-e157-7454-e2050218072b 48627f38-e759-e157-7454-e2050218072b}" error="failed to connect to peer: 48627f38-e759-e157-7454-e2050218072b" term=3 +2025-06-17T07:44:40.142Z [ERROR] server-6ab3ab76-8902-dcf5-b621-a2d82b152eca: failed to make requestVote RPC: target="{Voter server-370f4164-e543-05c3-f2e7-c9c93c63e16b 370f4164-e543-05c3-f2e7-c9c93c63e16b}" error="failed to connect to peer: 370f4164-e543-05c3-f2e7-c9c93c63e16b" term=3 +2025-06-17T07:44:40.150Z [ERROR] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: failed to appendEntries to: peer="{Voter server-6ab3ab76-8902-dcf5-b621-a2d82b152eca 6ab3ab76-8902-dcf5-b621-a2d82b152eca}" error="failed to connect to peer: 6ab3ab76-8902-dcf5-b621-a2d82b152eca" +2025-06-17T07:44:40.168Z [ERROR] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: failed to heartbeat to: peer=6ab3ab76-8902-dcf5-b621-a2d82b152eca backoff time=25ms error="failed to connect to peer: 6ab3ab76-8902-dcf5-b621-a2d82b152eca" +2025-06-17T07:44:40.170Z [WARN] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: failed to contact: server-id=server-6ab3ab76-8902-dcf5-b621-a2d82b152eca time=99.895629ms +2025-06-17T07:44:40.182Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: updating configuration: command=RemoveServer server-id=server-6ab3ab76-8902-dcf5-b621-a2d82b152eca server-addr= servers="[{Suffrage:Voter ID:server-370f4164-e543-05c3-f2e7-c9c93c63e16b Address:370f4164-e543-05c3-f2e7-c9c93c63e16b} {Suffrage:Voter ID:server-48627f38-e759-e157-7454-e2050218072b Address:48627f38-e759-e157-7454-e2050218072b}]" +2025-06-17T07:44:40.182Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: removed peer, stopping replication: peer=server-6ab3ab76-8902-dcf5-b621-a2d82b152eca last-index=3 +2025-06-17T07:44:40.183Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: updating configuration: command=AddNonvoter server-id=server-6ab3ab76-8902-dcf5-b621-a2d82b152eca server-addr=6ab3ab76-8902-dcf5-b621-a2d82b152eca servers="[{Suffrage:Voter ID:server-370f4164-e543-05c3-f2e7-c9c93c63e16b Address:370f4164-e543-05c3-f2e7-c9c93c63e16b} {Suffrage:Voter ID:server-48627f38-e759-e157-7454-e2050218072b Address:48627f38-e759-e157-7454-e2050218072b} {Suffrage:Nonvoter ID:server-6ab3ab76-8902-dcf5-b621-a2d82b152eca Address:6ab3ab76-8902-dcf5-b621-a2d82b152eca}]" +2025-06-17T07:44:40.183Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: added peer, starting replication: peer=server-6ab3ab76-8902-dcf5-b621-a2d82b152eca +2025-06-17T07:44:40.183Z [ERROR] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: failed to appendEntries to: peer="{Nonvoter server-6ab3ab76-8902-dcf5-b621-a2d82b152eca 6ab3ab76-8902-dcf5-b621-a2d82b152eca}" error="failed to connect to peer: 6ab3ab76-8902-dcf5-b621-a2d82b152eca" +2025-06-17T07:44:40.199Z [ERROR] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: peer has newer term, stopping replication: peer="{Nonvoter server-6ab3ab76-8902-dcf5-b621-a2d82b152eca 6ab3ab76-8902-dcf5-b621-a2d82b152eca}" +2025-06-17T07:44:40.199Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: entering follower state: follower="Node at 370f4164-e543-05c3-f2e7-c9c93c63e16b [Follower]" leader-address= leader-id= +2025-06-17T07:44:40.199Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: aborting pipeline replication: peer="{Voter server-48627f38-e759-e157-7454-e2050218072b 48627f38-e759-e157-7454-e2050218072b}" +2025-06-17T07:44:40.201Z [ERROR] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: peer has newer term, stopping replication: peer="{Voter server-6ab3ab76-8902-dcf5-b621-a2d82b152eca 6ab3ab76-8902-dcf5-b621-a2d82b152eca}" +2025-06-17T07:44:40.209Z [WARN] server-6ab3ab76-8902-dcf5-b621-a2d82b152eca: Election timeout reached, restarting election +2025-06-17T07:44:40.209Z [INFO] server-6ab3ab76-8902-dcf5-b621-a2d82b152eca: entering candidate state: node="Node at 6ab3ab76-8902-dcf5-b621-a2d82b152eca [Candidate]" term=23 +2025-06-17T07:44:40.210Z [WARN] server-48627f38-e759-e157-7454-e2050218072b: rejecting vote request since we have a leader: from=6ab3ab76-8902-dcf5-b621-a2d82b152eca leader=370f4164-e543-05c3-f2e7-c9c93c63e16b leader-id=server-370f4164-e543-05c3-f2e7-c9c93c63e16b +2025-06-17T07:44:40.210Z [WARN] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: rejecting vote request since node is not a voter: from=6ab3ab76-8902-dcf5-b621-a2d82b152eca +2025-06-17T07:44:40.265Z [WARN] server-48627f38-e759-e157-7454-e2050218072b: heartbeat timeout reached, starting election: last-leader-addr=370f4164-e543-05c3-f2e7-c9c93c63e16b last-leader-id=server-370f4164-e543-05c3-f2e7-c9c93c63e16b +2025-06-17T07:44:40.265Z [INFO] server-48627f38-e759-e157-7454-e2050218072b: entering candidate state: node="Node at 48627f38-e759-e157-7454-e2050218072b [Candidate]" term=3 +2025-06-17T07:44:40.265Z [INFO] server-48627f38-e759-e157-7454-e2050218072b: entering follower state: follower="Node at 48627f38-e759-e157-7454-e2050218072b [Follower]" leader-address= leader-id= +2025-06-17T07:44:40.288Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:40.290Z [WARN] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:40.290Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: entering candidate state: node="Node at 370f4164-e543-05c3-f2e7-c9c93c63e16b [Candidate]" term=24 +2025-06-17T07:44:40.290Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: election won: term=24 tally=2 +2025-06-17T07:44:40.290Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: entering leader state: leader="Node at 370f4164-e543-05c3-f2e7-c9c93c63e16b [Leader]" +2025-06-17T07:44:40.290Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: added peer, starting replication: peer=server-48627f38-e759-e157-7454-e2050218072b +2025-06-17T07:44:40.290Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: added peer, starting replication: peer=server-6ab3ab76-8902-dcf5-b621-a2d82b152eca +2025-06-17T07:44:40.290Z [WARN] server-6ab3ab76-8902-dcf5-b621-a2d82b152eca: failed to get previous log: previous-index=4 last-index=2 error="log not found" +2025-06-17T07:44:40.290Z [INFO] server-6ab3ab76-8902-dcf5-b621-a2d82b152eca: entering follower state: follower="Node at 6ab3ab76-8902-dcf5-b621-a2d82b152eca [Follower]" leader-address=370f4164-e543-05c3-f2e7-c9c93c63e16b leader-id=server-370f4164-e543-05c3-f2e7-c9c93c63e16b +2025-06-17T07:44:40.290Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: pipelining replication: peer="{Voter server-48627f38-e759-e157-7454-e2050218072b 48627f38-e759-e157-7454-e2050218072b}" +2025-06-17T07:44:40.290Z [WARN] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: appendEntries rejected, sending older logs: peer="{Nonvoter server-6ab3ab76-8902-dcf5-b621-a2d82b152eca 6ab3ab76-8902-dcf5-b621-a2d82b152eca}" next=3 +2025-06-17T07:44:40.291Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: pipelining replication: peer="{Nonvoter server-6ab3ab76-8902-dcf5-b621-a2d82b152eca 6ab3ab76-8902-dcf5-b621-a2d82b152eca}" +2025-06-17T07:44:40.397Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:40.290927196 +0000 UTC m=+108.894455915 (1 nodes), 2.449308ms from start of poll, 481.706449ms from cluster start. Timeout at 2025-06-17 07:44:40.396986068 +0000 UTC m=+109.000514760, 106.058845ms after stability +2025-06-17T07:44:40.420Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: updating configuration: command=RemoveServer server-id=server-6ab3ab76-8902-dcf5-b621-a2d82b152eca server-addr= servers="[{Suffrage:Voter ID:server-370f4164-e543-05c3-f2e7-c9c93c63e16b Address:370f4164-e543-05c3-f2e7-c9c93c63e16b} {Suffrage:Voter ID:server-48627f38-e759-e157-7454-e2050218072b Address:48627f38-e759-e157-7454-e2050218072b}]" +2025-06-17T07:44:40.420Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: removed peer, stopping replication: peer=server-6ab3ab76-8902-dcf5-b621-a2d82b152eca last-index=106 +2025-06-17T07:44:40.420Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: aborting pipeline replication: peer="{Nonvoter server-6ab3ab76-8902-dcf5-b621-a2d82b152eca 6ab3ab76-8902-dcf5-b621-a2d82b152eca}" +2025-06-17T07:44:40.420Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: updating configuration: command=AddVoter server-id=server-6ab3ab76-8902-dcf5-b621-a2d82b152eca server-addr=6ab3ab76-8902-dcf5-b621-a2d82b152eca servers="[{Suffrage:Voter ID:server-370f4164-e543-05c3-f2e7-c9c93c63e16b Address:370f4164-e543-05c3-f2e7-c9c93c63e16b} {Suffrage:Voter ID:server-48627f38-e759-e157-7454-e2050218072b Address:48627f38-e759-e157-7454-e2050218072b} {Suffrage:Voter ID:server-6ab3ab76-8902-dcf5-b621-a2d82b152eca Address:6ab3ab76-8902-dcf5-b621-a2d82b152eca}]" +2025-06-17T07:44:40.420Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: added peer, starting replication: peer=server-6ab3ab76-8902-dcf5-b621-a2d82b152eca +2025-06-17T07:44:40.420Z [WARN] server-6ab3ab76-8902-dcf5-b621-a2d82b152eca: failed to get previous log: previous-index=107 last-index=106 error="log not found" +2025-06-17T07:44:40.421Z [WARN] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: appendEntries rejected, sending older logs: peer="{Voter server-6ab3ab76-8902-dcf5-b621-a2d82b152eca 6ab3ab76-8902-dcf5-b621-a2d82b152eca}" next=107 +2025-06-17T07:44:40.421Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: pipelining replication: peer="{Voter server-6ab3ab76-8902-dcf5-b621-a2d82b152eca 6ab3ab76-8902-dcf5-b621-a2d82b152eca}" +2025-06-17T07:44:41.491Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:41.596Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:41.491258333 +0000 UTC m=+110.094787020 (1 nodes), 651ns from start of poll, 1.682037554s from cluster start. Timeout at 2025-06-17 07:44:41.596808958 +0000 UTC m=+110.200337643, 105.550623ms after stability +2025-06-17T07:44:41.596Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: aborting pipeline replication: peer="{Voter server-6ab3ab76-8902-dcf5-b621-a2d82b152eca 6ab3ab76-8902-dcf5-b621-a2d82b152eca}" +2025-06-17T07:44:41.596Z [INFO] server-370f4164-e543-05c3-f2e7-c9c93c63e16b: aborting pipeline replication: peer="{Voter server-48627f38-e759-e157-7454-e2050218072b 48627f38-e759-e157-7454-e2050218072b}" +--- PASS: TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin (1.79s) === RUN TestRaft_FollowerRemovalNoElection -2024-05-15T01:18:50.441Z [INFO] server-6f653f78-a4d7-0110-605e-6932e92c537e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f653f78-a4d7-0110-605e-6932e92c537e Address:6f653f78-a4d7-0110-605e-6932e92c537e} {Suffrage:Voter ID:server-2d439803-9d62-aefc-1a28-30abdeb13cf4 Address:2d439803-9d62-aefc-1a28-30abdeb13cf4} {Suffrage:Voter ID:server-8c5a024b-8a47-3a70-4b8b-636a8f1703d2 Address:8c5a024b-8a47-3a70-4b8b-636a8f1703d2}]" -2024-05-15T01:18:50.441Z [INFO] server-2d439803-9d62-aefc-1a28-30abdeb13cf4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f653f78-a4d7-0110-605e-6932e92c537e Address:6f653f78-a4d7-0110-605e-6932e92c537e} {Suffrage:Voter ID:server-2d439803-9d62-aefc-1a28-30abdeb13cf4 Address:2d439803-9d62-aefc-1a28-30abdeb13cf4} {Suffrage:Voter ID:server-8c5a024b-8a47-3a70-4b8b-636a8f1703d2 Address:8c5a024b-8a47-3a70-4b8b-636a8f1703d2}]" -2024-05-15T01:18:50.441Z [INFO] server-8c5a024b-8a47-3a70-4b8b-636a8f1703d2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f653f78-a4d7-0110-605e-6932e92c537e Address:6f653f78-a4d7-0110-605e-6932e92c537e} {Suffrage:Voter ID:server-2d439803-9d62-aefc-1a28-30abdeb13cf4 Address:2d439803-9d62-aefc-1a28-30abdeb13cf4} {Suffrage:Voter ID:server-8c5a024b-8a47-3a70-4b8b-636a8f1703d2 Address:8c5a024b-8a47-3a70-4b8b-636a8f1703d2}]" -2024-05-15T01:18:50.441Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:50.442Z [INFO] server-6f653f78-a4d7-0110-605e-6932e92c537e: entering follower state: follower="Node at 6f653f78-a4d7-0110-605e-6932e92c537e [Follower]" leader-address= leader-id= -2024-05-15T01:18:50.442Z [INFO] server-2d439803-9d62-aefc-1a28-30abdeb13cf4: entering follower state: follower="Node at 2d439803-9d62-aefc-1a28-30abdeb13cf4 [Follower]" leader-address= leader-id= -2024-05-15T01:18:50.442Z [INFO] server-8c5a024b-8a47-3a70-4b8b-636a8f1703d2: entering follower state: follower="Node at 8c5a024b-8a47-3a70-4b8b-636a8f1703d2 [Follower]" leader-address= leader-id= -2024-05-15T01:18:50.564Z [WARN] server-2d439803-9d62-aefc-1a28-30abdeb13cf4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:50.564Z [INFO] server-2d439803-9d62-aefc-1a28-30abdeb13cf4: entering candidate state: node="Node at 2d439803-9d62-aefc-1a28-30abdeb13cf4 [Candidate]" term=2 -2024-05-15T01:18:50.564Z [INFO] server-2d439803-9d62-aefc-1a28-30abdeb13cf4: election won: term=2 tally=2 -2024-05-15T01:18:50.564Z [INFO] server-2d439803-9d62-aefc-1a28-30abdeb13cf4: entering leader state: leader="Node at 2d439803-9d62-aefc-1a28-30abdeb13cf4 [Leader]" -2024-05-15T01:18:50.564Z [INFO] server-2d439803-9d62-aefc-1a28-30abdeb13cf4: added peer, starting replication: peer=server-6f653f78-a4d7-0110-605e-6932e92c537e -2024-05-15T01:18:50.564Z [INFO] server-2d439803-9d62-aefc-1a28-30abdeb13cf4: added peer, starting replication: peer=server-8c5a024b-8a47-3a70-4b8b-636a8f1703d2 -2024-05-15T01:18:50.564Z [INFO] server-2d439803-9d62-aefc-1a28-30abdeb13cf4: pipelining replication: peer="{Voter server-6f653f78-a4d7-0110-605e-6932e92c537e 6f653f78-a4d7-0110-605e-6932e92c537e}" -2024-05-15T01:18:50.564Z [INFO] server-2d439803-9d62-aefc-1a28-30abdeb13cf4: pipelining replication: peer="{Voter server-8c5a024b-8a47-3a70-4b8b-636a8f1703d2 8c5a024b-8a47-3a70-4b8b-636a8f1703d2}" -2024-05-15T01:18:50.769Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:50.564691506 +0000 UTC m=+111.106076342 (1 nodes), 122.723426ms from start of poll, 123.216403ms from cluster start. Timeout at 2024-05-15 01:18:50.769725677 +0000 UTC m=+111.311110566, 205.034224ms after stability -2024-05-15T01:18:50.769Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:50.975Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:50.769838793 +0000 UTC m=+111.311223451 (1 nodes), 1.788µs from start of poll, 328.363512ms from cluster start. Timeout at 2024-05-15 01:18:50.975264224 +0000 UTC m=+111.516649001, 205.42555ms after stability -2024-05-15T01:18:50.980Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:51.186Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:50.980505153 +0000 UTC m=+111.521889887 (2 nodes), 2.221µs from start of poll, 539.029948ms from cluster start. Timeout at 2024-05-15 01:18:51.186019079 +0000 UTC m=+111.727403845, 205.513958ms after stability -2024-05-15T01:18:51.186Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:51.391Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:51.186130218 +0000 UTC m=+111.727514957 (1 nodes), 1.814µs from start of poll, 744.655018ms from cluster start. Timeout at 2024-05-15 01:18:51.391520776 +0000 UTC m=+111.932905554, 205.390597ms after stability - raft_test.go:2961: [INFO] restarting Node at 6f653f78-a4d7-0110-605e-6932e92c537e [Follower] -2024-05-15T01:18:51.391Z [INFO] server-6f653f78-a4d7-0110-605e-6932e92c537e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f653f78-a4d7-0110-605e-6932e92c537e Address:6f653f78-a4d7-0110-605e-6932e92c537e} {Suffrage:Voter ID:server-2d439803-9d62-aefc-1a28-30abdeb13cf4 Address:2d439803-9d62-aefc-1a28-30abdeb13cf4} {Suffrage:Voter ID:server-8c5a024b-8a47-3a70-4b8b-636a8f1703d2 Address:8c5a024b-8a47-3a70-4b8b-636a8f1703d2}]" -2024-05-15T01:18:51.391Z [INFO] server-6f653f78-a4d7-0110-605e-6932e92c537e: entering follower state: follower="Node at 6f653f78-a4d7-0110-605e-6932e92c537e [Follower]" leader-address= leader-id= -2024-05-15T01:18:51.642Z [INFO] server-2d439803-9d62-aefc-1a28-30abdeb13cf4: aborting pipeline replication: peer="{Voter server-8c5a024b-8a47-3a70-4b8b-636a8f1703d2 8c5a024b-8a47-3a70-4b8b-636a8f1703d2}" -2024-05-15T01:18:52.022Z [ERROR] server-2d439803-9d62-aefc-1a28-30abdeb13cf4: failed to pipeline appendEntries: peer="{Voter server-6f653f78-a4d7-0110-605e-6932e92c537e 6f653f78-a4d7-0110-605e-6932e92c537e}" error="command enqueue timeout" -2024-05-15T01:18:52.022Z [INFO] server-2d439803-9d62-aefc-1a28-30abdeb13cf4: aborting pipeline replication: peer="{Voter server-6f653f78-a4d7-0110-605e-6932e92c537e 6f653f78-a4d7-0110-605e-6932e92c537e}" -2024-05-15T01:18:52.154Z [ERROR] server-2d439803-9d62-aefc-1a28-30abdeb13cf4: failed to heartbeat to: peer=6f653f78-a4d7-0110-605e-6932e92c537e backoff time=10ms error="command timed out" ---- PASS: TestRaft_FollowerRemovalNoElection (1.71s) +2025-06-17T07:44:41.597Z [INFO] server-efe73c71-bf10-8d9b-2edb-e7a1ae818f4e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-efe73c71-bf10-8d9b-2edb-e7a1ae818f4e Address:efe73c71-bf10-8d9b-2edb-e7a1ae818f4e} {Suffrage:Voter ID:server-2d280d74-9983-c19c-b29c-092bd77762b2 Address:2d280d74-9983-c19c-b29c-092bd77762b2} {Suffrage:Voter ID:server-ab579ac3-81f0-a6da-04d6-148b1ab6d13a Address:ab579ac3-81f0-a6da-04d6-148b1ab6d13a}]" +2025-06-17T07:44:41.597Z [INFO] server-efe73c71-bf10-8d9b-2edb-e7a1ae818f4e: entering follower state: follower="Node at efe73c71-bf10-8d9b-2edb-e7a1ae818f4e [Follower]" leader-address= leader-id= +2025-06-17T07:44:41.597Z [INFO] server-2d280d74-9983-c19c-b29c-092bd77762b2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-efe73c71-bf10-8d9b-2edb-e7a1ae818f4e Address:efe73c71-bf10-8d9b-2edb-e7a1ae818f4e} {Suffrage:Voter ID:server-2d280d74-9983-c19c-b29c-092bd77762b2 Address:2d280d74-9983-c19c-b29c-092bd77762b2} {Suffrage:Voter ID:server-ab579ac3-81f0-a6da-04d6-148b1ab6d13a Address:ab579ac3-81f0-a6da-04d6-148b1ab6d13a}]" +2025-06-17T07:44:41.598Z [INFO] server-2d280d74-9983-c19c-b29c-092bd77762b2: entering follower state: follower="Node at 2d280d74-9983-c19c-b29c-092bd77762b2 [Follower]" leader-address= leader-id= +2025-06-17T07:44:41.598Z [INFO] server-ab579ac3-81f0-a6da-04d6-148b1ab6d13a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-efe73c71-bf10-8d9b-2edb-e7a1ae818f4e Address:efe73c71-bf10-8d9b-2edb-e7a1ae818f4e} {Suffrage:Voter ID:server-2d280d74-9983-c19c-b29c-092bd77762b2 Address:2d280d74-9983-c19c-b29c-092bd77762b2} {Suffrage:Voter ID:server-ab579ac3-81f0-a6da-04d6-148b1ab6d13a Address:ab579ac3-81f0-a6da-04d6-148b1ab6d13a}]" +2025-06-17T07:44:41.598Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:41.598Z [INFO] server-ab579ac3-81f0-a6da-04d6-148b1ab6d13a: entering follower state: follower="Node at ab579ac3-81f0-a6da-04d6-148b1ab6d13a [Follower]" leader-address= leader-id= +2025-06-17T07:44:41.729Z [WARN] server-2d280d74-9983-c19c-b29c-092bd77762b2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:41.729Z [INFO] server-2d280d74-9983-c19c-b29c-092bd77762b2: entering candidate state: node="Node at 2d280d74-9983-c19c-b29c-092bd77762b2 [Candidate]" term=2 +2025-06-17T07:44:41.729Z [INFO] server-2d280d74-9983-c19c-b29c-092bd77762b2: election won: term=2 tally=2 +2025-06-17T07:44:41.729Z [INFO] server-2d280d74-9983-c19c-b29c-092bd77762b2: entering leader state: leader="Node at 2d280d74-9983-c19c-b29c-092bd77762b2 [Leader]" +2025-06-17T07:44:41.729Z [INFO] server-2d280d74-9983-c19c-b29c-092bd77762b2: added peer, starting replication: peer=server-efe73c71-bf10-8d9b-2edb-e7a1ae818f4e +2025-06-17T07:44:41.729Z [INFO] server-2d280d74-9983-c19c-b29c-092bd77762b2: added peer, starting replication: peer=server-ab579ac3-81f0-a6da-04d6-148b1ab6d13a +2025-06-17T07:44:41.729Z [INFO] server-2d280d74-9983-c19c-b29c-092bd77762b2: pipelining replication: peer="{Voter server-ab579ac3-81f0-a6da-04d6-148b1ab6d13a ab579ac3-81f0-a6da-04d6-148b1ab6d13a}" +2025-06-17T07:44:41.729Z [INFO] server-2d280d74-9983-c19c-b29c-092bd77762b2: pipelining replication: peer="{Voter server-efe73c71-bf10-8d9b-2edb-e7a1ae818f4e efe73c71-bf10-8d9b-2edb-e7a1ae818f4e}" +2025-06-17T07:44:41.935Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:41.729708184 +0000 UTC m=+110.333236884 (1 nodes), 131.57478ms from start of poll, 131.935879ms from cluster start. Timeout at 2025-06-17 07:44:41.93534555 +0000 UTC m=+110.538874252, 205.637368ms after stability +2025-06-17T07:44:41.935Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:42.141Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:41.935411902 +0000 UTC m=+110.538940587 (1 nodes), 614ns from start of poll, 337.639582ms from cluster start. Timeout at 2025-06-17 07:44:42.141402156 +0000 UTC m=+110.744930849, 205.990262ms after stability +2025-06-17T07:44:42.147Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:42.352Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:42.147209335 +0000 UTC m=+110.750738022 (2 nodes), 719ns from start of poll, 549.437017ms from cluster start. Timeout at 2025-06-17 07:44:42.352641038 +0000 UTC m=+110.956169740, 205.431718ms after stability +2025-06-17T07:44:42.352Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:42.557Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:42.352687772 +0000 UTC m=+110.956216463 (1 nodes), 561ns from start of poll, 754.915458ms from cluster start. Timeout at 2025-06-17 07:44:42.557864997 +0000 UTC m=+111.161393689, 205.177226ms after stability + raft_test.go:2961: [INFO] restarting Node at efe73c71-bf10-8d9b-2edb-e7a1ae818f4e [Follower] +2025-06-17T07:44:42.558Z [INFO] server-efe73c71-bf10-8d9b-2edb-e7a1ae818f4e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-efe73c71-bf10-8d9b-2edb-e7a1ae818f4e Address:efe73c71-bf10-8d9b-2edb-e7a1ae818f4e} {Suffrage:Voter ID:server-2d280d74-9983-c19c-b29c-092bd77762b2 Address:2d280d74-9983-c19c-b29c-092bd77762b2} {Suffrage:Voter ID:server-ab579ac3-81f0-a6da-04d6-148b1ab6d13a Address:ab579ac3-81f0-a6da-04d6-148b1ab6d13a}]" +2025-06-17T07:44:42.558Z [INFO] server-efe73c71-bf10-8d9b-2edb-e7a1ae818f4e: entering follower state: follower="Node at efe73c71-bf10-8d9b-2edb-e7a1ae818f4e [Follower]" leader-address= leader-id= +2025-06-17T07:44:42.809Z [INFO] server-2d280d74-9983-c19c-b29c-092bd77762b2: aborting pipeline replication: peer="{Voter server-ab579ac3-81f0-a6da-04d6-148b1ab6d13a ab579ac3-81f0-a6da-04d6-148b1ab6d13a}" +2025-06-17T07:44:43.186Z [ERROR] server-2d280d74-9983-c19c-b29c-092bd77762b2: failed to pipeline appendEntries: peer="{Voter server-efe73c71-bf10-8d9b-2edb-e7a1ae818f4e efe73c71-bf10-8d9b-2edb-e7a1ae818f4e}" error="command enqueue timeout" +2025-06-17T07:44:43.186Z [INFO] server-2d280d74-9983-c19c-b29c-092bd77762b2: aborting pipeline replication: peer="{Voter server-efe73c71-bf10-8d9b-2edb-e7a1ae818f4e efe73c71-bf10-8d9b-2edb-e7a1ae818f4e}" +2025-06-17T07:44:43.328Z [ERROR] server-2d280d74-9983-c19c-b29c-092bd77762b2: failed to heartbeat to: peer=efe73c71-bf10-8d9b-2edb-e7a1ae818f4e backoff time=10ms error="command timed out" +--- PASS: TestRaft_FollowerRemovalNoElection (1.73s) === RUN TestRaft_VoteWithNoIDNoAddr -2024-05-15T01:18:52.155Z [INFO] server-fe89db1e-8c45-e36e-5704-de15b20874a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fe89db1e-8c45-e36e-5704-de15b20874a1 Address:fe89db1e-8c45-e36e-5704-de15b20874a1} {Suffrage:Voter ID:server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f Address:b59aec8e-4a2f-fbe1-0380-af8b14d7c73f} {Suffrage:Voter ID:server-90b69d51-50f3-8e38-1ae5-fab9ca0bee0e Address:90b69d51-50f3-8e38-1ae5-fab9ca0bee0e}]" -2024-05-15T01:18:52.156Z [INFO] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fe89db1e-8c45-e36e-5704-de15b20874a1 Address:fe89db1e-8c45-e36e-5704-de15b20874a1} {Suffrage:Voter ID:server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f Address:b59aec8e-4a2f-fbe1-0380-af8b14d7c73f} {Suffrage:Voter ID:server-90b69d51-50f3-8e38-1ae5-fab9ca0bee0e Address:90b69d51-50f3-8e38-1ae5-fab9ca0bee0e}]" -2024-05-15T01:18:52.156Z [INFO] server-90b69d51-50f3-8e38-1ae5-fab9ca0bee0e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fe89db1e-8c45-e36e-5704-de15b20874a1 Address:fe89db1e-8c45-e36e-5704-de15b20874a1} {Suffrage:Voter ID:server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f Address:b59aec8e-4a2f-fbe1-0380-af8b14d7c73f} {Suffrage:Voter ID:server-90b69d51-50f3-8e38-1ae5-fab9ca0bee0e Address:90b69d51-50f3-8e38-1ae5-fab9ca0bee0e}]" -2024-05-15T01:18:52.156Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:52.156Z [INFO] server-fe89db1e-8c45-e36e-5704-de15b20874a1: entering follower state: follower="Node at fe89db1e-8c45-e36e-5704-de15b20874a1 [Follower]" leader-address= leader-id= -2024-05-15T01:18:52.156Z [INFO] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: entering follower state: follower="Node at b59aec8e-4a2f-fbe1-0380-af8b14d7c73f [Follower]" leader-address= leader-id= -2024-05-15T01:18:52.156Z [INFO] server-90b69d51-50f3-8e38-1ae5-fab9ca0bee0e: entering follower state: follower="Node at 90b69d51-50f3-8e38-1ae5-fab9ca0bee0e [Follower]" leader-address= leader-id= -2024-05-15T01:18:52.229Z [WARN] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-15T01:18:52.229Z [INFO] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: entering candidate state: node="Node at b59aec8e-4a2f-fbe1-0380-af8b14d7c73f [Candidate]" term=2 -2024-05-15T01:18:52.229Z [INFO] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: election won: term=2 tally=2 -2024-05-15T01:18:52.229Z [INFO] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: entering leader state: leader="Node at b59aec8e-4a2f-fbe1-0380-af8b14d7c73f [Leader]" -2024-05-15T01:18:52.229Z [INFO] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: added peer, starting replication: peer=server-fe89db1e-8c45-e36e-5704-de15b20874a1 -2024-05-15T01:18:52.229Z [INFO] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: added peer, starting replication: peer=server-90b69d51-50f3-8e38-1ae5-fab9ca0bee0e -2024-05-15T01:18:52.230Z [INFO] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: pipelining replication: peer="{Voter server-fe89db1e-8c45-e36e-5704-de15b20874a1 fe89db1e-8c45-e36e-5704-de15b20874a1}" -2024-05-15T01:18:52.230Z [INFO] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: pipelining replication: peer="{Voter server-90b69d51-50f3-8e38-1ae5-fab9ca0bee0e 90b69d51-50f3-8e38-1ae5-fab9ca0bee0e}" -2024-05-15T01:18:52.335Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:52.22991053 +0000 UTC m=+112.771295280 (1 nodes), 73.590367ms from start of poll, 74.122455ms from cluster start. Timeout at 2024-05-15 01:18:52.335007408 +0000 UTC m=+112.876392171, 105.096891ms after stability -2024-05-15T01:18:52.335Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-15T01:18:52.440Z [INFO] cluster: stable state for Leader reached at 2024-05-15 01:18:52.335142542 +0000 UTC m=+112.876527312 (1 nodes), 2.156µs from start of poll, 179.354487ms from cluster start. Timeout at 2024-05-15 01:18:52.440468284 +0000 UTC m=+112.981853026, 105.325714ms after stability -2024-05-15T01:18:52.446Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-15T01:18:52.553Z [INFO] cluster: stable state for Follower reached at 2024-05-15 01:18:52.446555469 +0000 UTC m=+112.987940522 (2 nodes), 2.462µs from start of poll, 290.767697ms from cluster start. Timeout at 2024-05-15 01:18:52.553051381 +0000 UTC m=+113.094436148, 106.495626ms after stability -2024-05-15T01:18:52.554Z [ERROR] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to heartbeat to: peer=fe89db1e-8c45-e36e-5704-de15b20874a1 backoff time=10ms error="failed to connect to peer: fe89db1e-8c45-e36e-5704-de15b20874a1" -2024-05-15T01:18:52.558Z [ERROR] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to heartbeat to: peer=90b69d51-50f3-8e38-1ae5-fab9ca0bee0e backoff time=10ms error="failed to connect to peer: 90b69d51-50f3-8e38-1ae5-fab9ca0bee0e" -2024-05-15T01:18:52.560Z [ERROR] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to pipeline appendEntries: peer="{Voter server-fe89db1e-8c45-e36e-5704-de15b20874a1 fe89db1e-8c45-e36e-5704-de15b20874a1}" error="append pipeline closed" -2024-05-15T01:18:52.560Z [INFO] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: aborting pipeline replication: peer="{Voter server-fe89db1e-8c45-e36e-5704-de15b20874a1 fe89db1e-8c45-e36e-5704-de15b20874a1}" -2024-05-15T01:18:52.562Z [ERROR] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to pipeline appendEntries: peer="{Voter server-90b69d51-50f3-8e38-1ae5-fab9ca0bee0e 90b69d51-50f3-8e38-1ae5-fab9ca0bee0e}" error="append pipeline closed" -2024-05-15T01:18:52.562Z [INFO] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: aborting pipeline replication: peer="{Voter server-90b69d51-50f3-8e38-1ae5-fab9ca0bee0e 90b69d51-50f3-8e38-1ae5-fab9ca0bee0e}" -2024-05-15T01:18:52.568Z [ERROR] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to appendEntries to: peer="{Voter server-fe89db1e-8c45-e36e-5704-de15b20874a1 fe89db1e-8c45-e36e-5704-de15b20874a1}" error="failed to connect to peer: fe89db1e-8c45-e36e-5704-de15b20874a1" -2024-05-15T01:18:52.570Z [ERROR] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to heartbeat to: peer=fe89db1e-8c45-e36e-5704-de15b20874a1 backoff time=10ms error="failed to connect to peer: fe89db1e-8c45-e36e-5704-de15b20874a1" -2024-05-15T01:18:52.571Z [ERROR] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to appendEntries to: peer="{Voter server-90b69d51-50f3-8e38-1ae5-fab9ca0bee0e 90b69d51-50f3-8e38-1ae5-fab9ca0bee0e}" error="failed to connect to peer: 90b69d51-50f3-8e38-1ae5-fab9ca0bee0e" -2024-05-15T01:18:52.578Z [ERROR] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to heartbeat to: peer=90b69d51-50f3-8e38-1ae5-fab9ca0bee0e backoff time=10ms error="failed to connect to peer: 90b69d51-50f3-8e38-1ae5-fab9ca0bee0e" -2024-05-15T01:18:52.585Z [ERROR] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to appendEntries to: peer="{Voter server-fe89db1e-8c45-e36e-5704-de15b20874a1 fe89db1e-8c45-e36e-5704-de15b20874a1}" error="failed to connect to peer: fe89db1e-8c45-e36e-5704-de15b20874a1" -2024-05-15T01:18:52.589Z [ERROR] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to heartbeat to: peer=fe89db1e-8c45-e36e-5704-de15b20874a1 backoff time=10ms error="failed to connect to peer: fe89db1e-8c45-e36e-5704-de15b20874a1" -2024-05-15T01:18:52.593Z [ERROR] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to appendEntries to: peer="{Voter server-90b69d51-50f3-8e38-1ae5-fab9ca0bee0e 90b69d51-50f3-8e38-1ae5-fab9ca0bee0e}" error="failed to connect to peer: 90b69d51-50f3-8e38-1ae5-fab9ca0bee0e" -2024-05-15T01:18:52.598Z [ERROR] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to heartbeat to: peer=90b69d51-50f3-8e38-1ae5-fab9ca0bee0e backoff time=10ms error="failed to connect to peer: 90b69d51-50f3-8e38-1ae5-fab9ca0bee0e" -2024-05-15T01:18:52.601Z [WARN] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to contact: server-id=server-fe89db1e-8c45-e36e-5704-de15b20874a1 time=50.133257ms -2024-05-15T01:18:52.605Z [ERROR] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to appendEntries to: peer="{Voter server-fe89db1e-8c45-e36e-5704-de15b20874a1 fe89db1e-8c45-e36e-5704-de15b20874a1}" error="failed to connect to peer: fe89db1e-8c45-e36e-5704-de15b20874a1" -2024-05-15T01:18:52.605Z [ERROR] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to heartbeat to: peer=fe89db1e-8c45-e36e-5704-de15b20874a1 backoff time=20ms error="failed to connect to peer: fe89db1e-8c45-e36e-5704-de15b20874a1" -2024-05-15T01:18:52.609Z [ERROR] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to appendEntries to: peer="{Voter server-90b69d51-50f3-8e38-1ae5-fab9ca0bee0e 90b69d51-50f3-8e38-1ae5-fab9ca0bee0e}" error="failed to connect to peer: 90b69d51-50f3-8e38-1ae5-fab9ca0bee0e" -2024-05-15T01:18:52.612Z [WARN] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to contact: server-id=server-fe89db1e-8c45-e36e-5704-de15b20874a1 time=60.923939ms -2024-05-15T01:18:52.612Z [WARN] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to contact: server-id=server-90b69d51-50f3-8e38-1ae5-fab9ca0bee0e time=59.470563ms -2024-05-15T01:18:52.612Z [WARN] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: failed to contact quorum of nodes, stepping down -2024-05-15T01:18:52.612Z [INFO] server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f: entering follower state: follower="Node at b59aec8e-4a2f-fbe1-0380-af8b14d7c73f [Follower]" leader-address= leader-id= -2024-05-15T01:18:52.635Z [WARN] server-90b69d51-50f3-8e38-1ae5-fab9ca0bee0e: heartbeat timeout reached, starting election: last-leader-addr=b59aec8e-4a2f-fbe1-0380-af8b14d7c73f last-leader-id=server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f -2024-05-15T01:18:52.635Z [INFO] server-90b69d51-50f3-8e38-1ae5-fab9ca0bee0e: entering candidate state: node="Node at 90b69d51-50f3-8e38-1ae5-fab9ca0bee0e [Candidate]" term=3 -2024-05-15T01:18:52.635Z [ERROR] server-90b69d51-50f3-8e38-1ae5-fab9ca0bee0e: failed to make requestVote RPC: target="{Voter server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f b59aec8e-4a2f-fbe1-0380-af8b14d7c73f}" error="failed to connect to peer: b59aec8e-4a2f-fbe1-0380-af8b14d7c73f" term=3 -2024-05-15T01:18:52.635Z [WARN] server-fe89db1e-8c45-e36e-5704-de15b20874a1: rejecting vote request since we have a leader: from=90b69d51-50f3-8e38-1ae5-fab9ca0bee0e leader=b59aec8e-4a2f-fbe1-0380-af8b14d7c73f leader-id=server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f -2024-05-15T01:18:52.668Z [WARN] server-fe89db1e-8c45-e36e-5704-de15b20874a1: heartbeat timeout reached, starting election: last-leader-addr=b59aec8e-4a2f-fbe1-0380-af8b14d7c73f last-leader-id=server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f -2024-05-15T01:18:52.668Z [INFO] server-fe89db1e-8c45-e36e-5704-de15b20874a1: entering candidate state: node="Node at fe89db1e-8c45-e36e-5704-de15b20874a1 [Candidate]" term=3 -2024-05-15T01:18:52.668Z [INFO] server-90b69d51-50f3-8e38-1ae5-fab9ca0bee0e: duplicate requestVote for same term: term=3 -2024-05-15T01:18:52.668Z [ERROR] server-fe89db1e-8c45-e36e-5704-de15b20874a1: failed to make requestVote RPC: target="{Voter server-b59aec8e-4a2f-fbe1-0380-af8b14d7c73f b59aec8e-4a2f-fbe1-0380-af8b14d7c73f}" error="failed to connect to peer: b59aec8e-4a2f-fbe1-0380-af8b14d7c73f" term=3 -2024-05-15T01:18:52.670Z [INFO] server-fe89db1e-8c45-e36e-5704-de15b20874a1: entering follower state: follower="Node at fe89db1e-8c45-e36e-5704-de15b20874a1 [Follower]" leader-address= leader-id= ---- PASS: TestRaft_VoteWithNoIDNoAddr (0.52s) +2025-06-17T07:44:43.331Z [INFO] server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d Address:f15e41c2-e2eb-b595-fdfd-8f98b39e511d} {Suffrage:Voter ID:server-0c13b324-a174-11a3-ab0e-800956a1f378 Address:0c13b324-a174-11a3-ab0e-800956a1f378} {Suffrage:Voter ID:server-2cf97996-1530-fff5-2bc4-9c02522cbc7b Address:2cf97996-1530-fff5-2bc4-9c02522cbc7b}]" +2025-06-17T07:44:43.331Z [INFO] server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d: entering follower state: follower="Node at f15e41c2-e2eb-b595-fdfd-8f98b39e511d [Follower]" leader-address= leader-id= +2025-06-17T07:44:43.331Z [INFO] server-0c13b324-a174-11a3-ab0e-800956a1f378: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d Address:f15e41c2-e2eb-b595-fdfd-8f98b39e511d} {Suffrage:Voter ID:server-0c13b324-a174-11a3-ab0e-800956a1f378 Address:0c13b324-a174-11a3-ab0e-800956a1f378} {Suffrage:Voter ID:server-2cf97996-1530-fff5-2bc4-9c02522cbc7b Address:2cf97996-1530-fff5-2bc4-9c02522cbc7b}]" +2025-06-17T07:44:43.331Z [INFO] server-0c13b324-a174-11a3-ab0e-800956a1f378: entering follower state: follower="Node at 0c13b324-a174-11a3-ab0e-800956a1f378 [Follower]" leader-address= leader-id= +2025-06-17T07:44:43.331Z [INFO] server-2cf97996-1530-fff5-2bc4-9c02522cbc7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d Address:f15e41c2-e2eb-b595-fdfd-8f98b39e511d} {Suffrage:Voter ID:server-0c13b324-a174-11a3-ab0e-800956a1f378 Address:0c13b324-a174-11a3-ab0e-800956a1f378} {Suffrage:Voter ID:server-2cf97996-1530-fff5-2bc4-9c02522cbc7b Address:2cf97996-1530-fff5-2bc4-9c02522cbc7b}]" +2025-06-17T07:44:43.331Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:43.331Z [INFO] server-2cf97996-1530-fff5-2bc4-9c02522cbc7b: entering follower state: follower="Node at 2cf97996-1530-fff5-2bc4-9c02522cbc7b [Follower]" leader-address= leader-id= +2025-06-17T07:44:43.390Z [WARN] server-0c13b324-a174-11a3-ab0e-800956a1f378: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:43.390Z [INFO] server-0c13b324-a174-11a3-ab0e-800956a1f378: entering candidate state: node="Node at 0c13b324-a174-11a3-ab0e-800956a1f378 [Candidate]" term=2 +2025-06-17T07:44:43.390Z [INFO] server-0c13b324-a174-11a3-ab0e-800956a1f378: election won: term=2 tally=2 +2025-06-17T07:44:43.390Z [INFO] server-0c13b324-a174-11a3-ab0e-800956a1f378: entering leader state: leader="Node at 0c13b324-a174-11a3-ab0e-800956a1f378 [Leader]" +2025-06-17T07:44:43.390Z [INFO] server-0c13b324-a174-11a3-ab0e-800956a1f378: added peer, starting replication: peer=server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d +2025-06-17T07:44:43.390Z [INFO] server-0c13b324-a174-11a3-ab0e-800956a1f378: added peer, starting replication: peer=server-2cf97996-1530-fff5-2bc4-9c02522cbc7b +2025-06-17T07:44:43.390Z [INFO] server-0c13b324-a174-11a3-ab0e-800956a1f378: pipelining replication: peer="{Voter server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d f15e41c2-e2eb-b595-fdfd-8f98b39e511d}" +2025-06-17T07:44:43.390Z [INFO] server-0c13b324-a174-11a3-ab0e-800956a1f378: pipelining replication: peer="{Voter server-2cf97996-1530-fff5-2bc4-9c02522cbc7b 2cf97996-1530-fff5-2bc4-9c02522cbc7b}" +2025-06-17T07:44:43.496Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:43.39062965 +0000 UTC m=+111.994158357 (1 nodes), 59.260494ms from start of poll, 59.628612ms from cluster start. Timeout at 2025-06-17 07:44:43.496519636 +0000 UTC m=+112.100048322, 105.889965ms after stability +2025-06-17T07:44:43.496Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-17T07:44:43.601Z [INFO] cluster: stable state for Leader reached at 2025-06-17 07:44:43.496574867 +0000 UTC m=+112.100103572 (1 nodes), 684ns from start of poll, 165.573827ms from cluster start. Timeout at 2025-06-17 07:44:43.601831488 +0000 UTC m=+112.205360168, 105.256596ms after stability +2025-06-17T07:44:43.607Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-17T07:44:43.712Z [INFO] cluster: stable state for Follower reached at 2025-06-17 07:44:43.607698002 +0000 UTC m=+112.211226705 (2 nodes), 819ns from start of poll, 276.69696ms from cluster start. Timeout at 2025-06-17 07:44:43.712734195 +0000 UTC m=+112.316262885, 105.03618ms after stability +2025-06-17T07:44:43.713Z [ERROR] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to pipeline appendEntries: peer="{Voter server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d f15e41c2-e2eb-b595-fdfd-8f98b39e511d}" error="append pipeline closed" +2025-06-17T07:44:43.713Z [INFO] server-0c13b324-a174-11a3-ab0e-800956a1f378: aborting pipeline replication: peer="{Voter server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d f15e41c2-e2eb-b595-fdfd-8f98b39e511d}" +2025-06-17T07:44:43.713Z [ERROR] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to heartbeat to: peer=2cf97996-1530-fff5-2bc4-9c02522cbc7b backoff time=10ms error="failed to connect to peer: 2cf97996-1530-fff5-2bc4-9c02522cbc7b" +2025-06-17T07:44:43.715Z [ERROR] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to heartbeat to: peer=f15e41c2-e2eb-b595-fdfd-8f98b39e511d backoff time=10ms error="failed to connect to peer: f15e41c2-e2eb-b595-fdfd-8f98b39e511d" +2025-06-17T07:44:43.717Z [ERROR] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to pipeline appendEntries: peer="{Voter server-2cf97996-1530-fff5-2bc4-9c02522cbc7b 2cf97996-1530-fff5-2bc4-9c02522cbc7b}" error="append pipeline closed" +2025-06-17T07:44:43.717Z [INFO] server-0c13b324-a174-11a3-ab0e-800956a1f378: aborting pipeline replication: peer="{Voter server-2cf97996-1530-fff5-2bc4-9c02522cbc7b 2cf97996-1530-fff5-2bc4-9c02522cbc7b}" +2025-06-17T07:44:43.723Z [ERROR] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to appendEntries to: peer="{Voter server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d f15e41c2-e2eb-b595-fdfd-8f98b39e511d}" error="failed to connect to peer: f15e41c2-e2eb-b595-fdfd-8f98b39e511d" +2025-06-17T07:44:43.724Z [ERROR] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to appendEntries to: peer="{Voter server-2cf97996-1530-fff5-2bc4-9c02522cbc7b 2cf97996-1530-fff5-2bc4-9c02522cbc7b}" error="failed to connect to peer: 2cf97996-1530-fff5-2bc4-9c02522cbc7b" +2025-06-17T07:44:43.733Z [ERROR] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to heartbeat to: peer=f15e41c2-e2eb-b595-fdfd-8f98b39e511d backoff time=10ms error="failed to connect to peer: f15e41c2-e2eb-b595-fdfd-8f98b39e511d" +2025-06-17T07:44:43.734Z [ERROR] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to heartbeat to: peer=2cf97996-1530-fff5-2bc4-9c02522cbc7b backoff time=10ms error="failed to connect to peer: 2cf97996-1530-fff5-2bc4-9c02522cbc7b" +2025-06-17T07:44:43.739Z [ERROR] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to appendEntries to: peer="{Voter server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d f15e41c2-e2eb-b595-fdfd-8f98b39e511d}" error="failed to connect to peer: f15e41c2-e2eb-b595-fdfd-8f98b39e511d" +2025-06-17T07:44:43.742Z [ERROR] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to appendEntries to: peer="{Voter server-2cf97996-1530-fff5-2bc4-9c02522cbc7b 2cf97996-1530-fff5-2bc4-9c02522cbc7b}" error="failed to connect to peer: 2cf97996-1530-fff5-2bc4-9c02522cbc7b" +2025-06-17T07:44:43.752Z [ERROR] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to heartbeat to: peer=2cf97996-1530-fff5-2bc4-9c02522cbc7b backoff time=10ms error="failed to connect to peer: 2cf97996-1530-fff5-2bc4-9c02522cbc7b" +2025-06-17T07:44:43.752Z [ERROR] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to heartbeat to: peer=f15e41c2-e2eb-b595-fdfd-8f98b39e511d backoff time=10ms error="failed to connect to peer: f15e41c2-e2eb-b595-fdfd-8f98b39e511d" +2025-06-17T07:44:43.755Z [WARN] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to contact: server-id=server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d time=50.133508ms +2025-06-17T07:44:43.757Z [ERROR] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to appendEntries to: peer="{Voter server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d f15e41c2-e2eb-b595-fdfd-8f98b39e511d}" error="failed to connect to peer: f15e41c2-e2eb-b595-fdfd-8f98b39e511d" +2025-06-17T07:44:43.759Z [ERROR] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to appendEntries to: peer="{Voter server-2cf97996-1530-fff5-2bc4-9c02522cbc7b 2cf97996-1530-fff5-2bc4-9c02522cbc7b}" error="failed to connect to peer: 2cf97996-1530-fff5-2bc4-9c02522cbc7b" +2025-06-17T07:44:43.766Z [WARN] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to contact: server-id=server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d time=60.831034ms +2025-06-17T07:44:43.766Z [WARN] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to contact: server-id=server-2cf97996-1530-fff5-2bc4-9c02522cbc7b time=56.350705ms +2025-06-17T07:44:43.766Z [WARN] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to contact quorum of nodes, stepping down +2025-06-17T07:44:43.766Z [INFO] server-0c13b324-a174-11a3-ab0e-800956a1f378: entering follower state: follower="Node at 0c13b324-a174-11a3-ab0e-800956a1f378 [Follower]" leader-address= leader-id= +2025-06-17T07:44:43.771Z [ERROR] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to heartbeat to: peer=f15e41c2-e2eb-b595-fdfd-8f98b39e511d backoff time=20ms error="failed to connect to peer: f15e41c2-e2eb-b595-fdfd-8f98b39e511d" +2025-06-17T07:44:43.772Z [ERROR] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to heartbeat to: peer=2cf97996-1530-fff5-2bc4-9c02522cbc7b backoff time=20ms error="failed to connect to peer: 2cf97996-1530-fff5-2bc4-9c02522cbc7b" +2025-06-17T07:44:43.779Z [WARN] server-2cf97996-1530-fff5-2bc4-9c02522cbc7b: heartbeat timeout reached, starting election: last-leader-addr=0c13b324-a174-11a3-ab0e-800956a1f378 last-leader-id=server-0c13b324-a174-11a3-ab0e-800956a1f378 +2025-06-17T07:44:43.779Z [INFO] server-2cf97996-1530-fff5-2bc4-9c02522cbc7b: entering candidate state: node="Node at 2cf97996-1530-fff5-2bc4-9c02522cbc7b [Candidate]" term=3 +2025-06-17T07:44:43.779Z [ERROR] server-2cf97996-1530-fff5-2bc4-9c02522cbc7b: failed to make requestVote RPC: target="{Voter server-0c13b324-a174-11a3-ab0e-800956a1f378 0c13b324-a174-11a3-ab0e-800956a1f378}" error="failed to connect to peer: 0c13b324-a174-11a3-ab0e-800956a1f378" term=3 +2025-06-17T07:44:43.779Z [WARN] server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d: rejecting vote request since we have a leader: from=2cf97996-1530-fff5-2bc4-9c02522cbc7b leader=0c13b324-a174-11a3-ab0e-800956a1f378 leader-id=server-0c13b324-a174-11a3-ab0e-800956a1f378 +2025-06-17T07:44:43.786Z [ERROR] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to appendEntries to: peer="{Voter server-2cf97996-1530-fff5-2bc4-9c02522cbc7b 2cf97996-1530-fff5-2bc4-9c02522cbc7b}" error="failed to connect to peer: 2cf97996-1530-fff5-2bc4-9c02522cbc7b" +2025-06-17T07:44:43.787Z [ERROR] server-0c13b324-a174-11a3-ab0e-800956a1f378: failed to appendEntries to: peer="{Voter server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d f15e41c2-e2eb-b595-fdfd-8f98b39e511d}" error="failed to connect to peer: f15e41c2-e2eb-b595-fdfd-8f98b39e511d" +2025-06-17T07:44:43.808Z [WARN] server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d: heartbeat timeout reached, starting election: last-leader-addr=0c13b324-a174-11a3-ab0e-800956a1f378 last-leader-id=server-0c13b324-a174-11a3-ab0e-800956a1f378 +2025-06-17T07:44:43.808Z [INFO] server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d: entering candidate state: node="Node at f15e41c2-e2eb-b595-fdfd-8f98b39e511d [Candidate]" term=3 +2025-06-17T07:44:43.808Z [INFO] server-2cf97996-1530-fff5-2bc4-9c02522cbc7b: duplicate requestVote for same term: term=3 +2025-06-17T07:44:43.808Z [ERROR] server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d: failed to make requestVote RPC: target="{Voter server-0c13b324-a174-11a3-ab0e-800956a1f378 0c13b324-a174-11a3-ab0e-800956a1f378}" error="failed to connect to peer: 0c13b324-a174-11a3-ab0e-800956a1f378" term=3 +2025-06-17T07:44:43.818Z [INFO] server-f15e41c2-e2eb-b595-fdfd-8f98b39e511d: entering follower state: follower="Node at f15e41c2-e2eb-b595-fdfd-8f98b39e511d [Follower]" leader-address= leader-id= +--- PASS: TestRaft_VoteWithNoIDNoAddr (0.49s) === RUN TestRaft_runFollower_State_Transition === RUN TestRaft_runFollower_State_Transition/NonVoter -2024-05-15T01:18:52.671Z [INFO] 127.0.0.1:33437 :: starting node: addr=127.0.0.1:33437 -2024-05-15T01:18:52.671Z [INFO] 127.0.0.1:33437 :: initial configuration: index=0 servers=[] -2024-05-15T01:18:52.672Z [INFO] 127.0.0.1:33437 :: entering follower state: follower="Node at 127.0.0.1:33437 [Follower]" leader-address= leader-id= -2024-05-15T01:18:52.739Z [WARN] 127.0.0.1:33437 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-06-17T07:44:43.819Z [INFO] 127.0.0.1:39923 :: starting node: addr=127.0.0.1:39923 +2025-06-17T07:44:43.819Z [INFO] 127.0.0.1:39923 :: initial configuration: index=0 servers=[] +2025-06-17T07:44:43.819Z [INFO] 127.0.0.1:39923 :: entering follower state: follower="Node at 127.0.0.1:39923 [Follower]" leader-address= leader-id= +2025-06-17T07:44:43.882Z [WARN] 127.0.0.1:39923 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election === RUN TestRaft_runFollower_State_Transition/Voter -2024-05-15T01:18:52.823Z [INFO] 127.0.0.1:33363 :: starting node: addr=127.0.0.1:33363 -2024-05-15T01:18:52.823Z [INFO] 127.0.0.1:33363 :: initial configuration: index=0 servers=[] -2024-05-15T01:18:52.823Z [INFO] 127.0.0.1:33363 :: entering follower state: follower="Node at 127.0.0.1:33363 [Follower]" leader-address= leader-id= -2024-05-15T01:18:52.883Z [WARN] 127.0.0.1:33363 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:43.970Z [INFO] 127.0.0.1:33899 :: starting node: addr=127.0.0.1:33899 +2025-06-17T07:44:43.970Z [INFO] 127.0.0.1:33899 :: initial configuration: index=0 servers=[] +2025-06-17T07:44:43.970Z [INFO] 127.0.0.1:33899 :: entering follower state: follower="Node at 127.0.0.1:33899 [Follower]" leader-address= leader-id= +2025-06-17T07:44:44.068Z [WARN] 127.0.0.1:33899 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= === RUN TestRaft_runFollower_State_Transition/Not_in_Config -2024-05-15T01:18:52.974Z [INFO] 127.0.0.1:44345 :: starting node: addr=127.0.0.1:44345 -2024-05-15T01:18:52.974Z [INFO] 127.0.0.1:44345 :: initial configuration: index=0 servers=[] -2024-05-15T01:18:52.974Z [INFO] 127.0.0.1:44345 :: entering follower state: follower="Node at 127.0.0.1:44345 [Follower]" leader-address= leader-id= -2024-05-15T01:18:53.051Z [WARN] 127.0.0.1:44345 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-06-17T07:44:44.121Z [INFO] 127.0.0.1:46481 :: starting node: addr=127.0.0.1:46481 +2025-06-17T07:44:44.121Z [INFO] 127.0.0.1:46481 :: initial configuration: index=0 servers=[] +2025-06-17T07:44:44.121Z [INFO] 127.0.0.1:46481 :: entering follower state: follower="Node at 127.0.0.1:46481 [Follower]" leader-address= leader-id= +2025-06-17T07:44:44.205Z [WARN] 127.0.0.1:46481 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election --- PASS: TestRaft_runFollower_State_Transition (0.45s) --- PASS: TestRaft_runFollower_State_Transition/NonVoter (0.15s) --- PASS: TestRaft_runFollower_State_Transition/Voter (0.15s) --- PASS: TestRaft_runFollower_State_Transition/Not_in_Config (0.15s) === RUN TestRaft_runFollower_ReloadTimeoutConfigs -2024-05-15T01:18:53.125Z [INFO] 127.0.0.1:43229 :: starting node: addr=127.0.0.1:43229 -2024-05-15T01:18:53.125Z [INFO] 127.0.0.1:43229 :: initial configuration: index=0 servers=[] -2024-05-15T01:18:53.125Z [INFO] 127.0.0.1:43229 :: entering follower state: follower="Node at 127.0.0.1:43229 [Follower]" leader-address= leader-id= -2024-05-15T01:18:53.125Z [WARN] 127.0.0.1:43229 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-17T07:44:44.271Z [INFO] 127.0.0.1:45031 :: starting node: addr=127.0.0.1:45031 +2025-06-17T07:44:44.271Z [INFO] 127.0.0.1:45031 :: initial configuration: index=0 servers=[] +2025-06-17T07:44:44.271Z [INFO] 127.0.0.1:45031 :: entering follower state: follower="Node at 127.0.0.1:45031 [Follower]" leader-address= leader-id= +2025-06-17T07:44:44.271Z [WARN] 127.0.0.1:45031 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= --- PASS: TestRaft_runFollower_ReloadTimeoutConfigs (0.15s) === RUN TestSaturationMetric === RUN TestSaturationMetric/without_smoothing @@ -4234,7 +4277,7 @@ === RUN TestOverrideNotifyBool --- PASS: TestOverrideNotifyBool (0.00s) PASS -ok github.com/hashicorp/raft 113.836s +ok github.com/hashicorp/raft 113.037s make[1]: Leaving directory '/build/reproducible-path/golang-github-hashicorp-raft-1.5.0' create-stamp debian/debhelper-build-stamp dh_testroot -O--buildsystem=golang @@ -4262,12 +4305,14 @@ dpkg-buildpackage: info: binary-only upload (no source included) dpkg-genchanges: info: including full source code in upload I: copying local configuration +I: user script /srv/workspace/pbuilder/102804/tmp/hooks/B01_cleanup starting +I: user script /srv/workspace/pbuilder/102804/tmp/hooks/B01_cleanup finished I: unmounting dev/ptmx filesystem I: unmounting dev/pts filesystem I: unmounting dev/shm filesystem I: unmounting proc filesystem I: unmounting sys filesystem I: cleaning the build env -I: removing directory /srv/workspace/pbuilder/53785 and its subdirectories -I: Current time: Tue May 14 13:19:00 -12 2024 -I: pbuilder-time-stamp: 1715735940 +I: removing directory /srv/workspace/pbuilder/102804 and its subdirectories +I: Current time: Tue Jun 17 21:44:48 +14 2025 +I: pbuilder-time-stamp: 1750146288