Diff of the two buildlogs: -- --- b1/build.log 2024-05-01 17:12:05.353047690 +0000 +++ b2/build.log 2024-05-01 17:15:03.861443933 +0000 @@ -1,6 +1,6 @@ I: pbuilder: network access will be disabled during build -I: Current time: Tue Jun 3 11:32:04 -12 2025 -I: pbuilder-time-stamp: 1748993524 +I: Current time: Wed Jun 4 13:35:08 +14 2025 +I: pbuilder-time-stamp: 1748993708 I: Building the build Environment I: extracting base tarball [/var/cache/pbuilder/unstable-reproducible-base.tgz] I: copying local configuration @@ -28,52 +28,84 @@ dpkg-source: info: applying 0001-Skip-unreliable-tests.patch I: Not using root during the build. I: Installing the build-deps -I: user script /srv/workspace/pbuilder/2735949/tmp/hooks/D02_print_environment starting +I: user script /srv/workspace/pbuilder/4129085/tmp/hooks/D01_modify_environment starting +debug: Running on codethink03-arm64. +I: Changing host+domainname to test build reproducibility +I: Adding a custom variable just for the fun of it... +I: Changing /bin/sh to bash +'/bin/sh' -> '/bin/bash' +lrwxrwxrwx 1 root root 9 Jun 3 23:35 /bin/sh -> /bin/bash +I: Setting pbuilder2's login shell to /bin/bash +I: Setting pbuilder2's GECOS to second user,second room,second work-phone,second home-phone,second other +I: user script /srv/workspace/pbuilder/4129085/tmp/hooks/D01_modify_environment finished +I: user script /srv/workspace/pbuilder/4129085/tmp/hooks/D02_print_environment starting I: set - BUILDDIR='/build/reproducible-path' - BUILDUSERGECOS='first user,first room,first work-phone,first home-phone,first other' - BUILDUSERNAME='pbuilder1' - BUILD_ARCH='arm64' - DEBIAN_FRONTEND='noninteractive' + BASH=/bin/sh + BASHOPTS=checkwinsize:cmdhist:complete_fullquote:extquote:force_fignore:globasciiranges:globskipdots:hostcomplete:interactive_comments:patsub_replacement:progcomp:promptvars:sourcepath + BASH_ALIASES=() + BASH_ARGC=() + BASH_ARGV=() + BASH_CMDS=() + BASH_LINENO=([0]="12" [1]="0") + BASH_LOADABLES_PATH=/usr/local/lib/bash:/usr/lib/bash:/opt/local/lib/bash:/usr/pkg/lib/bash:/opt/pkg/lib/bash:. + BASH_SOURCE=([0]="/tmp/hooks/D02_print_environment" [1]="/tmp/hooks/D02_print_environment") + BASH_VERSINFO=([0]="5" [1]="2" [2]="21" [3]="1" [4]="release" [5]="aarch64-unknown-linux-gnu") + BASH_VERSION='5.2.21(1)-release' + BUILDDIR=/build/reproducible-path + BUILDUSERGECOS='second user,second room,second work-phone,second home-phone,second other' + BUILDUSERNAME=pbuilder2 + BUILD_ARCH=arm64 + DEBIAN_FRONTEND=noninteractive DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=12 ' - DISTRIBUTION='unstable' - HOME='/root' - HOST_ARCH='arm64' + DIRSTACK=() + DISTRIBUTION=unstable + EUID=0 + FUNCNAME=([0]="Echo" [1]="main") + GROUPS=() + HOME=/root + HOSTNAME=i-capture-the-hostname + HOSTTYPE=aarch64 + HOST_ARCH=arm64 IFS=' ' - INVOCATION_ID='9c7cd8c5f7534c30a51584eb8f25dff3' - LANG='C' - LANGUAGE='en_US:en' - LC_ALL='C' - MAIL='/var/mail/root' - OPTIND='1' - PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/games' - PBCURRENTCOMMANDLINEOPERATION='build' - PBUILDER_OPERATION='build' - PBUILDER_PKGDATADIR='/usr/share/pbuilder' - PBUILDER_PKGLIBDIR='/usr/lib/pbuilder' - PBUILDER_SYSCONFDIR='/etc' - PPID='2735949' - PS1='# ' - PS2='> ' + INVOCATION_ID=f760209e527f41d1bffcd604fdce6f9d + LANG=C + LANGUAGE=nl_BE:nl + LC_ALL=C + MACHTYPE=aarch64-unknown-linux-gnu + MAIL=/var/mail/root + OPTERR=1 + OPTIND=1 + OSTYPE=linux-gnu + PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path + PBCURRENTCOMMANDLINEOPERATION=build + PBUILDER_OPERATION=build + PBUILDER_PKGDATADIR=/usr/share/pbuilder + PBUILDER_PKGLIBDIR=/usr/lib/pbuilder + PBUILDER_SYSCONFDIR=/etc + PIPESTATUS=([0]="0") + POSIXLY_CORRECT=y + PPID=4129085 PS4='+ ' - PWD='/' - SHELL='/bin/bash' - SHLVL='2' - SUDO_COMMAND='/usr/bin/timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.Jye48Rd2/pbuilderrc_6shd --distribution unstable --hookdir /etc/pbuilder/first-build-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/unstable-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.Jye48Rd2/b1 --logfile b1/build.log golang-github-hashicorp-raft_1.5.0-1.dsc' - SUDO_GID='109' - SUDO_UID='104' - SUDO_USER='jenkins' - TERM='unknown' - TZ='/usr/share/zoneinfo/Etc/GMT+12' - USER='root' - _='/usr/bin/systemd-run' - http_proxy='http://192.168.101.4:3128' + PWD=/ + SHELL=/bin/bash + SHELLOPTS=braceexpand:errexit:hashall:interactive-comments:posix + SHLVL=3 + SUDO_COMMAND='/usr/bin/timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.Jye48Rd2/pbuilderrc_knR3 --distribution unstable --hookdir /etc/pbuilder/rebuild-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/unstable-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.Jye48Rd2/b2 --logfile b2/build.log golang-github-hashicorp-raft_1.5.0-1.dsc' + SUDO_GID=109 + SUDO_UID=104 + SUDO_USER=jenkins + TERM=unknown + TZ=/usr/share/zoneinfo/Etc/GMT-14 + UID=0 + USER=root + _='I: set' + http_proxy=http://192.168.101.4:3128 I: uname -a - Linux codethink01-arm64 6.1.0-20-cloud-arm64 #1 SMP Debian 6.1.85-1 (2024-04-11) aarch64 GNU/Linux + Linux i-capture-the-hostname 6.1.0-20-cloud-arm64 #1 SMP Debian 6.1.85-1 (2024-04-11) aarch64 GNU/Linux I: ls -l /bin lrwxrwxrwx 1 root root 7 Jun 3 14:05 /bin -> usr/bin -I: user script /srv/workspace/pbuilder/2735949/tmp/hooks/D02_print_environment finished +I: user script /srv/workspace/pbuilder/4129085/tmp/hooks/D02_print_environment finished -> Attempting to satisfy build-dependencies -> Creating pbuilder-satisfydepends-dummy package Package: pbuilder-satisfydepends-dummy @@ -224,7 +256,7 @@ Get: 101 http://deb.debian.org/debian unstable/main arm64 golang-gopkg-check.v1-dev all 0.0+git20200902.038fdea-1 [32.1 kB] Get: 102 http://deb.debian.org/debian unstable/main arm64 golang-gopkg-mgo.v2-dev all 2016.08.01-7 [316 kB] Get: 103 http://deb.debian.org/debian unstable/main arm64 golang-github-hashicorp-go-msgpack-dev all 0.5.5-1 [43.3 kB] -Fetched 86.8 MB in 1s (119 MB/s) +Fetched 86.8 MB in 0s (223 MB/s) debconf: delaying package configuration, since apt-utils is not installed Selecting previously unselected package sensible-utils. (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 19735 files and directories currently installed.) @@ -651,7 +683,11 @@ Building tag database... -> Finished parsing the build-deps I: Building the package -I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.5.0/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.5.0-1_source.changes +I: user script /srv/workspace/pbuilder/4129085/tmp/hooks/A99_set_merged_usr starting +Not re-configuring usrmerge for unstable +I: user script /srv/workspace/pbuilder/4129085/tmp/hooks/A99_set_merged_usr finished +hostname: Name or service not known +I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.5.0/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.5.0-1_source.changes dpkg-buildpackage: info: source package golang-github-hashicorp-raft dpkg-buildpackage: info: source version 1.5.0-1 dpkg-buildpackage: info: source distribution unstable @@ -670,163 +706,163 @@ dh_auto_configure -O--buildsystem=golang dh_auto_build -O--buildsystem=golang cd obj-aarch64-linux-gnu && go install -trimpath -v -p 12 github.com/hashicorp/raft github.com/hashicorp/raft/bench -internal/goos -internal/unsafeheader +internal/goarch +internal/godebugs unicode/utf8 +internal/unsafeheader +internal/cpu internal/coverage/rtcov -internal/godebugs -container/list -internal/race -unicode -internal/goarch internal/goexperiment -sync/atomic -internal/itoa -math/bits -crypto/internal/alias -internal/abi +runtime/internal/syscall +internal/race internal/chacha8rand +container/list +internal/abi runtime/internal/math -internal/cpu -runtime/internal/syscall +internal/itoa +internal/goos +sync/atomic +unicode +math/bits runtime/internal/sys +crypto/internal/alias crypto/subtle -math crypto/internal/boring/sig cmp encoding unicode/utf16 log/internal -slices internal/nettrace vendor/golang.org/x/crypto/cryptobyte/asn1 vendor/golang.org/x/crypto/internal/alias +slices +math internal/bytealg runtime/internal/atomic runtime internal/reflectlite sync internal/testlog -internal/bisect internal/singleflight +internal/bisect runtime/cgo +internal/godebug errors sort -internal/godebug +math/rand io internal/oserror -vendor/golang.org/x/net/dns/dnsmessage -math/rand -internal/intern -crypto/internal/nistec/fiat -internal/safefilepath strconv -github.com/hashicorp/golang-lru/simplelru +internal/safefilepath +internal/intern path +github.com/hashicorp/golang-lru/simplelru syscall +vendor/golang.org/x/net/dns/dnsmessage +crypto/internal/nistec/fiat bytes -strings hash +strings crypto/internal/randutil internal/saferio hash/crc32 hash/crc64 vendor/golang.org/x/text/transform crypto -reflect net/netip +reflect crypto/rc4 bufio github.com/hashicorp/go-immutable-radix net/http/internal/ascii regexp/syntax html +regexp +internal/syscall/execenv time internal/syscall/unix -internal/syscall/execenv -regexp context -internal/poll io/fs -embed +internal/poll encoding/binary internal/fmtsort +embed os crypto/cipher -encoding/base64 crypto/internal/edwards25519/field -crypto/internal/nistec crypto/md5 vendor/golang.org/x/crypto/internal/poly1305 +crypto/internal/nistec +encoding/base64 golang.org/x/sys/unix +encoding/pem +crypto/internal/edwards25519 crypto/internal/boring crypto/des vendor/golang.org/x/crypto/chacha20 crypto/aes -crypto/sha512 crypto/hmac +crypto/sha512 crypto/sha1 -crypto/internal/edwards25519 -encoding/pem -vendor/golang.org/x/crypto/hkdf crypto/sha256 +vendor/golang.org/x/crypto/hkdf vendor/golang.org/x/crypto/chacha20poly1305 -fmt -path/filepath +io/ioutil os/signal +path/filepath internal/sysinfo -io/ioutil +fmt net crypto/ecdh -math/big -encoding/json -log -compress/flate encoding/hex net/url -vendor/golang.org/x/text/unicode/norm +math/big +encoding/json mime/quotedprintable +vendor/golang.org/x/text/unicode/norm mime vendor/golang.org/x/net/http2/hpack +compress/flate +log net/http/internal encoding/gob go/token -vendor/golang.org/x/text/unicode/bidi text/template/parse flag +vendor/golang.org/x/text/unicode/bidi runtime/debug compress/gzip runtime/trace testing vendor/golang.org/x/text/secure/bidirule -vendor/golang.org/x/net/idna text/template +vendor/golang.org/x/net/idna github.com/mattn/go-isatty github.com/mattn/go-colorable crypto/rand crypto/elliptic -crypto/internal/boring/bbig crypto/internal/bigmod +crypto/internal/boring/bbig encoding/asn1 crypto/dsa github.com/fatih/color crypto/ed25519 -github.com/hashicorp/go-hclog -html/template crypto/rsa vendor/golang.org/x/crypto/cryptobyte crypto/x509/pkix +github.com/hashicorp/go-hclog +html/template crypto/ecdsa -vendor/golang.org/x/net/http/httpproxy net/textproto +vendor/golang.org/x/net/http/httpproxy crypto/x509 vendor/golang.org/x/net/http/httpguts mime/multipart crypto/tls net/http/httptrace net/http -github.com/armon/go-metrics net/rpc +github.com/armon/go-metrics github.com/hashicorp/go-msgpack/codec github.com/hashicorp/raft github.com/hashicorp/raft/bench @@ -876,32 +912,32 @@ === RUN TestFileSnapshotSinkImpl --- PASS: TestFileSnapshotSinkImpl (0.00s) === RUN TestFileSS_CreateSnapshotMissingParentDir -2025-06-03T23:33:04.986Z [INFO] creating new snapshot: path=/tmp/raft1079390983/raft3110873911/snapshots/3-10-1748993584986.tmp +2025-06-03T23:36:04.685Z [INFO] creating new snapshot: path=/tmp/raft4268682134/raft2816422242/snapshots/3-10-1748993764685.tmp --- PASS: TestFileSS_CreateSnapshotMissingParentDir (0.00s) === RUN TestFileSS_CreateSnapshot -2025-06-03T23:33:04.986Z [INFO] creating new snapshot: path=/tmp/raft3538014368/snapshots/3-10-1748993584986.tmp -2025-06-03T23:33:04.987Z [WARN] found temporary snapshot: name=3-10-1748993584986.tmp +2025-06-03T23:36:04.685Z [INFO] creating new snapshot: path=/tmp/raft372426339/snapshots/3-10-1748993764685.tmp +2025-06-03T23:36:04.686Z [WARN] found temporary snapshot: name=3-10-1748993764685.tmp --- PASS: TestFileSS_CreateSnapshot (0.00s) === RUN TestFileSS_CancelSnapshot -2025-06-03T23:33:04.987Z [INFO] creating new snapshot: path=/tmp/raft125041875/snapshots/3-10-1748993584987.tmp +2025-06-03T23:36:04.686Z [INFO] creating new snapshot: path=/tmp/raft2702012852/snapshots/3-10-1748993764686.tmp --- PASS: TestFileSS_CancelSnapshot (0.00s) === RUN TestFileSS_Retention -2025-06-03T23:33:04.988Z [INFO] creating new snapshot: path=/tmp/raft3578083135/snapshots/3-10-1748993584988.tmp -2025-06-03T23:33:04.988Z [INFO] creating new snapshot: path=/tmp/raft3578083135/snapshots/3-11-1748993584988.tmp -2025-06-03T23:33:04.988Z [INFO] creating new snapshot: path=/tmp/raft3578083135/snapshots/3-12-1748993584988.tmp -2025-06-03T23:33:04.988Z [INFO] reaping snapshot: path=/tmp/raft3578083135/snapshots/3-10-1748993584988 -2025-06-03T23:33:04.988Z [INFO] creating new snapshot: path=/tmp/raft3578083135/snapshots/3-13-1748993584988.tmp -2025-06-03T23:33:04.989Z [INFO] reaping snapshot: path=/tmp/raft3578083135/snapshots/3-11-1748993584988 -2025-06-03T23:33:04.989Z [INFO] creating new snapshot: path=/tmp/raft3578083135/snapshots/3-14-1748993584989.tmp -2025-06-03T23:33:04.989Z [INFO] reaping snapshot: path=/tmp/raft3578083135/snapshots/3-12-1748993584988 +2025-06-03T23:36:04.687Z [INFO] creating new snapshot: path=/tmp/raft2174999238/snapshots/3-10-1748993764687.tmp +2025-06-03T23:36:04.687Z [INFO] creating new snapshot: path=/tmp/raft2174999238/snapshots/3-11-1748993764687.tmp +2025-06-03T23:36:04.687Z [INFO] creating new snapshot: path=/tmp/raft2174999238/snapshots/3-12-1748993764687.tmp +2025-06-03T23:36:04.687Z [INFO] reaping snapshot: path=/tmp/raft2174999238/snapshots/3-10-1748993764687 +2025-06-03T23:36:04.688Z [INFO] creating new snapshot: path=/tmp/raft2174999238/snapshots/3-13-1748993764688.tmp +2025-06-03T23:36:04.688Z [INFO] reaping snapshot: path=/tmp/raft2174999238/snapshots/3-11-1748993764687 +2025-06-03T23:36:04.688Z [INFO] creating new snapshot: path=/tmp/raft2174999238/snapshots/3-14-1748993764688.tmp +2025-06-03T23:36:04.688Z [INFO] reaping snapshot: path=/tmp/raft2174999238/snapshots/3-12-1748993764687 --- PASS: TestFileSS_Retention (0.00s) === RUN TestFileSS_BadPerm --- PASS: TestFileSS_BadPerm (0.00s) === RUN TestFileSS_MissingParentDir --- PASS: TestFileSS_MissingParentDir (0.00s) === RUN TestFileSS_Ordering -2025-06-03T23:33:04.990Z [INFO] creating new snapshot: path=/tmp/raft3962621337/snapshots/5-130350-1748993584990.tmp -2025-06-03T23:33:04.991Z [INFO] creating new snapshot: path=/tmp/raft3962621337/snapshots/36-204917-1748993584991.tmp +2025-06-03T23:36:04.689Z [INFO] creating new snapshot: path=/tmp/raft3239787093/snapshots/5-130350-1748993764689.tmp +2025-06-03T23:36:04.690Z [INFO] creating new snapshot: path=/tmp/raft3239787093/snapshots/36-204917-1748993764690.tmp --- PASS: TestFileSS_Ordering (0.00s) === RUN TestDeferFutureSuccess --- PASS: TestDeferFutureSuccess (0.00s) @@ -938,7 +974,7 @@ === RUN TestEmitsLogStoreMetrics --- PASS: TestEmitsLogStoreMetrics (0.01s) === RUN TestNetworkTransport_CloseStreams ---- PASS: TestNetworkTransport_CloseStreams (0.01s) +--- PASS: TestNetworkTransport_CloseStreams (0.00s) === RUN TestNetworkTransport_StartStop --- PASS: TestNetworkTransport_StartStop (0.00s) === RUN TestNetworkTransport_Heartbeat_FastPath @@ -951,12 +987,12 @@ --- PASS: TestNetworkTransport_AppendEntriesPipeline_CloseStreams (0.02s) === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=0 -2025-06-03T23:33:05.093Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" +2025-06-03T23:36:04.792Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=1 === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=2 -2025-06-03T23:33:05.145Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" +2025-06-03T23:36:04.844Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=10 -2025-06-03T23:33:05.199Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" +2025-06-03T23:36:04.897Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight (0.16s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=0 (0.05s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=1 (0.00s) @@ -973,27 +1009,27 @@ === RUN TestNetworkTransport_PooledConn --- PASS: TestNetworkTransport_PooledConn (0.00s) === RUN TestNetworkTransport_ListenBackoff - net_transport_test.go:839: countingWriter: 2025-06-03T23:33:05.206Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-03T23:36:04.904Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-06-03T23:33:05.211Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-03T23:36:04.910Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-06-03T23:33:05.222Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-03T23:36:04.921Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-06-03T23:33:05.243Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-03T23:36:04.941Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-06-03T23:33:05.283Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-03T23:36:04.982Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-06-03T23:33:05.364Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-03T23:36:05.063Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-06-03T23:33:05.525Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-03T23:36:05.223Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-06-03T23:33:05.846Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-03T23:36:05.544Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-06-03T23:33:06.486Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-03T23:36:06.185Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-06-03T23:33:07.489Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-03T23:36:07.187Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-06-03T23:33:08.491Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-03T23:36:08.189Z [ERROR] test: failed to accept connection: error="intentional error in test" --- PASS: TestNetworkTransport_ListenBackoff (4.00s) === RUN TestPeersJSON_BadConfiguration @@ -1003,2519 +1039,2543 @@ === RUN TestPeersJSON_ReadConfigJSON --- PASS: TestPeersJSON_ReadConfigJSON (0.00s) === RUN TestRaft_StartStop -2025-06-03T23:33:09.207Z [INFO] server-1ba76b79-c9c2-dd55-dd14-229afc6f5a67: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ba76b79-c9c2-dd55-dd14-229afc6f5a67 Address:1ba76b79-c9c2-dd55-dd14-229afc6f5a67}]" +2025-06-03T23:36:08.906Z [INFO] server-ec71bf06-daec-478d-02b2-c312cf78180f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec71bf06-daec-478d-02b2-c312cf78180f Address:ec71bf06-daec-478d-02b2-c312cf78180f}]" --- PASS: TestRaft_StartStop (0.00s) === RUN TestRaft_AfterShutdown -2025-06-03T23:33:09.208Z [INFO] server-172bbbe9-be6c-e268-9867-5b3df98f3606: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-172bbbe9-be6c-e268-9867-5b3df98f3606 Address:172bbbe9-be6c-e268-9867-5b3df98f3606}]" +2025-06-03T23:36:08.907Z [INFO] server-6a4b4aec-323c-01cf-20b4-80916d8a738c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6a4b4aec-323c-01cf-20b4-80916d8a738c Address:6a4b4aec-323c-01cf-20b4-80916d8a738c}]" --- PASS: TestRaft_AfterShutdown (0.00s) === RUN TestRaft_LiveBootstrap -2025-06-03T23:33:09.208Z [INFO] server-afc18362-99a4-0285-dd30-df9a4cf4d597: initial configuration: index=0 servers=[] -2025-06-03T23:33:09.208Z [INFO] server-63ed14f4-9e87-d57f-0742-04182ac99494: initial configuration: index=0 servers=[] -2025-06-03T23:33:09.208Z [INFO] server-9227f336-6744-b5b1-1ae6-4acd8487220c: initial configuration: index=0 servers=[] -2025-06-03T23:33:09.208Z [INFO] server-afc18362-99a4-0285-dd30-df9a4cf4d597: entering follower state: follower="Node at afc18362-99a4-0285-dd30-df9a4cf4d597 [Follower]" leader-address= leader-id= -2025-06-03T23:33:09.208Z [INFO] server-63ed14f4-9e87-d57f-0742-04182ac99494: entering follower state: follower="Node at 63ed14f4-9e87-d57f-0742-04182ac99494 [Follower]" leader-address= leader-id= -2025-06-03T23:33:09.208Z [INFO] server-9227f336-6744-b5b1-1ae6-4acd8487220c: entering follower state: follower="Node at 9227f336-6744-b5b1-1ae6-4acd8487220c [Follower]" leader-address= leader-id= -2025-06-03T23:33:09.209Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:09.261Z [WARN] server-afc18362-99a4-0285-dd30-df9a4cf4d597: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:09.261Z [INFO] server-afc18362-99a4-0285-dd30-df9a4cf4d597: entering candidate state: node="Node at afc18362-99a4-0285-dd30-df9a4cf4d597 [Candidate]" term=2 -2025-06-03T23:33:09.261Z [INFO] server-afc18362-99a4-0285-dd30-df9a4cf4d597: election won: term=2 tally=2 -2025-06-03T23:33:09.261Z [INFO] server-afc18362-99a4-0285-dd30-df9a4cf4d597: entering leader state: leader="Node at afc18362-99a4-0285-dd30-df9a4cf4d597 [Leader]" -2025-06-03T23:33:09.261Z [INFO] server-afc18362-99a4-0285-dd30-df9a4cf4d597: added peer, starting replication: peer=server-63ed14f4-9e87-d57f-0742-04182ac99494 -2025-06-03T23:33:09.261Z [INFO] server-afc18362-99a4-0285-dd30-df9a4cf4d597: added peer, starting replication: peer=server-9227f336-6744-b5b1-1ae6-4acd8487220c -2025-06-03T23:33:09.261Z [WARN] server-9227f336-6744-b5b1-1ae6-4acd8487220c: failed to get previous log: previous-index=1 last-index=0 error="log not found" -2025-06-03T23:33:09.261Z [WARN] server-63ed14f4-9e87-d57f-0742-04182ac99494: failed to get previous log: previous-index=1 last-index=0 error="log not found" -2025-06-03T23:33:09.261Z [WARN] server-afc18362-99a4-0285-dd30-df9a4cf4d597: appendEntries rejected, sending older logs: peer="{Voter server-9227f336-6744-b5b1-1ae6-4acd8487220c 9227f336-6744-b5b1-1ae6-4acd8487220c}" next=1 -2025-06-03T23:33:09.261Z [WARN] server-afc18362-99a4-0285-dd30-df9a4cf4d597: appendEntries rejected, sending older logs: peer="{Voter server-63ed14f4-9e87-d57f-0742-04182ac99494 63ed14f4-9e87-d57f-0742-04182ac99494}" next=1 -2025-06-03T23:33:09.261Z [INFO] server-afc18362-99a4-0285-dd30-df9a4cf4d597: pipelining replication: peer="{Voter server-9227f336-6744-b5b1-1ae6-4acd8487220c 9227f336-6744-b5b1-1ae6-4acd8487220c}" -2025-06-03T23:33:09.262Z [INFO] server-afc18362-99a4-0285-dd30-df9a4cf4d597: pipelining replication: peer="{Voter server-63ed14f4-9e87-d57f-0742-04182ac99494 63ed14f4-9e87-d57f-0742-04182ac99494}" -2025-06-03T23:33:09.367Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:09.261831671 +0000 UTC m=+4.282651081 (2 nodes), 52.6768ms from start of poll, 53.14372ms from cluster start. Timeout at 2025-06-03 23:33:09.367053831 +0000 UTC m=+4.387873201, 105.22212ms after stability -2025-06-03T23:33:09.367Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:09.472Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:09.367187351 +0000 UTC m=+4.388006721 (1 nodes), 2.96µs from start of poll, 158.49936ms from cluster start. Timeout at 2025-06-03 23:33:09.472834831 +0000 UTC m=+4.493654201, 105.64748ms after stability -2025-06-03T23:33:09.484Z [INFO] server-afc18362-99a4-0285-dd30-df9a4cf4d597: aborting pipeline replication: peer="{Voter server-9227f336-6744-b5b1-1ae6-4acd8487220c 9227f336-6744-b5b1-1ae6-4acd8487220c}" -2025-06-03T23:33:09.484Z [INFO] server-afc18362-99a4-0285-dd30-df9a4cf4d597: aborting pipeline replication: peer="{Voter server-63ed14f4-9e87-d57f-0742-04182ac99494 63ed14f4-9e87-d57f-0742-04182ac99494}" -2025-06-03T23:33:09.985Z [ERROR] server-afc18362-99a4-0285-dd30-df9a4cf4d597: failed to heartbeat to: peer=63ed14f4-9e87-d57f-0742-04182ac99494 backoff time=10ms error="command timed out" ---- PASS: TestRaft_LiveBootstrap (0.78s) +2025-06-03T23:36:08.908Z [INFO] server-491dc20b-ce1f-8544-f7a3-b702ac4b7a14: initial configuration: index=0 servers=[] +2025-06-03T23:36:08.908Z [INFO] server-b57293de-f0d7-423c-b939-fde0754c85b2: initial configuration: index=0 servers=[] +2025-06-03T23:36:08.908Z [INFO] server-3d06901b-3d6e-c2e2-1d3e-b0ace6aa697d: initial configuration: index=0 servers=[] +2025-06-03T23:36:08.908Z [INFO] server-b57293de-f0d7-423c-b939-fde0754c85b2: entering follower state: follower="Node at b57293de-f0d7-423c-b939-fde0754c85b2 [Follower]" leader-address= leader-id= +2025-06-03T23:36:08.908Z [INFO] server-3d06901b-3d6e-c2e2-1d3e-b0ace6aa697d: entering follower state: follower="Node at 3d06901b-3d6e-c2e2-1d3e-b0ace6aa697d [Follower]" leader-address= leader-id= +2025-06-03T23:36:08.908Z [INFO] server-491dc20b-ce1f-8544-f7a3-b702ac4b7a14: entering follower state: follower="Node at 491dc20b-ce1f-8544-f7a3-b702ac4b7a14 [Follower]" leader-address= leader-id= +2025-06-03T23:36:08.908Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:08.958Z [WARN] server-491dc20b-ce1f-8544-f7a3-b702ac4b7a14: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:08.958Z [INFO] server-491dc20b-ce1f-8544-f7a3-b702ac4b7a14: entering candidate state: node="Node at 491dc20b-ce1f-8544-f7a3-b702ac4b7a14 [Candidate]" term=2 +2025-06-03T23:36:08.958Z [INFO] server-491dc20b-ce1f-8544-f7a3-b702ac4b7a14: election won: term=2 tally=2 +2025-06-03T23:36:08.958Z [INFO] server-491dc20b-ce1f-8544-f7a3-b702ac4b7a14: entering leader state: leader="Node at 491dc20b-ce1f-8544-f7a3-b702ac4b7a14 [Leader]" +2025-06-03T23:36:08.958Z [INFO] server-491dc20b-ce1f-8544-f7a3-b702ac4b7a14: added peer, starting replication: peer=server-b57293de-f0d7-423c-b939-fde0754c85b2 +2025-06-03T23:36:08.958Z [INFO] server-491dc20b-ce1f-8544-f7a3-b702ac4b7a14: added peer, starting replication: peer=server-3d06901b-3d6e-c2e2-1d3e-b0ace6aa697d +2025-06-03T23:36:08.959Z [WARN] server-b57293de-f0d7-423c-b939-fde0754c85b2: failed to get previous log: previous-index=1 last-index=0 error="log not found" +2025-06-03T23:36:08.959Z [WARN] server-3d06901b-3d6e-c2e2-1d3e-b0ace6aa697d: failed to get previous log: previous-index=1 last-index=0 error="log not found" +2025-06-03T23:36:08.959Z [WARN] server-491dc20b-ce1f-8544-f7a3-b702ac4b7a14: appendEntries rejected, sending older logs: peer="{Voter server-b57293de-f0d7-423c-b939-fde0754c85b2 b57293de-f0d7-423c-b939-fde0754c85b2}" next=1 +2025-06-03T23:36:08.959Z [WARN] server-491dc20b-ce1f-8544-f7a3-b702ac4b7a14: appendEntries rejected, sending older logs: peer="{Voter server-3d06901b-3d6e-c2e2-1d3e-b0ace6aa697d 3d06901b-3d6e-c2e2-1d3e-b0ace6aa697d}" next=1 +2025-06-03T23:36:08.959Z [INFO] server-491dc20b-ce1f-8544-f7a3-b702ac4b7a14: pipelining replication: peer="{Voter server-b57293de-f0d7-423c-b939-fde0754c85b2 b57293de-f0d7-423c-b939-fde0754c85b2}" +2025-06-03T23:36:08.959Z [INFO] server-491dc20b-ce1f-8544-f7a3-b702ac4b7a14: pipelining replication: peer="{Voter server-3d06901b-3d6e-c2e2-1d3e-b0ace6aa697d 3d06901b-3d6e-c2e2-1d3e-b0ace6aa697d}" +2025-06-03T23:36:09.064Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:08.959044271 +0000 UTC m=+4.280799201 (2 nodes), 50.60684ms from start of poll, 51.07572ms from cluster start. Timeout at 2025-06-03 23:36:09.064816871 +0000 UTC m=+4.386571801, 105.7726ms after stability +2025-06-03T23:36:09.064Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:09.170Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:09.064948471 +0000 UTC m=+4.386703441 (1 nodes), 1.04µs from start of poll, 156.97996ms from cluster start. Timeout at 2025-06-03 23:36:09.169999871 +0000 UTC m=+4.491754801, 105.05136ms after stability +2025-06-03T23:36:09.181Z [INFO] server-491dc20b-ce1f-8544-f7a3-b702ac4b7a14: aborting pipeline replication: peer="{Voter server-3d06901b-3d6e-c2e2-1d3e-b0ace6aa697d 3d06901b-3d6e-c2e2-1d3e-b0ace6aa697d}" +2025-06-03T23:36:09.181Z [INFO] server-491dc20b-ce1f-8544-f7a3-b702ac4b7a14: aborting pipeline replication: peer="{Voter server-b57293de-f0d7-423c-b939-fde0754c85b2 b57293de-f0d7-423c-b939-fde0754c85b2}" +--- PASS: TestRaft_LiveBootstrap (0.27s) === RUN TestRaft_LiveBootstrap_From_NonVoter -2025-06-03T23:33:09.985Z [INFO] server-fb01afd6-c774-3b72-f0e6-84b9301d7f75: initial configuration: index=0 servers=[] -2025-06-03T23:33:09.986Z [INFO] server-0187d1a7-4f74-98d6-371c-2cbdc190787f: initial configuration: index=0 servers=[] -2025-06-03T23:33:09.986Z [INFO] server-0187d1a7-4f74-98d6-371c-2cbdc190787f: entering follower state: follower="Node at 0187d1a7-4f74-98d6-371c-2cbdc190787f [Follower]" leader-address= leader-id= -2025-06-03T23:33:09.986Z [INFO] server-fb01afd6-c774-3b72-f0e6-84b9301d7f75: entering follower state: follower="Node at fb01afd6-c774-3b72-f0e6-84b9301d7f75 [Follower]" leader-address= leader-id= +2025-06-03T23:36:09.181Z [INFO] server-a3f31aec-0db1-adc2-8c25-6a14a835c6a7: initial configuration: index=0 servers=[] +2025-06-03T23:36:09.181Z [INFO] server-43bdc3aa-8f14-7a7b-ed9d-da2c118c12d2: initial configuration: index=0 servers=[] +2025-06-03T23:36:09.181Z [INFO] server-a3f31aec-0db1-adc2-8c25-6a14a835c6a7: entering follower state: follower="Node at a3f31aec-0db1-adc2-8c25-6a14a835c6a7 [Follower]" leader-address= leader-id= +2025-06-03T23:36:09.181Z [INFO] server-43bdc3aa-8f14-7a7b-ed9d-da2c118c12d2: entering follower state: follower="Node at 43bdc3aa-8f14-7a7b-ed9d-da2c118c12d2 [Follower]" leader-address= leader-id= --- PASS: TestRaft_LiveBootstrap_From_NonVoter (0.00s) === RUN TestRaft_RecoverCluster_NoState -2025-06-03T23:33:09.987Z [INFO] server-37984a02-624a-3af0-7053-ead980816025: initial configuration: index=0 servers=[] -2025-06-03T23:33:09.987Z [INFO] server-37984a02-624a-3af0-7053-ead980816025: entering follower state: follower="Node at 37984a02-624a-3af0-7053-ead980816025 [Follower]" leader-address= leader-id= +2025-06-03T23:36:09.182Z [INFO] server-f1a8d22e-3076-3456-fa82-5240b96505b5: initial configuration: index=0 servers=[] --- PASS: TestRaft_RecoverCluster_NoState (0.00s) === RUN TestRaft_RecoverCluster === RUN TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs -2025-06-03T23:33:09.988Z [INFO] server-b52a7256-b2af-6f23-a3da-9702d1c96d9d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b52a7256-b2af-6f23-a3da-9702d1c96d9d Address:b52a7256-b2af-6f23-a3da-9702d1c96d9d} {Suffrage:Voter ID:server-59c331a3-2d38-1413-2ffe-5c2cf953e48a Address:59c331a3-2d38-1413-2ffe-5c2cf953e48a} {Suffrage:Voter ID:server-7e3a1cab-ee72-343c-42d9-5ee8432831ac Address:7e3a1cab-ee72-343c-42d9-5ee8432831ac}]" -2025-06-03T23:33:09.988Z [INFO] server-b52a7256-b2af-6f23-a3da-9702d1c96d9d: entering follower state: follower="Node at b52a7256-b2af-6f23-a3da-9702d1c96d9d [Follower]" leader-address= leader-id= -2025-06-03T23:33:09.988Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b52a7256-b2af-6f23-a3da-9702d1c96d9d Address:b52a7256-b2af-6f23-a3da-9702d1c96d9d} {Suffrage:Voter ID:server-59c331a3-2d38-1413-2ffe-5c2cf953e48a Address:59c331a3-2d38-1413-2ffe-5c2cf953e48a} {Suffrage:Voter ID:server-7e3a1cab-ee72-343c-42d9-5ee8432831ac Address:7e3a1cab-ee72-343c-42d9-5ee8432831ac}]" -2025-06-03T23:33:09.988Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: entering follower state: follower="Node at 59c331a3-2d38-1413-2ffe-5c2cf953e48a [Follower]" leader-address= leader-id= -2025-06-03T23:33:09.988Z [INFO] server-7e3a1cab-ee72-343c-42d9-5ee8432831ac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b52a7256-b2af-6f23-a3da-9702d1c96d9d Address:b52a7256-b2af-6f23-a3da-9702d1c96d9d} {Suffrage:Voter ID:server-59c331a3-2d38-1413-2ffe-5c2cf953e48a Address:59c331a3-2d38-1413-2ffe-5c2cf953e48a} {Suffrage:Voter ID:server-7e3a1cab-ee72-343c-42d9-5ee8432831ac Address:7e3a1cab-ee72-343c-42d9-5ee8432831ac}]" -2025-06-03T23:33:09.989Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:09.989Z [INFO] server-7e3a1cab-ee72-343c-42d9-5ee8432831ac: entering follower state: follower="Node at 7e3a1cab-ee72-343c-42d9-5ee8432831ac [Follower]" leader-address= leader-id= -2025-06-03T23:33:10.050Z [WARN] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:10.050Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: entering candidate state: node="Node at 59c331a3-2d38-1413-2ffe-5c2cf953e48a [Candidate]" term=2 -2025-06-03T23:33:10.050Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: election won: term=2 tally=2 -2025-06-03T23:33:10.050Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: entering leader state: leader="Node at 59c331a3-2d38-1413-2ffe-5c2cf953e48a [Leader]" -2025-06-03T23:33:10.050Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: added peer, starting replication: peer=server-b52a7256-b2af-6f23-a3da-9702d1c96d9d -2025-06-03T23:33:10.050Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: added peer, starting replication: peer=server-7e3a1cab-ee72-343c-42d9-5ee8432831ac -2025-06-03T23:33:10.050Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: pipelining replication: peer="{Voter server-b52a7256-b2af-6f23-a3da-9702d1c96d9d b52a7256-b2af-6f23-a3da-9702d1c96d9d}" -2025-06-03T23:33:10.050Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: pipelining replication: peer="{Voter server-7e3a1cab-ee72-343c-42d9-5ee8432831ac 7e3a1cab-ee72-343c-42d9-5ee8432831ac}" -2025-06-03T23:33:10.156Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:10.050284431 +0000 UTC m=+5.071103801 (1 nodes), 61.2386ms from start of poll, 61.84256ms from cluster start. Timeout at 2025-06-03 23:33:10.156238631 +0000 UTC m=+5.177058001, 105.9542ms after stability -2025-06-03T23:33:10.156Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: aborting pipeline replication: peer="{Voter server-7e3a1cab-ee72-343c-42d9-5ee8432831ac 7e3a1cab-ee72-343c-42d9-5ee8432831ac}" -2025-06-03T23:33:10.156Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: aborting pipeline replication: peer="{Voter server-b52a7256-b2af-6f23-a3da-9702d1c96d9d b52a7256-b2af-6f23-a3da-9702d1c96d9d}" -2025-06-03T23:33:10.156Z [INFO] creating new snapshot: path=/tmp/raft124056001/snapshots/2-2-1748993590156.tmp -2025-06-03T23:33:10.157Z [INFO] server-b52a7256-b2af-6f23-a3da-9702d1c96d9d: starting restore from snapshot: id=2-2-1748993590156 last-index=2 last-term=2 size-in-bytes=1 -2025-06-03T23:33:10.157Z [INFO] server-b52a7256-b2af-6f23-a3da-9702d1c96d9d: snapshot restore progress: id=2-2-1748993590156 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2025-06-03T23:33:10.157Z [INFO] server-b52a7256-b2af-6f23-a3da-9702d1c96d9d: restored from snapshot: id=2-2-1748993590156 last-index=2 last-term=2 size-in-bytes=1 -2025-06-03T23:33:10.157Z [INFO] server-b52a7256-b2af-6f23-a3da-9702d1c96d9d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b52a7256-b2af-6f23-a3da-9702d1c96d9d Address:b52a7256-b2af-6f23-a3da-9702d1c96d9d} {Suffrage:Voter ID:server-59c331a3-2d38-1413-2ffe-5c2cf953e48a Address:59c331a3-2d38-1413-2ffe-5c2cf953e48a} {Suffrage:Voter ID:server-7e3a1cab-ee72-343c-42d9-5ee8432831ac Address:7e3a1cab-ee72-343c-42d9-5ee8432831ac}]" -2025-06-03T23:33:10.157Z [INFO] creating new snapshot: path=/tmp/raft824244361/snapshots/2-2-1748993590157.tmp -2025-06-03T23:33:10.157Z [INFO] server-b52a7256-b2af-6f23-a3da-9702d1c96d9d: entering follower state: follower="Node at b52a7256-b2af-6f23-a3da-9702d1c96d9d [Follower]" leader-address= leader-id= -2025-06-03T23:33:10.157Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: starting restore from snapshot: id=2-2-1748993590157 last-index=2 last-term=2 size-in-bytes=1 -2025-06-03T23:33:10.158Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: snapshot restore progress: id=2-2-1748993590157 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2025-06-03T23:33:10.158Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: restored from snapshot: id=2-2-1748993590157 last-index=2 last-term=2 size-in-bytes=1 -2025-06-03T23:33:10.158Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b52a7256-b2af-6f23-a3da-9702d1c96d9d Address:b52a7256-b2af-6f23-a3da-9702d1c96d9d} {Suffrage:Voter ID:server-59c331a3-2d38-1413-2ffe-5c2cf953e48a Address:59c331a3-2d38-1413-2ffe-5c2cf953e48a} {Suffrage:Voter ID:server-7e3a1cab-ee72-343c-42d9-5ee8432831ac Address:7e3a1cab-ee72-343c-42d9-5ee8432831ac}]" -2025-06-03T23:33:10.158Z [INFO] creating new snapshot: path=/tmp/raft3951722141/snapshots/2-2-1748993590158.tmp -2025-06-03T23:33:10.158Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: entering follower state: follower="Node at 59c331a3-2d38-1413-2ffe-5c2cf953e48a [Follower]" leader-address= leader-id= -2025-06-03T23:33:10.158Z [INFO] server-7e3a1cab-ee72-343c-42d9-5ee8432831ac: starting restore from snapshot: id=2-2-1748993590158 last-index=2 last-term=2 size-in-bytes=1 -2025-06-03T23:33:10.158Z [INFO] server-7e3a1cab-ee72-343c-42d9-5ee8432831ac: snapshot restore progress: id=2-2-1748993590158 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2025-06-03T23:33:10.158Z [INFO] server-7e3a1cab-ee72-343c-42d9-5ee8432831ac: restored from snapshot: id=2-2-1748993590158 last-index=2 last-term=2 size-in-bytes=1 -2025-06-03T23:33:10.158Z [INFO] server-7e3a1cab-ee72-343c-42d9-5ee8432831ac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b52a7256-b2af-6f23-a3da-9702d1c96d9d Address:b52a7256-b2af-6f23-a3da-9702d1c96d9d} {Suffrage:Voter ID:server-59c331a3-2d38-1413-2ffe-5c2cf953e48a Address:59c331a3-2d38-1413-2ffe-5c2cf953e48a} {Suffrage:Voter ID:server-7e3a1cab-ee72-343c-42d9-5ee8432831ac Address:7e3a1cab-ee72-343c-42d9-5ee8432831ac}]" -2025-06-03T23:33:10.159Z [INFO] server-7e3a1cab-ee72-343c-42d9-5ee8432831ac: entering follower state: follower="Node at 7e3a1cab-ee72-343c-42d9-5ee8432831ac [Follower]" leader-address= leader-id= -2025-06-03T23:33:10.232Z [WARN] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:10.232Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: entering candidate state: node="Node at 59c331a3-2d38-1413-2ffe-5c2cf953e48a [Candidate]" term=3 -2025-06-03T23:33:10.232Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: election won: term=3 tally=2 -2025-06-03T23:33:10.233Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: entering leader state: leader="Node at 59c331a3-2d38-1413-2ffe-5c2cf953e48a [Leader]" -2025-06-03T23:33:10.233Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: added peer, starting replication: peer=server-b52a7256-b2af-6f23-a3da-9702d1c96d9d -2025-06-03T23:33:10.233Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: added peer, starting replication: peer=server-7e3a1cab-ee72-343c-42d9-5ee8432831ac -2025-06-03T23:33:10.233Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: pipelining replication: peer="{Voter server-7e3a1cab-ee72-343c-42d9-5ee8432831ac 7e3a1cab-ee72-343c-42d9-5ee8432831ac}" -2025-06-03T23:33:10.233Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: pipelining replication: peer="{Voter server-b52a7256-b2af-6f23-a3da-9702d1c96d9d b52a7256-b2af-6f23-a3da-9702d1c96d9d}" -2025-06-03T23:33:10.474Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:10.580Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:10.474621111 +0000 UTC m=+5.495440521 (1 nodes), 396.4µs from start of poll, 486.17928ms from cluster start. Timeout at 2025-06-03 23:33:10.580495911 +0000 UTC m=+5.601315281, 105.87476ms after stability -2025-06-03T23:33:10.580Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: aborting pipeline replication: peer="{Voter server-7e3a1cab-ee72-343c-42d9-5ee8432831ac 7e3a1cab-ee72-343c-42d9-5ee8432831ac}" -2025-06-03T23:33:10.580Z [INFO] server-59c331a3-2d38-1413-2ffe-5c2cf953e48a: aborting pipeline replication: peer="{Voter server-b52a7256-b2af-6f23-a3da-9702d1c96d9d b52a7256-b2af-6f23-a3da-9702d1c96d9d}" +2025-06-03T23:36:09.183Z [INFO] server-a7873211-6ab9-d792-aaec-8d12ff4f6b79: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a7873211-6ab9-d792-aaec-8d12ff4f6b79 Address:a7873211-6ab9-d792-aaec-8d12ff4f6b79} {Suffrage:Voter ID:server-8edee939-1348-732e-8c93-a7b9613f1d7b Address:8edee939-1348-732e-8c93-a7b9613f1d7b} {Suffrage:Voter ID:server-109c4b44-0e3e-e523-7729-d05c98b94541 Address:109c4b44-0e3e-e523-7729-d05c98b94541}]" +2025-06-03T23:36:09.183Z [INFO] server-a7873211-6ab9-d792-aaec-8d12ff4f6b79: entering follower state: follower="Node at a7873211-6ab9-d792-aaec-8d12ff4f6b79 [Follower]" leader-address= leader-id= +2025-06-03T23:36:09.183Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a7873211-6ab9-d792-aaec-8d12ff4f6b79 Address:a7873211-6ab9-d792-aaec-8d12ff4f6b79} {Suffrage:Voter ID:server-8edee939-1348-732e-8c93-a7b9613f1d7b Address:8edee939-1348-732e-8c93-a7b9613f1d7b} {Suffrage:Voter ID:server-109c4b44-0e3e-e523-7729-d05c98b94541 Address:109c4b44-0e3e-e523-7729-d05c98b94541}]" +2025-06-03T23:36:09.183Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: entering follower state: follower="Node at 8edee939-1348-732e-8c93-a7b9613f1d7b [Follower]" leader-address= leader-id= +2025-06-03T23:36:09.183Z [INFO] server-109c4b44-0e3e-e523-7729-d05c98b94541: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a7873211-6ab9-d792-aaec-8d12ff4f6b79 Address:a7873211-6ab9-d792-aaec-8d12ff4f6b79} {Suffrage:Voter ID:server-8edee939-1348-732e-8c93-a7b9613f1d7b Address:8edee939-1348-732e-8c93-a7b9613f1d7b} {Suffrage:Voter ID:server-109c4b44-0e3e-e523-7729-d05c98b94541 Address:109c4b44-0e3e-e523-7729-d05c98b94541}]" +2025-06-03T23:36:09.183Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:09.183Z [INFO] server-109c4b44-0e3e-e523-7729-d05c98b94541: entering follower state: follower="Node at 109c4b44-0e3e-e523-7729-d05c98b94541 [Follower]" leader-address= leader-id= +2025-06-03T23:36:09.250Z [WARN] server-8edee939-1348-732e-8c93-a7b9613f1d7b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:09.250Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: entering candidate state: node="Node at 8edee939-1348-732e-8c93-a7b9613f1d7b [Candidate]" term=2 +2025-06-03T23:36:09.250Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: election won: term=2 tally=2 +2025-06-03T23:36:09.250Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: entering leader state: leader="Node at 8edee939-1348-732e-8c93-a7b9613f1d7b [Leader]" +2025-06-03T23:36:09.250Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: added peer, starting replication: peer=server-a7873211-6ab9-d792-aaec-8d12ff4f6b79 +2025-06-03T23:36:09.250Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: added peer, starting replication: peer=server-109c4b44-0e3e-e523-7729-d05c98b94541 +2025-06-03T23:36:09.251Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: pipelining replication: peer="{Voter server-a7873211-6ab9-d792-aaec-8d12ff4f6b79 a7873211-6ab9-d792-aaec-8d12ff4f6b79}" +2025-06-03T23:36:09.251Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: pipelining replication: peer="{Voter server-109c4b44-0e3e-e523-7729-d05c98b94541 109c4b44-0e3e-e523-7729-d05c98b94541}" +2025-06-03T23:36:09.356Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:09.250918431 +0000 UTC m=+4.572673361 (1 nodes), 67.05068ms from start of poll, 67.5538ms from cluster start. Timeout at 2025-06-03 23:36:09.356541431 +0000 UTC m=+4.678296361, 105.623ms after stability +2025-06-03T23:36:09.356Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: aborting pipeline replication: peer="{Voter server-109c4b44-0e3e-e523-7729-d05c98b94541 109c4b44-0e3e-e523-7729-d05c98b94541}" +2025-06-03T23:36:09.356Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: aborting pipeline replication: peer="{Voter server-a7873211-6ab9-d792-aaec-8d12ff4f6b79 a7873211-6ab9-d792-aaec-8d12ff4f6b79}" +2025-06-03T23:36:09.356Z [INFO] creating new snapshot: path=/tmp/raft3987842855/snapshots/2-2-1748993769356.tmp +2025-06-03T23:36:09.357Z [INFO] server-a7873211-6ab9-d792-aaec-8d12ff4f6b79: starting restore from snapshot: id=2-2-1748993769356 last-index=2 last-term=2 size-in-bytes=1 +2025-06-03T23:36:09.357Z [INFO] server-a7873211-6ab9-d792-aaec-8d12ff4f6b79: snapshot restore progress: id=2-2-1748993769356 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-06-03T23:36:09.357Z [INFO] server-a7873211-6ab9-d792-aaec-8d12ff4f6b79: restored from snapshot: id=2-2-1748993769356 last-index=2 last-term=2 size-in-bytes=1 +2025-06-03T23:36:09.357Z [INFO] server-a7873211-6ab9-d792-aaec-8d12ff4f6b79: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a7873211-6ab9-d792-aaec-8d12ff4f6b79 Address:a7873211-6ab9-d792-aaec-8d12ff4f6b79} {Suffrage:Voter ID:server-8edee939-1348-732e-8c93-a7b9613f1d7b Address:8edee939-1348-732e-8c93-a7b9613f1d7b} {Suffrage:Voter ID:server-109c4b44-0e3e-e523-7729-d05c98b94541 Address:109c4b44-0e3e-e523-7729-d05c98b94541}]" +2025-06-03T23:36:09.357Z [INFO] creating new snapshot: path=/tmp/raft2091152715/snapshots/2-2-1748993769357.tmp +2025-06-03T23:36:09.357Z [INFO] server-a7873211-6ab9-d792-aaec-8d12ff4f6b79: entering follower state: follower="Node at a7873211-6ab9-d792-aaec-8d12ff4f6b79 [Follower]" leader-address= leader-id= +2025-06-03T23:36:09.358Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: starting restore from snapshot: id=2-2-1748993769357 last-index=2 last-term=2 size-in-bytes=1 +2025-06-03T23:36:09.358Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: snapshot restore progress: id=2-2-1748993769357 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-06-03T23:36:09.358Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: restored from snapshot: id=2-2-1748993769357 last-index=2 last-term=2 size-in-bytes=1 +2025-06-03T23:36:09.358Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a7873211-6ab9-d792-aaec-8d12ff4f6b79 Address:a7873211-6ab9-d792-aaec-8d12ff4f6b79} {Suffrage:Voter ID:server-8edee939-1348-732e-8c93-a7b9613f1d7b Address:8edee939-1348-732e-8c93-a7b9613f1d7b} {Suffrage:Voter ID:server-109c4b44-0e3e-e523-7729-d05c98b94541 Address:109c4b44-0e3e-e523-7729-d05c98b94541}]" +2025-06-03T23:36:09.358Z [INFO] creating new snapshot: path=/tmp/raft920540/snapshots/2-2-1748993769358.tmp +2025-06-03T23:36:09.358Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: entering follower state: follower="Node at 8edee939-1348-732e-8c93-a7b9613f1d7b [Follower]" leader-address= leader-id= +2025-06-03T23:36:09.359Z [INFO] server-109c4b44-0e3e-e523-7729-d05c98b94541: starting restore from snapshot: id=2-2-1748993769358 last-index=2 last-term=2 size-in-bytes=1 +2025-06-03T23:36:09.359Z [INFO] server-109c4b44-0e3e-e523-7729-d05c98b94541: snapshot restore progress: id=2-2-1748993769358 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-06-03T23:36:09.359Z [INFO] server-109c4b44-0e3e-e523-7729-d05c98b94541: restored from snapshot: id=2-2-1748993769358 last-index=2 last-term=2 size-in-bytes=1 +2025-06-03T23:36:09.359Z [INFO] server-109c4b44-0e3e-e523-7729-d05c98b94541: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a7873211-6ab9-d792-aaec-8d12ff4f6b79 Address:a7873211-6ab9-d792-aaec-8d12ff4f6b79} {Suffrage:Voter ID:server-8edee939-1348-732e-8c93-a7b9613f1d7b Address:8edee939-1348-732e-8c93-a7b9613f1d7b} {Suffrage:Voter ID:server-109c4b44-0e3e-e523-7729-d05c98b94541 Address:109c4b44-0e3e-e523-7729-d05c98b94541}]" +2025-06-03T23:36:09.359Z [INFO] server-109c4b44-0e3e-e523-7729-d05c98b94541: entering follower state: follower="Node at 109c4b44-0e3e-e523-7729-d05c98b94541 [Follower]" leader-address= leader-id= +2025-06-03T23:36:09.436Z [WARN] server-8edee939-1348-732e-8c93-a7b9613f1d7b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:09.436Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: entering candidate state: node="Node at 8edee939-1348-732e-8c93-a7b9613f1d7b [Candidate]" term=3 +2025-06-03T23:36:09.436Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: election won: term=3 tally=2 +2025-06-03T23:36:09.436Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: entering leader state: leader="Node at 8edee939-1348-732e-8c93-a7b9613f1d7b [Leader]" +2025-06-03T23:36:09.436Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: added peer, starting replication: peer=server-a7873211-6ab9-d792-aaec-8d12ff4f6b79 +2025-06-03T23:36:09.436Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: added peer, starting replication: peer=server-109c4b44-0e3e-e523-7729-d05c98b94541 +2025-06-03T23:36:09.436Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: pipelining replication: peer="{Voter server-109c4b44-0e3e-e523-7729-d05c98b94541 109c4b44-0e3e-e523-7729-d05c98b94541}" +2025-06-03T23:36:09.436Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: pipelining replication: peer="{Voter server-a7873211-6ab9-d792-aaec-8d12ff4f6b79 a7873211-6ab9-d792-aaec-8d12ff4f6b79}" +2025-06-03T23:36:09.675Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:09.780Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:09.675146191 +0000 UTC m=+4.996901121 (1 nodes), 45.44µs from start of poll, 491.78156ms from cluster start. Timeout at 2025-06-03 23:36:09.780805231 +0000 UTC m=+5.102560201, 105.65908ms after stability +2025-06-03T23:36:09.780Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: aborting pipeline replication: peer="{Voter server-a7873211-6ab9-d792-aaec-8d12ff4f6b79 a7873211-6ab9-d792-aaec-8d12ff4f6b79}" +2025-06-03T23:36:09.781Z [INFO] server-8edee939-1348-732e-8c93-a7b9613f1d7b: aborting pipeline replication: peer="{Voter server-109c4b44-0e3e-e523-7729-d05c98b94541 109c4b44-0e3e-e523-7729-d05c98b94541}" === RUN TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs -2025-06-03T23:33:10.581Z [INFO] server-417f79a1-27cf-4159-a9b5-54d5d0e9663e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-417f79a1-27cf-4159-a9b5-54d5d0e9663e Address:417f79a1-27cf-4159-a9b5-54d5d0e9663e} {Suffrage:Voter ID:server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e Address:b6bd6682-45dd-a67d-a5d6-dd3d252b690e} {Suffrage:Voter ID:server-87acc4e2-b230-5e68-68dc-7c9bda4ba096 Address:87acc4e2-b230-5e68-68dc-7c9bda4ba096}]" -2025-06-03T23:33:10.581Z [INFO] server-417f79a1-27cf-4159-a9b5-54d5d0e9663e: entering follower state: follower="Node at 417f79a1-27cf-4159-a9b5-54d5d0e9663e [Follower]" leader-address= leader-id= -2025-06-03T23:33:10.581Z [INFO] server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-417f79a1-27cf-4159-a9b5-54d5d0e9663e Address:417f79a1-27cf-4159-a9b5-54d5d0e9663e} {Suffrage:Voter ID:server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e Address:b6bd6682-45dd-a67d-a5d6-dd3d252b690e} {Suffrage:Voter ID:server-87acc4e2-b230-5e68-68dc-7c9bda4ba096 Address:87acc4e2-b230-5e68-68dc-7c9bda4ba096}]" -2025-06-03T23:33:10.581Z [INFO] server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e: entering follower state: follower="Node at b6bd6682-45dd-a67d-a5d6-dd3d252b690e [Follower]" leader-address= leader-id= -2025-06-03T23:33:10.582Z [INFO] server-87acc4e2-b230-5e68-68dc-7c9bda4ba096: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-417f79a1-27cf-4159-a9b5-54d5d0e9663e Address:417f79a1-27cf-4159-a9b5-54d5d0e9663e} {Suffrage:Voter ID:server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e Address:b6bd6682-45dd-a67d-a5d6-dd3d252b690e} {Suffrage:Voter ID:server-87acc4e2-b230-5e68-68dc-7c9bda4ba096 Address:87acc4e2-b230-5e68-68dc-7c9bda4ba096}]" -2025-06-03T23:33:10.582Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:10.582Z [INFO] server-87acc4e2-b230-5e68-68dc-7c9bda4ba096: entering follower state: follower="Node at 87acc4e2-b230-5e68-68dc-7c9bda4ba096 [Follower]" leader-address= leader-id= -2025-06-03T23:33:10.644Z [WARN] server-417f79a1-27cf-4159-a9b5-54d5d0e9663e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:10.644Z [INFO] server-417f79a1-27cf-4159-a9b5-54d5d0e9663e: entering candidate state: node="Node at 417f79a1-27cf-4159-a9b5-54d5d0e9663e [Candidate]" term=2 -2025-06-03T23:33:10.644Z [INFO] server-417f79a1-27cf-4159-a9b5-54d5d0e9663e: election won: term=2 tally=2 -2025-06-03T23:33:10.644Z [INFO] server-417f79a1-27cf-4159-a9b5-54d5d0e9663e: entering leader state: leader="Node at 417f79a1-27cf-4159-a9b5-54d5d0e9663e [Leader]" -2025-06-03T23:33:10.644Z [INFO] server-417f79a1-27cf-4159-a9b5-54d5d0e9663e: added peer, starting replication: peer=server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e -2025-06-03T23:33:10.644Z [INFO] server-417f79a1-27cf-4159-a9b5-54d5d0e9663e: added peer, starting replication: peer=server-87acc4e2-b230-5e68-68dc-7c9bda4ba096 -2025-06-03T23:33:10.644Z [INFO] server-417f79a1-27cf-4159-a9b5-54d5d0e9663e: pipelining replication: peer="{Voter server-87acc4e2-b230-5e68-68dc-7c9bda4ba096 87acc4e2-b230-5e68-68dc-7c9bda4ba096}" -2025-06-03T23:33:10.644Z [INFO] server-417f79a1-27cf-4159-a9b5-54d5d0e9663e: pipelining replication: peer="{Voter server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e b6bd6682-45dd-a67d-a5d6-dd3d252b690e}" -2025-06-03T23:33:10.750Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:10.644635551 +0000 UTC m=+5.665454921 (1 nodes), 62.46916ms from start of poll, 63.05452ms from cluster start. Timeout at 2025-06-03 23:33:10.750760551 +0000 UTC m=+5.771579921, 106.125ms after stability -2025-06-03T23:33:10.751Z [INFO] server-417f79a1-27cf-4159-a9b5-54d5d0e9663e: aborting pipeline replication: peer="{Voter server-87acc4e2-b230-5e68-68dc-7c9bda4ba096 87acc4e2-b230-5e68-68dc-7c9bda4ba096}" -2025-06-03T23:33:10.751Z [INFO] server-417f79a1-27cf-4159-a9b5-54d5d0e9663e: aborting pipeline replication: peer="{Voter server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e b6bd6682-45dd-a67d-a5d6-dd3d252b690e}" -2025-06-03T23:33:10.751Z [INFO] creating new snapshot: path=/tmp/raft3788824026/snapshots/2-6-1748993590751.tmp -2025-06-03T23:33:10.752Z [INFO] server-417f79a1-27cf-4159-a9b5-54d5d0e9663e: starting restore from snapshot: id=2-6-1748993590751 last-index=6 last-term=2 size-in-bytes=25 -2025-06-03T23:33:10.752Z [INFO] server-417f79a1-27cf-4159-a9b5-54d5d0e9663e: snapshot restore progress: id=2-6-1748993590751 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2025-06-03T23:33:10.752Z [INFO] server-417f79a1-27cf-4159-a9b5-54d5d0e9663e: restored from snapshot: id=2-6-1748993590751 last-index=6 last-term=2 size-in-bytes=25 -2025-06-03T23:33:10.752Z [INFO] server-417f79a1-27cf-4159-a9b5-54d5d0e9663e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-417f79a1-27cf-4159-a9b5-54d5d0e9663e Address:417f79a1-27cf-4159-a9b5-54d5d0e9663e} {Suffrage:Voter ID:server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e Address:b6bd6682-45dd-a67d-a5d6-dd3d252b690e} {Suffrage:Voter ID:server-87acc4e2-b230-5e68-68dc-7c9bda4ba096 Address:87acc4e2-b230-5e68-68dc-7c9bda4ba096}]" -2025-06-03T23:33:10.752Z [INFO] creating new snapshot: path=/tmp/raft3947968396/snapshots/2-6-1748993590752.tmp -2025-06-03T23:33:10.752Z [INFO] server-417f79a1-27cf-4159-a9b5-54d5d0e9663e: entering follower state: follower="Node at 417f79a1-27cf-4159-a9b5-54d5d0e9663e [Follower]" leader-address= leader-id= -2025-06-03T23:33:10.753Z [INFO] server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e: starting restore from snapshot: id=2-6-1748993590752 last-index=6 last-term=2 size-in-bytes=25 -2025-06-03T23:33:10.753Z [INFO] server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e: snapshot restore progress: id=2-6-1748993590752 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2025-06-03T23:33:10.753Z [INFO] server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e: restored from snapshot: id=2-6-1748993590752 last-index=6 last-term=2 size-in-bytes=25 -2025-06-03T23:33:10.753Z [INFO] server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-417f79a1-27cf-4159-a9b5-54d5d0e9663e Address:417f79a1-27cf-4159-a9b5-54d5d0e9663e} {Suffrage:Voter ID:server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e Address:b6bd6682-45dd-a67d-a5d6-dd3d252b690e} {Suffrage:Voter ID:server-87acc4e2-b230-5e68-68dc-7c9bda4ba096 Address:87acc4e2-b230-5e68-68dc-7c9bda4ba096}]" -2025-06-03T23:33:10.753Z [INFO] server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e: entering follower state: follower="Node at b6bd6682-45dd-a67d-a5d6-dd3d252b690e [Follower]" leader-address= leader-id= -2025-06-03T23:33:10.753Z [INFO] creating new snapshot: path=/tmp/raft2648491558/snapshots/2-6-1748993590753.tmp -2025-06-03T23:33:10.754Z [INFO] server-87acc4e2-b230-5e68-68dc-7c9bda4ba096: starting restore from snapshot: id=2-6-1748993590753 last-index=6 last-term=2 size-in-bytes=25 -2025-06-03T23:33:10.755Z [INFO] server-87acc4e2-b230-5e68-68dc-7c9bda4ba096: snapshot restore progress: id=2-6-1748993590753 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2025-06-03T23:33:10.755Z [INFO] server-87acc4e2-b230-5e68-68dc-7c9bda4ba096: restored from snapshot: id=2-6-1748993590753 last-index=6 last-term=2 size-in-bytes=25 -2025-06-03T23:33:10.755Z [INFO] server-87acc4e2-b230-5e68-68dc-7c9bda4ba096: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-417f79a1-27cf-4159-a9b5-54d5d0e9663e Address:417f79a1-27cf-4159-a9b5-54d5d0e9663e} {Suffrage:Voter ID:server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e Address:b6bd6682-45dd-a67d-a5d6-dd3d252b690e} {Suffrage:Voter ID:server-87acc4e2-b230-5e68-68dc-7c9bda4ba096 Address:87acc4e2-b230-5e68-68dc-7c9bda4ba096}]" -2025-06-03T23:33:10.755Z [INFO] server-87acc4e2-b230-5e68-68dc-7c9bda4ba096: entering follower state: follower="Node at 87acc4e2-b230-5e68-68dc-7c9bda4ba096 [Follower]" leader-address= leader-id= -2025-06-03T23:33:10.830Z [WARN] server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:10.830Z [INFO] server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e: entering candidate state: node="Node at b6bd6682-45dd-a67d-a5d6-dd3d252b690e [Candidate]" term=3 -2025-06-03T23:33:10.830Z [INFO] server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e: election won: term=3 tally=2 -2025-06-03T23:33:10.830Z [INFO] server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e: entering leader state: leader="Node at b6bd6682-45dd-a67d-a5d6-dd3d252b690e [Leader]" -2025-06-03T23:33:10.830Z [INFO] server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e: added peer, starting replication: peer=server-417f79a1-27cf-4159-a9b5-54d5d0e9663e -2025-06-03T23:33:10.830Z [INFO] server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e: added peer, starting replication: peer=server-87acc4e2-b230-5e68-68dc-7c9bda4ba096 -2025-06-03T23:33:10.831Z [INFO] server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e: pipelining replication: peer="{Voter server-87acc4e2-b230-5e68-68dc-7c9bda4ba096 87acc4e2-b230-5e68-68dc-7c9bda4ba096}" -2025-06-03T23:33:10.831Z [INFO] server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e: pipelining replication: peer="{Voter server-417f79a1-27cf-4159-a9b5-54d5d0e9663e 417f79a1-27cf-4159-a9b5-54d5d0e9663e}" -2025-06-03T23:33:11.070Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:11.179Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:11.070883151 +0000 UTC m=+6.091702521 (1 nodes), 46.92µs from start of poll, 489.30212ms from cluster start. Timeout at 2025-06-03 23:33:11.179707031 +0000 UTC m=+6.200526361, 108.82384ms after stability -2025-06-03T23:33:11.179Z [INFO] server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e: aborting pipeline replication: peer="{Voter server-417f79a1-27cf-4159-a9b5-54d5d0e9663e 417f79a1-27cf-4159-a9b5-54d5d0e9663e}" -2025-06-03T23:33:11.179Z [INFO] server-b6bd6682-45dd-a67d-a5d6-dd3d252b690e: aborting pipeline replication: peer="{Voter server-87acc4e2-b230-5e68-68dc-7c9bda4ba096 87acc4e2-b230-5e68-68dc-7c9bda4ba096}" +2025-06-03T23:36:09.782Z [INFO] server-45470987-44d3-440a-5e0b-05c50c1b769e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45470987-44d3-440a-5e0b-05c50c1b769e Address:45470987-44d3-440a-5e0b-05c50c1b769e} {Suffrage:Voter ID:server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1 Address:c9d48095-8ac8-10cf-1a20-f5501e7b5cf1} {Suffrage:Voter ID:server-66af4f98-3024-30d0-926e-1fedec9fa107 Address:66af4f98-3024-30d0-926e-1fedec9fa107}]" +2025-06-03T23:36:09.782Z [INFO] server-45470987-44d3-440a-5e0b-05c50c1b769e: entering follower state: follower="Node at 45470987-44d3-440a-5e0b-05c50c1b769e [Follower]" leader-address= leader-id= +2025-06-03T23:36:09.782Z [INFO] server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45470987-44d3-440a-5e0b-05c50c1b769e Address:45470987-44d3-440a-5e0b-05c50c1b769e} {Suffrage:Voter ID:server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1 Address:c9d48095-8ac8-10cf-1a20-f5501e7b5cf1} {Suffrage:Voter ID:server-66af4f98-3024-30d0-926e-1fedec9fa107 Address:66af4f98-3024-30d0-926e-1fedec9fa107}]" +2025-06-03T23:36:09.782Z [INFO] server-66af4f98-3024-30d0-926e-1fedec9fa107: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45470987-44d3-440a-5e0b-05c50c1b769e Address:45470987-44d3-440a-5e0b-05c50c1b769e} {Suffrage:Voter ID:server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1 Address:c9d48095-8ac8-10cf-1a20-f5501e7b5cf1} {Suffrage:Voter ID:server-66af4f98-3024-30d0-926e-1fedec9fa107 Address:66af4f98-3024-30d0-926e-1fedec9fa107}]" +2025-06-03T23:36:09.782Z [INFO] server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1: entering follower state: follower="Node at c9d48095-8ac8-10cf-1a20-f5501e7b5cf1 [Follower]" leader-address= leader-id= +2025-06-03T23:36:09.782Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:09.782Z [INFO] server-66af4f98-3024-30d0-926e-1fedec9fa107: entering follower state: follower="Node at 66af4f98-3024-30d0-926e-1fedec9fa107 [Follower]" leader-address= leader-id= +2025-06-03T23:36:09.848Z [WARN] server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:09.848Z [INFO] server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1: entering candidate state: node="Node at c9d48095-8ac8-10cf-1a20-f5501e7b5cf1 [Candidate]" term=2 +2025-06-03T23:36:09.848Z [INFO] server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1: election won: term=2 tally=2 +2025-06-03T23:36:09.848Z [INFO] server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1: entering leader state: leader="Node at c9d48095-8ac8-10cf-1a20-f5501e7b5cf1 [Leader]" +2025-06-03T23:36:09.848Z [INFO] server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1: added peer, starting replication: peer=server-45470987-44d3-440a-5e0b-05c50c1b769e +2025-06-03T23:36:09.848Z [INFO] server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1: added peer, starting replication: peer=server-66af4f98-3024-30d0-926e-1fedec9fa107 +2025-06-03T23:36:09.848Z [INFO] server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1: pipelining replication: peer="{Voter server-45470987-44d3-440a-5e0b-05c50c1b769e 45470987-44d3-440a-5e0b-05c50c1b769e}" +2025-06-03T23:36:09.848Z [INFO] server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1: pipelining replication: peer="{Voter server-66af4f98-3024-30d0-926e-1fedec9fa107 66af4f98-3024-30d0-926e-1fedec9fa107}" +2025-06-03T23:36:09.953Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:09.848440511 +0000 UTC m=+5.170195441 (1 nodes), 65.96272ms from start of poll, 66.45756ms from cluster start. Timeout at 2025-06-03 23:36:09.953496591 +0000 UTC m=+5.275251521, 105.05608ms after stability +2025-06-03T23:36:09.954Z [INFO] server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1: aborting pipeline replication: peer="{Voter server-66af4f98-3024-30d0-926e-1fedec9fa107 66af4f98-3024-30d0-926e-1fedec9fa107}" +2025-06-03T23:36:09.954Z [INFO] server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1: aborting pipeline replication: peer="{Voter server-45470987-44d3-440a-5e0b-05c50c1b769e 45470987-44d3-440a-5e0b-05c50c1b769e}" +2025-06-03T23:36:09.954Z [INFO] creating new snapshot: path=/tmp/raft2128789276/snapshots/2-6-1748993769954.tmp +2025-06-03T23:36:09.955Z [INFO] server-45470987-44d3-440a-5e0b-05c50c1b769e: starting restore from snapshot: id=2-6-1748993769954 last-index=6 last-term=2 size-in-bytes=25 +2025-06-03T23:36:09.955Z [INFO] server-45470987-44d3-440a-5e0b-05c50c1b769e: snapshot restore progress: id=2-6-1748993769954 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-06-03T23:36:09.955Z [INFO] server-45470987-44d3-440a-5e0b-05c50c1b769e: restored from snapshot: id=2-6-1748993769954 last-index=6 last-term=2 size-in-bytes=25 +2025-06-03T23:36:09.955Z [INFO] server-45470987-44d3-440a-5e0b-05c50c1b769e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45470987-44d3-440a-5e0b-05c50c1b769e Address:45470987-44d3-440a-5e0b-05c50c1b769e} {Suffrage:Voter ID:server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1 Address:c9d48095-8ac8-10cf-1a20-f5501e7b5cf1} {Suffrage:Voter ID:server-66af4f98-3024-30d0-926e-1fedec9fa107 Address:66af4f98-3024-30d0-926e-1fedec9fa107}]" +2025-06-03T23:36:09.955Z [INFO] creating new snapshot: path=/tmp/raft2122391154/snapshots/2-6-1748993769955.tmp +2025-06-03T23:36:09.955Z [INFO] server-45470987-44d3-440a-5e0b-05c50c1b769e: entering follower state: follower="Node at 45470987-44d3-440a-5e0b-05c50c1b769e [Follower]" leader-address= leader-id= +2025-06-03T23:36:09.956Z [INFO] server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1: starting restore from snapshot: id=2-6-1748993769955 last-index=6 last-term=2 size-in-bytes=25 +2025-06-03T23:36:09.956Z [INFO] server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1: snapshot restore progress: id=2-6-1748993769955 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-06-03T23:36:09.956Z [INFO] server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1: restored from snapshot: id=2-6-1748993769955 last-index=6 last-term=2 size-in-bytes=25 +2025-06-03T23:36:09.956Z [INFO] server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45470987-44d3-440a-5e0b-05c50c1b769e Address:45470987-44d3-440a-5e0b-05c50c1b769e} {Suffrage:Voter ID:server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1 Address:c9d48095-8ac8-10cf-1a20-f5501e7b5cf1} {Suffrage:Voter ID:server-66af4f98-3024-30d0-926e-1fedec9fa107 Address:66af4f98-3024-30d0-926e-1fedec9fa107}]" +2025-06-03T23:36:09.956Z [INFO] creating new snapshot: path=/tmp/raft3008275929/snapshots/2-6-1748993769956.tmp +2025-06-03T23:36:09.956Z [INFO] server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1: entering follower state: follower="Node at c9d48095-8ac8-10cf-1a20-f5501e7b5cf1 [Follower]" leader-address= leader-id= +2025-06-03T23:36:09.956Z [INFO] server-66af4f98-3024-30d0-926e-1fedec9fa107: starting restore from snapshot: id=2-6-1748993769956 last-index=6 last-term=2 size-in-bytes=25 +2025-06-03T23:36:09.956Z [INFO] server-66af4f98-3024-30d0-926e-1fedec9fa107: snapshot restore progress: id=2-6-1748993769956 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-06-03T23:36:09.957Z [INFO] server-66af4f98-3024-30d0-926e-1fedec9fa107: restored from snapshot: id=2-6-1748993769956 last-index=6 last-term=2 size-in-bytes=25 +2025-06-03T23:36:09.957Z [INFO] server-66af4f98-3024-30d0-926e-1fedec9fa107: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45470987-44d3-440a-5e0b-05c50c1b769e Address:45470987-44d3-440a-5e0b-05c50c1b769e} {Suffrage:Voter ID:server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1 Address:c9d48095-8ac8-10cf-1a20-f5501e7b5cf1} {Suffrage:Voter ID:server-66af4f98-3024-30d0-926e-1fedec9fa107 Address:66af4f98-3024-30d0-926e-1fedec9fa107}]" +2025-06-03T23:36:09.957Z [INFO] server-66af4f98-3024-30d0-926e-1fedec9fa107: entering follower state: follower="Node at 66af4f98-3024-30d0-926e-1fedec9fa107 [Follower]" leader-address= leader-id= +2025-06-03T23:36:10.023Z [WARN] server-45470987-44d3-440a-5e0b-05c50c1b769e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:10.023Z [INFO] server-45470987-44d3-440a-5e0b-05c50c1b769e: entering candidate state: node="Node at 45470987-44d3-440a-5e0b-05c50c1b769e [Candidate]" term=3 +2025-06-03T23:36:10.023Z [INFO] server-45470987-44d3-440a-5e0b-05c50c1b769e: election won: term=3 tally=2 +2025-06-03T23:36:10.023Z [INFO] server-45470987-44d3-440a-5e0b-05c50c1b769e: entering leader state: leader="Node at 45470987-44d3-440a-5e0b-05c50c1b769e [Leader]" +2025-06-03T23:36:10.023Z [INFO] server-45470987-44d3-440a-5e0b-05c50c1b769e: added peer, starting replication: peer=server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1 +2025-06-03T23:36:10.023Z [INFO] server-45470987-44d3-440a-5e0b-05c50c1b769e: added peer, starting replication: peer=server-66af4f98-3024-30d0-926e-1fedec9fa107 +2025-06-03T23:36:10.024Z [INFO] server-45470987-44d3-440a-5e0b-05c50c1b769e: pipelining replication: peer="{Voter server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1 c9d48095-8ac8-10cf-1a20-f5501e7b5cf1}" +2025-06-03T23:36:10.024Z [INFO] server-45470987-44d3-440a-5e0b-05c50c1b769e: pipelining replication: peer="{Voter server-66af4f98-3024-30d0-926e-1fedec9fa107 66af4f98-3024-30d0-926e-1fedec9fa107}" +2025-06-03T23:36:10.272Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:10.378Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:10.272181831 +0000 UTC m=+5.593936761 (1 nodes), 40.96µs from start of poll, 490.19888ms from cluster start. Timeout at 2025-06-03 23:36:10.378042831 +0000 UTC m=+5.699797761, 105.861ms after stability +2025-06-03T23:36:10.378Z [INFO] server-45470987-44d3-440a-5e0b-05c50c1b769e: aborting pipeline replication: peer="{Voter server-66af4f98-3024-30d0-926e-1fedec9fa107 66af4f98-3024-30d0-926e-1fedec9fa107}" +2025-06-03T23:36:10.378Z [INFO] server-45470987-44d3-440a-5e0b-05c50c1b769e: aborting pipeline replication: peer="{Voter server-c9d48095-8ac8-10cf-1a20-f5501e7b5cf1 c9d48095-8ac8-10cf-1a20-f5501e7b5cf1}" === RUN TestRaft_RecoverCluster/snapshot,_with_trailing_logs -2025-06-03T23:33:11.180Z [INFO] server-6f7753d6-9c41-f672-ad87-55dba0299469: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f7753d6-9c41-f672-ad87-55dba0299469 Address:6f7753d6-9c41-f672-ad87-55dba0299469} {Suffrage:Voter ID:server-40ac4783-e9fb-76b6-75c7-8f99de366500 Address:40ac4783-e9fb-76b6-75c7-8f99de366500} {Suffrage:Voter ID:server-294086ce-cd96-9e6b-e851-0939d2a9e970 Address:294086ce-cd96-9e6b-e851-0939d2a9e970}]" -2025-06-03T23:33:11.180Z [INFO] server-40ac4783-e9fb-76b6-75c7-8f99de366500: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f7753d6-9c41-f672-ad87-55dba0299469 Address:6f7753d6-9c41-f672-ad87-55dba0299469} {Suffrage:Voter ID:server-40ac4783-e9fb-76b6-75c7-8f99de366500 Address:40ac4783-e9fb-76b6-75c7-8f99de366500} {Suffrage:Voter ID:server-294086ce-cd96-9e6b-e851-0939d2a9e970 Address:294086ce-cd96-9e6b-e851-0939d2a9e970}]" -2025-06-03T23:33:11.180Z [INFO] server-294086ce-cd96-9e6b-e851-0939d2a9e970: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f7753d6-9c41-f672-ad87-55dba0299469 Address:6f7753d6-9c41-f672-ad87-55dba0299469} {Suffrage:Voter ID:server-40ac4783-e9fb-76b6-75c7-8f99de366500 Address:40ac4783-e9fb-76b6-75c7-8f99de366500} {Suffrage:Voter ID:server-294086ce-cd96-9e6b-e851-0939d2a9e970 Address:294086ce-cd96-9e6b-e851-0939d2a9e970}]" -2025-06-03T23:33:11.180Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:11.180Z [INFO] server-6f7753d6-9c41-f672-ad87-55dba0299469: entering follower state: follower="Node at 6f7753d6-9c41-f672-ad87-55dba0299469 [Follower]" leader-address= leader-id= -2025-06-03T23:33:11.180Z [INFO] server-40ac4783-e9fb-76b6-75c7-8f99de366500: entering follower state: follower="Node at 40ac4783-e9fb-76b6-75c7-8f99de366500 [Follower]" leader-address= leader-id= -2025-06-03T23:33:11.180Z [INFO] server-294086ce-cd96-9e6b-e851-0939d2a9e970: entering follower state: follower="Node at 294086ce-cd96-9e6b-e851-0939d2a9e970 [Follower]" leader-address= leader-id= -2025-06-03T23:33:11.237Z [WARN] server-294086ce-cd96-9e6b-e851-0939d2a9e970: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:11.237Z [INFO] server-294086ce-cd96-9e6b-e851-0939d2a9e970: entering candidate state: node="Node at 294086ce-cd96-9e6b-e851-0939d2a9e970 [Candidate]" term=2 -2025-06-03T23:33:11.238Z [INFO] server-294086ce-cd96-9e6b-e851-0939d2a9e970: election won: term=2 tally=2 -2025-06-03T23:33:11.238Z [INFO] server-294086ce-cd96-9e6b-e851-0939d2a9e970: entering leader state: leader="Node at 294086ce-cd96-9e6b-e851-0939d2a9e970 [Leader]" -2025-06-03T23:33:11.238Z [INFO] server-294086ce-cd96-9e6b-e851-0939d2a9e970: added peer, starting replication: peer=server-6f7753d6-9c41-f672-ad87-55dba0299469 -2025-06-03T23:33:11.238Z [INFO] server-294086ce-cd96-9e6b-e851-0939d2a9e970: added peer, starting replication: peer=server-40ac4783-e9fb-76b6-75c7-8f99de366500 -2025-06-03T23:33:11.238Z [INFO] server-294086ce-cd96-9e6b-e851-0939d2a9e970: pipelining replication: peer="{Voter server-40ac4783-e9fb-76b6-75c7-8f99de366500 40ac4783-e9fb-76b6-75c7-8f99de366500}" -2025-06-03T23:33:11.238Z [INFO] server-294086ce-cd96-9e6b-e851-0939d2a9e970: pipelining replication: peer="{Voter server-6f7753d6-9c41-f672-ad87-55dba0299469 6f7753d6-9c41-f672-ad87-55dba0299469}" -2025-06-03T23:33:11.344Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:11.238314231 +0000 UTC m=+6.259133601 (1 nodes), 57.58228ms from start of poll, 57.87668ms from cluster start. Timeout at 2025-06-03 23:33:11.344345431 +0000 UTC m=+6.365164801, 106.0312ms after stability -2025-06-03T23:33:11.348Z [INFO] server-294086ce-cd96-9e6b-e851-0939d2a9e970: aborting pipeline replication: peer="{Voter server-40ac4783-e9fb-76b6-75c7-8f99de366500 40ac4783-e9fb-76b6-75c7-8f99de366500}" -2025-06-03T23:33:11.348Z [INFO] server-294086ce-cd96-9e6b-e851-0939d2a9e970: aborting pipeline replication: peer="{Voter server-6f7753d6-9c41-f672-ad87-55dba0299469 6f7753d6-9c41-f672-ad87-55dba0299469}" -2025-06-03T23:33:11.348Z [INFO] creating new snapshot: path=/tmp/raft3547226876/snapshots/2-27-1748993591348.tmp -2025-06-03T23:33:11.349Z [INFO] server-6f7753d6-9c41-f672-ad87-55dba0299469: starting restore from snapshot: id=2-27-1748993591348 last-index=27 last-term=2 size-in-bytes=168 -2025-06-03T23:33:11.349Z [INFO] server-6f7753d6-9c41-f672-ad87-55dba0299469: snapshot restore progress: id=2-27-1748993591348 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2025-06-03T23:33:11.349Z [INFO] server-6f7753d6-9c41-f672-ad87-55dba0299469: restored from snapshot: id=2-27-1748993591348 last-index=27 last-term=2 size-in-bytes=168 -2025-06-03T23:33:11.349Z [INFO] server-6f7753d6-9c41-f672-ad87-55dba0299469: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f7753d6-9c41-f672-ad87-55dba0299469 Address:6f7753d6-9c41-f672-ad87-55dba0299469} {Suffrage:Voter ID:server-40ac4783-e9fb-76b6-75c7-8f99de366500 Address:40ac4783-e9fb-76b6-75c7-8f99de366500} {Suffrage:Voter ID:server-294086ce-cd96-9e6b-e851-0939d2a9e970 Address:294086ce-cd96-9e6b-e851-0939d2a9e970}]" -2025-06-03T23:33:11.350Z [INFO] server-6f7753d6-9c41-f672-ad87-55dba0299469: entering follower state: follower="Node at 6f7753d6-9c41-f672-ad87-55dba0299469 [Follower]" leader-address= leader-id= -2025-06-03T23:33:11.350Z [INFO] creating new snapshot: path=/tmp/raft3551764658/snapshots/2-27-1748993591350.tmp -2025-06-03T23:33:11.350Z [INFO] server-40ac4783-e9fb-76b6-75c7-8f99de366500: starting restore from snapshot: id=2-27-1748993591350 last-index=27 last-term=2 size-in-bytes=168 -2025-06-03T23:33:11.350Z [INFO] server-40ac4783-e9fb-76b6-75c7-8f99de366500: snapshot restore progress: id=2-27-1748993591350 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2025-06-03T23:33:11.351Z [INFO] server-40ac4783-e9fb-76b6-75c7-8f99de366500: restored from snapshot: id=2-27-1748993591350 last-index=27 last-term=2 size-in-bytes=168 -2025-06-03T23:33:11.351Z [INFO] server-40ac4783-e9fb-76b6-75c7-8f99de366500: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f7753d6-9c41-f672-ad87-55dba0299469 Address:6f7753d6-9c41-f672-ad87-55dba0299469} {Suffrage:Voter ID:server-40ac4783-e9fb-76b6-75c7-8f99de366500 Address:40ac4783-e9fb-76b6-75c7-8f99de366500} {Suffrage:Voter ID:server-294086ce-cd96-9e6b-e851-0939d2a9e970 Address:294086ce-cd96-9e6b-e851-0939d2a9e970}]" -2025-06-03T23:33:11.351Z [INFO] server-40ac4783-e9fb-76b6-75c7-8f99de366500: entering follower state: follower="Node at 40ac4783-e9fb-76b6-75c7-8f99de366500 [Follower]" leader-address= leader-id= -2025-06-03T23:33:11.351Z [INFO] creating new snapshot: path=/tmp/raft926312156/snapshots/2-27-1748993591351.tmp -2025-06-03T23:33:11.351Z [INFO] server-294086ce-cd96-9e6b-e851-0939d2a9e970: starting restore from snapshot: id=2-27-1748993591351 last-index=27 last-term=2 size-in-bytes=168 -2025-06-03T23:33:11.351Z [INFO] server-294086ce-cd96-9e6b-e851-0939d2a9e970: snapshot restore progress: id=2-27-1748993591351 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2025-06-03T23:33:11.351Z [INFO] server-294086ce-cd96-9e6b-e851-0939d2a9e970: restored from snapshot: id=2-27-1748993591351 last-index=27 last-term=2 size-in-bytes=168 -2025-06-03T23:33:11.351Z [INFO] server-294086ce-cd96-9e6b-e851-0939d2a9e970: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f7753d6-9c41-f672-ad87-55dba0299469 Address:6f7753d6-9c41-f672-ad87-55dba0299469} {Suffrage:Voter ID:server-40ac4783-e9fb-76b6-75c7-8f99de366500 Address:40ac4783-e9fb-76b6-75c7-8f99de366500} {Suffrage:Voter ID:server-294086ce-cd96-9e6b-e851-0939d2a9e970 Address:294086ce-cd96-9e6b-e851-0939d2a9e970}]" -2025-06-03T23:33:11.351Z [INFO] server-294086ce-cd96-9e6b-e851-0939d2a9e970: entering follower state: follower="Node at 294086ce-cd96-9e6b-e851-0939d2a9e970 [Follower]" leader-address= leader-id= -2025-06-03T23:33:11.419Z [WARN] server-6f7753d6-9c41-f672-ad87-55dba0299469: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:11.419Z [INFO] server-6f7753d6-9c41-f672-ad87-55dba0299469: entering candidate state: node="Node at 6f7753d6-9c41-f672-ad87-55dba0299469 [Candidate]" term=3 -2025-06-03T23:33:11.419Z [INFO] server-6f7753d6-9c41-f672-ad87-55dba0299469: election won: term=3 tally=2 -2025-06-03T23:33:11.419Z [INFO] server-6f7753d6-9c41-f672-ad87-55dba0299469: entering leader state: leader="Node at 6f7753d6-9c41-f672-ad87-55dba0299469 [Leader]" -2025-06-03T23:33:11.419Z [INFO] server-6f7753d6-9c41-f672-ad87-55dba0299469: added peer, starting replication: peer=server-40ac4783-e9fb-76b6-75c7-8f99de366500 -2025-06-03T23:33:11.419Z [INFO] server-6f7753d6-9c41-f672-ad87-55dba0299469: added peer, starting replication: peer=server-294086ce-cd96-9e6b-e851-0939d2a9e970 -2025-06-03T23:33:11.419Z [INFO] server-6f7753d6-9c41-f672-ad87-55dba0299469: pipelining replication: peer="{Voter server-294086ce-cd96-9e6b-e851-0939d2a9e970 294086ce-cd96-9e6b-e851-0939d2a9e970}" -2025-06-03T23:33:11.419Z [INFO] server-6f7753d6-9c41-f672-ad87-55dba0299469: pipelining replication: peer="{Voter server-40ac4783-e9fb-76b6-75c7-8f99de366500 40ac4783-e9fb-76b6-75c7-8f99de366500}" -2025-06-03T23:33:11.667Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:11.773Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:11.667505511 +0000 UTC m=+6.688324881 (1 nodes), 47.92µs from start of poll, 487.06796ms from cluster start. Timeout at 2025-06-03 23:33:11.773015551 +0000 UTC m=+6.793834921, 105.51004ms after stability -2025-06-03T23:33:11.773Z [INFO] server-6f7753d6-9c41-f672-ad87-55dba0299469: aborting pipeline replication: peer="{Voter server-40ac4783-e9fb-76b6-75c7-8f99de366500 40ac4783-e9fb-76b6-75c7-8f99de366500}" -2025-06-03T23:33:11.773Z [INFO] server-6f7753d6-9c41-f672-ad87-55dba0299469: aborting pipeline replication: peer="{Voter server-294086ce-cd96-9e6b-e851-0939d2a9e970 294086ce-cd96-9e6b-e851-0939d2a9e970}" +2025-06-03T23:36:10.379Z [INFO] server-c3a0e86c-a5f6-4394-80c0-937135d500f1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3a0e86c-a5f6-4394-80c0-937135d500f1 Address:c3a0e86c-a5f6-4394-80c0-937135d500f1} {Suffrage:Voter ID:server-37165c5a-9d22-7b92-c9b7-eaeae93c063a Address:37165c5a-9d22-7b92-c9b7-eaeae93c063a} {Suffrage:Voter ID:server-789fbd1b-a8e0-2863-d3af-011db7cf6157 Address:789fbd1b-a8e0-2863-d3af-011db7cf6157}]" +2025-06-03T23:36:10.379Z [INFO] server-37165c5a-9d22-7b92-c9b7-eaeae93c063a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3a0e86c-a5f6-4394-80c0-937135d500f1 Address:c3a0e86c-a5f6-4394-80c0-937135d500f1} {Suffrage:Voter ID:server-37165c5a-9d22-7b92-c9b7-eaeae93c063a Address:37165c5a-9d22-7b92-c9b7-eaeae93c063a} {Suffrage:Voter ID:server-789fbd1b-a8e0-2863-d3af-011db7cf6157 Address:789fbd1b-a8e0-2863-d3af-011db7cf6157}]" +2025-06-03T23:36:10.379Z [INFO] server-c3a0e86c-a5f6-4394-80c0-937135d500f1: entering follower state: follower="Node at c3a0e86c-a5f6-4394-80c0-937135d500f1 [Follower]" leader-address= leader-id= +2025-06-03T23:36:10.379Z [INFO] server-37165c5a-9d22-7b92-c9b7-eaeae93c063a: entering follower state: follower="Node at 37165c5a-9d22-7b92-c9b7-eaeae93c063a [Follower]" leader-address= leader-id= +2025-06-03T23:36:10.379Z [INFO] server-789fbd1b-a8e0-2863-d3af-011db7cf6157: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3a0e86c-a5f6-4394-80c0-937135d500f1 Address:c3a0e86c-a5f6-4394-80c0-937135d500f1} {Suffrage:Voter ID:server-37165c5a-9d22-7b92-c9b7-eaeae93c063a Address:37165c5a-9d22-7b92-c9b7-eaeae93c063a} {Suffrage:Voter ID:server-789fbd1b-a8e0-2863-d3af-011db7cf6157 Address:789fbd1b-a8e0-2863-d3af-011db7cf6157}]" +2025-06-03T23:36:10.379Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:10.379Z [INFO] server-789fbd1b-a8e0-2863-d3af-011db7cf6157: entering follower state: follower="Node at 789fbd1b-a8e0-2863-d3af-011db7cf6157 [Follower]" leader-address= leader-id= +2025-06-03T23:36:10.435Z [WARN] server-789fbd1b-a8e0-2863-d3af-011db7cf6157: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:10.435Z [INFO] server-789fbd1b-a8e0-2863-d3af-011db7cf6157: entering candidate state: node="Node at 789fbd1b-a8e0-2863-d3af-011db7cf6157 [Candidate]" term=2 +2025-06-03T23:36:10.435Z [INFO] server-789fbd1b-a8e0-2863-d3af-011db7cf6157: election won: term=2 tally=2 +2025-06-03T23:36:10.435Z [INFO] server-789fbd1b-a8e0-2863-d3af-011db7cf6157: entering leader state: leader="Node at 789fbd1b-a8e0-2863-d3af-011db7cf6157 [Leader]" +2025-06-03T23:36:10.435Z [INFO] server-789fbd1b-a8e0-2863-d3af-011db7cf6157: added peer, starting replication: peer=server-c3a0e86c-a5f6-4394-80c0-937135d500f1 +2025-06-03T23:36:10.436Z [INFO] server-789fbd1b-a8e0-2863-d3af-011db7cf6157: added peer, starting replication: peer=server-37165c5a-9d22-7b92-c9b7-eaeae93c063a +2025-06-03T23:36:10.436Z [INFO] server-789fbd1b-a8e0-2863-d3af-011db7cf6157: pipelining replication: peer="{Voter server-37165c5a-9d22-7b92-c9b7-eaeae93c063a 37165c5a-9d22-7b92-c9b7-eaeae93c063a}" +2025-06-03T23:36:10.436Z [INFO] server-789fbd1b-a8e0-2863-d3af-011db7cf6157: pipelining replication: peer="{Voter server-c3a0e86c-a5f6-4394-80c0-937135d500f1 c3a0e86c-a5f6-4394-80c0-937135d500f1}" +2025-06-03T23:36:10.542Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:10.436062431 +0000 UTC m=+5.757817361 (1 nodes), 56.4092ms from start of poll, 56.916ms from cluster start. Timeout at 2025-06-03 23:36:10.541995991 +0000 UTC m=+5.863750921, 105.93356ms after stability +2025-06-03T23:36:10.546Z [INFO] server-789fbd1b-a8e0-2863-d3af-011db7cf6157: aborting pipeline replication: peer="{Voter server-c3a0e86c-a5f6-4394-80c0-937135d500f1 c3a0e86c-a5f6-4394-80c0-937135d500f1}" +2025-06-03T23:36:10.546Z [INFO] server-789fbd1b-a8e0-2863-d3af-011db7cf6157: aborting pipeline replication: peer="{Voter server-37165c5a-9d22-7b92-c9b7-eaeae93c063a 37165c5a-9d22-7b92-c9b7-eaeae93c063a}" +2025-06-03T23:36:10.546Z [INFO] creating new snapshot: path=/tmp/raft1270472025/snapshots/2-27-1748993770546.tmp +2025-06-03T23:36:10.547Z [INFO] server-c3a0e86c-a5f6-4394-80c0-937135d500f1: starting restore from snapshot: id=2-27-1748993770546 last-index=27 last-term=2 size-in-bytes=168 +2025-06-03T23:36:10.547Z [INFO] server-c3a0e86c-a5f6-4394-80c0-937135d500f1: snapshot restore progress: id=2-27-1748993770546 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-06-03T23:36:10.547Z [INFO] server-c3a0e86c-a5f6-4394-80c0-937135d500f1: restored from snapshot: id=2-27-1748993770546 last-index=27 last-term=2 size-in-bytes=168 +2025-06-03T23:36:10.547Z [INFO] server-c3a0e86c-a5f6-4394-80c0-937135d500f1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3a0e86c-a5f6-4394-80c0-937135d500f1 Address:c3a0e86c-a5f6-4394-80c0-937135d500f1} {Suffrage:Voter ID:server-37165c5a-9d22-7b92-c9b7-eaeae93c063a Address:37165c5a-9d22-7b92-c9b7-eaeae93c063a} {Suffrage:Voter ID:server-789fbd1b-a8e0-2863-d3af-011db7cf6157 Address:789fbd1b-a8e0-2863-d3af-011db7cf6157}]" +2025-06-03T23:36:10.547Z [INFO] server-c3a0e86c-a5f6-4394-80c0-937135d500f1: entering follower state: follower="Node at c3a0e86c-a5f6-4394-80c0-937135d500f1 [Follower]" leader-address= leader-id= +2025-06-03T23:36:10.548Z [INFO] creating new snapshot: path=/tmp/raft438273918/snapshots/2-27-1748993770548.tmp +2025-06-03T23:36:10.548Z [INFO] server-37165c5a-9d22-7b92-c9b7-eaeae93c063a: starting restore from snapshot: id=2-27-1748993770548 last-index=27 last-term=2 size-in-bytes=168 +2025-06-03T23:36:10.548Z [INFO] server-37165c5a-9d22-7b92-c9b7-eaeae93c063a: snapshot restore progress: id=2-27-1748993770548 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-06-03T23:36:10.548Z [INFO] server-37165c5a-9d22-7b92-c9b7-eaeae93c063a: restored from snapshot: id=2-27-1748993770548 last-index=27 last-term=2 size-in-bytes=168 +2025-06-03T23:36:10.548Z [INFO] server-37165c5a-9d22-7b92-c9b7-eaeae93c063a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3a0e86c-a5f6-4394-80c0-937135d500f1 Address:c3a0e86c-a5f6-4394-80c0-937135d500f1} {Suffrage:Voter ID:server-37165c5a-9d22-7b92-c9b7-eaeae93c063a Address:37165c5a-9d22-7b92-c9b7-eaeae93c063a} {Suffrage:Voter ID:server-789fbd1b-a8e0-2863-d3af-011db7cf6157 Address:789fbd1b-a8e0-2863-d3af-011db7cf6157}]" +2025-06-03T23:36:10.548Z [INFO] server-37165c5a-9d22-7b92-c9b7-eaeae93c063a: entering follower state: follower="Node at 37165c5a-9d22-7b92-c9b7-eaeae93c063a [Follower]" leader-address= leader-id= +2025-06-03T23:36:10.548Z [INFO] creating new snapshot: path=/tmp/raft314256331/snapshots/2-27-1748993770548.tmp +2025-06-03T23:36:10.549Z [INFO] server-789fbd1b-a8e0-2863-d3af-011db7cf6157: starting restore from snapshot: id=2-27-1748993770548 last-index=27 last-term=2 size-in-bytes=168 +2025-06-03T23:36:10.549Z [INFO] server-789fbd1b-a8e0-2863-d3af-011db7cf6157: snapshot restore progress: id=2-27-1748993770548 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-06-03T23:36:10.549Z [INFO] server-789fbd1b-a8e0-2863-d3af-011db7cf6157: restored from snapshot: id=2-27-1748993770548 last-index=27 last-term=2 size-in-bytes=168 +2025-06-03T23:36:10.549Z [INFO] server-789fbd1b-a8e0-2863-d3af-011db7cf6157: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3a0e86c-a5f6-4394-80c0-937135d500f1 Address:c3a0e86c-a5f6-4394-80c0-937135d500f1} {Suffrage:Voter ID:server-37165c5a-9d22-7b92-c9b7-eaeae93c063a Address:37165c5a-9d22-7b92-c9b7-eaeae93c063a} {Suffrage:Voter ID:server-789fbd1b-a8e0-2863-d3af-011db7cf6157 Address:789fbd1b-a8e0-2863-d3af-011db7cf6157}]" +2025-06-03T23:36:10.549Z [INFO] server-789fbd1b-a8e0-2863-d3af-011db7cf6157: entering follower state: follower="Node at 789fbd1b-a8e0-2863-d3af-011db7cf6157 [Follower]" leader-address= leader-id= +2025-06-03T23:36:10.603Z [WARN] server-c3a0e86c-a5f6-4394-80c0-937135d500f1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:10.603Z [INFO] server-c3a0e86c-a5f6-4394-80c0-937135d500f1: entering candidate state: node="Node at c3a0e86c-a5f6-4394-80c0-937135d500f1 [Candidate]" term=3 +2025-06-03T23:36:10.603Z [INFO] server-c3a0e86c-a5f6-4394-80c0-937135d500f1: election won: term=3 tally=2 +2025-06-03T23:36:10.603Z [INFO] server-c3a0e86c-a5f6-4394-80c0-937135d500f1: entering leader state: leader="Node at c3a0e86c-a5f6-4394-80c0-937135d500f1 [Leader]" +2025-06-03T23:36:10.603Z [INFO] server-c3a0e86c-a5f6-4394-80c0-937135d500f1: added peer, starting replication: peer=server-37165c5a-9d22-7b92-c9b7-eaeae93c063a +2025-06-03T23:36:10.603Z [INFO] server-c3a0e86c-a5f6-4394-80c0-937135d500f1: added peer, starting replication: peer=server-789fbd1b-a8e0-2863-d3af-011db7cf6157 +2025-06-03T23:36:10.603Z [INFO] server-c3a0e86c-a5f6-4394-80c0-937135d500f1: pipelining replication: peer="{Voter server-37165c5a-9d22-7b92-c9b7-eaeae93c063a 37165c5a-9d22-7b92-c9b7-eaeae93c063a}" +2025-06-03T23:36:10.603Z [INFO] server-c3a0e86c-a5f6-4394-80c0-937135d500f1: pipelining replication: peer="{Voter server-789fbd1b-a8e0-2863-d3af-011db7cf6157 789fbd1b-a8e0-2863-d3af-011db7cf6157}" +2025-06-03T23:36:10.865Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:10.970Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:10.865247511 +0000 UTC m=+6.187002441 (1 nodes), 44.88µs from start of poll, 486.10108ms from cluster start. Timeout at 2025-06-03 23:36:10.970507231 +0000 UTC m=+6.292262161, 105.25972ms after stability +2025-06-03T23:36:10.970Z [INFO] server-c3a0e86c-a5f6-4394-80c0-937135d500f1: aborting pipeline replication: peer="{Voter server-37165c5a-9d22-7b92-c9b7-eaeae93c063a 37165c5a-9d22-7b92-c9b7-eaeae93c063a}" +2025-06-03T23:36:10.970Z [INFO] server-c3a0e86c-a5f6-4394-80c0-937135d500f1: aborting pipeline replication: peer="{Voter server-789fbd1b-a8e0-2863-d3af-011db7cf6157 789fbd1b-a8e0-2863-d3af-011db7cf6157}" --- PASS: TestRaft_RecoverCluster (1.79s) - --- PASS: TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs (0.59s) + --- PASS: TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs (0.60s) --- PASS: TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs (0.60s) --- PASS: TestRaft_RecoverCluster/snapshot,_with_trailing_logs (0.59s) === RUN TestRaft_HasExistingState raft_test.go:258: Skipping unreliable test --- SKIP: TestRaft_HasExistingState (0.00s) === RUN TestRaft_SingleNode -2025-06-03T23:33:11.774Z [INFO] server-d008b5b6-8386-e5d2-0211-fc105d719961: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d008b5b6-8386-e5d2-0211-fc105d719961 Address:d008b5b6-8386-e5d2-0211-fc105d719961}]" -2025-06-03T23:33:11.774Z [INFO] server-d008b5b6-8386-e5d2-0211-fc105d719961: entering follower state: follower="Node at d008b5b6-8386-e5d2-0211-fc105d719961 [Follower]" leader-address= leader-id= -2025-06-03T23:33:11.828Z [WARN] server-d008b5b6-8386-e5d2-0211-fc105d719961: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:11.828Z [INFO] server-d008b5b6-8386-e5d2-0211-fc105d719961: entering candidate state: node="Node at d008b5b6-8386-e5d2-0211-fc105d719961 [Candidate]" term=2 -2025-06-03T23:33:11.828Z [INFO] server-d008b5b6-8386-e5d2-0211-fc105d719961: election won: term=2 tally=1 -2025-06-03T23:33:11.828Z [INFO] server-d008b5b6-8386-e5d2-0211-fc105d719961: entering leader state: leader="Node at d008b5b6-8386-e5d2-0211-fc105d719961 [Leader]" ---- PASS: TestRaft_SingleNode (0.05s) +2025-06-03T23:36:10.972Z [INFO] server-a9bf9656-035a-03ac-787d-802ec2d3abea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a9bf9656-035a-03ac-787d-802ec2d3abea Address:a9bf9656-035a-03ac-787d-802ec2d3abea}]" +2025-06-03T23:36:10.972Z [INFO] server-a9bf9656-035a-03ac-787d-802ec2d3abea: entering follower state: follower="Node at a9bf9656-035a-03ac-787d-802ec2d3abea [Follower]" leader-address= leader-id= +2025-06-03T23:36:11.032Z [WARN] server-a9bf9656-035a-03ac-787d-802ec2d3abea: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:11.032Z [INFO] server-a9bf9656-035a-03ac-787d-802ec2d3abea: entering candidate state: node="Node at a9bf9656-035a-03ac-787d-802ec2d3abea [Candidate]" term=2 +2025-06-03T23:36:11.033Z [INFO] server-a9bf9656-035a-03ac-787d-802ec2d3abea: election won: term=2 tally=1 +2025-06-03T23:36:11.033Z [INFO] server-a9bf9656-035a-03ac-787d-802ec2d3abea: entering leader state: leader="Node at a9bf9656-035a-03ac-787d-802ec2d3abea [Leader]" +--- PASS: TestRaft_SingleNode (0.06s) === RUN TestRaft_TripleNode -2025-06-03T23:33:11.829Z [INFO] server-0bebd902-aa09-5cec-a978-ba230923ee3d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0bebd902-aa09-5cec-a978-ba230923ee3d Address:0bebd902-aa09-5cec-a978-ba230923ee3d} {Suffrage:Voter ID:server-967bb40c-0aa7-b4c6-e22d-8a419be582e0 Address:967bb40c-0aa7-b4c6-e22d-8a419be582e0} {Suffrage:Voter ID:server-48f823ee-5b06-5acf-9086-ee4c1ed82d20 Address:48f823ee-5b06-5acf-9086-ee4c1ed82d20}]" -2025-06-03T23:33:11.829Z [INFO] server-0bebd902-aa09-5cec-a978-ba230923ee3d: entering follower state: follower="Node at 0bebd902-aa09-5cec-a978-ba230923ee3d [Follower]" leader-address= leader-id= -2025-06-03T23:33:11.829Z [INFO] server-967bb40c-0aa7-b4c6-e22d-8a419be582e0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0bebd902-aa09-5cec-a978-ba230923ee3d Address:0bebd902-aa09-5cec-a978-ba230923ee3d} {Suffrage:Voter ID:server-967bb40c-0aa7-b4c6-e22d-8a419be582e0 Address:967bb40c-0aa7-b4c6-e22d-8a419be582e0} {Suffrage:Voter ID:server-48f823ee-5b06-5acf-9086-ee4c1ed82d20 Address:48f823ee-5b06-5acf-9086-ee4c1ed82d20}]" -2025-06-03T23:33:11.829Z [INFO] server-967bb40c-0aa7-b4c6-e22d-8a419be582e0: entering follower state: follower="Node at 967bb40c-0aa7-b4c6-e22d-8a419be582e0 [Follower]" leader-address= leader-id= -2025-06-03T23:33:11.829Z [INFO] server-48f823ee-5b06-5acf-9086-ee4c1ed82d20: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0bebd902-aa09-5cec-a978-ba230923ee3d Address:0bebd902-aa09-5cec-a978-ba230923ee3d} {Suffrage:Voter ID:server-967bb40c-0aa7-b4c6-e22d-8a419be582e0 Address:967bb40c-0aa7-b4c6-e22d-8a419be582e0} {Suffrage:Voter ID:server-48f823ee-5b06-5acf-9086-ee4c1ed82d20 Address:48f823ee-5b06-5acf-9086-ee4c1ed82d20}]" -2025-06-03T23:33:11.829Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:11.829Z [INFO] server-48f823ee-5b06-5acf-9086-ee4c1ed82d20: entering follower state: follower="Node at 48f823ee-5b06-5acf-9086-ee4c1ed82d20 [Follower]" leader-address= leader-id= -2025-06-03T23:33:11.883Z [WARN] server-967bb40c-0aa7-b4c6-e22d-8a419be582e0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:11.883Z [INFO] server-967bb40c-0aa7-b4c6-e22d-8a419be582e0: entering candidate state: node="Node at 967bb40c-0aa7-b4c6-e22d-8a419be582e0 [Candidate]" term=2 -2025-06-03T23:33:11.883Z [INFO] server-967bb40c-0aa7-b4c6-e22d-8a419be582e0: election won: term=2 tally=2 -2025-06-03T23:33:11.883Z [INFO] server-967bb40c-0aa7-b4c6-e22d-8a419be582e0: entering leader state: leader="Node at 967bb40c-0aa7-b4c6-e22d-8a419be582e0 [Leader]" -2025-06-03T23:33:11.883Z [INFO] server-967bb40c-0aa7-b4c6-e22d-8a419be582e0: added peer, starting replication: peer=server-0bebd902-aa09-5cec-a978-ba230923ee3d -2025-06-03T23:33:11.883Z [INFO] server-967bb40c-0aa7-b4c6-e22d-8a419be582e0: added peer, starting replication: peer=server-48f823ee-5b06-5acf-9086-ee4c1ed82d20 -2025-06-03T23:33:11.883Z [INFO] server-967bb40c-0aa7-b4c6-e22d-8a419be582e0: pipelining replication: peer="{Voter server-48f823ee-5b06-5acf-9086-ee4c1ed82d20 48f823ee-5b06-5acf-9086-ee4c1ed82d20}" -2025-06-03T23:33:11.883Z [INFO] server-967bb40c-0aa7-b4c6-e22d-8a419be582e0: pipelining replication: peer="{Voter server-0bebd902-aa09-5cec-a978-ba230923ee3d 0bebd902-aa09-5cec-a978-ba230923ee3d}" -2025-06-03T23:33:11.988Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:11.883647231 +0000 UTC m=+6.904466601 (2 nodes), 54.0016ms from start of poll, 54.52048ms from cluster start. Timeout at 2025-06-03 23:33:11.988872471 +0000 UTC m=+7.009691841, 105.22524ms after stability -2025-06-03T23:33:11.988Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:12.094Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:11.989007151 +0000 UTC m=+7.009826521 (1 nodes), 1.44µs from start of poll, 159.8804ms from cluster start. Timeout at 2025-06-03 23:33:12.094053551 +0000 UTC m=+7.114872961, 105.04644ms after stability -2025-06-03T23:33:12.105Z [INFO] server-967bb40c-0aa7-b4c6-e22d-8a419be582e0: aborting pipeline replication: peer="{Voter server-48f823ee-5b06-5acf-9086-ee4c1ed82d20 48f823ee-5b06-5acf-9086-ee4c1ed82d20}" -2025-06-03T23:33:12.105Z [INFO] server-967bb40c-0aa7-b4c6-e22d-8a419be582e0: aborting pipeline replication: peer="{Voter server-0bebd902-aa09-5cec-a978-ba230923ee3d 0bebd902-aa09-5cec-a978-ba230923ee3d}" ---- PASS: TestRaft_TripleNode (0.28s) +2025-06-03T23:36:11.033Z [INFO] server-8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9 Address:8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9} {Suffrage:Voter ID:server-5ff022a1-f5a5-b802-901d-194885979bbe Address:5ff022a1-f5a5-b802-901d-194885979bbe} {Suffrage:Voter ID:server-b98e5d23-cafb-9076-8ed2-b783fcf61f13 Address:b98e5d23-cafb-9076-8ed2-b783fcf61f13}]" +2025-06-03T23:36:11.034Z [INFO] server-8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9: entering follower state: follower="Node at 8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9 [Follower]" leader-address= leader-id= +2025-06-03T23:36:11.034Z [INFO] server-5ff022a1-f5a5-b802-901d-194885979bbe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9 Address:8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9} {Suffrage:Voter ID:server-5ff022a1-f5a5-b802-901d-194885979bbe Address:5ff022a1-f5a5-b802-901d-194885979bbe} {Suffrage:Voter ID:server-b98e5d23-cafb-9076-8ed2-b783fcf61f13 Address:b98e5d23-cafb-9076-8ed2-b783fcf61f13}]" +2025-06-03T23:36:11.034Z [INFO] server-b98e5d23-cafb-9076-8ed2-b783fcf61f13: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9 Address:8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9} {Suffrage:Voter ID:server-5ff022a1-f5a5-b802-901d-194885979bbe Address:5ff022a1-f5a5-b802-901d-194885979bbe} {Suffrage:Voter ID:server-b98e5d23-cafb-9076-8ed2-b783fcf61f13 Address:b98e5d23-cafb-9076-8ed2-b783fcf61f13}]" +2025-06-03T23:36:11.034Z [INFO] server-5ff022a1-f5a5-b802-901d-194885979bbe: entering follower state: follower="Node at 5ff022a1-f5a5-b802-901d-194885979bbe [Follower]" leader-address= leader-id= +2025-06-03T23:36:11.034Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:11.034Z [INFO] server-b98e5d23-cafb-9076-8ed2-b783fcf61f13: entering follower state: follower="Node at b98e5d23-cafb-9076-8ed2-b783fcf61f13 [Follower]" leader-address= leader-id= +2025-06-03T23:36:11.103Z [WARN] server-8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:11.103Z [INFO] server-8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9: entering candidate state: node="Node at 8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9 [Candidate]" term=2 +2025-06-03T23:36:11.103Z [INFO] server-8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9: election won: term=2 tally=2 +2025-06-03T23:36:11.103Z [INFO] server-8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9: entering leader state: leader="Node at 8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9 [Leader]" +2025-06-03T23:36:11.103Z [INFO] server-8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9: added peer, starting replication: peer=server-5ff022a1-f5a5-b802-901d-194885979bbe +2025-06-03T23:36:11.103Z [INFO] server-8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9: added peer, starting replication: peer=server-b98e5d23-cafb-9076-8ed2-b783fcf61f13 +2025-06-03T23:36:11.104Z [INFO] server-8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9: pipelining replication: peer="{Voter server-b98e5d23-cafb-9076-8ed2-b783fcf61f13 b98e5d23-cafb-9076-8ed2-b783fcf61f13}" +2025-06-03T23:36:11.104Z [INFO] server-8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9: pipelining replication: peer="{Voter server-5ff022a1-f5a5-b802-901d-194885979bbe 5ff022a1-f5a5-b802-901d-194885979bbe}" +2025-06-03T23:36:11.209Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:11.104063351 +0000 UTC m=+6.425818281 (2 nodes), 69.77736ms from start of poll, 70.22068ms from cluster start. Timeout at 2025-06-03 23:36:11.209396431 +0000 UTC m=+6.531151361, 105.33308ms after stability +2025-06-03T23:36:11.209Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:11.314Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:11.209516471 +0000 UTC m=+6.531271401 (1 nodes), 960ns from start of poll, 175.6738ms from cluster start. Timeout at 2025-06-03 23:36:11.314525191 +0000 UTC m=+6.636280121, 105.00872ms after stability +2025-06-03T23:36:11.326Z [INFO] server-8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9: aborting pipeline replication: peer="{Voter server-5ff022a1-f5a5-b802-901d-194885979bbe 5ff022a1-f5a5-b802-901d-194885979bbe}" +2025-06-03T23:36:11.326Z [INFO] server-8556e5f8-0b19-f7a3-db2a-9ccb8c9f29c9: aborting pipeline replication: peer="{Voter server-b98e5d23-cafb-9076-8ed2-b783fcf61f13 b98e5d23-cafb-9076-8ed2-b783fcf61f13}" +--- PASS: TestRaft_TripleNode (0.29s) === RUN TestRaft_LeaderFail -2025-06-03T23:33:12.106Z [INFO] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9 Address:52c63f1d-88b7-8bde-9b10-35417f5b8ba9} {Suffrage:Voter ID:server-01671222-c73f-ede3-f3da-c97116c2ba2d Address:01671222-c73f-ede3-f3da-c97116c2ba2d} {Suffrage:Voter ID:server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c Address:13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c}]" -2025-06-03T23:33:12.106Z [INFO] server-01671222-c73f-ede3-f3da-c97116c2ba2d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9 Address:52c63f1d-88b7-8bde-9b10-35417f5b8ba9} {Suffrage:Voter ID:server-01671222-c73f-ede3-f3da-c97116c2ba2d Address:01671222-c73f-ede3-f3da-c97116c2ba2d} {Suffrage:Voter ID:server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c Address:13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c}]" -2025-06-03T23:33:12.106Z [INFO] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: entering follower state: follower="Node at 52c63f1d-88b7-8bde-9b10-35417f5b8ba9 [Follower]" leader-address= leader-id= -2025-06-03T23:33:12.106Z [INFO] server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9 Address:52c63f1d-88b7-8bde-9b10-35417f5b8ba9} {Suffrage:Voter ID:server-01671222-c73f-ede3-f3da-c97116c2ba2d Address:01671222-c73f-ede3-f3da-c97116c2ba2d} {Suffrage:Voter ID:server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c Address:13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c}]" -2025-06-03T23:33:12.106Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:12.106Z [INFO] server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c: entering follower state: follower="Node at 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c [Follower]" leader-address= leader-id= -2025-06-03T23:33:12.106Z [INFO] server-01671222-c73f-ede3-f3da-c97116c2ba2d: entering follower state: follower="Node at 01671222-c73f-ede3-f3da-c97116c2ba2d [Follower]" leader-address= leader-id= -2025-06-03T23:33:12.162Z [WARN] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:12.162Z [INFO] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: entering candidate state: node="Node at 52c63f1d-88b7-8bde-9b10-35417f5b8ba9 [Candidate]" term=2 -2025-06-03T23:33:12.162Z [INFO] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: election won: term=2 tally=2 -2025-06-03T23:33:12.162Z [INFO] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: entering leader state: leader="Node at 52c63f1d-88b7-8bde-9b10-35417f5b8ba9 [Leader]" -2025-06-03T23:33:12.162Z [INFO] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: added peer, starting replication: peer=server-01671222-c73f-ede3-f3da-c97116c2ba2d -2025-06-03T23:33:12.162Z [INFO] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: added peer, starting replication: peer=server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c -2025-06-03T23:33:12.162Z [INFO] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: pipelining replication: peer="{Voter server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c}" -2025-06-03T23:33:12.163Z [INFO] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: pipelining replication: peer="{Voter server-01671222-c73f-ede3-f3da-c97116c2ba2d 01671222-c73f-ede3-f3da-c97116c2ba2d}" -2025-06-03T23:33:12.268Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:12.163204591 +0000 UTC m=+7.184023961 (2 nodes), 56.483ms from start of poll, 56.99948ms from cluster start. Timeout at 2025-06-03 23:33:12.268840951 +0000 UTC m=+7.289660321, 105.63636ms after stability -2025-06-03T23:33:12.268Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:12.374Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:12.268977231 +0000 UTC m=+7.289796601 (1 nodes), 2.44µs from start of poll, 162.77212ms from cluster start. Timeout at 2025-06-03 23:33:12.374205351 +0000 UTC m=+7.395024721, 105.22812ms after stability - raft_test.go:378: [INFO] Disconnecting Node at 52c63f1d-88b7-8bde-9b10-35417f5b8ba9 [Leader] -2025-06-03T23:33:12.386Z [ERROR] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to pipeline appendEntries: peer="{Voter server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c}" error="append pipeline closed" -2025-06-03T23:33:12.386Z [INFO] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: aborting pipeline replication: peer="{Voter server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c}" -2025-06-03T23:33:12.387Z [ERROR] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to pipeline appendEntries: peer="{Voter server-01671222-c73f-ede3-f3da-c97116c2ba2d 01671222-c73f-ede3-f3da-c97116c2ba2d}" error="append pipeline closed" -2025-06-03T23:33:12.387Z [INFO] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: aborting pipeline replication: peer="{Voter server-01671222-c73f-ede3-f3da-c97116c2ba2d 01671222-c73f-ede3-f3da-c97116c2ba2d}" -2025-06-03T23:33:12.388Z [ERROR] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to heartbeat to: peer=13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c backoff time=10ms error="failed to connect to peer: 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c" -2025-06-03T23:33:12.388Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:12.389Z [ERROR] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to heartbeat to: peer=01671222-c73f-ede3-f3da-c97116c2ba2d backoff time=10ms error="failed to connect to peer: 01671222-c73f-ede3-f3da-c97116c2ba2d" -2025-06-03T23:33:12.393Z [ERROR] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to appendEntries to: peer="{Voter server-01671222-c73f-ede3-f3da-c97116c2ba2d 01671222-c73f-ede3-f3da-c97116c2ba2d}" error="failed to connect to peer: 01671222-c73f-ede3-f3da-c97116c2ba2d" -2025-06-03T23:33:12.394Z [ERROR] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to appendEntries to: peer="{Voter server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c}" error="failed to connect to peer: 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c" -2025-06-03T23:33:12.404Z [ERROR] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to heartbeat to: peer=01671222-c73f-ede3-f3da-c97116c2ba2d backoff time=10ms error="failed to connect to peer: 01671222-c73f-ede3-f3da-c97116c2ba2d" -2025-06-03T23:33:12.409Z [ERROR] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to heartbeat to: peer=13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c backoff time=10ms error="failed to connect to peer: 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c" -2025-06-03T23:33:12.410Z [ERROR] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to appendEntries to: peer="{Voter server-01671222-c73f-ede3-f3da-c97116c2ba2d 01671222-c73f-ede3-f3da-c97116c2ba2d}" error="failed to connect to peer: 01671222-c73f-ede3-f3da-c97116c2ba2d" -2025-06-03T23:33:12.412Z [ERROR] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to appendEntries to: peer="{Voter server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c}" error="failed to connect to peer: 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c" -2025-06-03T23:33:12.421Z [ERROR] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to heartbeat to: peer=01671222-c73f-ede3-f3da-c97116c2ba2d backoff time=10ms error="failed to connect to peer: 01671222-c73f-ede3-f3da-c97116c2ba2d" -2025-06-03T23:33:12.428Z [ERROR] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to heartbeat to: peer=13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c backoff time=10ms error="failed to connect to peer: 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c" -2025-06-03T23:33:12.429Z [ERROR] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to appendEntries to: peer="{Voter server-01671222-c73f-ede3-f3da-c97116c2ba2d 01671222-c73f-ede3-f3da-c97116c2ba2d}" error="failed to connect to peer: 01671222-c73f-ede3-f3da-c97116c2ba2d" -2025-06-03T23:33:12.432Z [WARN] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to contact: server-id=server-01671222-c73f-ede3-f3da-c97116c2ba2d time=51.07964ms -2025-06-03T23:33:12.432Z [WARN] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to contact: server-id=server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c time=51.1184ms -2025-06-03T23:33:12.432Z [WARN] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to contact quorum of nodes, stepping down -2025-06-03T23:33:12.432Z [INFO] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: entering follower state: follower="Node at 52c63f1d-88b7-8bde-9b10-35417f5b8ba9 [Follower]" leader-address= leader-id= -2025-06-03T23:33:12.433Z [ERROR] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to appendEntries to: peer="{Voter server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c}" error="failed to connect to peer: 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c" -2025-06-03T23:33:12.459Z [WARN] server-01671222-c73f-ede3-f3da-c97116c2ba2d: heartbeat timeout reached, starting election: last-leader-addr=52c63f1d-88b7-8bde-9b10-35417f5b8ba9 last-leader-id=server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9 -2025-06-03T23:33:12.459Z [INFO] server-01671222-c73f-ede3-f3da-c97116c2ba2d: entering candidate state: node="Node at 01671222-c73f-ede3-f3da-c97116c2ba2d [Candidate]" term=3 -2025-06-03T23:33:12.459Z [WARN] server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c: rejecting vote request since we have a leader: from=01671222-c73f-ede3-f3da-c97116c2ba2d leader=52c63f1d-88b7-8bde-9b10-35417f5b8ba9 leader-id=server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9 -2025-06-03T23:33:12.459Z [ERROR] server-01671222-c73f-ede3-f3da-c97116c2ba2d: failed to make requestVote RPC: target="{Voter server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9 52c63f1d-88b7-8bde-9b10-35417f5b8ba9}" error="failed to connect to peer: 52c63f1d-88b7-8bde-9b10-35417f5b8ba9" term=3 -2025-06-03T23:33:12.480Z [WARN] server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c: heartbeat timeout reached, starting election: last-leader-addr=52c63f1d-88b7-8bde-9b10-35417f5b8ba9 last-leader-id=server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9 -2025-06-03T23:33:12.480Z [INFO] server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c: entering candidate state: node="Node at 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c [Candidate]" term=3 -2025-06-03T23:33:12.480Z [INFO] server-01671222-c73f-ede3-f3da-c97116c2ba2d: duplicate requestVote for same term: term=3 -2025-06-03T23:33:12.480Z [ERROR] server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c: failed to make requestVote RPC: target="{Voter server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9 52c63f1d-88b7-8bde-9b10-35417f5b8ba9}" error="failed to connect to peer: 52c63f1d-88b7-8bde-9b10-35417f5b8ba9" term=3 -2025-06-03T23:33:12.520Z [WARN] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:12.520Z [INFO] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: entering candidate state: node="Node at 52c63f1d-88b7-8bde-9b10-35417f5b8ba9 [Candidate]" term=3 -2025-06-03T23:33:12.520Z [ERROR] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to make requestVote RPC: target="{Voter server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c}" error="failed to connect to peer: 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c" term=3 -2025-06-03T23:33:12.521Z [ERROR] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to make requestVote RPC: target="{Voter server-01671222-c73f-ede3-f3da-c97116c2ba2d 01671222-c73f-ede3-f3da-c97116c2ba2d}" error="failed to connect to peer: 01671222-c73f-ede3-f3da-c97116c2ba2d" term=3 -2025-06-03T23:33:12.543Z [WARN] server-01671222-c73f-ede3-f3da-c97116c2ba2d: Election timeout reached, restarting election -2025-06-03T23:33:12.543Z [INFO] server-01671222-c73f-ede3-f3da-c97116c2ba2d: entering candidate state: node="Node at 01671222-c73f-ede3-f3da-c97116c2ba2d [Candidate]" term=4 -2025-06-03T23:33:12.543Z [INFO] server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c: entering follower state: follower="Node at 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c [Follower]" leader-address= leader-id= -2025-06-03T23:33:12.543Z [INFO] server-01671222-c73f-ede3-f3da-c97116c2ba2d: election won: term=4 tally=2 -2025-06-03T23:33:12.543Z [INFO] server-01671222-c73f-ede3-f3da-c97116c2ba2d: entering leader state: leader="Node at 01671222-c73f-ede3-f3da-c97116c2ba2d [Leader]" -2025-06-03T23:33:12.543Z [INFO] server-01671222-c73f-ede3-f3da-c97116c2ba2d: added peer, starting replication: peer=server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9 -2025-06-03T23:33:12.543Z [INFO] server-01671222-c73f-ede3-f3da-c97116c2ba2d: added peer, starting replication: peer=server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c -2025-06-03T23:33:12.544Z [INFO] server-01671222-c73f-ede3-f3da-c97116c2ba2d: pipelining replication: peer="{Voter server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c}" -2025-06-03T23:33:12.544Z [ERROR] server-01671222-c73f-ede3-f3da-c97116c2ba2d: failed to make requestVote RPC: target="{Voter server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9 52c63f1d-88b7-8bde-9b10-35417f5b8ba9}" error="failed to connect to peer: 52c63f1d-88b7-8bde-9b10-35417f5b8ba9" term=4 -2025-06-03T23:33:12.544Z [ERROR] server-01671222-c73f-ede3-f3da-c97116c2ba2d: failed to appendEntries to: peer="{Voter server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9 52c63f1d-88b7-8bde-9b10-35417f5b8ba9}" error="failed to connect to peer: 52c63f1d-88b7-8bde-9b10-35417f5b8ba9" -2025-06-03T23:33:12.552Z [ERROR] server-01671222-c73f-ede3-f3da-c97116c2ba2d: failed to heartbeat to: peer=52c63f1d-88b7-8bde-9b10-35417f5b8ba9 backoff time=10ms error="failed to connect to peer: 52c63f1d-88b7-8bde-9b10-35417f5b8ba9" -2025-06-03T23:33:12.560Z [ERROR] server-01671222-c73f-ede3-f3da-c97116c2ba2d: failed to appendEntries to: peer="{Voter server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9 52c63f1d-88b7-8bde-9b10-35417f5b8ba9}" error="failed to connect to peer: 52c63f1d-88b7-8bde-9b10-35417f5b8ba9" -2025-06-03T23:33:12.569Z [ERROR] server-01671222-c73f-ede3-f3da-c97116c2ba2d: failed to heartbeat to: peer=52c63f1d-88b7-8bde-9b10-35417f5b8ba9 backoff time=10ms error="failed to connect to peer: 52c63f1d-88b7-8bde-9b10-35417f5b8ba9" -2025-06-03T23:33:12.578Z [ERROR] server-01671222-c73f-ede3-f3da-c97116c2ba2d: failed to appendEntries to: peer="{Voter server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9 52c63f1d-88b7-8bde-9b10-35417f5b8ba9}" error="failed to connect to peer: 52c63f1d-88b7-8bde-9b10-35417f5b8ba9" -2025-06-03T23:33:12.590Z [ERROR] server-01671222-c73f-ede3-f3da-c97116c2ba2d: failed to heartbeat to: peer=52c63f1d-88b7-8bde-9b10-35417f5b8ba9 backoff time=10ms error="failed to connect to peer: 52c63f1d-88b7-8bde-9b10-35417f5b8ba9" -2025-06-03T23:33:12.594Z [WARN] server-01671222-c73f-ede3-f3da-c97116c2ba2d: failed to contact: server-id=server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9 time=50.563ms -2025-06-03T23:33:12.601Z [WARN] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: Election timeout reached, restarting election -2025-06-03T23:33:12.601Z [INFO] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: entering candidate state: node="Node at 52c63f1d-88b7-8bde-9b10-35417f5b8ba9 [Candidate]" term=4 -2025-06-03T23:33:12.601Z [ERROR] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to make requestVote RPC: target="{Voter server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c}" error="failed to connect to peer: 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c" term=4 -2025-06-03T23:33:12.601Z [ERROR] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: failed to make requestVote RPC: target="{Voter server-01671222-c73f-ede3-f3da-c97116c2ba2d 01671222-c73f-ede3-f3da-c97116c2ba2d}" error="failed to connect to peer: 01671222-c73f-ede3-f3da-c97116c2ba2d" term=4 -2025-06-03T23:33:12.607Z [ERROR] server-01671222-c73f-ede3-f3da-c97116c2ba2d: failed to appendEntries to: peer="{Voter server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9 52c63f1d-88b7-8bde-9b10-35417f5b8ba9}" error="failed to connect to peer: 52c63f1d-88b7-8bde-9b10-35417f5b8ba9" -2025-06-03T23:33:12.608Z [ERROR] server-01671222-c73f-ede3-f3da-c97116c2ba2d: failed to heartbeat to: peer=52c63f1d-88b7-8bde-9b10-35417f5b8ba9 backoff time=20ms error="failed to connect to peer: 52c63f1d-88b7-8bde-9b10-35417f5b8ba9" -2025-06-03T23:33:12.636Z [WARN] server-01671222-c73f-ede3-f3da-c97116c2ba2d: failed to contact: server-id=server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9 time=92.00592ms -2025-06-03T23:33:12.638Z [ERROR] server-01671222-c73f-ede3-f3da-c97116c2ba2d: failed to heartbeat to: peer=52c63f1d-88b7-8bde-9b10-35417f5b8ba9 backoff time=25ms error="failed to connect to peer: 52c63f1d-88b7-8bde-9b10-35417f5b8ba9" -2025-06-03T23:33:12.649Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:12.544225351 +0000 UTC m=+7.565044761 (1 nodes), 155.2566ms from start of poll, 438.02028ms from cluster start. Timeout at 2025-06-03 23:33:12.649811351 +0000 UTC m=+7.670630721, 105.58596ms after stability - raft_test.go:413: [INFO] Reconnecting Node at 52c63f1d-88b7-8bde-9b10-35417f5b8ba9 [Candidate] -2025-06-03T23:33:12.655Z [INFO] server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9: entering follower state: follower="Node at 52c63f1d-88b7-8bde-9b10-35417f5b8ba9 [Follower]" leader-address=01671222-c73f-ede3-f3da-c97116c2ba2d leader-id=server-01671222-c73f-ede3-f3da-c97116c2ba2d -2025-06-03T23:33:12.655Z [INFO] server-01671222-c73f-ede3-f3da-c97116c2ba2d: pipelining replication: peer="{Voter server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9 52c63f1d-88b7-8bde-9b10-35417f5b8ba9}" -2025-06-03T23:33:12.661Z [INFO] server-01671222-c73f-ede3-f3da-c97116c2ba2d: aborting pipeline replication: peer="{Voter server-13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c 13c051f5-e0a9-7bad-b8b5-4cb8f74e5b1c}" -2025-06-03T23:33:12.661Z [INFO] server-01671222-c73f-ede3-f3da-c97116c2ba2d: aborting pipeline replication: peer="{Voter server-52c63f1d-88b7-8bde-9b10-35417f5b8ba9 52c63f1d-88b7-8bde-9b10-35417f5b8ba9}" ---- PASS: TestRaft_LeaderFail (0.56s) +2025-06-03T23:36:11.327Z [INFO] server-cf845591-38d9-caa4-dd0a-ff2816669258: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf845591-38d9-caa4-dd0a-ff2816669258 Address:cf845591-38d9-caa4-dd0a-ff2816669258} {Suffrage:Voter ID:server-0124c84b-4ee6-ba0b-d409-c1e41835a863 Address:0124c84b-4ee6-ba0b-d409-c1e41835a863} {Suffrage:Voter ID:server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd Address:b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd}]" +2025-06-03T23:36:11.327Z [INFO] server-cf845591-38d9-caa4-dd0a-ff2816669258: entering follower state: follower="Node at cf845591-38d9-caa4-dd0a-ff2816669258 [Follower]" leader-address= leader-id= +2025-06-03T23:36:11.327Z [INFO] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf845591-38d9-caa4-dd0a-ff2816669258 Address:cf845591-38d9-caa4-dd0a-ff2816669258} {Suffrage:Voter ID:server-0124c84b-4ee6-ba0b-d409-c1e41835a863 Address:0124c84b-4ee6-ba0b-d409-c1e41835a863} {Suffrage:Voter ID:server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd Address:b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd}]" +2025-06-03T23:36:11.327Z [INFO] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: entering follower state: follower="Node at 0124c84b-4ee6-ba0b-d409-c1e41835a863 [Follower]" leader-address= leader-id= +2025-06-03T23:36:11.327Z [INFO] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf845591-38d9-caa4-dd0a-ff2816669258 Address:cf845591-38d9-caa4-dd0a-ff2816669258} {Suffrage:Voter ID:server-0124c84b-4ee6-ba0b-d409-c1e41835a863 Address:0124c84b-4ee6-ba0b-d409-c1e41835a863} {Suffrage:Voter ID:server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd Address:b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd}]" +2025-06-03T23:36:11.327Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:11.327Z [INFO] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: entering follower state: follower="Node at b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd [Follower]" leader-address= leader-id= +2025-06-03T23:36:11.386Z [WARN] server-cf845591-38d9-caa4-dd0a-ff2816669258: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:11.386Z [INFO] server-cf845591-38d9-caa4-dd0a-ff2816669258: entering candidate state: node="Node at cf845591-38d9-caa4-dd0a-ff2816669258 [Candidate]" term=2 +2025-06-03T23:36:11.386Z [INFO] server-cf845591-38d9-caa4-dd0a-ff2816669258: election won: term=2 tally=2 +2025-06-03T23:36:11.386Z [INFO] server-cf845591-38d9-caa4-dd0a-ff2816669258: entering leader state: leader="Node at cf845591-38d9-caa4-dd0a-ff2816669258 [Leader]" +2025-06-03T23:36:11.386Z [INFO] server-cf845591-38d9-caa4-dd0a-ff2816669258: added peer, starting replication: peer=server-0124c84b-4ee6-ba0b-d409-c1e41835a863 +2025-06-03T23:36:11.386Z [INFO] server-cf845591-38d9-caa4-dd0a-ff2816669258: added peer, starting replication: peer=server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd +2025-06-03T23:36:11.386Z [INFO] server-cf845591-38d9-caa4-dd0a-ff2816669258: pipelining replication: peer="{Voter server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd}" +2025-06-03T23:36:11.386Z [INFO] server-cf845591-38d9-caa4-dd0a-ff2816669258: pipelining replication: peer="{Voter server-0124c84b-4ee6-ba0b-d409-c1e41835a863 0124c84b-4ee6-ba0b-d409-c1e41835a863}" +2025-06-03T23:36:11.492Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:11.386856711 +0000 UTC m=+6.708611641 (2 nodes), 59.3768ms from start of poll, 59.94732ms from cluster start. Timeout at 2025-06-03 23:36:11.492060351 +0000 UTC m=+6.813815281, 105.20364ms after stability +2025-06-03T23:36:11.492Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:11.598Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:11.492507871 +0000 UTC m=+6.814262801 (1 nodes), 1.04µs from start of poll, 165.59848ms from cluster start. Timeout at 2025-06-03 23:36:11.598143351 +0000 UTC m=+6.919898281, 105.63548ms after stability + raft_test.go:378: [INFO] Disconnecting Node at cf845591-38d9-caa4-dd0a-ff2816669258 [Leader] +2025-06-03T23:36:11.611Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to heartbeat to: peer=0124c84b-4ee6-ba0b-d409-c1e41835a863 backoff time=10ms error="failed to connect to peer: 0124c84b-4ee6-ba0b-d409-c1e41835a863" +2025-06-03T23:36:11.611Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:11.612Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to pipeline appendEntries: peer="{Voter server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd}" error="append pipeline closed" +2025-06-03T23:36:11.612Z [INFO] server-cf845591-38d9-caa4-dd0a-ff2816669258: aborting pipeline replication: peer="{Voter server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd}" +2025-06-03T23:36:11.613Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to heartbeat to: peer=b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd backoff time=10ms error="failed to connect to peer: b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd" +2025-06-03T23:36:11.617Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to appendEntries to: peer="{Voter server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd}" error="failed to connect to peer: b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd" +2025-06-03T23:36:11.617Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to pipeline appendEntries: peer="{Voter server-0124c84b-4ee6-ba0b-d409-c1e41835a863 0124c84b-4ee6-ba0b-d409-c1e41835a863}" error="append pipeline closed" +2025-06-03T23:36:11.617Z [INFO] server-cf845591-38d9-caa4-dd0a-ff2816669258: aborting pipeline replication: peer="{Voter server-0124c84b-4ee6-ba0b-d409-c1e41835a863 0124c84b-4ee6-ba0b-d409-c1e41835a863}" +2025-06-03T23:36:11.623Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to appendEntries to: peer="{Voter server-0124c84b-4ee6-ba0b-d409-c1e41835a863 0124c84b-4ee6-ba0b-d409-c1e41835a863}" error="failed to connect to peer: 0124c84b-4ee6-ba0b-d409-c1e41835a863" +2025-06-03T23:36:11.629Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to heartbeat to: peer=0124c84b-4ee6-ba0b-d409-c1e41835a863 backoff time=10ms error="failed to connect to peer: 0124c84b-4ee6-ba0b-d409-c1e41835a863" +2025-06-03T23:36:11.633Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to heartbeat to: peer=b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd backoff time=10ms error="failed to connect to peer: b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd" +2025-06-03T23:36:11.637Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to appendEntries to: peer="{Voter server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd}" error="failed to connect to peer: b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd" +2025-06-03T23:36:11.644Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to appendEntries to: peer="{Voter server-0124c84b-4ee6-ba0b-d409-c1e41835a863 0124c84b-4ee6-ba0b-d409-c1e41835a863}" error="failed to connect to peer: 0124c84b-4ee6-ba0b-d409-c1e41835a863" +2025-06-03T23:36:11.650Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to heartbeat to: peer=0124c84b-4ee6-ba0b-d409-c1e41835a863 backoff time=10ms error="failed to connect to peer: 0124c84b-4ee6-ba0b-d409-c1e41835a863" +2025-06-03T23:36:11.655Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to appendEntries to: peer="{Voter server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd}" error="failed to connect to peer: b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd" +2025-06-03T23:36:11.655Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to heartbeat to: peer=b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd backoff time=10ms error="failed to connect to peer: b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd" +2025-06-03T23:36:11.656Z [WARN] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to contact: server-id=server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd time=50.71064ms +2025-06-03T23:36:11.665Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to appendEntries to: peer="{Voter server-0124c84b-4ee6-ba0b-d409-c1e41835a863 0124c84b-4ee6-ba0b-d409-c1e41835a863}" error="failed to connect to peer: 0124c84b-4ee6-ba0b-d409-c1e41835a863" +2025-06-03T23:36:11.667Z [WARN] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to contact: server-id=server-0124c84b-4ee6-ba0b-d409-c1e41835a863 time=59.25564ms +2025-06-03T23:36:11.667Z [WARN] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to contact: server-id=server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd time=61.7884ms +2025-06-03T23:36:11.667Z [WARN] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to contact quorum of nodes, stepping down +2025-06-03T23:36:11.667Z [INFO] server-cf845591-38d9-caa4-dd0a-ff2816669258: entering follower state: follower="Node at cf845591-38d9-caa4-dd0a-ff2816669258 [Follower]" leader-address= leader-id= +2025-06-03T23:36:11.673Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to heartbeat to: peer=b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd backoff time=20ms error="failed to connect to peer: b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd" +2025-06-03T23:36:11.686Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to appendEntries to: peer="{Voter server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd}" error="failed to connect to peer: b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd" +2025-06-03T23:36:11.723Z [WARN] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: heartbeat timeout reached, starting election: last-leader-addr=cf845591-38d9-caa4-dd0a-ff2816669258 last-leader-id=server-cf845591-38d9-caa4-dd0a-ff2816669258 +2025-06-03T23:36:11.723Z [INFO] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: entering candidate state: node="Node at b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd [Candidate]" term=3 +2025-06-03T23:36:11.723Z [WARN] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: rejecting vote request since we have a leader: from=b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd leader=cf845591-38d9-caa4-dd0a-ff2816669258 leader-id=server-cf845591-38d9-caa4-dd0a-ff2816669258 +2025-06-03T23:36:11.723Z [ERROR] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: failed to make requestVote RPC: target="{Voter server-cf845591-38d9-caa4-dd0a-ff2816669258 cf845591-38d9-caa4-dd0a-ff2816669258}" error="failed to connect to peer: cf845591-38d9-caa4-dd0a-ff2816669258" term=3 +2025-06-03T23:36:11.735Z [WARN] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: heartbeat timeout reached, starting election: last-leader-addr=cf845591-38d9-caa4-dd0a-ff2816669258 last-leader-id=server-cf845591-38d9-caa4-dd0a-ff2816669258 +2025-06-03T23:36:11.735Z [INFO] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: entering candidate state: node="Node at 0124c84b-4ee6-ba0b-d409-c1e41835a863 [Candidate]" term=3 +2025-06-03T23:36:11.735Z [INFO] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: duplicate requestVote for same term: term=3 +2025-06-03T23:36:11.735Z [ERROR] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: failed to make requestVote RPC: target="{Voter server-cf845591-38d9-caa4-dd0a-ff2816669258 cf845591-38d9-caa4-dd0a-ff2816669258}" error="failed to connect to peer: cf845591-38d9-caa4-dd0a-ff2816669258" term=3 +2025-06-03T23:36:11.759Z [WARN] server-cf845591-38d9-caa4-dd0a-ff2816669258: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:11.759Z [INFO] server-cf845591-38d9-caa4-dd0a-ff2816669258: entering candidate state: node="Node at cf845591-38d9-caa4-dd0a-ff2816669258 [Candidate]" term=3 +2025-06-03T23:36:11.759Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to make requestVote RPC: target="{Voter server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd}" error="failed to connect to peer: b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd" term=3 +2025-06-03T23:36:11.759Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to make requestVote RPC: target="{Voter server-0124c84b-4ee6-ba0b-d409-c1e41835a863 0124c84b-4ee6-ba0b-d409-c1e41835a863}" error="failed to connect to peer: 0124c84b-4ee6-ba0b-d409-c1e41835a863" term=3 +2025-06-03T23:36:11.799Z [WARN] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: Election timeout reached, restarting election +2025-06-03T23:36:11.799Z [INFO] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: entering candidate state: node="Node at b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd [Candidate]" term=4 +2025-06-03T23:36:11.799Z [INFO] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: entering follower state: follower="Node at 0124c84b-4ee6-ba0b-d409-c1e41835a863 [Follower]" leader-address= leader-id= +2025-06-03T23:36:11.799Z [ERROR] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: failed to make requestVote RPC: target="{Voter server-cf845591-38d9-caa4-dd0a-ff2816669258 cf845591-38d9-caa4-dd0a-ff2816669258}" error="failed to connect to peer: cf845591-38d9-caa4-dd0a-ff2816669258" term=4 +2025-06-03T23:36:11.800Z [INFO] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: election won: term=4 tally=2 +2025-06-03T23:36:11.800Z [INFO] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: entering leader state: leader="Node at b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd [Leader]" +2025-06-03T23:36:11.800Z [INFO] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: added peer, starting replication: peer=server-cf845591-38d9-caa4-dd0a-ff2816669258 +2025-06-03T23:36:11.800Z [INFO] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: added peer, starting replication: peer=server-0124c84b-4ee6-ba0b-d409-c1e41835a863 +2025-06-03T23:36:11.800Z [INFO] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: pipelining replication: peer="{Voter server-0124c84b-4ee6-ba0b-d409-c1e41835a863 0124c84b-4ee6-ba0b-d409-c1e41835a863}" +2025-06-03T23:36:11.800Z [ERROR] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: failed to appendEntries to: peer="{Voter server-cf845591-38d9-caa4-dd0a-ff2816669258 cf845591-38d9-caa4-dd0a-ff2816669258}" error="failed to connect to peer: cf845591-38d9-caa4-dd0a-ff2816669258" +2025-06-03T23:36:11.806Z [ERROR] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: failed to heartbeat to: peer=cf845591-38d9-caa4-dd0a-ff2816669258 backoff time=10ms error="failed to connect to peer: cf845591-38d9-caa4-dd0a-ff2816669258" +2025-06-03T23:36:11.820Z [ERROR] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: failed to appendEntries to: peer="{Voter server-cf845591-38d9-caa4-dd0a-ff2816669258 cf845591-38d9-caa4-dd0a-ff2816669258}" error="failed to connect to peer: cf845591-38d9-caa4-dd0a-ff2816669258" +2025-06-03T23:36:11.822Z [WARN] server-cf845591-38d9-caa4-dd0a-ff2816669258: Election timeout reached, restarting election +2025-06-03T23:36:11.822Z [INFO] server-cf845591-38d9-caa4-dd0a-ff2816669258: entering candidate state: node="Node at cf845591-38d9-caa4-dd0a-ff2816669258 [Candidate]" term=4 +2025-06-03T23:36:11.822Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to make requestVote RPC: target="{Voter server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd}" error="failed to connect to peer: b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd" term=4 +2025-06-03T23:36:11.822Z [ERROR] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to make requestVote RPC: target="{Voter server-0124c84b-4ee6-ba0b-d409-c1e41835a863 0124c84b-4ee6-ba0b-d409-c1e41835a863}" error="failed to connect to peer: 0124c84b-4ee6-ba0b-d409-c1e41835a863" term=4 +2025-06-03T23:36:11.826Z [ERROR] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: failed to heartbeat to: peer=cf845591-38d9-caa4-dd0a-ff2816669258 backoff time=10ms error="failed to connect to peer: cf845591-38d9-caa4-dd0a-ff2816669258" +2025-06-03T23:36:11.838Z [ERROR] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: failed to appendEntries to: peer="{Voter server-cf845591-38d9-caa4-dd0a-ff2816669258 cf845591-38d9-caa4-dd0a-ff2816669258}" error="failed to connect to peer: cf845591-38d9-caa4-dd0a-ff2816669258" +2025-06-03T23:36:11.847Z [ERROR] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: failed to heartbeat to: peer=cf845591-38d9-caa4-dd0a-ff2816669258 backoff time=10ms error="failed to connect to peer: cf845591-38d9-caa4-dd0a-ff2816669258" +2025-06-03T23:36:11.850Z [WARN] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: failed to contact: server-id=server-cf845591-38d9-caa4-dd0a-ff2816669258 time=50.63976ms +2025-06-03T23:36:11.867Z [ERROR] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: failed to heartbeat to: peer=cf845591-38d9-caa4-dd0a-ff2816669258 backoff time=20ms error="failed to connect to peer: cf845591-38d9-caa4-dd0a-ff2816669258" +2025-06-03T23:36:11.869Z [ERROR] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: failed to appendEntries to: peer="{Voter server-cf845591-38d9-caa4-dd0a-ff2816669258 cf845591-38d9-caa4-dd0a-ff2816669258}" error="failed to connect to peer: cf845591-38d9-caa4-dd0a-ff2816669258" +2025-06-03T23:36:11.897Z [ERROR] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: failed to heartbeat to: peer=cf845591-38d9-caa4-dd0a-ff2816669258 backoff time=25ms error="failed to connect to peer: cf845591-38d9-caa4-dd0a-ff2816669258" +2025-06-03T23:36:11.900Z [WARN] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: failed to contact: server-id=server-cf845591-38d9-caa4-dd0a-ff2816669258 time=100.14452ms +2025-06-03T23:36:11.905Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:11.800194751 +0000 UTC m=+7.121949681 (1 nodes), 188.7838ms from start of poll, 473.28536ms from cluster start. Timeout at 2025-06-03 23:36:11.905646351 +0000 UTC m=+7.227401281, 105.4516ms after stability + raft_test.go:413: [INFO] Reconnecting Node at cf845591-38d9-caa4-dd0a-ff2816669258 [Candidate] +2025-06-03T23:36:11.911Z [WARN] server-cf845591-38d9-caa4-dd0a-ff2816669258: Election timeout reached, restarting election +2025-06-03T23:36:11.911Z [INFO] server-cf845591-38d9-caa4-dd0a-ff2816669258: entering candidate state: node="Node at cf845591-38d9-caa4-dd0a-ff2816669258 [Candidate]" term=5 +2025-06-03T23:36:11.911Z [WARN] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: rejecting vote request since we have a leader: from=cf845591-38d9-caa4-dd0a-ff2816669258 leader=b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd leader-id=server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd +2025-06-03T23:36:11.911Z [WARN] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: rejecting vote request since we have a leader: from=cf845591-38d9-caa4-dd0a-ff2816669258 leader=b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd leader-id=server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd +2025-06-03T23:36:11.920Z [ERROR] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: peer has newer term, stopping replication: peer="{Voter server-cf845591-38d9-caa4-dd0a-ff2816669258 cf845591-38d9-caa4-dd0a-ff2816669258}" +2025-06-03T23:36:11.920Z [INFO] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: entering follower state: follower="Node at b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd [Follower]" leader-address= leader-id= +2025-06-03T23:36:11.920Z [INFO] server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd: aborting pipeline replication: peer="{Voter server-0124c84b-4ee6-ba0b-d409-c1e41835a863 0124c84b-4ee6-ba0b-d409-c1e41835a863}" +2025-06-03T23:36:11.974Z [WARN] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: heartbeat timeout reached, starting election: last-leader-addr=b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd last-leader-id=server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd +2025-06-03T23:36:11.974Z [INFO] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: entering candidate state: node="Node at 0124c84b-4ee6-ba0b-d409-c1e41835a863 [Candidate]" term=5 +2025-06-03T23:36:11.974Z [INFO] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: election won: term=5 tally=2 +2025-06-03T23:36:11.974Z [INFO] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: entering leader state: leader="Node at 0124c84b-4ee6-ba0b-d409-c1e41835a863 [Leader]" +2025-06-03T23:36:11.974Z [INFO] server-cf845591-38d9-caa4-dd0a-ff2816669258: duplicate requestVote for same term: term=5 +2025-06-03T23:36:11.974Z [INFO] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: added peer, starting replication: peer=server-cf845591-38d9-caa4-dd0a-ff2816669258 +2025-06-03T23:36:11.974Z [INFO] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: added peer, starting replication: peer=server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd +2025-06-03T23:36:11.974Z [WARN] server-cf845591-38d9-caa4-dd0a-ff2816669258: failed to get previous log: previous-index=5 last-index=3 error="log not found" +2025-06-03T23:36:11.974Z [INFO] server-cf845591-38d9-caa4-dd0a-ff2816669258: entering follower state: follower="Node at cf845591-38d9-caa4-dd0a-ff2816669258 [Follower]" leader-address=0124c84b-4ee6-ba0b-d409-c1e41835a863 leader-id=server-0124c84b-4ee6-ba0b-d409-c1e41835a863 +2025-06-03T23:36:11.974Z [INFO] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: pipelining replication: peer="{Voter server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd}" +2025-06-03T23:36:11.974Z [WARN] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: appendEntries rejected, sending older logs: peer="{Voter server-cf845591-38d9-caa4-dd0a-ff2816669258 cf845591-38d9-caa4-dd0a-ff2816669258}" next=4 +2025-06-03T23:36:11.974Z [INFO] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: pipelining replication: peer="{Voter server-cf845591-38d9-caa4-dd0a-ff2816669258 cf845591-38d9-caa4-dd0a-ff2816669258}" +2025-06-03T23:36:11.979Z [INFO] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: aborting pipeline replication: peer="{Voter server-b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd b7fb0ee0-f902-27be-bcf0-1a2d2bf0d6bd}" +2025-06-03T23:36:11.980Z [INFO] server-0124c84b-4ee6-ba0b-d409-c1e41835a863: aborting pipeline replication: peer="{Voter server-cf845591-38d9-caa4-dd0a-ff2816669258 cf845591-38d9-caa4-dd0a-ff2816669258}" +--- PASS: TestRaft_LeaderFail (0.65s) === RUN TestRaft_BehindFollower -2025-06-03T23:33:12.662Z [INFO] server-0da54206-c2be-e5d5-a1aa-c49d11852bfc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0da54206-c2be-e5d5-a1aa-c49d11852bfc Address:0da54206-c2be-e5d5-a1aa-c49d11852bfc} {Suffrage:Voter ID:server-10f94b31-16ab-c0f1-8b64-8080fa8b8e81 Address:10f94b31-16ab-c0f1-8b64-8080fa8b8e81} {Suffrage:Voter ID:server-ced8bcf0-4414-5594-d1be-dd142da35540 Address:ced8bcf0-4414-5594-d1be-dd142da35540}]" -2025-06-03T23:33:12.662Z [INFO] server-0da54206-c2be-e5d5-a1aa-c49d11852bfc: entering follower state: follower="Node at 0da54206-c2be-e5d5-a1aa-c49d11852bfc [Follower]" leader-address= leader-id= -2025-06-03T23:33:12.662Z [INFO] server-10f94b31-16ab-c0f1-8b64-8080fa8b8e81: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0da54206-c2be-e5d5-a1aa-c49d11852bfc Address:0da54206-c2be-e5d5-a1aa-c49d11852bfc} {Suffrage:Voter ID:server-10f94b31-16ab-c0f1-8b64-8080fa8b8e81 Address:10f94b31-16ab-c0f1-8b64-8080fa8b8e81} {Suffrage:Voter ID:server-ced8bcf0-4414-5594-d1be-dd142da35540 Address:ced8bcf0-4414-5594-d1be-dd142da35540}]" -2025-06-03T23:33:12.662Z [INFO] server-10f94b31-16ab-c0f1-8b64-8080fa8b8e81: entering follower state: follower="Node at 10f94b31-16ab-c0f1-8b64-8080fa8b8e81 [Follower]" leader-address= leader-id= -2025-06-03T23:33:12.662Z [INFO] server-ced8bcf0-4414-5594-d1be-dd142da35540: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0da54206-c2be-e5d5-a1aa-c49d11852bfc Address:0da54206-c2be-e5d5-a1aa-c49d11852bfc} {Suffrage:Voter ID:server-10f94b31-16ab-c0f1-8b64-8080fa8b8e81 Address:10f94b31-16ab-c0f1-8b64-8080fa8b8e81} {Suffrage:Voter ID:server-ced8bcf0-4414-5594-d1be-dd142da35540 Address:ced8bcf0-4414-5594-d1be-dd142da35540}]" -2025-06-03T23:33:12.662Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:12.662Z [INFO] server-ced8bcf0-4414-5594-d1be-dd142da35540: entering follower state: follower="Node at ced8bcf0-4414-5594-d1be-dd142da35540 [Follower]" leader-address= leader-id= -2025-06-03T23:33:12.715Z [WARN] server-0da54206-c2be-e5d5-a1aa-c49d11852bfc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:12.715Z [INFO] server-0da54206-c2be-e5d5-a1aa-c49d11852bfc: entering candidate state: node="Node at 0da54206-c2be-e5d5-a1aa-c49d11852bfc [Candidate]" term=2 -2025-06-03T23:33:12.715Z [INFO] server-0da54206-c2be-e5d5-a1aa-c49d11852bfc: election won: term=2 tally=2 -2025-06-03T23:33:12.715Z [INFO] server-0da54206-c2be-e5d5-a1aa-c49d11852bfc: entering leader state: leader="Node at 0da54206-c2be-e5d5-a1aa-c49d11852bfc [Leader]" -2025-06-03T23:33:12.715Z [INFO] server-0da54206-c2be-e5d5-a1aa-c49d11852bfc: added peer, starting replication: peer=server-10f94b31-16ab-c0f1-8b64-8080fa8b8e81 -2025-06-03T23:33:12.715Z [INFO] server-0da54206-c2be-e5d5-a1aa-c49d11852bfc: added peer, starting replication: peer=server-ced8bcf0-4414-5594-d1be-dd142da35540 -2025-06-03T23:33:12.715Z [INFO] server-0da54206-c2be-e5d5-a1aa-c49d11852bfc: pipelining replication: peer="{Voter server-10f94b31-16ab-c0f1-8b64-8080fa8b8e81 10f94b31-16ab-c0f1-8b64-8080fa8b8e81}" -2025-06-03T23:33:12.715Z [INFO] server-0da54206-c2be-e5d5-a1aa-c49d11852bfc: pipelining replication: peer="{Voter server-ced8bcf0-4414-5594-d1be-dd142da35540 ced8bcf0-4414-5594-d1be-dd142da35540}" -2025-06-03T23:33:12.820Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:12.715301711 +0000 UTC m=+7.736121081 (1 nodes), 52.83756ms from start of poll, 53.32972ms from cluster start. Timeout at 2025-06-03 23:33:12.820769551 +0000 UTC m=+7.841588921, 105.46784ms after stability -2025-06-03T23:33:12.820Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:12.926Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:12.820868951 +0000 UTC m=+7.841688321 (2 nodes), 3.32µs from start of poll, 158.89696ms from cluster start. Timeout at 2025-06-03 23:33:12.926742431 +0000 UTC m=+7.947561801, 105.87348ms after stability -2025-06-03T23:33:12.926Z [ERROR] server-0da54206-c2be-e5d5-a1aa-c49d11852bfc: failed to pipeline appendEntries: peer="{Voter server-10f94b31-16ab-c0f1-8b64-8080fa8b8e81 10f94b31-16ab-c0f1-8b64-8080fa8b8e81}" error="append pipeline closed" -2025-06-03T23:33:12.926Z [INFO] server-0da54206-c2be-e5d5-a1aa-c49d11852bfc: aborting pipeline replication: peer="{Voter server-10f94b31-16ab-c0f1-8b64-8080fa8b8e81 10f94b31-16ab-c0f1-8b64-8080fa8b8e81}" -2025-06-03T23:33:12.926Z [ERROR] server-0da54206-c2be-e5d5-a1aa-c49d11852bfc: failed to appendEntries to: peer="{Voter server-10f94b31-16ab-c0f1-8b64-8080fa8b8e81 10f94b31-16ab-c0f1-8b64-8080fa8b8e81}" error="failed to connect to peer: 10f94b31-16ab-c0f1-8b64-8080fa8b8e81" +2025-06-03T23:36:11.980Z [INFO] server-334d8b33-dea3-bc35-d471-450121b8eccb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-334d8b33-dea3-bc35-d471-450121b8eccb Address:334d8b33-dea3-bc35-d471-450121b8eccb} {Suffrage:Voter ID:server-01ac1206-4c97-9bc4-e339-cce5e6d50f0c Address:01ac1206-4c97-9bc4-e339-cce5e6d50f0c} {Suffrage:Voter ID:server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121 Address:0bc821f7-8ae0-efe2-5aca-a75e53bd2121}]" +2025-06-03T23:36:11.980Z [INFO] server-334d8b33-dea3-bc35-d471-450121b8eccb: entering follower state: follower="Node at 334d8b33-dea3-bc35-d471-450121b8eccb [Follower]" leader-address= leader-id= +2025-06-03T23:36:11.981Z [INFO] server-01ac1206-4c97-9bc4-e339-cce5e6d50f0c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-334d8b33-dea3-bc35-d471-450121b8eccb Address:334d8b33-dea3-bc35-d471-450121b8eccb} {Suffrage:Voter ID:server-01ac1206-4c97-9bc4-e339-cce5e6d50f0c Address:01ac1206-4c97-9bc4-e339-cce5e6d50f0c} {Suffrage:Voter ID:server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121 Address:0bc821f7-8ae0-efe2-5aca-a75e53bd2121}]" +2025-06-03T23:36:11.981Z [INFO] server-01ac1206-4c97-9bc4-e339-cce5e6d50f0c: entering follower state: follower="Node at 01ac1206-4c97-9bc4-e339-cce5e6d50f0c [Follower]" leader-address= leader-id= +2025-06-03T23:36:11.981Z [INFO] server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-334d8b33-dea3-bc35-d471-450121b8eccb Address:334d8b33-dea3-bc35-d471-450121b8eccb} {Suffrage:Voter ID:server-01ac1206-4c97-9bc4-e339-cce5e6d50f0c Address:01ac1206-4c97-9bc4-e339-cce5e6d50f0c} {Suffrage:Voter ID:server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121 Address:0bc821f7-8ae0-efe2-5aca-a75e53bd2121}]" +2025-06-03T23:36:11.981Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:11.981Z [INFO] server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121: entering follower state: follower="Node at 0bc821f7-8ae0-efe2-5aca-a75e53bd2121 [Follower]" leader-address= leader-id= +2025-06-03T23:36:12.033Z [WARN] server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:12.033Z [INFO] server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121: entering candidate state: node="Node at 0bc821f7-8ae0-efe2-5aca-a75e53bd2121 [Candidate]" term=2 +2025-06-03T23:36:12.033Z [INFO] server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121: election won: term=2 tally=2 +2025-06-03T23:36:12.033Z [INFO] server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121: entering leader state: leader="Node at 0bc821f7-8ae0-efe2-5aca-a75e53bd2121 [Leader]" +2025-06-03T23:36:12.033Z [INFO] server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121: added peer, starting replication: peer=server-334d8b33-dea3-bc35-d471-450121b8eccb +2025-06-03T23:36:12.033Z [INFO] server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121: added peer, starting replication: peer=server-01ac1206-4c97-9bc4-e339-cce5e6d50f0c +2025-06-03T23:36:12.033Z [INFO] server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121: pipelining replication: peer="{Voter server-01ac1206-4c97-9bc4-e339-cce5e6d50f0c 01ac1206-4c97-9bc4-e339-cce5e6d50f0c}" +2025-06-03T23:36:12.033Z [INFO] server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121: pipelining replication: peer="{Voter server-334d8b33-dea3-bc35-d471-450121b8eccb 334d8b33-dea3-bc35-d471-450121b8eccb}" +2025-06-03T23:36:12.138Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:12.033647471 +0000 UTC m=+7.355402361 (1 nodes), 52.48204ms from start of poll, 52.9304ms from cluster start. Timeout at 2025-06-03 23:36:12.138783031 +0000 UTC m=+7.460538001, 105.13564ms after stability +2025-06-03T23:36:12.138Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:12.244Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:12.138904751 +0000 UTC m=+7.460659681 (2 nodes), 1.64µs from start of poll, 158.18772ms from cluster start. Timeout at 2025-06-03 23:36:12.244269751 +0000 UTC m=+7.566024681, 105.365ms after stability +2025-06-03T23:36:12.244Z [ERROR] server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121: failed to pipeline appendEntries: peer="{Voter server-334d8b33-dea3-bc35-d471-450121b8eccb 334d8b33-dea3-bc35-d471-450121b8eccb}" error="append pipeline closed" +2025-06-03T23:36:12.244Z [INFO] server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121: aborting pipeline replication: peer="{Voter server-334d8b33-dea3-bc35-d471-450121b8eccb 334d8b33-dea3-bc35-d471-450121b8eccb}" +2025-06-03T23:36:12.244Z [ERROR] server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121: failed to appendEntries to: peer="{Voter server-334d8b33-dea3-bc35-d471-450121b8eccb 334d8b33-dea3-bc35-d471-450121b8eccb}" error="failed to connect to peer: 334d8b33-dea3-bc35-d471-450121b8eccb" +2025-06-03T23:36:12.249Z [ERROR] server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121: failed to heartbeat to: peer=334d8b33-dea3-bc35-d471-450121b8eccb backoff time=10ms error="failed to connect to peer: 334d8b33-dea3-bc35-d471-450121b8eccb" raft_test.go:459: [INFO] Finished apply without behind follower -2025-06-03T23:33:12.931Z [ERROR] server-0da54206-c2be-e5d5-a1aa-c49d11852bfc: failed to heartbeat to: peer=10f94b31-16ab-c0f1-8b64-8080fa8b8e81 backoff time=10ms error="failed to connect to peer: 10f94b31-16ab-c0f1-8b64-8080fa8b8e81" -2025-06-03T23:33:12.937Z [INFO] server-0da54206-c2be-e5d5-a1aa-c49d11852bfc: pipelining replication: peer="{Voter server-10f94b31-16ab-c0f1-8b64-8080fa8b8e81 10f94b31-16ab-c0f1-8b64-8080fa8b8e81}" -2025-06-03T23:33:12.948Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:13.054Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:12.948842071 +0000 UTC m=+7.969661441 (1 nodes), 1.08µs from start of poll, 286.87008ms from cluster start. Timeout at 2025-06-03 23:33:13.053951311 +0000 UTC m=+8.074770681, 105.10924ms after stability -2025-06-03T23:33:13.054Z [INFO] server-0da54206-c2be-e5d5-a1aa-c49d11852bfc: aborting pipeline replication: peer="{Voter server-10f94b31-16ab-c0f1-8b64-8080fa8b8e81 10f94b31-16ab-c0f1-8b64-8080fa8b8e81}" -2025-06-03T23:33:13.054Z [INFO] server-0da54206-c2be-e5d5-a1aa-c49d11852bfc: aborting pipeline replication: peer="{Voter server-ced8bcf0-4414-5594-d1be-dd142da35540 ced8bcf0-4414-5594-d1be-dd142da35540}" +2025-06-03T23:36:12.255Z [INFO] server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121: pipelining replication: peer="{Voter server-334d8b33-dea3-bc35-d471-450121b8eccb 334d8b33-dea3-bc35-d471-450121b8eccb}" +2025-06-03T23:36:12.265Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:12.371Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:12.265979991 +0000 UTC m=+7.587734921 (1 nodes), 2.64µs from start of poll, 285.26296ms from cluster start. Timeout at 2025-06-03 23:36:12.371050671 +0000 UTC m=+7.692805601, 105.07068ms after stability +2025-06-03T23:36:12.371Z [INFO] server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121: aborting pipeline replication: peer="{Voter server-01ac1206-4c97-9bc4-e339-cce5e6d50f0c 01ac1206-4c97-9bc4-e339-cce5e6d50f0c}" +2025-06-03T23:36:12.371Z [INFO] server-0bc821f7-8ae0-efe2-5aca-a75e53bd2121: aborting pipeline replication: peer="{Voter server-334d8b33-dea3-bc35-d471-450121b8eccb 334d8b33-dea3-bc35-d471-450121b8eccb}" --- PASS: TestRaft_BehindFollower (0.39s) === RUN TestRaft_ApplyNonLeader -2025-06-03T23:33:13.055Z [INFO] server-d2b8f883-dbf7-e363-0e2f-f3968165acd8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d2b8f883-dbf7-e363-0e2f-f3968165acd8 Address:d2b8f883-dbf7-e363-0e2f-f3968165acd8} {Suffrage:Voter ID:server-d8fa7f66-212e-9d21-eacb-5947a8fbb2ad Address:d8fa7f66-212e-9d21-eacb-5947a8fbb2ad} {Suffrage:Voter ID:server-a269dca0-fe92-fadb-c736-7fbafeb9d1f0 Address:a269dca0-fe92-fadb-c736-7fbafeb9d1f0}]" -2025-06-03T23:33:13.055Z [INFO] server-d2b8f883-dbf7-e363-0e2f-f3968165acd8: entering follower state: follower="Node at d2b8f883-dbf7-e363-0e2f-f3968165acd8 [Follower]" leader-address= leader-id= -2025-06-03T23:33:13.055Z [INFO] server-d8fa7f66-212e-9d21-eacb-5947a8fbb2ad: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d2b8f883-dbf7-e363-0e2f-f3968165acd8 Address:d2b8f883-dbf7-e363-0e2f-f3968165acd8} {Suffrage:Voter ID:server-d8fa7f66-212e-9d21-eacb-5947a8fbb2ad Address:d8fa7f66-212e-9d21-eacb-5947a8fbb2ad} {Suffrage:Voter ID:server-a269dca0-fe92-fadb-c736-7fbafeb9d1f0 Address:a269dca0-fe92-fadb-c736-7fbafeb9d1f0}]" -2025-06-03T23:33:13.055Z [INFO] server-d8fa7f66-212e-9d21-eacb-5947a8fbb2ad: entering follower state: follower="Node at d8fa7f66-212e-9d21-eacb-5947a8fbb2ad [Follower]" leader-address= leader-id= -2025-06-03T23:33:13.055Z [INFO] server-a269dca0-fe92-fadb-c736-7fbafeb9d1f0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d2b8f883-dbf7-e363-0e2f-f3968165acd8 Address:d2b8f883-dbf7-e363-0e2f-f3968165acd8} {Suffrage:Voter ID:server-d8fa7f66-212e-9d21-eacb-5947a8fbb2ad Address:d8fa7f66-212e-9d21-eacb-5947a8fbb2ad} {Suffrage:Voter ID:server-a269dca0-fe92-fadb-c736-7fbafeb9d1f0 Address:a269dca0-fe92-fadb-c736-7fbafeb9d1f0}]" -2025-06-03T23:33:13.055Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:13.055Z [INFO] server-a269dca0-fe92-fadb-c736-7fbafeb9d1f0: entering follower state: follower="Node at a269dca0-fe92-fadb-c736-7fbafeb9d1f0 [Follower]" leader-address= leader-id= -2025-06-03T23:33:13.107Z [WARN] server-d8fa7f66-212e-9d21-eacb-5947a8fbb2ad: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:13.107Z [INFO] server-d8fa7f66-212e-9d21-eacb-5947a8fbb2ad: entering candidate state: node="Node at d8fa7f66-212e-9d21-eacb-5947a8fbb2ad [Candidate]" term=2 -2025-06-03T23:33:13.107Z [INFO] server-d8fa7f66-212e-9d21-eacb-5947a8fbb2ad: election won: term=2 tally=2 -2025-06-03T23:33:13.107Z [INFO] server-d8fa7f66-212e-9d21-eacb-5947a8fbb2ad: entering leader state: leader="Node at d8fa7f66-212e-9d21-eacb-5947a8fbb2ad [Leader]" -2025-06-03T23:33:13.107Z [INFO] server-d8fa7f66-212e-9d21-eacb-5947a8fbb2ad: added peer, starting replication: peer=server-d2b8f883-dbf7-e363-0e2f-f3968165acd8 -2025-06-03T23:33:13.107Z [INFO] server-d8fa7f66-212e-9d21-eacb-5947a8fbb2ad: added peer, starting replication: peer=server-a269dca0-fe92-fadb-c736-7fbafeb9d1f0 -2025-06-03T23:33:13.107Z [INFO] server-d8fa7f66-212e-9d21-eacb-5947a8fbb2ad: pipelining replication: peer="{Voter server-d2b8f883-dbf7-e363-0e2f-f3968165acd8 d2b8f883-dbf7-e363-0e2f-f3968165acd8}" -2025-06-03T23:33:13.107Z [INFO] server-d8fa7f66-212e-9d21-eacb-5947a8fbb2ad: pipelining replication: peer="{Voter server-a269dca0-fe92-fadb-c736-7fbafeb9d1f0 a269dca0-fe92-fadb-c736-7fbafeb9d1f0}" -2025-06-03T23:33:13.213Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:13.107480671 +0000 UTC m=+8.128300041 (1 nodes), 51.87804ms from start of poll, 52.49664ms from cluster start. Timeout at 2025-06-03 23:33:13.213531991 +0000 UTC m=+8.234351361, 106.05132ms after stability -2025-06-03T23:33:13.213Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:13.319Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:13.213626071 +0000 UTC m=+8.234445441 (2 nodes), 1.64µs from start of poll, 158.64204ms from cluster start. Timeout at 2025-06-03 23:33:13.319374191 +0000 UTC m=+8.340193561, 105.74812ms after stability -2025-06-03T23:33:13.319Z [INFO] server-d8fa7f66-212e-9d21-eacb-5947a8fbb2ad: aborting pipeline replication: peer="{Voter server-d2b8f883-dbf7-e363-0e2f-f3968165acd8 d2b8f883-dbf7-e363-0e2f-f3968165acd8}" -2025-06-03T23:33:13.319Z [INFO] server-d8fa7f66-212e-9d21-eacb-5947a8fbb2ad: aborting pipeline replication: peer="{Voter server-a269dca0-fe92-fadb-c736-7fbafeb9d1f0 a269dca0-fe92-fadb-c736-7fbafeb9d1f0}" +2025-06-03T23:36:12.372Z [INFO] server-a0805523-941f-0802-6468-2b114fce5dbf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0805523-941f-0802-6468-2b114fce5dbf Address:a0805523-941f-0802-6468-2b114fce5dbf} {Suffrage:Voter ID:server-7db16826-8960-f547-287b-97a28c0c1aa4 Address:7db16826-8960-f547-287b-97a28c0c1aa4} {Suffrage:Voter ID:server-5cf485d2-d760-b527-10f7-1cebd0094f9b Address:5cf485d2-d760-b527-10f7-1cebd0094f9b}]" +2025-06-03T23:36:12.372Z [INFO] server-a0805523-941f-0802-6468-2b114fce5dbf: entering follower state: follower="Node at a0805523-941f-0802-6468-2b114fce5dbf [Follower]" leader-address= leader-id= +2025-06-03T23:36:12.372Z [INFO] server-7db16826-8960-f547-287b-97a28c0c1aa4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0805523-941f-0802-6468-2b114fce5dbf Address:a0805523-941f-0802-6468-2b114fce5dbf} {Suffrage:Voter ID:server-7db16826-8960-f547-287b-97a28c0c1aa4 Address:7db16826-8960-f547-287b-97a28c0c1aa4} {Suffrage:Voter ID:server-5cf485d2-d760-b527-10f7-1cebd0094f9b Address:5cf485d2-d760-b527-10f7-1cebd0094f9b}]" +2025-06-03T23:36:12.372Z [INFO] server-7db16826-8960-f547-287b-97a28c0c1aa4: entering follower state: follower="Node at 7db16826-8960-f547-287b-97a28c0c1aa4 [Follower]" leader-address= leader-id= +2025-06-03T23:36:12.372Z [INFO] server-5cf485d2-d760-b527-10f7-1cebd0094f9b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0805523-941f-0802-6468-2b114fce5dbf Address:a0805523-941f-0802-6468-2b114fce5dbf} {Suffrage:Voter ID:server-7db16826-8960-f547-287b-97a28c0c1aa4 Address:7db16826-8960-f547-287b-97a28c0c1aa4} {Suffrage:Voter ID:server-5cf485d2-d760-b527-10f7-1cebd0094f9b Address:5cf485d2-d760-b527-10f7-1cebd0094f9b}]" +2025-06-03T23:36:12.372Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:12.372Z [INFO] server-5cf485d2-d760-b527-10f7-1cebd0094f9b: entering follower state: follower="Node at 5cf485d2-d760-b527-10f7-1cebd0094f9b [Follower]" leader-address= leader-id= +2025-06-03T23:36:12.433Z [WARN] server-7db16826-8960-f547-287b-97a28c0c1aa4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:12.433Z [INFO] server-7db16826-8960-f547-287b-97a28c0c1aa4: entering candidate state: node="Node at 7db16826-8960-f547-287b-97a28c0c1aa4 [Candidate]" term=2 +2025-06-03T23:36:12.433Z [INFO] server-7db16826-8960-f547-287b-97a28c0c1aa4: election won: term=2 tally=2 +2025-06-03T23:36:12.433Z [INFO] server-7db16826-8960-f547-287b-97a28c0c1aa4: entering leader state: leader="Node at 7db16826-8960-f547-287b-97a28c0c1aa4 [Leader]" +2025-06-03T23:36:12.433Z [INFO] server-7db16826-8960-f547-287b-97a28c0c1aa4: added peer, starting replication: peer=server-a0805523-941f-0802-6468-2b114fce5dbf +2025-06-03T23:36:12.434Z [INFO] server-7db16826-8960-f547-287b-97a28c0c1aa4: added peer, starting replication: peer=server-5cf485d2-d760-b527-10f7-1cebd0094f9b +2025-06-03T23:36:12.434Z [INFO] server-7db16826-8960-f547-287b-97a28c0c1aa4: pipelining replication: peer="{Voter server-a0805523-941f-0802-6468-2b114fce5dbf a0805523-941f-0802-6468-2b114fce5dbf}" +2025-06-03T23:36:12.434Z [INFO] server-7db16826-8960-f547-287b-97a28c0c1aa4: pipelining replication: peer="{Voter server-5cf485d2-d760-b527-10f7-1cebd0094f9b 5cf485d2-d760-b527-10f7-1cebd0094f9b}" +2025-06-03T23:36:12.539Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:12.434018311 +0000 UTC m=+7.755773241 (1 nodes), 61.44852ms from start of poll, 61.94324ms from cluster start. Timeout at 2025-06-03 23:36:12.539327591 +0000 UTC m=+7.861082561, 105.30932ms after stability +2025-06-03T23:36:12.539Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:12.645Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:12.539466471 +0000 UTC m=+7.861221401 (2 nodes), 3.88µs from start of poll, 167.3914ms from cluster start. Timeout at 2025-06-03 23:36:12.645095631 +0000 UTC m=+7.966850561, 105.62916ms after stability +2025-06-03T23:36:12.645Z [INFO] server-7db16826-8960-f547-287b-97a28c0c1aa4: aborting pipeline replication: peer="{Voter server-5cf485d2-d760-b527-10f7-1cebd0094f9b 5cf485d2-d760-b527-10f7-1cebd0094f9b}" +2025-06-03T23:36:12.645Z [INFO] server-7db16826-8960-f547-287b-97a28c0c1aa4: aborting pipeline replication: peer="{Voter server-a0805523-941f-0802-6468-2b114fce5dbf a0805523-941f-0802-6468-2b114fce5dbf}" --- PASS: TestRaft_ApplyNonLeader (0.27s) === RUN TestRaft_ApplyConcurrent -2025-06-03T23:33:13.320Z [INFO] server-9f6d4955-9db3-9c41-9872-5d3fdd4acf75: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9f6d4955-9db3-9c41-9872-5d3fdd4acf75 Address:9f6d4955-9db3-9c41-9872-5d3fdd4acf75} {Suffrage:Voter ID:server-b405b69a-5b1e-0ff9-9b10-97e5c191cf1d Address:b405b69a-5b1e-0ff9-9b10-97e5c191cf1d} {Suffrage:Voter ID:server-ecf8c6d8-56f8-af7b-c35e-3d3901b7e058 Address:ecf8c6d8-56f8-af7b-c35e-3d3901b7e058}]" -2025-06-03T23:33:13.320Z [INFO] server-9f6d4955-9db3-9c41-9872-5d3fdd4acf75: entering follower state: follower="Node at 9f6d4955-9db3-9c41-9872-5d3fdd4acf75 [Follower]" leader-address= leader-id= -2025-06-03T23:33:13.320Z [INFO] server-b405b69a-5b1e-0ff9-9b10-97e5c191cf1d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9f6d4955-9db3-9c41-9872-5d3fdd4acf75 Address:9f6d4955-9db3-9c41-9872-5d3fdd4acf75} {Suffrage:Voter ID:server-b405b69a-5b1e-0ff9-9b10-97e5c191cf1d Address:b405b69a-5b1e-0ff9-9b10-97e5c191cf1d} {Suffrage:Voter ID:server-ecf8c6d8-56f8-af7b-c35e-3d3901b7e058 Address:ecf8c6d8-56f8-af7b-c35e-3d3901b7e058}]" -2025-06-03T23:33:13.320Z [INFO] server-b405b69a-5b1e-0ff9-9b10-97e5c191cf1d: entering follower state: follower="Node at b405b69a-5b1e-0ff9-9b10-97e5c191cf1d [Follower]" leader-address= leader-id= -2025-06-03T23:33:13.320Z [INFO] server-ecf8c6d8-56f8-af7b-c35e-3d3901b7e058: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9f6d4955-9db3-9c41-9872-5d3fdd4acf75 Address:9f6d4955-9db3-9c41-9872-5d3fdd4acf75} {Suffrage:Voter ID:server-b405b69a-5b1e-0ff9-9b10-97e5c191cf1d Address:b405b69a-5b1e-0ff9-9b10-97e5c191cf1d} {Suffrage:Voter ID:server-ecf8c6d8-56f8-af7b-c35e-3d3901b7e058 Address:ecf8c6d8-56f8-af7b-c35e-3d3901b7e058}]" -2025-06-03T23:33:13.320Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:13.320Z [INFO] server-ecf8c6d8-56f8-af7b-c35e-3d3901b7e058: entering follower state: follower="Node at ecf8c6d8-56f8-af7b-c35e-3d3901b7e058 [Follower]" leader-address= leader-id= -2025-06-03T23:33:13.483Z [WARN] server-b405b69a-5b1e-0ff9-9b10-97e5c191cf1d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:13.483Z [INFO] server-b405b69a-5b1e-0ff9-9b10-97e5c191cf1d: entering candidate state: node="Node at b405b69a-5b1e-0ff9-9b10-97e5c191cf1d [Candidate]" term=2 -2025-06-03T23:33:13.483Z [INFO] server-b405b69a-5b1e-0ff9-9b10-97e5c191cf1d: election won: term=2 tally=2 -2025-06-03T23:33:13.483Z [INFO] server-b405b69a-5b1e-0ff9-9b10-97e5c191cf1d: entering leader state: leader="Node at b405b69a-5b1e-0ff9-9b10-97e5c191cf1d [Leader]" -2025-06-03T23:33:13.483Z [INFO] server-b405b69a-5b1e-0ff9-9b10-97e5c191cf1d: added peer, starting replication: peer=server-9f6d4955-9db3-9c41-9872-5d3fdd4acf75 -2025-06-03T23:33:13.483Z [INFO] server-b405b69a-5b1e-0ff9-9b10-97e5c191cf1d: added peer, starting replication: peer=server-ecf8c6d8-56f8-af7b-c35e-3d3901b7e058 -2025-06-03T23:33:13.484Z [INFO] server-b405b69a-5b1e-0ff9-9b10-97e5c191cf1d: pipelining replication: peer="{Voter server-ecf8c6d8-56f8-af7b-c35e-3d3901b7e058 ecf8c6d8-56f8-af7b-c35e-3d3901b7e058}" -2025-06-03T23:33:13.484Z [INFO] server-b405b69a-5b1e-0ff9-9b10-97e5c191cf1d: pipelining replication: peer="{Voter server-9f6d4955-9db3-9c41-9872-5d3fdd4acf75 9f6d4955-9db3-9c41-9872-5d3fdd4acf75}" -2025-06-03T23:33:13.689Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:13.483975871 +0000 UTC m=+8.504795241 (1 nodes), 163.03728ms from start of poll, 163.62924ms from cluster start. Timeout at 2025-06-03 23:33:13.689614191 +0000 UTC m=+8.710433561, 205.63832ms after stability -2025-06-03T23:33:13.704Z [INFO] server-b405b69a-5b1e-0ff9-9b10-97e5c191cf1d: aborting pipeline replication: peer="{Voter server-ecf8c6d8-56f8-af7b-c35e-3d3901b7e058 ecf8c6d8-56f8-af7b-c35e-3d3901b7e058}" -2025-06-03T23:33:13.704Z [INFO] server-b405b69a-5b1e-0ff9-9b10-97e5c191cf1d: aborting pipeline replication: peer="{Voter server-9f6d4955-9db3-9c41-9872-5d3fdd4acf75 9f6d4955-9db3-9c41-9872-5d3fdd4acf75}" ---- PASS: TestRaft_ApplyConcurrent (0.38s) +2025-06-03T23:36:12.646Z [INFO] server-77d73e66-626f-ec79-452b-627487d9a8db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-77d73e66-626f-ec79-452b-627487d9a8db Address:77d73e66-626f-ec79-452b-627487d9a8db} {Suffrage:Voter ID:server-38d8ed99-6da5-d97d-56fb-9961de881394 Address:38d8ed99-6da5-d97d-56fb-9961de881394} {Suffrage:Voter ID:server-5b3bebdd-745e-d222-21bd-bae9260151b0 Address:5b3bebdd-745e-d222-21bd-bae9260151b0}]" +2025-06-03T23:36:12.646Z [INFO] server-77d73e66-626f-ec79-452b-627487d9a8db: entering follower state: follower="Node at 77d73e66-626f-ec79-452b-627487d9a8db [Follower]" leader-address= leader-id= +2025-06-03T23:36:12.646Z [INFO] server-38d8ed99-6da5-d97d-56fb-9961de881394: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-77d73e66-626f-ec79-452b-627487d9a8db Address:77d73e66-626f-ec79-452b-627487d9a8db} {Suffrage:Voter ID:server-38d8ed99-6da5-d97d-56fb-9961de881394 Address:38d8ed99-6da5-d97d-56fb-9961de881394} {Suffrage:Voter ID:server-5b3bebdd-745e-d222-21bd-bae9260151b0 Address:5b3bebdd-745e-d222-21bd-bae9260151b0}]" +2025-06-03T23:36:12.646Z [INFO] server-38d8ed99-6da5-d97d-56fb-9961de881394: entering follower state: follower="Node at 38d8ed99-6da5-d97d-56fb-9961de881394 [Follower]" leader-address= leader-id= +2025-06-03T23:36:12.646Z [INFO] server-5b3bebdd-745e-d222-21bd-bae9260151b0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-77d73e66-626f-ec79-452b-627487d9a8db Address:77d73e66-626f-ec79-452b-627487d9a8db} {Suffrage:Voter ID:server-38d8ed99-6da5-d97d-56fb-9961de881394 Address:38d8ed99-6da5-d97d-56fb-9961de881394} {Suffrage:Voter ID:server-5b3bebdd-745e-d222-21bd-bae9260151b0 Address:5b3bebdd-745e-d222-21bd-bae9260151b0}]" +2025-06-03T23:36:12.646Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:12.646Z [INFO] server-5b3bebdd-745e-d222-21bd-bae9260151b0: entering follower state: follower="Node at 5b3bebdd-745e-d222-21bd-bae9260151b0 [Follower]" leader-address= leader-id= +2025-06-03T23:36:12.752Z [WARN] server-5b3bebdd-745e-d222-21bd-bae9260151b0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:12.752Z [INFO] server-5b3bebdd-745e-d222-21bd-bae9260151b0: entering candidate state: node="Node at 5b3bebdd-745e-d222-21bd-bae9260151b0 [Candidate]" term=2 +2025-06-03T23:36:12.753Z [INFO] server-5b3bebdd-745e-d222-21bd-bae9260151b0: election won: term=2 tally=2 +2025-06-03T23:36:12.753Z [INFO] server-5b3bebdd-745e-d222-21bd-bae9260151b0: entering leader state: leader="Node at 5b3bebdd-745e-d222-21bd-bae9260151b0 [Leader]" +2025-06-03T23:36:12.753Z [INFO] server-5b3bebdd-745e-d222-21bd-bae9260151b0: added peer, starting replication: peer=server-77d73e66-626f-ec79-452b-627487d9a8db +2025-06-03T23:36:12.753Z [INFO] server-5b3bebdd-745e-d222-21bd-bae9260151b0: added peer, starting replication: peer=server-38d8ed99-6da5-d97d-56fb-9961de881394 +2025-06-03T23:36:12.753Z [INFO] server-5b3bebdd-745e-d222-21bd-bae9260151b0: pipelining replication: peer="{Voter server-38d8ed99-6da5-d97d-56fb-9961de881394 38d8ed99-6da5-d97d-56fb-9961de881394}" +2025-06-03T23:36:12.753Z [INFO] server-5b3bebdd-745e-d222-21bd-bae9260151b0: pipelining replication: peer="{Voter server-77d73e66-626f-ec79-452b-627487d9a8db 77d73e66-626f-ec79-452b-627487d9a8db}" +2025-06-03T23:36:12.958Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:12.753241871 +0000 UTC m=+8.074996841 (1 nodes), 106.55772ms from start of poll, 107.20672ms from cluster start. Timeout at 2025-06-03 23:36:12.958637071 +0000 UTC m=+8.280392001, 205.39516ms after stability +2025-06-03T23:36:12.973Z [INFO] server-5b3bebdd-745e-d222-21bd-bae9260151b0: aborting pipeline replication: peer="{Voter server-38d8ed99-6da5-d97d-56fb-9961de881394 38d8ed99-6da5-d97d-56fb-9961de881394}" +2025-06-03T23:36:12.973Z [INFO] server-5b3bebdd-745e-d222-21bd-bae9260151b0: aborting pipeline replication: peer="{Voter server-77d73e66-626f-ec79-452b-627487d9a8db 77d73e66-626f-ec79-452b-627487d9a8db}" +--- PASS: TestRaft_ApplyConcurrent (0.33s) === RUN TestRaft_ApplyConcurrent_Timeout -2025-06-03T23:33:13.705Z [INFO] server-69a31cad-7044-bb67-a29c-2edda71296c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-69a31cad-7044-bb67-a29c-2edda71296c3 Address:69a31cad-7044-bb67-a29c-2edda71296c3}]" -2025-06-03T23:33:13.705Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:13.705Z [INFO] server-69a31cad-7044-bb67-a29c-2edda71296c3: entering follower state: follower="Node at 69a31cad-7044-bb67-a29c-2edda71296c3 [Follower]" leader-address= leader-id= -2025-06-03T23:33:13.834Z [WARN] server-69a31cad-7044-bb67-a29c-2edda71296c3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:13.834Z [INFO] server-69a31cad-7044-bb67-a29c-2edda71296c3: entering candidate state: node="Node at 69a31cad-7044-bb67-a29c-2edda71296c3 [Candidate]" term=2 -2025-06-03T23:33:13.834Z [INFO] server-69a31cad-7044-bb67-a29c-2edda71296c3: election won: term=2 tally=1 -2025-06-03T23:33:13.834Z [INFO] server-69a31cad-7044-bb67-a29c-2edda71296c3: entering leader state: leader="Node at 69a31cad-7044-bb67-a29c-2edda71296c3 [Leader]" -2025-06-03T23:33:14.036Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:13.834616031 +0000 UTC m=+8.855435401 (1 nodes), 129.20644ms from start of poll, 129.41724ms from cluster start. Timeout at 2025-06-03 23:33:14.036110231 +0000 UTC m=+9.056929601, 201.4942ms after stability ---- PASS: TestRaft_ApplyConcurrent_Timeout (0.33s) +2025-06-03T23:36:12.974Z [INFO] server-f01f57a6-0224-0be7-a2cf-a3cb91037e9e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f01f57a6-0224-0be7-a2cf-a3cb91037e9e Address:f01f57a6-0224-0be7-a2cf-a3cb91037e9e}]" +2025-06-03T23:36:12.974Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:12.974Z [INFO] server-f01f57a6-0224-0be7-a2cf-a3cb91037e9e: entering follower state: follower="Node at f01f57a6-0224-0be7-a2cf-a3cb91037e9e [Follower]" leader-address= leader-id= +2025-06-03T23:36:13.115Z [WARN] server-f01f57a6-0224-0be7-a2cf-a3cb91037e9e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:13.115Z [INFO] server-f01f57a6-0224-0be7-a2cf-a3cb91037e9e: entering candidate state: node="Node at f01f57a6-0224-0be7-a2cf-a3cb91037e9e [Candidate]" term=2 +2025-06-03T23:36:13.115Z [INFO] server-f01f57a6-0224-0be7-a2cf-a3cb91037e9e: election won: term=2 tally=1 +2025-06-03T23:36:13.115Z [INFO] server-f01f57a6-0224-0be7-a2cf-a3cb91037e9e: entering leader state: leader="Node at f01f57a6-0224-0be7-a2cf-a3cb91037e9e [Leader]" +2025-06-03T23:36:13.317Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:13.115846911 +0000 UTC m=+8.437601841 (1 nodes), 141.548ms from start of poll, 141.75188ms from cluster start. Timeout at 2025-06-03 23:36:13.317654031 +0000 UTC m=+8.639409001, 201.80716ms after stability +--- PASS: TestRaft_ApplyConcurrent_Timeout (0.34s) === RUN TestRaft_JoinNode -2025-06-03T23:33:14.037Z [INFO] server-d5522d3e-288f-1503-56c0-3f42c8b3e8f2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d5522d3e-288f-1503-56c0-3f42c8b3e8f2 Address:d5522d3e-288f-1503-56c0-3f42c8b3e8f2} {Suffrage:Voter ID:server-2db41c68-b795-80b5-585d-b33233c30f8b Address:2db41c68-b795-80b5-585d-b33233c30f8b}]" -2025-06-03T23:33:14.037Z [INFO] server-2db41c68-b795-80b5-585d-b33233c30f8b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d5522d3e-288f-1503-56c0-3f42c8b3e8f2 Address:d5522d3e-288f-1503-56c0-3f42c8b3e8f2} {Suffrage:Voter ID:server-2db41c68-b795-80b5-585d-b33233c30f8b Address:2db41c68-b795-80b5-585d-b33233c30f8b}]" -2025-06-03T23:33:14.037Z [INFO] server-3c42ba81-21d3-c362-9acd-41f92eae91f9: initial configuration: index=0 servers=[] -2025-06-03T23:33:14.037Z [INFO] server-d5522d3e-288f-1503-56c0-3f42c8b3e8f2: entering follower state: follower="Node at d5522d3e-288f-1503-56c0-3f42c8b3e8f2 [Follower]" leader-address= leader-id= -2025-06-03T23:33:14.037Z [INFO] server-2db41c68-b795-80b5-585d-b33233c30f8b: entering follower state: follower="Node at 2db41c68-b795-80b5-585d-b33233c30f8b [Follower]" leader-address= leader-id= -2025-06-03T23:33:14.037Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:14.037Z [INFO] server-3c42ba81-21d3-c362-9acd-41f92eae91f9: entering follower state: follower="Node at 3c42ba81-21d3-c362-9acd-41f92eae91f9 [Follower]" leader-address= leader-id= -2025-06-03T23:33:14.101Z [WARN] server-d5522d3e-288f-1503-56c0-3f42c8b3e8f2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:14.101Z [INFO] server-d5522d3e-288f-1503-56c0-3f42c8b3e8f2: entering candidate state: node="Node at d5522d3e-288f-1503-56c0-3f42c8b3e8f2 [Candidate]" term=2 -2025-06-03T23:33:14.101Z [INFO] server-d5522d3e-288f-1503-56c0-3f42c8b3e8f2: election won: term=2 tally=2 -2025-06-03T23:33:14.101Z [INFO] server-d5522d3e-288f-1503-56c0-3f42c8b3e8f2: entering leader state: leader="Node at d5522d3e-288f-1503-56c0-3f42c8b3e8f2 [Leader]" -2025-06-03T23:33:14.101Z [INFO] server-d5522d3e-288f-1503-56c0-3f42c8b3e8f2: added peer, starting replication: peer=server-2db41c68-b795-80b5-585d-b33233c30f8b -2025-06-03T23:33:14.102Z [INFO] server-d5522d3e-288f-1503-56c0-3f42c8b3e8f2: pipelining replication: peer="{Voter server-2db41c68-b795-80b5-585d-b33233c30f8b 2db41c68-b795-80b5-585d-b33233c30f8b}" -2025-06-03T23:33:14.103Z [WARN] server-3c42ba81-21d3-c362-9acd-41f92eae91f9: no known peers, aborting election -2025-06-03T23:33:14.207Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:14.101978071 +0000 UTC m=+9.122797441 (1 nodes), 64.07356ms from start of poll, 64.58744ms from cluster start. Timeout at 2025-06-03 23:33:14.207772591 +0000 UTC m=+9.228591961, 105.79452ms after stability -2025-06-03T23:33:14.207Z [INFO] server-d5522d3e-288f-1503-56c0-3f42c8b3e8f2: updating configuration: command=AddVoter server-id=server-3c42ba81-21d3-c362-9acd-41f92eae91f9 server-addr=3c42ba81-21d3-c362-9acd-41f92eae91f9 servers="[{Suffrage:Voter ID:server-d5522d3e-288f-1503-56c0-3f42c8b3e8f2 Address:d5522d3e-288f-1503-56c0-3f42c8b3e8f2} {Suffrage:Voter ID:server-2db41c68-b795-80b5-585d-b33233c30f8b Address:2db41c68-b795-80b5-585d-b33233c30f8b} {Suffrage:Voter ID:server-3c42ba81-21d3-c362-9acd-41f92eae91f9 Address:3c42ba81-21d3-c362-9acd-41f92eae91f9}]" -2025-06-03T23:33:14.207Z [INFO] server-d5522d3e-288f-1503-56c0-3f42c8b3e8f2: added peer, starting replication: peer=server-3c42ba81-21d3-c362-9acd-41f92eae91f9 -2025-06-03T23:33:14.208Z [WARN] server-3c42ba81-21d3-c362-9acd-41f92eae91f9: failed to get previous log: previous-index=3 last-index=0 error="log not found" -2025-06-03T23:33:14.208Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:14.208Z [WARN] server-d5522d3e-288f-1503-56c0-3f42c8b3e8f2: appendEntries rejected, sending older logs: peer="{Voter server-3c42ba81-21d3-c362-9acd-41f92eae91f9 3c42ba81-21d3-c362-9acd-41f92eae91f9}" next=1 -2025-06-03T23:33:14.208Z [INFO] server-d5522d3e-288f-1503-56c0-3f42c8b3e8f2: pipelining replication: peer="{Voter server-3c42ba81-21d3-c362-9acd-41f92eae91f9 3c42ba81-21d3-c362-9acd-41f92eae91f9}" -2025-06-03T23:33:14.313Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:14.208233471 +0000 UTC m=+9.229052841 (1 nodes), 1µs from start of poll, 170.84284ms from cluster start. Timeout at 2025-06-03 23:33:14.313834111 +0000 UTC m=+9.334653561, 105.60072ms after stability -2025-06-03T23:33:14.313Z [INFO] server-d5522d3e-288f-1503-56c0-3f42c8b3e8f2: aborting pipeline replication: peer="{Voter server-3c42ba81-21d3-c362-9acd-41f92eae91f9 3c42ba81-21d3-c362-9acd-41f92eae91f9}" -2025-06-03T23:33:14.314Z [INFO] server-d5522d3e-288f-1503-56c0-3f42c8b3e8f2: aborting pipeline replication: peer="{Voter server-2db41c68-b795-80b5-585d-b33233c30f8b 2db41c68-b795-80b5-585d-b33233c30f8b}" ---- PASS: TestRaft_JoinNode (0.28s) +2025-06-03T23:36:13.318Z [INFO] server-0354be2d-9832-1b1f-e53e-da693bba0288: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0354be2d-9832-1b1f-e53e-da693bba0288 Address:0354be2d-9832-1b1f-e53e-da693bba0288} {Suffrage:Voter ID:server-24668516-317a-33fb-9572-5e84484579c8 Address:24668516-317a-33fb-9572-5e84484579c8}]" +2025-06-03T23:36:13.318Z [INFO] server-24668516-317a-33fb-9572-5e84484579c8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0354be2d-9832-1b1f-e53e-da693bba0288 Address:0354be2d-9832-1b1f-e53e-da693bba0288} {Suffrage:Voter ID:server-24668516-317a-33fb-9572-5e84484579c8 Address:24668516-317a-33fb-9572-5e84484579c8}]" +2025-06-03T23:36:13.318Z [INFO] server-0354be2d-9832-1b1f-e53e-da693bba0288: entering follower state: follower="Node at 0354be2d-9832-1b1f-e53e-da693bba0288 [Follower]" leader-address= leader-id= +2025-06-03T23:36:13.319Z [INFO] server-24668516-317a-33fb-9572-5e84484579c8: entering follower state: follower="Node at 24668516-317a-33fb-9572-5e84484579c8 [Follower]" leader-address= leader-id= +2025-06-03T23:36:13.319Z [INFO] server-3c490b14-2a31-9e0b-024d-687f0a94b57f: initial configuration: index=0 servers=[] +2025-06-03T23:36:13.319Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:13.319Z [INFO] server-3c490b14-2a31-9e0b-024d-687f0a94b57f: entering follower state: follower="Node at 3c490b14-2a31-9e0b-024d-687f0a94b57f [Follower]" leader-address= leader-id= +2025-06-03T23:36:13.369Z [WARN] server-0354be2d-9832-1b1f-e53e-da693bba0288: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:13.369Z [INFO] server-0354be2d-9832-1b1f-e53e-da693bba0288: entering candidate state: node="Node at 0354be2d-9832-1b1f-e53e-da693bba0288 [Candidate]" term=2 +2025-06-03T23:36:13.369Z [INFO] server-0354be2d-9832-1b1f-e53e-da693bba0288: election won: term=2 tally=2 +2025-06-03T23:36:13.369Z [INFO] server-0354be2d-9832-1b1f-e53e-da693bba0288: entering leader state: leader="Node at 0354be2d-9832-1b1f-e53e-da693bba0288 [Leader]" +2025-06-03T23:36:13.369Z [INFO] server-0354be2d-9832-1b1f-e53e-da693bba0288: added peer, starting replication: peer=server-24668516-317a-33fb-9572-5e84484579c8 +2025-06-03T23:36:13.370Z [INFO] server-0354be2d-9832-1b1f-e53e-da693bba0288: pipelining replication: peer="{Voter server-24668516-317a-33fb-9572-5e84484579c8 24668516-317a-33fb-9572-5e84484579c8}" +2025-06-03T23:36:13.373Z [WARN] server-3c490b14-2a31-9e0b-024d-687f0a94b57f: no known peers, aborting election +2025-06-03T23:36:13.476Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:13.370003551 +0000 UTC m=+8.691758481 (1 nodes), 50.86828ms from start of poll, 51.39864ms from cluster start. Timeout at 2025-06-03 23:36:13.475995231 +0000 UTC m=+8.797750161, 105.99168ms after stability +2025-06-03T23:36:13.476Z [INFO] server-0354be2d-9832-1b1f-e53e-da693bba0288: updating configuration: command=AddVoter server-id=server-3c490b14-2a31-9e0b-024d-687f0a94b57f server-addr=3c490b14-2a31-9e0b-024d-687f0a94b57f servers="[{Suffrage:Voter ID:server-0354be2d-9832-1b1f-e53e-da693bba0288 Address:0354be2d-9832-1b1f-e53e-da693bba0288} {Suffrage:Voter ID:server-24668516-317a-33fb-9572-5e84484579c8 Address:24668516-317a-33fb-9572-5e84484579c8} {Suffrage:Voter ID:server-3c490b14-2a31-9e0b-024d-687f0a94b57f Address:3c490b14-2a31-9e0b-024d-687f0a94b57f}]" +2025-06-03T23:36:13.476Z [INFO] server-0354be2d-9832-1b1f-e53e-da693bba0288: added peer, starting replication: peer=server-3c490b14-2a31-9e0b-024d-687f0a94b57f +2025-06-03T23:36:13.476Z [WARN] server-3c490b14-2a31-9e0b-024d-687f0a94b57f: failed to get previous log: previous-index=3 last-index=0 error="log not found" +2025-06-03T23:36:13.476Z [WARN] server-0354be2d-9832-1b1f-e53e-da693bba0288: appendEntries rejected, sending older logs: peer="{Voter server-3c490b14-2a31-9e0b-024d-687f0a94b57f 3c490b14-2a31-9e0b-024d-687f0a94b57f}" next=1 +2025-06-03T23:36:13.476Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:13.476Z [INFO] server-0354be2d-9832-1b1f-e53e-da693bba0288: pipelining replication: peer="{Voter server-3c490b14-2a31-9e0b-024d-687f0a94b57f 3c490b14-2a31-9e0b-024d-687f0a94b57f}" +2025-06-03T23:36:13.582Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:13.476587071 +0000 UTC m=+8.798342001 (1 nodes), 1.24µs from start of poll, 157.98216ms from cluster start. Timeout at 2025-06-03 23:36:13.582045151 +0000 UTC m=+8.903800081, 105.45808ms after stability +2025-06-03T23:36:13.582Z [INFO] server-0354be2d-9832-1b1f-e53e-da693bba0288: aborting pipeline replication: peer="{Voter server-3c490b14-2a31-9e0b-024d-687f0a94b57f 3c490b14-2a31-9e0b-024d-687f0a94b57f}" +2025-06-03T23:36:13.582Z [INFO] server-0354be2d-9832-1b1f-e53e-da693bba0288: aborting pipeline replication: peer="{Voter server-24668516-317a-33fb-9572-5e84484579c8 24668516-317a-33fb-9572-5e84484579c8}" +--- PASS: TestRaft_JoinNode (0.26s) === RUN TestRaft_JoinNode_ConfigStore -2025-06-03T23:33:14.314Z [INFO] server-afc52ca8-ed75-63ec-a836-f3c881c92341: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-afc52ca8-ed75-63ec-a836-f3c881c92341 Address:afc52ca8-ed75-63ec-a836-f3c881c92341}]" -2025-06-03T23:33:14.314Z [INFO] server-afc52ca8-ed75-63ec-a836-f3c881c92341: entering follower state: follower="Node at afc52ca8-ed75-63ec-a836-f3c881c92341 [Follower]" leader-address= leader-id= -2025-06-03T23:33:14.314Z [INFO] server-efda1cf7-af79-0097-f642-8aeca493c80e: initial configuration: index=0 servers=[] -2025-06-03T23:33:14.314Z [INFO] server-efda1cf7-af79-0097-f642-8aeca493c80e: entering follower state: follower="Node at efda1cf7-af79-0097-f642-8aeca493c80e [Follower]" leader-address= leader-id= -2025-06-03T23:33:14.315Z [INFO] server-8b2eae71-f3e6-c275-4de4-f21a6530c37f: initial configuration: index=0 servers=[] -2025-06-03T23:33:14.315Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:14.315Z [INFO] server-8b2eae71-f3e6-c275-4de4-f21a6530c37f: entering follower state: follower="Node at 8b2eae71-f3e6-c275-4de4-f21a6530c37f [Follower]" leader-address= leader-id= -2025-06-03T23:33:14.371Z [WARN] server-efda1cf7-af79-0097-f642-8aeca493c80e: no known peers, aborting election -2025-06-03T23:33:14.401Z [WARN] server-afc52ca8-ed75-63ec-a836-f3c881c92341: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:14.402Z [INFO] server-afc52ca8-ed75-63ec-a836-f3c881c92341: entering candidate state: node="Node at afc52ca8-ed75-63ec-a836-f3c881c92341 [Candidate]" term=2 -2025-06-03T23:33:14.402Z [INFO] server-afc52ca8-ed75-63ec-a836-f3c881c92341: election won: term=2 tally=1 -2025-06-03T23:33:14.402Z [INFO] server-afc52ca8-ed75-63ec-a836-f3c881c92341: entering leader state: leader="Node at afc52ca8-ed75-63ec-a836-f3c881c92341 [Leader]" -2025-06-03T23:33:14.415Z [WARN] server-8b2eae71-f3e6-c275-4de4-f21a6530c37f: no known peers, aborting election -2025-06-03T23:33:14.508Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:14.402151791 +0000 UTC m=+9.422971161 (1 nodes), 87.01028ms from start of poll, 87.52428ms from cluster start. Timeout at 2025-06-03 23:33:14.508045591 +0000 UTC m=+9.528864961, 105.8938ms after stability -2025-06-03T23:33:14.508Z [INFO] server-afc52ca8-ed75-63ec-a836-f3c881c92341: updating configuration: command=AddVoter server-id=server-efda1cf7-af79-0097-f642-8aeca493c80e server-addr=efda1cf7-af79-0097-f642-8aeca493c80e servers="[{Suffrage:Voter ID:server-afc52ca8-ed75-63ec-a836-f3c881c92341 Address:afc52ca8-ed75-63ec-a836-f3c881c92341} {Suffrage:Voter ID:server-efda1cf7-af79-0097-f642-8aeca493c80e Address:efda1cf7-af79-0097-f642-8aeca493c80e}]" -2025-06-03T23:33:14.508Z [INFO] server-afc52ca8-ed75-63ec-a836-f3c881c92341: added peer, starting replication: peer=server-efda1cf7-af79-0097-f642-8aeca493c80e -2025-06-03T23:33:14.508Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:14.508Z [WARN] server-efda1cf7-af79-0097-f642-8aeca493c80e: failed to get previous log: previous-index=3 last-index=0 error="log not found" -2025-06-03T23:33:14.508Z [WARN] server-afc52ca8-ed75-63ec-a836-f3c881c92341: appendEntries rejected, sending older logs: peer="{Voter server-efda1cf7-af79-0097-f642-8aeca493c80e efda1cf7-af79-0097-f642-8aeca493c80e}" next=1 -2025-06-03T23:33:14.508Z [INFO] server-afc52ca8-ed75-63ec-a836-f3c881c92341: pipelining replication: peer="{Voter server-efda1cf7-af79-0097-f642-8aeca493c80e efda1cf7-af79-0097-f642-8aeca493c80e}" -2025-06-03T23:33:14.613Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:14.508340591 +0000 UTC m=+9.529159961 (1 nodes), 800ns from start of poll, 193.71308ms from cluster start. Timeout at 2025-06-03 23:33:14.613371791 +0000 UTC m=+9.634191161, 105.0312ms after stability -2025-06-03T23:33:14.613Z [INFO] server-afc52ca8-ed75-63ec-a836-f3c881c92341: updating configuration: command=AddVoter server-id=server-8b2eae71-f3e6-c275-4de4-f21a6530c37f server-addr=8b2eae71-f3e6-c275-4de4-f21a6530c37f servers="[{Suffrage:Voter ID:server-afc52ca8-ed75-63ec-a836-f3c881c92341 Address:afc52ca8-ed75-63ec-a836-f3c881c92341} {Suffrage:Voter ID:server-efda1cf7-af79-0097-f642-8aeca493c80e Address:efda1cf7-af79-0097-f642-8aeca493c80e} {Suffrage:Voter ID:server-8b2eae71-f3e6-c275-4de4-f21a6530c37f Address:8b2eae71-f3e6-c275-4de4-f21a6530c37f}]" -2025-06-03T23:33:14.613Z [INFO] server-afc52ca8-ed75-63ec-a836-f3c881c92341: added peer, starting replication: peer=server-8b2eae71-f3e6-c275-4de4-f21a6530c37f -2025-06-03T23:33:14.613Z [WARN] server-8b2eae71-f3e6-c275-4de4-f21a6530c37f: failed to get previous log: previous-index=4 last-index=0 error="log not found" -2025-06-03T23:33:14.613Z [WARN] server-afc52ca8-ed75-63ec-a836-f3c881c92341: appendEntries rejected, sending older logs: peer="{Voter server-8b2eae71-f3e6-c275-4de4-f21a6530c37f 8b2eae71-f3e6-c275-4de4-f21a6530c37f}" next=1 -2025-06-03T23:33:14.613Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:14.613Z [INFO] server-afc52ca8-ed75-63ec-a836-f3c881c92341: pipelining replication: peer="{Voter server-8b2eae71-f3e6-c275-4de4-f21a6530c37f 8b2eae71-f3e6-c275-4de4-f21a6530c37f}" -2025-06-03T23:33:14.719Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:14.613797071 +0000 UTC m=+9.634616441 (1 nodes), 1.12µs from start of poll, 299.16956ms from cluster start. Timeout at 2025-06-03 23:33:14.719240231 +0000 UTC m=+9.740059601, 105.44316ms after stability -2025-06-03T23:33:14.719Z [INFO] server-afc52ca8-ed75-63ec-a836-f3c881c92341: aborting pipeline replication: peer="{Voter server-8b2eae71-f3e6-c275-4de4-f21a6530c37f 8b2eae71-f3e6-c275-4de4-f21a6530c37f}" -2025-06-03T23:33:14.719Z [INFO] server-afc52ca8-ed75-63ec-a836-f3c881c92341: aborting pipeline replication: peer="{Voter server-efda1cf7-af79-0097-f642-8aeca493c80e efda1cf7-af79-0097-f642-8aeca493c80e}" ---- PASS: TestRaft_JoinNode_ConfigStore (0.41s) +2025-06-03T23:36:13.583Z [INFO] server-b8f8c672-56e9-f7ff-818d-f609048ce826: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b8f8c672-56e9-f7ff-818d-f609048ce826 Address:b8f8c672-56e9-f7ff-818d-f609048ce826}]" +2025-06-03T23:36:13.583Z [INFO] server-b8f8c672-56e9-f7ff-818d-f609048ce826: entering follower state: follower="Node at b8f8c672-56e9-f7ff-818d-f609048ce826 [Follower]" leader-address= leader-id= +2025-06-03T23:36:13.583Z [INFO] server-889917f7-4934-7754-2f0c-0ec712afa762: initial configuration: index=0 servers=[] +2025-06-03T23:36:13.583Z [INFO] server-889917f7-4934-7754-2f0c-0ec712afa762: entering follower state: follower="Node at 889917f7-4934-7754-2f0c-0ec712afa762 [Follower]" leader-address= leader-id= +2025-06-03T23:36:13.583Z [INFO] server-a0e879fe-7d09-2a65-0464-a8b42835e9ca: initial configuration: index=0 servers=[] +2025-06-03T23:36:13.583Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:13.583Z [INFO] server-a0e879fe-7d09-2a65-0464-a8b42835e9ca: entering follower state: follower="Node at a0e879fe-7d09-2a65-0464-a8b42835e9ca [Follower]" leader-address= leader-id= +2025-06-03T23:36:13.637Z [WARN] server-a0e879fe-7d09-2a65-0464-a8b42835e9ca: no known peers, aborting election +2025-06-03T23:36:13.640Z [WARN] server-889917f7-4934-7754-2f0c-0ec712afa762: no known peers, aborting election +2025-06-03T23:36:13.652Z [WARN] server-b8f8c672-56e9-f7ff-818d-f609048ce826: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:13.652Z [INFO] server-b8f8c672-56e9-f7ff-818d-f609048ce826: entering candidate state: node="Node at b8f8c672-56e9-f7ff-818d-f609048ce826 [Candidate]" term=2 +2025-06-03T23:36:13.652Z [INFO] server-b8f8c672-56e9-f7ff-818d-f609048ce826: election won: term=2 tally=1 +2025-06-03T23:36:13.652Z [INFO] server-b8f8c672-56e9-f7ff-818d-f609048ce826: entering leader state: leader="Node at b8f8c672-56e9-f7ff-818d-f609048ce826 [Leader]" +2025-06-03T23:36:13.758Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:13.652499991 +0000 UTC m=+8.974254921 (1 nodes), 69.03244ms from start of poll, 69.6118ms from cluster start. Timeout at 2025-06-03 23:36:13.758426191 +0000 UTC m=+9.080181121, 105.9262ms after stability +2025-06-03T23:36:13.758Z [INFO] server-b8f8c672-56e9-f7ff-818d-f609048ce826: updating configuration: command=AddVoter server-id=server-889917f7-4934-7754-2f0c-0ec712afa762 server-addr=889917f7-4934-7754-2f0c-0ec712afa762 servers="[{Suffrage:Voter ID:server-b8f8c672-56e9-f7ff-818d-f609048ce826 Address:b8f8c672-56e9-f7ff-818d-f609048ce826} {Suffrage:Voter ID:server-889917f7-4934-7754-2f0c-0ec712afa762 Address:889917f7-4934-7754-2f0c-0ec712afa762}]" +2025-06-03T23:36:13.758Z [INFO] server-b8f8c672-56e9-f7ff-818d-f609048ce826: added peer, starting replication: peer=server-889917f7-4934-7754-2f0c-0ec712afa762 +2025-06-03T23:36:13.758Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:13.758Z [WARN] server-889917f7-4934-7754-2f0c-0ec712afa762: failed to get previous log: previous-index=3 last-index=0 error="log not found" +2025-06-03T23:36:13.758Z [WARN] server-b8f8c672-56e9-f7ff-818d-f609048ce826: appendEntries rejected, sending older logs: peer="{Voter server-889917f7-4934-7754-2f0c-0ec712afa762 889917f7-4934-7754-2f0c-0ec712afa762}" next=1 +2025-06-03T23:36:13.759Z [INFO] server-b8f8c672-56e9-f7ff-818d-f609048ce826: pipelining replication: peer="{Voter server-889917f7-4934-7754-2f0c-0ec712afa762 889917f7-4934-7754-2f0c-0ec712afa762}" +2025-06-03T23:36:13.864Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:13.758765631 +0000 UTC m=+9.080520561 (1 nodes), 920ns from start of poll, 175.87744ms from cluster start. Timeout at 2025-06-03 23:36:13.864162391 +0000 UTC m=+9.185917321, 105.39676ms after stability +2025-06-03T23:36:13.864Z [INFO] server-b8f8c672-56e9-f7ff-818d-f609048ce826: updating configuration: command=AddVoter server-id=server-a0e879fe-7d09-2a65-0464-a8b42835e9ca server-addr=a0e879fe-7d09-2a65-0464-a8b42835e9ca servers="[{Suffrage:Voter ID:server-b8f8c672-56e9-f7ff-818d-f609048ce826 Address:b8f8c672-56e9-f7ff-818d-f609048ce826} {Suffrage:Voter ID:server-889917f7-4934-7754-2f0c-0ec712afa762 Address:889917f7-4934-7754-2f0c-0ec712afa762} {Suffrage:Voter ID:server-a0e879fe-7d09-2a65-0464-a8b42835e9ca Address:a0e879fe-7d09-2a65-0464-a8b42835e9ca}]" +2025-06-03T23:36:13.864Z [INFO] server-b8f8c672-56e9-f7ff-818d-f609048ce826: added peer, starting replication: peer=server-a0e879fe-7d09-2a65-0464-a8b42835e9ca +2025-06-03T23:36:13.864Z [WARN] server-a0e879fe-7d09-2a65-0464-a8b42835e9ca: failed to get previous log: previous-index=4 last-index=0 error="log not found" +2025-06-03T23:36:13.864Z [WARN] server-b8f8c672-56e9-f7ff-818d-f609048ce826: appendEntries rejected, sending older logs: peer="{Voter server-a0e879fe-7d09-2a65-0464-a8b42835e9ca a0e879fe-7d09-2a65-0464-a8b42835e9ca}" next=1 +2025-06-03T23:36:13.864Z [INFO] server-b8f8c672-56e9-f7ff-818d-f609048ce826: pipelining replication: peer="{Voter server-a0e879fe-7d09-2a65-0464-a8b42835e9ca a0e879fe-7d09-2a65-0464-a8b42835e9ca}" +2025-06-03T23:36:13.864Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:13.970Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:13.864744711 +0000 UTC m=+9.186499681 (1 nodes), 880ns from start of poll, 281.85656ms from cluster start. Timeout at 2025-06-03 23:36:13.970317751 +0000 UTC m=+9.292072681, 105.573ms after stability +2025-06-03T23:36:13.970Z [INFO] server-b8f8c672-56e9-f7ff-818d-f609048ce826: aborting pipeline replication: peer="{Voter server-a0e879fe-7d09-2a65-0464-a8b42835e9ca a0e879fe-7d09-2a65-0464-a8b42835e9ca}" +2025-06-03T23:36:13.970Z [INFO] server-b8f8c672-56e9-f7ff-818d-f609048ce826: aborting pipeline replication: peer="{Voter server-889917f7-4934-7754-2f0c-0ec712afa762 889917f7-4934-7754-2f0c-0ec712afa762}" +--- PASS: TestRaft_JoinNode_ConfigStore (0.39s) === RUN TestRaft_RemoveFollower -2025-06-03T23:33:14.720Z [INFO] server-49e806ca-f5d9-28ac-0dee-56bf29d86b08: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-49e806ca-f5d9-28ac-0dee-56bf29d86b08 Address:49e806ca-f5d9-28ac-0dee-56bf29d86b08} {Suffrage:Voter ID:server-28b2e23d-b798-3ba6-475f-e04ddff08c4a Address:28b2e23d-b798-3ba6-475f-e04ddff08c4a} {Suffrage:Voter ID:server-2d54dd8a-bfae-7955-601b-325dc6ac05e8 Address:2d54dd8a-bfae-7955-601b-325dc6ac05e8}]" -2025-06-03T23:33:14.720Z [INFO] server-49e806ca-f5d9-28ac-0dee-56bf29d86b08: entering follower state: follower="Node at 49e806ca-f5d9-28ac-0dee-56bf29d86b08 [Follower]" leader-address= leader-id= -2025-06-03T23:33:14.720Z [INFO] server-28b2e23d-b798-3ba6-475f-e04ddff08c4a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-49e806ca-f5d9-28ac-0dee-56bf29d86b08 Address:49e806ca-f5d9-28ac-0dee-56bf29d86b08} {Suffrage:Voter ID:server-28b2e23d-b798-3ba6-475f-e04ddff08c4a Address:28b2e23d-b798-3ba6-475f-e04ddff08c4a} {Suffrage:Voter ID:server-2d54dd8a-bfae-7955-601b-325dc6ac05e8 Address:2d54dd8a-bfae-7955-601b-325dc6ac05e8}]" -2025-06-03T23:33:14.720Z [INFO] server-28b2e23d-b798-3ba6-475f-e04ddff08c4a: entering follower state: follower="Node at 28b2e23d-b798-3ba6-475f-e04ddff08c4a [Follower]" leader-address= leader-id= -2025-06-03T23:33:14.720Z [INFO] server-2d54dd8a-bfae-7955-601b-325dc6ac05e8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-49e806ca-f5d9-28ac-0dee-56bf29d86b08 Address:49e806ca-f5d9-28ac-0dee-56bf29d86b08} {Suffrage:Voter ID:server-28b2e23d-b798-3ba6-475f-e04ddff08c4a Address:28b2e23d-b798-3ba6-475f-e04ddff08c4a} {Suffrage:Voter ID:server-2d54dd8a-bfae-7955-601b-325dc6ac05e8 Address:2d54dd8a-bfae-7955-601b-325dc6ac05e8}]" -2025-06-03T23:33:14.720Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:14.720Z [INFO] server-2d54dd8a-bfae-7955-601b-325dc6ac05e8: entering follower state: follower="Node at 2d54dd8a-bfae-7955-601b-325dc6ac05e8 [Follower]" leader-address= leader-id= -2025-06-03T23:33:14.784Z [WARN] server-2d54dd8a-bfae-7955-601b-325dc6ac05e8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:14.784Z [INFO] server-2d54dd8a-bfae-7955-601b-325dc6ac05e8: entering candidate state: node="Node at 2d54dd8a-bfae-7955-601b-325dc6ac05e8 [Candidate]" term=2 -2025-06-03T23:33:14.784Z [INFO] server-2d54dd8a-bfae-7955-601b-325dc6ac05e8: election won: term=2 tally=2 -2025-06-03T23:33:14.784Z [INFO] server-2d54dd8a-bfae-7955-601b-325dc6ac05e8: entering leader state: leader="Node at 2d54dd8a-bfae-7955-601b-325dc6ac05e8 [Leader]" -2025-06-03T23:33:14.784Z [INFO] server-2d54dd8a-bfae-7955-601b-325dc6ac05e8: added peer, starting replication: peer=server-49e806ca-f5d9-28ac-0dee-56bf29d86b08 -2025-06-03T23:33:14.784Z [INFO] server-2d54dd8a-bfae-7955-601b-325dc6ac05e8: added peer, starting replication: peer=server-28b2e23d-b798-3ba6-475f-e04ddff08c4a -2025-06-03T23:33:14.784Z [INFO] server-2d54dd8a-bfae-7955-601b-325dc6ac05e8: pipelining replication: peer="{Voter server-49e806ca-f5d9-28ac-0dee-56bf29d86b08 49e806ca-f5d9-28ac-0dee-56bf29d86b08}" -2025-06-03T23:33:14.784Z [INFO] server-2d54dd8a-bfae-7955-601b-325dc6ac05e8: pipelining replication: peer="{Voter server-28b2e23d-b798-3ba6-475f-e04ddff08c4a 28b2e23d-b798-3ba6-475f-e04ddff08c4a}" -2025-06-03T23:33:14.890Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:14.784621871 +0000 UTC m=+9.805441241 (1 nodes), 64.04384ms from start of poll, 64.49184ms from cluster start. Timeout at 2025-06-03 23:33:14.890478871 +0000 UTC m=+9.911298241, 105.857ms after stability -2025-06-03T23:33:14.896Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:15.002Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:14.896101631 +0000 UTC m=+9.916921001 (2 nodes), 1.88µs from start of poll, 175.9716ms from cluster start. Timeout at 2025-06-03 23:33:15.002121231 +0000 UTC m=+10.022940641, 106.01964ms after stability -2025-06-03T23:33:15.002Z [INFO] server-2d54dd8a-bfae-7955-601b-325dc6ac05e8: updating configuration: command=RemoveServer server-id=server-49e806ca-f5d9-28ac-0dee-56bf29d86b08 server-addr= servers="[{Suffrage:Voter ID:server-28b2e23d-b798-3ba6-475f-e04ddff08c4a Address:28b2e23d-b798-3ba6-475f-e04ddff08c4a} {Suffrage:Voter ID:server-2d54dd8a-bfae-7955-601b-325dc6ac05e8 Address:2d54dd8a-bfae-7955-601b-325dc6ac05e8}]" -2025-06-03T23:33:15.002Z [INFO] server-2d54dd8a-bfae-7955-601b-325dc6ac05e8: removed peer, stopping replication: peer=server-49e806ca-f5d9-28ac-0dee-56bf29d86b08 last-index=3 -2025-06-03T23:33:15.002Z [INFO] server-2d54dd8a-bfae-7955-601b-325dc6ac05e8: aborting pipeline replication: peer="{Voter server-49e806ca-f5d9-28ac-0dee-56bf29d86b08 49e806ca-f5d9-28ac-0dee-56bf29d86b08}" -2025-06-03T23:33:15.108Z [INFO] server-2d54dd8a-bfae-7955-601b-325dc6ac05e8: aborting pipeline replication: peer="{Voter server-28b2e23d-b798-3ba6-475f-e04ddff08c4a 28b2e23d-b798-3ba6-475f-e04ddff08c4a}" +2025-06-03T23:36:13.971Z [INFO] server-ab6fdd4f-657c-d057-cf69-f5f57efd0ecd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ab6fdd4f-657c-d057-cf69-f5f57efd0ecd Address:ab6fdd4f-657c-d057-cf69-f5f57efd0ecd} {Suffrage:Voter ID:server-55af84db-8a3d-d360-d01c-ea672b30624c Address:55af84db-8a3d-d360-d01c-ea672b30624c} {Suffrage:Voter ID:server-992ea8e7-4a14-4bc2-d517-43c84b1212e4 Address:992ea8e7-4a14-4bc2-d517-43c84b1212e4}]" +2025-06-03T23:36:13.971Z [INFO] server-ab6fdd4f-657c-d057-cf69-f5f57efd0ecd: entering follower state: follower="Node at ab6fdd4f-657c-d057-cf69-f5f57efd0ecd [Follower]" leader-address= leader-id= +2025-06-03T23:36:13.971Z [INFO] server-55af84db-8a3d-d360-d01c-ea672b30624c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ab6fdd4f-657c-d057-cf69-f5f57efd0ecd Address:ab6fdd4f-657c-d057-cf69-f5f57efd0ecd} {Suffrage:Voter ID:server-55af84db-8a3d-d360-d01c-ea672b30624c Address:55af84db-8a3d-d360-d01c-ea672b30624c} {Suffrage:Voter ID:server-992ea8e7-4a14-4bc2-d517-43c84b1212e4 Address:992ea8e7-4a14-4bc2-d517-43c84b1212e4}]" +2025-06-03T23:36:13.971Z [INFO] server-55af84db-8a3d-d360-d01c-ea672b30624c: entering follower state: follower="Node at 55af84db-8a3d-d360-d01c-ea672b30624c [Follower]" leader-address= leader-id= +2025-06-03T23:36:13.971Z [INFO] server-992ea8e7-4a14-4bc2-d517-43c84b1212e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ab6fdd4f-657c-d057-cf69-f5f57efd0ecd Address:ab6fdd4f-657c-d057-cf69-f5f57efd0ecd} {Suffrage:Voter ID:server-55af84db-8a3d-d360-d01c-ea672b30624c Address:55af84db-8a3d-d360-d01c-ea672b30624c} {Suffrage:Voter ID:server-992ea8e7-4a14-4bc2-d517-43c84b1212e4 Address:992ea8e7-4a14-4bc2-d517-43c84b1212e4}]" +2025-06-03T23:36:13.971Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:13.971Z [INFO] server-992ea8e7-4a14-4bc2-d517-43c84b1212e4: entering follower state: follower="Node at 992ea8e7-4a14-4bc2-d517-43c84b1212e4 [Follower]" leader-address= leader-id= +2025-06-03T23:36:14.037Z [WARN] server-992ea8e7-4a14-4bc2-d517-43c84b1212e4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:14.037Z [INFO] server-992ea8e7-4a14-4bc2-d517-43c84b1212e4: entering candidate state: node="Node at 992ea8e7-4a14-4bc2-d517-43c84b1212e4 [Candidate]" term=2 +2025-06-03T23:36:14.037Z [INFO] server-992ea8e7-4a14-4bc2-d517-43c84b1212e4: election won: term=2 tally=2 +2025-06-03T23:36:14.037Z [INFO] server-992ea8e7-4a14-4bc2-d517-43c84b1212e4: entering leader state: leader="Node at 992ea8e7-4a14-4bc2-d517-43c84b1212e4 [Leader]" +2025-06-03T23:36:14.037Z [INFO] server-992ea8e7-4a14-4bc2-d517-43c84b1212e4: added peer, starting replication: peer=server-ab6fdd4f-657c-d057-cf69-f5f57efd0ecd +2025-06-03T23:36:14.037Z [INFO] server-992ea8e7-4a14-4bc2-d517-43c84b1212e4: added peer, starting replication: peer=server-55af84db-8a3d-d360-d01c-ea672b30624c +2025-06-03T23:36:14.037Z [INFO] server-992ea8e7-4a14-4bc2-d517-43c84b1212e4: pipelining replication: peer="{Voter server-ab6fdd4f-657c-d057-cf69-f5f57efd0ecd ab6fdd4f-657c-d057-cf69-f5f57efd0ecd}" +2025-06-03T23:36:14.037Z [INFO] server-992ea8e7-4a14-4bc2-d517-43c84b1212e4: pipelining replication: peer="{Voter server-55af84db-8a3d-d360-d01c-ea672b30624c 55af84db-8a3d-d360-d01c-ea672b30624c}" +2025-06-03T23:36:14.142Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:14.037342431 +0000 UTC m=+9.359097361 (1 nodes), 65.56984ms from start of poll, 66.08064ms from cluster start. Timeout at 2025-06-03 23:36:14.142755351 +0000 UTC m=+9.464510281, 105.41292ms after stability +2025-06-03T23:36:14.148Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:14.253Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:14.148295431 +0000 UTC m=+9.470050361 (2 nodes), 1.96µs from start of poll, 177.03364ms from cluster start. Timeout at 2025-06-03 23:36:14.253620431 +0000 UTC m=+9.575375401, 105.32504ms after stability +2025-06-03T23:36:14.253Z [INFO] server-992ea8e7-4a14-4bc2-d517-43c84b1212e4: updating configuration: command=RemoveServer server-id=server-ab6fdd4f-657c-d057-cf69-f5f57efd0ecd server-addr= servers="[{Suffrage:Voter ID:server-55af84db-8a3d-d360-d01c-ea672b30624c Address:55af84db-8a3d-d360-d01c-ea672b30624c} {Suffrage:Voter ID:server-992ea8e7-4a14-4bc2-d517-43c84b1212e4 Address:992ea8e7-4a14-4bc2-d517-43c84b1212e4}]" +2025-06-03T23:36:14.253Z [INFO] server-992ea8e7-4a14-4bc2-d517-43c84b1212e4: removed peer, stopping replication: peer=server-ab6fdd4f-657c-d057-cf69-f5f57efd0ecd last-index=3 +2025-06-03T23:36:14.253Z [INFO] server-992ea8e7-4a14-4bc2-d517-43c84b1212e4: aborting pipeline replication: peer="{Voter server-ab6fdd4f-657c-d057-cf69-f5f57efd0ecd ab6fdd4f-657c-d057-cf69-f5f57efd0ecd}" +2025-06-03T23:36:14.359Z [INFO] server-992ea8e7-4a14-4bc2-d517-43c84b1212e4: aborting pipeline replication: peer="{Voter server-55af84db-8a3d-d360-d01c-ea672b30624c 55af84db-8a3d-d360-d01c-ea672b30624c}" --- PASS: TestRaft_RemoveFollower (0.39s) === RUN TestRaft_RemoveLeader -2025-06-03T23:33:15.109Z [INFO] server-0582b1c8-cf82-8efd-0824-288cbf8c5e57: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0582b1c8-cf82-8efd-0824-288cbf8c5e57 Address:0582b1c8-cf82-8efd-0824-288cbf8c5e57} {Suffrage:Voter ID:server-87dd7580-3266-83ec-7f8c-d5de941fffaf Address:87dd7580-3266-83ec-7f8c-d5de941fffaf} {Suffrage:Voter ID:server-829348be-9c67-2d5d-7a4f-e92746be9b9a Address:829348be-9c67-2d5d-7a4f-e92746be9b9a}]" -2025-06-03T23:33:15.109Z [INFO] server-0582b1c8-cf82-8efd-0824-288cbf8c5e57: entering follower state: follower="Node at 0582b1c8-cf82-8efd-0824-288cbf8c5e57 [Follower]" leader-address= leader-id= -2025-06-03T23:33:15.109Z [INFO] server-87dd7580-3266-83ec-7f8c-d5de941fffaf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0582b1c8-cf82-8efd-0824-288cbf8c5e57 Address:0582b1c8-cf82-8efd-0824-288cbf8c5e57} {Suffrage:Voter ID:server-87dd7580-3266-83ec-7f8c-d5de941fffaf Address:87dd7580-3266-83ec-7f8c-d5de941fffaf} {Suffrage:Voter ID:server-829348be-9c67-2d5d-7a4f-e92746be9b9a Address:829348be-9c67-2d5d-7a4f-e92746be9b9a}]" -2025-06-03T23:33:15.109Z [INFO] server-87dd7580-3266-83ec-7f8c-d5de941fffaf: entering follower state: follower="Node at 87dd7580-3266-83ec-7f8c-d5de941fffaf [Follower]" leader-address= leader-id= -2025-06-03T23:33:15.109Z [INFO] server-829348be-9c67-2d5d-7a4f-e92746be9b9a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0582b1c8-cf82-8efd-0824-288cbf8c5e57 Address:0582b1c8-cf82-8efd-0824-288cbf8c5e57} {Suffrage:Voter ID:server-87dd7580-3266-83ec-7f8c-d5de941fffaf Address:87dd7580-3266-83ec-7f8c-d5de941fffaf} {Suffrage:Voter ID:server-829348be-9c67-2d5d-7a4f-e92746be9b9a Address:829348be-9c67-2d5d-7a4f-e92746be9b9a}]" -2025-06-03T23:33:15.109Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:15.109Z [INFO] server-829348be-9c67-2d5d-7a4f-e92746be9b9a: entering follower state: follower="Node at 829348be-9c67-2d5d-7a4f-e92746be9b9a [Follower]" leader-address= leader-id= -2025-06-03T23:33:15.169Z [WARN] server-0582b1c8-cf82-8efd-0824-288cbf8c5e57: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:15.169Z [INFO] server-0582b1c8-cf82-8efd-0824-288cbf8c5e57: entering candidate state: node="Node at 0582b1c8-cf82-8efd-0824-288cbf8c5e57 [Candidate]" term=2 -2025-06-03T23:33:15.169Z [INFO] server-0582b1c8-cf82-8efd-0824-288cbf8c5e57: election won: term=2 tally=2 -2025-06-03T23:33:15.169Z [INFO] server-0582b1c8-cf82-8efd-0824-288cbf8c5e57: entering leader state: leader="Node at 0582b1c8-cf82-8efd-0824-288cbf8c5e57 [Leader]" -2025-06-03T23:33:15.169Z [INFO] server-0582b1c8-cf82-8efd-0824-288cbf8c5e57: added peer, starting replication: peer=server-87dd7580-3266-83ec-7f8c-d5de941fffaf -2025-06-03T23:33:15.169Z [INFO] server-0582b1c8-cf82-8efd-0824-288cbf8c5e57: added peer, starting replication: peer=server-829348be-9c67-2d5d-7a4f-e92746be9b9a -2025-06-03T23:33:15.170Z [INFO] server-0582b1c8-cf82-8efd-0824-288cbf8c5e57: pipelining replication: peer="{Voter server-829348be-9c67-2d5d-7a4f-e92746be9b9a 829348be-9c67-2d5d-7a4f-e92746be9b9a}" -2025-06-03T23:33:15.170Z [INFO] server-0582b1c8-cf82-8efd-0824-288cbf8c5e57: pipelining replication: peer="{Voter server-87dd7580-3266-83ec-7f8c-d5de941fffaf 87dd7580-3266-83ec-7f8c-d5de941fffaf}" -2025-06-03T23:33:15.276Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:15.169967151 +0000 UTC m=+10.190786561 (1 nodes), 60.63996ms from start of poll, 61.08452ms from cluster start. Timeout at 2025-06-03 23:33:15.276063391 +0000 UTC m=+10.296882801, 106.09624ms after stability -2025-06-03T23:33:15.281Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:15.387Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:15.281911871 +0000 UTC m=+10.302731281 (2 nodes), 1.36µs from start of poll, 173.02924ms from cluster start. Timeout at 2025-06-03 23:33:15.386979831 +0000 UTC m=+10.407799201, 105.06792ms after stability -2025-06-03T23:33:15.387Z [INFO] server-0582b1c8-cf82-8efd-0824-288cbf8c5e57: updating configuration: command=RemoveServer server-id=server-0582b1c8-cf82-8efd-0824-288cbf8c5e57 server-addr= servers="[{Suffrage:Voter ID:server-87dd7580-3266-83ec-7f8c-d5de941fffaf Address:87dd7580-3266-83ec-7f8c-d5de941fffaf} {Suffrage:Voter ID:server-829348be-9c67-2d5d-7a4f-e92746be9b9a Address:829348be-9c67-2d5d-7a4f-e92746be9b9a}]" -2025-06-03T23:33:15.387Z [INFO] server-0582b1c8-cf82-8efd-0824-288cbf8c5e57: removed ourself, shutting down -2025-06-03T23:33:15.387Z [INFO] server-0582b1c8-cf82-8efd-0824-288cbf8c5e57: aborting pipeline replication: peer="{Voter server-829348be-9c67-2d5d-7a4f-e92746be9b9a 829348be-9c67-2d5d-7a4f-e92746be9b9a}" -2025-06-03T23:33:15.387Z [INFO] server-0582b1c8-cf82-8efd-0824-288cbf8c5e57: aborting pipeline replication: peer="{Voter server-87dd7580-3266-83ec-7f8c-d5de941fffaf 87dd7580-3266-83ec-7f8c-d5de941fffaf}" -2025-06-03T23:33:15.452Z [WARN] server-87dd7580-3266-83ec-7f8c-d5de941fffaf: heartbeat timeout reached, starting election: last-leader-addr=0582b1c8-cf82-8efd-0824-288cbf8c5e57 last-leader-id=server-0582b1c8-cf82-8efd-0824-288cbf8c5e57 -2025-06-03T23:33:15.452Z [INFO] server-87dd7580-3266-83ec-7f8c-d5de941fffaf: entering candidate state: node="Node at 87dd7580-3266-83ec-7f8c-d5de941fffaf [Candidate]" term=3 -2025-06-03T23:33:15.452Z [WARN] server-829348be-9c67-2d5d-7a4f-e92746be9b9a: rejecting vote request since we have a leader: from=87dd7580-3266-83ec-7f8c-d5de941fffaf leader=0582b1c8-cf82-8efd-0824-288cbf8c5e57 leader-id=server-0582b1c8-cf82-8efd-0824-288cbf8c5e57 -2025-06-03T23:33:15.476Z [WARN] server-829348be-9c67-2d5d-7a4f-e92746be9b9a: heartbeat timeout reached, starting election: last-leader-addr=0582b1c8-cf82-8efd-0824-288cbf8c5e57 last-leader-id=server-0582b1c8-cf82-8efd-0824-288cbf8c5e57 -2025-06-03T23:33:15.476Z [INFO] server-829348be-9c67-2d5d-7a4f-e92746be9b9a: entering candidate state: node="Node at 829348be-9c67-2d5d-7a4f-e92746be9b9a [Candidate]" term=3 -2025-06-03T23:33:15.477Z [INFO] server-87dd7580-3266-83ec-7f8c-d5de941fffaf: duplicate requestVote for same term: term=3 -2025-06-03T23:33:15.537Z [WARN] server-87dd7580-3266-83ec-7f8c-d5de941fffaf: Election timeout reached, restarting election -2025-06-03T23:33:15.537Z [INFO] server-87dd7580-3266-83ec-7f8c-d5de941fffaf: entering candidate state: node="Node at 87dd7580-3266-83ec-7f8c-d5de941fffaf [Candidate]" term=4 -2025-06-03T23:33:15.537Z [INFO] server-829348be-9c67-2d5d-7a4f-e92746be9b9a: entering follower state: follower="Node at 829348be-9c67-2d5d-7a4f-e92746be9b9a [Follower]" leader-address= leader-id= -2025-06-03T23:33:15.537Z [INFO] server-87dd7580-3266-83ec-7f8c-d5de941fffaf: election won: term=4 tally=2 -2025-06-03T23:33:15.537Z [INFO] server-87dd7580-3266-83ec-7f8c-d5de941fffaf: entering leader state: leader="Node at 87dd7580-3266-83ec-7f8c-d5de941fffaf [Leader]" -2025-06-03T23:33:15.537Z [INFO] server-87dd7580-3266-83ec-7f8c-d5de941fffaf: added peer, starting replication: peer=server-829348be-9c67-2d5d-7a4f-e92746be9b9a -2025-06-03T23:33:15.538Z [INFO] server-87dd7580-3266-83ec-7f8c-d5de941fffaf: pipelining replication: peer="{Voter server-829348be-9c67-2d5d-7a4f-e92746be9b9a 829348be-9c67-2d5d-7a4f-e92746be9b9a}" -2025-06-03T23:33:15.598Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:15.703Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:15.598336751 +0000 UTC m=+10.619156121 (1 nodes), 61.28µs from start of poll, 489.45408ms from cluster start. Timeout at 2025-06-03 23:33:15.703783151 +0000 UTC m=+10.724602521, 105.4464ms after stability -2025-06-03T23:33:15.703Z [INFO] server-87dd7580-3266-83ec-7f8c-d5de941fffaf: aborting pipeline replication: peer="{Voter server-829348be-9c67-2d5d-7a4f-e92746be9b9a 829348be-9c67-2d5d-7a4f-e92746be9b9a}" ---- PASS: TestRaft_RemoveLeader (0.60s) +2025-06-03T23:36:14.360Z [INFO] server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa Address:5963a783-b94f-9c4d-c3bf-3f956f6c74fa} {Suffrage:Voter ID:server-90b7c997-28d1-7cff-3cad-3d5e969b39d1 Address:90b7c997-28d1-7cff-3cad-3d5e969b39d1} {Suffrage:Voter ID:server-3b2a0c0e-bd65-bf28-da92-a2b574a4046f Address:3b2a0c0e-bd65-bf28-da92-a2b574a4046f}]" +2025-06-03T23:36:14.361Z [INFO] server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa: entering follower state: follower="Node at 5963a783-b94f-9c4d-c3bf-3f956f6c74fa [Follower]" leader-address= leader-id= +2025-06-03T23:36:14.361Z [INFO] server-90b7c997-28d1-7cff-3cad-3d5e969b39d1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa Address:5963a783-b94f-9c4d-c3bf-3f956f6c74fa} {Suffrage:Voter ID:server-90b7c997-28d1-7cff-3cad-3d5e969b39d1 Address:90b7c997-28d1-7cff-3cad-3d5e969b39d1} {Suffrage:Voter ID:server-3b2a0c0e-bd65-bf28-da92-a2b574a4046f Address:3b2a0c0e-bd65-bf28-da92-a2b574a4046f}]" +2025-06-03T23:36:14.361Z [INFO] server-90b7c997-28d1-7cff-3cad-3d5e969b39d1: entering follower state: follower="Node at 90b7c997-28d1-7cff-3cad-3d5e969b39d1 [Follower]" leader-address= leader-id= +2025-06-03T23:36:14.361Z [INFO] server-3b2a0c0e-bd65-bf28-da92-a2b574a4046f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa Address:5963a783-b94f-9c4d-c3bf-3f956f6c74fa} {Suffrage:Voter ID:server-90b7c997-28d1-7cff-3cad-3d5e969b39d1 Address:90b7c997-28d1-7cff-3cad-3d5e969b39d1} {Suffrage:Voter ID:server-3b2a0c0e-bd65-bf28-da92-a2b574a4046f Address:3b2a0c0e-bd65-bf28-da92-a2b574a4046f}]" +2025-06-03T23:36:14.361Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:14.361Z [INFO] server-3b2a0c0e-bd65-bf28-da92-a2b574a4046f: entering follower state: follower="Node at 3b2a0c0e-bd65-bf28-da92-a2b574a4046f [Follower]" leader-address= leader-id= +2025-06-03T23:36:14.445Z [WARN] server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:14.445Z [INFO] server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa: entering candidate state: node="Node at 5963a783-b94f-9c4d-c3bf-3f956f6c74fa [Candidate]" term=2 +2025-06-03T23:36:14.445Z [INFO] server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa: election won: term=2 tally=2 +2025-06-03T23:36:14.445Z [INFO] server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa: entering leader state: leader="Node at 5963a783-b94f-9c4d-c3bf-3f956f6c74fa [Leader]" +2025-06-03T23:36:14.445Z [INFO] server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa: added peer, starting replication: peer=server-90b7c997-28d1-7cff-3cad-3d5e969b39d1 +2025-06-03T23:36:14.445Z [INFO] server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa: added peer, starting replication: peer=server-3b2a0c0e-bd65-bf28-da92-a2b574a4046f +2025-06-03T23:36:14.445Z [INFO] server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa: pipelining replication: peer="{Voter server-3b2a0c0e-bd65-bf28-da92-a2b574a4046f 3b2a0c0e-bd65-bf28-da92-a2b574a4046f}" +2025-06-03T23:36:14.445Z [INFO] server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa: pipelining replication: peer="{Voter server-90b7c997-28d1-7cff-3cad-3d5e969b39d1 90b7c997-28d1-7cff-3cad-3d5e969b39d1}" +2025-06-03T23:36:14.551Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:14.445721231 +0000 UTC m=+9.767476161 (1 nodes), 84.34132ms from start of poll, 84.95972ms from cluster start. Timeout at 2025-06-03 23:36:14.551450071 +0000 UTC m=+9.873205001, 105.72884ms after stability +2025-06-03T23:36:14.557Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:14.662Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:14.557221351 +0000 UTC m=+9.878976321 (2 nodes), 1.72µs from start of poll, 196.45988ms from cluster start. Timeout at 2025-06-03 23:36:14.662328671 +0000 UTC m=+9.984083601, 105.10728ms after stability +2025-06-03T23:36:14.662Z [INFO] server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa: updating configuration: command=RemoveServer server-id=server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa server-addr= servers="[{Suffrage:Voter ID:server-90b7c997-28d1-7cff-3cad-3d5e969b39d1 Address:90b7c997-28d1-7cff-3cad-3d5e969b39d1} {Suffrage:Voter ID:server-3b2a0c0e-bd65-bf28-da92-a2b574a4046f Address:3b2a0c0e-bd65-bf28-da92-a2b574a4046f}]" +2025-06-03T23:36:14.662Z [INFO] server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa: removed ourself, shutting down +2025-06-03T23:36:14.662Z [INFO] server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa: aborting pipeline replication: peer="{Voter server-3b2a0c0e-bd65-bf28-da92-a2b574a4046f 3b2a0c0e-bd65-bf28-da92-a2b574a4046f}" +2025-06-03T23:36:14.662Z [INFO] server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa: aborting pipeline replication: peer="{Voter server-90b7c997-28d1-7cff-3cad-3d5e969b39d1 90b7c997-28d1-7cff-3cad-3d5e969b39d1}" +2025-06-03T23:36:14.748Z [WARN] server-90b7c997-28d1-7cff-3cad-3d5e969b39d1: heartbeat timeout reached, starting election: last-leader-addr=5963a783-b94f-9c4d-c3bf-3f956f6c74fa last-leader-id=server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa +2025-06-03T23:36:14.748Z [INFO] server-90b7c997-28d1-7cff-3cad-3d5e969b39d1: entering candidate state: node="Node at 90b7c997-28d1-7cff-3cad-3d5e969b39d1 [Candidate]" term=3 +2025-06-03T23:36:14.749Z [WARN] server-3b2a0c0e-bd65-bf28-da92-a2b574a4046f: rejecting vote request since we have a leader: from=90b7c997-28d1-7cff-3cad-3d5e969b39d1 leader=5963a783-b94f-9c4d-c3bf-3f956f6c74fa leader-id=server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa +2025-06-03T23:36:14.760Z [WARN] server-3b2a0c0e-bd65-bf28-da92-a2b574a4046f: heartbeat timeout reached, starting election: last-leader-addr=5963a783-b94f-9c4d-c3bf-3f956f6c74fa last-leader-id=server-5963a783-b94f-9c4d-c3bf-3f956f6c74fa +2025-06-03T23:36:14.760Z [INFO] server-3b2a0c0e-bd65-bf28-da92-a2b574a4046f: entering candidate state: node="Node at 3b2a0c0e-bd65-bf28-da92-a2b574a4046f [Candidate]" term=3 +2025-06-03T23:36:14.760Z [INFO] server-90b7c997-28d1-7cff-3cad-3d5e969b39d1: duplicate requestVote for same term: term=3 +2025-06-03T23:36:14.821Z [WARN] server-90b7c997-28d1-7cff-3cad-3d5e969b39d1: Election timeout reached, restarting election +2025-06-03T23:36:14.821Z [INFO] server-90b7c997-28d1-7cff-3cad-3d5e969b39d1: entering candidate state: node="Node at 90b7c997-28d1-7cff-3cad-3d5e969b39d1 [Candidate]" term=4 +2025-06-03T23:36:14.821Z [INFO] server-3b2a0c0e-bd65-bf28-da92-a2b574a4046f: entering follower state: follower="Node at 3b2a0c0e-bd65-bf28-da92-a2b574a4046f [Follower]" leader-address= leader-id= +2025-06-03T23:36:14.821Z [INFO] server-90b7c997-28d1-7cff-3cad-3d5e969b39d1: election won: term=4 tally=2 +2025-06-03T23:36:14.821Z [INFO] server-90b7c997-28d1-7cff-3cad-3d5e969b39d1: entering leader state: leader="Node at 90b7c997-28d1-7cff-3cad-3d5e969b39d1 [Leader]" +2025-06-03T23:36:14.821Z [INFO] server-90b7c997-28d1-7cff-3cad-3d5e969b39d1: added peer, starting replication: peer=server-3b2a0c0e-bd65-bf28-da92-a2b574a4046f +2025-06-03T23:36:14.822Z [INFO] server-90b7c997-28d1-7cff-3cad-3d5e969b39d1: pipelining replication: peer="{Voter server-3b2a0c0e-bd65-bf28-da92-a2b574a4046f 3b2a0c0e-bd65-bf28-da92-a2b574a4046f}" +2025-06-03T23:36:14.874Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:14.979Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:14.874390671 +0000 UTC m=+10.196145601 (1 nodes), 62.96µs from start of poll, 513.62916ms from cluster start. Timeout at 2025-06-03 23:36:14.979676031 +0000 UTC m=+10.301430961, 105.28536ms after stability +2025-06-03T23:36:14.979Z [INFO] server-90b7c997-28d1-7cff-3cad-3d5e969b39d1: aborting pipeline replication: peer="{Voter server-3b2a0c0e-bd65-bf28-da92-a2b574a4046f 3b2a0c0e-bd65-bf28-da92-a2b574a4046f}" +--- PASS: TestRaft_RemoveLeader (0.62s) === RUN TestRaft_RemoveLeader_NoShutdown -2025-06-03T23:33:15.704Z [INFO] server-690d21e1-1210-0769-54d5-fa0f326febf6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-690d21e1-1210-0769-54d5-fa0f326febf6 Address:690d21e1-1210-0769-54d5-fa0f326febf6} {Suffrage:Voter ID:server-21ca567d-7996-108e-a347-01c5744b0d86 Address:21ca567d-7996-108e-a347-01c5744b0d86} {Suffrage:Voter ID:server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31 Address:62f38a3b-1bcd-2688-96a7-3e08b5e65e31}]" -2025-06-03T23:33:15.704Z [INFO] server-690d21e1-1210-0769-54d5-fa0f326febf6: entering follower state: follower="Node at 690d21e1-1210-0769-54d5-fa0f326febf6 [Follower]" leader-address= leader-id= -2025-06-03T23:33:15.704Z [INFO] server-21ca567d-7996-108e-a347-01c5744b0d86: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-690d21e1-1210-0769-54d5-fa0f326febf6 Address:690d21e1-1210-0769-54d5-fa0f326febf6} {Suffrage:Voter ID:server-21ca567d-7996-108e-a347-01c5744b0d86 Address:21ca567d-7996-108e-a347-01c5744b0d86} {Suffrage:Voter ID:server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31 Address:62f38a3b-1bcd-2688-96a7-3e08b5e65e31}]" -2025-06-03T23:33:15.705Z [INFO] server-21ca567d-7996-108e-a347-01c5744b0d86: entering follower state: follower="Node at 21ca567d-7996-108e-a347-01c5744b0d86 [Follower]" leader-address= leader-id= -2025-06-03T23:33:15.705Z [INFO] server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-690d21e1-1210-0769-54d5-fa0f326febf6 Address:690d21e1-1210-0769-54d5-fa0f326febf6} {Suffrage:Voter ID:server-21ca567d-7996-108e-a347-01c5744b0d86 Address:21ca567d-7996-108e-a347-01c5744b0d86} {Suffrage:Voter ID:server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31 Address:62f38a3b-1bcd-2688-96a7-3e08b5e65e31}]" -2025-06-03T23:33:15.705Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:15.705Z [INFO] server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31: entering follower state: follower="Node at 62f38a3b-1bcd-2688-96a7-3e08b5e65e31 [Follower]" leader-address= leader-id= -2025-06-03T23:33:15.756Z [WARN] server-21ca567d-7996-108e-a347-01c5744b0d86: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:15.756Z [INFO] server-21ca567d-7996-108e-a347-01c5744b0d86: entering candidate state: node="Node at 21ca567d-7996-108e-a347-01c5744b0d86 [Candidate]" term=2 -2025-06-03T23:33:15.757Z [INFO] server-21ca567d-7996-108e-a347-01c5744b0d86: election won: term=2 tally=2 -2025-06-03T23:33:15.757Z [INFO] server-21ca567d-7996-108e-a347-01c5744b0d86: entering leader state: leader="Node at 21ca567d-7996-108e-a347-01c5744b0d86 [Leader]" -2025-06-03T23:33:15.757Z [INFO] server-21ca567d-7996-108e-a347-01c5744b0d86: added peer, starting replication: peer=server-690d21e1-1210-0769-54d5-fa0f326febf6 -2025-06-03T23:33:15.757Z [INFO] server-21ca567d-7996-108e-a347-01c5744b0d86: added peer, starting replication: peer=server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31 -2025-06-03T23:33:15.757Z [INFO] server-21ca567d-7996-108e-a347-01c5744b0d86: pipelining replication: peer="{Voter server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31 62f38a3b-1bcd-2688-96a7-3e08b5e65e31}" -2025-06-03T23:33:15.757Z [INFO] server-21ca567d-7996-108e-a347-01c5744b0d86: pipelining replication: peer="{Voter server-690d21e1-1210-0769-54d5-fa0f326febf6 690d21e1-1210-0769-54d5-fa0f326febf6}" -2025-06-03T23:33:15.863Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:15.757156831 +0000 UTC m=+10.777976201 (2 nodes), 51.96972ms from start of poll, 52.45792ms from cluster start. Timeout at 2025-06-03 23:33:15.863156831 +0000 UTC m=+10.883976201, 106ms after stability -2025-06-03T23:33:15.863Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:15.969Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:15.863272391 +0000 UTC m=+10.884091761 (1 nodes), 1.04µs from start of poll, 158.57348ms from cluster start. Timeout at 2025-06-03 23:33:15.969056911 +0000 UTC m=+10.989876281, 105.78452ms after stability -2025-06-03T23:33:15.973Z [INFO] server-21ca567d-7996-108e-a347-01c5744b0d86: updating configuration: command=RemoveServer server-id=server-21ca567d-7996-108e-a347-01c5744b0d86 server-addr= servers="[{Suffrage:Voter ID:server-690d21e1-1210-0769-54d5-fa0f326febf6 Address:690d21e1-1210-0769-54d5-fa0f326febf6} {Suffrage:Voter ID:server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31 Address:62f38a3b-1bcd-2688-96a7-3e08b5e65e31}]" -2025-06-03T23:33:15.974Z [INFO] server-21ca567d-7996-108e-a347-01c5744b0d86: removed ourself, transitioning to follower -2025-06-03T23:33:15.974Z [INFO] server-21ca567d-7996-108e-a347-01c5744b0d86: entering follower state: follower="Node at 21ca567d-7996-108e-a347-01c5744b0d86 [Follower]" leader-address= leader-id= -2025-06-03T23:33:15.974Z [INFO] server-21ca567d-7996-108e-a347-01c5744b0d86: aborting pipeline replication: peer="{Voter server-690d21e1-1210-0769-54d5-fa0f326febf6 690d21e1-1210-0769-54d5-fa0f326febf6}" -2025-06-03T23:33:15.974Z [INFO] server-21ca567d-7996-108e-a347-01c5744b0d86: aborting pipeline replication: peer="{Voter server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31 62f38a3b-1bcd-2688-96a7-3e08b5e65e31}" -2025-06-03T23:33:16.053Z [WARN] server-690d21e1-1210-0769-54d5-fa0f326febf6: heartbeat timeout reached, starting election: last-leader-addr=21ca567d-7996-108e-a347-01c5744b0d86 last-leader-id=server-21ca567d-7996-108e-a347-01c5744b0d86 -2025-06-03T23:33:16.053Z [INFO] server-690d21e1-1210-0769-54d5-fa0f326febf6: entering candidate state: node="Node at 690d21e1-1210-0769-54d5-fa0f326febf6 [Candidate]" term=3 -2025-06-03T23:33:16.053Z [WARN] server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31: rejecting vote request since we have a leader: from=690d21e1-1210-0769-54d5-fa0f326febf6 leader=21ca567d-7996-108e-a347-01c5744b0d86 leader-id=server-21ca567d-7996-108e-a347-01c5744b0d86 -2025-06-03T23:33:16.053Z [WARN] server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31: heartbeat timeout reached, starting election: last-leader-addr=21ca567d-7996-108e-a347-01c5744b0d86 last-leader-id=server-21ca567d-7996-108e-a347-01c5744b0d86 -2025-06-03T23:33:16.053Z [INFO] server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31: entering candidate state: node="Node at 62f38a3b-1bcd-2688-96a7-3e08b5e65e31 [Candidate]" term=3 -2025-06-03T23:33:16.053Z [INFO] server-690d21e1-1210-0769-54d5-fa0f326febf6: duplicate requestVote for same term: term=3 -2025-06-03T23:33:16.057Z [WARN] server-21ca567d-7996-108e-a347-01c5744b0d86: not part of stable configuration, aborting election -2025-06-03T23:33:16.080Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:16.128Z [WARN] server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31: Election timeout reached, restarting election -2025-06-03T23:33:16.128Z [INFO] server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31: entering candidate state: node="Node at 62f38a3b-1bcd-2688-96a7-3e08b5e65e31 [Candidate]" term=4 -2025-06-03T23:33:16.128Z [INFO] server-690d21e1-1210-0769-54d5-fa0f326febf6: entering follower state: follower="Node at 690d21e1-1210-0769-54d5-fa0f326febf6 [Follower]" leader-address= leader-id= -2025-06-03T23:33:16.128Z [INFO] server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31: election won: term=4 tally=2 -2025-06-03T23:33:16.128Z [INFO] server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31: entering leader state: leader="Node at 62f38a3b-1bcd-2688-96a7-3e08b5e65e31 [Leader]" -2025-06-03T23:33:16.128Z [INFO] server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31: added peer, starting replication: peer=server-690d21e1-1210-0769-54d5-fa0f326febf6 -2025-06-03T23:33:16.128Z [INFO] server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31: pipelining replication: peer="{Voter server-690d21e1-1210-0769-54d5-fa0f326febf6 690d21e1-1210-0769-54d5-fa0f326febf6}" -2025-06-03T23:33:16.234Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:16.128300231 +0000 UTC m=+11.149119641 (1 nodes), 48.12404ms from start of poll, 423.60136ms from cluster start. Timeout at 2025-06-03 23:33:16.233977231 +0000 UTC m=+11.254796601, 105.67696ms after stability -2025-06-03T23:33:16.340Z [INFO] server-62f38a3b-1bcd-2688-96a7-3e08b5e65e31: aborting pipeline replication: peer="{Voter server-690d21e1-1210-0769-54d5-fa0f326febf6 690d21e1-1210-0769-54d5-fa0f326febf6}" ---- PASS: TestRaft_RemoveLeader_NoShutdown (0.64s) +2025-06-03T23:36:14.980Z [INFO] server-b306b72d-17c6-955a-dcd7-5b10973405ef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b306b72d-17c6-955a-dcd7-5b10973405ef Address:b306b72d-17c6-955a-dcd7-5b10973405ef} {Suffrage:Voter ID:server-348046da-3993-18b5-f889-ee84f9cf6b22 Address:348046da-3993-18b5-f889-ee84f9cf6b22} {Suffrage:Voter ID:server-5aad4832-8aa9-9ead-9522-a02784c432ab Address:5aad4832-8aa9-9ead-9522-a02784c432ab}]" +2025-06-03T23:36:14.980Z [INFO] server-b306b72d-17c6-955a-dcd7-5b10973405ef: entering follower state: follower="Node at b306b72d-17c6-955a-dcd7-5b10973405ef [Follower]" leader-address= leader-id= +2025-06-03T23:36:14.980Z [INFO] server-348046da-3993-18b5-f889-ee84f9cf6b22: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b306b72d-17c6-955a-dcd7-5b10973405ef Address:b306b72d-17c6-955a-dcd7-5b10973405ef} {Suffrage:Voter ID:server-348046da-3993-18b5-f889-ee84f9cf6b22 Address:348046da-3993-18b5-f889-ee84f9cf6b22} {Suffrage:Voter ID:server-5aad4832-8aa9-9ead-9522-a02784c432ab Address:5aad4832-8aa9-9ead-9522-a02784c432ab}]" +2025-06-03T23:36:14.980Z [INFO] server-348046da-3993-18b5-f889-ee84f9cf6b22: entering follower state: follower="Node at 348046da-3993-18b5-f889-ee84f9cf6b22 [Follower]" leader-address= leader-id= +2025-06-03T23:36:14.980Z [INFO] server-5aad4832-8aa9-9ead-9522-a02784c432ab: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b306b72d-17c6-955a-dcd7-5b10973405ef Address:b306b72d-17c6-955a-dcd7-5b10973405ef} {Suffrage:Voter ID:server-348046da-3993-18b5-f889-ee84f9cf6b22 Address:348046da-3993-18b5-f889-ee84f9cf6b22} {Suffrage:Voter ID:server-5aad4832-8aa9-9ead-9522-a02784c432ab Address:5aad4832-8aa9-9ead-9522-a02784c432ab}]" +2025-06-03T23:36:14.980Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:14.981Z [INFO] server-5aad4832-8aa9-9ead-9522-a02784c432ab: entering follower state: follower="Node at 5aad4832-8aa9-9ead-9522-a02784c432ab [Follower]" leader-address= leader-id= +2025-06-03T23:36:15.036Z [WARN] server-b306b72d-17c6-955a-dcd7-5b10973405ef: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:15.036Z [INFO] server-b306b72d-17c6-955a-dcd7-5b10973405ef: entering candidate state: node="Node at b306b72d-17c6-955a-dcd7-5b10973405ef [Candidate]" term=2 +2025-06-03T23:36:15.036Z [INFO] server-b306b72d-17c6-955a-dcd7-5b10973405ef: election won: term=2 tally=2 +2025-06-03T23:36:15.036Z [INFO] server-b306b72d-17c6-955a-dcd7-5b10973405ef: entering leader state: leader="Node at b306b72d-17c6-955a-dcd7-5b10973405ef [Leader]" +2025-06-03T23:36:15.036Z [INFO] server-b306b72d-17c6-955a-dcd7-5b10973405ef: added peer, starting replication: peer=server-348046da-3993-18b5-f889-ee84f9cf6b22 +2025-06-03T23:36:15.036Z [INFO] server-b306b72d-17c6-955a-dcd7-5b10973405ef: added peer, starting replication: peer=server-5aad4832-8aa9-9ead-9522-a02784c432ab +2025-06-03T23:36:15.036Z [INFO] server-b306b72d-17c6-955a-dcd7-5b10973405ef: pipelining replication: peer="{Voter server-5aad4832-8aa9-9ead-9522-a02784c432ab 5aad4832-8aa9-9ead-9522-a02784c432ab}" +2025-06-03T23:36:15.036Z [INFO] server-b306b72d-17c6-955a-dcd7-5b10973405ef: pipelining replication: peer="{Voter server-348046da-3993-18b5-f889-ee84f9cf6b22 348046da-3993-18b5-f889-ee84f9cf6b22}" +2025-06-03T23:36:15.142Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:15.036553431 +0000 UTC m=+10.358308361 (2 nodes), 55.54576ms from start of poll, 56.02936ms from cluster start. Timeout at 2025-06-03 23:36:15.142358311 +0000 UTC m=+10.464113241, 105.80488ms after stability +2025-06-03T23:36:15.142Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:15.248Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:15.142470271 +0000 UTC m=+10.464225201 (1 nodes), 2.92µs from start of poll, 161.9462ms from cluster start. Timeout at 2025-06-03 23:36:15.247966431 +0000 UTC m=+10.569721361, 105.49616ms after stability +2025-06-03T23:36:15.252Z [INFO] server-b306b72d-17c6-955a-dcd7-5b10973405ef: updating configuration: command=RemoveServer server-id=server-b306b72d-17c6-955a-dcd7-5b10973405ef server-addr= servers="[{Suffrage:Voter ID:server-348046da-3993-18b5-f889-ee84f9cf6b22 Address:348046da-3993-18b5-f889-ee84f9cf6b22} {Suffrage:Voter ID:server-5aad4832-8aa9-9ead-9522-a02784c432ab Address:5aad4832-8aa9-9ead-9522-a02784c432ab}]" +2025-06-03T23:36:15.253Z [INFO] server-b306b72d-17c6-955a-dcd7-5b10973405ef: removed ourself, transitioning to follower +2025-06-03T23:36:15.253Z [INFO] server-b306b72d-17c6-955a-dcd7-5b10973405ef: entering follower state: follower="Node at b306b72d-17c6-955a-dcd7-5b10973405ef [Follower]" leader-address= leader-id= +2025-06-03T23:36:15.253Z [INFO] server-b306b72d-17c6-955a-dcd7-5b10973405ef: aborting pipeline replication: peer="{Voter server-5aad4832-8aa9-9ead-9522-a02784c432ab 5aad4832-8aa9-9ead-9522-a02784c432ab}" +2025-06-03T23:36:15.253Z [INFO] server-b306b72d-17c6-955a-dcd7-5b10973405ef: aborting pipeline replication: peer="{Voter server-348046da-3993-18b5-f889-ee84f9cf6b22 348046da-3993-18b5-f889-ee84f9cf6b22}" +2025-06-03T23:36:15.330Z [WARN] server-348046da-3993-18b5-f889-ee84f9cf6b22: heartbeat timeout reached, starting election: last-leader-addr=b306b72d-17c6-955a-dcd7-5b10973405ef last-leader-id=server-b306b72d-17c6-955a-dcd7-5b10973405ef +2025-06-03T23:36:15.330Z [INFO] server-348046da-3993-18b5-f889-ee84f9cf6b22: entering candidate state: node="Node at 348046da-3993-18b5-f889-ee84f9cf6b22 [Candidate]" term=3 +2025-06-03T23:36:15.330Z [WARN] server-5aad4832-8aa9-9ead-9522-a02784c432ab: rejecting vote request since we have a leader: from=348046da-3993-18b5-f889-ee84f9cf6b22 leader=b306b72d-17c6-955a-dcd7-5b10973405ef leader-id=server-b306b72d-17c6-955a-dcd7-5b10973405ef +2025-06-03T23:36:15.332Z [WARN] server-b306b72d-17c6-955a-dcd7-5b10973405ef: not part of stable configuration, aborting election +2025-06-03T23:36:15.338Z [WARN] server-5aad4832-8aa9-9ead-9522-a02784c432ab: heartbeat timeout reached, starting election: last-leader-addr=b306b72d-17c6-955a-dcd7-5b10973405ef last-leader-id=server-b306b72d-17c6-955a-dcd7-5b10973405ef +2025-06-03T23:36:15.338Z [INFO] server-5aad4832-8aa9-9ead-9522-a02784c432ab: entering candidate state: node="Node at 5aad4832-8aa9-9ead-9522-a02784c432ab [Candidate]" term=3 +2025-06-03T23:36:15.338Z [INFO] server-348046da-3993-18b5-f889-ee84f9cf6b22: duplicate requestVote for same term: term=3 +2025-06-03T23:36:15.359Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:15.388Z [WARN] server-348046da-3993-18b5-f889-ee84f9cf6b22: Election timeout reached, restarting election +2025-06-03T23:36:15.388Z [INFO] server-348046da-3993-18b5-f889-ee84f9cf6b22: entering candidate state: node="Node at 348046da-3993-18b5-f889-ee84f9cf6b22 [Candidate]" term=4 +2025-06-03T23:36:15.388Z [INFO] server-5aad4832-8aa9-9ead-9522-a02784c432ab: entering follower state: follower="Node at 5aad4832-8aa9-9ead-9522-a02784c432ab [Follower]" leader-address= leader-id= +2025-06-03T23:36:15.388Z [INFO] server-348046da-3993-18b5-f889-ee84f9cf6b22: election won: term=4 tally=2 +2025-06-03T23:36:15.388Z [INFO] server-348046da-3993-18b5-f889-ee84f9cf6b22: entering leader state: leader="Node at 348046da-3993-18b5-f889-ee84f9cf6b22 [Leader]" +2025-06-03T23:36:15.388Z [INFO] server-348046da-3993-18b5-f889-ee84f9cf6b22: added peer, starting replication: peer=server-5aad4832-8aa9-9ead-9522-a02784c432ab +2025-06-03T23:36:15.389Z [INFO] server-348046da-3993-18b5-f889-ee84f9cf6b22: pipelining replication: peer="{Voter server-5aad4832-8aa9-9ead-9522-a02784c432ab 5aad4832-8aa9-9ead-9522-a02784c432ab}" +2025-06-03T23:36:15.494Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:15.388963791 +0000 UTC m=+10.710718721 (1 nodes), 29.66068ms from start of poll, 408.43972ms from cluster start. Timeout at 2025-06-03 23:36:15.494136311 +0000 UTC m=+10.815891241, 105.17252ms after stability +2025-06-03T23:36:15.600Z [INFO] server-348046da-3993-18b5-f889-ee84f9cf6b22: aborting pipeline replication: peer="{Voter server-5aad4832-8aa9-9ead-9522-a02784c432ab 5aad4832-8aa9-9ead-9522-a02784c432ab}" +--- PASS: TestRaft_RemoveLeader_NoShutdown (0.62s) === RUN TestRaft_RemoveFollower_SplitCluster -2025-06-03T23:33:16.341Z [INFO] server-ba297ef7-9ed9-35f6-f0c6-a6ce330e954a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba297ef7-9ed9-35f6-f0c6-a6ce330e954a Address:ba297ef7-9ed9-35f6-f0c6-a6ce330e954a} {Suffrage:Voter ID:server-8b30d35b-8644-e8a2-5f27-686f363f727b Address:8b30d35b-8644-e8a2-5f27-686f363f727b} {Suffrage:Voter ID:server-6e26e93a-659c-a93a-fde3-919014e9afda Address:6e26e93a-659c-a93a-fde3-919014e9afda} {Suffrage:Voter ID:server-59b1261d-ddad-f198-393b-bcf9017f9e5d Address:59b1261d-ddad-f198-393b-bcf9017f9e5d}]" -2025-06-03T23:33:16.342Z [INFO] server-ba297ef7-9ed9-35f6-f0c6-a6ce330e954a: entering follower state: follower="Node at ba297ef7-9ed9-35f6-f0c6-a6ce330e954a [Follower]" leader-address= leader-id= -2025-06-03T23:33:16.342Z [INFO] server-8b30d35b-8644-e8a2-5f27-686f363f727b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba297ef7-9ed9-35f6-f0c6-a6ce330e954a Address:ba297ef7-9ed9-35f6-f0c6-a6ce330e954a} {Suffrage:Voter ID:server-8b30d35b-8644-e8a2-5f27-686f363f727b Address:8b30d35b-8644-e8a2-5f27-686f363f727b} {Suffrage:Voter ID:server-6e26e93a-659c-a93a-fde3-919014e9afda Address:6e26e93a-659c-a93a-fde3-919014e9afda} {Suffrage:Voter ID:server-59b1261d-ddad-f198-393b-bcf9017f9e5d Address:59b1261d-ddad-f198-393b-bcf9017f9e5d}]" -2025-06-03T23:33:16.342Z [INFO] server-8b30d35b-8644-e8a2-5f27-686f363f727b: entering follower state: follower="Node at 8b30d35b-8644-e8a2-5f27-686f363f727b [Follower]" leader-address= leader-id= -2025-06-03T23:33:16.342Z [INFO] server-6e26e93a-659c-a93a-fde3-919014e9afda: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba297ef7-9ed9-35f6-f0c6-a6ce330e954a Address:ba297ef7-9ed9-35f6-f0c6-a6ce330e954a} {Suffrage:Voter ID:server-8b30d35b-8644-e8a2-5f27-686f363f727b Address:8b30d35b-8644-e8a2-5f27-686f363f727b} {Suffrage:Voter ID:server-6e26e93a-659c-a93a-fde3-919014e9afda Address:6e26e93a-659c-a93a-fde3-919014e9afda} {Suffrage:Voter ID:server-59b1261d-ddad-f198-393b-bcf9017f9e5d Address:59b1261d-ddad-f198-393b-bcf9017f9e5d}]" -2025-06-03T23:33:16.342Z [INFO] server-6e26e93a-659c-a93a-fde3-919014e9afda: entering follower state: follower="Node at 6e26e93a-659c-a93a-fde3-919014e9afda [Follower]" leader-address= leader-id= -2025-06-03T23:33:16.342Z [INFO] server-59b1261d-ddad-f198-393b-bcf9017f9e5d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba297ef7-9ed9-35f6-f0c6-a6ce330e954a Address:ba297ef7-9ed9-35f6-f0c6-a6ce330e954a} {Suffrage:Voter ID:server-8b30d35b-8644-e8a2-5f27-686f363f727b Address:8b30d35b-8644-e8a2-5f27-686f363f727b} {Suffrage:Voter ID:server-6e26e93a-659c-a93a-fde3-919014e9afda Address:6e26e93a-659c-a93a-fde3-919014e9afda} {Suffrage:Voter ID:server-59b1261d-ddad-f198-393b-bcf9017f9e5d Address:59b1261d-ddad-f198-393b-bcf9017f9e5d}]" -2025-06-03T23:33:16.342Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:16.343Z [INFO] server-59b1261d-ddad-f198-393b-bcf9017f9e5d: entering follower state: follower="Node at 59b1261d-ddad-f198-393b-bcf9017f9e5d [Follower]" leader-address= leader-id= -2025-06-03T23:33:16.401Z [WARN] server-6e26e93a-659c-a93a-fde3-919014e9afda: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:16.401Z [INFO] server-6e26e93a-659c-a93a-fde3-919014e9afda: entering candidate state: node="Node at 6e26e93a-659c-a93a-fde3-919014e9afda [Candidate]" term=2 -2025-06-03T23:33:16.401Z [INFO] server-6e26e93a-659c-a93a-fde3-919014e9afda: election won: term=2 tally=3 -2025-06-03T23:33:16.401Z [INFO] server-6e26e93a-659c-a93a-fde3-919014e9afda: entering leader state: leader="Node at 6e26e93a-659c-a93a-fde3-919014e9afda [Leader]" -2025-06-03T23:33:16.401Z [INFO] server-6e26e93a-659c-a93a-fde3-919014e9afda: added peer, starting replication: peer=server-ba297ef7-9ed9-35f6-f0c6-a6ce330e954a -2025-06-03T23:33:16.401Z [INFO] server-6e26e93a-659c-a93a-fde3-919014e9afda: added peer, starting replication: peer=server-8b30d35b-8644-e8a2-5f27-686f363f727b -2025-06-03T23:33:16.401Z [INFO] server-6e26e93a-659c-a93a-fde3-919014e9afda: added peer, starting replication: peer=server-59b1261d-ddad-f198-393b-bcf9017f9e5d -2025-06-03T23:33:16.401Z [INFO] server-6e26e93a-659c-a93a-fde3-919014e9afda: pipelining replication: peer="{Voter server-59b1261d-ddad-f198-393b-bcf9017f9e5d 59b1261d-ddad-f198-393b-bcf9017f9e5d}" -2025-06-03T23:33:16.401Z [INFO] server-6e26e93a-659c-a93a-fde3-919014e9afda: pipelining replication: peer="{Voter server-8b30d35b-8644-e8a2-5f27-686f363f727b 8b30d35b-8644-e8a2-5f27-686f363f727b}" -2025-06-03T23:33:16.401Z [INFO] server-6e26e93a-659c-a93a-fde3-919014e9afda: pipelining replication: peer="{Voter server-ba297ef7-9ed9-35f6-f0c6-a6ce330e954a ba297ef7-9ed9-35f6-f0c6-a6ce330e954a}" -2025-06-03T23:33:16.506Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:16.401380591 +0000 UTC m=+11.422199961 (1 nodes), 58.3808ms from start of poll, 60.45036ms from cluster start. Timeout at 2025-06-03 23:33:16.506654431 +0000 UTC m=+11.527473801, 105.27384ms after stability -2025-06-03T23:33:16.612Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:16.718Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:16.612882471 +0000 UTC m=+11.633701841 (3 nodes), 1.76µs from start of poll, 271.95224ms from cluster start. Timeout at 2025-06-03 23:33:16.718605951 +0000 UTC m=+11.739425281, 105.72344ms after stability -2025-06-03T23:33:16.718Z [INFO] server-6e26e93a-659c-a93a-fde3-919014e9afda: updating configuration: command=RemoveServer server-id=server-59b1261d-ddad-f198-393b-bcf9017f9e5d server-addr= servers="[{Suffrage:Voter ID:server-ba297ef7-9ed9-35f6-f0c6-a6ce330e954a Address:ba297ef7-9ed9-35f6-f0c6-a6ce330e954a} {Suffrage:Voter ID:server-8b30d35b-8644-e8a2-5f27-686f363f727b Address:8b30d35b-8644-e8a2-5f27-686f363f727b} {Suffrage:Voter ID:server-6e26e93a-659c-a93a-fde3-919014e9afda Address:6e26e93a-659c-a93a-fde3-919014e9afda}]" -2025-06-03T23:33:16.718Z [INFO] server-6e26e93a-659c-a93a-fde3-919014e9afda: removed peer, stopping replication: peer=server-59b1261d-ddad-f198-393b-bcf9017f9e5d last-index=3 -2025-06-03T23:33:16.718Z [ERROR] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to pipeline appendEntries: peer="{Voter server-ba297ef7-9ed9-35f6-f0c6-a6ce330e954a ba297ef7-9ed9-35f6-f0c6-a6ce330e954a}" error="append pipeline closed" -2025-06-03T23:33:16.718Z [INFO] server-6e26e93a-659c-a93a-fde3-919014e9afda: aborting pipeline replication: peer="{Voter server-ba297ef7-9ed9-35f6-f0c6-a6ce330e954a ba297ef7-9ed9-35f6-f0c6-a6ce330e954a}" -2025-06-03T23:33:16.718Z [ERROR] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to pipeline appendEntries: peer="{Voter server-8b30d35b-8644-e8a2-5f27-686f363f727b 8b30d35b-8644-e8a2-5f27-686f363f727b}" error="append pipeline closed" -2025-06-03T23:33:16.718Z [INFO] server-6e26e93a-659c-a93a-fde3-919014e9afda: aborting pipeline replication: peer="{Voter server-8b30d35b-8644-e8a2-5f27-686f363f727b 8b30d35b-8644-e8a2-5f27-686f363f727b}" -2025-06-03T23:33:16.718Z [ERROR] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to appendEntries to: peer="{Voter server-8b30d35b-8644-e8a2-5f27-686f363f727b 8b30d35b-8644-e8a2-5f27-686f363f727b}" error="failed to connect to peer: 8b30d35b-8644-e8a2-5f27-686f363f727b" -2025-06-03T23:33:16.718Z [INFO] server-6e26e93a-659c-a93a-fde3-919014e9afda: aborting pipeline replication: peer="{Voter server-59b1261d-ddad-f198-393b-bcf9017f9e5d 59b1261d-ddad-f198-393b-bcf9017f9e5d}" -2025-06-03T23:33:16.719Z [ERROR] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to heartbeat to: peer=ba297ef7-9ed9-35f6-f0c6-a6ce330e954a backoff time=10ms error="failed to connect to peer: ba297ef7-9ed9-35f6-f0c6-a6ce330e954a" -2025-06-03T23:33:16.724Z [ERROR] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to heartbeat to: peer=8b30d35b-8644-e8a2-5f27-686f363f727b backoff time=10ms error="failed to connect to peer: 8b30d35b-8644-e8a2-5f27-686f363f727b" -2025-06-03T23:33:16.727Z [ERROR] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to appendEntries to: peer="{Voter server-ba297ef7-9ed9-35f6-f0c6-a6ce330e954a ba297ef7-9ed9-35f6-f0c6-a6ce330e954a}" error="failed to connect to peer: ba297ef7-9ed9-35f6-f0c6-a6ce330e954a" -2025-06-03T23:33:16.738Z [ERROR] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to heartbeat to: peer=ba297ef7-9ed9-35f6-f0c6-a6ce330e954a backoff time=10ms error="failed to connect to peer: ba297ef7-9ed9-35f6-f0c6-a6ce330e954a" -2025-06-03T23:33:16.738Z [ERROR] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to appendEntries to: peer="{Voter server-8b30d35b-8644-e8a2-5f27-686f363f727b 8b30d35b-8644-e8a2-5f27-686f363f727b}" error="failed to connect to peer: 8b30d35b-8644-e8a2-5f27-686f363f727b" -2025-06-03T23:33:16.742Z [ERROR] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to heartbeat to: peer=8b30d35b-8644-e8a2-5f27-686f363f727b backoff time=10ms error="failed to connect to peer: 8b30d35b-8644-e8a2-5f27-686f363f727b" -2025-06-03T23:33:16.749Z [ERROR] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to appendEntries to: peer="{Voter server-ba297ef7-9ed9-35f6-f0c6-a6ce330e954a ba297ef7-9ed9-35f6-f0c6-a6ce330e954a}" error="failed to connect to peer: ba297ef7-9ed9-35f6-f0c6-a6ce330e954a" -2025-06-03T23:33:16.756Z [ERROR] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to heartbeat to: peer=ba297ef7-9ed9-35f6-f0c6-a6ce330e954a backoff time=10ms error="failed to connect to peer: ba297ef7-9ed9-35f6-f0c6-a6ce330e954a" -2025-06-03T23:33:16.757Z [ERROR] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to appendEntries to: peer="{Voter server-8b30d35b-8644-e8a2-5f27-686f363f727b 8b30d35b-8644-e8a2-5f27-686f363f727b}" error="failed to connect to peer: 8b30d35b-8644-e8a2-5f27-686f363f727b" -2025-06-03T23:33:16.763Z [ERROR] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to heartbeat to: peer=8b30d35b-8644-e8a2-5f27-686f363f727b backoff time=10ms error="failed to connect to peer: 8b30d35b-8644-e8a2-5f27-686f363f727b" -2025-06-03T23:33:16.763Z [WARN] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to contact: server-id=server-8b30d35b-8644-e8a2-5f27-686f363f727b time=50.80252ms -2025-06-03T23:33:16.767Z [ERROR] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to appendEntries to: peer="{Voter server-ba297ef7-9ed9-35f6-f0c6-a6ce330e954a ba297ef7-9ed9-35f6-f0c6-a6ce330e954a}" error="failed to connect to peer: ba297ef7-9ed9-35f6-f0c6-a6ce330e954a" -2025-06-03T23:33:16.773Z [ERROR] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to heartbeat to: peer=ba297ef7-9ed9-35f6-f0c6-a6ce330e954a backoff time=20ms error="failed to connect to peer: ba297ef7-9ed9-35f6-f0c6-a6ce330e954a" -2025-06-03T23:33:16.774Z [WARN] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to contact: server-id=server-ba297ef7-9ed9-35f6-f0c6-a6ce330e954a time=55.65588ms -2025-06-03T23:33:16.774Z [WARN] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to contact: server-id=server-8b30d35b-8644-e8a2-5f27-686f363f727b time=61.34884ms -2025-06-03T23:33:16.774Z [WARN] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to contact quorum of nodes, stepping down -2025-06-03T23:33:16.774Z [INFO] server-6e26e93a-659c-a93a-fde3-919014e9afda: entering follower state: follower="Node at 6e26e93a-659c-a93a-fde3-919014e9afda [Follower]" leader-address= leader-id= -2025-06-03T23:33:16.774Z [ERROR] server-6e26e93a-659c-a93a-fde3-919014e9afda: failed to appendEntries to: peer="{Voter server-8b30d35b-8644-e8a2-5f27-686f363f727b 8b30d35b-8644-e8a2-5f27-686f363f727b}" error="failed to connect to peer: 8b30d35b-8644-e8a2-5f27-686f363f727b" +2025-06-03T23:36:15.601Z [INFO] server-b648497d-26b7-aff5-3367-038ae14763e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b648497d-26b7-aff5-3367-038ae14763e9 Address:b648497d-26b7-aff5-3367-038ae14763e9} {Suffrage:Voter ID:server-768bea95-0add-036d-fd9a-d5479b38a325 Address:768bea95-0add-036d-fd9a-d5479b38a325} {Suffrage:Voter ID:server-cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 Address:cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0} {Suffrage:Voter ID:server-da0c8b90-aa9b-7b56-65f5-c687e999918c Address:da0c8b90-aa9b-7b56-65f5-c687e999918c}]" +2025-06-03T23:36:15.601Z [INFO] server-b648497d-26b7-aff5-3367-038ae14763e9: entering follower state: follower="Node at b648497d-26b7-aff5-3367-038ae14763e9 [Follower]" leader-address= leader-id= +2025-06-03T23:36:15.601Z [INFO] server-768bea95-0add-036d-fd9a-d5479b38a325: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b648497d-26b7-aff5-3367-038ae14763e9 Address:b648497d-26b7-aff5-3367-038ae14763e9} {Suffrage:Voter ID:server-768bea95-0add-036d-fd9a-d5479b38a325 Address:768bea95-0add-036d-fd9a-d5479b38a325} {Suffrage:Voter ID:server-cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 Address:cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0} {Suffrage:Voter ID:server-da0c8b90-aa9b-7b56-65f5-c687e999918c Address:da0c8b90-aa9b-7b56-65f5-c687e999918c}]" +2025-06-03T23:36:15.601Z [INFO] server-768bea95-0add-036d-fd9a-d5479b38a325: entering follower state: follower="Node at 768bea95-0add-036d-fd9a-d5479b38a325 [Follower]" leader-address= leader-id= +2025-06-03T23:36:15.601Z [INFO] server-cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b648497d-26b7-aff5-3367-038ae14763e9 Address:b648497d-26b7-aff5-3367-038ae14763e9} {Suffrage:Voter ID:server-768bea95-0add-036d-fd9a-d5479b38a325 Address:768bea95-0add-036d-fd9a-d5479b38a325} {Suffrage:Voter ID:server-cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 Address:cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0} {Suffrage:Voter ID:server-da0c8b90-aa9b-7b56-65f5-c687e999918c Address:da0c8b90-aa9b-7b56-65f5-c687e999918c}]" +2025-06-03T23:36:15.602Z [INFO] server-da0c8b90-aa9b-7b56-65f5-c687e999918c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b648497d-26b7-aff5-3367-038ae14763e9 Address:b648497d-26b7-aff5-3367-038ae14763e9} {Suffrage:Voter ID:server-768bea95-0add-036d-fd9a-d5479b38a325 Address:768bea95-0add-036d-fd9a-d5479b38a325} {Suffrage:Voter ID:server-cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 Address:cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0} {Suffrage:Voter ID:server-da0c8b90-aa9b-7b56-65f5-c687e999918c Address:da0c8b90-aa9b-7b56-65f5-c687e999918c}]" +2025-06-03T23:36:15.602Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:15.602Z [INFO] server-cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0: entering follower state: follower="Node at cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 [Follower]" leader-address= leader-id= +2025-06-03T23:36:15.602Z [INFO] server-da0c8b90-aa9b-7b56-65f5-c687e999918c: entering follower state: follower="Node at da0c8b90-aa9b-7b56-65f5-c687e999918c [Follower]" leader-address= leader-id= +2025-06-03T23:36:15.652Z [WARN] server-768bea95-0add-036d-fd9a-d5479b38a325: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:15.652Z [INFO] server-768bea95-0add-036d-fd9a-d5479b38a325: entering candidate state: node="Node at 768bea95-0add-036d-fd9a-d5479b38a325 [Candidate]" term=2 +2025-06-03T23:36:15.652Z [INFO] server-768bea95-0add-036d-fd9a-d5479b38a325: election won: term=2 tally=3 +2025-06-03T23:36:15.652Z [INFO] server-768bea95-0add-036d-fd9a-d5479b38a325: entering leader state: leader="Node at 768bea95-0add-036d-fd9a-d5479b38a325 [Leader]" +2025-06-03T23:36:15.652Z [INFO] server-768bea95-0add-036d-fd9a-d5479b38a325: added peer, starting replication: peer=server-b648497d-26b7-aff5-3367-038ae14763e9 +2025-06-03T23:36:15.652Z [INFO] server-768bea95-0add-036d-fd9a-d5479b38a325: added peer, starting replication: peer=server-cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 +2025-06-03T23:36:15.652Z [INFO] server-768bea95-0add-036d-fd9a-d5479b38a325: added peer, starting replication: peer=server-da0c8b90-aa9b-7b56-65f5-c687e999918c +2025-06-03T23:36:15.652Z [INFO] server-768bea95-0add-036d-fd9a-d5479b38a325: pipelining replication: peer="{Voter server-b648497d-26b7-aff5-3367-038ae14763e9 b648497d-26b7-aff5-3367-038ae14763e9}" +2025-06-03T23:36:15.652Z [INFO] server-768bea95-0add-036d-fd9a-d5479b38a325: pipelining replication: peer="{Voter server-cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0}" +2025-06-03T23:36:15.653Z [INFO] server-768bea95-0add-036d-fd9a-d5479b38a325: pipelining replication: peer="{Voter server-da0c8b90-aa9b-7b56-65f5-c687e999918c da0c8b90-aa9b-7b56-65f5-c687e999918c}" +2025-06-03T23:36:15.758Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:15.652716911 +0000 UTC m=+10.974471841 (1 nodes), 50.61316ms from start of poll, 51.85156ms from cluster start. Timeout at 2025-06-03 23:36:15.758681031 +0000 UTC m=+11.080436001, 105.96416ms after stability +2025-06-03T23:36:15.864Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:15.970Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:15.864558231 +0000 UTC m=+11.186313161 (3 nodes), 3.72µs from start of poll, 263.69288ms from cluster start. Timeout at 2025-06-03 23:36:15.970588511 +0000 UTC m=+11.292343441, 106.03028ms after stability +2025-06-03T23:36:15.970Z [INFO] server-768bea95-0add-036d-fd9a-d5479b38a325: updating configuration: command=RemoveServer server-id=server-da0c8b90-aa9b-7b56-65f5-c687e999918c server-addr= servers="[{Suffrage:Voter ID:server-b648497d-26b7-aff5-3367-038ae14763e9 Address:b648497d-26b7-aff5-3367-038ae14763e9} {Suffrage:Voter ID:server-768bea95-0add-036d-fd9a-d5479b38a325 Address:768bea95-0add-036d-fd9a-d5479b38a325} {Suffrage:Voter ID:server-cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 Address:cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0}]" +2025-06-03T23:36:15.970Z [INFO] server-768bea95-0add-036d-fd9a-d5479b38a325: removed peer, stopping replication: peer=server-da0c8b90-aa9b-7b56-65f5-c687e999918c last-index=3 +2025-06-03T23:36:15.970Z [ERROR] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to pipeline appendEntries: peer="{Voter server-b648497d-26b7-aff5-3367-038ae14763e9 b648497d-26b7-aff5-3367-038ae14763e9}" error="append pipeline closed" +2025-06-03T23:36:15.970Z [INFO] server-768bea95-0add-036d-fd9a-d5479b38a325: aborting pipeline replication: peer="{Voter server-b648497d-26b7-aff5-3367-038ae14763e9 b648497d-26b7-aff5-3367-038ae14763e9}" +2025-06-03T23:36:15.970Z [INFO] server-768bea95-0add-036d-fd9a-d5479b38a325: aborting pipeline replication: peer="{Voter server-da0c8b90-aa9b-7b56-65f5-c687e999918c da0c8b90-aa9b-7b56-65f5-c687e999918c}" +2025-06-03T23:36:15.970Z [ERROR] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to pipeline appendEntries: peer="{Voter server-cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0}" error="append pipeline closed" +2025-06-03T23:36:15.971Z [INFO] server-768bea95-0add-036d-fd9a-d5479b38a325: aborting pipeline replication: peer="{Voter server-cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0}" +2025-06-03T23:36:15.977Z [ERROR] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to heartbeat to: peer=b648497d-26b7-aff5-3367-038ae14763e9 backoff time=10ms error="failed to connect to peer: b648497d-26b7-aff5-3367-038ae14763e9" +2025-06-03T23:36:15.977Z [ERROR] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to heartbeat to: peer=cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 backoff time=10ms error="failed to connect to peer: cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0" +2025-06-03T23:36:15.977Z [ERROR] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to appendEntries to: peer="{Voter server-cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0}" error="failed to connect to peer: cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0" +2025-06-03T23:36:15.978Z [ERROR] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to appendEntries to: peer="{Voter server-b648497d-26b7-aff5-3367-038ae14763e9 b648497d-26b7-aff5-3367-038ae14763e9}" error="failed to connect to peer: b648497d-26b7-aff5-3367-038ae14763e9" +2025-06-03T23:36:15.994Z [ERROR] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to heartbeat to: peer=b648497d-26b7-aff5-3367-038ae14763e9 backoff time=10ms error="failed to connect to peer: b648497d-26b7-aff5-3367-038ae14763e9" +2025-06-03T23:36:15.995Z [ERROR] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to heartbeat to: peer=cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 backoff time=10ms error="failed to connect to peer: cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0" +2025-06-03T23:36:15.995Z [ERROR] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to appendEntries to: peer="{Voter server-b648497d-26b7-aff5-3367-038ae14763e9 b648497d-26b7-aff5-3367-038ae14763e9}" error="failed to connect to peer: b648497d-26b7-aff5-3367-038ae14763e9" +2025-06-03T23:36:15.997Z [ERROR] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to appendEntries to: peer="{Voter server-cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0}" error="failed to connect to peer: cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0" +2025-06-03T23:36:16.014Z [ERROR] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to heartbeat to: peer=b648497d-26b7-aff5-3367-038ae14763e9 backoff time=10ms error="failed to connect to peer: b648497d-26b7-aff5-3367-038ae14763e9" +2025-06-03T23:36:16.014Z [ERROR] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to heartbeat to: peer=cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 backoff time=10ms error="failed to connect to peer: cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0" +2025-06-03T23:36:16.016Z [ERROR] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to appendEntries to: peer="{Voter server-cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0}" error="failed to connect to peer: cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0" +2025-06-03T23:36:16.017Z [ERROR] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to appendEntries to: peer="{Voter server-b648497d-26b7-aff5-3367-038ae14763e9 b648497d-26b7-aff5-3367-038ae14763e9}" error="failed to connect to peer: b648497d-26b7-aff5-3367-038ae14763e9" +2025-06-03T23:36:16.018Z [WARN] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to contact: server-id=server-cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 time=50.3812ms +2025-06-03T23:36:16.029Z [WARN] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to contact: server-id=server-b648497d-26b7-aff5-3367-038ae14763e9 time=60.10292ms +2025-06-03T23:36:16.029Z [WARN] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to contact: server-id=server-cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 time=61.33976ms +2025-06-03T23:36:16.029Z [WARN] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to contact quorum of nodes, stepping down +2025-06-03T23:36:16.029Z [INFO] server-768bea95-0add-036d-fd9a-d5479b38a325: entering follower state: follower="Node at 768bea95-0add-036d-fd9a-d5479b38a325 [Follower]" leader-address= leader-id= +2025-06-03T23:36:16.029Z [ERROR] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to appendEntries to: peer="{Voter server-cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0 cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0}" error="failed to connect to peer: cbaf3de0-9194-cc5e-2a89-ff1cd29fcbd0" +2025-06-03T23:36:16.029Z [ERROR] server-768bea95-0add-036d-fd9a-d5479b38a325: failed to appendEntries to: peer="{Voter server-b648497d-26b7-aff5-3367-038ae14763e9 b648497d-26b7-aff5-3367-038ae14763e9}" error="failed to connect to peer: b648497d-26b7-aff5-3367-038ae14763e9" --- PASS: TestRaft_RemoveFollower_SplitCluster (0.43s) === RUN TestRaft_AddKnownPeer -2025-06-03T23:33:16.775Z [INFO] server-b1cfb755-4673-2628-6811-684110356638: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b1cfb755-4673-2628-6811-684110356638 Address:b1cfb755-4673-2628-6811-684110356638} {Suffrage:Voter ID:server-a90c8f81-7525-f1f3-d24c-2f9c2f0a65e3 Address:a90c8f81-7525-f1f3-d24c-2f9c2f0a65e3} {Suffrage:Voter ID:server-508b1792-f995-5b14-57a5-6b60bb6ea8a9 Address:508b1792-f995-5b14-57a5-6b60bb6ea8a9}]" -2025-06-03T23:33:16.775Z [INFO] server-b1cfb755-4673-2628-6811-684110356638: entering follower state: follower="Node at b1cfb755-4673-2628-6811-684110356638 [Follower]" leader-address= leader-id= -2025-06-03T23:33:16.775Z [INFO] server-a90c8f81-7525-f1f3-d24c-2f9c2f0a65e3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b1cfb755-4673-2628-6811-684110356638 Address:b1cfb755-4673-2628-6811-684110356638} {Suffrage:Voter ID:server-a90c8f81-7525-f1f3-d24c-2f9c2f0a65e3 Address:a90c8f81-7525-f1f3-d24c-2f9c2f0a65e3} {Suffrage:Voter ID:server-508b1792-f995-5b14-57a5-6b60bb6ea8a9 Address:508b1792-f995-5b14-57a5-6b60bb6ea8a9}]" -2025-06-03T23:33:16.775Z [INFO] server-a90c8f81-7525-f1f3-d24c-2f9c2f0a65e3: entering follower state: follower="Node at a90c8f81-7525-f1f3-d24c-2f9c2f0a65e3 [Follower]" leader-address= leader-id= -2025-06-03T23:33:16.775Z [INFO] server-508b1792-f995-5b14-57a5-6b60bb6ea8a9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b1cfb755-4673-2628-6811-684110356638 Address:b1cfb755-4673-2628-6811-684110356638} {Suffrage:Voter ID:server-a90c8f81-7525-f1f3-d24c-2f9c2f0a65e3 Address:a90c8f81-7525-f1f3-d24c-2f9c2f0a65e3} {Suffrage:Voter ID:server-508b1792-f995-5b14-57a5-6b60bb6ea8a9 Address:508b1792-f995-5b14-57a5-6b60bb6ea8a9}]" -2025-06-03T23:33:16.775Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:16.775Z [INFO] server-508b1792-f995-5b14-57a5-6b60bb6ea8a9: entering follower state: follower="Node at 508b1792-f995-5b14-57a5-6b60bb6ea8a9 [Follower]" leader-address= leader-id= -2025-06-03T23:33:16.854Z [WARN] server-508b1792-f995-5b14-57a5-6b60bb6ea8a9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:16.854Z [INFO] server-508b1792-f995-5b14-57a5-6b60bb6ea8a9: entering candidate state: node="Node at 508b1792-f995-5b14-57a5-6b60bb6ea8a9 [Candidate]" term=2 -2025-06-03T23:33:16.854Z [INFO] server-508b1792-f995-5b14-57a5-6b60bb6ea8a9: election won: term=2 tally=2 -2025-06-03T23:33:16.854Z [INFO] server-508b1792-f995-5b14-57a5-6b60bb6ea8a9: entering leader state: leader="Node at 508b1792-f995-5b14-57a5-6b60bb6ea8a9 [Leader]" -2025-06-03T23:33:16.854Z [INFO] server-508b1792-f995-5b14-57a5-6b60bb6ea8a9: added peer, starting replication: peer=server-b1cfb755-4673-2628-6811-684110356638 -2025-06-03T23:33:16.854Z [INFO] server-508b1792-f995-5b14-57a5-6b60bb6ea8a9: added peer, starting replication: peer=server-a90c8f81-7525-f1f3-d24c-2f9c2f0a65e3 -2025-06-03T23:33:16.854Z [INFO] server-508b1792-f995-5b14-57a5-6b60bb6ea8a9: pipelining replication: peer="{Voter server-a90c8f81-7525-f1f3-d24c-2f9c2f0a65e3 a90c8f81-7525-f1f3-d24c-2f9c2f0a65e3}" -2025-06-03T23:33:16.854Z [INFO] server-508b1792-f995-5b14-57a5-6b60bb6ea8a9: pipelining replication: peer="{Voter server-b1cfb755-4673-2628-6811-684110356638 b1cfb755-4673-2628-6811-684110356638}" -2025-06-03T23:33:16.960Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:16.855029311 +0000 UTC m=+11.875848681 (1 nodes), 79.14144ms from start of poll, 80.10096ms from cluster start. Timeout at 2025-06-03 23:33:16.960909231 +0000 UTC m=+11.981728561, 105.87988ms after stability -2025-06-03T23:33:16.960Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:17.066Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:16.961002511 +0000 UTC m=+11.981821841 (2 nodes), 2.04µs from start of poll, 186.07412ms from cluster start. Timeout at 2025-06-03 23:33:17.066216871 +0000 UTC m=+12.087036201, 105.21436ms after stability -2025-06-03T23:33:17.066Z [INFO] server-508b1792-f995-5b14-57a5-6b60bb6ea8a9: updating configuration: command=AddVoter server-id=server-b1cfb755-4673-2628-6811-684110356638 server-addr=b1cfb755-4673-2628-6811-684110356638 servers="[{Suffrage:Voter ID:server-b1cfb755-4673-2628-6811-684110356638 Address:b1cfb755-4673-2628-6811-684110356638} {Suffrage:Voter ID:server-a90c8f81-7525-f1f3-d24c-2f9c2f0a65e3 Address:a90c8f81-7525-f1f3-d24c-2f9c2f0a65e3} {Suffrage:Voter ID:server-508b1792-f995-5b14-57a5-6b60bb6ea8a9 Address:508b1792-f995-5b14-57a5-6b60bb6ea8a9}]" -2025-06-03T23:33:17.066Z [INFO] server-508b1792-f995-5b14-57a5-6b60bb6ea8a9: aborting pipeline replication: peer="{Voter server-a90c8f81-7525-f1f3-d24c-2f9c2f0a65e3 a90c8f81-7525-f1f3-d24c-2f9c2f0a65e3}" -2025-06-03T23:33:17.066Z [INFO] server-508b1792-f995-5b14-57a5-6b60bb6ea8a9: aborting pipeline replication: peer="{Voter server-b1cfb755-4673-2628-6811-684110356638 b1cfb755-4673-2628-6811-684110356638}" -2025-06-03T23:33:17.570Z [ERROR] server-508b1792-f995-5b14-57a5-6b60bb6ea8a9: failed to heartbeat to: peer=a90c8f81-7525-f1f3-d24c-2f9c2f0a65e3 backoff time=10ms error="command timed out" ---- PASS: TestRaft_AddKnownPeer (0.80s) +2025-06-03T23:36:16.030Z [INFO] server-d781db1e-a470-0d61-d3a3-cf6c6cff9109: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d781db1e-a470-0d61-d3a3-cf6c6cff9109 Address:d781db1e-a470-0d61-d3a3-cf6c6cff9109} {Suffrage:Voter ID:server-8ec349b8-9bc7-627b-62d2-092075cead3f Address:8ec349b8-9bc7-627b-62d2-092075cead3f} {Suffrage:Voter ID:server-20e3aa89-591f-2618-de00-c9e87a4d59bb Address:20e3aa89-591f-2618-de00-c9e87a4d59bb}]" +2025-06-03T23:36:16.030Z [INFO] server-d781db1e-a470-0d61-d3a3-cf6c6cff9109: entering follower state: follower="Node at d781db1e-a470-0d61-d3a3-cf6c6cff9109 [Follower]" leader-address= leader-id= +2025-06-03T23:36:16.030Z [INFO] server-8ec349b8-9bc7-627b-62d2-092075cead3f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d781db1e-a470-0d61-d3a3-cf6c6cff9109 Address:d781db1e-a470-0d61-d3a3-cf6c6cff9109} {Suffrage:Voter ID:server-8ec349b8-9bc7-627b-62d2-092075cead3f Address:8ec349b8-9bc7-627b-62d2-092075cead3f} {Suffrage:Voter ID:server-20e3aa89-591f-2618-de00-c9e87a4d59bb Address:20e3aa89-591f-2618-de00-c9e87a4d59bb}]" +2025-06-03T23:36:16.030Z [INFO] server-8ec349b8-9bc7-627b-62d2-092075cead3f: entering follower state: follower="Node at 8ec349b8-9bc7-627b-62d2-092075cead3f [Follower]" leader-address= leader-id= +2025-06-03T23:36:16.031Z [INFO] server-20e3aa89-591f-2618-de00-c9e87a4d59bb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d781db1e-a470-0d61-d3a3-cf6c6cff9109 Address:d781db1e-a470-0d61-d3a3-cf6c6cff9109} {Suffrage:Voter ID:server-8ec349b8-9bc7-627b-62d2-092075cead3f Address:8ec349b8-9bc7-627b-62d2-092075cead3f} {Suffrage:Voter ID:server-20e3aa89-591f-2618-de00-c9e87a4d59bb Address:20e3aa89-591f-2618-de00-c9e87a4d59bb}]" +2025-06-03T23:36:16.031Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:16.031Z [INFO] server-20e3aa89-591f-2618-de00-c9e87a4d59bb: entering follower state: follower="Node at 20e3aa89-591f-2618-de00-c9e87a4d59bb [Follower]" leader-address= leader-id= +2025-06-03T23:36:16.096Z [WARN] server-20e3aa89-591f-2618-de00-c9e87a4d59bb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:16.096Z [INFO] server-20e3aa89-591f-2618-de00-c9e87a4d59bb: entering candidate state: node="Node at 20e3aa89-591f-2618-de00-c9e87a4d59bb [Candidate]" term=2 +2025-06-03T23:36:16.096Z [INFO] server-20e3aa89-591f-2618-de00-c9e87a4d59bb: election won: term=2 tally=2 +2025-06-03T23:36:16.096Z [INFO] server-20e3aa89-591f-2618-de00-c9e87a4d59bb: entering leader state: leader="Node at 20e3aa89-591f-2618-de00-c9e87a4d59bb [Leader]" +2025-06-03T23:36:16.096Z [INFO] server-20e3aa89-591f-2618-de00-c9e87a4d59bb: added peer, starting replication: peer=server-d781db1e-a470-0d61-d3a3-cf6c6cff9109 +2025-06-03T23:36:16.096Z [INFO] server-20e3aa89-591f-2618-de00-c9e87a4d59bb: added peer, starting replication: peer=server-8ec349b8-9bc7-627b-62d2-092075cead3f +2025-06-03T23:36:16.096Z [INFO] server-20e3aa89-591f-2618-de00-c9e87a4d59bb: pipelining replication: peer="{Voter server-8ec349b8-9bc7-627b-62d2-092075cead3f 8ec349b8-9bc7-627b-62d2-092075cead3f}" +2025-06-03T23:36:16.096Z [INFO] server-20e3aa89-591f-2618-de00-c9e87a4d59bb: pipelining replication: peer="{Voter server-d781db1e-a470-0d61-d3a3-cf6c6cff9109 d781db1e-a470-0d61-d3a3-cf6c6cff9109}" +2025-06-03T23:36:16.202Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:16.096241631 +0000 UTC m=+11.417996561 (1 nodes), 65.0766ms from start of poll, 65.70288ms from cluster start. Timeout at 2025-06-03 23:36:16.202162391 +0000 UTC m=+11.523917321, 105.92076ms after stability +2025-06-03T23:36:16.202Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:16.308Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:16.202324071 +0000 UTC m=+11.524079001 (2 nodes), 3.48µs from start of poll, 171.78532ms from cluster start. Timeout at 2025-06-03 23:36:16.308342751 +0000 UTC m=+11.630097721, 106.01872ms after stability +2025-06-03T23:36:16.308Z [INFO] server-20e3aa89-591f-2618-de00-c9e87a4d59bb: updating configuration: command=AddVoter server-id=server-d781db1e-a470-0d61-d3a3-cf6c6cff9109 server-addr=d781db1e-a470-0d61-d3a3-cf6c6cff9109 servers="[{Suffrage:Voter ID:server-d781db1e-a470-0d61-d3a3-cf6c6cff9109 Address:d781db1e-a470-0d61-d3a3-cf6c6cff9109} {Suffrage:Voter ID:server-8ec349b8-9bc7-627b-62d2-092075cead3f Address:8ec349b8-9bc7-627b-62d2-092075cead3f} {Suffrage:Voter ID:server-20e3aa89-591f-2618-de00-c9e87a4d59bb Address:20e3aa89-591f-2618-de00-c9e87a4d59bb}]" +2025-06-03T23:36:16.308Z [INFO] server-20e3aa89-591f-2618-de00-c9e87a4d59bb: aborting pipeline replication: peer="{Voter server-8ec349b8-9bc7-627b-62d2-092075cead3f 8ec349b8-9bc7-627b-62d2-092075cead3f}" +2025-06-03T23:36:16.308Z [INFO] server-20e3aa89-591f-2618-de00-c9e87a4d59bb: aborting pipeline replication: peer="{Voter server-d781db1e-a470-0d61-d3a3-cf6c6cff9109 d781db1e-a470-0d61-d3a3-cf6c6cff9109}" +--- PASS: TestRaft_AddKnownPeer (0.28s) === RUN TestRaft_RemoveUnknownPeer -2025-06-03T23:33:17.570Z [INFO] server-de761e11-5ff5-b6da-d36a-b89b16552bba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-de761e11-5ff5-b6da-d36a-b89b16552bba Address:de761e11-5ff5-b6da-d36a-b89b16552bba} {Suffrage:Voter ID:server-ed58fa80-b702-3f2f-2c90-07eb83ce6859 Address:ed58fa80-b702-3f2f-2c90-07eb83ce6859} {Suffrage:Voter ID:server-d51c5fd5-b883-c92a-a989-d67bafa6e381 Address:d51c5fd5-b883-c92a-a989-d67bafa6e381}]" -2025-06-03T23:33:17.571Z [INFO] server-ed58fa80-b702-3f2f-2c90-07eb83ce6859: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-de761e11-5ff5-b6da-d36a-b89b16552bba Address:de761e11-5ff5-b6da-d36a-b89b16552bba} {Suffrage:Voter ID:server-ed58fa80-b702-3f2f-2c90-07eb83ce6859 Address:ed58fa80-b702-3f2f-2c90-07eb83ce6859} {Suffrage:Voter ID:server-d51c5fd5-b883-c92a-a989-d67bafa6e381 Address:d51c5fd5-b883-c92a-a989-d67bafa6e381}]" -2025-06-03T23:33:17.571Z [INFO] server-d51c5fd5-b883-c92a-a989-d67bafa6e381: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-de761e11-5ff5-b6da-d36a-b89b16552bba Address:de761e11-5ff5-b6da-d36a-b89b16552bba} {Suffrage:Voter ID:server-ed58fa80-b702-3f2f-2c90-07eb83ce6859 Address:ed58fa80-b702-3f2f-2c90-07eb83ce6859} {Suffrage:Voter ID:server-d51c5fd5-b883-c92a-a989-d67bafa6e381 Address:d51c5fd5-b883-c92a-a989-d67bafa6e381}]" -2025-06-03T23:33:17.571Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:17.571Z [INFO] server-de761e11-5ff5-b6da-d36a-b89b16552bba: entering follower state: follower="Node at de761e11-5ff5-b6da-d36a-b89b16552bba [Follower]" leader-address= leader-id= -2025-06-03T23:33:17.571Z [INFO] server-ed58fa80-b702-3f2f-2c90-07eb83ce6859: entering follower state: follower="Node at ed58fa80-b702-3f2f-2c90-07eb83ce6859 [Follower]" leader-address= leader-id= -2025-06-03T23:33:17.571Z [INFO] server-d51c5fd5-b883-c92a-a989-d67bafa6e381: entering follower state: follower="Node at d51c5fd5-b883-c92a-a989-d67bafa6e381 [Follower]" leader-address= leader-id= -2025-06-03T23:33:17.642Z [WARN] server-de761e11-5ff5-b6da-d36a-b89b16552bba: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:17.642Z [INFO] server-de761e11-5ff5-b6da-d36a-b89b16552bba: entering candidate state: node="Node at de761e11-5ff5-b6da-d36a-b89b16552bba [Candidate]" term=2 -2025-06-03T23:33:17.642Z [INFO] server-de761e11-5ff5-b6da-d36a-b89b16552bba: election won: term=2 tally=2 -2025-06-03T23:33:17.642Z [INFO] server-de761e11-5ff5-b6da-d36a-b89b16552bba: entering leader state: leader="Node at de761e11-5ff5-b6da-d36a-b89b16552bba [Leader]" -2025-06-03T23:33:17.642Z [INFO] server-de761e11-5ff5-b6da-d36a-b89b16552bba: added peer, starting replication: peer=server-ed58fa80-b702-3f2f-2c90-07eb83ce6859 -2025-06-03T23:33:17.642Z [INFO] server-de761e11-5ff5-b6da-d36a-b89b16552bba: added peer, starting replication: peer=server-d51c5fd5-b883-c92a-a989-d67bafa6e381 -2025-06-03T23:33:17.642Z [INFO] server-de761e11-5ff5-b6da-d36a-b89b16552bba: pipelining replication: peer="{Voter server-d51c5fd5-b883-c92a-a989-d67bafa6e381 d51c5fd5-b883-c92a-a989-d67bafa6e381}" -2025-06-03T23:33:17.642Z [INFO] server-de761e11-5ff5-b6da-d36a-b89b16552bba: pipelining replication: peer="{Voter server-ed58fa80-b702-3f2f-2c90-07eb83ce6859 ed58fa80-b702-3f2f-2c90-07eb83ce6859}" -2025-06-03T23:33:17.750Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:17.642901951 +0000 UTC m=+12.663721281 (1 nodes), 71.773ms from start of poll, 71.9918ms from cluster start. Timeout at 2025-06-03 23:33:17.750418071 +0000 UTC m=+12.771237401, 107.51612ms after stability -2025-06-03T23:33:17.750Z [INFO] server-de761e11-5ff5-b6da-d36a-b89b16552bba: updating configuration: command=RemoveServer server-id=37a835e2-0786-2894-d5fa-61c6e547017d server-addr= servers="[{Suffrage:Voter ID:server-de761e11-5ff5-b6da-d36a-b89b16552bba Address:de761e11-5ff5-b6da-d36a-b89b16552bba} {Suffrage:Voter ID:server-ed58fa80-b702-3f2f-2c90-07eb83ce6859 Address:ed58fa80-b702-3f2f-2c90-07eb83ce6859} {Suffrage:Voter ID:server-d51c5fd5-b883-c92a-a989-d67bafa6e381 Address:d51c5fd5-b883-c92a-a989-d67bafa6e381}]" -2025-06-03T23:33:17.750Z [INFO] server-de761e11-5ff5-b6da-d36a-b89b16552bba: aborting pipeline replication: peer="{Voter server-ed58fa80-b702-3f2f-2c90-07eb83ce6859 ed58fa80-b702-3f2f-2c90-07eb83ce6859}" -2025-06-03T23:33:17.750Z [INFO] server-de761e11-5ff5-b6da-d36a-b89b16552bba: aborting pipeline replication: peer="{Voter server-d51c5fd5-b883-c92a-a989-d67bafa6e381 d51c5fd5-b883-c92a-a989-d67bafa6e381}" ---- PASS: TestRaft_RemoveUnknownPeer (0.18s) +2025-06-03T23:36:16.309Z [INFO] server-537ae327-4611-d802-cba3-05a5738c581a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-537ae327-4611-d802-cba3-05a5738c581a Address:537ae327-4611-d802-cba3-05a5738c581a} {Suffrage:Voter ID:server-07db3335-50d0-9e28-a2be-28b04e85e373 Address:07db3335-50d0-9e28-a2be-28b04e85e373} {Suffrage:Voter ID:server-52719bba-8c14-b0f9-37ee-28401db2343a Address:52719bba-8c14-b0f9-37ee-28401db2343a}]" +2025-06-03T23:36:16.309Z [INFO] server-537ae327-4611-d802-cba3-05a5738c581a: entering follower state: follower="Node at 537ae327-4611-d802-cba3-05a5738c581a [Follower]" leader-address= leader-id= +2025-06-03T23:36:16.310Z [INFO] server-07db3335-50d0-9e28-a2be-28b04e85e373: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-537ae327-4611-d802-cba3-05a5738c581a Address:537ae327-4611-d802-cba3-05a5738c581a} {Suffrage:Voter ID:server-07db3335-50d0-9e28-a2be-28b04e85e373 Address:07db3335-50d0-9e28-a2be-28b04e85e373} {Suffrage:Voter ID:server-52719bba-8c14-b0f9-37ee-28401db2343a Address:52719bba-8c14-b0f9-37ee-28401db2343a}]" +2025-06-03T23:36:16.310Z [INFO] server-52719bba-8c14-b0f9-37ee-28401db2343a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-537ae327-4611-d802-cba3-05a5738c581a Address:537ae327-4611-d802-cba3-05a5738c581a} {Suffrage:Voter ID:server-07db3335-50d0-9e28-a2be-28b04e85e373 Address:07db3335-50d0-9e28-a2be-28b04e85e373} {Suffrage:Voter ID:server-52719bba-8c14-b0f9-37ee-28401db2343a Address:52719bba-8c14-b0f9-37ee-28401db2343a}]" +2025-06-03T23:36:16.310Z [INFO] server-07db3335-50d0-9e28-a2be-28b04e85e373: entering follower state: follower="Node at 07db3335-50d0-9e28-a2be-28b04e85e373 [Follower]" leader-address= leader-id= +2025-06-03T23:36:16.310Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:16.310Z [INFO] server-52719bba-8c14-b0f9-37ee-28401db2343a: entering follower state: follower="Node at 52719bba-8c14-b0f9-37ee-28401db2343a [Follower]" leader-address= leader-id= +2025-06-03T23:36:16.372Z [WARN] server-537ae327-4611-d802-cba3-05a5738c581a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:16.373Z [INFO] server-537ae327-4611-d802-cba3-05a5738c581a: entering candidate state: node="Node at 537ae327-4611-d802-cba3-05a5738c581a [Candidate]" term=2 +2025-06-03T23:36:16.373Z [WARN] server-07db3335-50d0-9e28-a2be-28b04e85e373: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:16.373Z [INFO] server-07db3335-50d0-9e28-a2be-28b04e85e373: entering candidate state: node="Node at 07db3335-50d0-9e28-a2be-28b04e85e373 [Candidate]" term=2 +2025-06-03T23:36:16.373Z [INFO] server-52719bba-8c14-b0f9-37ee-28401db2343a: duplicate requestVote for same term: term=2 +2025-06-03T23:36:16.373Z [INFO] server-537ae327-4611-d802-cba3-05a5738c581a: duplicate requestVote for same term: term=2 +2025-06-03T23:36:16.373Z [INFO] server-07db3335-50d0-9e28-a2be-28b04e85e373: duplicate requestVote for same term: term=2 +2025-06-03T23:36:16.373Z [INFO] server-537ae327-4611-d802-cba3-05a5738c581a: election won: term=2 tally=2 +2025-06-03T23:36:16.373Z [INFO] server-537ae327-4611-d802-cba3-05a5738c581a: entering leader state: leader="Node at 537ae327-4611-d802-cba3-05a5738c581a [Leader]" +2025-06-03T23:36:16.373Z [INFO] server-537ae327-4611-d802-cba3-05a5738c581a: added peer, starting replication: peer=server-07db3335-50d0-9e28-a2be-28b04e85e373 +2025-06-03T23:36:16.373Z [INFO] server-537ae327-4611-d802-cba3-05a5738c581a: added peer, starting replication: peer=server-52719bba-8c14-b0f9-37ee-28401db2343a +2025-06-03T23:36:16.373Z [INFO] server-07db3335-50d0-9e28-a2be-28b04e85e373: entering follower state: follower="Node at 07db3335-50d0-9e28-a2be-28b04e85e373 [Follower]" leader-address=537ae327-4611-d802-cba3-05a5738c581a leader-id=server-537ae327-4611-d802-cba3-05a5738c581a +2025-06-03T23:36:16.373Z [INFO] server-537ae327-4611-d802-cba3-05a5738c581a: pipelining replication: peer="{Voter server-52719bba-8c14-b0f9-37ee-28401db2343a 52719bba-8c14-b0f9-37ee-28401db2343a}" +2025-06-03T23:36:16.373Z [INFO] server-537ae327-4611-d802-cba3-05a5738c581a: pipelining replication: peer="{Voter server-07db3335-50d0-9e28-a2be-28b04e85e373 07db3335-50d0-9e28-a2be-28b04e85e373}" +2025-06-03T23:36:16.479Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:16.373578991 +0000 UTC m=+11.695333921 (1 nodes), 63.41148ms from start of poll, 63.8976ms from cluster start. Timeout at 2025-06-03 23:36:16.479359031 +0000 UTC m=+11.801113961, 105.78004ms after stability +2025-06-03T23:36:16.479Z [INFO] server-537ae327-4611-d802-cba3-05a5738c581a: updating configuration: command=RemoveServer server-id=efb5e9ec-9826-8c41-ad99-1aff967fe2c8 server-addr= servers="[{Suffrage:Voter ID:server-537ae327-4611-d802-cba3-05a5738c581a Address:537ae327-4611-d802-cba3-05a5738c581a} {Suffrage:Voter ID:server-07db3335-50d0-9e28-a2be-28b04e85e373 Address:07db3335-50d0-9e28-a2be-28b04e85e373} {Suffrage:Voter ID:server-52719bba-8c14-b0f9-37ee-28401db2343a Address:52719bba-8c14-b0f9-37ee-28401db2343a}]" +2025-06-03T23:36:16.479Z [INFO] server-537ae327-4611-d802-cba3-05a5738c581a: aborting pipeline replication: peer="{Voter server-52719bba-8c14-b0f9-37ee-28401db2343a 52719bba-8c14-b0f9-37ee-28401db2343a}" +2025-06-03T23:36:16.479Z [INFO] server-537ae327-4611-d802-cba3-05a5738c581a: aborting pipeline replication: peer="{Voter server-07db3335-50d0-9e28-a2be-28b04e85e373 07db3335-50d0-9e28-a2be-28b04e85e373}" +--- PASS: TestRaft_RemoveUnknownPeer (0.17s) === RUN TestRaft_SnapshotRestore -2025-06-03T23:33:17.751Z [INFO] server-76357380-6ccf-b19f-8fb1-0569c76ee8f4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-76357380-6ccf-b19f-8fb1-0569c76ee8f4 Address:76357380-6ccf-b19f-8fb1-0569c76ee8f4}]" -2025-06-03T23:33:17.751Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:17.751Z [INFO] server-76357380-6ccf-b19f-8fb1-0569c76ee8f4: entering follower state: follower="Node at 76357380-6ccf-b19f-8fb1-0569c76ee8f4 [Follower]" leader-address= leader-id= -2025-06-03T23:33:17.850Z [WARN] server-76357380-6ccf-b19f-8fb1-0569c76ee8f4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:17.850Z [INFO] server-76357380-6ccf-b19f-8fb1-0569c76ee8f4: entering candidate state: node="Node at 76357380-6ccf-b19f-8fb1-0569c76ee8f4 [Candidate]" term=2 -2025-06-03T23:33:17.850Z [INFO] server-76357380-6ccf-b19f-8fb1-0569c76ee8f4: election won: term=2 tally=1 -2025-06-03T23:33:17.850Z [INFO] server-76357380-6ccf-b19f-8fb1-0569c76ee8f4: entering leader state: leader="Node at 76357380-6ccf-b19f-8fb1-0569c76ee8f4 [Leader]" -2025-06-03T23:33:17.956Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:17.850600711 +0000 UTC m=+12.871420041 (1 nodes), 99.47368ms from start of poll, 99.5672ms from cluster start. Timeout at 2025-06-03 23:33:17.956238271 +0000 UTC m=+12.977057641, 105.6376ms after stability -2025-06-03T23:33:17.958Z [INFO] server-76357380-6ccf-b19f-8fb1-0569c76ee8f4: starting snapshot up to: index=102 -2025-06-03T23:33:17.958Z [INFO] creating new snapshot: path=/tmp/raft3011364377/snapshots/2-102-1748993597958.tmp -2025-06-03T23:33:17.959Z [INFO] server-76357380-6ccf-b19f-8fb1-0569c76ee8f4: compacting logs: from=1 to=92 -2025-06-03T23:33:17.959Z [INFO] server-76357380-6ccf-b19f-8fb1-0569c76ee8f4: snapshot complete up to: index=102 -2025-06-03T23:33:17.960Z [INFO] server-76357380-6ccf-b19f-8fb1-0569c76ee8f4: starting restore from snapshot: id=2-102-1748993597958 last-index=102 last-term=2 size-in-bytes=693 -2025-06-03T23:33:17.962Z [INFO] server-76357380-6ccf-b19f-8fb1-0569c76ee8f4: snapshot restore progress: id=2-102-1748993597958 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2025-06-03T23:33:17.962Z [INFO] server-76357380-6ccf-b19f-8fb1-0569c76ee8f4: restored from snapshot: id=2-102-1748993597958 last-index=102 last-term=2 size-in-bytes=693 -2025-06-03T23:33:17.962Z [INFO] server-76357380-6ccf-b19f-8fb1-0569c76ee8f4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-76357380-6ccf-b19f-8fb1-0569c76ee8f4 Address:76357380-6ccf-b19f-8fb1-0569c76ee8f4}]" ---- PASS: TestRaft_SnapshotRestore (0.21s) +2025-06-03T23:36:16.480Z [INFO] server-ac067815-1f20-cff7-14cd-dbb11afdbb09: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ac067815-1f20-cff7-14cd-dbb11afdbb09 Address:ac067815-1f20-cff7-14cd-dbb11afdbb09}]" +2025-06-03T23:36:16.480Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:16.480Z [INFO] server-ac067815-1f20-cff7-14cd-dbb11afdbb09: entering follower state: follower="Node at ac067815-1f20-cff7-14cd-dbb11afdbb09 [Follower]" leader-address= leader-id= +2025-06-03T23:36:16.555Z [WARN] server-ac067815-1f20-cff7-14cd-dbb11afdbb09: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:16.555Z [INFO] server-ac067815-1f20-cff7-14cd-dbb11afdbb09: entering candidate state: node="Node at ac067815-1f20-cff7-14cd-dbb11afdbb09 [Candidate]" term=2 +2025-06-03T23:36:16.555Z [INFO] server-ac067815-1f20-cff7-14cd-dbb11afdbb09: election won: term=2 tally=1 +2025-06-03T23:36:16.555Z [INFO] server-ac067815-1f20-cff7-14cd-dbb11afdbb09: entering leader state: leader="Node at ac067815-1f20-cff7-14cd-dbb11afdbb09 [Leader]" +2025-06-03T23:36:16.660Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:16.555343871 +0000 UTC m=+11.877098801 (1 nodes), 74.77596ms from start of poll, 74.96384ms from cluster start. Timeout at 2025-06-03 23:36:16.660353511 +0000 UTC m=+11.982108441, 105.00964ms after stability +2025-06-03T23:36:16.663Z [INFO] server-ac067815-1f20-cff7-14cd-dbb11afdbb09: starting snapshot up to: index=102 +2025-06-03T23:36:16.663Z [INFO] creating new snapshot: path=/tmp/raft889715503/snapshots/2-102-1748993776663.tmp +2025-06-03T23:36:16.664Z [INFO] server-ac067815-1f20-cff7-14cd-dbb11afdbb09: compacting logs: from=1 to=92 +2025-06-03T23:36:16.664Z [INFO] server-ac067815-1f20-cff7-14cd-dbb11afdbb09: snapshot complete up to: index=102 +2025-06-03T23:36:16.664Z [INFO] server-ac067815-1f20-cff7-14cd-dbb11afdbb09: starting restore from snapshot: id=2-102-1748993776663 last-index=102 last-term=2 size-in-bytes=693 +2025-06-03T23:36:16.665Z [INFO] server-ac067815-1f20-cff7-14cd-dbb11afdbb09: snapshot restore progress: id=2-102-1748993776663 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-06-03T23:36:16.665Z [INFO] server-ac067815-1f20-cff7-14cd-dbb11afdbb09: restored from snapshot: id=2-102-1748993776663 last-index=102 last-term=2 size-in-bytes=693 +2025-06-03T23:36:16.665Z [INFO] server-ac067815-1f20-cff7-14cd-dbb11afdbb09: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ac067815-1f20-cff7-14cd-dbb11afdbb09 Address:ac067815-1f20-cff7-14cd-dbb11afdbb09}]" +--- PASS: TestRaft_SnapshotRestore (0.19s) === RUN TestRaft_RestoreSnapshotOnStartup_Monotonic -2025-06-03T23:33:17.962Z [INFO] server-a67d1ed3-9c29-d881-0d57-38ae35f45088: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a67d1ed3-9c29-d881-0d57-38ae35f45088 Address:a67d1ed3-9c29-d881-0d57-38ae35f45088}]" -2025-06-03T23:33:17.962Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:17.963Z [INFO] server-a67d1ed3-9c29-d881-0d57-38ae35f45088: entering follower state: follower="Node at a67d1ed3-9c29-d881-0d57-38ae35f45088 [Follower]" leader-address= leader-id= -2025-06-03T23:33:18.036Z [WARN] server-a67d1ed3-9c29-d881-0d57-38ae35f45088: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:18.036Z [INFO] server-a67d1ed3-9c29-d881-0d57-38ae35f45088: entering candidate state: node="Node at a67d1ed3-9c29-d881-0d57-38ae35f45088 [Candidate]" term=2 -2025-06-03T23:33:18.036Z [INFO] server-a67d1ed3-9c29-d881-0d57-38ae35f45088: election won: term=2 tally=1 -2025-06-03T23:33:18.036Z [INFO] server-a67d1ed3-9c29-d881-0d57-38ae35f45088: entering leader state: leader="Node at a67d1ed3-9c29-d881-0d57-38ae35f45088 [Leader]" -2025-06-03T23:33:18.141Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:18.036229311 +0000 UTC m=+13.057048681 (1 nodes), 73.21868ms from start of poll, 73.47944ms from cluster start. Timeout at 2025-06-03 23:33:18.141302951 +0000 UTC m=+13.162122281, 105.0736ms after stability -2025-06-03T23:33:18.144Z [INFO] server-a67d1ed3-9c29-d881-0d57-38ae35f45088: starting snapshot up to: index=102 -2025-06-03T23:33:18.144Z [INFO] creating new snapshot: path=/tmp/raft1823609920/snapshots/2-102-1748993598144.tmp -2025-06-03T23:33:18.145Z [INFO] server-a67d1ed3-9c29-d881-0d57-38ae35f45088: compacting logs: from=1 to=92 -2025-06-03T23:33:18.145Z [INFO] server-a67d1ed3-9c29-d881-0d57-38ae35f45088: snapshot complete up to: index=102 -2025-06-03T23:33:18.145Z [INFO] server-a67d1ed3-9c29-d881-0d57-38ae35f45088: starting restore from snapshot: id=2-102-1748993598144 last-index=102 last-term=2 size-in-bytes=693 -2025-06-03T23:33:18.145Z [INFO] server-a67d1ed3-9c29-d881-0d57-38ae35f45088: snapshot restore progress: id=2-102-1748993598144 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2025-06-03T23:33:18.145Z [INFO] server-a67d1ed3-9c29-d881-0d57-38ae35f45088: restored from snapshot: id=2-102-1748993598144 last-index=102 last-term=2 size-in-bytes=693 -2025-06-03T23:33:18.145Z [INFO] server-a67d1ed3-9c29-d881-0d57-38ae35f45088: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a67d1ed3-9c29-d881-0d57-38ae35f45088 Address:a67d1ed3-9c29-d881-0d57-38ae35f45088}]" ---- PASS: TestRaft_RestoreSnapshotOnStartup_Monotonic (0.18s) +2025-06-03T23:36:16.666Z [INFO] server-297e1894-517d-95d0-d29a-cdf64658c987: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-297e1894-517d-95d0-d29a-cdf64658c987 Address:297e1894-517d-95d0-d29a-cdf64658c987}]" +2025-06-03T23:36:16.666Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:16.666Z [INFO] server-297e1894-517d-95d0-d29a-cdf64658c987: entering follower state: follower="Node at 297e1894-517d-95d0-d29a-cdf64658c987 [Follower]" leader-address= leader-id= +2025-06-03T23:36:16.755Z [WARN] server-297e1894-517d-95d0-d29a-cdf64658c987: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:16.755Z [INFO] server-297e1894-517d-95d0-d29a-cdf64658c987: entering candidate state: node="Node at 297e1894-517d-95d0-d29a-cdf64658c987 [Candidate]" term=2 +2025-06-03T23:36:16.755Z [INFO] server-297e1894-517d-95d0-d29a-cdf64658c987: election won: term=2 tally=1 +2025-06-03T23:36:16.755Z [INFO] server-297e1894-517d-95d0-d29a-cdf64658c987: entering leader state: leader="Node at 297e1894-517d-95d0-d29a-cdf64658c987 [Leader]" +2025-06-03T23:36:16.860Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:16.755405271 +0000 UTC m=+12.077160201 (1 nodes), 89.02608ms from start of poll, 89.23732ms from cluster start. Timeout at 2025-06-03 23:36:16.860669991 +0000 UTC m=+12.182424961, 105.26476ms after stability +2025-06-03T23:36:16.863Z [INFO] server-297e1894-517d-95d0-d29a-cdf64658c987: starting snapshot up to: index=102 +2025-06-03T23:36:16.863Z [INFO] creating new snapshot: path=/tmp/raft2454924093/snapshots/2-102-1748993776863.tmp +2025-06-03T23:36:16.864Z [INFO] server-297e1894-517d-95d0-d29a-cdf64658c987: compacting logs: from=1 to=92 +2025-06-03T23:36:16.864Z [INFO] server-297e1894-517d-95d0-d29a-cdf64658c987: snapshot complete up to: index=102 +2025-06-03T23:36:16.864Z [INFO] server-297e1894-517d-95d0-d29a-cdf64658c987: starting restore from snapshot: id=2-102-1748993776863 last-index=102 last-term=2 size-in-bytes=693 +2025-06-03T23:36:16.864Z [INFO] server-297e1894-517d-95d0-d29a-cdf64658c987: snapshot restore progress: id=2-102-1748993776863 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-06-03T23:36:16.865Z [INFO] server-297e1894-517d-95d0-d29a-cdf64658c987: restored from snapshot: id=2-102-1748993776863 last-index=102 last-term=2 size-in-bytes=693 +2025-06-03T23:36:16.865Z [INFO] server-297e1894-517d-95d0-d29a-cdf64658c987: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-297e1894-517d-95d0-d29a-cdf64658c987 Address:297e1894-517d-95d0-d29a-cdf64658c987}]" +2025-06-03T23:36:16.865Z [INFO] server-297e1894-517d-95d0-d29a-cdf64658c987: entering follower state: follower="Node at 297e1894-517d-95d0-d29a-cdf64658c987 [Shutdown]" leader-address= leader-id= +--- PASS: TestRaft_RestoreSnapshotOnStartup_Monotonic (0.20s) === RUN TestRaft_SnapshotRestore_Progress -2025-06-03T23:33:18.146Z [INFO] server-666e00e0-df1f-e706-6d79-e195ccec2cc6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-666e00e0-df1f-e706-6d79-e195ccec2cc6 Address:666e00e0-df1f-e706-6d79-e195ccec2cc6}]" -2025-06-03T23:33:18.146Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:18.146Z [INFO] server-666e00e0-df1f-e706-6d79-e195ccec2cc6: entering follower state: follower="Node at 666e00e0-df1f-e706-6d79-e195ccec2cc6 [Follower]" leader-address= leader-id= -2025-06-03T23:33:18.204Z [WARN] server-666e00e0-df1f-e706-6d79-e195ccec2cc6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:18.204Z [INFO] server-666e00e0-df1f-e706-6d79-e195ccec2cc6: entering candidate state: node="Node at 666e00e0-df1f-e706-6d79-e195ccec2cc6 [Candidate]" term=2 -2025-06-03T23:33:18.204Z [INFO] server-666e00e0-df1f-e706-6d79-e195ccec2cc6: election won: term=2 tally=1 -2025-06-03T23:33:18.204Z [INFO] server-666e00e0-df1f-e706-6d79-e195ccec2cc6: entering leader state: leader="Node at 666e00e0-df1f-e706-6d79-e195ccec2cc6 [Leader]" -2025-06-03T23:33:18.311Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:18.204901111 +0000 UTC m=+13.225720481 (1 nodes), 58.47672ms from start of poll, 58.6586ms from cluster start. Timeout at 2025-06-03 23:33:18.310936791 +0000 UTC m=+13.331756201, 106.03572ms after stability -2025-06-03T23:33:18.313Z [INFO] server-666e00e0-df1f-e706-6d79-e195ccec2cc6: starting snapshot up to: index=102 -2025-06-03T23:33:18.314Z [INFO] creating new snapshot: path=/tmp/raft1690784684/snapshots/2-102-1748993598314.tmp -2025-06-03T23:33:18.314Z [INFO] server-666e00e0-df1f-e706-6d79-e195ccec2cc6: compacting logs: from=1 to=92 -2025-06-03T23:33:18.314Z [INFO] server-666e00e0-df1f-e706-6d79-e195ccec2cc6: snapshot complete up to: index=102 ---- PASS: TestRaft_SnapshotRestore_Progress (0.17s) +2025-06-03T23:36:16.865Z [INFO] server-e87d0d51-21d1-b67b-708e-8a5295959267: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e87d0d51-21d1-b67b-708e-8a5295959267 Address:e87d0d51-21d1-b67b-708e-8a5295959267}]" +2025-06-03T23:36:16.865Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:16.865Z [INFO] server-e87d0d51-21d1-b67b-708e-8a5295959267: entering follower state: follower="Node at e87d0d51-21d1-b67b-708e-8a5295959267 [Follower]" leader-address= leader-id= +2025-06-03T23:36:16.963Z [WARN] server-e87d0d51-21d1-b67b-708e-8a5295959267: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:16.963Z [INFO] server-e87d0d51-21d1-b67b-708e-8a5295959267: entering candidate state: node="Node at e87d0d51-21d1-b67b-708e-8a5295959267 [Candidate]" term=2 +2025-06-03T23:36:16.963Z [INFO] server-e87d0d51-21d1-b67b-708e-8a5295959267: election won: term=2 tally=1 +2025-06-03T23:36:16.963Z [INFO] server-e87d0d51-21d1-b67b-708e-8a5295959267: entering leader state: leader="Node at e87d0d51-21d1-b67b-708e-8a5295959267 [Leader]" +2025-06-03T23:36:17.069Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:16.963787191 +0000 UTC m=+12.285542121 (1 nodes), 98.0036ms from start of poll, 98.21444ms from cluster start. Timeout at 2025-06-03 23:36:17.068983631 +0000 UTC m=+12.390738561, 105.19644ms after stability +2025-06-03T23:36:17.072Z [INFO] server-e87d0d51-21d1-b67b-708e-8a5295959267: starting snapshot up to: index=102 +2025-06-03T23:36:17.072Z [INFO] creating new snapshot: path=/tmp/raft2121132402/snapshots/2-102-1748993777072.tmp +2025-06-03T23:36:17.072Z [INFO] server-e87d0d51-21d1-b67b-708e-8a5295959267: compacting logs: from=1 to=92 +2025-06-03T23:36:17.072Z [INFO] server-e87d0d51-21d1-b67b-708e-8a5295959267: snapshot complete up to: index=102 +--- PASS: TestRaft_SnapshotRestore_Progress (0.21s) === RUN TestRaft_NoRestoreOnStart -2025-06-03T23:33:18.316Z [INFO] server-3e027815-5c84-875c-2f42-5262246b3c11: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3e027815-5c84-875c-2f42-5262246b3c11 Address:3e027815-5c84-875c-2f42-5262246b3c11}]" -2025-06-03T23:33:18.316Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:18.316Z [INFO] server-3e027815-5c84-875c-2f42-5262246b3c11: entering follower state: follower="Node at 3e027815-5c84-875c-2f42-5262246b3c11 [Follower]" leader-address= leader-id= -2025-06-03T23:33:18.376Z [WARN] server-3e027815-5c84-875c-2f42-5262246b3c11: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:18.376Z [INFO] server-3e027815-5c84-875c-2f42-5262246b3c11: entering candidate state: node="Node at 3e027815-5c84-875c-2f42-5262246b3c11 [Candidate]" term=2 -2025-06-03T23:33:18.376Z [INFO] server-3e027815-5c84-875c-2f42-5262246b3c11: election won: term=2 tally=1 -2025-06-03T23:33:18.376Z [INFO] server-3e027815-5c84-875c-2f42-5262246b3c11: entering leader state: leader="Node at 3e027815-5c84-875c-2f42-5262246b3c11 [Leader]" -2025-06-03T23:33:18.482Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:18.376961911 +0000 UTC m=+13.397781321 (1 nodes), 60.74232ms from start of poll, 60.9406ms from cluster start. Timeout at 2025-06-03 23:33:18.482376991 +0000 UTC m=+13.503196361, 105.41504ms after stability -2025-06-03T23:33:18.486Z [INFO] server-3e027815-5c84-875c-2f42-5262246b3c11: starting snapshot up to: index=102 -2025-06-03T23:33:18.486Z [INFO] creating new snapshot: path=/tmp/raft1018683795/snapshots/2-102-1748993598486.tmp -2025-06-03T23:33:18.487Z [INFO] server-3e027815-5c84-875c-2f42-5262246b3c11: compacting logs: from=1 to=92 -2025-06-03T23:33:18.487Z [INFO] server-3e027815-5c84-875c-2f42-5262246b3c11: snapshot complete up to: index=102 -2025-06-03T23:33:18.487Z [INFO] server-3e027815-5c84-875c-2f42-5262246b3c11: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3e027815-5c84-875c-2f42-5262246b3c11 Address:3e027815-5c84-875c-2f42-5262246b3c11}]" ---- PASS: TestRaft_NoRestoreOnStart (0.17s) +2025-06-03T23:36:17.074Z [INFO] server-eaf7ccb2-e292-0490-4b04-ad5fa4e6a271: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eaf7ccb2-e292-0490-4b04-ad5fa4e6a271 Address:eaf7ccb2-e292-0490-4b04-ad5fa4e6a271}]" +2025-06-03T23:36:17.074Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:17.074Z [INFO] server-eaf7ccb2-e292-0490-4b04-ad5fa4e6a271: entering follower state: follower="Node at eaf7ccb2-e292-0490-4b04-ad5fa4e6a271 [Follower]" leader-address= leader-id= +2025-06-03T23:36:17.159Z [WARN] server-eaf7ccb2-e292-0490-4b04-ad5fa4e6a271: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:17.159Z [INFO] server-eaf7ccb2-e292-0490-4b04-ad5fa4e6a271: entering candidate state: node="Node at eaf7ccb2-e292-0490-4b04-ad5fa4e6a271 [Candidate]" term=2 +2025-06-03T23:36:17.159Z [INFO] server-eaf7ccb2-e292-0490-4b04-ad5fa4e6a271: election won: term=2 tally=1 +2025-06-03T23:36:17.159Z [INFO] server-eaf7ccb2-e292-0490-4b04-ad5fa4e6a271: entering leader state: leader="Node at eaf7ccb2-e292-0490-4b04-ad5fa4e6a271 [Leader]" +2025-06-03T23:36:17.264Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:17.159263351 +0000 UTC m=+12.481018321 (1 nodes), 84.90808ms from start of poll, 85.11108ms from cluster start. Timeout at 2025-06-03 23:36:17.264667831 +0000 UTC m=+12.586422761, 105.40444ms after stability +2025-06-03T23:36:17.267Z [INFO] server-eaf7ccb2-e292-0490-4b04-ad5fa4e6a271: starting snapshot up to: index=102 +2025-06-03T23:36:17.267Z [INFO] creating new snapshot: path=/tmp/raft3737840908/snapshots/2-102-1748993777267.tmp +2025-06-03T23:36:17.268Z [INFO] server-eaf7ccb2-e292-0490-4b04-ad5fa4e6a271: compacting logs: from=1 to=92 +2025-06-03T23:36:17.268Z [INFO] server-eaf7ccb2-e292-0490-4b04-ad5fa4e6a271: snapshot complete up to: index=102 +2025-06-03T23:36:17.269Z [INFO] server-eaf7ccb2-e292-0490-4b04-ad5fa4e6a271: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eaf7ccb2-e292-0490-4b04-ad5fa4e6a271 Address:eaf7ccb2-e292-0490-4b04-ad5fa4e6a271}]" +--- PASS: TestRaft_NoRestoreOnStart (0.20s) === RUN TestRaft_SnapshotRestore_PeerChange -2025-06-03T23:33:18.487Z [INFO] server-3e027815-5c84-875c-2f42-5262246b3c11: entering follower state: follower="Node at 3e027815-5c84-875c-2f42-5262246b3c11 [Follower]" leader-address= leader-id= +2025-06-03T23:36:17.269Z [INFO] server-eaf7ccb2-e292-0490-4b04-ad5fa4e6a271: entering follower state: follower="Node at eaf7ccb2-e292-0490-4b04-ad5fa4e6a271 [Follower]" leader-address= leader-id= raft_test.go:1266: Skipping unreliable test --- SKIP: TestRaft_SnapshotRestore_PeerChange (0.00s) === RUN TestRaft_AutoSnapshot -2025-06-03T23:33:18.487Z [INFO] server-f38c5fbb-eff1-8ab6-8e28-fbc675095077: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f38c5fbb-eff1-8ab6-8e28-fbc675095077 Address:f38c5fbb-eff1-8ab6-8e28-fbc675095077}]" -2025-06-03T23:33:18.487Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:18.487Z [INFO] server-f38c5fbb-eff1-8ab6-8e28-fbc675095077: entering follower state: follower="Node at f38c5fbb-eff1-8ab6-8e28-fbc675095077 [Follower]" leader-address= leader-id= -2025-06-03T23:33:18.579Z [WARN] server-f38c5fbb-eff1-8ab6-8e28-fbc675095077: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:18.579Z [INFO] server-f38c5fbb-eff1-8ab6-8e28-fbc675095077: entering candidate state: node="Node at f38c5fbb-eff1-8ab6-8e28-fbc675095077 [Candidate]" term=2 -2025-06-03T23:33:18.579Z [INFO] server-f38c5fbb-eff1-8ab6-8e28-fbc675095077: election won: term=2 tally=1 -2025-06-03T23:33:18.579Z [INFO] server-f38c5fbb-eff1-8ab6-8e28-fbc675095077: entering leader state: leader="Node at f38c5fbb-eff1-8ab6-8e28-fbc675095077 [Leader]" -2025-06-03T23:33:18.587Z [WARN] server-3e027815-5c84-875c-2f42-5262246b3c11: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:18.587Z [INFO] server-3e027815-5c84-875c-2f42-5262246b3c11: entering candidate state: node="Node at 3e027815-5c84-875c-2f42-5262246b3c11 [Candidate]" term=3 -2025-06-03T23:33:18.587Z [INFO] server-3e027815-5c84-875c-2f42-5262246b3c11: election won: term=3 tally=1 -2025-06-03T23:33:18.587Z [INFO] server-3e027815-5c84-875c-2f42-5262246b3c11: entering leader state: leader="Node at 3e027815-5c84-875c-2f42-5262246b3c11 [Leader]" -2025-06-03T23:33:18.685Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:18.579796231 +0000 UTC m=+13.600615601 (1 nodes), 91.88796ms from start of poll, 92.07304ms from cluster start. Timeout at 2025-06-03 23:33:18.685784191 +0000 UTC m=+13.706603561, 105.98796ms after stability -2025-06-03T23:33:18.688Z [INFO] server-f38c5fbb-eff1-8ab6-8e28-fbc675095077: starting snapshot up to: index=102 -2025-06-03T23:33:18.688Z [INFO] creating new snapshot: path=/tmp/raft1231645706/snapshots/2-102-1748993598688.tmp -2025-06-03T23:33:18.689Z [INFO] server-f38c5fbb-eff1-8ab6-8e28-fbc675095077: compacting logs: from=1 to=92 -2025-06-03T23:33:18.689Z [INFO] server-f38c5fbb-eff1-8ab6-8e28-fbc675095077: snapshot complete up to: index=102 ---- PASS: TestRaft_AutoSnapshot (0.31s) +2025-06-03T23:36:17.270Z [INFO] server-d4e55654-6b22-e249-bbc5-5fe1e9000d8b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d4e55654-6b22-e249-bbc5-5fe1e9000d8b Address:d4e55654-6b22-e249-bbc5-5fe1e9000d8b}]" +2025-06-03T23:36:17.270Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:17.270Z [INFO] server-d4e55654-6b22-e249-bbc5-5fe1e9000d8b: entering follower state: follower="Node at d4e55654-6b22-e249-bbc5-5fe1e9000d8b [Follower]" leader-address= leader-id= +2025-06-03T23:36:17.327Z [WARN] server-d4e55654-6b22-e249-bbc5-5fe1e9000d8b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:17.327Z [INFO] server-d4e55654-6b22-e249-bbc5-5fe1e9000d8b: entering candidate state: node="Node at d4e55654-6b22-e249-bbc5-5fe1e9000d8b [Candidate]" term=2 +2025-06-03T23:36:17.327Z [INFO] server-d4e55654-6b22-e249-bbc5-5fe1e9000d8b: election won: term=2 tally=1 +2025-06-03T23:36:17.327Z [INFO] server-d4e55654-6b22-e249-bbc5-5fe1e9000d8b: entering leader state: leader="Node at d4e55654-6b22-e249-bbc5-5fe1e9000d8b [Leader]" +2025-06-03T23:36:17.337Z [WARN] server-eaf7ccb2-e292-0490-4b04-ad5fa4e6a271: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:17.337Z [INFO] server-eaf7ccb2-e292-0490-4b04-ad5fa4e6a271: entering candidate state: node="Node at eaf7ccb2-e292-0490-4b04-ad5fa4e6a271 [Candidate]" term=3 +2025-06-03T23:36:17.338Z [INFO] server-eaf7ccb2-e292-0490-4b04-ad5fa4e6a271: election won: term=3 tally=1 +2025-06-03T23:36:17.338Z [INFO] server-eaf7ccb2-e292-0490-4b04-ad5fa4e6a271: entering leader state: leader="Node at eaf7ccb2-e292-0490-4b04-ad5fa4e6a271 [Leader]" +2025-06-03T23:36:17.433Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:17.327818711 +0000 UTC m=+12.649573641 (1 nodes), 57.51196ms from start of poll, 57.74256ms from cluster start. Timeout at 2025-06-03 23:36:17.433355631 +0000 UTC m=+12.755110561, 105.53692ms after stability +2025-06-03T23:36:17.441Z [INFO] server-d4e55654-6b22-e249-bbc5-5fe1e9000d8b: starting snapshot up to: index=102 +2025-06-03T23:36:17.441Z [INFO] creating new snapshot: path=/tmp/raft1938832060/snapshots/2-102-1748993777441.tmp +2025-06-03T23:36:17.442Z [INFO] server-d4e55654-6b22-e249-bbc5-5fe1e9000d8b: compacting logs: from=1 to=92 +2025-06-03T23:36:17.442Z [INFO] server-d4e55654-6b22-e249-bbc5-5fe1e9000d8b: snapshot complete up to: index=102 +--- PASS: TestRaft_AutoSnapshot (0.27s) === RUN TestRaft_UserSnapshot -2025-06-03T23:33:18.795Z [INFO] server-d404ea34-419a-5d0e-850f-7eb3066dbb7e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d404ea34-419a-5d0e-850f-7eb3066dbb7e Address:d404ea34-419a-5d0e-850f-7eb3066dbb7e}]" -2025-06-03T23:33:18.795Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:18.795Z [INFO] server-d404ea34-419a-5d0e-850f-7eb3066dbb7e: entering follower state: follower="Node at d404ea34-419a-5d0e-850f-7eb3066dbb7e [Follower]" leader-address= leader-id= -2025-06-03T23:33:18.881Z [WARN] server-d404ea34-419a-5d0e-850f-7eb3066dbb7e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:18.881Z [INFO] server-d404ea34-419a-5d0e-850f-7eb3066dbb7e: entering candidate state: node="Node at d404ea34-419a-5d0e-850f-7eb3066dbb7e [Candidate]" term=2 -2025-06-03T23:33:18.881Z [INFO] server-d404ea34-419a-5d0e-850f-7eb3066dbb7e: election won: term=2 tally=1 -2025-06-03T23:33:18.881Z [INFO] server-d404ea34-419a-5d0e-850f-7eb3066dbb7e: entering leader state: leader="Node at d404ea34-419a-5d0e-850f-7eb3066dbb7e [Leader]" -2025-06-03T23:33:18.987Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:18.881746191 +0000 UTC m=+13.902565561 (1 nodes), 86.35932ms from start of poll, 86.57184ms from cluster start. Timeout at 2025-06-03 23:33:18.987360151 +0000 UTC m=+14.008179521, 105.61396ms after stability -2025-06-03T23:33:18.987Z [ERROR] server-d404ea34-419a-5d0e-850f-7eb3066dbb7e: failed to take snapshot: error="nothing new to snapshot" -2025-06-03T23:33:18.987Z [INFO] server-d404ea34-419a-5d0e-850f-7eb3066dbb7e: starting snapshot up to: index=12 -2025-06-03T23:33:18.987Z [INFO] creating new snapshot: path=/tmp/raft973825394/snapshots/2-12-1748993598987.tmp -2025-06-03T23:33:18.988Z [INFO] server-d404ea34-419a-5d0e-850f-7eb3066dbb7e: compacting logs: from=1 to=2 -2025-06-03T23:33:18.988Z [INFO] server-d404ea34-419a-5d0e-850f-7eb3066dbb7e: snapshot complete up to: index=12 ---- PASS: TestRaft_UserSnapshot (0.19s) +2025-06-03T23:36:17.542Z [INFO] server-b007c69e-f1b1-4a2d-f54c-60e60dd80e2d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b007c69e-f1b1-4a2d-f54c-60e60dd80e2d Address:b007c69e-f1b1-4a2d-f54c-60e60dd80e2d}]" +2025-06-03T23:36:17.542Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:17.542Z [INFO] server-b007c69e-f1b1-4a2d-f54c-60e60dd80e2d: entering follower state: follower="Node at b007c69e-f1b1-4a2d-f54c-60e60dd80e2d [Follower]" leader-address= leader-id= +2025-06-03T23:36:17.633Z [WARN] server-b007c69e-f1b1-4a2d-f54c-60e60dd80e2d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:17.633Z [INFO] server-b007c69e-f1b1-4a2d-f54c-60e60dd80e2d: entering candidate state: node="Node at b007c69e-f1b1-4a2d-f54c-60e60dd80e2d [Candidate]" term=2 +2025-06-03T23:36:17.633Z [INFO] server-b007c69e-f1b1-4a2d-f54c-60e60dd80e2d: election won: term=2 tally=1 +2025-06-03T23:36:17.633Z [INFO] server-b007c69e-f1b1-4a2d-f54c-60e60dd80e2d: entering leader state: leader="Node at b007c69e-f1b1-4a2d-f54c-60e60dd80e2d [Leader]" +2025-06-03T23:36:17.739Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:17.633967951 +0000 UTC m=+12.955722881 (1 nodes), 91.34228ms from start of poll, 91.56004ms from cluster start. Timeout at 2025-06-03 23:36:17.739802751 +0000 UTC m=+13.061557681, 105.8348ms after stability +2025-06-03T23:36:17.739Z [ERROR] server-b007c69e-f1b1-4a2d-f54c-60e60dd80e2d: failed to take snapshot: error="nothing new to snapshot" +2025-06-03T23:36:17.740Z [INFO] server-b007c69e-f1b1-4a2d-f54c-60e60dd80e2d: starting snapshot up to: index=12 +2025-06-03T23:36:17.740Z [INFO] creating new snapshot: path=/tmp/raft4271668541/snapshots/2-12-1748993777740.tmp +2025-06-03T23:36:17.740Z [INFO] server-b007c69e-f1b1-4a2d-f54c-60e60dd80e2d: compacting logs: from=1 to=2 +2025-06-03T23:36:17.740Z [INFO] server-b007c69e-f1b1-4a2d-f54c-60e60dd80e2d: snapshot complete up to: index=12 +--- PASS: TestRaft_UserSnapshot (0.20s) === RUN TestRaft_UserRestore === RUN TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false -2025-06-03T23:33:18.989Z [INFO] server-a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3 Address:a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3} {Suffrage:Voter ID:server-7aea9213-85fe-8250-3116-046ee5979da2 Address:7aea9213-85fe-8250-3116-046ee5979da2} {Suffrage:Voter ID:server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba Address:6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba}]" -2025-06-03T23:33:18.989Z [INFO] server-a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3: entering follower state: follower="Node at a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3 [Follower]" leader-address= leader-id= -2025-06-03T23:33:18.989Z [INFO] server-7aea9213-85fe-8250-3116-046ee5979da2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3 Address:a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3} {Suffrage:Voter ID:server-7aea9213-85fe-8250-3116-046ee5979da2 Address:7aea9213-85fe-8250-3116-046ee5979da2} {Suffrage:Voter ID:server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba Address:6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba}]" -2025-06-03T23:33:18.989Z [INFO] server-7aea9213-85fe-8250-3116-046ee5979da2: entering follower state: follower="Node at 7aea9213-85fe-8250-3116-046ee5979da2 [Follower]" leader-address= leader-id= -2025-06-03T23:33:18.989Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3 Address:a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3} {Suffrage:Voter ID:server-7aea9213-85fe-8250-3116-046ee5979da2 Address:7aea9213-85fe-8250-3116-046ee5979da2} {Suffrage:Voter ID:server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba Address:6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba}]" -2025-06-03T23:33:18.989Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:18.989Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: entering follower state: follower="Node at 6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba [Follower]" leader-address= leader-id= -2025-06-03T23:33:19.496Z [WARN] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:19.496Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: entering candidate state: node="Node at 6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba [Candidate]" term=2 -2025-06-03T23:33:19.496Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: election won: term=2 tally=2 -2025-06-03T23:33:19.496Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: entering leader state: leader="Node at 6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba [Leader]" -2025-06-03T23:33:19.496Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: added peer, starting replication: peer=server-a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3 -2025-06-03T23:33:19.496Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: added peer, starting replication: peer=server-7aea9213-85fe-8250-3116-046ee5979da2 -2025-06-03T23:33:19.496Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: pipelining replication: peer="{Voter server-7aea9213-85fe-8250-3116-046ee5979da2 7aea9213-85fe-8250-3116-046ee5979da2}" -2025-06-03T23:33:19.496Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: pipelining replication: peer="{Voter server-a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3 a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3}" -2025-06-03T23:33:20.501Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:19.496365871 +0000 UTC m=+14.517185281 (1 nodes), 506.74776ms from start of poll, 507.24668ms from cluster start. Timeout at 2025-06-03 23:33:20.501380951 +0000 UTC m=+15.522200321, 1.00501504s after stability -2025-06-03T23:33:20.502Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: starting snapshot up to: index=12 -2025-06-03T23:33:20.502Z [INFO] creating new snapshot: path=/tmp/raft2121999210/snapshots/2-12-1748993600502.tmp -2025-06-03T23:33:20.502Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: snapshot complete up to: index=12 -2025-06-03T23:33:20.505Z [INFO] creating new snapshot: path=/tmp/raft2121999210/snapshots/2-23-1748993600505.tmp -2025-06-03T23:33:20.505Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: copied to local snapshot: bytes=71 -2025-06-03T23:33:20.505Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: snapshot restore progress: id=2-23-1748993600505 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:20.506Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: restored user snapshot: index=23 -2025-06-03T23:33:20.506Z [ERROR] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: failed to get log: index=23 error="log not found" -2025-06-03T23:33:20.506Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: aborting pipeline replication: peer="{Voter server-7aea9213-85fe-8250-3116-046ee5979da2 7aea9213-85fe-8250-3116-046ee5979da2}" -2025-06-03T23:33:20.506Z [ERROR] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: failed to get log: index=23 error="log not found" -2025-06-03T23:33:20.506Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: aborting pipeline replication: peer="{Voter server-a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3 a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3}" -2025-06-03T23:33:20.513Z [ERROR] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: failed to get log: index=23 error="log not found" -2025-06-03T23:33:20.513Z [ERROR] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: failed to get log: index=23 error="log not found" -2025-06-03T23:33:20.513Z [INFO] creating new snapshot: path=/tmp/raft69116329/snapshots/2-23-1748993600513.tmp -2025-06-03T23:33:20.513Z [INFO] creating new snapshot: path=/tmp/raft3222983108/snapshots/2-23-1748993600513.tmp -2025-06-03T23:33:20.514Z [INFO] server-a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:20.514Z [INFO] server-7aea9213-85fe-8250-3116-046ee5979da2: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:20.514Z [INFO] server-a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3: copied to local snapshot: bytes=71 -2025-06-03T23:33:20.514Z [INFO] server-7aea9213-85fe-8250-3116-046ee5979da2: copied to local snapshot: bytes=71 -2025-06-03T23:33:20.514Z [INFO] server-7aea9213-85fe-8250-3116-046ee5979da2: snapshot restore progress: id=2-23-1748993600513 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:20.514Z [INFO] server-7aea9213-85fe-8250-3116-046ee5979da2: Installed remote snapshot -2025-06-03T23:33:20.514Z [INFO] server-a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3: snapshot restore progress: id=2-23-1748993600513 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:20.514Z [INFO] server-a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3: Installed remote snapshot -2025-06-03T23:33:20.514Z [WARN] server-7aea9213-85fe-8250-3116-046ee5979da2: skipping application of old log: index=22 -2025-06-03T23:33:20.514Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: pipelining replication: peer="{Voter server-7aea9213-85fe-8250-3116-046ee5979da2 7aea9213-85fe-8250-3116-046ee5979da2}" -2025-06-03T23:33:20.514Z [WARN] server-a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3: skipping application of old log: index=22 -2025-06-03T23:33:20.514Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:20.514Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: pipelining replication: peer="{Voter server-a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3 a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3}" -2025-06-03T23:33:21.520Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:20.515000791 +0000 UTC m=+15.535820161 (2 nodes), 1µs from start of poll, 1.52588156s from cluster start. Timeout at 2025-06-03 23:33:21.520219591 +0000 UTC m=+16.541039001, 1.00521884s after stability -2025-06-03T23:33:21.532Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: aborting pipeline replication: peer="{Voter server-7aea9213-85fe-8250-3116-046ee5979da2 7aea9213-85fe-8250-3116-046ee5979da2}" -2025-06-03T23:33:21.532Z [INFO] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: aborting pipeline replication: peer="{Voter server-a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3 a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3}" -2025-06-03T23:33:22.033Z [ERROR] server-6fd882fb-1de9-7f0a-e3b2-1f61a72da8ba: failed to heartbeat to: peer=a6f6ada3-7dd6-f82f-3b3d-d01b4389ead3 backoff time=10ms error="command timed out" +2025-06-03T23:36:17.741Z [INFO] server-5cae0cba-a8fd-a416-3e77-ee0c69394035: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5cae0cba-a8fd-a416-3e77-ee0c69394035 Address:5cae0cba-a8fd-a416-3e77-ee0c69394035} {Suffrage:Voter ID:server-0c7f11fc-ac3a-7848-4417-55c9bfeb3739 Address:0c7f11fc-ac3a-7848-4417-55c9bfeb3739} {Suffrage:Voter ID:server-37051d4f-f925-6936-2496-4186b5ae18fa Address:37051d4f-f925-6936-2496-4186b5ae18fa}]" +2025-06-03T23:36:17.741Z [INFO] server-5cae0cba-a8fd-a416-3e77-ee0c69394035: entering follower state: follower="Node at 5cae0cba-a8fd-a416-3e77-ee0c69394035 [Follower]" leader-address= leader-id= +2025-06-03T23:36:17.741Z [INFO] server-0c7f11fc-ac3a-7848-4417-55c9bfeb3739: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5cae0cba-a8fd-a416-3e77-ee0c69394035 Address:5cae0cba-a8fd-a416-3e77-ee0c69394035} {Suffrage:Voter ID:server-0c7f11fc-ac3a-7848-4417-55c9bfeb3739 Address:0c7f11fc-ac3a-7848-4417-55c9bfeb3739} {Suffrage:Voter ID:server-37051d4f-f925-6936-2496-4186b5ae18fa Address:37051d4f-f925-6936-2496-4186b5ae18fa}]" +2025-06-03T23:36:17.742Z [INFO] server-0c7f11fc-ac3a-7848-4417-55c9bfeb3739: entering follower state: follower="Node at 0c7f11fc-ac3a-7848-4417-55c9bfeb3739 [Follower]" leader-address= leader-id= +2025-06-03T23:36:17.742Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5cae0cba-a8fd-a416-3e77-ee0c69394035 Address:5cae0cba-a8fd-a416-3e77-ee0c69394035} {Suffrage:Voter ID:server-0c7f11fc-ac3a-7848-4417-55c9bfeb3739 Address:0c7f11fc-ac3a-7848-4417-55c9bfeb3739} {Suffrage:Voter ID:server-37051d4f-f925-6936-2496-4186b5ae18fa Address:37051d4f-f925-6936-2496-4186b5ae18fa}]" +2025-06-03T23:36:17.742Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:17.742Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: entering follower state: follower="Node at 37051d4f-f925-6936-2496-4186b5ae18fa [Follower]" leader-address= leader-id= +2025-06-03T23:36:18.690Z [WARN] server-37051d4f-f925-6936-2496-4186b5ae18fa: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:18.690Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: entering candidate state: node="Node at 37051d4f-f925-6936-2496-4186b5ae18fa [Candidate]" term=2 +2025-06-03T23:36:18.690Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: election won: term=2 tally=2 +2025-06-03T23:36:18.690Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: entering leader state: leader="Node at 37051d4f-f925-6936-2496-4186b5ae18fa [Leader]" +2025-06-03T23:36:18.690Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: added peer, starting replication: peer=server-5cae0cba-a8fd-a416-3e77-ee0c69394035 +2025-06-03T23:36:18.690Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: added peer, starting replication: peer=server-0c7f11fc-ac3a-7848-4417-55c9bfeb3739 +2025-06-03T23:36:18.691Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: pipelining replication: peer="{Voter server-0c7f11fc-ac3a-7848-4417-55c9bfeb3739 0c7f11fc-ac3a-7848-4417-55c9bfeb3739}" +2025-06-03T23:36:18.691Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: pipelining replication: peer="{Voter server-5cae0cba-a8fd-a416-3e77-ee0c69394035 5cae0cba-a8fd-a416-3e77-ee0c69394035}" +2025-06-03T23:36:19.696Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:18.690996431 +0000 UTC m=+14.012751361 (1 nodes), 948.8432ms from start of poll, 949.3862ms from cluster start. Timeout at 2025-06-03 23:36:19.696338391 +0000 UTC m=+15.018093321, 1.00534196s after stability +2025-06-03T23:36:19.697Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: starting snapshot up to: index=12 +2025-06-03T23:36:19.697Z [INFO] creating new snapshot: path=/tmp/raft3854969871/snapshots/2-12-1748993779697.tmp +2025-06-03T23:36:19.697Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: snapshot complete up to: index=12 +2025-06-03T23:36:19.698Z [INFO] creating new snapshot: path=/tmp/raft3854969871/snapshots/2-23-1748993779698.tmp +2025-06-03T23:36:19.699Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: copied to local snapshot: bytes=71 +2025-06-03T23:36:19.699Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: snapshot restore progress: id=2-23-1748993779698 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:19.699Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: restored user snapshot: index=23 +2025-06-03T23:36:19.699Z [ERROR] server-37051d4f-f925-6936-2496-4186b5ae18fa: failed to get log: index=23 error="log not found" +2025-06-03T23:36:19.699Z [ERROR] server-37051d4f-f925-6936-2496-4186b5ae18fa: failed to get log: index=23 error="log not found" +2025-06-03T23:36:19.699Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: aborting pipeline replication: peer="{Voter server-5cae0cba-a8fd-a416-3e77-ee0c69394035 5cae0cba-a8fd-a416-3e77-ee0c69394035}" +2025-06-03T23:36:19.699Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: aborting pipeline replication: peer="{Voter server-0c7f11fc-ac3a-7848-4417-55c9bfeb3739 0c7f11fc-ac3a-7848-4417-55c9bfeb3739}" +2025-06-03T23:36:19.709Z [ERROR] server-37051d4f-f925-6936-2496-4186b5ae18fa: failed to get log: index=23 error="log not found" +2025-06-03T23:36:19.709Z [ERROR] server-37051d4f-f925-6936-2496-4186b5ae18fa: failed to get log: index=23 error="log not found" +2025-06-03T23:36:19.709Z [INFO] creating new snapshot: path=/tmp/raft3311350762/snapshots/2-23-1748993779709.tmp +2025-06-03T23:36:19.710Z [INFO] creating new snapshot: path=/tmp/raft2265139159/snapshots/2-23-1748993779710.tmp +2025-06-03T23:36:19.710Z [INFO] server-5cae0cba-a8fd-a416-3e77-ee0c69394035: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:19.710Z [INFO] server-0c7f11fc-ac3a-7848-4417-55c9bfeb3739: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:19.710Z [INFO] server-5cae0cba-a8fd-a416-3e77-ee0c69394035: copied to local snapshot: bytes=71 +2025-06-03T23:36:19.710Z [INFO] server-0c7f11fc-ac3a-7848-4417-55c9bfeb3739: copied to local snapshot: bytes=71 +2025-06-03T23:36:19.710Z [INFO] server-5cae0cba-a8fd-a416-3e77-ee0c69394035: snapshot restore progress: id=2-23-1748993779709 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:19.710Z [INFO] server-5cae0cba-a8fd-a416-3e77-ee0c69394035: Installed remote snapshot +2025-06-03T23:36:19.710Z [INFO] server-0c7f11fc-ac3a-7848-4417-55c9bfeb3739: snapshot restore progress: id=2-23-1748993779710 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:19.711Z [INFO] server-0c7f11fc-ac3a-7848-4417-55c9bfeb3739: Installed remote snapshot +2025-06-03T23:36:19.711Z [WARN] server-5cae0cba-a8fd-a416-3e77-ee0c69394035: skipping application of old log: index=22 +2025-06-03T23:36:19.711Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: pipelining replication: peer="{Voter server-5cae0cba-a8fd-a416-3e77-ee0c69394035 5cae0cba-a8fd-a416-3e77-ee0c69394035}" +2025-06-03T23:36:19.711Z [WARN] server-0c7f11fc-ac3a-7848-4417-55c9bfeb3739: skipping application of old log: index=22 +2025-06-03T23:36:19.711Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: pipelining replication: peer="{Voter server-0c7f11fc-ac3a-7848-4417-55c9bfeb3739 0c7f11fc-ac3a-7848-4417-55c9bfeb3739}" +2025-06-03T23:36:19.711Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:20.716Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:19.711186751 +0000 UTC m=+15.032941681 (2 nodes), 1.28µs from start of poll, 1.96957652s from cluster start. Timeout at 2025-06-03 23:36:20.716763351 +0000 UTC m=+16.038518281, 1.0055766s after stability +2025-06-03T23:36:20.728Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: aborting pipeline replication: peer="{Voter server-0c7f11fc-ac3a-7848-4417-55c9bfeb3739 0c7f11fc-ac3a-7848-4417-55c9bfeb3739}" +2025-06-03T23:36:20.728Z [INFO] server-37051d4f-f925-6936-2496-4186b5ae18fa: aborting pipeline replication: peer="{Voter server-5cae0cba-a8fd-a416-3e77-ee0c69394035 5cae0cba-a8fd-a416-3e77-ee0c69394035}" === RUN TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false -2025-06-03T23:33:22.034Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a277c387-70c5-c656-3586-f92e88f65f8c Address:a277c387-70c5-c656-3586-f92e88f65f8c} {Suffrage:Voter ID:server-b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf Address:b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf} {Suffrage:Voter ID:server-5f485f2f-54a0-001b-6824-8dd51f8c3960 Address:5f485f2f-54a0-001b-6824-8dd51f8c3960}]" -2025-06-03T23:33:22.034Z [INFO] server-b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a277c387-70c5-c656-3586-f92e88f65f8c Address:a277c387-70c5-c656-3586-f92e88f65f8c} {Suffrage:Voter ID:server-b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf Address:b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf} {Suffrage:Voter ID:server-5f485f2f-54a0-001b-6824-8dd51f8c3960 Address:5f485f2f-54a0-001b-6824-8dd51f8c3960}]" -2025-06-03T23:33:22.034Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: entering follower state: follower="Node at a277c387-70c5-c656-3586-f92e88f65f8c [Follower]" leader-address= leader-id= -2025-06-03T23:33:22.034Z [INFO] server-5f485f2f-54a0-001b-6824-8dd51f8c3960: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a277c387-70c5-c656-3586-f92e88f65f8c Address:a277c387-70c5-c656-3586-f92e88f65f8c} {Suffrage:Voter ID:server-b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf Address:b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf} {Suffrage:Voter ID:server-5f485f2f-54a0-001b-6824-8dd51f8c3960 Address:5f485f2f-54a0-001b-6824-8dd51f8c3960}]" -2025-06-03T23:33:22.034Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:22.034Z [INFO] server-b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf: entering follower state: follower="Node at b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf [Follower]" leader-address= leader-id= -2025-06-03T23:33:22.034Z [INFO] server-5f485f2f-54a0-001b-6824-8dd51f8c3960: entering follower state: follower="Node at 5f485f2f-54a0-001b-6824-8dd51f8c3960 [Follower]" leader-address= leader-id= -2025-06-03T23:33:22.568Z [WARN] server-a277c387-70c5-c656-3586-f92e88f65f8c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:22.568Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: entering candidate state: node="Node at a277c387-70c5-c656-3586-f92e88f65f8c [Candidate]" term=2 -2025-06-03T23:33:22.568Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: election won: term=2 tally=2 -2025-06-03T23:33:22.568Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: entering leader state: leader="Node at a277c387-70c5-c656-3586-f92e88f65f8c [Leader]" -2025-06-03T23:33:22.568Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: added peer, starting replication: peer=server-b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf -2025-06-03T23:33:22.568Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: added peer, starting replication: peer=server-5f485f2f-54a0-001b-6824-8dd51f8c3960 -2025-06-03T23:33:22.569Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: pipelining replication: peer="{Voter server-b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf}" -2025-06-03T23:33:22.569Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: pipelining replication: peer="{Voter server-5f485f2f-54a0-001b-6824-8dd51f8c3960 5f485f2f-54a0-001b-6824-8dd51f8c3960}" -2025-06-03T23:33:23.574Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:22.568783391 +0000 UTC m=+17.589602761 (1 nodes), 534.24232ms from start of poll, 534.75072ms from cluster start. Timeout at 2025-06-03 23:33:23.574526311 +0000 UTC m=+18.595345681, 1.00574292s after stability -2025-06-03T23:33:23.576Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: starting snapshot up to: index=12 -2025-06-03T23:33:23.576Z [INFO] creating new snapshot: path=/tmp/raft1286544017/snapshots/2-12-1748993603576.tmp -2025-06-03T23:33:23.577Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: snapshot complete up to: index=12 -2025-06-03T23:33:23.578Z [INFO] creating new snapshot: path=/tmp/raft1286544017/snapshots/2-23-1748993603578.tmp -2025-06-03T23:33:23.578Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: copied to local snapshot: bytes=71 -2025-06-03T23:33:23.579Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: snapshot restore progress: id=2-23-1748993603578 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:23.579Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: removing all old logs from log store -2025-06-03T23:33:23.579Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: compacting logs: from=1 to=22 -2025-06-03T23:33:23.579Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: restored user snapshot: index=23 -2025-06-03T23:33:23.579Z [ERROR] server-a277c387-70c5-c656-3586-f92e88f65f8c: failed to get log: index=22 error="log not found" -2025-06-03T23:33:23.579Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: aborting pipeline replication: peer="{Voter server-5f485f2f-54a0-001b-6824-8dd51f8c3960 5f485f2f-54a0-001b-6824-8dd51f8c3960}" -2025-06-03T23:33:23.579Z [ERROR] server-a277c387-70c5-c656-3586-f92e88f65f8c: failed to get log: index=22 error="log not found" -2025-06-03T23:33:23.579Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: aborting pipeline replication: peer="{Voter server-b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf}" -2025-06-03T23:33:23.585Z [ERROR] server-a277c387-70c5-c656-3586-f92e88f65f8c: failed to get log: index=22 error="log not found" -2025-06-03T23:33:23.585Z [ERROR] server-a277c387-70c5-c656-3586-f92e88f65f8c: failed to get log: index=22 error="log not found" -2025-06-03T23:33:23.586Z [INFO] creating new snapshot: path=/tmp/raft2815401465/snapshots/2-23-1748993603586.tmp -2025-06-03T23:33:23.586Z [INFO] creating new snapshot: path=/tmp/raft1170716926/snapshots/2-23-1748993603586.tmp -2025-06-03T23:33:23.586Z [INFO] server-b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:23.586Z [INFO] server-5f485f2f-54a0-001b-6824-8dd51f8c3960: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:23.587Z [INFO] server-b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf: copied to local snapshot: bytes=71 -2025-06-03T23:33:23.587Z [INFO] server-5f485f2f-54a0-001b-6824-8dd51f8c3960: copied to local snapshot: bytes=71 -2025-06-03T23:33:23.587Z [INFO] server-5f485f2f-54a0-001b-6824-8dd51f8c3960: snapshot restore progress: id=2-23-1748993603586 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:23.587Z [INFO] server-5f485f2f-54a0-001b-6824-8dd51f8c3960: removing all old logs from log store -2025-06-03T23:33:23.587Z [INFO] server-5f485f2f-54a0-001b-6824-8dd51f8c3960: compacting logs: from=1 to=22 -2025-06-03T23:33:23.587Z [INFO] server-5f485f2f-54a0-001b-6824-8dd51f8c3960: Installed remote snapshot -2025-06-03T23:33:23.587Z [INFO] server-b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf: snapshot restore progress: id=2-23-1748993603586 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:23.587Z [INFO] server-b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf: removing all old logs from log store -2025-06-03T23:33:23.587Z [INFO] server-b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf: compacting logs: from=1 to=22 -2025-06-03T23:33:23.587Z [INFO] server-b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf: Installed remote snapshot -2025-06-03T23:33:23.588Z [WARN] server-5f485f2f-54a0-001b-6824-8dd51f8c3960: skipping application of old log: index=23 -2025-06-03T23:33:23.588Z [WARN] server-b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf: skipping application of old log: index=23 -2025-06-03T23:33:23.588Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: pipelining replication: peer="{Voter server-5f485f2f-54a0-001b-6824-8dd51f8c3960 5f485f2f-54a0-001b-6824-8dd51f8c3960}" -2025-06-03T23:33:23.588Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:23.588Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: pipelining replication: peer="{Voter server-b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf}" -2025-06-03T23:33:24.593Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:23.588205471 +0000 UTC m=+18.609024841 (2 nodes), 1.16µs from start of poll, 1.5541728s from cluster start. Timeout at 2025-06-03 23:33:24.593857311 +0000 UTC m=+19.614676681, 1.00565184s after stability -2025-06-03T23:33:24.606Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: aborting pipeline replication: peer="{Voter server-5f485f2f-54a0-001b-6824-8dd51f8c3960 5f485f2f-54a0-001b-6824-8dd51f8c3960}" -2025-06-03T23:33:24.606Z [INFO] server-a277c387-70c5-c656-3586-f92e88f65f8c: aborting pipeline replication: peer="{Voter server-b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf b5bda890-1d4e-5d3e-28a8-a5f1f4458dcf}" +2025-06-03T23:36:20.729Z [INFO] server-08eefb7a-2649-6195-3e45-b88186bad415: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08eefb7a-2649-6195-3e45-b88186bad415 Address:08eefb7a-2649-6195-3e45-b88186bad415} {Suffrage:Voter ID:server-49478ce0-00af-ab07-32f3-1370b9d0691e Address:49478ce0-00af-ab07-32f3-1370b9d0691e} {Suffrage:Voter ID:server-2b1c53fc-42d2-dd0b-bd20-a17e3478be0a Address:2b1c53fc-42d2-dd0b-bd20-a17e3478be0a}]" +2025-06-03T23:36:20.729Z [INFO] server-08eefb7a-2649-6195-3e45-b88186bad415: entering follower state: follower="Node at 08eefb7a-2649-6195-3e45-b88186bad415 [Follower]" leader-address= leader-id= +2025-06-03T23:36:20.729Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08eefb7a-2649-6195-3e45-b88186bad415 Address:08eefb7a-2649-6195-3e45-b88186bad415} {Suffrage:Voter ID:server-49478ce0-00af-ab07-32f3-1370b9d0691e Address:49478ce0-00af-ab07-32f3-1370b9d0691e} {Suffrage:Voter ID:server-2b1c53fc-42d2-dd0b-bd20-a17e3478be0a Address:2b1c53fc-42d2-dd0b-bd20-a17e3478be0a}]" +2025-06-03T23:36:20.730Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: entering follower state: follower="Node at 49478ce0-00af-ab07-32f3-1370b9d0691e [Follower]" leader-address= leader-id= +2025-06-03T23:36:20.730Z [INFO] server-2b1c53fc-42d2-dd0b-bd20-a17e3478be0a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08eefb7a-2649-6195-3e45-b88186bad415 Address:08eefb7a-2649-6195-3e45-b88186bad415} {Suffrage:Voter ID:server-49478ce0-00af-ab07-32f3-1370b9d0691e Address:49478ce0-00af-ab07-32f3-1370b9d0691e} {Suffrage:Voter ID:server-2b1c53fc-42d2-dd0b-bd20-a17e3478be0a Address:2b1c53fc-42d2-dd0b-bd20-a17e3478be0a}]" +2025-06-03T23:36:20.730Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:20.730Z [INFO] server-2b1c53fc-42d2-dd0b-bd20-a17e3478be0a: entering follower state: follower="Node at 2b1c53fc-42d2-dd0b-bd20-a17e3478be0a [Follower]" leader-address= leader-id= +2025-06-03T23:36:21.331Z [WARN] server-49478ce0-00af-ab07-32f3-1370b9d0691e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:21.331Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: entering candidate state: node="Node at 49478ce0-00af-ab07-32f3-1370b9d0691e [Candidate]" term=2 +2025-06-03T23:36:21.331Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: election won: term=2 tally=2 +2025-06-03T23:36:21.331Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: entering leader state: leader="Node at 49478ce0-00af-ab07-32f3-1370b9d0691e [Leader]" +2025-06-03T23:36:21.331Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: added peer, starting replication: peer=server-08eefb7a-2649-6195-3e45-b88186bad415 +2025-06-03T23:36:21.331Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: added peer, starting replication: peer=server-2b1c53fc-42d2-dd0b-bd20-a17e3478be0a +2025-06-03T23:36:21.332Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: pipelining replication: peer="{Voter server-08eefb7a-2649-6195-3e45-b88186bad415 08eefb7a-2649-6195-3e45-b88186bad415}" +2025-06-03T23:36:21.332Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: pipelining replication: peer="{Voter server-2b1c53fc-42d2-dd0b-bd20-a17e3478be0a 2b1c53fc-42d2-dd0b-bd20-a17e3478be0a}" +2025-06-03T23:36:22.337Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:21.331980231 +0000 UTC m=+16.653735161 (1 nodes), 601.71248ms from start of poll, 602.34196ms from cluster start. Timeout at 2025-06-03 23:36:22.337367751 +0000 UTC m=+17.659122721, 1.00538756s after stability +2025-06-03T23:36:22.338Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: starting snapshot up to: index=12 +2025-06-03T23:36:22.338Z [INFO] creating new snapshot: path=/tmp/raft4017983386/snapshots/2-12-1748993782338.tmp +2025-06-03T23:36:22.338Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: snapshot complete up to: index=12 +2025-06-03T23:36:22.339Z [INFO] creating new snapshot: path=/tmp/raft4017983386/snapshots/2-23-1748993782339.tmp +2025-06-03T23:36:22.340Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: copied to local snapshot: bytes=71 +2025-06-03T23:36:22.341Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: snapshot restore progress: id=2-23-1748993782339 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:22.341Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: removing all old logs from log store +2025-06-03T23:36:22.341Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: compacting logs: from=1 to=22 +2025-06-03T23:36:22.341Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: restored user snapshot: index=23 +2025-06-03T23:36:22.341Z [ERROR] server-49478ce0-00af-ab07-32f3-1370b9d0691e: failed to get log: index=22 error="log not found" +2025-06-03T23:36:22.341Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: aborting pipeline replication: peer="{Voter server-2b1c53fc-42d2-dd0b-bd20-a17e3478be0a 2b1c53fc-42d2-dd0b-bd20-a17e3478be0a}" +2025-06-03T23:36:22.341Z [ERROR] server-49478ce0-00af-ab07-32f3-1370b9d0691e: failed to get log: index=22 error="log not found" +2025-06-03T23:36:22.341Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: aborting pipeline replication: peer="{Voter server-08eefb7a-2649-6195-3e45-b88186bad415 08eefb7a-2649-6195-3e45-b88186bad415}" +2025-06-03T23:36:22.348Z [ERROR] server-49478ce0-00af-ab07-32f3-1370b9d0691e: failed to get log: index=22 error="log not found" +2025-06-03T23:36:22.348Z [ERROR] server-49478ce0-00af-ab07-32f3-1370b9d0691e: failed to get log: index=22 error="log not found" +2025-06-03T23:36:22.349Z [INFO] creating new snapshot: path=/tmp/raft1830966938/snapshots/2-23-1748993782349.tmp +2025-06-03T23:36:22.349Z [INFO] creating new snapshot: path=/tmp/raft2958154394/snapshots/2-23-1748993782349.tmp +2025-06-03T23:36:22.349Z [INFO] server-08eefb7a-2649-6195-3e45-b88186bad415: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:22.349Z [INFO] server-2b1c53fc-42d2-dd0b-bd20-a17e3478be0a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:22.349Z [INFO] server-08eefb7a-2649-6195-3e45-b88186bad415: copied to local snapshot: bytes=71 +2025-06-03T23:36:22.350Z [INFO] server-2b1c53fc-42d2-dd0b-bd20-a17e3478be0a: copied to local snapshot: bytes=71 +2025-06-03T23:36:22.350Z [INFO] server-08eefb7a-2649-6195-3e45-b88186bad415: snapshot restore progress: id=2-23-1748993782349 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:22.350Z [INFO] server-08eefb7a-2649-6195-3e45-b88186bad415: removing all old logs from log store +2025-06-03T23:36:22.350Z [INFO] server-08eefb7a-2649-6195-3e45-b88186bad415: compacting logs: from=1 to=22 +2025-06-03T23:36:22.350Z [INFO] server-08eefb7a-2649-6195-3e45-b88186bad415: Installed remote snapshot +2025-06-03T23:36:22.350Z [INFO] server-2b1c53fc-42d2-dd0b-bd20-a17e3478be0a: snapshot restore progress: id=2-23-1748993782349 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:22.350Z [INFO] server-2b1c53fc-42d2-dd0b-bd20-a17e3478be0a: removing all old logs from log store +2025-06-03T23:36:22.350Z [INFO] server-2b1c53fc-42d2-dd0b-bd20-a17e3478be0a: compacting logs: from=1 to=22 +2025-06-03T23:36:22.350Z [WARN] server-08eefb7a-2649-6195-3e45-b88186bad415: skipping application of old log: index=22 +2025-06-03T23:36:22.350Z [INFO] server-2b1c53fc-42d2-dd0b-bd20-a17e3478be0a: Installed remote snapshot +2025-06-03T23:36:22.350Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: pipelining replication: peer="{Voter server-08eefb7a-2649-6195-3e45-b88186bad415 08eefb7a-2649-6195-3e45-b88186bad415}" +2025-06-03T23:36:22.350Z [WARN] server-2b1c53fc-42d2-dd0b-bd20-a17e3478be0a: skipping application of old log: index=23 +2025-06-03T23:36:22.350Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:22.350Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: pipelining replication: peer="{Voter server-2b1c53fc-42d2-dd0b-bd20-a17e3478be0a 2b1c53fc-42d2-dd0b-bd20-a17e3478be0a}" +2025-06-03T23:36:23.355Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:22.350538031 +0000 UTC m=+17.672292961 (2 nodes), 5.32µs from start of poll, 1.62089976s from cluster start. Timeout at 2025-06-03 23:36:23.355885751 +0000 UTC m=+18.677640681, 1.00534772s after stability +2025-06-03T23:36:23.368Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: aborting pipeline replication: peer="{Voter server-2b1c53fc-42d2-dd0b-bd20-a17e3478be0a 2b1c53fc-42d2-dd0b-bd20-a17e3478be0a}" +2025-06-03T23:36:23.368Z [INFO] server-49478ce0-00af-ab07-32f3-1370b9d0691e: aborting pipeline replication: peer="{Voter server-08eefb7a-2649-6195-3e45-b88186bad415 08eefb7a-2649-6195-3e45-b88186bad415}" === RUN TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true -2025-06-03T23:33:24.607Z [INFO] server-a79068ed-386d-b6c0-80cf-6976fba192d8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a79068ed-386d-b6c0-80cf-6976fba192d8 Address:a79068ed-386d-b6c0-80cf-6976fba192d8} {Suffrage:Voter ID:server-40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7 Address:40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7} {Suffrage:Voter ID:server-4d865d30-7813-aa37-cd6d-15c4b4c4ca7d Address:4d865d30-7813-aa37-cd6d-15c4b4c4ca7d}]" -2025-06-03T23:33:24.607Z [INFO] server-a79068ed-386d-b6c0-80cf-6976fba192d8: entering follower state: follower="Node at a79068ed-386d-b6c0-80cf-6976fba192d8 [Follower]" leader-address= leader-id= -2025-06-03T23:33:24.607Z [INFO] server-40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a79068ed-386d-b6c0-80cf-6976fba192d8 Address:a79068ed-386d-b6c0-80cf-6976fba192d8} {Suffrage:Voter ID:server-40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7 Address:40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7} {Suffrage:Voter ID:server-4d865d30-7813-aa37-cd6d-15c4b4c4ca7d Address:4d865d30-7813-aa37-cd6d-15c4b4c4ca7d}]" -2025-06-03T23:33:24.607Z [INFO] server-40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7: entering follower state: follower="Node at 40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7 [Follower]" leader-address= leader-id= -2025-06-03T23:33:24.607Z [INFO] server-4d865d30-7813-aa37-cd6d-15c4b4c4ca7d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a79068ed-386d-b6c0-80cf-6976fba192d8 Address:a79068ed-386d-b6c0-80cf-6976fba192d8} {Suffrage:Voter ID:server-40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7 Address:40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7} {Suffrage:Voter ID:server-4d865d30-7813-aa37-cd6d-15c4b4c4ca7d Address:4d865d30-7813-aa37-cd6d-15c4b4c4ca7d}]" -2025-06-03T23:33:24.607Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:24.607Z [INFO] server-4d865d30-7813-aa37-cd6d-15c4b4c4ca7d: entering follower state: follower="Node at 4d865d30-7813-aa37-cd6d-15c4b4c4ca7d [Follower]" leader-address= leader-id= -2025-06-03T23:33:25.321Z [WARN] server-40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:25.321Z [INFO] server-40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7: entering candidate state: node="Node at 40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7 [Candidate]" term=2 -2025-06-03T23:33:25.321Z [INFO] server-40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7: election won: term=2 tally=2 -2025-06-03T23:33:25.321Z [INFO] server-40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7: entering leader state: leader="Node at 40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7 [Leader]" -2025-06-03T23:33:25.321Z [INFO] server-40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7: added peer, starting replication: peer=server-a79068ed-386d-b6c0-80cf-6976fba192d8 -2025-06-03T23:33:25.321Z [INFO] server-40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7: added peer, starting replication: peer=server-4d865d30-7813-aa37-cd6d-15c4b4c4ca7d -2025-06-03T23:33:25.321Z [INFO] server-40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7: pipelining replication: peer="{Voter server-a79068ed-386d-b6c0-80cf-6976fba192d8 a79068ed-386d-b6c0-80cf-6976fba192d8}" -2025-06-03T23:33:25.321Z [INFO] server-40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7: pipelining replication: peer="{Voter server-4d865d30-7813-aa37-cd6d-15c4b4c4ca7d 4d865d30-7813-aa37-cd6d-15c4b4c4ca7d}" -2025-06-03T23:33:26.326Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:25.321411071 +0000 UTC m=+20.342230441 (1 nodes), 713.51028ms from start of poll, 713.95308ms from cluster start. Timeout at 2025-06-03 23:33:26.326614871 +0000 UTC m=+21.347434281, 1.00520384s after stability -2025-06-03T23:33:26.327Z [INFO] server-40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7: starting snapshot up to: index=12 -2025-06-03T23:33:26.327Z [INFO] creating new snapshot: path=/tmp/raft54678450/snapshots/2-12-1748993606327.tmp -2025-06-03T23:33:26.328Z [INFO] server-40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7: snapshot complete up to: index=12 -2025-06-03T23:33:26.329Z [INFO] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-56a226e2-db30-f2cf-ee7b-3991f56a7f73 Address:56a226e2-db30-f2cf-ee7b-3991f56a7f73} {Suffrage:Voter ID:server-ed082504-4970-0080-a8d8-77b8eea40e20 Address:ed082504-4970-0080-a8d8-77b8eea40e20} {Suffrage:Voter ID:server-c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba Address:c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba}]" -2025-06-03T23:33:26.329Z [INFO] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: entering follower state: follower="Node at 56a226e2-db30-f2cf-ee7b-3991f56a7f73 [Follower]" leader-address= leader-id= -2025-06-03T23:33:26.329Z [INFO] server-ed082504-4970-0080-a8d8-77b8eea40e20: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-56a226e2-db30-f2cf-ee7b-3991f56a7f73 Address:56a226e2-db30-f2cf-ee7b-3991f56a7f73} {Suffrage:Voter ID:server-ed082504-4970-0080-a8d8-77b8eea40e20 Address:ed082504-4970-0080-a8d8-77b8eea40e20} {Suffrage:Voter ID:server-c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba Address:c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba}]" -2025-06-03T23:33:26.330Z [INFO] server-c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-56a226e2-db30-f2cf-ee7b-3991f56a7f73 Address:56a226e2-db30-f2cf-ee7b-3991f56a7f73} {Suffrage:Voter ID:server-ed082504-4970-0080-a8d8-77b8eea40e20 Address:ed082504-4970-0080-a8d8-77b8eea40e20} {Suffrage:Voter ID:server-c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba Address:c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba}]" -2025-06-03T23:33:26.330Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:26.330Z [INFO] server-c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba: entering follower state: follower="Node at c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba [Follower]" leader-address= leader-id= -2025-06-03T23:33:26.329Z [INFO] server-ed082504-4970-0080-a8d8-77b8eea40e20: entering follower state: follower="Node at ed082504-4970-0080-a8d8-77b8eea40e20 [Follower]" leader-address= leader-id= -2025-06-03T23:33:26.904Z [WARN] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:26.904Z [INFO] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: entering candidate state: node="Node at 56a226e2-db30-f2cf-ee7b-3991f56a7f73 [Candidate]" term=2 -2025-06-03T23:33:26.904Z [INFO] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: election won: term=2 tally=2 -2025-06-03T23:33:26.904Z [INFO] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: entering leader state: leader="Node at 56a226e2-db30-f2cf-ee7b-3991f56a7f73 [Leader]" -2025-06-03T23:33:26.904Z [INFO] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: added peer, starting replication: peer=server-ed082504-4970-0080-a8d8-77b8eea40e20 -2025-06-03T23:33:26.904Z [INFO] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: added peer, starting replication: peer=server-c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba -2025-06-03T23:33:26.904Z [INFO] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: pipelining replication: peer="{Voter server-c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba}" -2025-06-03T23:33:26.904Z [INFO] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: pipelining replication: peer="{Voter server-ed082504-4970-0080-a8d8-77b8eea40e20 ed082504-4970-0080-a8d8-77b8eea40e20}" -2025-06-03T23:33:27.910Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:26.904836431 +0000 UTC m=+21.925655801 (1 nodes), 574.46756ms from start of poll, 575.97152ms from cluster start. Timeout at 2025-06-03 23:33:27.910083991 +0000 UTC m=+22.930903321, 1.00524752s after stability -2025-06-03T23:33:27.910Z [INFO] creating new snapshot: path=/tmp/raft3566082691/snapshots/2-13-1748993607910.tmp -2025-06-03T23:33:27.910Z [INFO] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: copied to local snapshot: bytes=71 -2025-06-03T23:33:27.911Z [INFO] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: snapshot restore progress: id=2-13-1748993607910 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:27.911Z [INFO] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: restored user snapshot: index=13 -2025-06-03T23:33:27.911Z [ERROR] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: failed to get log: index=3 error="log not found" -2025-06-03T23:33:27.911Z [INFO] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: aborting pipeline replication: peer="{Voter server-c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba}" -2025-06-03T23:33:27.911Z [ERROR] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: failed to get log: index=3 error="log not found" -2025-06-03T23:33:27.911Z [INFO] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: aborting pipeline replication: peer="{Voter server-ed082504-4970-0080-a8d8-77b8eea40e20 ed082504-4970-0080-a8d8-77b8eea40e20}" -2025-06-03T23:33:27.921Z [ERROR] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: failed to get log: index=3 error="log not found" -2025-06-03T23:33:27.921Z [ERROR] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: failed to get log: index=3 error="log not found" -2025-06-03T23:33:27.922Z [INFO] creating new snapshot: path=/tmp/raft58438219/snapshots/2-13-1748993607922.tmp -2025-06-03T23:33:27.922Z [INFO] creating new snapshot: path=/tmp/raft540112306/snapshots/2-13-1748993607922.tmp -2025-06-03T23:33:27.922Z [INFO] server-c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:27.922Z [INFO] server-ed082504-4970-0080-a8d8-77b8eea40e20: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:27.922Z [INFO] server-c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba: copied to local snapshot: bytes=71 -2025-06-03T23:33:27.922Z [INFO] server-ed082504-4970-0080-a8d8-77b8eea40e20: copied to local snapshot: bytes=71 -2025-06-03T23:33:27.923Z [INFO] server-c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba: snapshot restore progress: id=2-13-1748993607922 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:27.923Z [INFO] server-ed082504-4970-0080-a8d8-77b8eea40e20: snapshot restore progress: id=2-13-1748993607922 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:27.923Z [INFO] server-c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba: Installed remote snapshot -2025-06-03T23:33:27.923Z [INFO] server-ed082504-4970-0080-a8d8-77b8eea40e20: Installed remote snapshot -2025-06-03T23:33:27.923Z [INFO] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: pipelining replication: peer="{Voter server-c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba c8083b30-cb3d-6b9e-cc64-e0cc7bcda5ba}" -2025-06-03T23:33:27.923Z [INFO] server-56a226e2-db30-f2cf-ee7b-3991f56a7f73: pipelining replication: peer="{Voter server-ed082504-4970-0080-a8d8-77b8eea40e20 ed082504-4970-0080-a8d8-77b8eea40e20}" -2025-06-03T23:33:27.923Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:28.929Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:27.923661071 +0000 UTC m=+22.944480441 (2 nodes), 1.2µs from start of poll, 1.59479616s from cluster start. Timeout at 2025-06-03 23:33:28.929210991 +0000 UTC m=+23.950030401, 1.00554996s after stability -2025-06-03T23:33:28.942Z [INFO] server-40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7: aborting pipeline replication: peer="{Voter server-4d865d30-7813-aa37-cd6d-15c4b4c4ca7d 4d865d30-7813-aa37-cd6d-15c4b4c4ca7d}" -2025-06-03T23:33:28.942Z [INFO] server-40cc57d4-ddc9-b748-0faf-6d7cefd6a0b7: aborting pipeline replication: peer="{Voter server-a79068ed-386d-b6c0-80cf-6976fba192d8 a79068ed-386d-b6c0-80cf-6976fba192d8}" +2025-06-03T23:36:23.369Z [INFO] server-450c5e42-b834-6145-ce91-d6b1dfaa086c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-450c5e42-b834-6145-ce91-d6b1dfaa086c Address:450c5e42-b834-6145-ce91-d6b1dfaa086c} {Suffrage:Voter ID:server-628a8592-e787-20aa-f18f-4c74883a1681 Address:628a8592-e787-20aa-f18f-4c74883a1681} {Suffrage:Voter ID:server-02b11b45-10e7-41cc-a770-017c90890c17 Address:02b11b45-10e7-41cc-a770-017c90890c17}]" +2025-06-03T23:36:23.369Z [INFO] server-450c5e42-b834-6145-ce91-d6b1dfaa086c: entering follower state: follower="Node at 450c5e42-b834-6145-ce91-d6b1dfaa086c [Follower]" leader-address= leader-id= +2025-06-03T23:36:23.369Z [INFO] server-628a8592-e787-20aa-f18f-4c74883a1681: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-450c5e42-b834-6145-ce91-d6b1dfaa086c Address:450c5e42-b834-6145-ce91-d6b1dfaa086c} {Suffrage:Voter ID:server-628a8592-e787-20aa-f18f-4c74883a1681 Address:628a8592-e787-20aa-f18f-4c74883a1681} {Suffrage:Voter ID:server-02b11b45-10e7-41cc-a770-017c90890c17 Address:02b11b45-10e7-41cc-a770-017c90890c17}]" +2025-06-03T23:36:23.369Z [INFO] server-628a8592-e787-20aa-f18f-4c74883a1681: entering follower state: follower="Node at 628a8592-e787-20aa-f18f-4c74883a1681 [Follower]" leader-address= leader-id= +2025-06-03T23:36:23.369Z [INFO] server-02b11b45-10e7-41cc-a770-017c90890c17: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-450c5e42-b834-6145-ce91-d6b1dfaa086c Address:450c5e42-b834-6145-ce91-d6b1dfaa086c} {Suffrage:Voter ID:server-628a8592-e787-20aa-f18f-4c74883a1681 Address:628a8592-e787-20aa-f18f-4c74883a1681} {Suffrage:Voter ID:server-02b11b45-10e7-41cc-a770-017c90890c17 Address:02b11b45-10e7-41cc-a770-017c90890c17}]" +2025-06-03T23:36:23.369Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:23.369Z [INFO] server-02b11b45-10e7-41cc-a770-017c90890c17: entering follower state: follower="Node at 02b11b45-10e7-41cc-a770-017c90890c17 [Follower]" leader-address= leader-id= +2025-06-03T23:36:23.924Z [WARN] server-450c5e42-b834-6145-ce91-d6b1dfaa086c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:23.924Z [INFO] server-450c5e42-b834-6145-ce91-d6b1dfaa086c: entering candidate state: node="Node at 450c5e42-b834-6145-ce91-d6b1dfaa086c [Candidate]" term=2 +2025-06-03T23:36:23.924Z [INFO] server-450c5e42-b834-6145-ce91-d6b1dfaa086c: election won: term=2 tally=2 +2025-06-03T23:36:23.924Z [INFO] server-450c5e42-b834-6145-ce91-d6b1dfaa086c: entering leader state: leader="Node at 450c5e42-b834-6145-ce91-d6b1dfaa086c [Leader]" +2025-06-03T23:36:23.924Z [INFO] server-450c5e42-b834-6145-ce91-d6b1dfaa086c: added peer, starting replication: peer=server-628a8592-e787-20aa-f18f-4c74883a1681 +2025-06-03T23:36:23.924Z [INFO] server-450c5e42-b834-6145-ce91-d6b1dfaa086c: added peer, starting replication: peer=server-02b11b45-10e7-41cc-a770-017c90890c17 +2025-06-03T23:36:23.924Z [INFO] server-450c5e42-b834-6145-ce91-d6b1dfaa086c: pipelining replication: peer="{Voter server-02b11b45-10e7-41cc-a770-017c90890c17 02b11b45-10e7-41cc-a770-017c90890c17}" +2025-06-03T23:36:23.924Z [INFO] server-450c5e42-b834-6145-ce91-d6b1dfaa086c: pipelining replication: peer="{Voter server-628a8592-e787-20aa-f18f-4c74883a1681 628a8592-e787-20aa-f18f-4c74883a1681}" +2025-06-03T23:36:24.930Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:23.924730551 +0000 UTC m=+19.246485481 (1 nodes), 555.0224ms from start of poll, 555.6412ms from cluster start. Timeout at 2025-06-03 23:36:24.930473151 +0000 UTC m=+20.252228081, 1.0057426s after stability +2025-06-03T23:36:24.931Z [INFO] server-450c5e42-b834-6145-ce91-d6b1dfaa086c: starting snapshot up to: index=12 +2025-06-03T23:36:24.931Z [INFO] creating new snapshot: path=/tmp/raft76826245/snapshots/2-12-1748993784931.tmp +2025-06-03T23:36:24.931Z [INFO] server-450c5e42-b834-6145-ce91-d6b1dfaa086c: snapshot complete up to: index=12 +2025-06-03T23:36:24.932Z [INFO] server-9bbc8fa8-2607-d93e-304d-08293302a5fa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9bbc8fa8-2607-d93e-304d-08293302a5fa Address:9bbc8fa8-2607-d93e-304d-08293302a5fa} {Suffrage:Voter ID:server-85476bda-54ac-de65-ab6e-a3d20cffdcab Address:85476bda-54ac-de65-ab6e-a3d20cffdcab} {Suffrage:Voter ID:server-3533d119-ffb3-a0f2-d541-5613a1d61ca1 Address:3533d119-ffb3-a0f2-d541-5613a1d61ca1}]" +2025-06-03T23:36:24.932Z [INFO] server-9bbc8fa8-2607-d93e-304d-08293302a5fa: entering follower state: follower="Node at 9bbc8fa8-2607-d93e-304d-08293302a5fa [Follower]" leader-address= leader-id= +2025-06-03T23:36:24.933Z [INFO] server-85476bda-54ac-de65-ab6e-a3d20cffdcab: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9bbc8fa8-2607-d93e-304d-08293302a5fa Address:9bbc8fa8-2607-d93e-304d-08293302a5fa} {Suffrage:Voter ID:server-85476bda-54ac-de65-ab6e-a3d20cffdcab Address:85476bda-54ac-de65-ab6e-a3d20cffdcab} {Suffrage:Voter ID:server-3533d119-ffb3-a0f2-d541-5613a1d61ca1 Address:3533d119-ffb3-a0f2-d541-5613a1d61ca1}]" +2025-06-03T23:36:24.933Z [INFO] server-85476bda-54ac-de65-ab6e-a3d20cffdcab: entering follower state: follower="Node at 85476bda-54ac-de65-ab6e-a3d20cffdcab [Follower]" leader-address= leader-id= +2025-06-03T23:36:24.933Z [INFO] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9bbc8fa8-2607-d93e-304d-08293302a5fa Address:9bbc8fa8-2607-d93e-304d-08293302a5fa} {Suffrage:Voter ID:server-85476bda-54ac-de65-ab6e-a3d20cffdcab Address:85476bda-54ac-de65-ab6e-a3d20cffdcab} {Suffrage:Voter ID:server-3533d119-ffb3-a0f2-d541-5613a1d61ca1 Address:3533d119-ffb3-a0f2-d541-5613a1d61ca1}]" +2025-06-03T23:36:24.933Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:24.933Z [INFO] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: entering follower state: follower="Node at 3533d119-ffb3-a0f2-d541-5613a1d61ca1 [Follower]" leader-address= leader-id= +2025-06-03T23:36:25.674Z [WARN] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:25.675Z [INFO] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: entering candidate state: node="Node at 3533d119-ffb3-a0f2-d541-5613a1d61ca1 [Candidate]" term=2 +2025-06-03T23:36:25.675Z [INFO] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: election won: term=2 tally=2 +2025-06-03T23:36:25.675Z [INFO] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: entering leader state: leader="Node at 3533d119-ffb3-a0f2-d541-5613a1d61ca1 [Leader]" +2025-06-03T23:36:25.675Z [INFO] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: added peer, starting replication: peer=server-9bbc8fa8-2607-d93e-304d-08293302a5fa +2025-06-03T23:36:25.675Z [INFO] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: added peer, starting replication: peer=server-85476bda-54ac-de65-ab6e-a3d20cffdcab +2025-06-03T23:36:25.675Z [INFO] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: pipelining replication: peer="{Voter server-9bbc8fa8-2607-d93e-304d-08293302a5fa 9bbc8fa8-2607-d93e-304d-08293302a5fa}" +2025-06-03T23:36:25.675Z [INFO] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: pipelining replication: peer="{Voter server-85476bda-54ac-de65-ab6e-a3d20cffdcab 85476bda-54ac-de65-ab6e-a3d20cffdcab}" +2025-06-03T23:36:26.680Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:25.675239271 +0000 UTC m=+20.996994201 (1 nodes), 742.0802ms from start of poll, 742.54844ms from cluster start. Timeout at 2025-06-03 23:36:26.680275951 +0000 UTC m=+22.002030921, 1.00503672s after stability +2025-06-03T23:36:26.680Z [INFO] creating new snapshot: path=/tmp/raft3029333533/snapshots/2-13-1748993786680.tmp +2025-06-03T23:36:26.681Z [INFO] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: copied to local snapshot: bytes=71 +2025-06-03T23:36:26.681Z [INFO] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: snapshot restore progress: id=2-13-1748993786680 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:26.681Z [INFO] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: restored user snapshot: index=13 +2025-06-03T23:36:26.681Z [ERROR] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: failed to get log: index=3 error="log not found" +2025-06-03T23:36:26.681Z [INFO] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: aborting pipeline replication: peer="{Voter server-85476bda-54ac-de65-ab6e-a3d20cffdcab 85476bda-54ac-de65-ab6e-a3d20cffdcab}" +2025-06-03T23:36:26.681Z [ERROR] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: failed to get log: index=3 error="log not found" +2025-06-03T23:36:26.681Z [INFO] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: aborting pipeline replication: peer="{Voter server-9bbc8fa8-2607-d93e-304d-08293302a5fa 9bbc8fa8-2607-d93e-304d-08293302a5fa}" +2025-06-03T23:36:26.689Z [ERROR] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: failed to get log: index=3 error="log not found" +2025-06-03T23:36:26.689Z [ERROR] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: failed to get log: index=3 error="log not found" +2025-06-03T23:36:26.689Z [INFO] creating new snapshot: path=/tmp/raft2273470030/snapshots/2-13-1748993786689.tmp +2025-06-03T23:36:26.689Z [INFO] creating new snapshot: path=/tmp/raft2528862560/snapshots/2-13-1748993786689.tmp +2025-06-03T23:36:26.689Z [INFO] server-85476bda-54ac-de65-ab6e-a3d20cffdcab: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:26.689Z [INFO] server-9bbc8fa8-2607-d93e-304d-08293302a5fa: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:26.690Z [INFO] server-9bbc8fa8-2607-d93e-304d-08293302a5fa: copied to local snapshot: bytes=71 +2025-06-03T23:36:26.690Z [INFO] server-85476bda-54ac-de65-ab6e-a3d20cffdcab: copied to local snapshot: bytes=71 +2025-06-03T23:36:26.690Z [INFO] server-9bbc8fa8-2607-d93e-304d-08293302a5fa: snapshot restore progress: id=2-13-1748993786689 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:26.690Z [INFO] server-85476bda-54ac-de65-ab6e-a3d20cffdcab: snapshot restore progress: id=2-13-1748993786689 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:26.690Z [INFO] server-85476bda-54ac-de65-ab6e-a3d20cffdcab: Installed remote snapshot +2025-06-03T23:36:26.690Z [INFO] server-9bbc8fa8-2607-d93e-304d-08293302a5fa: Installed remote snapshot +2025-06-03T23:36:26.690Z [INFO] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: pipelining replication: peer="{Voter server-85476bda-54ac-de65-ab6e-a3d20cffdcab 85476bda-54ac-de65-ab6e-a3d20cffdcab}" +2025-06-03T23:36:26.690Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:26.690Z [INFO] server-3533d119-ffb3-a0f2-d541-5613a1d61ca1: pipelining replication: peer="{Voter server-9bbc8fa8-2607-d93e-304d-08293302a5fa 9bbc8fa8-2607-d93e-304d-08293302a5fa}" +2025-06-03T23:36:27.695Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:26.690789911 +0000 UTC m=+22.012544841 (2 nodes), 1.4µs from start of poll, 1.75809908s from cluster start. Timeout at 2025-06-03 23:36:27.695888711 +0000 UTC m=+23.017643681, 1.00509884s after stability +2025-06-03T23:36:27.708Z [INFO] server-450c5e42-b834-6145-ce91-d6b1dfaa086c: aborting pipeline replication: peer="{Voter server-02b11b45-10e7-41cc-a770-017c90890c17 02b11b45-10e7-41cc-a770-017c90890c17}" +2025-06-03T23:36:27.708Z [INFO] server-450c5e42-b834-6145-ce91-d6b1dfaa086c: aborting pipeline replication: peer="{Voter server-628a8592-e787-20aa-f18f-4c74883a1681 628a8592-e787-20aa-f18f-4c74883a1681}" === RUN TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true -2025-06-03T23:33:28.943Z [INFO] server-fc4db19b-5629-d0d2-d5cb-7fb6b5959fd9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc4db19b-5629-d0d2-d5cb-7fb6b5959fd9 Address:fc4db19b-5629-d0d2-d5cb-7fb6b5959fd9} {Suffrage:Voter ID:server-3b002a5f-d4d0-378b-c685-d90abb62381e Address:3b002a5f-d4d0-378b-c685-d90abb62381e} {Suffrage:Voter ID:server-801ce673-57b8-b835-51ce-2c5c67046702 Address:801ce673-57b8-b835-51ce-2c5c67046702}]" -2025-06-03T23:33:28.943Z [INFO] server-fc4db19b-5629-d0d2-d5cb-7fb6b5959fd9: entering follower state: follower="Node at fc4db19b-5629-d0d2-d5cb-7fb6b5959fd9 [Follower]" leader-address= leader-id= -2025-06-03T23:33:28.943Z [INFO] server-3b002a5f-d4d0-378b-c685-d90abb62381e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc4db19b-5629-d0d2-d5cb-7fb6b5959fd9 Address:fc4db19b-5629-d0d2-d5cb-7fb6b5959fd9} {Suffrage:Voter ID:server-3b002a5f-d4d0-378b-c685-d90abb62381e Address:3b002a5f-d4d0-378b-c685-d90abb62381e} {Suffrage:Voter ID:server-801ce673-57b8-b835-51ce-2c5c67046702 Address:801ce673-57b8-b835-51ce-2c5c67046702}]" -2025-06-03T23:33:28.943Z [INFO] server-3b002a5f-d4d0-378b-c685-d90abb62381e: entering follower state: follower="Node at 3b002a5f-d4d0-378b-c685-d90abb62381e [Follower]" leader-address= leader-id= -2025-06-03T23:33:28.943Z [INFO] server-801ce673-57b8-b835-51ce-2c5c67046702: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc4db19b-5629-d0d2-d5cb-7fb6b5959fd9 Address:fc4db19b-5629-d0d2-d5cb-7fb6b5959fd9} {Suffrage:Voter ID:server-3b002a5f-d4d0-378b-c685-d90abb62381e Address:3b002a5f-d4d0-378b-c685-d90abb62381e} {Suffrage:Voter ID:server-801ce673-57b8-b835-51ce-2c5c67046702 Address:801ce673-57b8-b835-51ce-2c5c67046702}]" -2025-06-03T23:33:28.943Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:28.943Z [INFO] server-801ce673-57b8-b835-51ce-2c5c67046702: entering follower state: follower="Node at 801ce673-57b8-b835-51ce-2c5c67046702 [Follower]" leader-address= leader-id= -2025-06-03T23:33:29.470Z [WARN] server-3b002a5f-d4d0-378b-c685-d90abb62381e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:29.470Z [INFO] server-3b002a5f-d4d0-378b-c685-d90abb62381e: entering candidate state: node="Node at 3b002a5f-d4d0-378b-c685-d90abb62381e [Candidate]" term=2 -2025-06-03T23:33:29.470Z [INFO] server-3b002a5f-d4d0-378b-c685-d90abb62381e: election won: term=2 tally=2 -2025-06-03T23:33:29.470Z [INFO] server-3b002a5f-d4d0-378b-c685-d90abb62381e: entering leader state: leader="Node at 3b002a5f-d4d0-378b-c685-d90abb62381e [Leader]" -2025-06-03T23:33:29.470Z [INFO] server-3b002a5f-d4d0-378b-c685-d90abb62381e: added peer, starting replication: peer=server-fc4db19b-5629-d0d2-d5cb-7fb6b5959fd9 -2025-06-03T23:33:29.470Z [INFO] server-3b002a5f-d4d0-378b-c685-d90abb62381e: added peer, starting replication: peer=server-801ce673-57b8-b835-51ce-2c5c67046702 -2025-06-03T23:33:29.470Z [INFO] server-3b002a5f-d4d0-378b-c685-d90abb62381e: pipelining replication: peer="{Voter server-801ce673-57b8-b835-51ce-2c5c67046702 801ce673-57b8-b835-51ce-2c5c67046702}" -2025-06-03T23:33:29.470Z [INFO] server-3b002a5f-d4d0-378b-c685-d90abb62381e: pipelining replication: peer="{Voter server-fc4db19b-5629-d0d2-d5cb-7fb6b5959fd9 fc4db19b-5629-d0d2-d5cb-7fb6b5959fd9}" -2025-06-03T23:33:30.476Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:29.470692271 +0000 UTC m=+24.491511641 (1 nodes), 526.9368ms from start of poll, 527.5504ms from cluster start. Timeout at 2025-06-03 23:33:30.476748591 +0000 UTC m=+25.497568001, 1.00605636s after stability -2025-06-03T23:33:30.478Z [INFO] server-3b002a5f-d4d0-378b-c685-d90abb62381e: starting snapshot up to: index=12 -2025-06-03T23:33:30.478Z [INFO] creating new snapshot: path=/tmp/raft2268642434/snapshots/2-12-1748993610478.tmp -2025-06-03T23:33:30.479Z [INFO] server-3b002a5f-d4d0-378b-c685-d90abb62381e: snapshot complete up to: index=12 -2025-06-03T23:33:30.480Z [INFO] server-b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a Address:b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a} {Suffrage:Voter ID:server-8d837ac5-81d9-bc5b-b080-038f0fd008ed Address:8d837ac5-81d9-bc5b-b080-038f0fd008ed} {Suffrage:Voter ID:server-14471c9d-5ee8-516b-e2cd-7a633e36c697 Address:14471c9d-5ee8-516b-e2cd-7a633e36c697}]" -2025-06-03T23:33:30.480Z [INFO] server-b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a: entering follower state: follower="Node at b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a [Follower]" leader-address= leader-id= -2025-06-03T23:33:30.480Z [INFO] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a Address:b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a} {Suffrage:Voter ID:server-8d837ac5-81d9-bc5b-b080-038f0fd008ed Address:8d837ac5-81d9-bc5b-b080-038f0fd008ed} {Suffrage:Voter ID:server-14471c9d-5ee8-516b-e2cd-7a633e36c697 Address:14471c9d-5ee8-516b-e2cd-7a633e36c697}]" -2025-06-03T23:33:30.480Z [INFO] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: entering follower state: follower="Node at 8d837ac5-81d9-bc5b-b080-038f0fd008ed [Follower]" leader-address= leader-id= -2025-06-03T23:33:30.480Z [INFO] server-14471c9d-5ee8-516b-e2cd-7a633e36c697: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a Address:b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a} {Suffrage:Voter ID:server-8d837ac5-81d9-bc5b-b080-038f0fd008ed Address:8d837ac5-81d9-bc5b-b080-038f0fd008ed} {Suffrage:Voter ID:server-14471c9d-5ee8-516b-e2cd-7a633e36c697 Address:14471c9d-5ee8-516b-e2cd-7a633e36c697}]" -2025-06-03T23:33:30.480Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:30.480Z [INFO] server-14471c9d-5ee8-516b-e2cd-7a633e36c697: entering follower state: follower="Node at 14471c9d-5ee8-516b-e2cd-7a633e36c697 [Follower]" leader-address= leader-id= -2025-06-03T23:33:31.137Z [WARN] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:31.137Z [INFO] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: entering candidate state: node="Node at 8d837ac5-81d9-bc5b-b080-038f0fd008ed [Candidate]" term=2 -2025-06-03T23:33:31.137Z [INFO] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: election won: term=2 tally=2 -2025-06-03T23:33:31.137Z [INFO] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: entering leader state: leader="Node at 8d837ac5-81d9-bc5b-b080-038f0fd008ed [Leader]" -2025-06-03T23:33:31.137Z [INFO] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: added peer, starting replication: peer=server-b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a -2025-06-03T23:33:31.137Z [INFO] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: added peer, starting replication: peer=server-14471c9d-5ee8-516b-e2cd-7a633e36c697 -2025-06-03T23:33:31.137Z [INFO] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: pipelining replication: peer="{Voter server-14471c9d-5ee8-516b-e2cd-7a633e36c697 14471c9d-5ee8-516b-e2cd-7a633e36c697}" -2025-06-03T23:33:31.138Z [INFO] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: pipelining replication: peer="{Voter server-b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a}" -2025-06-03T23:33:32.143Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:31.137895631 +0000 UTC m=+26.158715001 (1 nodes), 657.21976ms from start of poll, 657.86776ms from cluster start. Timeout at 2025-06-03 23:33:32.143482191 +0000 UTC m=+27.164301561, 1.00558656s after stability -2025-06-03T23:33:32.143Z [INFO] creating new snapshot: path=/tmp/raft1837524032/snapshots/2-13-1748993612143.tmp -2025-06-03T23:33:32.144Z [INFO] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: copied to local snapshot: bytes=71 -2025-06-03T23:33:32.144Z [INFO] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: snapshot restore progress: id=2-13-1748993612143 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:32.144Z [INFO] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: removing all old logs from log store -2025-06-03T23:33:32.144Z [INFO] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: compacting logs: from=1 to=2 -2025-06-03T23:33:32.144Z [INFO] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: restored user snapshot: index=13 -2025-06-03T23:33:32.144Z [ERROR] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: failed to get log: index=2 error="log not found" -2025-06-03T23:33:32.144Z [ERROR] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: failed to get log: index=2 error="log not found" -2025-06-03T23:33:32.144Z [INFO] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: aborting pipeline replication: peer="{Voter server-b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a}" -2025-06-03T23:33:32.144Z [INFO] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: aborting pipeline replication: peer="{Voter server-14471c9d-5ee8-516b-e2cd-7a633e36c697 14471c9d-5ee8-516b-e2cd-7a633e36c697}" -2025-06-03T23:33:32.152Z [ERROR] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: failed to get log: index=2 error="log not found" -2025-06-03T23:33:32.153Z [INFO] creating new snapshot: path=/tmp/raft3248475961/snapshots/2-13-1748993612153.tmp -2025-06-03T23:33:32.153Z [ERROR] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: failed to get log: index=2 error="log not found" -2025-06-03T23:33:32.153Z [INFO] server-b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:32.154Z [INFO] server-b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a: copied to local snapshot: bytes=71 -2025-06-03T23:33:32.155Z [INFO] creating new snapshot: path=/tmp/raft3093388244/snapshots/2-13-1748993612154.tmp -2025-06-03T23:33:32.155Z [INFO] server-14471c9d-5ee8-516b-e2cd-7a633e36c697: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:32.155Z [INFO] server-b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a: snapshot restore progress: id=2-13-1748993612153 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:32.155Z [INFO] server-b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a: removing all old logs from log store -2025-06-03T23:33:32.155Z [INFO] server-b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a: compacting logs: from=1 to=2 -2025-06-03T23:33:32.155Z [INFO] server-b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a: Installed remote snapshot -2025-06-03T23:33:32.155Z [INFO] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: pipelining replication: peer="{Voter server-b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a b25ca3b6-4830-82ae-ffeb-e8460a1d9d6a}" -2025-06-03T23:33:32.155Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:32.155Z [INFO] server-14471c9d-5ee8-516b-e2cd-7a633e36c697: copied to local snapshot: bytes=71 -2025-06-03T23:33:32.156Z [INFO] server-14471c9d-5ee8-516b-e2cd-7a633e36c697: snapshot restore progress: id=2-13-1748993612154 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:32.156Z [INFO] server-14471c9d-5ee8-516b-e2cd-7a633e36c697: removing all old logs from log store -2025-06-03T23:33:32.156Z [INFO] server-14471c9d-5ee8-516b-e2cd-7a633e36c697: compacting logs: from=1 to=2 -2025-06-03T23:33:32.156Z [INFO] server-14471c9d-5ee8-516b-e2cd-7a633e36c697: Installed remote snapshot -2025-06-03T23:33:32.156Z [INFO] server-8d837ac5-81d9-bc5b-b080-038f0fd008ed: pipelining replication: peer="{Voter server-14471c9d-5ee8-516b-e2cd-7a633e36c697 14471c9d-5ee8-516b-e2cd-7a633e36c697}" -2025-06-03T23:33:33.161Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:32.155806951 +0000 UTC m=+27.176626321 (2 nodes), 1.04µs from start of poll, 1.67577908s from cluster start. Timeout at 2025-06-03 23:33:33.161382671 +0000 UTC m=+28.182202081, 1.00557576s after stability -2025-06-03T23:33:33.174Z [INFO] server-3b002a5f-d4d0-378b-c685-d90abb62381e: aborting pipeline replication: peer="{Voter server-801ce673-57b8-b835-51ce-2c5c67046702 801ce673-57b8-b835-51ce-2c5c67046702}" -2025-06-03T23:33:33.174Z [INFO] server-3b002a5f-d4d0-378b-c685-d90abb62381e: aborting pipeline replication: peer="{Voter server-fc4db19b-5629-d0d2-d5cb-7fb6b5959fd9 fc4db19b-5629-d0d2-d5cb-7fb6b5959fd9}" +2025-06-03T23:36:27.709Z [INFO] server-dc90abbb-1a7b-2f81-da6d-ab406c1e9a4d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dc90abbb-1a7b-2f81-da6d-ab406c1e9a4d Address:dc90abbb-1a7b-2f81-da6d-ab406c1e9a4d} {Suffrage:Voter ID:server-1459aa8c-b168-505e-a5bc-26d086362971 Address:1459aa8c-b168-505e-a5bc-26d086362971} {Suffrage:Voter ID:server-612907ea-ae79-9177-859c-cc232916d290 Address:612907ea-ae79-9177-859c-cc232916d290}]" +2025-06-03T23:36:27.709Z [INFO] server-dc90abbb-1a7b-2f81-da6d-ab406c1e9a4d: entering follower state: follower="Node at dc90abbb-1a7b-2f81-da6d-ab406c1e9a4d [Follower]" leader-address= leader-id= +2025-06-03T23:36:27.709Z [INFO] server-1459aa8c-b168-505e-a5bc-26d086362971: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dc90abbb-1a7b-2f81-da6d-ab406c1e9a4d Address:dc90abbb-1a7b-2f81-da6d-ab406c1e9a4d} {Suffrage:Voter ID:server-1459aa8c-b168-505e-a5bc-26d086362971 Address:1459aa8c-b168-505e-a5bc-26d086362971} {Suffrage:Voter ID:server-612907ea-ae79-9177-859c-cc232916d290 Address:612907ea-ae79-9177-859c-cc232916d290}]" +2025-06-03T23:36:27.709Z [INFO] server-1459aa8c-b168-505e-a5bc-26d086362971: entering follower state: follower="Node at 1459aa8c-b168-505e-a5bc-26d086362971 [Follower]" leader-address= leader-id= +2025-06-03T23:36:27.709Z [INFO] server-612907ea-ae79-9177-859c-cc232916d290: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dc90abbb-1a7b-2f81-da6d-ab406c1e9a4d Address:dc90abbb-1a7b-2f81-da6d-ab406c1e9a4d} {Suffrage:Voter ID:server-1459aa8c-b168-505e-a5bc-26d086362971 Address:1459aa8c-b168-505e-a5bc-26d086362971} {Suffrage:Voter ID:server-612907ea-ae79-9177-859c-cc232916d290 Address:612907ea-ae79-9177-859c-cc232916d290}]" +2025-06-03T23:36:27.709Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:27.709Z [INFO] server-612907ea-ae79-9177-859c-cc232916d290: entering follower state: follower="Node at 612907ea-ae79-9177-859c-cc232916d290 [Follower]" leader-address= leader-id= +2025-06-03T23:36:28.392Z [WARN] server-612907ea-ae79-9177-859c-cc232916d290: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:28.392Z [INFO] server-612907ea-ae79-9177-859c-cc232916d290: entering candidate state: node="Node at 612907ea-ae79-9177-859c-cc232916d290 [Candidate]" term=2 +2025-06-03T23:36:28.392Z [INFO] server-612907ea-ae79-9177-859c-cc232916d290: election won: term=2 tally=2 +2025-06-03T23:36:28.392Z [INFO] server-612907ea-ae79-9177-859c-cc232916d290: entering leader state: leader="Node at 612907ea-ae79-9177-859c-cc232916d290 [Leader]" +2025-06-03T23:36:28.392Z [INFO] server-612907ea-ae79-9177-859c-cc232916d290: added peer, starting replication: peer=server-dc90abbb-1a7b-2f81-da6d-ab406c1e9a4d +2025-06-03T23:36:28.392Z [INFO] server-612907ea-ae79-9177-859c-cc232916d290: added peer, starting replication: peer=server-1459aa8c-b168-505e-a5bc-26d086362971 +2025-06-03T23:36:28.393Z [INFO] server-612907ea-ae79-9177-859c-cc232916d290: pipelining replication: peer="{Voter server-dc90abbb-1a7b-2f81-da6d-ab406c1e9a4d dc90abbb-1a7b-2f81-da6d-ab406c1e9a4d}" +2025-06-03T23:36:28.393Z [INFO] server-612907ea-ae79-9177-859c-cc232916d290: pipelining replication: peer="{Voter server-1459aa8c-b168-505e-a5bc-26d086362971 1459aa8c-b168-505e-a5bc-26d086362971}" +2025-06-03T23:36:29.398Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:28.392955431 +0000 UTC m=+23.714710401 (1 nodes), 683.12396ms from start of poll, 683.5952ms from cluster start. Timeout at 2025-06-03 23:36:29.398925911 +0000 UTC m=+24.720680841, 1.00597044s after stability +2025-06-03T23:36:29.399Z [INFO] server-612907ea-ae79-9177-859c-cc232916d290: starting snapshot up to: index=12 +2025-06-03T23:36:29.399Z [INFO] creating new snapshot: path=/tmp/raft2938103395/snapshots/2-12-1748993789399.tmp +2025-06-03T23:36:29.400Z [INFO] server-612907ea-ae79-9177-859c-cc232916d290: snapshot complete up to: index=12 +2025-06-03T23:36:29.401Z [INFO] server-4e66fb50-dd8c-506a-9813-4e11b7bed875: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4e66fb50-dd8c-506a-9813-4e11b7bed875 Address:4e66fb50-dd8c-506a-9813-4e11b7bed875} {Suffrage:Voter ID:server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a Address:1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a} {Suffrage:Voter ID:server-633ceb4a-241c-bfb6-50fb-eba4c897c325 Address:633ceb4a-241c-bfb6-50fb-eba4c897c325}]" +2025-06-03T23:36:29.401Z [INFO] server-4e66fb50-dd8c-506a-9813-4e11b7bed875: entering follower state: follower="Node at 4e66fb50-dd8c-506a-9813-4e11b7bed875 [Follower]" leader-address= leader-id= +2025-06-03T23:36:29.401Z [INFO] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4e66fb50-dd8c-506a-9813-4e11b7bed875 Address:4e66fb50-dd8c-506a-9813-4e11b7bed875} {Suffrage:Voter ID:server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a Address:1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a} {Suffrage:Voter ID:server-633ceb4a-241c-bfb6-50fb-eba4c897c325 Address:633ceb4a-241c-bfb6-50fb-eba4c897c325}]" +2025-06-03T23:36:29.401Z [INFO] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: entering follower state: follower="Node at 1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a [Follower]" leader-address= leader-id= +2025-06-03T23:36:29.401Z [INFO] server-633ceb4a-241c-bfb6-50fb-eba4c897c325: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4e66fb50-dd8c-506a-9813-4e11b7bed875 Address:4e66fb50-dd8c-506a-9813-4e11b7bed875} {Suffrage:Voter ID:server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a Address:1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a} {Suffrage:Voter ID:server-633ceb4a-241c-bfb6-50fb-eba4c897c325 Address:633ceb4a-241c-bfb6-50fb-eba4c897c325}]" +2025-06-03T23:36:29.401Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:29.401Z [INFO] server-633ceb4a-241c-bfb6-50fb-eba4c897c325: entering follower state: follower="Node at 633ceb4a-241c-bfb6-50fb-eba4c897c325 [Follower]" leader-address= leader-id= +2025-06-03T23:36:29.906Z [WARN] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:29.906Z [INFO] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: entering candidate state: node="Node at 1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a [Candidate]" term=2 +2025-06-03T23:36:29.906Z [INFO] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: election won: term=2 tally=2 +2025-06-03T23:36:29.906Z [INFO] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: entering leader state: leader="Node at 1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a [Leader]" +2025-06-03T23:36:29.906Z [INFO] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: added peer, starting replication: peer=server-4e66fb50-dd8c-506a-9813-4e11b7bed875 +2025-06-03T23:36:29.906Z [INFO] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: added peer, starting replication: peer=server-633ceb4a-241c-bfb6-50fb-eba4c897c325 +2025-06-03T23:36:29.907Z [INFO] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: pipelining replication: peer="{Voter server-633ceb4a-241c-bfb6-50fb-eba4c897c325 633ceb4a-241c-bfb6-50fb-eba4c897c325}" +2025-06-03T23:36:29.907Z [INFO] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: pipelining replication: peer="{Voter server-4e66fb50-dd8c-506a-9813-4e11b7bed875 4e66fb50-dd8c-506a-9813-4e11b7bed875}" +2025-06-03T23:36:30.912Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:29.906517951 +0000 UTC m=+25.228272961 (1 nodes), 504.78288ms from start of poll, 505.43968ms from cluster start. Timeout at 2025-06-03 23:36:30.912188151 +0000 UTC m=+26.233943081, 1.00567012s after stability +2025-06-03T23:36:30.912Z [INFO] creating new snapshot: path=/tmp/raft3405740795/snapshots/2-13-1748993790912.tmp +2025-06-03T23:36:30.913Z [INFO] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: copied to local snapshot: bytes=71 +2025-06-03T23:36:30.914Z [INFO] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: snapshot restore progress: id=2-13-1748993790912 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:30.914Z [INFO] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: removing all old logs from log store +2025-06-03T23:36:30.914Z [INFO] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: compacting logs: from=1 to=2 +2025-06-03T23:36:30.914Z [INFO] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: restored user snapshot: index=13 +2025-06-03T23:36:30.914Z [ERROR] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: failed to get log: index=2 error="log not found" +2025-06-03T23:36:30.914Z [INFO] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: aborting pipeline replication: peer="{Voter server-633ceb4a-241c-bfb6-50fb-eba4c897c325 633ceb4a-241c-bfb6-50fb-eba4c897c325}" +2025-06-03T23:36:30.914Z [ERROR] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: failed to get log: index=2 error="log not found" +2025-06-03T23:36:30.914Z [INFO] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: aborting pipeline replication: peer="{Voter server-4e66fb50-dd8c-506a-9813-4e11b7bed875 4e66fb50-dd8c-506a-9813-4e11b7bed875}" +2025-06-03T23:36:30.922Z [ERROR] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: failed to get log: index=2 error="log not found" +2025-06-03T23:36:30.922Z [INFO] creating new snapshot: path=/tmp/raft1896818909/snapshots/2-13-1748993790922.tmp +2025-06-03T23:36:30.922Z [ERROR] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: failed to get log: index=2 error="log not found" +2025-06-03T23:36:30.923Z [INFO] server-633ceb4a-241c-bfb6-50fb-eba4c897c325: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:30.923Z [INFO] creating new snapshot: path=/tmp/raft1868052945/snapshots/2-13-1748993790923.tmp +2025-06-03T23:36:30.923Z [INFO] server-633ceb4a-241c-bfb6-50fb-eba4c897c325: copied to local snapshot: bytes=71 +2025-06-03T23:36:30.923Z [INFO] server-4e66fb50-dd8c-506a-9813-4e11b7bed875: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:30.923Z [INFO] server-633ceb4a-241c-bfb6-50fb-eba4c897c325: snapshot restore progress: id=2-13-1748993790922 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:30.923Z [INFO] server-633ceb4a-241c-bfb6-50fb-eba4c897c325: removing all old logs from log store +2025-06-03T23:36:30.923Z [INFO] server-633ceb4a-241c-bfb6-50fb-eba4c897c325: compacting logs: from=1 to=2 +2025-06-03T23:36:30.923Z [INFO] server-633ceb4a-241c-bfb6-50fb-eba4c897c325: Installed remote snapshot +2025-06-03T23:36:30.923Z [INFO] server-4e66fb50-dd8c-506a-9813-4e11b7bed875: copied to local snapshot: bytes=71 +2025-06-03T23:36:30.923Z [INFO] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: pipelining replication: peer="{Voter server-633ceb4a-241c-bfb6-50fb-eba4c897c325 633ceb4a-241c-bfb6-50fb-eba4c897c325}" +2025-06-03T23:36:30.924Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:30.924Z [INFO] server-4e66fb50-dd8c-506a-9813-4e11b7bed875: snapshot restore progress: id=2-13-1748993790923 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:30.924Z [INFO] server-4e66fb50-dd8c-506a-9813-4e11b7bed875: removing all old logs from log store +2025-06-03T23:36:30.924Z [INFO] server-4e66fb50-dd8c-506a-9813-4e11b7bed875: compacting logs: from=1 to=2 +2025-06-03T23:36:30.924Z [INFO] server-4e66fb50-dd8c-506a-9813-4e11b7bed875: Installed remote snapshot +2025-06-03T23:36:30.924Z [INFO] server-1a0ad9cc-e324-5b49-fde6-91e1b4d7af7a: pipelining replication: peer="{Voter server-4e66fb50-dd8c-506a-9813-4e11b7bed875 4e66fb50-dd8c-506a-9813-4e11b7bed875}" +2025-06-03T23:36:31.929Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:30.924023271 +0000 UTC m=+26.245778201 (2 nodes), 1.28µs from start of poll, 1.52294492s from cluster start. Timeout at 2025-06-03 23:36:31.929655511 +0000 UTC m=+27.251410441, 1.00563224s after stability +2025-06-03T23:36:31.942Z [INFO] server-612907ea-ae79-9177-859c-cc232916d290: aborting pipeline replication: peer="{Voter server-1459aa8c-b168-505e-a5bc-26d086362971 1459aa8c-b168-505e-a5bc-26d086362971}" +2025-06-03T23:36:31.942Z [INFO] server-612907ea-ae79-9177-859c-cc232916d290: aborting pipeline replication: peer="{Voter server-dc90abbb-1a7b-2f81-da6d-ab406c1e9a4d dc90abbb-1a7b-2f81-da6d-ab406c1e9a4d}" === RUN TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false -2025-06-03T23:33:33.175Z [INFO] server-e2c7607f-d446-70e4-98ee-db17dffe1ecd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e2c7607f-d446-70e4-98ee-db17dffe1ecd Address:e2c7607f-d446-70e4-98ee-db17dffe1ecd} {Suffrage:Voter ID:server-92e31615-95ee-3f5f-18b8-76c06d642c65 Address:92e31615-95ee-3f5f-18b8-76c06d642c65} {Suffrage:Voter ID:server-3b7102ab-b85f-eda8-87b5-c670ecefea9c Address:3b7102ab-b85f-eda8-87b5-c670ecefea9c}]" -2025-06-03T23:33:33.175Z [INFO] server-e2c7607f-d446-70e4-98ee-db17dffe1ecd: entering follower state: follower="Node at e2c7607f-d446-70e4-98ee-db17dffe1ecd [Follower]" leader-address= leader-id= -2025-06-03T23:33:33.175Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e2c7607f-d446-70e4-98ee-db17dffe1ecd Address:e2c7607f-d446-70e4-98ee-db17dffe1ecd} {Suffrage:Voter ID:server-92e31615-95ee-3f5f-18b8-76c06d642c65 Address:92e31615-95ee-3f5f-18b8-76c06d642c65} {Suffrage:Voter ID:server-3b7102ab-b85f-eda8-87b5-c670ecefea9c Address:3b7102ab-b85f-eda8-87b5-c670ecefea9c}]" -2025-06-03T23:33:33.175Z [INFO] server-3b7102ab-b85f-eda8-87b5-c670ecefea9c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e2c7607f-d446-70e4-98ee-db17dffe1ecd Address:e2c7607f-d446-70e4-98ee-db17dffe1ecd} {Suffrage:Voter ID:server-92e31615-95ee-3f5f-18b8-76c06d642c65 Address:92e31615-95ee-3f5f-18b8-76c06d642c65} {Suffrage:Voter ID:server-3b7102ab-b85f-eda8-87b5-c670ecefea9c Address:3b7102ab-b85f-eda8-87b5-c670ecefea9c}]" -2025-06-03T23:33:33.175Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: entering follower state: follower="Node at 92e31615-95ee-3f5f-18b8-76c06d642c65 [Follower]" leader-address= leader-id= -2025-06-03T23:33:33.175Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:33.175Z [INFO] server-3b7102ab-b85f-eda8-87b5-c670ecefea9c: entering follower state: follower="Node at 3b7102ab-b85f-eda8-87b5-c670ecefea9c [Follower]" leader-address= leader-id= -2025-06-03T23:33:33.807Z [WARN] server-92e31615-95ee-3f5f-18b8-76c06d642c65: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:33.807Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: entering candidate state: node="Node at 92e31615-95ee-3f5f-18b8-76c06d642c65 [Candidate]" term=2 -2025-06-03T23:33:33.807Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: election won: term=2 tally=2 -2025-06-03T23:33:33.807Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: entering leader state: leader="Node at 92e31615-95ee-3f5f-18b8-76c06d642c65 [Leader]" -2025-06-03T23:33:33.807Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: added peer, starting replication: peer=server-e2c7607f-d446-70e4-98ee-db17dffe1ecd -2025-06-03T23:33:33.807Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: added peer, starting replication: peer=server-3b7102ab-b85f-eda8-87b5-c670ecefea9c -2025-06-03T23:33:33.807Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: pipelining replication: peer="{Voter server-e2c7607f-d446-70e4-98ee-db17dffe1ecd e2c7607f-d446-70e4-98ee-db17dffe1ecd}" -2025-06-03T23:33:33.807Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: pipelining replication: peer="{Voter server-3b7102ab-b85f-eda8-87b5-c670ecefea9c 3b7102ab-b85f-eda8-87b5-c670ecefea9c}" -2025-06-03T23:33:34.812Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:33.807339951 +0000 UTC m=+28.828159321 (1 nodes), 631.71756ms from start of poll, 632.21928ms from cluster start. Timeout at 2025-06-03 23:33:34.812566631 +0000 UTC m=+29.833386001, 1.00522668s after stability -2025-06-03T23:33:34.813Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: starting snapshot up to: index=12 -2025-06-03T23:33:34.813Z [INFO] creating new snapshot: path=/tmp/raft4011383960/snapshots/2-12-1748993614813.tmp -2025-06-03T23:33:34.813Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: snapshot complete up to: index=12 -2025-06-03T23:33:34.815Z [INFO] creating new snapshot: path=/tmp/raft4011383960/snapshots/2-23-1748993614815.tmp -2025-06-03T23:33:34.816Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: copied to local snapshot: bytes=71 -2025-06-03T23:33:34.816Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: snapshot restore progress: id=2-23-1748993614815 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:34.816Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: restored user snapshot: index=23 -2025-06-03T23:33:34.816Z [ERROR] server-92e31615-95ee-3f5f-18b8-76c06d642c65: failed to get log: index=23 error="log not found" -2025-06-03T23:33:34.816Z [ERROR] server-92e31615-95ee-3f5f-18b8-76c06d642c65: failed to get log: index=23 error="log not found" -2025-06-03T23:33:34.816Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: aborting pipeline replication: peer="{Voter server-3b7102ab-b85f-eda8-87b5-c670ecefea9c 3b7102ab-b85f-eda8-87b5-c670ecefea9c}" -2025-06-03T23:33:34.816Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: aborting pipeline replication: peer="{Voter server-e2c7607f-d446-70e4-98ee-db17dffe1ecd e2c7607f-d446-70e4-98ee-db17dffe1ecd}" -2025-06-03T23:33:34.823Z [ERROR] server-92e31615-95ee-3f5f-18b8-76c06d642c65: failed to get log: index=23 error="log not found" -2025-06-03T23:33:34.823Z [ERROR] server-92e31615-95ee-3f5f-18b8-76c06d642c65: failed to get log: index=23 error="log not found" -2025-06-03T23:33:34.823Z [INFO] creating new snapshot: path=/tmp/raft1062171615/snapshots/2-23-1748993614823.tmp -2025-06-03T23:33:34.823Z [INFO] creating new snapshot: path=/tmp/raft3259099063/snapshots/2-23-1748993614823.tmp -2025-06-03T23:33:34.824Z [INFO] server-e2c7607f-d446-70e4-98ee-db17dffe1ecd: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:34.824Z [INFO] server-3b7102ab-b85f-eda8-87b5-c670ecefea9c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:34.824Z [INFO] server-e2c7607f-d446-70e4-98ee-db17dffe1ecd: copied to local snapshot: bytes=71 -2025-06-03T23:33:34.824Z [INFO] server-3b7102ab-b85f-eda8-87b5-c670ecefea9c: copied to local snapshot: bytes=71 -2025-06-03T23:33:34.824Z [INFO] server-e2c7607f-d446-70e4-98ee-db17dffe1ecd: snapshot restore progress: id=2-23-1748993614823 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:34.824Z [INFO] server-e2c7607f-d446-70e4-98ee-db17dffe1ecd: Installed remote snapshot -2025-06-03T23:33:34.824Z [WARN] server-e2c7607f-d446-70e4-98ee-db17dffe1ecd: skipping application of old log: index=22 -2025-06-03T23:33:34.824Z [INFO] server-3b7102ab-b85f-eda8-87b5-c670ecefea9c: snapshot restore progress: id=2-23-1748993614823 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:34.824Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: pipelining replication: peer="{Voter server-e2c7607f-d446-70e4-98ee-db17dffe1ecd e2c7607f-d446-70e4-98ee-db17dffe1ecd}" -2025-06-03T23:33:34.824Z [INFO] server-3b7102ab-b85f-eda8-87b5-c670ecefea9c: Installed remote snapshot -2025-06-03T23:33:34.824Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:34.824Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: pipelining replication: peer="{Voter server-3b7102ab-b85f-eda8-87b5-c670ecefea9c 3b7102ab-b85f-eda8-87b5-c670ecefea9c}" -2025-06-03T23:33:35.830Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:34.824835911 +0000 UTC m=+29.845655241 (2 nodes), 920ns from start of poll, 1.6497152s from cluster start. Timeout at 2025-06-03 23:33:35.830782271 +0000 UTC m=+30.851601641, 1.0059464s after stability -2025-06-03T23:33:35.843Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: aborting pipeline replication: peer="{Voter server-e2c7607f-d446-70e4-98ee-db17dffe1ecd e2c7607f-d446-70e4-98ee-db17dffe1ecd}" -2025-06-03T23:33:35.843Z [INFO] server-92e31615-95ee-3f5f-18b8-76c06d642c65: aborting pipeline replication: peer="{Voter server-3b7102ab-b85f-eda8-87b5-c670ecefea9c 3b7102ab-b85f-eda8-87b5-c670ecefea9c}" +2025-06-03T23:36:31.943Z [INFO] server-203e3fdb-e0ec-849c-0c19-48fd93d83828: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-203e3fdb-e0ec-849c-0c19-48fd93d83828 Address:203e3fdb-e0ec-849c-0c19-48fd93d83828} {Suffrage:Voter ID:server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4 Address:4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4} {Suffrage:Voter ID:server-ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1 Address:ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1}]" +2025-06-03T23:36:31.943Z [INFO] server-203e3fdb-e0ec-849c-0c19-48fd93d83828: entering follower state: follower="Node at 203e3fdb-e0ec-849c-0c19-48fd93d83828 [Follower]" leader-address= leader-id= +2025-06-03T23:36:31.943Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-203e3fdb-e0ec-849c-0c19-48fd93d83828 Address:203e3fdb-e0ec-849c-0c19-48fd93d83828} {Suffrage:Voter ID:server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4 Address:4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4} {Suffrage:Voter ID:server-ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1 Address:ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1}]" +2025-06-03T23:36:31.944Z [INFO] server-ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-203e3fdb-e0ec-849c-0c19-48fd93d83828 Address:203e3fdb-e0ec-849c-0c19-48fd93d83828} {Suffrage:Voter ID:server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4 Address:4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4} {Suffrage:Voter ID:server-ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1 Address:ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1}]" +2025-06-03T23:36:31.944Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:31.944Z [INFO] server-ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1: entering follower state: follower="Node at ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1 [Follower]" leader-address= leader-id= +2025-06-03T23:36:31.944Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: entering follower state: follower="Node at 4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4 [Follower]" leader-address= leader-id= +2025-06-03T23:36:32.776Z [WARN] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:32.776Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: entering candidate state: node="Node at 4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4 [Candidate]" term=2 +2025-06-03T23:36:32.777Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: election won: term=2 tally=2 +2025-06-03T23:36:32.777Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: entering leader state: leader="Node at 4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4 [Leader]" +2025-06-03T23:36:32.777Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: added peer, starting replication: peer=server-203e3fdb-e0ec-849c-0c19-48fd93d83828 +2025-06-03T23:36:32.777Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: added peer, starting replication: peer=server-ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1 +2025-06-03T23:36:32.777Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: pipelining replication: peer="{Voter server-ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1 ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1}" +2025-06-03T23:36:32.777Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: pipelining replication: peer="{Voter server-203e3fdb-e0ec-849c-0c19-48fd93d83828 203e3fdb-e0ec-849c-0c19-48fd93d83828}" +2025-06-03T23:36:33.782Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:32.777204991 +0000 UTC m=+28.098959921 (1 nodes), 833.08932ms from start of poll, 833.56404ms from cluster start. Timeout at 2025-06-03 23:36:33.782920671 +0000 UTC m=+29.104675601, 1.00571568s after stability +2025-06-03T23:36:33.785Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: starting snapshot up to: index=12 +2025-06-03T23:36:33.785Z [INFO] creating new snapshot: path=/tmp/raft718290336/snapshots/2-12-1748993793785.tmp +2025-06-03T23:36:33.785Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: snapshot complete up to: index=12 +2025-06-03T23:36:33.786Z [INFO] creating new snapshot: path=/tmp/raft718290336/snapshots/2-23-1748993793786.tmp +2025-06-03T23:36:33.787Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: copied to local snapshot: bytes=71 +2025-06-03T23:36:33.787Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: snapshot restore progress: id=2-23-1748993793786 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:33.787Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: restored user snapshot: index=23 +2025-06-03T23:36:33.787Z [ERROR] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: failed to get log: index=23 error="log not found" +2025-06-03T23:36:33.787Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: aborting pipeline replication: peer="{Voter server-ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1 ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1}" +2025-06-03T23:36:33.787Z [ERROR] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: failed to get log: index=23 error="log not found" +2025-06-03T23:36:33.787Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: aborting pipeline replication: peer="{Voter server-203e3fdb-e0ec-849c-0c19-48fd93d83828 203e3fdb-e0ec-849c-0c19-48fd93d83828}" +2025-06-03T23:36:33.794Z [ERROR] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: failed to get log: index=23 error="log not found" +2025-06-03T23:36:33.795Z [ERROR] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: failed to get log: index=23 error="log not found" +2025-06-03T23:36:33.795Z [INFO] creating new snapshot: path=/tmp/raft404404999/snapshots/2-23-1748993793795.tmp +2025-06-03T23:36:33.795Z [INFO] creating new snapshot: path=/tmp/raft4033883192/snapshots/2-23-1748993793795.tmp +2025-06-03T23:36:33.795Z [INFO] server-203e3fdb-e0ec-849c-0c19-48fd93d83828: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:33.795Z [INFO] server-ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:33.796Z [INFO] server-203e3fdb-e0ec-849c-0c19-48fd93d83828: copied to local snapshot: bytes=71 +2025-06-03T23:36:33.796Z [INFO] server-203e3fdb-e0ec-849c-0c19-48fd93d83828: snapshot restore progress: id=2-23-1748993793795 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:33.796Z [INFO] server-203e3fdb-e0ec-849c-0c19-48fd93d83828: Installed remote snapshot +2025-06-03T23:36:33.796Z [INFO] server-ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1: copied to local snapshot: bytes=71 +2025-06-03T23:36:33.796Z [WARN] server-203e3fdb-e0ec-849c-0c19-48fd93d83828: skipping application of old log: index=22 +2025-06-03T23:36:33.796Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: pipelining replication: peer="{Voter server-203e3fdb-e0ec-849c-0c19-48fd93d83828 203e3fdb-e0ec-849c-0c19-48fd93d83828}" +2025-06-03T23:36:33.796Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:33.796Z [INFO] server-ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1: snapshot restore progress: id=2-23-1748993793795 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:33.796Z [INFO] server-ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1: Installed remote snapshot +2025-06-03T23:36:33.796Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: pipelining replication: peer="{Voter server-ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1 ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1}" +2025-06-03T23:36:34.802Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:33.796675831 +0000 UTC m=+29.118430761 (2 nodes), 960ns from start of poll, 1.85303488s from cluster start. Timeout at 2025-06-03 23:36:34.802667311 +0000 UTC m=+30.124422241, 1.00599148s after stability +2025-06-03T23:36:34.814Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: aborting pipeline replication: peer="{Voter server-ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1 ae04bd43-cd4b-9216-ebbc-9d8c5c8d15b1}" +2025-06-03T23:36:34.814Z [INFO] server-4d9ee53a-a3e1-f6f2-08f5-84e1040db1d4: aborting pipeline replication: peer="{Voter server-203e3fdb-e0ec-849c-0c19-48fd93d83828 203e3fdb-e0ec-849c-0c19-48fd93d83828}" === RUN TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false -2025-06-03T23:33:35.844Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-747efe47-d407-5522-8c67-dcc075431822 Address:747efe47-d407-5522-8c67-dcc075431822} {Suffrage:Voter ID:server-5fc5e2c0-5db0-5416-0a13-340f760c0474 Address:5fc5e2c0-5db0-5416-0a13-340f760c0474} {Suffrage:Voter ID:server-707a04bc-ada7-10d7-20ac-310680d53a60 Address:707a04bc-ada7-10d7-20ac-310680d53a60}]" -2025-06-03T23:33:35.844Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: entering follower state: follower="Node at 747efe47-d407-5522-8c67-dcc075431822 [Follower]" leader-address= leader-id= -2025-06-03T23:33:35.844Z [INFO] server-5fc5e2c0-5db0-5416-0a13-340f760c0474: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-747efe47-d407-5522-8c67-dcc075431822 Address:747efe47-d407-5522-8c67-dcc075431822} {Suffrage:Voter ID:server-5fc5e2c0-5db0-5416-0a13-340f760c0474 Address:5fc5e2c0-5db0-5416-0a13-340f760c0474} {Suffrage:Voter ID:server-707a04bc-ada7-10d7-20ac-310680d53a60 Address:707a04bc-ada7-10d7-20ac-310680d53a60}]" -2025-06-03T23:33:35.844Z [INFO] server-5fc5e2c0-5db0-5416-0a13-340f760c0474: entering follower state: follower="Node at 5fc5e2c0-5db0-5416-0a13-340f760c0474 [Follower]" leader-address= leader-id= -2025-06-03T23:33:35.845Z [INFO] server-707a04bc-ada7-10d7-20ac-310680d53a60: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-747efe47-d407-5522-8c67-dcc075431822 Address:747efe47-d407-5522-8c67-dcc075431822} {Suffrage:Voter ID:server-5fc5e2c0-5db0-5416-0a13-340f760c0474 Address:5fc5e2c0-5db0-5416-0a13-340f760c0474} {Suffrage:Voter ID:server-707a04bc-ada7-10d7-20ac-310680d53a60 Address:707a04bc-ada7-10d7-20ac-310680d53a60}]" -2025-06-03T23:33:35.845Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:35.845Z [INFO] server-707a04bc-ada7-10d7-20ac-310680d53a60: entering follower state: follower="Node at 707a04bc-ada7-10d7-20ac-310680d53a60 [Follower]" leader-address= leader-id= -2025-06-03T23:33:36.697Z [WARN] server-747efe47-d407-5522-8c67-dcc075431822: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:36.697Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: entering candidate state: node="Node at 747efe47-d407-5522-8c67-dcc075431822 [Candidate]" term=2 -2025-06-03T23:33:36.697Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: election won: term=2 tally=2 -2025-06-03T23:33:36.697Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: entering leader state: leader="Node at 747efe47-d407-5522-8c67-dcc075431822 [Leader]" -2025-06-03T23:33:36.697Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: added peer, starting replication: peer=server-5fc5e2c0-5db0-5416-0a13-340f760c0474 -2025-06-03T23:33:36.697Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: added peer, starting replication: peer=server-707a04bc-ada7-10d7-20ac-310680d53a60 -2025-06-03T23:33:36.698Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: pipelining replication: peer="{Voter server-707a04bc-ada7-10d7-20ac-310680d53a60 707a04bc-ada7-10d7-20ac-310680d53a60}" -2025-06-03T23:33:36.698Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: pipelining replication: peer="{Voter server-5fc5e2c0-5db0-5416-0a13-340f760c0474 5fc5e2c0-5db0-5416-0a13-340f760c0474}" -2025-06-03T23:33:37.704Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:36.697953911 +0000 UTC m=+31.718773281 (1 nodes), 852.86044ms from start of poll, 853.34252ms from cluster start. Timeout at 2025-06-03 23:33:37.703998711 +0000 UTC m=+32.724818081, 1.0060448s after stability -2025-06-03T23:33:37.704Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: starting snapshot up to: index=12 -2025-06-03T23:33:37.704Z [INFO] creating new snapshot: path=/tmp/raft3645003988/snapshots/2-12-1748993617704.tmp -2025-06-03T23:33:37.705Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: snapshot complete up to: index=12 -2025-06-03T23:33:37.707Z [INFO] creating new snapshot: path=/tmp/raft3645003988/snapshots/2-23-1748993617707.tmp -2025-06-03T23:33:37.707Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: copied to local snapshot: bytes=71 -2025-06-03T23:33:37.708Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: snapshot restore progress: id=2-23-1748993617707 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:37.708Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: removing all old logs from log store -2025-06-03T23:33:37.708Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: compacting logs: from=1 to=22 -2025-06-03T23:33:37.708Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: restored user snapshot: index=23 -2025-06-03T23:33:37.708Z [ERROR] server-747efe47-d407-5522-8c67-dcc075431822: failed to get log: index=22 error="log not found" -2025-06-03T23:33:37.708Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: aborting pipeline replication: peer="{Voter server-707a04bc-ada7-10d7-20ac-310680d53a60 707a04bc-ada7-10d7-20ac-310680d53a60}" -2025-06-03T23:33:37.708Z [ERROR] server-747efe47-d407-5522-8c67-dcc075431822: failed to get log: index=22 error="log not found" -2025-06-03T23:33:37.708Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: aborting pipeline replication: peer="{Voter server-5fc5e2c0-5db0-5416-0a13-340f760c0474 5fc5e2c0-5db0-5416-0a13-340f760c0474}" -2025-06-03T23:33:37.716Z [ERROR] server-747efe47-d407-5522-8c67-dcc075431822: failed to get log: index=22 error="log not found" -2025-06-03T23:33:37.716Z [INFO] creating new snapshot: path=/tmp/raft3572754334/snapshots/2-23-1748993617716.tmp -2025-06-03T23:33:37.716Z [INFO] server-707a04bc-ada7-10d7-20ac-310680d53a60: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:37.717Z [INFO] server-707a04bc-ada7-10d7-20ac-310680d53a60: copied to local snapshot: bytes=71 -2025-06-03T23:33:37.717Z [ERROR] server-747efe47-d407-5522-8c67-dcc075431822: failed to get log: index=22 error="log not found" -2025-06-03T23:33:37.717Z [INFO] server-707a04bc-ada7-10d7-20ac-310680d53a60: snapshot restore progress: id=2-23-1748993617716 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:37.717Z [INFO] server-707a04bc-ada7-10d7-20ac-310680d53a60: removing all old logs from log store -2025-06-03T23:33:37.717Z [INFO] server-707a04bc-ada7-10d7-20ac-310680d53a60: compacting logs: from=1 to=22 -2025-06-03T23:33:37.717Z [INFO] server-707a04bc-ada7-10d7-20ac-310680d53a60: Installed remote snapshot -2025-06-03T23:33:37.717Z [WARN] server-707a04bc-ada7-10d7-20ac-310680d53a60: skipping application of old log: index=22 -2025-06-03T23:33:37.717Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: pipelining replication: peer="{Voter server-707a04bc-ada7-10d7-20ac-310680d53a60 707a04bc-ada7-10d7-20ac-310680d53a60}" -2025-06-03T23:33:37.717Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:37.717Z [INFO] creating new snapshot: path=/tmp/raft711929527/snapshots/2-23-1748993617717.tmp -2025-06-03T23:33:37.717Z [INFO] server-5fc5e2c0-5db0-5416-0a13-340f760c0474: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:37.718Z [INFO] server-5fc5e2c0-5db0-5416-0a13-340f760c0474: copied to local snapshot: bytes=71 -2025-06-03T23:33:37.718Z [INFO] server-5fc5e2c0-5db0-5416-0a13-340f760c0474: snapshot restore progress: id=2-23-1748993617717 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:37.718Z [INFO] server-5fc5e2c0-5db0-5416-0a13-340f760c0474: removing all old logs from log store -2025-06-03T23:33:37.718Z [INFO] server-5fc5e2c0-5db0-5416-0a13-340f760c0474: compacting logs: from=1 to=22 -2025-06-03T23:33:37.718Z [INFO] server-5fc5e2c0-5db0-5416-0a13-340f760c0474: Installed remote snapshot -2025-06-03T23:33:37.718Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: pipelining replication: peer="{Voter server-5fc5e2c0-5db0-5416-0a13-340f760c0474 5fc5e2c0-5db0-5416-0a13-340f760c0474}" -2025-06-03T23:33:38.723Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:37.717648471 +0000 UTC m=+32.738467841 (2 nodes), 1.24µs from start of poll, 1.87303708s from cluster start. Timeout at 2025-06-03 23:33:38.723444751 +0000 UTC m=+33.744264121, 1.00579628s after stability -2025-06-03T23:33:38.735Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: aborting pipeline replication: peer="{Voter server-707a04bc-ada7-10d7-20ac-310680d53a60 707a04bc-ada7-10d7-20ac-310680d53a60}" -2025-06-03T23:33:38.735Z [INFO] server-747efe47-d407-5522-8c67-dcc075431822: aborting pipeline replication: peer="{Voter server-5fc5e2c0-5db0-5416-0a13-340f760c0474 5fc5e2c0-5db0-5416-0a13-340f760c0474}" +2025-06-03T23:36:34.817Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c979709e-2a34-35ef-5888-4f91a1d76fa6 Address:c979709e-2a34-35ef-5888-4f91a1d76fa6} {Suffrage:Voter ID:server-8c246903-5a8e-5d14-ca99-c0eeaee0266d Address:8c246903-5a8e-5d14-ca99-c0eeaee0266d} {Suffrage:Voter ID:server-b493324a-e301-20c4-a92e-ee8cf3c029f1 Address:b493324a-e301-20c4-a92e-ee8cf3c029f1}]" +2025-06-03T23:36:34.817Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: entering follower state: follower="Node at c979709e-2a34-35ef-5888-4f91a1d76fa6 [Follower]" leader-address= leader-id= +2025-06-03T23:36:34.817Z [INFO] server-8c246903-5a8e-5d14-ca99-c0eeaee0266d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c979709e-2a34-35ef-5888-4f91a1d76fa6 Address:c979709e-2a34-35ef-5888-4f91a1d76fa6} {Suffrage:Voter ID:server-8c246903-5a8e-5d14-ca99-c0eeaee0266d Address:8c246903-5a8e-5d14-ca99-c0eeaee0266d} {Suffrage:Voter ID:server-b493324a-e301-20c4-a92e-ee8cf3c029f1 Address:b493324a-e301-20c4-a92e-ee8cf3c029f1}]" +2025-06-03T23:36:34.817Z [INFO] server-8c246903-5a8e-5d14-ca99-c0eeaee0266d: entering follower state: follower="Node at 8c246903-5a8e-5d14-ca99-c0eeaee0266d [Follower]" leader-address= leader-id= +2025-06-03T23:36:34.817Z [INFO] server-b493324a-e301-20c4-a92e-ee8cf3c029f1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c979709e-2a34-35ef-5888-4f91a1d76fa6 Address:c979709e-2a34-35ef-5888-4f91a1d76fa6} {Suffrage:Voter ID:server-8c246903-5a8e-5d14-ca99-c0eeaee0266d Address:8c246903-5a8e-5d14-ca99-c0eeaee0266d} {Suffrage:Voter ID:server-b493324a-e301-20c4-a92e-ee8cf3c029f1 Address:b493324a-e301-20c4-a92e-ee8cf3c029f1}]" +2025-06-03T23:36:34.817Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:34.817Z [INFO] server-b493324a-e301-20c4-a92e-ee8cf3c029f1: entering follower state: follower="Node at b493324a-e301-20c4-a92e-ee8cf3c029f1 [Follower]" leader-address= leader-id= +2025-06-03T23:36:35.421Z [WARN] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:35.421Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: entering candidate state: node="Node at c979709e-2a34-35ef-5888-4f91a1d76fa6 [Candidate]" term=2 +2025-06-03T23:36:35.422Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: election won: term=2 tally=2 +2025-06-03T23:36:35.422Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: entering leader state: leader="Node at c979709e-2a34-35ef-5888-4f91a1d76fa6 [Leader]" +2025-06-03T23:36:35.422Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: added peer, starting replication: peer=server-8c246903-5a8e-5d14-ca99-c0eeaee0266d +2025-06-03T23:36:35.422Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: added peer, starting replication: peer=server-b493324a-e301-20c4-a92e-ee8cf3c029f1 +2025-06-03T23:36:35.422Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: pipelining replication: peer="{Voter server-b493324a-e301-20c4-a92e-ee8cf3c029f1 b493324a-e301-20c4-a92e-ee8cf3c029f1}" +2025-06-03T23:36:35.422Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: pipelining replication: peer="{Voter server-8c246903-5a8e-5d14-ca99-c0eeaee0266d 8c246903-5a8e-5d14-ca99-c0eeaee0266d}" +2025-06-03T23:36:36.427Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:35.422116151 +0000 UTC m=+30.743871081 (1 nodes), 604.53832ms from start of poll, 605.76028ms from cluster start. Timeout at 2025-06-03 23:36:36.427907431 +0000 UTC m=+31.749662361, 1.00579128s after stability +2025-06-03T23:36:36.429Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: starting snapshot up to: index=12 +2025-06-03T23:36:36.429Z [INFO] creating new snapshot: path=/tmp/raft829612317/snapshots/2-12-1748993796429.tmp +2025-06-03T23:36:36.430Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: snapshot complete up to: index=12 +2025-06-03T23:36:36.431Z [INFO] creating new snapshot: path=/tmp/raft829612317/snapshots/2-23-1748993796431.tmp +2025-06-03T23:36:36.432Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: copied to local snapshot: bytes=71 +2025-06-03T23:36:36.432Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: snapshot restore progress: id=2-23-1748993796431 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:36.432Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: removing all old logs from log store +2025-06-03T23:36:36.432Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: compacting logs: from=1 to=22 +2025-06-03T23:36:36.432Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: restored user snapshot: index=23 +2025-06-03T23:36:36.432Z [ERROR] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: failed to get log: index=22 error="log not found" +2025-06-03T23:36:36.432Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: aborting pipeline replication: peer="{Voter server-8c246903-5a8e-5d14-ca99-c0eeaee0266d 8c246903-5a8e-5d14-ca99-c0eeaee0266d}" +2025-06-03T23:36:36.432Z [ERROR] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: failed to get log: index=22 error="log not found" +2025-06-03T23:36:36.432Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: aborting pipeline replication: peer="{Voter server-b493324a-e301-20c4-a92e-ee8cf3c029f1 b493324a-e301-20c4-a92e-ee8cf3c029f1}" +2025-06-03T23:36:36.441Z [ERROR] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: failed to get log: index=22 error="log not found" +2025-06-03T23:36:36.442Z [INFO] creating new snapshot: path=/tmp/raft3300187196/snapshots/2-23-1748993796442.tmp +2025-06-03T23:36:36.442Z [ERROR] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: failed to get log: index=22 error="log not found" +2025-06-03T23:36:36.442Z [INFO] server-b493324a-e301-20c4-a92e-ee8cf3c029f1: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:36.442Z [INFO] creating new snapshot: path=/tmp/raft3273558242/snapshots/2-23-1748993796442.tmp +2025-06-03T23:36:36.443Z [INFO] server-b493324a-e301-20c4-a92e-ee8cf3c029f1: copied to local snapshot: bytes=71 +2025-06-03T23:36:36.443Z [INFO] server-8c246903-5a8e-5d14-ca99-c0eeaee0266d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:36.443Z [INFO] server-b493324a-e301-20c4-a92e-ee8cf3c029f1: snapshot restore progress: id=2-23-1748993796442 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:36.443Z [INFO] server-b493324a-e301-20c4-a92e-ee8cf3c029f1: removing all old logs from log store +2025-06-03T23:36:36.443Z [INFO] server-b493324a-e301-20c4-a92e-ee8cf3c029f1: compacting logs: from=1 to=22 +2025-06-03T23:36:36.443Z [INFO] server-b493324a-e301-20c4-a92e-ee8cf3c029f1: Installed remote snapshot +2025-06-03T23:36:36.443Z [INFO] server-8c246903-5a8e-5d14-ca99-c0eeaee0266d: copied to local snapshot: bytes=71 +2025-06-03T23:36:36.443Z [WARN] server-b493324a-e301-20c4-a92e-ee8cf3c029f1: skipping application of old log: index=22 +2025-06-03T23:36:36.443Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: pipelining replication: peer="{Voter server-b493324a-e301-20c4-a92e-ee8cf3c029f1 b493324a-e301-20c4-a92e-ee8cf3c029f1}" +2025-06-03T23:36:36.443Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:36.443Z [INFO] server-8c246903-5a8e-5d14-ca99-c0eeaee0266d: snapshot restore progress: id=2-23-1748993796442 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:36.443Z [INFO] server-8c246903-5a8e-5d14-ca99-c0eeaee0266d: removing all old logs from log store +2025-06-03T23:36:36.443Z [INFO] server-8c246903-5a8e-5d14-ca99-c0eeaee0266d: compacting logs: from=1 to=22 +2025-06-03T23:36:36.443Z [INFO] server-8c246903-5a8e-5d14-ca99-c0eeaee0266d: Installed remote snapshot +2025-06-03T23:36:36.444Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: pipelining replication: peer="{Voter server-8c246903-5a8e-5d14-ca99-c0eeaee0266d 8c246903-5a8e-5d14-ca99-c0eeaee0266d}" +2025-06-03T23:36:37.449Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:36.443766391 +0000 UTC m=+31.765521321 (2 nodes), 1.24µs from start of poll, 1.62741052s from cluster start. Timeout at 2025-06-03 23:36:37.449806231 +0000 UTC m=+32.771561161, 1.00603984s after stability +2025-06-03T23:36:37.461Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: aborting pipeline replication: peer="{Voter server-b493324a-e301-20c4-a92e-ee8cf3c029f1 b493324a-e301-20c4-a92e-ee8cf3c029f1}" +2025-06-03T23:36:37.461Z [INFO] server-c979709e-2a34-35ef-5888-4f91a1d76fa6: aborting pipeline replication: peer="{Voter server-8c246903-5a8e-5d14-ca99-c0eeaee0266d 8c246903-5a8e-5d14-ca99-c0eeaee0266d}" === RUN TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true -2025-06-03T23:33:38.737Z [INFO] server-922b3217-6014-6aa8-2275-c67f69caf62d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-922b3217-6014-6aa8-2275-c67f69caf62d Address:922b3217-6014-6aa8-2275-c67f69caf62d} {Suffrage:Voter ID:server-cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d Address:cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d} {Suffrage:Voter ID:server-fb5c6c7a-b003-0554-bb6e-7d45f3ca5e5c Address:fb5c6c7a-b003-0554-bb6e-7d45f3ca5e5c}]" -2025-06-03T23:33:38.737Z [INFO] server-922b3217-6014-6aa8-2275-c67f69caf62d: entering follower state: follower="Node at 922b3217-6014-6aa8-2275-c67f69caf62d [Follower]" leader-address= leader-id= -2025-06-03T23:33:38.737Z [INFO] server-cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-922b3217-6014-6aa8-2275-c67f69caf62d Address:922b3217-6014-6aa8-2275-c67f69caf62d} {Suffrage:Voter ID:server-cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d Address:cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d} {Suffrage:Voter ID:server-fb5c6c7a-b003-0554-bb6e-7d45f3ca5e5c Address:fb5c6c7a-b003-0554-bb6e-7d45f3ca5e5c}]" -2025-06-03T23:33:38.737Z [INFO] server-cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d: entering follower state: follower="Node at cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d [Follower]" leader-address= leader-id= -2025-06-03T23:33:38.737Z [INFO] server-fb5c6c7a-b003-0554-bb6e-7d45f3ca5e5c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-922b3217-6014-6aa8-2275-c67f69caf62d Address:922b3217-6014-6aa8-2275-c67f69caf62d} {Suffrage:Voter ID:server-cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d Address:cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d} {Suffrage:Voter ID:server-fb5c6c7a-b003-0554-bb6e-7d45f3ca5e5c Address:fb5c6c7a-b003-0554-bb6e-7d45f3ca5e5c}]" -2025-06-03T23:33:38.737Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:38.737Z [INFO] server-fb5c6c7a-b003-0554-bb6e-7d45f3ca5e5c: entering follower state: follower="Node at fb5c6c7a-b003-0554-bb6e-7d45f3ca5e5c [Follower]" leader-address= leader-id= -2025-06-03T23:33:39.419Z [WARN] server-cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:39.419Z [INFO] server-cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d: entering candidate state: node="Node at cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d [Candidate]" term=2 -2025-06-03T23:33:39.419Z [INFO] server-cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d: election won: term=2 tally=2 -2025-06-03T23:33:39.419Z [INFO] server-cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d: entering leader state: leader="Node at cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d [Leader]" -2025-06-03T23:33:39.419Z [INFO] server-cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d: added peer, starting replication: peer=server-922b3217-6014-6aa8-2275-c67f69caf62d -2025-06-03T23:33:39.419Z [INFO] server-cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d: added peer, starting replication: peer=server-fb5c6c7a-b003-0554-bb6e-7d45f3ca5e5c -2025-06-03T23:33:39.419Z [INFO] server-cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d: pipelining replication: peer="{Voter server-922b3217-6014-6aa8-2275-c67f69caf62d 922b3217-6014-6aa8-2275-c67f69caf62d}" -2025-06-03T23:33:39.419Z [INFO] server-cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d: pipelining replication: peer="{Voter server-fb5c6c7a-b003-0554-bb6e-7d45f3ca5e5c fb5c6c7a-b003-0554-bb6e-7d45f3ca5e5c}" -2025-06-03T23:33:40.425Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:39.419815071 +0000 UTC m=+34.440634441 (1 nodes), 682.45872ms from start of poll, 682.92628ms from cluster start. Timeout at 2025-06-03 23:33:40.425497071 +0000 UTC m=+35.446316441, 1.005682s after stability -2025-06-03T23:33:40.426Z [INFO] server-cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d: starting snapshot up to: index=12 -2025-06-03T23:33:40.426Z [INFO] creating new snapshot: path=/tmp/raft592681826/snapshots/2-12-1748993620426.tmp -2025-06-03T23:33:40.426Z [INFO] server-cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d: snapshot complete up to: index=12 -2025-06-03T23:33:40.428Z [INFO] server-e4d16f43-c036-1763-c5fa-b5344c39f861: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e4d16f43-c036-1763-c5fa-b5344c39f861 Address:e4d16f43-c036-1763-c5fa-b5344c39f861} {Suffrage:Voter ID:server-6715519e-6fbe-07e5-aeee-71c4bd194f66 Address:6715519e-6fbe-07e5-aeee-71c4bd194f66} {Suffrage:Voter ID:server-67866b5c-4086-5764-c82a-b143e2271058 Address:67866b5c-4086-5764-c82a-b143e2271058}]" -2025-06-03T23:33:40.428Z [INFO] server-e4d16f43-c036-1763-c5fa-b5344c39f861: entering follower state: follower="Node at e4d16f43-c036-1763-c5fa-b5344c39f861 [Follower]" leader-address= leader-id= -2025-06-03T23:33:40.428Z [INFO] server-6715519e-6fbe-07e5-aeee-71c4bd194f66: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e4d16f43-c036-1763-c5fa-b5344c39f861 Address:e4d16f43-c036-1763-c5fa-b5344c39f861} {Suffrage:Voter ID:server-6715519e-6fbe-07e5-aeee-71c4bd194f66 Address:6715519e-6fbe-07e5-aeee-71c4bd194f66} {Suffrage:Voter ID:server-67866b5c-4086-5764-c82a-b143e2271058 Address:67866b5c-4086-5764-c82a-b143e2271058}]" -2025-06-03T23:33:40.428Z [INFO] server-6715519e-6fbe-07e5-aeee-71c4bd194f66: entering follower state: follower="Node at 6715519e-6fbe-07e5-aeee-71c4bd194f66 [Follower]" leader-address= leader-id= -2025-06-03T23:33:40.429Z [INFO] server-67866b5c-4086-5764-c82a-b143e2271058: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e4d16f43-c036-1763-c5fa-b5344c39f861 Address:e4d16f43-c036-1763-c5fa-b5344c39f861} {Suffrage:Voter ID:server-6715519e-6fbe-07e5-aeee-71c4bd194f66 Address:6715519e-6fbe-07e5-aeee-71c4bd194f66} {Suffrage:Voter ID:server-67866b5c-4086-5764-c82a-b143e2271058 Address:67866b5c-4086-5764-c82a-b143e2271058}]" -2025-06-03T23:33:40.429Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:40.429Z [INFO] server-67866b5c-4086-5764-c82a-b143e2271058: entering follower state: follower="Node at 67866b5c-4086-5764-c82a-b143e2271058 [Follower]" leader-address= leader-id= -2025-06-03T23:33:41.149Z [WARN] server-67866b5c-4086-5764-c82a-b143e2271058: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:41.149Z [INFO] server-67866b5c-4086-5764-c82a-b143e2271058: entering candidate state: node="Node at 67866b5c-4086-5764-c82a-b143e2271058 [Candidate]" term=2 -2025-06-03T23:33:41.149Z [INFO] server-67866b5c-4086-5764-c82a-b143e2271058: election won: term=2 tally=2 -2025-06-03T23:33:41.149Z [INFO] server-67866b5c-4086-5764-c82a-b143e2271058: entering leader state: leader="Node at 67866b5c-4086-5764-c82a-b143e2271058 [Leader]" -2025-06-03T23:33:41.149Z [INFO] server-67866b5c-4086-5764-c82a-b143e2271058: added peer, starting replication: peer=server-e4d16f43-c036-1763-c5fa-b5344c39f861 -2025-06-03T23:33:41.149Z [INFO] server-67866b5c-4086-5764-c82a-b143e2271058: added peer, starting replication: peer=server-6715519e-6fbe-07e5-aeee-71c4bd194f66 -2025-06-03T23:33:41.149Z [INFO] server-67866b5c-4086-5764-c82a-b143e2271058: pipelining replication: peer="{Voter server-6715519e-6fbe-07e5-aeee-71c4bd194f66 6715519e-6fbe-07e5-aeee-71c4bd194f66}" -2025-06-03T23:33:41.149Z [INFO] server-67866b5c-4086-5764-c82a-b143e2271058: pipelining replication: peer="{Voter server-e4d16f43-c036-1763-c5fa-b5344c39f861 e4d16f43-c036-1763-c5fa-b5344c39f861}" -2025-06-03T23:33:42.155Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:41.149823431 +0000 UTC m=+36.170642801 (1 nodes), 720.3712ms from start of poll, 722.03544ms from cluster start. Timeout at 2025-06-03 23:33:42.155741191 +0000 UTC m=+37.176560561, 1.00591776s after stability -2025-06-03T23:33:42.156Z [INFO] creating new snapshot: path=/tmp/raft732690721/snapshots/2-14-1748993622156.tmp -2025-06-03T23:33:42.156Z [INFO] server-67866b5c-4086-5764-c82a-b143e2271058: copied to local snapshot: bytes=71 -2025-06-03T23:33:42.156Z [INFO] server-67866b5c-4086-5764-c82a-b143e2271058: snapshot restore progress: id=2-14-1748993622156 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:42.156Z [INFO] server-67866b5c-4086-5764-c82a-b143e2271058: restored user snapshot: index=14 -2025-06-03T23:33:42.156Z [ERROR] server-67866b5c-4086-5764-c82a-b143e2271058: failed to get log: index=3 error="log not found" -2025-06-03T23:33:42.156Z [INFO] server-67866b5c-4086-5764-c82a-b143e2271058: aborting pipeline replication: peer="{Voter server-6715519e-6fbe-07e5-aeee-71c4bd194f66 6715519e-6fbe-07e5-aeee-71c4bd194f66}" -2025-06-03T23:33:42.156Z [ERROR] server-67866b5c-4086-5764-c82a-b143e2271058: failed to get log: index=3 error="log not found" -2025-06-03T23:33:42.157Z [INFO] server-67866b5c-4086-5764-c82a-b143e2271058: aborting pipeline replication: peer="{Voter server-e4d16f43-c036-1763-c5fa-b5344c39f861 e4d16f43-c036-1763-c5fa-b5344c39f861}" -2025-06-03T23:33:42.163Z [ERROR] server-67866b5c-4086-5764-c82a-b143e2271058: failed to get log: index=3 error="log not found" -2025-06-03T23:33:42.163Z [INFO] creating new snapshot: path=/tmp/raft1221915467/snapshots/2-14-1748993622163.tmp -2025-06-03T23:33:42.163Z [INFO] server-e4d16f43-c036-1763-c5fa-b5344c39f861: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:42.164Z [INFO] server-e4d16f43-c036-1763-c5fa-b5344c39f861: copied to local snapshot: bytes=71 -2025-06-03T23:33:42.164Z [INFO] server-e4d16f43-c036-1763-c5fa-b5344c39f861: snapshot restore progress: id=2-14-1748993622163 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:42.164Z [INFO] server-e4d16f43-c036-1763-c5fa-b5344c39f861: Installed remote snapshot -2025-06-03T23:33:42.164Z [ERROR] server-67866b5c-4086-5764-c82a-b143e2271058: failed to get log: index=3 error="log not found" -2025-06-03T23:33:42.164Z [INFO] server-67866b5c-4086-5764-c82a-b143e2271058: pipelining replication: peer="{Voter server-e4d16f43-c036-1763-c5fa-b5344c39f861 e4d16f43-c036-1763-c5fa-b5344c39f861}" -2025-06-03T23:33:42.164Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:42.165Z [INFO] creating new snapshot: path=/tmp/raft1788145580/snapshots/2-14-1748993622165.tmp -2025-06-03T23:33:42.165Z [INFO] server-6715519e-6fbe-07e5-aeee-71c4bd194f66: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:42.165Z [INFO] server-6715519e-6fbe-07e5-aeee-71c4bd194f66: copied to local snapshot: bytes=71 -2025-06-03T23:33:42.165Z [INFO] server-6715519e-6fbe-07e5-aeee-71c4bd194f66: snapshot restore progress: id=2-14-1748993622165 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:42.165Z [INFO] server-6715519e-6fbe-07e5-aeee-71c4bd194f66: Installed remote snapshot -2025-06-03T23:33:42.166Z [INFO] server-67866b5c-4086-5764-c82a-b143e2271058: pipelining replication: peer="{Voter server-6715519e-6fbe-07e5-aeee-71c4bd194f66 6715519e-6fbe-07e5-aeee-71c4bd194f66}" -2025-06-03T23:33:43.170Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:42.164828311 +0000 UTC m=+37.185647681 (2 nodes), 1.12µs from start of poll, 1.73704032s from cluster start. Timeout at 2025-06-03 23:33:43.170859831 +0000 UTC m=+38.191679201, 1.00603152s after stability -2025-06-03T23:33:43.183Z [INFO] server-cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d: aborting pipeline replication: peer="{Voter server-922b3217-6014-6aa8-2275-c67f69caf62d 922b3217-6014-6aa8-2275-c67f69caf62d}" -2025-06-03T23:33:43.183Z [INFO] server-cfff6cee-3ab2-740f-0fe0-eae4d09b3e6d: aborting pipeline replication: peer="{Voter server-fb5c6c7a-b003-0554-bb6e-7d45f3ca5e5c fb5c6c7a-b003-0554-bb6e-7d45f3ca5e5c}" +2025-06-03T23:36:37.463Z [INFO] server-83870856-48a9-2eec-6400-7ff0cc634067: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-83870856-48a9-2eec-6400-7ff0cc634067 Address:83870856-48a9-2eec-6400-7ff0cc634067} {Suffrage:Voter ID:server-9afa250d-43a4-1f59-a0e4-486f5c3aad6c Address:9afa250d-43a4-1f59-a0e4-486f5c3aad6c} {Suffrage:Voter ID:server-75650069-2d47-ea0f-ddee-5e2aca100923 Address:75650069-2d47-ea0f-ddee-5e2aca100923}]" +2025-06-03T23:36:37.463Z [INFO] server-83870856-48a9-2eec-6400-7ff0cc634067: entering follower state: follower="Node at 83870856-48a9-2eec-6400-7ff0cc634067 [Follower]" leader-address= leader-id= +2025-06-03T23:36:37.463Z [INFO] server-9afa250d-43a4-1f59-a0e4-486f5c3aad6c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-83870856-48a9-2eec-6400-7ff0cc634067 Address:83870856-48a9-2eec-6400-7ff0cc634067} {Suffrage:Voter ID:server-9afa250d-43a4-1f59-a0e4-486f5c3aad6c Address:9afa250d-43a4-1f59-a0e4-486f5c3aad6c} {Suffrage:Voter ID:server-75650069-2d47-ea0f-ddee-5e2aca100923 Address:75650069-2d47-ea0f-ddee-5e2aca100923}]" +2025-06-03T23:36:37.463Z [INFO] server-9afa250d-43a4-1f59-a0e4-486f5c3aad6c: entering follower state: follower="Node at 9afa250d-43a4-1f59-a0e4-486f5c3aad6c [Follower]" leader-address= leader-id= +2025-06-03T23:36:37.464Z [INFO] server-75650069-2d47-ea0f-ddee-5e2aca100923: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-83870856-48a9-2eec-6400-7ff0cc634067 Address:83870856-48a9-2eec-6400-7ff0cc634067} {Suffrage:Voter ID:server-9afa250d-43a4-1f59-a0e4-486f5c3aad6c Address:9afa250d-43a4-1f59-a0e4-486f5c3aad6c} {Suffrage:Voter ID:server-75650069-2d47-ea0f-ddee-5e2aca100923 Address:75650069-2d47-ea0f-ddee-5e2aca100923}]" +2025-06-03T23:36:37.464Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:37.464Z [INFO] server-75650069-2d47-ea0f-ddee-5e2aca100923: entering follower state: follower="Node at 75650069-2d47-ea0f-ddee-5e2aca100923 [Follower]" leader-address= leader-id= +2025-06-03T23:36:38.170Z [WARN] server-75650069-2d47-ea0f-ddee-5e2aca100923: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:38.170Z [INFO] server-75650069-2d47-ea0f-ddee-5e2aca100923: entering candidate state: node="Node at 75650069-2d47-ea0f-ddee-5e2aca100923 [Candidate]" term=2 +2025-06-03T23:36:38.170Z [INFO] server-75650069-2d47-ea0f-ddee-5e2aca100923: election won: term=2 tally=2 +2025-06-03T23:36:38.170Z [INFO] server-75650069-2d47-ea0f-ddee-5e2aca100923: entering leader state: leader="Node at 75650069-2d47-ea0f-ddee-5e2aca100923 [Leader]" +2025-06-03T23:36:38.170Z [INFO] server-75650069-2d47-ea0f-ddee-5e2aca100923: added peer, starting replication: peer=server-83870856-48a9-2eec-6400-7ff0cc634067 +2025-06-03T23:36:38.170Z [INFO] server-75650069-2d47-ea0f-ddee-5e2aca100923: added peer, starting replication: peer=server-9afa250d-43a4-1f59-a0e4-486f5c3aad6c +2025-06-03T23:36:38.170Z [INFO] server-75650069-2d47-ea0f-ddee-5e2aca100923: pipelining replication: peer="{Voter server-9afa250d-43a4-1f59-a0e4-486f5c3aad6c 9afa250d-43a4-1f59-a0e4-486f5c3aad6c}" +2025-06-03T23:36:38.170Z [INFO] server-75650069-2d47-ea0f-ddee-5e2aca100923: pipelining replication: peer="{Voter server-83870856-48a9-2eec-6400-7ff0cc634067 83870856-48a9-2eec-6400-7ff0cc634067}" +2025-06-03T23:36:39.175Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:38.170757871 +0000 UTC m=+33.492512801 (1 nodes), 706.29056ms from start of poll, 707.80104ms from cluster start. Timeout at 2025-06-03 23:36:39.175815151 +0000 UTC m=+34.497570081, 1.00505728s after stability +2025-06-03T23:36:39.176Z [INFO] server-75650069-2d47-ea0f-ddee-5e2aca100923: starting snapshot up to: index=12 +2025-06-03T23:36:39.176Z [INFO] creating new snapshot: path=/tmp/raft1424633988/snapshots/2-12-1748993799176.tmp +2025-06-03T23:36:39.177Z [INFO] server-75650069-2d47-ea0f-ddee-5e2aca100923: snapshot complete up to: index=12 +2025-06-03T23:36:39.178Z [INFO] server-35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c Address:35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c} {Suffrage:Voter ID:server-0e60c94e-a569-4657-0b3f-69bd74854c68 Address:0e60c94e-a569-4657-0b3f-69bd74854c68} {Suffrage:Voter ID:server-e14aa510-0041-7ae5-a6b3-3a177757dce0 Address:e14aa510-0041-7ae5-a6b3-3a177757dce0}]" +2025-06-03T23:36:39.178Z [INFO] server-35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c: entering follower state: follower="Node at 35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c [Follower]" leader-address= leader-id= +2025-06-03T23:36:39.178Z [INFO] server-0e60c94e-a569-4657-0b3f-69bd74854c68: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c Address:35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c} {Suffrage:Voter ID:server-0e60c94e-a569-4657-0b3f-69bd74854c68 Address:0e60c94e-a569-4657-0b3f-69bd74854c68} {Suffrage:Voter ID:server-e14aa510-0041-7ae5-a6b3-3a177757dce0 Address:e14aa510-0041-7ae5-a6b3-3a177757dce0}]" +2025-06-03T23:36:39.178Z [INFO] server-0e60c94e-a569-4657-0b3f-69bd74854c68: entering follower state: follower="Node at 0e60c94e-a569-4657-0b3f-69bd74854c68 [Follower]" leader-address= leader-id= +2025-06-03T23:36:39.178Z [INFO] server-e14aa510-0041-7ae5-a6b3-3a177757dce0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c Address:35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c} {Suffrage:Voter ID:server-0e60c94e-a569-4657-0b3f-69bd74854c68 Address:0e60c94e-a569-4657-0b3f-69bd74854c68} {Suffrage:Voter ID:server-e14aa510-0041-7ae5-a6b3-3a177757dce0 Address:e14aa510-0041-7ae5-a6b3-3a177757dce0}]" +2025-06-03T23:36:39.178Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:39.178Z [INFO] server-e14aa510-0041-7ae5-a6b3-3a177757dce0: entering follower state: follower="Node at e14aa510-0041-7ae5-a6b3-3a177757dce0 [Follower]" leader-address= leader-id= +2025-06-03T23:36:39.855Z [WARN] server-0e60c94e-a569-4657-0b3f-69bd74854c68: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:39.855Z [INFO] server-0e60c94e-a569-4657-0b3f-69bd74854c68: entering candidate state: node="Node at 0e60c94e-a569-4657-0b3f-69bd74854c68 [Candidate]" term=2 +2025-06-03T23:36:39.855Z [INFO] server-0e60c94e-a569-4657-0b3f-69bd74854c68: election won: term=2 tally=2 +2025-06-03T23:36:39.855Z [INFO] server-0e60c94e-a569-4657-0b3f-69bd74854c68: entering leader state: leader="Node at 0e60c94e-a569-4657-0b3f-69bd74854c68 [Leader]" +2025-06-03T23:36:39.855Z [INFO] server-0e60c94e-a569-4657-0b3f-69bd74854c68: added peer, starting replication: peer=server-35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c +2025-06-03T23:36:39.855Z [INFO] server-0e60c94e-a569-4657-0b3f-69bd74854c68: added peer, starting replication: peer=server-e14aa510-0041-7ae5-a6b3-3a177757dce0 +2025-06-03T23:36:39.855Z [INFO] server-0e60c94e-a569-4657-0b3f-69bd74854c68: pipelining replication: peer="{Voter server-e14aa510-0041-7ae5-a6b3-3a177757dce0 e14aa510-0041-7ae5-a6b3-3a177757dce0}" +2025-06-03T23:36:39.855Z [INFO] server-0e60c94e-a569-4657-0b3f-69bd74854c68: pipelining replication: peer="{Voter server-35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c 35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c}" +2025-06-03T23:36:40.860Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:39.855388791 +0000 UTC m=+35.177143721 (1 nodes), 676.68552ms from start of poll, 677.19064ms from cluster start. Timeout at 2025-06-03 23:36:40.860601431 +0000 UTC m=+36.182356401, 1.00521268s after stability +2025-06-03T23:36:40.861Z [INFO] creating new snapshot: path=/tmp/raft4234401231/snapshots/2-14-1748993800861.tmp +2025-06-03T23:36:40.861Z [INFO] server-0e60c94e-a569-4657-0b3f-69bd74854c68: copied to local snapshot: bytes=71 +2025-06-03T23:36:40.861Z [INFO] server-0e60c94e-a569-4657-0b3f-69bd74854c68: snapshot restore progress: id=2-14-1748993800861 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:40.861Z [INFO] server-0e60c94e-a569-4657-0b3f-69bd74854c68: restored user snapshot: index=14 +2025-06-03T23:36:40.861Z [ERROR] server-0e60c94e-a569-4657-0b3f-69bd74854c68: failed to get log: index=3 error="log not found" +2025-06-03T23:36:40.861Z [INFO] server-0e60c94e-a569-4657-0b3f-69bd74854c68: aborting pipeline replication: peer="{Voter server-e14aa510-0041-7ae5-a6b3-3a177757dce0 e14aa510-0041-7ae5-a6b3-3a177757dce0}" +2025-06-03T23:36:40.861Z [ERROR] server-0e60c94e-a569-4657-0b3f-69bd74854c68: failed to get log: index=3 error="log not found" +2025-06-03T23:36:40.861Z [INFO] server-0e60c94e-a569-4657-0b3f-69bd74854c68: aborting pipeline replication: peer="{Voter server-35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c 35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c}" +2025-06-03T23:36:40.868Z [ERROR] server-0e60c94e-a569-4657-0b3f-69bd74854c68: failed to get log: index=3 error="log not found" +2025-06-03T23:36:40.869Z [INFO] creating new snapshot: path=/tmp/raft502026517/snapshots/2-14-1748993800869.tmp +2025-06-03T23:36:40.869Z [ERROR] server-0e60c94e-a569-4657-0b3f-69bd74854c68: failed to get log: index=3 error="log not found" +2025-06-03T23:36:40.869Z [INFO] server-e14aa510-0041-7ae5-a6b3-3a177757dce0: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:40.870Z [INFO] server-e14aa510-0041-7ae5-a6b3-3a177757dce0: copied to local snapshot: bytes=71 +2025-06-03T23:36:40.870Z [INFO] creating new snapshot: path=/tmp/raft2251203685/snapshots/2-14-1748993800870.tmp +2025-06-03T23:36:40.870Z [INFO] server-35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:40.870Z [INFO] server-e14aa510-0041-7ae5-a6b3-3a177757dce0: snapshot restore progress: id=2-14-1748993800869 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:40.870Z [INFO] server-e14aa510-0041-7ae5-a6b3-3a177757dce0: Installed remote snapshot +2025-06-03T23:36:40.870Z [INFO] server-0e60c94e-a569-4657-0b3f-69bd74854c68: pipelining replication: peer="{Voter server-e14aa510-0041-7ae5-a6b3-3a177757dce0 e14aa510-0041-7ae5-a6b3-3a177757dce0}" +2025-06-03T23:36:40.870Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:40.870Z [INFO] server-35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c: copied to local snapshot: bytes=71 +2025-06-03T23:36:40.870Z [INFO] server-35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c: snapshot restore progress: id=2-14-1748993800870 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:40.870Z [INFO] server-35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c: Installed remote snapshot +2025-06-03T23:36:40.870Z [INFO] server-0e60c94e-a569-4657-0b3f-69bd74854c68: pipelining replication: peer="{Voter server-35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c 35fbe457-2ee8-6fe0-c8a5-dc3550fa3d3c}" +2025-06-03T23:36:41.875Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:40.870578271 +0000 UTC m=+36.192333201 (2 nodes), 1.48µs from start of poll, 1.69238012s from cluster start. Timeout at 2025-06-03 23:36:41.875716711 +0000 UTC m=+37.197471641, 1.00513844s after stability +2025-06-03T23:36:41.887Z [INFO] server-75650069-2d47-ea0f-ddee-5e2aca100923: aborting pipeline replication: peer="{Voter server-9afa250d-43a4-1f59-a0e4-486f5c3aad6c 9afa250d-43a4-1f59-a0e4-486f5c3aad6c}" +2025-06-03T23:36:41.887Z [INFO] server-75650069-2d47-ea0f-ddee-5e2aca100923: aborting pipeline replication: peer="{Voter server-83870856-48a9-2eec-6400-7ff0cc634067 83870856-48a9-2eec-6400-7ff0cc634067}" === RUN TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true -2025-06-03T23:33:43.184Z [INFO] server-566fb2c0-77b5-d95e-0680-15d2cc3f6967: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-566fb2c0-77b5-d95e-0680-15d2cc3f6967 Address:566fb2c0-77b5-d95e-0680-15d2cc3f6967} {Suffrage:Voter ID:server-f1b58fb6-e55f-0dc3-688f-3eae412cf734 Address:f1b58fb6-e55f-0dc3-688f-3eae412cf734} {Suffrage:Voter ID:server-6c5744e8-3603-b2e4-4572-8933f13dfdb5 Address:6c5744e8-3603-b2e4-4572-8933f13dfdb5}]" -2025-06-03T23:33:43.184Z [INFO] server-566fb2c0-77b5-d95e-0680-15d2cc3f6967: entering follower state: follower="Node at 566fb2c0-77b5-d95e-0680-15d2cc3f6967 [Follower]" leader-address= leader-id= -2025-06-03T23:33:43.184Z [INFO] server-f1b58fb6-e55f-0dc3-688f-3eae412cf734: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-566fb2c0-77b5-d95e-0680-15d2cc3f6967 Address:566fb2c0-77b5-d95e-0680-15d2cc3f6967} {Suffrage:Voter ID:server-f1b58fb6-e55f-0dc3-688f-3eae412cf734 Address:f1b58fb6-e55f-0dc3-688f-3eae412cf734} {Suffrage:Voter ID:server-6c5744e8-3603-b2e4-4572-8933f13dfdb5 Address:6c5744e8-3603-b2e4-4572-8933f13dfdb5}]" -2025-06-03T23:33:43.184Z [INFO] server-f1b58fb6-e55f-0dc3-688f-3eae412cf734: entering follower state: follower="Node at f1b58fb6-e55f-0dc3-688f-3eae412cf734 [Follower]" leader-address= leader-id= -2025-06-03T23:33:43.184Z [INFO] server-6c5744e8-3603-b2e4-4572-8933f13dfdb5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-566fb2c0-77b5-d95e-0680-15d2cc3f6967 Address:566fb2c0-77b5-d95e-0680-15d2cc3f6967} {Suffrage:Voter ID:server-f1b58fb6-e55f-0dc3-688f-3eae412cf734 Address:f1b58fb6-e55f-0dc3-688f-3eae412cf734} {Suffrage:Voter ID:server-6c5744e8-3603-b2e4-4572-8933f13dfdb5 Address:6c5744e8-3603-b2e4-4572-8933f13dfdb5}]" -2025-06-03T23:33:43.184Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:43.184Z [INFO] server-6c5744e8-3603-b2e4-4572-8933f13dfdb5: entering follower state: follower="Node at 6c5744e8-3603-b2e4-4572-8933f13dfdb5 [Follower]" leader-address= leader-id= -2025-06-03T23:33:43.854Z [WARN] server-6c5744e8-3603-b2e4-4572-8933f13dfdb5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:43.854Z [INFO] server-6c5744e8-3603-b2e4-4572-8933f13dfdb5: entering candidate state: node="Node at 6c5744e8-3603-b2e4-4572-8933f13dfdb5 [Candidate]" term=2 -2025-06-03T23:33:43.854Z [INFO] server-6c5744e8-3603-b2e4-4572-8933f13dfdb5: election won: term=2 tally=2 -2025-06-03T23:33:43.854Z [INFO] server-6c5744e8-3603-b2e4-4572-8933f13dfdb5: entering leader state: leader="Node at 6c5744e8-3603-b2e4-4572-8933f13dfdb5 [Leader]" -2025-06-03T23:33:43.854Z [INFO] server-6c5744e8-3603-b2e4-4572-8933f13dfdb5: added peer, starting replication: peer=server-566fb2c0-77b5-d95e-0680-15d2cc3f6967 -2025-06-03T23:33:43.854Z [INFO] server-6c5744e8-3603-b2e4-4572-8933f13dfdb5: added peer, starting replication: peer=server-f1b58fb6-e55f-0dc3-688f-3eae412cf734 -2025-06-03T23:33:43.855Z [INFO] server-6c5744e8-3603-b2e4-4572-8933f13dfdb5: pipelining replication: peer="{Voter server-f1b58fb6-e55f-0dc3-688f-3eae412cf734 f1b58fb6-e55f-0dc3-688f-3eae412cf734}" -2025-06-03T23:33:43.855Z [INFO] server-6c5744e8-3603-b2e4-4572-8933f13dfdb5: pipelining replication: peer="{Voter server-566fb2c0-77b5-d95e-0680-15d2cc3f6967 566fb2c0-77b5-d95e-0680-15d2cc3f6967}" -2025-06-03T23:33:44.861Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:43.854997671 +0000 UTC m=+38.875817041 (1 nodes), 670.2916ms from start of poll, 670.75336ms from cluster start. Timeout at 2025-06-03 23:33:44.861046991 +0000 UTC m=+39.881866361, 1.00604932s after stability -2025-06-03T23:33:44.861Z [INFO] server-6c5744e8-3603-b2e4-4572-8933f13dfdb5: starting snapshot up to: index=12 -2025-06-03T23:33:44.861Z [INFO] creating new snapshot: path=/tmp/raft2030756434/snapshots/2-12-1748993624861.tmp -2025-06-03T23:33:44.862Z [INFO] server-6c5744e8-3603-b2e4-4572-8933f13dfdb5: snapshot complete up to: index=12 -2025-06-03T23:33:44.863Z [INFO] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d Address:dd6a86b7-1e6b-5ab2-42c5-849a8675c71d} {Suffrage:Voter ID:server-e885eb1b-10c3-18fb-0817-6dd7f4cb799d Address:e885eb1b-10c3-18fb-0817-6dd7f4cb799d} {Suffrage:Voter ID:server-46b17517-9d34-8958-94ac-845adc00ae9f Address:46b17517-9d34-8958-94ac-845adc00ae9f}]" -2025-06-03T23:33:44.863Z [INFO] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: entering follower state: follower="Node at dd6a86b7-1e6b-5ab2-42c5-849a8675c71d [Follower]" leader-address= leader-id= -2025-06-03T23:33:44.863Z [INFO] server-e885eb1b-10c3-18fb-0817-6dd7f4cb799d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d Address:dd6a86b7-1e6b-5ab2-42c5-849a8675c71d} {Suffrage:Voter ID:server-e885eb1b-10c3-18fb-0817-6dd7f4cb799d Address:e885eb1b-10c3-18fb-0817-6dd7f4cb799d} {Suffrage:Voter ID:server-46b17517-9d34-8958-94ac-845adc00ae9f Address:46b17517-9d34-8958-94ac-845adc00ae9f}]" -2025-06-03T23:33:44.863Z [INFO] server-e885eb1b-10c3-18fb-0817-6dd7f4cb799d: entering follower state: follower="Node at e885eb1b-10c3-18fb-0817-6dd7f4cb799d [Follower]" leader-address= leader-id= -2025-06-03T23:33:44.863Z [INFO] server-46b17517-9d34-8958-94ac-845adc00ae9f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d Address:dd6a86b7-1e6b-5ab2-42c5-849a8675c71d} {Suffrage:Voter ID:server-e885eb1b-10c3-18fb-0817-6dd7f4cb799d Address:e885eb1b-10c3-18fb-0817-6dd7f4cb799d} {Suffrage:Voter ID:server-46b17517-9d34-8958-94ac-845adc00ae9f Address:46b17517-9d34-8958-94ac-845adc00ae9f}]" -2025-06-03T23:33:44.863Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:44.863Z [INFO] server-46b17517-9d34-8958-94ac-845adc00ae9f: entering follower state: follower="Node at 46b17517-9d34-8958-94ac-845adc00ae9f [Follower]" leader-address= leader-id= -2025-06-03T23:33:45.575Z [WARN] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:45.575Z [INFO] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: entering candidate state: node="Node at dd6a86b7-1e6b-5ab2-42c5-849a8675c71d [Candidate]" term=2 -2025-06-03T23:33:45.575Z [INFO] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: election won: term=2 tally=2 -2025-06-03T23:33:45.575Z [INFO] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: entering leader state: leader="Node at dd6a86b7-1e6b-5ab2-42c5-849a8675c71d [Leader]" -2025-06-03T23:33:45.575Z [INFO] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: added peer, starting replication: peer=server-e885eb1b-10c3-18fb-0817-6dd7f4cb799d -2025-06-03T23:33:45.575Z [INFO] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: added peer, starting replication: peer=server-46b17517-9d34-8958-94ac-845adc00ae9f -2025-06-03T23:33:45.575Z [INFO] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: pipelining replication: peer="{Voter server-e885eb1b-10c3-18fb-0817-6dd7f4cb799d e885eb1b-10c3-18fb-0817-6dd7f4cb799d}" -2025-06-03T23:33:45.575Z [INFO] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: pipelining replication: peer="{Voter server-46b17517-9d34-8958-94ac-845adc00ae9f 46b17517-9d34-8958-94ac-845adc00ae9f}" -2025-06-03T23:33:46.580Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:45.575369591 +0000 UTC m=+40.596188961 (1 nodes), 711.625ms from start of poll, 712.23912ms from cluster start. Timeout at 2025-06-03 23:33:46.580832351 +0000 UTC m=+41.601651721, 1.00546276s after stability -2025-06-03T23:33:46.581Z [INFO] creating new snapshot: path=/tmp/raft3917011904/snapshots/2-14-1748993626581.tmp -2025-06-03T23:33:46.582Z [INFO] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: copied to local snapshot: bytes=71 -2025-06-03T23:33:46.583Z [INFO] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: snapshot restore progress: id=2-14-1748993626581 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:46.583Z [INFO] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: removing all old logs from log store -2025-06-03T23:33:46.583Z [INFO] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: compacting logs: from=1 to=2 -2025-06-03T23:33:46.583Z [INFO] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: restored user snapshot: index=14 -2025-06-03T23:33:46.583Z [ERROR] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: failed to get log: index=2 error="log not found" -2025-06-03T23:33:46.583Z [INFO] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: aborting pipeline replication: peer="{Voter server-46b17517-9d34-8958-94ac-845adc00ae9f 46b17517-9d34-8958-94ac-845adc00ae9f}" -2025-06-03T23:33:46.583Z [ERROR] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: failed to get log: index=2 error="log not found" -2025-06-03T23:33:46.583Z [INFO] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: aborting pipeline replication: peer="{Voter server-e885eb1b-10c3-18fb-0817-6dd7f4cb799d e885eb1b-10c3-18fb-0817-6dd7f4cb799d}" -2025-06-03T23:33:46.590Z [ERROR] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: failed to get log: index=2 error="log not found" -2025-06-03T23:33:46.591Z [INFO] creating new snapshot: path=/tmp/raft1070768575/snapshots/2-14-1748993626591.tmp -2025-06-03T23:33:46.591Z [INFO] server-46b17517-9d34-8958-94ac-845adc00ae9f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:46.591Z [INFO] server-46b17517-9d34-8958-94ac-845adc00ae9f: copied to local snapshot: bytes=71 -2025-06-03T23:33:46.591Z [INFO] server-46b17517-9d34-8958-94ac-845adc00ae9f: snapshot restore progress: id=2-14-1748993626591 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:46.591Z [INFO] server-46b17517-9d34-8958-94ac-845adc00ae9f: removing all old logs from log store -2025-06-03T23:33:46.591Z [INFO] server-46b17517-9d34-8958-94ac-845adc00ae9f: compacting logs: from=1 to=2 -2025-06-03T23:33:46.591Z [INFO] server-46b17517-9d34-8958-94ac-845adc00ae9f: Installed remote snapshot -2025-06-03T23:33:46.592Z [INFO] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: pipelining replication: peer="{Voter server-46b17517-9d34-8958-94ac-845adc00ae9f 46b17517-9d34-8958-94ac-845adc00ae9f}" -2025-06-03T23:33:46.592Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:46.592Z [ERROR] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: failed to get log: index=2 error="log not found" -2025-06-03T23:33:46.593Z [INFO] creating new snapshot: path=/tmp/raft765260384/snapshots/2-14-1748993626593.tmp -2025-06-03T23:33:46.593Z [INFO] server-e885eb1b-10c3-18fb-0817-6dd7f4cb799d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:46.593Z [INFO] server-e885eb1b-10c3-18fb-0817-6dd7f4cb799d: copied to local snapshot: bytes=71 -2025-06-03T23:33:46.594Z [INFO] server-e885eb1b-10c3-18fb-0817-6dd7f4cb799d: snapshot restore progress: id=2-14-1748993626593 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:46.594Z [INFO] server-e885eb1b-10c3-18fb-0817-6dd7f4cb799d: removing all old logs from log store -2025-06-03T23:33:46.594Z [INFO] server-e885eb1b-10c3-18fb-0817-6dd7f4cb799d: compacting logs: from=1 to=2 -2025-06-03T23:33:46.594Z [INFO] server-e885eb1b-10c3-18fb-0817-6dd7f4cb799d: Installed remote snapshot -2025-06-03T23:33:46.594Z [INFO] server-dd6a86b7-1e6b-5ab2-42c5-849a8675c71d: pipelining replication: peer="{Voter server-e885eb1b-10c3-18fb-0817-6dd7f4cb799d e885eb1b-10c3-18fb-0817-6dd7f4cb799d}" -2025-06-03T23:33:47.597Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:46.592180511 +0000 UTC m=+41.612999881 (2 nodes), 1.28µs from start of poll, 1.72905004s from cluster start. Timeout at 2025-06-03 23:33:47.597568231 +0000 UTC m=+42.618387601, 1.00538772s after stability -2025-06-03T23:33:47.609Z [INFO] server-6c5744e8-3603-b2e4-4572-8933f13dfdb5: aborting pipeline replication: peer="{Voter server-f1b58fb6-e55f-0dc3-688f-3eae412cf734 f1b58fb6-e55f-0dc3-688f-3eae412cf734}" -2025-06-03T23:33:47.609Z [INFO] server-6c5744e8-3603-b2e4-4572-8933f13dfdb5: aborting pipeline replication: peer="{Voter server-566fb2c0-77b5-d95e-0680-15d2cc3f6967 566fb2c0-77b5-d95e-0680-15d2cc3f6967}" +2025-06-03T23:36:41.888Z [INFO] server-0280e289-2de6-7aa6-31bc-3c0d3e0486a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0280e289-2de6-7aa6-31bc-3c0d3e0486a1 Address:0280e289-2de6-7aa6-31bc-3c0d3e0486a1} {Suffrage:Voter ID:server-dddfa048-ece4-f5e9-aa37-11f0ec964a26 Address:dddfa048-ece4-f5e9-aa37-11f0ec964a26} {Suffrage:Voter ID:server-1bd8e82a-8037-6303-5254-243a28cbeac2 Address:1bd8e82a-8037-6303-5254-243a28cbeac2}]" +2025-06-03T23:36:41.888Z [INFO] server-0280e289-2de6-7aa6-31bc-3c0d3e0486a1: entering follower state: follower="Node at 0280e289-2de6-7aa6-31bc-3c0d3e0486a1 [Follower]" leader-address= leader-id= +2025-06-03T23:36:41.888Z [INFO] server-dddfa048-ece4-f5e9-aa37-11f0ec964a26: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0280e289-2de6-7aa6-31bc-3c0d3e0486a1 Address:0280e289-2de6-7aa6-31bc-3c0d3e0486a1} {Suffrage:Voter ID:server-dddfa048-ece4-f5e9-aa37-11f0ec964a26 Address:dddfa048-ece4-f5e9-aa37-11f0ec964a26} {Suffrage:Voter ID:server-1bd8e82a-8037-6303-5254-243a28cbeac2 Address:1bd8e82a-8037-6303-5254-243a28cbeac2}]" +2025-06-03T23:36:41.888Z [INFO] server-dddfa048-ece4-f5e9-aa37-11f0ec964a26: entering follower state: follower="Node at dddfa048-ece4-f5e9-aa37-11f0ec964a26 [Follower]" leader-address= leader-id= +2025-06-03T23:36:41.888Z [INFO] server-1bd8e82a-8037-6303-5254-243a28cbeac2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0280e289-2de6-7aa6-31bc-3c0d3e0486a1 Address:0280e289-2de6-7aa6-31bc-3c0d3e0486a1} {Suffrage:Voter ID:server-dddfa048-ece4-f5e9-aa37-11f0ec964a26 Address:dddfa048-ece4-f5e9-aa37-11f0ec964a26} {Suffrage:Voter ID:server-1bd8e82a-8037-6303-5254-243a28cbeac2 Address:1bd8e82a-8037-6303-5254-243a28cbeac2}]" +2025-06-03T23:36:41.888Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:41.889Z [INFO] server-1bd8e82a-8037-6303-5254-243a28cbeac2: entering follower state: follower="Node at 1bd8e82a-8037-6303-5254-243a28cbeac2 [Follower]" leader-address= leader-id= +2025-06-03T23:36:42.426Z [WARN] server-1bd8e82a-8037-6303-5254-243a28cbeac2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:42.426Z [INFO] server-1bd8e82a-8037-6303-5254-243a28cbeac2: entering candidate state: node="Node at 1bd8e82a-8037-6303-5254-243a28cbeac2 [Candidate]" term=2 +2025-06-03T23:36:42.426Z [INFO] server-1bd8e82a-8037-6303-5254-243a28cbeac2: election won: term=2 tally=2 +2025-06-03T23:36:42.426Z [INFO] server-1bd8e82a-8037-6303-5254-243a28cbeac2: entering leader state: leader="Node at 1bd8e82a-8037-6303-5254-243a28cbeac2 [Leader]" +2025-06-03T23:36:42.426Z [INFO] server-1bd8e82a-8037-6303-5254-243a28cbeac2: added peer, starting replication: peer=server-0280e289-2de6-7aa6-31bc-3c0d3e0486a1 +2025-06-03T23:36:42.426Z [INFO] server-1bd8e82a-8037-6303-5254-243a28cbeac2: added peer, starting replication: peer=server-dddfa048-ece4-f5e9-aa37-11f0ec964a26 +2025-06-03T23:36:42.426Z [INFO] server-1bd8e82a-8037-6303-5254-243a28cbeac2: pipelining replication: peer="{Voter server-0280e289-2de6-7aa6-31bc-3c0d3e0486a1 0280e289-2de6-7aa6-31bc-3c0d3e0486a1}" +2025-06-03T23:36:42.426Z [INFO] server-1bd8e82a-8037-6303-5254-243a28cbeac2: pipelining replication: peer="{Voter server-dddfa048-ece4-f5e9-aa37-11f0ec964a26 dddfa048-ece4-f5e9-aa37-11f0ec964a26}" +2025-06-03T23:36:43.432Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:42.426756231 +0000 UTC m=+37.748511161 (1 nodes), 537.74596ms from start of poll, 538.34368ms from cluster start. Timeout at 2025-06-03 23:36:43.432177311 +0000 UTC m=+38.753932241, 1.00542108s after stability +2025-06-03T23:36:43.432Z [INFO] server-1bd8e82a-8037-6303-5254-243a28cbeac2: starting snapshot up to: index=12 +2025-06-03T23:36:43.432Z [INFO] creating new snapshot: path=/tmp/raft371737671/snapshots/2-12-1748993803432.tmp +2025-06-03T23:36:43.433Z [INFO] server-1bd8e82a-8037-6303-5254-243a28cbeac2: snapshot complete up to: index=12 +2025-06-03T23:36:43.434Z [INFO] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87 Address:8cbc9b5a-bc7a-89e1-dbde-3c259d721b87} {Suffrage:Voter ID:server-3dff7c24-8303-bb97-8cb8-cea1739963c5 Address:3dff7c24-8303-bb97-8cb8-cea1739963c5} {Suffrage:Voter ID:server-3922a75c-89d7-4cff-8637-1af8bb6e8972 Address:3922a75c-89d7-4cff-8637-1af8bb6e8972}]" +2025-06-03T23:36:43.434Z [INFO] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: entering follower state: follower="Node at 8cbc9b5a-bc7a-89e1-dbde-3c259d721b87 [Follower]" leader-address= leader-id= +2025-06-03T23:36:43.434Z [INFO] server-3dff7c24-8303-bb97-8cb8-cea1739963c5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87 Address:8cbc9b5a-bc7a-89e1-dbde-3c259d721b87} {Suffrage:Voter ID:server-3dff7c24-8303-bb97-8cb8-cea1739963c5 Address:3dff7c24-8303-bb97-8cb8-cea1739963c5} {Suffrage:Voter ID:server-3922a75c-89d7-4cff-8637-1af8bb6e8972 Address:3922a75c-89d7-4cff-8637-1af8bb6e8972}]" +2025-06-03T23:36:43.434Z [INFO] server-3dff7c24-8303-bb97-8cb8-cea1739963c5: entering follower state: follower="Node at 3dff7c24-8303-bb97-8cb8-cea1739963c5 [Follower]" leader-address= leader-id= +2025-06-03T23:36:43.434Z [INFO] server-3922a75c-89d7-4cff-8637-1af8bb6e8972: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87 Address:8cbc9b5a-bc7a-89e1-dbde-3c259d721b87} {Suffrage:Voter ID:server-3dff7c24-8303-bb97-8cb8-cea1739963c5 Address:3dff7c24-8303-bb97-8cb8-cea1739963c5} {Suffrage:Voter ID:server-3922a75c-89d7-4cff-8637-1af8bb6e8972 Address:3922a75c-89d7-4cff-8637-1af8bb6e8972}]" +2025-06-03T23:36:43.434Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:43.434Z [INFO] server-3922a75c-89d7-4cff-8637-1af8bb6e8972: entering follower state: follower="Node at 3922a75c-89d7-4cff-8637-1af8bb6e8972 [Follower]" leader-address= leader-id= +2025-06-03T23:36:43.956Z [WARN] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:43.956Z [INFO] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: entering candidate state: node="Node at 8cbc9b5a-bc7a-89e1-dbde-3c259d721b87 [Candidate]" term=2 +2025-06-03T23:36:43.956Z [INFO] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: election won: term=2 tally=2 +2025-06-03T23:36:43.956Z [INFO] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: entering leader state: leader="Node at 8cbc9b5a-bc7a-89e1-dbde-3c259d721b87 [Leader]" +2025-06-03T23:36:43.956Z [INFO] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: added peer, starting replication: peer=server-3dff7c24-8303-bb97-8cb8-cea1739963c5 +2025-06-03T23:36:43.956Z [INFO] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: added peer, starting replication: peer=server-3922a75c-89d7-4cff-8637-1af8bb6e8972 +2025-06-03T23:36:43.956Z [INFO] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: pipelining replication: peer="{Voter server-3dff7c24-8303-bb97-8cb8-cea1739963c5 3dff7c24-8303-bb97-8cb8-cea1739963c5}" +2025-06-03T23:36:43.956Z [INFO] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: pipelining replication: peer="{Voter server-3922a75c-89d7-4cff-8637-1af8bb6e8972 3922a75c-89d7-4cff-8637-1af8bb6e8972}" +2025-06-03T23:36:44.961Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:43.956610271 +0000 UTC m=+39.278365201 (1 nodes), 521.67388ms from start of poll, 522.16752ms from cluster start. Timeout at 2025-06-03 23:36:44.961675791 +0000 UTC m=+40.283430721, 1.00506552s after stability +2025-06-03T23:36:44.962Z [INFO] creating new snapshot: path=/tmp/raft2013999241/snapshots/2-14-1748993804962.tmp +2025-06-03T23:36:44.962Z [INFO] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: copied to local snapshot: bytes=71 +2025-06-03T23:36:44.962Z [INFO] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: snapshot restore progress: id=2-14-1748993804962 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:44.962Z [INFO] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: removing all old logs from log store +2025-06-03T23:36:44.962Z [INFO] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: compacting logs: from=1 to=2 +2025-06-03T23:36:44.962Z [INFO] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: restored user snapshot: index=14 +2025-06-03T23:36:44.962Z [ERROR] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: failed to get log: index=2 error="log not found" +2025-06-03T23:36:44.962Z [ERROR] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: failed to get log: index=2 error="log not found" +2025-06-03T23:36:44.962Z [INFO] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: aborting pipeline replication: peer="{Voter server-3922a75c-89d7-4cff-8637-1af8bb6e8972 3922a75c-89d7-4cff-8637-1af8bb6e8972}" +2025-06-03T23:36:44.962Z [INFO] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: aborting pipeline replication: peer="{Voter server-3dff7c24-8303-bb97-8cb8-cea1739963c5 3dff7c24-8303-bb97-8cb8-cea1739963c5}" +2025-06-03T23:36:44.968Z [ERROR] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: failed to get log: index=2 error="log not found" +2025-06-03T23:36:44.969Z [INFO] creating new snapshot: path=/tmp/raft913252032/snapshots/2-14-1748993804969.tmp +2025-06-03T23:36:44.970Z [INFO] server-3922a75c-89d7-4cff-8637-1af8bb6e8972: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:44.970Z [INFO] server-3922a75c-89d7-4cff-8637-1af8bb6e8972: copied to local snapshot: bytes=71 +2025-06-03T23:36:44.970Z [ERROR] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: failed to get log: index=2 error="log not found" +2025-06-03T23:36:44.970Z [INFO] server-3922a75c-89d7-4cff-8637-1af8bb6e8972: snapshot restore progress: id=2-14-1748993804969 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:44.971Z [INFO] server-3922a75c-89d7-4cff-8637-1af8bb6e8972: removing all old logs from log store +2025-06-03T23:36:44.971Z [INFO] server-3922a75c-89d7-4cff-8637-1af8bb6e8972: compacting logs: from=1 to=2 +2025-06-03T23:36:44.971Z [INFO] server-3922a75c-89d7-4cff-8637-1af8bb6e8972: Installed remote snapshot +2025-06-03T23:36:44.971Z [INFO] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: pipelining replication: peer="{Voter server-3922a75c-89d7-4cff-8637-1af8bb6e8972 3922a75c-89d7-4cff-8637-1af8bb6e8972}" +2025-06-03T23:36:44.971Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:44.971Z [INFO] creating new snapshot: path=/tmp/raft3705875152/snapshots/2-14-1748993804971.tmp +2025-06-03T23:36:44.971Z [INFO] server-3dff7c24-8303-bb97-8cb8-cea1739963c5: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:44.971Z [INFO] server-3dff7c24-8303-bb97-8cb8-cea1739963c5: copied to local snapshot: bytes=71 +2025-06-03T23:36:44.972Z [INFO] server-3dff7c24-8303-bb97-8cb8-cea1739963c5: snapshot restore progress: id=2-14-1748993804971 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:44.972Z [INFO] server-3dff7c24-8303-bb97-8cb8-cea1739963c5: removing all old logs from log store +2025-06-03T23:36:44.972Z [INFO] server-3dff7c24-8303-bb97-8cb8-cea1739963c5: compacting logs: from=1 to=2 +2025-06-03T23:36:44.972Z [INFO] server-3dff7c24-8303-bb97-8cb8-cea1739963c5: Installed remote snapshot +2025-06-03T23:36:44.972Z [INFO] server-8cbc9b5a-bc7a-89e1-dbde-3c259d721b87: pipelining replication: peer="{Voter server-3dff7c24-8303-bb97-8cb8-cea1739963c5 3dff7c24-8303-bb97-8cb8-cea1739963c5}" +2025-06-03T23:36:45.976Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:44.971361591 +0000 UTC m=+40.293116521 (2 nodes), 2.04µs from start of poll, 1.53691884s from cluster start. Timeout at 2025-06-03 23:36:45.976892311 +0000 UTC m=+41.298647241, 1.00553072s after stability +2025-06-03T23:36:45.987Z [INFO] server-1bd8e82a-8037-6303-5254-243a28cbeac2: aborting pipeline replication: peer="{Voter server-dddfa048-ece4-f5e9-aa37-11f0ec964a26 dddfa048-ece4-f5e9-aa37-11f0ec964a26}" +2025-06-03T23:36:45.987Z [INFO] server-1bd8e82a-8037-6303-5254-243a28cbeac2: aborting pipeline replication: peer="{Voter server-0280e289-2de6-7aa6-31bc-3c0d3e0486a1 0280e289-2de6-7aa6-31bc-3c0d3e0486a1}" === RUN TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false -2025-06-03T23:33:47.610Z [INFO] server-bf2a0a7c-8a19-bf62-a008-c5a48df1133f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bf2a0a7c-8a19-bf62-a008-c5a48df1133f Address:bf2a0a7c-8a19-bf62-a008-c5a48df1133f} {Suffrage:Voter ID:server-33b051fd-bd77-79ac-7463-e883204a3ca3 Address:33b051fd-bd77-79ac-7463-e883204a3ca3} {Suffrage:Voter ID:server-3f05604a-6bde-0a12-f241-12830bc3975d Address:3f05604a-6bde-0a12-f241-12830bc3975d}]" -2025-06-03T23:33:47.610Z [INFO] server-bf2a0a7c-8a19-bf62-a008-c5a48df1133f: entering follower state: follower="Node at bf2a0a7c-8a19-bf62-a008-c5a48df1133f [Follower]" leader-address= leader-id= -2025-06-03T23:33:47.610Z [INFO] server-33b051fd-bd77-79ac-7463-e883204a3ca3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bf2a0a7c-8a19-bf62-a008-c5a48df1133f Address:bf2a0a7c-8a19-bf62-a008-c5a48df1133f} {Suffrage:Voter ID:server-33b051fd-bd77-79ac-7463-e883204a3ca3 Address:33b051fd-bd77-79ac-7463-e883204a3ca3} {Suffrage:Voter ID:server-3f05604a-6bde-0a12-f241-12830bc3975d Address:3f05604a-6bde-0a12-f241-12830bc3975d}]" -2025-06-03T23:33:47.610Z [INFO] server-33b051fd-bd77-79ac-7463-e883204a3ca3: entering follower state: follower="Node at 33b051fd-bd77-79ac-7463-e883204a3ca3 [Follower]" leader-address= leader-id= -2025-06-03T23:33:47.611Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bf2a0a7c-8a19-bf62-a008-c5a48df1133f Address:bf2a0a7c-8a19-bf62-a008-c5a48df1133f} {Suffrage:Voter ID:server-33b051fd-bd77-79ac-7463-e883204a3ca3 Address:33b051fd-bd77-79ac-7463-e883204a3ca3} {Suffrage:Voter ID:server-3f05604a-6bde-0a12-f241-12830bc3975d Address:3f05604a-6bde-0a12-f241-12830bc3975d}]" -2025-06-03T23:33:47.611Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:47.611Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: entering follower state: follower="Node at 3f05604a-6bde-0a12-f241-12830bc3975d [Follower]" leader-address= leader-id= -2025-06-03T23:33:48.378Z [WARN] server-3f05604a-6bde-0a12-f241-12830bc3975d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:48.378Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: entering candidate state: node="Node at 3f05604a-6bde-0a12-f241-12830bc3975d [Candidate]" term=2 -2025-06-03T23:33:48.378Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: election won: term=2 tally=2 -2025-06-03T23:33:48.378Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: entering leader state: leader="Node at 3f05604a-6bde-0a12-f241-12830bc3975d [Leader]" -2025-06-03T23:33:48.378Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: added peer, starting replication: peer=server-bf2a0a7c-8a19-bf62-a008-c5a48df1133f -2025-06-03T23:33:48.378Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: added peer, starting replication: peer=server-33b051fd-bd77-79ac-7463-e883204a3ca3 -2025-06-03T23:33:48.378Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: pipelining replication: peer="{Voter server-33b051fd-bd77-79ac-7463-e883204a3ca3 33b051fd-bd77-79ac-7463-e883204a3ca3}" -2025-06-03T23:33:48.378Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: pipelining replication: peer="{Voter server-bf2a0a7c-8a19-bf62-a008-c5a48df1133f bf2a0a7c-8a19-bf62-a008-c5a48df1133f}" -2025-06-03T23:33:49.383Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:48.378793191 +0000 UTC m=+43.399612521 (1 nodes), 767.63828ms from start of poll, 768.29652ms from cluster start. Timeout at 2025-06-03 23:33:49.383830991 +0000 UTC m=+44.404650361, 1.00503784s after stability -2025-06-03T23:33:49.384Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: starting snapshot up to: index=12 -2025-06-03T23:33:49.384Z [INFO] creating new snapshot: path=/tmp/raft1104907738/snapshots/2-12-1748993629384.tmp -2025-06-03T23:33:49.385Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: snapshot complete up to: index=12 -2025-06-03T23:33:49.385Z [INFO] creating new snapshot: path=/tmp/raft1104907738/snapshots/2-23-1748993629385.tmp -2025-06-03T23:33:49.386Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: copied to local snapshot: bytes=71 -2025-06-03T23:33:49.386Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: snapshot restore progress: id=2-23-1748993629385 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:49.386Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: restored user snapshot: index=23 -2025-06-03T23:33:49.386Z [ERROR] server-3f05604a-6bde-0a12-f241-12830bc3975d: failed to get log: index=23 error="log not found" -2025-06-03T23:33:49.386Z [ERROR] server-3f05604a-6bde-0a12-f241-12830bc3975d: failed to get log: index=23 error="log not found" -2025-06-03T23:33:49.386Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: aborting pipeline replication: peer="{Voter server-bf2a0a7c-8a19-bf62-a008-c5a48df1133f bf2a0a7c-8a19-bf62-a008-c5a48df1133f}" -2025-06-03T23:33:49.386Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: aborting pipeline replication: peer="{Voter server-33b051fd-bd77-79ac-7463-e883204a3ca3 33b051fd-bd77-79ac-7463-e883204a3ca3}" -2025-06-03T23:33:49.392Z [ERROR] server-3f05604a-6bde-0a12-f241-12830bc3975d: failed to get log: index=23 error="log not found" -2025-06-03T23:33:49.393Z [INFO] creating new snapshot: path=/tmp/raft4238810956/snapshots/2-23-1748993629393.tmp -2025-06-03T23:33:49.393Z [INFO] server-bf2a0a7c-8a19-bf62-a008-c5a48df1133f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:49.393Z [INFO] server-bf2a0a7c-8a19-bf62-a008-c5a48df1133f: copied to local snapshot: bytes=71 -2025-06-03T23:33:49.394Z [INFO] server-bf2a0a7c-8a19-bf62-a008-c5a48df1133f: snapshot restore progress: id=2-23-1748993629393 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:49.394Z [INFO] server-bf2a0a7c-8a19-bf62-a008-c5a48df1133f: Installed remote snapshot -2025-06-03T23:33:49.394Z [WARN] server-bf2a0a7c-8a19-bf62-a008-c5a48df1133f: skipping application of old log: index=22 -2025-06-03T23:33:49.394Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: pipelining replication: peer="{Voter server-bf2a0a7c-8a19-bf62-a008-c5a48df1133f bf2a0a7c-8a19-bf62-a008-c5a48df1133f}" -2025-06-03T23:33:49.394Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:49.396Z [ERROR] server-3f05604a-6bde-0a12-f241-12830bc3975d: failed to get log: index=23 error="log not found" -2025-06-03T23:33:49.396Z [INFO] creating new snapshot: path=/tmp/raft4287158068/snapshots/2-23-1748993629396.tmp -2025-06-03T23:33:49.397Z [INFO] server-33b051fd-bd77-79ac-7463-e883204a3ca3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:49.397Z [INFO] server-33b051fd-bd77-79ac-7463-e883204a3ca3: copied to local snapshot: bytes=71 -2025-06-03T23:33:49.397Z [INFO] server-33b051fd-bd77-79ac-7463-e883204a3ca3: snapshot restore progress: id=2-23-1748993629396 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:49.397Z [INFO] server-33b051fd-bd77-79ac-7463-e883204a3ca3: Installed remote snapshot -2025-06-03T23:33:49.397Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: pipelining replication: peer="{Voter server-33b051fd-bd77-79ac-7463-e883204a3ca3 33b051fd-bd77-79ac-7463-e883204a3ca3}" -2025-06-03T23:33:50.400Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:49.394564391 +0000 UTC m=+44.415383761 (2 nodes), 920ns from start of poll, 1.78406776s from cluster start. Timeout at 2025-06-03 23:33:50.400210791 +0000 UTC m=+45.421030161, 1.0056464s after stability -2025-06-03T23:33:50.412Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: aborting pipeline replication: peer="{Voter server-33b051fd-bd77-79ac-7463-e883204a3ca3 33b051fd-bd77-79ac-7463-e883204a3ca3}" -2025-06-03T23:33:50.412Z [INFO] server-3f05604a-6bde-0a12-f241-12830bc3975d: aborting pipeline replication: peer="{Voter server-bf2a0a7c-8a19-bf62-a008-c5a48df1133f bf2a0a7c-8a19-bf62-a008-c5a48df1133f}" +2025-06-03T23:36:45.989Z [INFO] server-a50a27a4-bd9f-a8d3-9a7b-da7498fef409: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a50a27a4-bd9f-a8d3-9a7b-da7498fef409 Address:a50a27a4-bd9f-a8d3-9a7b-da7498fef409} {Suffrage:Voter ID:server-671239d4-567f-0cd4-339e-541ac4abe0c9 Address:671239d4-567f-0cd4-339e-541ac4abe0c9} {Suffrage:Voter ID:server-c22c5b60-61df-41b1-4100-b881ba44cd98 Address:c22c5b60-61df-41b1-4100-b881ba44cd98}]" +2025-06-03T23:36:45.989Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a50a27a4-bd9f-a8d3-9a7b-da7498fef409 Address:a50a27a4-bd9f-a8d3-9a7b-da7498fef409} {Suffrage:Voter ID:server-671239d4-567f-0cd4-339e-541ac4abe0c9 Address:671239d4-567f-0cd4-339e-541ac4abe0c9} {Suffrage:Voter ID:server-c22c5b60-61df-41b1-4100-b881ba44cd98 Address:c22c5b60-61df-41b1-4100-b881ba44cd98}]" +2025-06-03T23:36:45.989Z [INFO] server-c22c5b60-61df-41b1-4100-b881ba44cd98: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a50a27a4-bd9f-a8d3-9a7b-da7498fef409 Address:a50a27a4-bd9f-a8d3-9a7b-da7498fef409} {Suffrage:Voter ID:server-671239d4-567f-0cd4-339e-541ac4abe0c9 Address:671239d4-567f-0cd4-339e-541ac4abe0c9} {Suffrage:Voter ID:server-c22c5b60-61df-41b1-4100-b881ba44cd98 Address:c22c5b60-61df-41b1-4100-b881ba44cd98}]" +2025-06-03T23:36:45.989Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:45.989Z [INFO] server-a50a27a4-bd9f-a8d3-9a7b-da7498fef409: entering follower state: follower="Node at a50a27a4-bd9f-a8d3-9a7b-da7498fef409 [Follower]" leader-address= leader-id= +2025-06-03T23:36:45.989Z [INFO] server-c22c5b60-61df-41b1-4100-b881ba44cd98: entering follower state: follower="Node at c22c5b60-61df-41b1-4100-b881ba44cd98 [Follower]" leader-address= leader-id= +2025-06-03T23:36:45.989Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: entering follower state: follower="Node at 671239d4-567f-0cd4-339e-541ac4abe0c9 [Follower]" leader-address= leader-id= +2025-06-03T23:36:46.672Z [WARN] server-671239d4-567f-0cd4-339e-541ac4abe0c9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:46.673Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: entering candidate state: node="Node at 671239d4-567f-0cd4-339e-541ac4abe0c9 [Candidate]" term=2 +2025-06-03T23:36:46.673Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: election won: term=2 tally=2 +2025-06-03T23:36:46.673Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: entering leader state: leader="Node at 671239d4-567f-0cd4-339e-541ac4abe0c9 [Leader]" +2025-06-03T23:36:46.673Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: added peer, starting replication: peer=server-a50a27a4-bd9f-a8d3-9a7b-da7498fef409 +2025-06-03T23:36:46.673Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: added peer, starting replication: peer=server-c22c5b60-61df-41b1-4100-b881ba44cd98 +2025-06-03T23:36:46.673Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: pipelining replication: peer="{Voter server-a50a27a4-bd9f-a8d3-9a7b-da7498fef409 a50a27a4-bd9f-a8d3-9a7b-da7498fef409}" +2025-06-03T23:36:46.673Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: pipelining replication: peer="{Voter server-c22c5b60-61df-41b1-4100-b881ba44cd98 c22c5b60-61df-41b1-4100-b881ba44cd98}" +2025-06-03T23:36:47.679Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:46.673633751 +0000 UTC m=+41.995388681 (1 nodes), 683.8292ms from start of poll, 684.8966ms from cluster start. Timeout at 2025-06-03 23:36:47.678952431 +0000 UTC m=+43.000707361, 1.00531868s after stability +2025-06-03T23:36:47.679Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: starting snapshot up to: index=12 +2025-06-03T23:36:47.679Z [INFO] creating new snapshot: path=/tmp/raft832670062/snapshots/2-12-1748993807679.tmp +2025-06-03T23:36:47.680Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: snapshot complete up to: index=12 +2025-06-03T23:36:47.681Z [INFO] creating new snapshot: path=/tmp/raft832670062/snapshots/2-23-1748993807681.tmp +2025-06-03T23:36:47.682Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: copied to local snapshot: bytes=71 +2025-06-03T23:36:47.682Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: snapshot restore progress: id=2-23-1748993807681 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:47.682Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: restored user snapshot: index=23 +2025-06-03T23:36:47.682Z [ERROR] server-671239d4-567f-0cd4-339e-541ac4abe0c9: failed to get log: index=23 error="log not found" +2025-06-03T23:36:47.682Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: aborting pipeline replication: peer="{Voter server-c22c5b60-61df-41b1-4100-b881ba44cd98 c22c5b60-61df-41b1-4100-b881ba44cd98}" +2025-06-03T23:36:47.682Z [ERROR] server-671239d4-567f-0cd4-339e-541ac4abe0c9: failed to get log: index=23 error="log not found" +2025-06-03T23:36:47.683Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: aborting pipeline replication: peer="{Voter server-a50a27a4-bd9f-a8d3-9a7b-da7498fef409 a50a27a4-bd9f-a8d3-9a7b-da7498fef409}" +2025-06-03T23:36:47.690Z [ERROR] server-671239d4-567f-0cd4-339e-541ac4abe0c9: failed to get log: index=23 error="log not found" +2025-06-03T23:36:47.691Z [INFO] creating new snapshot: path=/tmp/raft3658711064/snapshots/2-23-1748993807691.tmp +2025-06-03T23:36:47.691Z [INFO] server-a50a27a4-bd9f-a8d3-9a7b-da7498fef409: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:47.691Z [INFO] server-a50a27a4-bd9f-a8d3-9a7b-da7498fef409: copied to local snapshot: bytes=71 +2025-06-03T23:36:47.691Z [INFO] server-a50a27a4-bd9f-a8d3-9a7b-da7498fef409: snapshot restore progress: id=2-23-1748993807691 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:47.692Z [INFO] server-a50a27a4-bd9f-a8d3-9a7b-da7498fef409: Installed remote snapshot +2025-06-03T23:36:47.692Z [WARN] server-a50a27a4-bd9f-a8d3-9a7b-da7498fef409: skipping application of old log: index=22 +2025-06-03T23:36:47.692Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: pipelining replication: peer="{Voter server-a50a27a4-bd9f-a8d3-9a7b-da7498fef409 a50a27a4-bd9f-a8d3-9a7b-da7498fef409}" +2025-06-03T23:36:47.692Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:47.692Z [ERROR] server-671239d4-567f-0cd4-339e-541ac4abe0c9: failed to get log: index=23 error="log not found" +2025-06-03T23:36:47.693Z [INFO] creating new snapshot: path=/tmp/raft2709077927/snapshots/2-23-1748993807693.tmp +2025-06-03T23:36:47.693Z [INFO] server-c22c5b60-61df-41b1-4100-b881ba44cd98: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:47.693Z [INFO] server-c22c5b60-61df-41b1-4100-b881ba44cd98: copied to local snapshot: bytes=71 +2025-06-03T23:36:47.694Z [INFO] server-c22c5b60-61df-41b1-4100-b881ba44cd98: snapshot restore progress: id=2-23-1748993807693 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:47.694Z [INFO] server-c22c5b60-61df-41b1-4100-b881ba44cd98: Installed remote snapshot +2025-06-03T23:36:47.694Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: pipelining replication: peer="{Voter server-c22c5b60-61df-41b1-4100-b881ba44cd98 c22c5b60-61df-41b1-4100-b881ba44cd98}" +2025-06-03T23:36:48.697Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:47.692223671 +0000 UTC m=+43.013978561 (2 nodes), 920ns from start of poll, 1.70348648s from cluster start. Timeout at 2025-06-03 23:36:48.697292671 +0000 UTC m=+44.019047601, 1.00506904s after stability +2025-06-03T23:36:48.713Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: aborting pipeline replication: peer="{Voter server-c22c5b60-61df-41b1-4100-b881ba44cd98 c22c5b60-61df-41b1-4100-b881ba44cd98}" +2025-06-03T23:36:48.713Z [INFO] server-671239d4-567f-0cd4-339e-541ac4abe0c9: aborting pipeline replication: peer="{Voter server-a50a27a4-bd9f-a8d3-9a7b-da7498fef409 a50a27a4-bd9f-a8d3-9a7b-da7498fef409}" === RUN TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false -2025-06-03T23:33:50.413Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-123171e0-1368-5c12-f3d1-ffa2bcf49189 Address:123171e0-1368-5c12-f3d1-ffa2bcf49189} {Suffrage:Voter ID:server-676bc847-f8b6-48a3-df6b-99f49b1a963a Address:676bc847-f8b6-48a3-df6b-99f49b1a963a} {Suffrage:Voter ID:server-97a25d2d-a1f5-d918-5a2d-eb2899c909ef Address:97a25d2d-a1f5-d918-5a2d-eb2899c909ef}]" -2025-06-03T23:33:50.413Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: entering follower state: follower="Node at 123171e0-1368-5c12-f3d1-ffa2bcf49189 [Follower]" leader-address= leader-id= -2025-06-03T23:33:50.413Z [INFO] server-676bc847-f8b6-48a3-df6b-99f49b1a963a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-123171e0-1368-5c12-f3d1-ffa2bcf49189 Address:123171e0-1368-5c12-f3d1-ffa2bcf49189} {Suffrage:Voter ID:server-676bc847-f8b6-48a3-df6b-99f49b1a963a Address:676bc847-f8b6-48a3-df6b-99f49b1a963a} {Suffrage:Voter ID:server-97a25d2d-a1f5-d918-5a2d-eb2899c909ef Address:97a25d2d-a1f5-d918-5a2d-eb2899c909ef}]" -2025-06-03T23:33:50.413Z [INFO] server-676bc847-f8b6-48a3-df6b-99f49b1a963a: entering follower state: follower="Node at 676bc847-f8b6-48a3-df6b-99f49b1a963a [Follower]" leader-address= leader-id= -2025-06-03T23:33:50.413Z [INFO] server-97a25d2d-a1f5-d918-5a2d-eb2899c909ef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-123171e0-1368-5c12-f3d1-ffa2bcf49189 Address:123171e0-1368-5c12-f3d1-ffa2bcf49189} {Suffrage:Voter ID:server-676bc847-f8b6-48a3-df6b-99f49b1a963a Address:676bc847-f8b6-48a3-df6b-99f49b1a963a} {Suffrage:Voter ID:server-97a25d2d-a1f5-d918-5a2d-eb2899c909ef Address:97a25d2d-a1f5-d918-5a2d-eb2899c909ef}]" -2025-06-03T23:33:50.413Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:50.413Z [INFO] server-97a25d2d-a1f5-d918-5a2d-eb2899c909ef: entering follower state: follower="Node at 97a25d2d-a1f5-d918-5a2d-eb2899c909ef [Follower]" leader-address= leader-id= -2025-06-03T23:33:50.926Z [WARN] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:50.926Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: entering candidate state: node="Node at 123171e0-1368-5c12-f3d1-ffa2bcf49189 [Candidate]" term=2 -2025-06-03T23:33:50.926Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: election won: term=2 tally=2 -2025-06-03T23:33:50.926Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: entering leader state: leader="Node at 123171e0-1368-5c12-f3d1-ffa2bcf49189 [Leader]" -2025-06-03T23:33:50.926Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: added peer, starting replication: peer=server-676bc847-f8b6-48a3-df6b-99f49b1a963a -2025-06-03T23:33:50.926Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: added peer, starting replication: peer=server-97a25d2d-a1f5-d918-5a2d-eb2899c909ef -2025-06-03T23:33:50.926Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: pipelining replication: peer="{Voter server-676bc847-f8b6-48a3-df6b-99f49b1a963a 676bc847-f8b6-48a3-df6b-99f49b1a963a}" -2025-06-03T23:33:50.927Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: pipelining replication: peer="{Voter server-97a25d2d-a1f5-d918-5a2d-eb2899c909ef 97a25d2d-a1f5-d918-5a2d-eb2899c909ef}" -2025-06-03T23:33:51.932Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:50.926767671 +0000 UTC m=+45.947587041 (1 nodes), 513.07064ms from start of poll, 513.56236ms from cluster start. Timeout at 2025-06-03 23:33:51.932223591 +0000 UTC m=+46.953042961, 1.00545592s after stability -2025-06-03T23:33:51.933Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: starting snapshot up to: index=12 -2025-06-03T23:33:51.933Z [INFO] creating new snapshot: path=/tmp/raft3563023616/snapshots/2-12-1748993631933.tmp -2025-06-03T23:33:51.933Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: snapshot complete up to: index=12 -2025-06-03T23:33:51.934Z [INFO] creating new snapshot: path=/tmp/raft3563023616/snapshots/2-23-1748993631934.tmp -2025-06-03T23:33:51.934Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: copied to local snapshot: bytes=71 -2025-06-03T23:33:51.935Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: snapshot restore progress: id=2-23-1748993631934 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:51.935Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: removing all old logs from log store -2025-06-03T23:33:51.935Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: compacting logs: from=1 to=22 -2025-06-03T23:33:51.935Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: restored user snapshot: index=23 -2025-06-03T23:33:51.935Z [ERROR] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: failed to get log: index=22 error="log not found" -2025-06-03T23:33:51.935Z [ERROR] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: failed to get log: index=22 error="log not found" -2025-06-03T23:33:51.935Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: aborting pipeline replication: peer="{Voter server-676bc847-f8b6-48a3-df6b-99f49b1a963a 676bc847-f8b6-48a3-df6b-99f49b1a963a}" -2025-06-03T23:33:51.935Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: aborting pipeline replication: peer="{Voter server-97a25d2d-a1f5-d918-5a2d-eb2899c909ef 97a25d2d-a1f5-d918-5a2d-eb2899c909ef}" -2025-06-03T23:33:51.944Z [ERROR] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: failed to get log: index=22 error="log not found" -2025-06-03T23:33:51.944Z [ERROR] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: failed to get log: index=22 error="log not found" -2025-06-03T23:33:51.944Z [INFO] creating new snapshot: path=/tmp/raft3191318130/snapshots/2-23-1748993631944.tmp -2025-06-03T23:33:51.944Z [INFO] creating new snapshot: path=/tmp/raft3311697438/snapshots/2-23-1748993631944.tmp -2025-06-03T23:33:51.944Z [INFO] server-97a25d2d-a1f5-d918-5a2d-eb2899c909ef: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:51.944Z [INFO] server-676bc847-f8b6-48a3-df6b-99f49b1a963a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:51.945Z [INFO] server-97a25d2d-a1f5-d918-5a2d-eb2899c909ef: copied to local snapshot: bytes=71 -2025-06-03T23:33:51.945Z [INFO] server-676bc847-f8b6-48a3-df6b-99f49b1a963a: copied to local snapshot: bytes=71 -2025-06-03T23:33:51.945Z [INFO] server-676bc847-f8b6-48a3-df6b-99f49b1a963a: snapshot restore progress: id=2-23-1748993631944 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:51.945Z [INFO] server-676bc847-f8b6-48a3-df6b-99f49b1a963a: removing all old logs from log store -2025-06-03T23:33:51.945Z [INFO] server-676bc847-f8b6-48a3-df6b-99f49b1a963a: compacting logs: from=1 to=22 -2025-06-03T23:33:51.945Z [INFO] server-97a25d2d-a1f5-d918-5a2d-eb2899c909ef: snapshot restore progress: id=2-23-1748993631944 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:51.945Z [INFO] server-97a25d2d-a1f5-d918-5a2d-eb2899c909ef: removing all old logs from log store -2025-06-03T23:33:51.945Z [INFO] server-97a25d2d-a1f5-d918-5a2d-eb2899c909ef: compacting logs: from=1 to=22 -2025-06-03T23:33:51.945Z [INFO] server-97a25d2d-a1f5-d918-5a2d-eb2899c909ef: Installed remote snapshot -2025-06-03T23:33:51.945Z [INFO] server-676bc847-f8b6-48a3-df6b-99f49b1a963a: Installed remote snapshot -2025-06-03T23:33:51.945Z [WARN] server-97a25d2d-a1f5-d918-5a2d-eb2899c909ef: skipping application of old log: index=22 -2025-06-03T23:33:51.945Z [WARN] server-676bc847-f8b6-48a3-df6b-99f49b1a963a: skipping application of old log: index=22 -2025-06-03T23:33:51.945Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: pipelining replication: peer="{Voter server-97a25d2d-a1f5-d918-5a2d-eb2899c909ef 97a25d2d-a1f5-d918-5a2d-eb2899c909ef}" -2025-06-03T23:33:51.945Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: pipelining replication: peer="{Voter server-676bc847-f8b6-48a3-df6b-99f49b1a963a 676bc847-f8b6-48a3-df6b-99f49b1a963a}" -2025-06-03T23:33:51.945Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:52.951Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:51.945798271 +0000 UTC m=+46.966617641 (2 nodes), 1µs from start of poll, 1.53259296s from cluster start. Timeout at 2025-06-03 23:33:52.951278791 +0000 UTC m=+47.972098201, 1.00548056s after stability -2025-06-03T23:33:52.968Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: aborting pipeline replication: peer="{Voter server-97a25d2d-a1f5-d918-5a2d-eb2899c909ef 97a25d2d-a1f5-d918-5a2d-eb2899c909ef}" -2025-06-03T23:33:52.968Z [INFO] server-123171e0-1368-5c12-f3d1-ffa2bcf49189: aborting pipeline replication: peer="{Voter server-676bc847-f8b6-48a3-df6b-99f49b1a963a 676bc847-f8b6-48a3-df6b-99f49b1a963a}" +2025-06-03T23:36:48.714Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2945dbdf-688a-2409-3959-52bf99ef6e14 Address:2945dbdf-688a-2409-3959-52bf99ef6e14} {Suffrage:Voter ID:server-6e475b4a-1833-722c-1492-ed99bda17f88 Address:6e475b4a-1833-722c-1492-ed99bda17f88} {Suffrage:Voter ID:server-2da4518a-d1ca-4a8b-de2d-1af716fa0cd0 Address:2da4518a-d1ca-4a8b-de2d-1af716fa0cd0}]" +2025-06-03T23:36:48.714Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: entering follower state: follower="Node at 2945dbdf-688a-2409-3959-52bf99ef6e14 [Follower]" leader-address= leader-id= +2025-06-03T23:36:48.714Z [INFO] server-6e475b4a-1833-722c-1492-ed99bda17f88: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2945dbdf-688a-2409-3959-52bf99ef6e14 Address:2945dbdf-688a-2409-3959-52bf99ef6e14} {Suffrage:Voter ID:server-6e475b4a-1833-722c-1492-ed99bda17f88 Address:6e475b4a-1833-722c-1492-ed99bda17f88} {Suffrage:Voter ID:server-2da4518a-d1ca-4a8b-de2d-1af716fa0cd0 Address:2da4518a-d1ca-4a8b-de2d-1af716fa0cd0}]" +2025-06-03T23:36:48.714Z [INFO] server-6e475b4a-1833-722c-1492-ed99bda17f88: entering follower state: follower="Node at 6e475b4a-1833-722c-1492-ed99bda17f88 [Follower]" leader-address= leader-id= +2025-06-03T23:36:48.714Z [INFO] server-2da4518a-d1ca-4a8b-de2d-1af716fa0cd0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2945dbdf-688a-2409-3959-52bf99ef6e14 Address:2945dbdf-688a-2409-3959-52bf99ef6e14} {Suffrage:Voter ID:server-6e475b4a-1833-722c-1492-ed99bda17f88 Address:6e475b4a-1833-722c-1492-ed99bda17f88} {Suffrage:Voter ID:server-2da4518a-d1ca-4a8b-de2d-1af716fa0cd0 Address:2da4518a-d1ca-4a8b-de2d-1af716fa0cd0}]" +2025-06-03T23:36:48.714Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:48.714Z [INFO] server-2da4518a-d1ca-4a8b-de2d-1af716fa0cd0: entering follower state: follower="Node at 2da4518a-d1ca-4a8b-de2d-1af716fa0cd0 [Follower]" leader-address= leader-id= +2025-06-03T23:36:49.353Z [WARN] server-2945dbdf-688a-2409-3959-52bf99ef6e14: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:49.353Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: entering candidate state: node="Node at 2945dbdf-688a-2409-3959-52bf99ef6e14 [Candidate]" term=2 +2025-06-03T23:36:49.353Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: election won: term=2 tally=2 +2025-06-03T23:36:49.353Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: entering leader state: leader="Node at 2945dbdf-688a-2409-3959-52bf99ef6e14 [Leader]" +2025-06-03T23:36:49.353Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: added peer, starting replication: peer=server-6e475b4a-1833-722c-1492-ed99bda17f88 +2025-06-03T23:36:49.353Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: added peer, starting replication: peer=server-2da4518a-d1ca-4a8b-de2d-1af716fa0cd0 +2025-06-03T23:36:49.353Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: pipelining replication: peer="{Voter server-6e475b4a-1833-722c-1492-ed99bda17f88 6e475b4a-1833-722c-1492-ed99bda17f88}" +2025-06-03T23:36:49.353Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: pipelining replication: peer="{Voter server-2da4518a-d1ca-4a8b-de2d-1af716fa0cd0 2da4518a-d1ca-4a8b-de2d-1af716fa0cd0}" +2025-06-03T23:36:50.359Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:49.353376871 +0000 UTC m=+44.675131841 (1 nodes), 638.40108ms from start of poll, 638.88932ms from cluster start. Timeout at 2025-06-03 23:36:50.359362711 +0000 UTC m=+45.681117641, 1.0059858s after stability +2025-06-03T23:36:50.360Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: starting snapshot up to: index=12 +2025-06-03T23:36:50.360Z [INFO] creating new snapshot: path=/tmp/raft1530460419/snapshots/2-12-1748993810360.tmp +2025-06-03T23:36:50.360Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: snapshot complete up to: index=12 +2025-06-03T23:36:50.361Z [INFO] creating new snapshot: path=/tmp/raft1530460419/snapshots/2-23-1748993810361.tmp +2025-06-03T23:36:50.362Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: copied to local snapshot: bytes=71 +2025-06-03T23:36:50.362Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: snapshot restore progress: id=2-23-1748993810361 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:50.362Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: removing all old logs from log store +2025-06-03T23:36:50.362Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: compacting logs: from=1 to=22 +2025-06-03T23:36:50.362Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: restored user snapshot: index=23 +2025-06-03T23:36:50.362Z [ERROR] server-2945dbdf-688a-2409-3959-52bf99ef6e14: failed to get log: index=22 error="log not found" +2025-06-03T23:36:50.362Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: aborting pipeline replication: peer="{Voter server-2da4518a-d1ca-4a8b-de2d-1af716fa0cd0 2da4518a-d1ca-4a8b-de2d-1af716fa0cd0}" +2025-06-03T23:36:50.362Z [ERROR] server-2945dbdf-688a-2409-3959-52bf99ef6e14: failed to get log: index=22 error="log not found" +2025-06-03T23:36:50.362Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: aborting pipeline replication: peer="{Voter server-6e475b4a-1833-722c-1492-ed99bda17f88 6e475b4a-1833-722c-1492-ed99bda17f88}" +2025-06-03T23:36:50.370Z [ERROR] server-2945dbdf-688a-2409-3959-52bf99ef6e14: failed to get log: index=22 error="log not found" +2025-06-03T23:36:50.371Z [INFO] creating new snapshot: path=/tmp/raft3858774805/snapshots/2-23-1748993810371.tmp +2025-06-03T23:36:50.371Z [INFO] server-6e475b4a-1833-722c-1492-ed99bda17f88: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:50.371Z [INFO] server-6e475b4a-1833-722c-1492-ed99bda17f88: copied to local snapshot: bytes=71 +2025-06-03T23:36:50.372Z [ERROR] server-2945dbdf-688a-2409-3959-52bf99ef6e14: failed to get log: index=22 error="log not found" +2025-06-03T23:36:50.372Z [INFO] server-6e475b4a-1833-722c-1492-ed99bda17f88: snapshot restore progress: id=2-23-1748993810371 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:50.372Z [INFO] server-6e475b4a-1833-722c-1492-ed99bda17f88: removing all old logs from log store +2025-06-03T23:36:50.372Z [INFO] server-6e475b4a-1833-722c-1492-ed99bda17f88: compacting logs: from=1 to=22 +2025-06-03T23:36:50.372Z [INFO] server-6e475b4a-1833-722c-1492-ed99bda17f88: Installed remote snapshot +2025-06-03T23:36:50.372Z [WARN] server-6e475b4a-1833-722c-1492-ed99bda17f88: skipping application of old log: index=22 +2025-06-03T23:36:50.372Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: pipelining replication: peer="{Voter server-6e475b4a-1833-722c-1492-ed99bda17f88 6e475b4a-1833-722c-1492-ed99bda17f88}" +2025-06-03T23:36:50.372Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:50.372Z [INFO] creating new snapshot: path=/tmp/raft1672546064/snapshots/2-23-1748993810372.tmp +2025-06-03T23:36:50.372Z [INFO] server-2da4518a-d1ca-4a8b-de2d-1af716fa0cd0: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:50.373Z [INFO] server-2da4518a-d1ca-4a8b-de2d-1af716fa0cd0: copied to local snapshot: bytes=71 +2025-06-03T23:36:50.373Z [INFO] server-2da4518a-d1ca-4a8b-de2d-1af716fa0cd0: snapshot restore progress: id=2-23-1748993810372 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:50.373Z [INFO] server-2da4518a-d1ca-4a8b-de2d-1af716fa0cd0: removing all old logs from log store +2025-06-03T23:36:50.373Z [INFO] server-2da4518a-d1ca-4a8b-de2d-1af716fa0cd0: compacting logs: from=1 to=22 +2025-06-03T23:36:50.373Z [INFO] server-2da4518a-d1ca-4a8b-de2d-1af716fa0cd0: Installed remote snapshot +2025-06-03T23:36:50.373Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: pipelining replication: peer="{Voter server-2da4518a-d1ca-4a8b-de2d-1af716fa0cd0 2da4518a-d1ca-4a8b-de2d-1af716fa0cd0}" +2025-06-03T23:36:51.377Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:50.372441111 +0000 UTC m=+45.694196041 (2 nodes), 1.24µs from start of poll, 1.65795352s from cluster start. Timeout at 2025-06-03 23:36:51.377587591 +0000 UTC m=+46.699342521, 1.00514648s after stability +2025-06-03T23:36:51.390Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: aborting pipeline replication: peer="{Voter server-2da4518a-d1ca-4a8b-de2d-1af716fa0cd0 2da4518a-d1ca-4a8b-de2d-1af716fa0cd0}" +2025-06-03T23:36:51.390Z [INFO] server-2945dbdf-688a-2409-3959-52bf99ef6e14: aborting pipeline replication: peer="{Voter server-6e475b4a-1833-722c-1492-ed99bda17f88 6e475b4a-1833-722c-1492-ed99bda17f88}" === RUN TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true -2025-06-03T23:33:52.970Z [INFO] server-b6efdf20-f439-0d05-30ea-4b5885c7787f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6efdf20-f439-0d05-30ea-4b5885c7787f Address:b6efdf20-f439-0d05-30ea-4b5885c7787f} {Suffrage:Voter ID:server-3bf24e5d-8e73-4af7-380a-dfd94f54d1fd Address:3bf24e5d-8e73-4af7-380a-dfd94f54d1fd} {Suffrage:Voter ID:server-1685d8b3-cbfd-c81a-5fa5-64f2766c8bfb Address:1685d8b3-cbfd-c81a-5fa5-64f2766c8bfb}]" -2025-06-03T23:33:52.970Z [INFO] server-b6efdf20-f439-0d05-30ea-4b5885c7787f: entering follower state: follower="Node at b6efdf20-f439-0d05-30ea-4b5885c7787f [Follower]" leader-address= leader-id= -2025-06-03T23:33:52.970Z [INFO] server-3bf24e5d-8e73-4af7-380a-dfd94f54d1fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6efdf20-f439-0d05-30ea-4b5885c7787f Address:b6efdf20-f439-0d05-30ea-4b5885c7787f} {Suffrage:Voter ID:server-3bf24e5d-8e73-4af7-380a-dfd94f54d1fd Address:3bf24e5d-8e73-4af7-380a-dfd94f54d1fd} {Suffrage:Voter ID:server-1685d8b3-cbfd-c81a-5fa5-64f2766c8bfb Address:1685d8b3-cbfd-c81a-5fa5-64f2766c8bfb}]" -2025-06-03T23:33:52.970Z [INFO] server-3bf24e5d-8e73-4af7-380a-dfd94f54d1fd: entering follower state: follower="Node at 3bf24e5d-8e73-4af7-380a-dfd94f54d1fd [Follower]" leader-address= leader-id= -2025-06-03T23:33:52.970Z [INFO] server-1685d8b3-cbfd-c81a-5fa5-64f2766c8bfb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6efdf20-f439-0d05-30ea-4b5885c7787f Address:b6efdf20-f439-0d05-30ea-4b5885c7787f} {Suffrage:Voter ID:server-3bf24e5d-8e73-4af7-380a-dfd94f54d1fd Address:3bf24e5d-8e73-4af7-380a-dfd94f54d1fd} {Suffrage:Voter ID:server-1685d8b3-cbfd-c81a-5fa5-64f2766c8bfb Address:1685d8b3-cbfd-c81a-5fa5-64f2766c8bfb}]" -2025-06-03T23:33:52.970Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:52.970Z [INFO] server-1685d8b3-cbfd-c81a-5fa5-64f2766c8bfb: entering follower state: follower="Node at 1685d8b3-cbfd-c81a-5fa5-64f2766c8bfb [Follower]" leader-address= leader-id= -2025-06-03T23:33:53.515Z [WARN] server-3bf24e5d-8e73-4af7-380a-dfd94f54d1fd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:53.515Z [INFO] server-3bf24e5d-8e73-4af7-380a-dfd94f54d1fd: entering candidate state: node="Node at 3bf24e5d-8e73-4af7-380a-dfd94f54d1fd [Candidate]" term=2 -2025-06-03T23:33:53.515Z [INFO] server-3bf24e5d-8e73-4af7-380a-dfd94f54d1fd: election won: term=2 tally=2 -2025-06-03T23:33:53.515Z [INFO] server-3bf24e5d-8e73-4af7-380a-dfd94f54d1fd: entering leader state: leader="Node at 3bf24e5d-8e73-4af7-380a-dfd94f54d1fd [Leader]" -2025-06-03T23:33:53.515Z [INFO] server-3bf24e5d-8e73-4af7-380a-dfd94f54d1fd: added peer, starting replication: peer=server-b6efdf20-f439-0d05-30ea-4b5885c7787f -2025-06-03T23:33:53.515Z [INFO] server-3bf24e5d-8e73-4af7-380a-dfd94f54d1fd: added peer, starting replication: peer=server-1685d8b3-cbfd-c81a-5fa5-64f2766c8bfb -2025-06-03T23:33:53.516Z [INFO] server-3bf24e5d-8e73-4af7-380a-dfd94f54d1fd: pipelining replication: peer="{Voter server-b6efdf20-f439-0d05-30ea-4b5885c7787f b6efdf20-f439-0d05-30ea-4b5885c7787f}" -2025-06-03T23:33:53.516Z [INFO] server-3bf24e5d-8e73-4af7-380a-dfd94f54d1fd: pipelining replication: peer="{Voter server-1685d8b3-cbfd-c81a-5fa5-64f2766c8bfb 1685d8b3-cbfd-c81a-5fa5-64f2766c8bfb}" -2025-06-03T23:33:54.522Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:53.515935991 +0000 UTC m=+48.536755361 (1 nodes), 545.38304ms from start of poll, 546.05116ms from cluster start. Timeout at 2025-06-03 23:33:54.522103831 +0000 UTC m=+49.542923201, 1.00616784s after stability -2025-06-03T23:33:54.523Z [INFO] server-3bf24e5d-8e73-4af7-380a-dfd94f54d1fd: starting snapshot up to: index=12 -2025-06-03T23:33:54.523Z [INFO] creating new snapshot: path=/tmp/raft1733890824/snapshots/2-12-1748993634523.tmp -2025-06-03T23:33:54.523Z [INFO] server-3bf24e5d-8e73-4af7-380a-dfd94f54d1fd: snapshot complete up to: index=12 -2025-06-03T23:33:54.524Z [INFO] server-431dcd0d-04e5-a21e-e716-e111c44813fc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-431dcd0d-04e5-a21e-e716-e111c44813fc Address:431dcd0d-04e5-a21e-e716-e111c44813fc} {Suffrage:Voter ID:server-1ac1f355-f903-6831-991e-0cef60e6215e Address:1ac1f355-f903-6831-991e-0cef60e6215e} {Suffrage:Voter ID:server-21c419c0-386b-9db6-aa97-9aca55db71bc Address:21c419c0-386b-9db6-aa97-9aca55db71bc}]" -2025-06-03T23:33:54.524Z [INFO] server-1ac1f355-f903-6831-991e-0cef60e6215e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-431dcd0d-04e5-a21e-e716-e111c44813fc Address:431dcd0d-04e5-a21e-e716-e111c44813fc} {Suffrage:Voter ID:server-1ac1f355-f903-6831-991e-0cef60e6215e Address:1ac1f355-f903-6831-991e-0cef60e6215e} {Suffrage:Voter ID:server-21c419c0-386b-9db6-aa97-9aca55db71bc Address:21c419c0-386b-9db6-aa97-9aca55db71bc}]" -2025-06-03T23:33:54.524Z [INFO] server-21c419c0-386b-9db6-aa97-9aca55db71bc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-431dcd0d-04e5-a21e-e716-e111c44813fc Address:431dcd0d-04e5-a21e-e716-e111c44813fc} {Suffrage:Voter ID:server-1ac1f355-f903-6831-991e-0cef60e6215e Address:1ac1f355-f903-6831-991e-0cef60e6215e} {Suffrage:Voter ID:server-21c419c0-386b-9db6-aa97-9aca55db71bc Address:21c419c0-386b-9db6-aa97-9aca55db71bc}]" -2025-06-03T23:33:54.525Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:54.525Z [INFO] server-431dcd0d-04e5-a21e-e716-e111c44813fc: entering follower state: follower="Node at 431dcd0d-04e5-a21e-e716-e111c44813fc [Follower]" leader-address= leader-id= -2025-06-03T23:33:54.525Z [INFO] server-1ac1f355-f903-6831-991e-0cef60e6215e: entering follower state: follower="Node at 1ac1f355-f903-6831-991e-0cef60e6215e [Follower]" leader-address= leader-id= -2025-06-03T23:33:54.525Z [INFO] server-21c419c0-386b-9db6-aa97-9aca55db71bc: entering follower state: follower="Node at 21c419c0-386b-9db6-aa97-9aca55db71bc [Follower]" leader-address= leader-id= -2025-06-03T23:33:55.179Z [WARN] server-1ac1f355-f903-6831-991e-0cef60e6215e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:55.179Z [INFO] server-1ac1f355-f903-6831-991e-0cef60e6215e: entering candidate state: node="Node at 1ac1f355-f903-6831-991e-0cef60e6215e [Candidate]" term=2 -2025-06-03T23:33:55.179Z [INFO] server-1ac1f355-f903-6831-991e-0cef60e6215e: election won: term=2 tally=2 -2025-06-03T23:33:55.179Z [INFO] server-1ac1f355-f903-6831-991e-0cef60e6215e: entering leader state: leader="Node at 1ac1f355-f903-6831-991e-0cef60e6215e [Leader]" -2025-06-03T23:33:55.179Z [INFO] server-1ac1f355-f903-6831-991e-0cef60e6215e: added peer, starting replication: peer=server-431dcd0d-04e5-a21e-e716-e111c44813fc -2025-06-03T23:33:55.179Z [INFO] server-1ac1f355-f903-6831-991e-0cef60e6215e: added peer, starting replication: peer=server-21c419c0-386b-9db6-aa97-9aca55db71bc -2025-06-03T23:33:55.179Z [INFO] server-1ac1f355-f903-6831-991e-0cef60e6215e: pipelining replication: peer="{Voter server-21c419c0-386b-9db6-aa97-9aca55db71bc 21c419c0-386b-9db6-aa97-9aca55db71bc}" -2025-06-03T23:33:55.179Z [INFO] server-1ac1f355-f903-6831-991e-0cef60e6215e: pipelining replication: peer="{Voter server-431dcd0d-04e5-a21e-e716-e111c44813fc 431dcd0d-04e5-a21e-e716-e111c44813fc}" -2025-06-03T23:33:56.184Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:55.179613031 +0000 UTC m=+50.200432401 (1 nodes), 654.58488ms from start of poll, 655.0306ms from cluster start. Timeout at 2025-06-03 23:33:56.184692511 +0000 UTC m=+51.205511881, 1.00507948s after stability -2025-06-03T23:33:56.185Z [INFO] creating new snapshot: path=/tmp/raft3772950296/snapshots/2-15-1748993636185.tmp -2025-06-03T23:33:56.185Z [INFO] server-1ac1f355-f903-6831-991e-0cef60e6215e: copied to local snapshot: bytes=71 -2025-06-03T23:33:56.185Z [INFO] server-1ac1f355-f903-6831-991e-0cef60e6215e: snapshot restore progress: id=2-15-1748993636185 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:56.185Z [INFO] server-1ac1f355-f903-6831-991e-0cef60e6215e: restored user snapshot: index=15 -2025-06-03T23:33:56.185Z [ERROR] server-1ac1f355-f903-6831-991e-0cef60e6215e: failed to get log: index=3 error="log not found" -2025-06-03T23:33:56.185Z [ERROR] server-1ac1f355-f903-6831-991e-0cef60e6215e: failed to get log: index=3 error="log not found" -2025-06-03T23:33:56.186Z [INFO] server-1ac1f355-f903-6831-991e-0cef60e6215e: aborting pipeline replication: peer="{Voter server-431dcd0d-04e5-a21e-e716-e111c44813fc 431dcd0d-04e5-a21e-e716-e111c44813fc}" -2025-06-03T23:33:56.185Z [INFO] server-1ac1f355-f903-6831-991e-0cef60e6215e: aborting pipeline replication: peer="{Voter server-21c419c0-386b-9db6-aa97-9aca55db71bc 21c419c0-386b-9db6-aa97-9aca55db71bc}" -2025-06-03T23:33:56.191Z [ERROR] server-1ac1f355-f903-6831-991e-0cef60e6215e: failed to get log: index=3 error="log not found" -2025-06-03T23:33:56.192Z [INFO] creating new snapshot: path=/tmp/raft4094490693/snapshots/2-15-1748993636192.tmp -2025-06-03T23:33:56.192Z [INFO] server-431dcd0d-04e5-a21e-e716-e111c44813fc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:56.192Z [INFO] server-431dcd0d-04e5-a21e-e716-e111c44813fc: copied to local snapshot: bytes=71 -2025-06-03T23:33:56.193Z [INFO] server-431dcd0d-04e5-a21e-e716-e111c44813fc: snapshot restore progress: id=2-15-1748993636192 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:56.193Z [INFO] server-431dcd0d-04e5-a21e-e716-e111c44813fc: Installed remote snapshot -2025-06-03T23:33:56.193Z [INFO] server-1ac1f355-f903-6831-991e-0cef60e6215e: pipelining replication: peer="{Voter server-431dcd0d-04e5-a21e-e716-e111c44813fc 431dcd0d-04e5-a21e-e716-e111c44813fc}" -2025-06-03T23:33:56.193Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:33:56.196Z [ERROR] server-1ac1f355-f903-6831-991e-0cef60e6215e: failed to get log: index=3 error="log not found" -2025-06-03T23:33:56.197Z [INFO] creating new snapshot: path=/tmp/raft1129856935/snapshots/2-15-1748993636196.tmp -2025-06-03T23:33:56.197Z [INFO] server-21c419c0-386b-9db6-aa97-9aca55db71bc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:56.197Z [INFO] server-21c419c0-386b-9db6-aa97-9aca55db71bc: copied to local snapshot: bytes=71 -2025-06-03T23:33:56.197Z [INFO] server-21c419c0-386b-9db6-aa97-9aca55db71bc: snapshot restore progress: id=2-15-1748993636196 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:33:56.197Z [INFO] server-21c419c0-386b-9db6-aa97-9aca55db71bc: Installed remote snapshot -2025-06-03T23:33:56.197Z [INFO] server-1ac1f355-f903-6831-991e-0cef60e6215e: pipelining replication: peer="{Voter server-21c419c0-386b-9db6-aa97-9aca55db71bc 21c419c0-386b-9db6-aa97-9aca55db71bc}" -2025-06-03T23:33:57.199Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:33:56.193513711 +0000 UTC m=+51.214333041 (2 nodes), 1µs from start of poll, 1.66893124s from cluster start. Timeout at 2025-06-03 23:33:57.199066591 +0000 UTC m=+52.219885961, 1.00555292s after stability -2025-06-03T23:33:57.210Z [INFO] server-3bf24e5d-8e73-4af7-380a-dfd94f54d1fd: aborting pipeline replication: peer="{Voter server-1685d8b3-cbfd-c81a-5fa5-64f2766c8bfb 1685d8b3-cbfd-c81a-5fa5-64f2766c8bfb}" -2025-06-03T23:33:57.210Z [INFO] server-3bf24e5d-8e73-4af7-380a-dfd94f54d1fd: aborting pipeline replication: peer="{Voter server-b6efdf20-f439-0d05-30ea-4b5885c7787f b6efdf20-f439-0d05-30ea-4b5885c7787f}" +2025-06-03T23:36:51.391Z [INFO] server-da9dd419-58d3-9030-dcde-7db9cb73cd82: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-da9dd419-58d3-9030-dcde-7db9cb73cd82 Address:da9dd419-58d3-9030-dcde-7db9cb73cd82} {Suffrage:Voter ID:server-fbbca7c8-ac6a-bcd1-dfba-2e3593376a63 Address:fbbca7c8-ac6a-bcd1-dfba-2e3593376a63} {Suffrage:Voter ID:server-ebd13c86-264c-9f48-c9ce-8e8451abe3dc Address:ebd13c86-264c-9f48-c9ce-8e8451abe3dc}]" +2025-06-03T23:36:51.391Z [INFO] server-da9dd419-58d3-9030-dcde-7db9cb73cd82: entering follower state: follower="Node at da9dd419-58d3-9030-dcde-7db9cb73cd82 [Follower]" leader-address= leader-id= +2025-06-03T23:36:51.391Z [INFO] server-fbbca7c8-ac6a-bcd1-dfba-2e3593376a63: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-da9dd419-58d3-9030-dcde-7db9cb73cd82 Address:da9dd419-58d3-9030-dcde-7db9cb73cd82} {Suffrage:Voter ID:server-fbbca7c8-ac6a-bcd1-dfba-2e3593376a63 Address:fbbca7c8-ac6a-bcd1-dfba-2e3593376a63} {Suffrage:Voter ID:server-ebd13c86-264c-9f48-c9ce-8e8451abe3dc Address:ebd13c86-264c-9f48-c9ce-8e8451abe3dc}]" +2025-06-03T23:36:51.391Z [INFO] server-fbbca7c8-ac6a-bcd1-dfba-2e3593376a63: entering follower state: follower="Node at fbbca7c8-ac6a-bcd1-dfba-2e3593376a63 [Follower]" leader-address= leader-id= +2025-06-03T23:36:51.391Z [INFO] server-ebd13c86-264c-9f48-c9ce-8e8451abe3dc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-da9dd419-58d3-9030-dcde-7db9cb73cd82 Address:da9dd419-58d3-9030-dcde-7db9cb73cd82} {Suffrage:Voter ID:server-fbbca7c8-ac6a-bcd1-dfba-2e3593376a63 Address:fbbca7c8-ac6a-bcd1-dfba-2e3593376a63} {Suffrage:Voter ID:server-ebd13c86-264c-9f48-c9ce-8e8451abe3dc Address:ebd13c86-264c-9f48-c9ce-8e8451abe3dc}]" +2025-06-03T23:36:51.391Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:51.391Z [INFO] server-ebd13c86-264c-9f48-c9ce-8e8451abe3dc: entering follower state: follower="Node at ebd13c86-264c-9f48-c9ce-8e8451abe3dc [Follower]" leader-address= leader-id= +2025-06-03T23:36:52.061Z [WARN] server-fbbca7c8-ac6a-bcd1-dfba-2e3593376a63: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:52.061Z [INFO] server-fbbca7c8-ac6a-bcd1-dfba-2e3593376a63: entering candidate state: node="Node at fbbca7c8-ac6a-bcd1-dfba-2e3593376a63 [Candidate]" term=2 +2025-06-03T23:36:52.061Z [INFO] server-fbbca7c8-ac6a-bcd1-dfba-2e3593376a63: election won: term=2 tally=2 +2025-06-03T23:36:52.061Z [INFO] server-fbbca7c8-ac6a-bcd1-dfba-2e3593376a63: entering leader state: leader="Node at fbbca7c8-ac6a-bcd1-dfba-2e3593376a63 [Leader]" +2025-06-03T23:36:52.061Z [INFO] server-fbbca7c8-ac6a-bcd1-dfba-2e3593376a63: added peer, starting replication: peer=server-da9dd419-58d3-9030-dcde-7db9cb73cd82 +2025-06-03T23:36:52.061Z [INFO] server-fbbca7c8-ac6a-bcd1-dfba-2e3593376a63: added peer, starting replication: peer=server-ebd13c86-264c-9f48-c9ce-8e8451abe3dc +2025-06-03T23:36:52.061Z [INFO] server-fbbca7c8-ac6a-bcd1-dfba-2e3593376a63: pipelining replication: peer="{Voter server-ebd13c86-264c-9f48-c9ce-8e8451abe3dc ebd13c86-264c-9f48-c9ce-8e8451abe3dc}" +2025-06-03T23:36:52.061Z [INFO] server-fbbca7c8-ac6a-bcd1-dfba-2e3593376a63: pipelining replication: peer="{Voter server-da9dd419-58d3-9030-dcde-7db9cb73cd82 da9dd419-58d3-9030-dcde-7db9cb73cd82}" +2025-06-03T23:36:53.066Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:52.061823711 +0000 UTC m=+47.383578641 (1 nodes), 670.05576ms from start of poll, 670.53744ms from cluster start. Timeout at 2025-06-03 23:36:53.066930471 +0000 UTC m=+48.388685361, 1.00510672s after stability +2025-06-03T23:36:53.067Z [INFO] server-fbbca7c8-ac6a-bcd1-dfba-2e3593376a63: starting snapshot up to: index=12 +2025-06-03T23:36:53.067Z [INFO] creating new snapshot: path=/tmp/raft896882972/snapshots/2-12-1748993813067.tmp +2025-06-03T23:36:53.068Z [INFO] server-fbbca7c8-ac6a-bcd1-dfba-2e3593376a63: snapshot complete up to: index=12 +2025-06-03T23:36:53.069Z [INFO] server-582287de-5ec5-5c65-c12f-c5be37fa7781: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-582287de-5ec5-5c65-c12f-c5be37fa7781 Address:582287de-5ec5-5c65-c12f-c5be37fa7781} {Suffrage:Voter ID:server-f49349f9-ceb6-f2d1-6e57-17151eb4b3cc Address:f49349f9-ceb6-f2d1-6e57-17151eb4b3cc} {Suffrage:Voter ID:server-64602769-33bd-eee9-e8bf-dbd27ff81f33 Address:64602769-33bd-eee9-e8bf-dbd27ff81f33}]" +2025-06-03T23:36:53.069Z [INFO] server-582287de-5ec5-5c65-c12f-c5be37fa7781: entering follower state: follower="Node at 582287de-5ec5-5c65-c12f-c5be37fa7781 [Follower]" leader-address= leader-id= +2025-06-03T23:36:53.069Z [INFO] server-f49349f9-ceb6-f2d1-6e57-17151eb4b3cc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-582287de-5ec5-5c65-c12f-c5be37fa7781 Address:582287de-5ec5-5c65-c12f-c5be37fa7781} {Suffrage:Voter ID:server-f49349f9-ceb6-f2d1-6e57-17151eb4b3cc Address:f49349f9-ceb6-f2d1-6e57-17151eb4b3cc} {Suffrage:Voter ID:server-64602769-33bd-eee9-e8bf-dbd27ff81f33 Address:64602769-33bd-eee9-e8bf-dbd27ff81f33}]" +2025-06-03T23:36:53.069Z [INFO] server-f49349f9-ceb6-f2d1-6e57-17151eb4b3cc: entering follower state: follower="Node at f49349f9-ceb6-f2d1-6e57-17151eb4b3cc [Follower]" leader-address= leader-id= +2025-06-03T23:36:53.069Z [INFO] server-64602769-33bd-eee9-e8bf-dbd27ff81f33: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-582287de-5ec5-5c65-c12f-c5be37fa7781 Address:582287de-5ec5-5c65-c12f-c5be37fa7781} {Suffrage:Voter ID:server-f49349f9-ceb6-f2d1-6e57-17151eb4b3cc Address:f49349f9-ceb6-f2d1-6e57-17151eb4b3cc} {Suffrage:Voter ID:server-64602769-33bd-eee9-e8bf-dbd27ff81f33 Address:64602769-33bd-eee9-e8bf-dbd27ff81f33}]" +2025-06-03T23:36:53.069Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:53.069Z [INFO] server-64602769-33bd-eee9-e8bf-dbd27ff81f33: entering follower state: follower="Node at 64602769-33bd-eee9-e8bf-dbd27ff81f33 [Follower]" leader-address= leader-id= +2025-06-03T23:36:53.658Z [WARN] server-582287de-5ec5-5c65-c12f-c5be37fa7781: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:53.658Z [INFO] server-582287de-5ec5-5c65-c12f-c5be37fa7781: entering candidate state: node="Node at 582287de-5ec5-5c65-c12f-c5be37fa7781 [Candidate]" term=2 +2025-06-03T23:36:53.658Z [INFO] server-582287de-5ec5-5c65-c12f-c5be37fa7781: election won: term=2 tally=2 +2025-06-03T23:36:53.658Z [INFO] server-582287de-5ec5-5c65-c12f-c5be37fa7781: entering leader state: leader="Node at 582287de-5ec5-5c65-c12f-c5be37fa7781 [Leader]" +2025-06-03T23:36:53.658Z [INFO] server-582287de-5ec5-5c65-c12f-c5be37fa7781: added peer, starting replication: peer=server-f49349f9-ceb6-f2d1-6e57-17151eb4b3cc +2025-06-03T23:36:53.659Z [INFO] server-582287de-5ec5-5c65-c12f-c5be37fa7781: added peer, starting replication: peer=server-64602769-33bd-eee9-e8bf-dbd27ff81f33 +2025-06-03T23:36:53.659Z [INFO] server-582287de-5ec5-5c65-c12f-c5be37fa7781: pipelining replication: peer="{Voter server-64602769-33bd-eee9-e8bf-dbd27ff81f33 64602769-33bd-eee9-e8bf-dbd27ff81f33}" +2025-06-03T23:36:53.659Z [INFO] server-582287de-5ec5-5c65-c12f-c5be37fa7781: pipelining replication: peer="{Voter server-f49349f9-ceb6-f2d1-6e57-17151eb4b3cc f49349f9-ceb6-f2d1-6e57-17151eb4b3cc}" +2025-06-03T23:36:54.664Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:53.659109031 +0000 UTC m=+48.980863961 (1 nodes), 589.3272ms from start of poll, 589.9904ms from cluster start. Timeout at 2025-06-03 23:36:54.664534831 +0000 UTC m=+49.986289761, 1.0054258s after stability +2025-06-03T23:36:54.664Z [INFO] creating new snapshot: path=/tmp/raft3001051793/snapshots/2-15-1748993814664.tmp +2025-06-03T23:36:54.665Z [INFO] server-582287de-5ec5-5c65-c12f-c5be37fa7781: copied to local snapshot: bytes=71 +2025-06-03T23:36:54.665Z [INFO] server-582287de-5ec5-5c65-c12f-c5be37fa7781: snapshot restore progress: id=2-15-1748993814664 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:54.665Z [INFO] server-582287de-5ec5-5c65-c12f-c5be37fa7781: restored user snapshot: index=15 +2025-06-03T23:36:54.665Z [ERROR] server-582287de-5ec5-5c65-c12f-c5be37fa7781: failed to get log: index=3 error="log not found" +2025-06-03T23:36:54.665Z [INFO] server-582287de-5ec5-5c65-c12f-c5be37fa7781: aborting pipeline replication: peer="{Voter server-64602769-33bd-eee9-e8bf-dbd27ff81f33 64602769-33bd-eee9-e8bf-dbd27ff81f33}" +2025-06-03T23:36:54.665Z [ERROR] server-582287de-5ec5-5c65-c12f-c5be37fa7781: failed to get log: index=3 error="log not found" +2025-06-03T23:36:54.665Z [INFO] server-582287de-5ec5-5c65-c12f-c5be37fa7781: aborting pipeline replication: peer="{Voter server-f49349f9-ceb6-f2d1-6e57-17151eb4b3cc f49349f9-ceb6-f2d1-6e57-17151eb4b3cc}" +2025-06-03T23:36:54.675Z [ERROR] server-582287de-5ec5-5c65-c12f-c5be37fa7781: failed to get log: index=3 error="log not found" +2025-06-03T23:36:54.675Z [ERROR] server-582287de-5ec5-5c65-c12f-c5be37fa7781: failed to get log: index=3 error="log not found" +2025-06-03T23:36:54.675Z [INFO] creating new snapshot: path=/tmp/raft1622130016/snapshots/2-15-1748993814675.tmp +2025-06-03T23:36:54.675Z [INFO] creating new snapshot: path=/tmp/raft2515530177/snapshots/2-15-1748993814675.tmp +2025-06-03T23:36:54.676Z [INFO] server-f49349f9-ceb6-f2d1-6e57-17151eb4b3cc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:54.676Z [INFO] server-64602769-33bd-eee9-e8bf-dbd27ff81f33: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:54.676Z [INFO] server-64602769-33bd-eee9-e8bf-dbd27ff81f33: copied to local snapshot: bytes=71 +2025-06-03T23:36:54.676Z [INFO] server-f49349f9-ceb6-f2d1-6e57-17151eb4b3cc: copied to local snapshot: bytes=71 +2025-06-03T23:36:54.676Z [INFO] server-64602769-33bd-eee9-e8bf-dbd27ff81f33: snapshot restore progress: id=2-15-1748993814675 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:54.676Z [INFO] server-64602769-33bd-eee9-e8bf-dbd27ff81f33: Installed remote snapshot +2025-06-03T23:36:54.676Z [INFO] server-f49349f9-ceb6-f2d1-6e57-17151eb4b3cc: snapshot restore progress: id=2-15-1748993814675 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:54.676Z [INFO] server-f49349f9-ceb6-f2d1-6e57-17151eb4b3cc: Installed remote snapshot +2025-06-03T23:36:54.676Z [WARN] server-f49349f9-ceb6-f2d1-6e57-17151eb4b3cc: skipping application of old log: index=15 +2025-06-03T23:36:54.676Z [INFO] server-582287de-5ec5-5c65-c12f-c5be37fa7781: pipelining replication: peer="{Voter server-64602769-33bd-eee9-e8bf-dbd27ff81f33 64602769-33bd-eee9-e8bf-dbd27ff81f33}" +2025-06-03T23:36:54.676Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:36:54.676Z [INFO] server-582287de-5ec5-5c65-c12f-c5be37fa7781: pipelining replication: peer="{Voter server-f49349f9-ceb6-f2d1-6e57-17151eb4b3cc f49349f9-ceb6-f2d1-6e57-17151eb4b3cc}" +2025-06-03T23:36:55.683Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:54.676882231 +0000 UTC m=+49.998637161 (2 nodes), 1.04µs from start of poll, 1.6077636s from cluster start. Timeout at 2025-06-03 23:36:55.683619071 +0000 UTC m=+51.005373961, 1.0067368s after stability +2025-06-03T23:36:55.703Z [INFO] server-fbbca7c8-ac6a-bcd1-dfba-2e3593376a63: aborting pipeline replication: peer="{Voter server-ebd13c86-264c-9f48-c9ce-8e8451abe3dc ebd13c86-264c-9f48-c9ce-8e8451abe3dc}" +2025-06-03T23:36:55.703Z [INFO] server-fbbca7c8-ac6a-bcd1-dfba-2e3593376a63: aborting pipeline replication: peer="{Voter server-da9dd419-58d3-9030-dcde-7db9cb73cd82 da9dd419-58d3-9030-dcde-7db9cb73cd82}" === RUN TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true -2025-06-03T23:33:57.211Z [INFO] server-3fdec502-4b4d-2bde-0b8e-1cbfaac9be25: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3fdec502-4b4d-2bde-0b8e-1cbfaac9be25 Address:3fdec502-4b4d-2bde-0b8e-1cbfaac9be25} {Suffrage:Voter ID:server-a01ba2ed-663f-1858-966a-7260a5d0a6e2 Address:a01ba2ed-663f-1858-966a-7260a5d0a6e2} {Suffrage:Voter ID:server-ce19c4eb-d7e2-a1f9-fc9a-8f4e749b82e2 Address:ce19c4eb-d7e2-a1f9-fc9a-8f4e749b82e2}]" -2025-06-03T23:33:57.211Z [INFO] server-3fdec502-4b4d-2bde-0b8e-1cbfaac9be25: entering follower state: follower="Node at 3fdec502-4b4d-2bde-0b8e-1cbfaac9be25 [Follower]" leader-address= leader-id= -2025-06-03T23:33:57.211Z [INFO] server-a01ba2ed-663f-1858-966a-7260a5d0a6e2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3fdec502-4b4d-2bde-0b8e-1cbfaac9be25 Address:3fdec502-4b4d-2bde-0b8e-1cbfaac9be25} {Suffrage:Voter ID:server-a01ba2ed-663f-1858-966a-7260a5d0a6e2 Address:a01ba2ed-663f-1858-966a-7260a5d0a6e2} {Suffrage:Voter ID:server-ce19c4eb-d7e2-a1f9-fc9a-8f4e749b82e2 Address:ce19c4eb-d7e2-a1f9-fc9a-8f4e749b82e2}]" -2025-06-03T23:33:57.211Z [INFO] server-a01ba2ed-663f-1858-966a-7260a5d0a6e2: entering follower state: follower="Node at a01ba2ed-663f-1858-966a-7260a5d0a6e2 [Follower]" leader-address= leader-id= -2025-06-03T23:33:57.212Z [INFO] server-ce19c4eb-d7e2-a1f9-fc9a-8f4e749b82e2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3fdec502-4b4d-2bde-0b8e-1cbfaac9be25 Address:3fdec502-4b4d-2bde-0b8e-1cbfaac9be25} {Suffrage:Voter ID:server-a01ba2ed-663f-1858-966a-7260a5d0a6e2 Address:a01ba2ed-663f-1858-966a-7260a5d0a6e2} {Suffrage:Voter ID:server-ce19c4eb-d7e2-a1f9-fc9a-8f4e749b82e2 Address:ce19c4eb-d7e2-a1f9-fc9a-8f4e749b82e2}]" -2025-06-03T23:33:57.212Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:57.212Z [INFO] server-ce19c4eb-d7e2-a1f9-fc9a-8f4e749b82e2: entering follower state: follower="Node at ce19c4eb-d7e2-a1f9-fc9a-8f4e749b82e2 [Follower]" leader-address= leader-id= -2025-06-03T23:33:57.717Z [WARN] server-a01ba2ed-663f-1858-966a-7260a5d0a6e2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:57.717Z [INFO] server-a01ba2ed-663f-1858-966a-7260a5d0a6e2: entering candidate state: node="Node at a01ba2ed-663f-1858-966a-7260a5d0a6e2 [Candidate]" term=2 -2025-06-03T23:33:57.717Z [INFO] server-a01ba2ed-663f-1858-966a-7260a5d0a6e2: election won: term=2 tally=2 -2025-06-03T23:33:57.717Z [INFO] server-a01ba2ed-663f-1858-966a-7260a5d0a6e2: entering leader state: leader="Node at a01ba2ed-663f-1858-966a-7260a5d0a6e2 [Leader]" -2025-06-03T23:33:57.717Z [INFO] server-a01ba2ed-663f-1858-966a-7260a5d0a6e2: added peer, starting replication: peer=server-3fdec502-4b4d-2bde-0b8e-1cbfaac9be25 -2025-06-03T23:33:57.717Z [INFO] server-a01ba2ed-663f-1858-966a-7260a5d0a6e2: added peer, starting replication: peer=server-ce19c4eb-d7e2-a1f9-fc9a-8f4e749b82e2 -2025-06-03T23:33:57.717Z [INFO] server-a01ba2ed-663f-1858-966a-7260a5d0a6e2: pipelining replication: peer="{Voter server-3fdec502-4b4d-2bde-0b8e-1cbfaac9be25 3fdec502-4b4d-2bde-0b8e-1cbfaac9be25}" -2025-06-03T23:33:57.717Z [INFO] server-a01ba2ed-663f-1858-966a-7260a5d0a6e2: pipelining replication: peer="{Voter server-ce19c4eb-d7e2-a1f9-fc9a-8f4e749b82e2 ce19c4eb-d7e2-a1f9-fc9a-8f4e749b82e2}" -2025-06-03T23:33:58.723Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:57.717831871 +0000 UTC m=+52.738651241 (1 nodes), 505.77592ms from start of poll, 506.22244ms from cluster start. Timeout at 2025-06-03 23:33:58.723288191 +0000 UTC m=+53.744107561, 1.00545632s after stability -2025-06-03T23:33:58.724Z [INFO] server-a01ba2ed-663f-1858-966a-7260a5d0a6e2: starting snapshot up to: index=12 -2025-06-03T23:33:58.724Z [INFO] creating new snapshot: path=/tmp/raft1414886196/snapshots/2-12-1748993638724.tmp -2025-06-03T23:33:58.724Z [INFO] server-a01ba2ed-663f-1858-966a-7260a5d0a6e2: snapshot complete up to: index=12 -2025-06-03T23:33:58.725Z [INFO] server-70b68156-8420-0a77-0511-520cabef21e3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-70b68156-8420-0a77-0511-520cabef21e3 Address:70b68156-8420-0a77-0511-520cabef21e3} {Suffrage:Voter ID:server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c Address:3ec28349-8e5f-9bc8-c557-d74ac6460e6c} {Suffrage:Voter ID:server-66171a47-6a42-b033-ba2a-2ef36b2f72eb Address:66171a47-6a42-b033-ba2a-2ef36b2f72eb}]" -2025-06-03T23:33:58.726Z [INFO] server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-70b68156-8420-0a77-0511-520cabef21e3 Address:70b68156-8420-0a77-0511-520cabef21e3} {Suffrage:Voter ID:server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c Address:3ec28349-8e5f-9bc8-c557-d74ac6460e6c} {Suffrage:Voter ID:server-66171a47-6a42-b033-ba2a-2ef36b2f72eb Address:66171a47-6a42-b033-ba2a-2ef36b2f72eb}]" -2025-06-03T23:33:58.726Z [INFO] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-70b68156-8420-0a77-0511-520cabef21e3 Address:70b68156-8420-0a77-0511-520cabef21e3} {Suffrage:Voter ID:server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c Address:3ec28349-8e5f-9bc8-c557-d74ac6460e6c} {Suffrage:Voter ID:server-66171a47-6a42-b033-ba2a-2ef36b2f72eb Address:66171a47-6a42-b033-ba2a-2ef36b2f72eb}]" -2025-06-03T23:33:58.726Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:33:58.726Z [INFO] server-70b68156-8420-0a77-0511-520cabef21e3: entering follower state: follower="Node at 70b68156-8420-0a77-0511-520cabef21e3 [Follower]" leader-address= leader-id= -2025-06-03T23:33:58.726Z [INFO] server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c: entering follower state: follower="Node at 3ec28349-8e5f-9bc8-c557-d74ac6460e6c [Follower]" leader-address= leader-id= -2025-06-03T23:33:58.726Z [INFO] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: entering follower state: follower="Node at 66171a47-6a42-b033-ba2a-2ef36b2f72eb [Follower]" leader-address= leader-id= -2025-06-03T23:33:59.384Z [WARN] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:59.384Z [INFO] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: entering candidate state: node="Node at 66171a47-6a42-b033-ba2a-2ef36b2f72eb [Candidate]" term=2 -2025-06-03T23:33:59.384Z [WARN] server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:33:59.384Z [INFO] server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c: entering candidate state: node="Node at 3ec28349-8e5f-9bc8-c557-d74ac6460e6c [Candidate]" term=2 -2025-06-03T23:33:59.385Z [INFO] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: election won: term=2 tally=2 -2025-06-03T23:33:59.385Z [INFO] server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c: duplicate requestVote for same term: term=2 -2025-06-03T23:33:59.385Z [INFO] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: entering leader state: leader="Node at 66171a47-6a42-b033-ba2a-2ef36b2f72eb [Leader]" -2025-06-03T23:33:59.385Z [INFO] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: added peer, starting replication: peer=server-70b68156-8420-0a77-0511-520cabef21e3 -2025-06-03T23:33:59.385Z [INFO] server-70b68156-8420-0a77-0511-520cabef21e3: duplicate requestVote for same term: term=2 -2025-06-03T23:33:59.385Z [INFO] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: added peer, starting replication: peer=server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c -2025-06-03T23:33:59.385Z [WARN] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: rejecting vote request since we have a leader: from=3ec28349-8e5f-9bc8-c557-d74ac6460e6c leader=66171a47-6a42-b033-ba2a-2ef36b2f72eb leader-id=server-66171a47-6a42-b033-ba2a-2ef36b2f72eb -2025-06-03T23:33:59.385Z [INFO] server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c: entering follower state: follower="Node at 3ec28349-8e5f-9bc8-c557-d74ac6460e6c [Follower]" leader-address=66171a47-6a42-b033-ba2a-2ef36b2f72eb leader-id=server-66171a47-6a42-b033-ba2a-2ef36b2f72eb -2025-06-03T23:33:59.385Z [INFO] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: pipelining replication: peer="{Voter server-70b68156-8420-0a77-0511-520cabef21e3 70b68156-8420-0a77-0511-520cabef21e3}" -2025-06-03T23:33:59.385Z [INFO] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: pipelining replication: peer="{Voter server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c 3ec28349-8e5f-9bc8-c557-d74ac6460e6c}" -2025-06-03T23:34:00.390Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:33:59.385357031 +0000 UTC m=+54.406176401 (1 nodes), 659.1106ms from start of poll, 659.53588ms from cluster start. Timeout at 2025-06-03 23:34:00.390671751 +0000 UTC m=+55.411491121, 1.00531472s after stability -2025-06-03T23:34:00.391Z [INFO] creating new snapshot: path=/tmp/raft3519818619/snapshots/2-15-1748993640391.tmp -2025-06-03T23:34:00.391Z [INFO] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: copied to local snapshot: bytes=71 -2025-06-03T23:34:00.391Z [INFO] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: snapshot restore progress: id=2-15-1748993640391 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:00.391Z [INFO] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: removing all old logs from log store -2025-06-03T23:34:00.391Z [INFO] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: compacting logs: from=1 to=2 -2025-06-03T23:34:00.391Z [INFO] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: restored user snapshot: index=15 -2025-06-03T23:34:00.391Z [ERROR] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: failed to get log: index=2 error="log not found" -2025-06-03T23:34:00.391Z [INFO] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: aborting pipeline replication: peer="{Voter server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c 3ec28349-8e5f-9bc8-c557-d74ac6460e6c}" -2025-06-03T23:34:00.392Z [ERROR] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: failed to get log: index=2 error="log not found" -2025-06-03T23:34:00.392Z [INFO] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: aborting pipeline replication: peer="{Voter server-70b68156-8420-0a77-0511-520cabef21e3 70b68156-8420-0a77-0511-520cabef21e3}" -2025-06-03T23:34:00.401Z [ERROR] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: failed to get log: index=2 error="log not found" -2025-06-03T23:34:00.402Z [ERROR] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: failed to get log: index=2 error="log not found" -2025-06-03T23:34:00.402Z [INFO] creating new snapshot: path=/tmp/raft3945645190/snapshots/2-15-1748993640402.tmp -2025-06-03T23:34:00.402Z [INFO] creating new snapshot: path=/tmp/raft3438176933/snapshots/2-15-1748993640402.tmp -2025-06-03T23:34:00.405Z [INFO] server-70b68156-8420-0a77-0511-520cabef21e3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:00.405Z [INFO] server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:00.406Z [INFO] server-70b68156-8420-0a77-0511-520cabef21e3: copied to local snapshot: bytes=71 -2025-06-03T23:34:00.406Z [INFO] server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c: copied to local snapshot: bytes=71 -2025-06-03T23:34:00.406Z [INFO] server-70b68156-8420-0a77-0511-520cabef21e3: snapshot restore progress: id=2-15-1748993640402 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:00.406Z [INFO] server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c: snapshot restore progress: id=2-15-1748993640402 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:00.406Z [INFO] server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c: removing all old logs from log store -2025-06-03T23:34:00.406Z [INFO] server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c: compacting logs: from=1 to=2 -2025-06-03T23:34:00.406Z [INFO] server-70b68156-8420-0a77-0511-520cabef21e3: removing all old logs from log store -2025-06-03T23:34:00.406Z [INFO] server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c: Installed remote snapshot -2025-06-03T23:34:00.406Z [INFO] server-70b68156-8420-0a77-0511-520cabef21e3: compacting logs: from=1 to=2 -2025-06-03T23:34:00.406Z [INFO] server-70b68156-8420-0a77-0511-520cabef21e3: Installed remote snapshot -2025-06-03T23:34:00.407Z [INFO] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: pipelining replication: peer="{Voter server-70b68156-8420-0a77-0511-520cabef21e3 70b68156-8420-0a77-0511-520cabef21e3}" -2025-06-03T23:34:00.407Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:00.407Z [INFO] server-66171a47-6a42-b033-ba2a-2ef36b2f72eb: pipelining replication: peer="{Voter server-3ec28349-8e5f-9bc8-c557-d74ac6460e6c 3ec28349-8e5f-9bc8-c557-d74ac6460e6c}" -2025-06-03T23:34:01.413Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:00.407114511 +0000 UTC m=+55.427933881 (2 nodes), 1.12µs from start of poll, 1.68129336s from cluster start. Timeout at 2025-06-03 23:34:01.413119071 +0000 UTC m=+56.433938441, 1.00600456s after stability -2025-06-03T23:34:01.423Z [INFO] server-a01ba2ed-663f-1858-966a-7260a5d0a6e2: aborting pipeline replication: peer="{Voter server-ce19c4eb-d7e2-a1f9-fc9a-8f4e749b82e2 ce19c4eb-d7e2-a1f9-fc9a-8f4e749b82e2}" -2025-06-03T23:34:01.424Z [INFO] server-a01ba2ed-663f-1858-966a-7260a5d0a6e2: aborting pipeline replication: peer="{Voter server-3fdec502-4b4d-2bde-0b8e-1cbfaac9be25 3fdec502-4b4d-2bde-0b8e-1cbfaac9be25}" +2025-06-03T23:36:55.704Z [INFO] server-8ec15b1e-857b-ee03-d1b4-baf16b67527b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8ec15b1e-857b-ee03-d1b4-baf16b67527b Address:8ec15b1e-857b-ee03-d1b4-baf16b67527b} {Suffrage:Voter ID:server-41f7f64b-4efd-6d8f-f37d-4a762831ac9c Address:41f7f64b-4efd-6d8f-f37d-4a762831ac9c} {Suffrage:Voter ID:server-336c4dbb-29a6-a22f-08d3-f41322b52ff7 Address:336c4dbb-29a6-a22f-08d3-f41322b52ff7}]" +2025-06-03T23:36:55.704Z [INFO] server-41f7f64b-4efd-6d8f-f37d-4a762831ac9c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8ec15b1e-857b-ee03-d1b4-baf16b67527b Address:8ec15b1e-857b-ee03-d1b4-baf16b67527b} {Suffrage:Voter ID:server-41f7f64b-4efd-6d8f-f37d-4a762831ac9c Address:41f7f64b-4efd-6d8f-f37d-4a762831ac9c} {Suffrage:Voter ID:server-336c4dbb-29a6-a22f-08d3-f41322b52ff7 Address:336c4dbb-29a6-a22f-08d3-f41322b52ff7}]" +2025-06-03T23:36:55.704Z [INFO] server-336c4dbb-29a6-a22f-08d3-f41322b52ff7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8ec15b1e-857b-ee03-d1b4-baf16b67527b Address:8ec15b1e-857b-ee03-d1b4-baf16b67527b} {Suffrage:Voter ID:server-41f7f64b-4efd-6d8f-f37d-4a762831ac9c Address:41f7f64b-4efd-6d8f-f37d-4a762831ac9c} {Suffrage:Voter ID:server-336c4dbb-29a6-a22f-08d3-f41322b52ff7 Address:336c4dbb-29a6-a22f-08d3-f41322b52ff7}]" +2025-06-03T23:36:55.704Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:55.704Z [INFO] server-8ec15b1e-857b-ee03-d1b4-baf16b67527b: entering follower state: follower="Node at 8ec15b1e-857b-ee03-d1b4-baf16b67527b [Follower]" leader-address= leader-id= +2025-06-03T23:36:55.704Z [INFO] server-41f7f64b-4efd-6d8f-f37d-4a762831ac9c: entering follower state: follower="Node at 41f7f64b-4efd-6d8f-f37d-4a762831ac9c [Follower]" leader-address= leader-id= +2025-06-03T23:36:55.704Z [INFO] server-336c4dbb-29a6-a22f-08d3-f41322b52ff7: entering follower state: follower="Node at 336c4dbb-29a6-a22f-08d3-f41322b52ff7 [Follower]" leader-address= leader-id= +2025-06-03T23:36:56.532Z [WARN] server-336c4dbb-29a6-a22f-08d3-f41322b52ff7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:56.532Z [INFO] server-336c4dbb-29a6-a22f-08d3-f41322b52ff7: entering candidate state: node="Node at 336c4dbb-29a6-a22f-08d3-f41322b52ff7 [Candidate]" term=2 +2025-06-03T23:36:56.532Z [INFO] server-336c4dbb-29a6-a22f-08d3-f41322b52ff7: election won: term=2 tally=2 +2025-06-03T23:36:56.532Z [INFO] server-336c4dbb-29a6-a22f-08d3-f41322b52ff7: entering leader state: leader="Node at 336c4dbb-29a6-a22f-08d3-f41322b52ff7 [Leader]" +2025-06-03T23:36:56.532Z [INFO] server-336c4dbb-29a6-a22f-08d3-f41322b52ff7: added peer, starting replication: peer=server-8ec15b1e-857b-ee03-d1b4-baf16b67527b +2025-06-03T23:36:56.532Z [INFO] server-336c4dbb-29a6-a22f-08d3-f41322b52ff7: added peer, starting replication: peer=server-41f7f64b-4efd-6d8f-f37d-4a762831ac9c +2025-06-03T23:36:56.532Z [INFO] server-336c4dbb-29a6-a22f-08d3-f41322b52ff7: pipelining replication: peer="{Voter server-41f7f64b-4efd-6d8f-f37d-4a762831ac9c 41f7f64b-4efd-6d8f-f37d-4a762831ac9c}" +2025-06-03T23:36:56.533Z [INFO] server-336c4dbb-29a6-a22f-08d3-f41322b52ff7: pipelining replication: peer="{Voter server-8ec15b1e-857b-ee03-d1b4-baf16b67527b 8ec15b1e-857b-ee03-d1b4-baf16b67527b}" +2025-06-03T23:36:57.539Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:56.533081231 +0000 UTC m=+51.854836121 (1 nodes), 828.6118ms from start of poll, 828.80704ms from cluster start. Timeout at 2025-06-03 23:36:57.539613751 +0000 UTC m=+52.861368641, 1.00653252s after stability +2025-06-03T23:36:57.540Z [INFO] server-336c4dbb-29a6-a22f-08d3-f41322b52ff7: starting snapshot up to: index=12 +2025-06-03T23:36:57.540Z [INFO] creating new snapshot: path=/tmp/raft3049862299/snapshots/2-12-1748993817540.tmp +2025-06-03T23:36:57.540Z [INFO] server-336c4dbb-29a6-a22f-08d3-f41322b52ff7: snapshot complete up to: index=12 +2025-06-03T23:36:57.541Z [INFO] server-61d8bdfe-7df6-9421-931e-afbe754e0e50: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-61d8bdfe-7df6-9421-931e-afbe754e0e50 Address:61d8bdfe-7df6-9421-931e-afbe754e0e50} {Suffrage:Voter ID:server-91300b6e-b6f7-f6f9-190b-d83f473c1f80 Address:91300b6e-b6f7-f6f9-190b-d83f473c1f80} {Suffrage:Voter ID:server-05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7 Address:05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7}]" +2025-06-03T23:36:57.541Z [INFO] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-61d8bdfe-7df6-9421-931e-afbe754e0e50 Address:61d8bdfe-7df6-9421-931e-afbe754e0e50} {Suffrage:Voter ID:server-91300b6e-b6f7-f6f9-190b-d83f473c1f80 Address:91300b6e-b6f7-f6f9-190b-d83f473c1f80} {Suffrage:Voter ID:server-05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7 Address:05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7}]" +2025-06-03T23:36:57.541Z [INFO] server-05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-61d8bdfe-7df6-9421-931e-afbe754e0e50 Address:61d8bdfe-7df6-9421-931e-afbe754e0e50} {Suffrage:Voter ID:server-91300b6e-b6f7-f6f9-190b-d83f473c1f80 Address:91300b6e-b6f7-f6f9-190b-d83f473c1f80} {Suffrage:Voter ID:server-05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7 Address:05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7}]" +2025-06-03T23:36:57.541Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:36:57.541Z [INFO] server-61d8bdfe-7df6-9421-931e-afbe754e0e50: entering follower state: follower="Node at 61d8bdfe-7df6-9421-931e-afbe754e0e50 [Follower]" leader-address= leader-id= +2025-06-03T23:36:57.541Z [INFO] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: entering follower state: follower="Node at 91300b6e-b6f7-f6f9-190b-d83f473c1f80 [Follower]" leader-address= leader-id= +2025-06-03T23:36:57.541Z [INFO] server-05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7: entering follower state: follower="Node at 05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7 [Follower]" leader-address= leader-id= +2025-06-03T23:36:58.383Z [WARN] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:36:58.383Z [INFO] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: entering candidate state: node="Node at 91300b6e-b6f7-f6f9-190b-d83f473c1f80 [Candidate]" term=2 +2025-06-03T23:36:58.383Z [INFO] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: election won: term=2 tally=2 +2025-06-03T23:36:58.383Z [INFO] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: entering leader state: leader="Node at 91300b6e-b6f7-f6f9-190b-d83f473c1f80 [Leader]" +2025-06-03T23:36:58.383Z [INFO] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: added peer, starting replication: peer=server-61d8bdfe-7df6-9421-931e-afbe754e0e50 +2025-06-03T23:36:58.383Z [INFO] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: added peer, starting replication: peer=server-05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7 +2025-06-03T23:36:58.383Z [INFO] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: pipelining replication: peer="{Voter server-05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7 05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7}" +2025-06-03T23:36:58.383Z [INFO] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: pipelining replication: peer="{Voter server-61d8bdfe-7df6-9421-931e-afbe754e0e50 61d8bdfe-7df6-9421-931e-afbe754e0e50}" +2025-06-03T23:36:59.391Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:36:58.383833031 +0000 UTC m=+53.705587921 (1 nodes), 842.63904ms from start of poll, 842.8256ms from cluster start. Timeout at 2025-06-03 23:36:59.391344431 +0000 UTC m=+54.713099321, 1.0075114s after stability +2025-06-03T23:36:59.391Z [INFO] creating new snapshot: path=/tmp/raft373059507/snapshots/2-15-1748993819391.tmp +2025-06-03T23:36:59.391Z [INFO] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: copied to local snapshot: bytes=71 +2025-06-03T23:36:59.391Z [INFO] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: snapshot restore progress: id=2-15-1748993819391 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:59.391Z [INFO] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: removing all old logs from log store +2025-06-03T23:36:59.391Z [INFO] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: compacting logs: from=1 to=2 +2025-06-03T23:36:59.391Z [INFO] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: restored user snapshot: index=15 +2025-06-03T23:36:59.391Z [ERROR] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: failed to get log: index=2 error="log not found" +2025-06-03T23:36:59.391Z [INFO] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: aborting pipeline replication: peer="{Voter server-05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7 05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7}" +2025-06-03T23:36:59.392Z [ERROR] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: failed to get log: index=2 error="log not found" +2025-06-03T23:36:59.392Z [INFO] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: aborting pipeline replication: peer="{Voter server-61d8bdfe-7df6-9421-931e-afbe754e0e50 61d8bdfe-7df6-9421-931e-afbe754e0e50}" +2025-06-03T23:36:59.399Z [ERROR] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: failed to get log: index=2 error="log not found" +2025-06-03T23:36:59.399Z [ERROR] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: failed to get log: index=2 error="log not found" +2025-06-03T23:36:59.399Z [INFO] creating new snapshot: path=/tmp/raft3579559306/snapshots/2-15-1748993819399.tmp +2025-06-03T23:36:59.399Z [INFO] server-61d8bdfe-7df6-9421-931e-afbe754e0e50: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:59.400Z [INFO] server-61d8bdfe-7df6-9421-931e-afbe754e0e50: copied to local snapshot: bytes=71 +2025-06-03T23:36:59.400Z [INFO] server-61d8bdfe-7df6-9421-931e-afbe754e0e50: snapshot restore progress: id=2-15-1748993819399 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:59.400Z [INFO] server-61d8bdfe-7df6-9421-931e-afbe754e0e50: removing all old logs from log store +2025-06-03T23:36:59.400Z [INFO] server-61d8bdfe-7df6-9421-931e-afbe754e0e50: compacting logs: from=1 to=2 +2025-06-03T23:36:59.400Z [INFO] server-61d8bdfe-7df6-9421-931e-afbe754e0e50: Installed remote snapshot +2025-06-03T23:36:59.400Z [INFO] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: pipelining replication: peer="{Voter server-61d8bdfe-7df6-9421-931e-afbe754e0e50 61d8bdfe-7df6-9421-931e-afbe754e0e50}" +2025-06-03T23:36:59.400Z [INFO] creating new snapshot: path=/tmp/raft2985445566/snapshots/2-15-1748993819400.tmp +2025-06-03T23:36:59.400Z [INFO] server-05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:59.400Z [INFO] server-05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7: copied to local snapshot: bytes=71 +2025-06-03T23:36:59.400Z [INFO] server-05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7: snapshot restore progress: id=2-15-1748993819400 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:36:59.400Z [INFO] server-05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7: removing all old logs from log store +2025-06-03T23:36:59.400Z [INFO] server-05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7: compacting logs: from=1 to=2 +2025-06-03T23:36:59.400Z [INFO] server-05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7: Installed remote snapshot +2025-06-03T23:36:59.400Z [INFO] server-91300b6e-b6f7-f6f9-190b-d83f473c1f80: pipelining replication: peer="{Voter server-05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7 05dfb8ad-8c8a-a61a-9b20-f8fd33a010d7}" +2025-06-03T23:36:59.400Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:00.407Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:36:59.400722911 +0000 UTC m=+54.722477801 (2 nodes), 920ns from start of poll, 1.85971548s from cluster start. Timeout at 2025-06-03 23:37:00.407343031 +0000 UTC m=+55.729097921, 1.00662012s after stability +2025-06-03T23:37:00.427Z [INFO] server-336c4dbb-29a6-a22f-08d3-f41322b52ff7: aborting pipeline replication: peer="{Voter server-41f7f64b-4efd-6d8f-f37d-4a762831ac9c 41f7f64b-4efd-6d8f-f37d-4a762831ac9c}" +2025-06-03T23:37:00.427Z [INFO] server-336c4dbb-29a6-a22f-08d3-f41322b52ff7: aborting pipeline replication: peer="{Voter server-8ec15b1e-857b-ee03-d1b4-baf16b67527b 8ec15b1e-857b-ee03-d1b4-baf16b67527b}" === RUN TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false -2025-06-03T23:34:01.425Z [INFO] server-f82e82cc-6b6d-f677-d552-a92eb6f2dea3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f82e82cc-6b6d-f677-d552-a92eb6f2dea3 Address:f82e82cc-6b6d-f677-d552-a92eb6f2dea3} {Suffrage:Voter ID:server-37641f7a-b9ff-7384-288b-9fd4c4485e73 Address:37641f7a-b9ff-7384-288b-9fd4c4485e73} {Suffrage:Voter ID:server-360122d0-4b64-3c96-f80a-ca00fb5aebb9 Address:360122d0-4b64-3c96-f80a-ca00fb5aebb9}]" -2025-06-03T23:34:01.425Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f82e82cc-6b6d-f677-d552-a92eb6f2dea3 Address:f82e82cc-6b6d-f677-d552-a92eb6f2dea3} {Suffrage:Voter ID:server-37641f7a-b9ff-7384-288b-9fd4c4485e73 Address:37641f7a-b9ff-7384-288b-9fd4c4485e73} {Suffrage:Voter ID:server-360122d0-4b64-3c96-f80a-ca00fb5aebb9 Address:360122d0-4b64-3c96-f80a-ca00fb5aebb9}]" -2025-06-03T23:34:01.425Z [INFO] server-360122d0-4b64-3c96-f80a-ca00fb5aebb9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f82e82cc-6b6d-f677-d552-a92eb6f2dea3 Address:f82e82cc-6b6d-f677-d552-a92eb6f2dea3} {Suffrage:Voter ID:server-37641f7a-b9ff-7384-288b-9fd4c4485e73 Address:37641f7a-b9ff-7384-288b-9fd4c4485e73} {Suffrage:Voter ID:server-360122d0-4b64-3c96-f80a-ca00fb5aebb9 Address:360122d0-4b64-3c96-f80a-ca00fb5aebb9}]" -2025-06-03T23:34:01.425Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:01.425Z [INFO] server-f82e82cc-6b6d-f677-d552-a92eb6f2dea3: entering follower state: follower="Node at f82e82cc-6b6d-f677-d552-a92eb6f2dea3 [Follower]" leader-address= leader-id= -2025-06-03T23:34:01.425Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: entering follower state: follower="Node at 37641f7a-b9ff-7384-288b-9fd4c4485e73 [Follower]" leader-address= leader-id= -2025-06-03T23:34:01.425Z [INFO] server-360122d0-4b64-3c96-f80a-ca00fb5aebb9: entering follower state: follower="Node at 360122d0-4b64-3c96-f80a-ca00fb5aebb9 [Follower]" leader-address= leader-id= -2025-06-03T23:34:02.337Z [WARN] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:02.337Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: entering candidate state: node="Node at 37641f7a-b9ff-7384-288b-9fd4c4485e73 [Candidate]" term=2 -2025-06-03T23:34:02.337Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: election won: term=2 tally=2 -2025-06-03T23:34:02.337Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: entering leader state: leader="Node at 37641f7a-b9ff-7384-288b-9fd4c4485e73 [Leader]" -2025-06-03T23:34:02.337Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: added peer, starting replication: peer=server-f82e82cc-6b6d-f677-d552-a92eb6f2dea3 -2025-06-03T23:34:02.337Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: added peer, starting replication: peer=server-360122d0-4b64-3c96-f80a-ca00fb5aebb9 -2025-06-03T23:34:02.337Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: pipelining replication: peer="{Voter server-f82e82cc-6b6d-f677-d552-a92eb6f2dea3 f82e82cc-6b6d-f677-d552-a92eb6f2dea3}" -2025-06-03T23:34:02.337Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: pipelining replication: peer="{Voter server-360122d0-4b64-3c96-f80a-ca00fb5aebb9 360122d0-4b64-3c96-f80a-ca00fb5aebb9}" -2025-06-03T23:34:03.343Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:02.337533791 +0000 UTC m=+57.358353201 (1 nodes), 912.23508ms from start of poll, 912.69024ms from cluster start. Timeout at 2025-06-03 23:34:03.343517311 +0000 UTC m=+58.364336681, 1.00598348s after stability -2025-06-03T23:34:03.344Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: starting snapshot up to: index=12 -2025-06-03T23:34:03.344Z [INFO] creating new snapshot: path=/tmp/raft2427269089/snapshots/2-12-1748993643344.tmp -2025-06-03T23:34:03.344Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: snapshot complete up to: index=12 -2025-06-03T23:34:03.345Z [INFO] creating new snapshot: path=/tmp/raft2427269089/snapshots/2-113-1748993643345.tmp -2025-06-03T23:34:03.346Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: copied to local snapshot: bytes=71 -2025-06-03T23:34:03.346Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: snapshot restore progress: id=2-113-1748993643345 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:03.346Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: restored user snapshot: index=113 -2025-06-03T23:34:03.346Z [ERROR] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: failed to get log: index=23 error="log not found" -2025-06-03T23:34:03.346Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: aborting pipeline replication: peer="{Voter server-360122d0-4b64-3c96-f80a-ca00fb5aebb9 360122d0-4b64-3c96-f80a-ca00fb5aebb9}" -2025-06-03T23:34:03.346Z [ERROR] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: failed to get log: index=23 error="log not found" -2025-06-03T23:34:03.346Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: aborting pipeline replication: peer="{Voter server-f82e82cc-6b6d-f677-d552-a92eb6f2dea3 f82e82cc-6b6d-f677-d552-a92eb6f2dea3}" -2025-06-03T23:34:03.354Z [ERROR] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: failed to get log: index=23 error="log not found" -2025-06-03T23:34:03.354Z [ERROR] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: failed to get log: index=23 error="log not found" -2025-06-03T23:34:03.355Z [INFO] creating new snapshot: path=/tmp/raft2822000531/snapshots/2-113-1748993643355.tmp -2025-06-03T23:34:03.355Z [INFO] server-f82e82cc-6b6d-f677-d552-a92eb6f2dea3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:03.355Z [INFO] creating new snapshot: path=/tmp/raft3886143670/snapshots/2-113-1748993643355.tmp -2025-06-03T23:34:03.355Z [INFO] server-f82e82cc-6b6d-f677-d552-a92eb6f2dea3: copied to local snapshot: bytes=71 -2025-06-03T23:34:03.355Z [INFO] server-360122d0-4b64-3c96-f80a-ca00fb5aebb9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:03.355Z [INFO] server-f82e82cc-6b6d-f677-d552-a92eb6f2dea3: snapshot restore progress: id=2-113-1748993643355 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:03.355Z [INFO] server-f82e82cc-6b6d-f677-d552-a92eb6f2dea3: Installed remote snapshot -2025-06-03T23:34:03.355Z [WARN] server-f82e82cc-6b6d-f677-d552-a92eb6f2dea3: skipping application of old log: index=22 -2025-06-03T23:34:03.355Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: pipelining replication: peer="{Voter server-f82e82cc-6b6d-f677-d552-a92eb6f2dea3 f82e82cc-6b6d-f677-d552-a92eb6f2dea3}" -2025-06-03T23:34:03.355Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:03.356Z [INFO] server-360122d0-4b64-3c96-f80a-ca00fb5aebb9: copied to local snapshot: bytes=71 -2025-06-03T23:34:03.360Z [INFO] server-360122d0-4b64-3c96-f80a-ca00fb5aebb9: snapshot restore progress: id=2-113-1748993643355 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:03.360Z [INFO] server-360122d0-4b64-3c96-f80a-ca00fb5aebb9: Installed remote snapshot -2025-06-03T23:34:03.360Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: pipelining replication: peer="{Voter server-360122d0-4b64-3c96-f80a-ca00fb5aebb9 360122d0-4b64-3c96-f80a-ca00fb5aebb9}" -2025-06-03T23:34:04.360Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:03.355759791 +0000 UTC m=+58.376579121 (2 nodes), 480ns from start of poll, 1.93091616s from cluster start. Timeout at 2025-06-03 23:34:04.360813991 +0000 UTC m=+59.381633361, 1.00505424s after stability -2025-06-03T23:34:04.373Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: aborting pipeline replication: peer="{Voter server-f82e82cc-6b6d-f677-d552-a92eb6f2dea3 f82e82cc-6b6d-f677-d552-a92eb6f2dea3}" -2025-06-03T23:34:04.373Z [INFO] server-37641f7a-b9ff-7384-288b-9fd4c4485e73: aborting pipeline replication: peer="{Voter server-360122d0-4b64-3c96-f80a-ca00fb5aebb9 360122d0-4b64-3c96-f80a-ca00fb5aebb9}" +2025-06-03T23:37:00.427Z [INFO] server-6ed6c291-99c4-2a3d-eb85-8d4bb3813824: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ed6c291-99c4-2a3d-eb85-8d4bb3813824 Address:6ed6c291-99c4-2a3d-eb85-8d4bb3813824} {Suffrage:Voter ID:server-2a703600-eab3-ef81-a95e-f9e273aa6743 Address:2a703600-eab3-ef81-a95e-f9e273aa6743} {Suffrage:Voter ID:server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4 Address:b90c668f-5f43-2877-c3c7-9a29ea33ddc4}]" +2025-06-03T23:37:00.428Z [INFO] server-2a703600-eab3-ef81-a95e-f9e273aa6743: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ed6c291-99c4-2a3d-eb85-8d4bb3813824 Address:6ed6c291-99c4-2a3d-eb85-8d4bb3813824} {Suffrage:Voter ID:server-2a703600-eab3-ef81-a95e-f9e273aa6743 Address:2a703600-eab3-ef81-a95e-f9e273aa6743} {Suffrage:Voter ID:server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4 Address:b90c668f-5f43-2877-c3c7-9a29ea33ddc4}]" +2025-06-03T23:37:00.428Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ed6c291-99c4-2a3d-eb85-8d4bb3813824 Address:6ed6c291-99c4-2a3d-eb85-8d4bb3813824} {Suffrage:Voter ID:server-2a703600-eab3-ef81-a95e-f9e273aa6743 Address:2a703600-eab3-ef81-a95e-f9e273aa6743} {Suffrage:Voter ID:server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4 Address:b90c668f-5f43-2877-c3c7-9a29ea33ddc4}]" +2025-06-03T23:37:00.428Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:00.428Z [INFO] server-6ed6c291-99c4-2a3d-eb85-8d4bb3813824: entering follower state: follower="Node at 6ed6c291-99c4-2a3d-eb85-8d4bb3813824 [Follower]" leader-address= leader-id= +2025-06-03T23:37:00.428Z [INFO] server-2a703600-eab3-ef81-a95e-f9e273aa6743: entering follower state: follower="Node at 2a703600-eab3-ef81-a95e-f9e273aa6743 [Follower]" leader-address= leader-id= +2025-06-03T23:37:00.428Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: entering follower state: follower="Node at b90c668f-5f43-2877-c3c7-9a29ea33ddc4 [Follower]" leader-address= leader-id= +2025-06-03T23:37:01.075Z [WARN] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:01.075Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: entering candidate state: node="Node at b90c668f-5f43-2877-c3c7-9a29ea33ddc4 [Candidate]" term=2 +2025-06-03T23:37:01.075Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: election won: term=2 tally=2 +2025-06-03T23:37:01.075Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: entering leader state: leader="Node at b90c668f-5f43-2877-c3c7-9a29ea33ddc4 [Leader]" +2025-06-03T23:37:01.075Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: added peer, starting replication: peer=server-6ed6c291-99c4-2a3d-eb85-8d4bb3813824 +2025-06-03T23:37:01.075Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: added peer, starting replication: peer=server-2a703600-eab3-ef81-a95e-f9e273aa6743 +2025-06-03T23:37:01.075Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: pipelining replication: peer="{Voter server-2a703600-eab3-ef81-a95e-f9e273aa6743 2a703600-eab3-ef81-a95e-f9e273aa6743}" +2025-06-03T23:37:01.075Z [WARN] server-6ed6c291-99c4-2a3d-eb85-8d4bb3813824: rejecting vote request since our last term is greater: candidate=b90c668f-5f43-2877-c3c7-9a29ea33ddc4 last-term=2 last-candidate-term=1 +2025-06-03T23:37:01.075Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: pipelining replication: peer="{Voter server-6ed6c291-99c4-2a3d-eb85-8d4bb3813824 6ed6c291-99c4-2a3d-eb85-8d4bb3813824}" +2025-06-03T23:37:02.083Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:01.075690871 +0000 UTC m=+56.397445721 (1 nodes), 647.60696ms from start of poll, 647.80676ms from cluster start. Timeout at 2025-06-03 23:37:02.083746671 +0000 UTC m=+57.405501561, 1.00805584s after stability +2025-06-03T23:37:02.084Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: starting snapshot up to: index=12 +2025-06-03T23:37:02.084Z [INFO] creating new snapshot: path=/tmp/raft603323998/snapshots/2-12-1748993822084.tmp +2025-06-03T23:37:02.084Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: snapshot complete up to: index=12 +2025-06-03T23:37:02.084Z [INFO] creating new snapshot: path=/tmp/raft603323998/snapshots/2-113-1748993822084.tmp +2025-06-03T23:37:02.085Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: copied to local snapshot: bytes=71 +2025-06-03T23:37:02.085Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: snapshot restore progress: id=2-113-1748993822084 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:02.085Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: restored user snapshot: index=113 +2025-06-03T23:37:02.085Z [ERROR] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: failed to get log: index=23 error="log not found" +2025-06-03T23:37:02.085Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: aborting pipeline replication: peer="{Voter server-2a703600-eab3-ef81-a95e-f9e273aa6743 2a703600-eab3-ef81-a95e-f9e273aa6743}" +2025-06-03T23:37:02.085Z [ERROR] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: failed to get log: index=23 error="log not found" +2025-06-03T23:37:02.085Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: aborting pipeline replication: peer="{Voter server-6ed6c291-99c4-2a3d-eb85-8d4bb3813824 6ed6c291-99c4-2a3d-eb85-8d4bb3813824}" +2025-06-03T23:37:02.107Z [ERROR] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: failed to get log: index=23 error="log not found" +2025-06-03T23:37:02.107Z [INFO] creating new snapshot: path=/tmp/raft2814018786/snapshots/2-113-1748993822107.tmp +2025-06-03T23:37:02.107Z [INFO] server-6ed6c291-99c4-2a3d-eb85-8d4bb3813824: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:02.108Z [INFO] server-6ed6c291-99c4-2a3d-eb85-8d4bb3813824: copied to local snapshot: bytes=71 +2025-06-03T23:37:02.108Z [INFO] server-6ed6c291-99c4-2a3d-eb85-8d4bb3813824: snapshot restore progress: id=2-113-1748993822107 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:02.108Z [INFO] server-6ed6c291-99c4-2a3d-eb85-8d4bb3813824: Installed remote snapshot +2025-06-03T23:37:02.108Z [WARN] server-6ed6c291-99c4-2a3d-eb85-8d4bb3813824: skipping application of old log: index=22 +2025-06-03T23:37:02.108Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: pipelining replication: peer="{Voter server-6ed6c291-99c4-2a3d-eb85-8d4bb3813824 6ed6c291-99c4-2a3d-eb85-8d4bb3813824}" +2025-06-03T23:37:02.108Z [ERROR] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: failed to get log: index=23 error="log not found" +2025-06-03T23:37:02.108Z [INFO] creating new snapshot: path=/tmp/raft4135932596/snapshots/2-113-1748993822108.tmp +2025-06-03T23:37:02.108Z [INFO] server-2a703600-eab3-ef81-a95e-f9e273aa6743: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:02.108Z [INFO] server-2a703600-eab3-ef81-a95e-f9e273aa6743: copied to local snapshot: bytes=71 +2025-06-03T23:37:02.108Z [INFO] server-2a703600-eab3-ef81-a95e-f9e273aa6743: snapshot restore progress: id=2-113-1748993822108 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:02.108Z [INFO] server-2a703600-eab3-ef81-a95e-f9e273aa6743: Installed remote snapshot +2025-06-03T23:37:02.108Z [WARN] server-2a703600-eab3-ef81-a95e-f9e273aa6743: skipping application of old log: index=22 +2025-06-03T23:37:02.108Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: pipelining replication: peer="{Voter server-2a703600-eab3-ef81-a95e-f9e273aa6743 2a703600-eab3-ef81-a95e-f9e273aa6743}" +2025-06-03T23:37:02.109Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:03.115Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:02.109049911 +0000 UTC m=+57.430804761 (2 nodes), 520ns from start of poll, 1.6811658s from cluster start. Timeout at 2025-06-03 23:37:03.115362871 +0000 UTC m=+58.437117761, 1.006313s after stability +2025-06-03T23:37:03.131Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: aborting pipeline replication: peer="{Voter server-2a703600-eab3-ef81-a95e-f9e273aa6743 2a703600-eab3-ef81-a95e-f9e273aa6743}" +2025-06-03T23:37:03.131Z [INFO] server-b90c668f-5f43-2877-c3c7-9a29ea33ddc4: aborting pipeline replication: peer="{Voter server-6ed6c291-99c4-2a3d-eb85-8d4bb3813824 6ed6c291-99c4-2a3d-eb85-8d4bb3813824}" === RUN TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false -2025-06-03T23:34:04.374Z [INFO] server-e86497a6-9add-7c41-3a88-3f0873475c2e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e86497a6-9add-7c41-3a88-3f0873475c2e Address:e86497a6-9add-7c41-3a88-3f0873475c2e} {Suffrage:Voter ID:server-110ac8fc-39f7-baee-511d-07000eae7692 Address:110ac8fc-39f7-baee-511d-07000eae7692} {Suffrage:Voter ID:server-f1f944b9-7162-fa59-3101-8b95565cbf28 Address:f1f944b9-7162-fa59-3101-8b95565cbf28}]" -2025-06-03T23:34:04.374Z [INFO] server-e86497a6-9add-7c41-3a88-3f0873475c2e: entering follower state: follower="Node at e86497a6-9add-7c41-3a88-3f0873475c2e [Follower]" leader-address= leader-id= -2025-06-03T23:34:04.374Z [INFO] server-110ac8fc-39f7-baee-511d-07000eae7692: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e86497a6-9add-7c41-3a88-3f0873475c2e Address:e86497a6-9add-7c41-3a88-3f0873475c2e} {Suffrage:Voter ID:server-110ac8fc-39f7-baee-511d-07000eae7692 Address:110ac8fc-39f7-baee-511d-07000eae7692} {Suffrage:Voter ID:server-f1f944b9-7162-fa59-3101-8b95565cbf28 Address:f1f944b9-7162-fa59-3101-8b95565cbf28}]" -2025-06-03T23:34:04.374Z [INFO] server-110ac8fc-39f7-baee-511d-07000eae7692: entering follower state: follower="Node at 110ac8fc-39f7-baee-511d-07000eae7692 [Follower]" leader-address= leader-id= -2025-06-03T23:34:04.374Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e86497a6-9add-7c41-3a88-3f0873475c2e Address:e86497a6-9add-7c41-3a88-3f0873475c2e} {Suffrage:Voter ID:server-110ac8fc-39f7-baee-511d-07000eae7692 Address:110ac8fc-39f7-baee-511d-07000eae7692} {Suffrage:Voter ID:server-f1f944b9-7162-fa59-3101-8b95565cbf28 Address:f1f944b9-7162-fa59-3101-8b95565cbf28}]" -2025-06-03T23:34:04.374Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:04.374Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: entering follower state: follower="Node at f1f944b9-7162-fa59-3101-8b95565cbf28 [Follower]" leader-address= leader-id= -2025-06-03T23:34:05.015Z [WARN] server-f1f944b9-7162-fa59-3101-8b95565cbf28: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:05.015Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: entering candidate state: node="Node at f1f944b9-7162-fa59-3101-8b95565cbf28 [Candidate]" term=2 -2025-06-03T23:34:05.015Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: election won: term=2 tally=2 -2025-06-03T23:34:05.015Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: entering leader state: leader="Node at f1f944b9-7162-fa59-3101-8b95565cbf28 [Leader]" -2025-06-03T23:34:05.015Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: added peer, starting replication: peer=server-e86497a6-9add-7c41-3a88-3f0873475c2e -2025-06-03T23:34:05.015Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: added peer, starting replication: peer=server-110ac8fc-39f7-baee-511d-07000eae7692 -2025-06-03T23:34:05.016Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: pipelining replication: peer="{Voter server-110ac8fc-39f7-baee-511d-07000eae7692 110ac8fc-39f7-baee-511d-07000eae7692}" -2025-06-03T23:34:05.016Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: pipelining replication: peer="{Voter server-e86497a6-9add-7c41-3a88-3f0873475c2e e86497a6-9add-7c41-3a88-3f0873475c2e}" -2025-06-03T23:34:06.021Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:05.016140991 +0000 UTC m=+60.036960361 (1 nodes), 641.38712ms from start of poll, 641.98348ms from cluster start. Timeout at 2025-06-03 23:34:06.021370031 +0000 UTC m=+61.042189441, 1.00522908s after stability -2025-06-03T23:34:06.022Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: starting snapshot up to: index=12 -2025-06-03T23:34:06.022Z [INFO] creating new snapshot: path=/tmp/raft3691643975/snapshots/2-12-1748993646022.tmp -2025-06-03T23:34:06.022Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: snapshot complete up to: index=12 -2025-06-03T23:34:06.023Z [INFO] creating new snapshot: path=/tmp/raft3691643975/snapshots/2-113-1748993646023.tmp -2025-06-03T23:34:06.024Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: copied to local snapshot: bytes=71 -2025-06-03T23:34:06.024Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: snapshot restore progress: id=2-113-1748993646023 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:06.024Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: removing all old logs from log store -2025-06-03T23:34:06.024Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: compacting logs: from=1 to=22 -2025-06-03T23:34:06.024Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: restored user snapshot: index=113 -2025-06-03T23:34:06.024Z [ERROR] server-f1f944b9-7162-fa59-3101-8b95565cbf28: failed to get log: index=22 error="log not found" -2025-06-03T23:34:06.024Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: aborting pipeline replication: peer="{Voter server-110ac8fc-39f7-baee-511d-07000eae7692 110ac8fc-39f7-baee-511d-07000eae7692}" -2025-06-03T23:34:06.024Z [ERROR] server-f1f944b9-7162-fa59-3101-8b95565cbf28: failed to get log: index=22 error="log not found" -2025-06-03T23:34:06.024Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: aborting pipeline replication: peer="{Voter server-e86497a6-9add-7c41-3a88-3f0873475c2e e86497a6-9add-7c41-3a88-3f0873475c2e}" -2025-06-03T23:34:06.032Z [ERROR] server-f1f944b9-7162-fa59-3101-8b95565cbf28: failed to get log: index=22 error="log not found" -2025-06-03T23:34:06.033Z [INFO] creating new snapshot: path=/tmp/raft1262201727/snapshots/2-113-1748993646033.tmp -2025-06-03T23:34:06.033Z [INFO] server-110ac8fc-39f7-baee-511d-07000eae7692: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:06.033Z [INFO] server-110ac8fc-39f7-baee-511d-07000eae7692: copied to local snapshot: bytes=71 -2025-06-03T23:34:06.033Z [INFO] server-110ac8fc-39f7-baee-511d-07000eae7692: snapshot restore progress: id=2-113-1748993646033 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:06.034Z [INFO] server-110ac8fc-39f7-baee-511d-07000eae7692: removing all old logs from log store -2025-06-03T23:34:06.034Z [INFO] server-110ac8fc-39f7-baee-511d-07000eae7692: compacting logs: from=1 to=22 -2025-06-03T23:34:06.034Z [INFO] server-110ac8fc-39f7-baee-511d-07000eae7692: Installed remote snapshot -2025-06-03T23:34:06.034Z [WARN] server-110ac8fc-39f7-baee-511d-07000eae7692: skipping application of old log: index=22 -2025-06-03T23:34:06.034Z [ERROR] server-f1f944b9-7162-fa59-3101-8b95565cbf28: failed to get log: index=22 error="log not found" -2025-06-03T23:34:06.034Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: pipelining replication: peer="{Voter server-110ac8fc-39f7-baee-511d-07000eae7692 110ac8fc-39f7-baee-511d-07000eae7692}" -2025-06-03T23:34:06.034Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:06.034Z [INFO] creating new snapshot: path=/tmp/raft3500301781/snapshots/2-113-1748993646034.tmp -2025-06-03T23:34:06.034Z [INFO] server-e86497a6-9add-7c41-3a88-3f0873475c2e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:06.035Z [INFO] server-e86497a6-9add-7c41-3a88-3f0873475c2e: copied to local snapshot: bytes=71 -2025-06-03T23:34:06.035Z [INFO] server-e86497a6-9add-7c41-3a88-3f0873475c2e: snapshot restore progress: id=2-113-1748993646034 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:06.035Z [INFO] server-e86497a6-9add-7c41-3a88-3f0873475c2e: removing all old logs from log store -2025-06-03T23:34:06.035Z [INFO] server-e86497a6-9add-7c41-3a88-3f0873475c2e: compacting logs: from=1 to=22 -2025-06-03T23:34:06.035Z [INFO] server-e86497a6-9add-7c41-3a88-3f0873475c2e: Installed remote snapshot -2025-06-03T23:34:06.037Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: pipelining replication: peer="{Voter server-e86497a6-9add-7c41-3a88-3f0873475c2e e86497a6-9add-7c41-3a88-3f0873475c2e}" -2025-06-03T23:34:07.039Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:06.034289351 +0000 UTC m=+61.055108721 (2 nodes), 1.48µs from start of poll, 1.66013184s from cluster start. Timeout at 2025-06-03 23:34:07.039673671 +0000 UTC m=+62.060493041, 1.00538432s after stability -2025-06-03T23:34:07.051Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: aborting pipeline replication: peer="{Voter server-110ac8fc-39f7-baee-511d-07000eae7692 110ac8fc-39f7-baee-511d-07000eae7692}" -2025-06-03T23:34:07.051Z [INFO] server-f1f944b9-7162-fa59-3101-8b95565cbf28: aborting pipeline replication: peer="{Voter server-e86497a6-9add-7c41-3a88-3f0873475c2e e86497a6-9add-7c41-3a88-3f0873475c2e}" +2025-06-03T23:37:03.132Z [INFO] server-b181b3e3-ef8c-9eca-d046-ace67869b333: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b181b3e3-ef8c-9eca-d046-ace67869b333 Address:b181b3e3-ef8c-9eca-d046-ace67869b333} {Suffrage:Voter ID:server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d Address:26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d} {Suffrage:Voter ID:server-117aa9d2-21ca-eba3-1d12-6a912f16a347 Address:117aa9d2-21ca-eba3-1d12-6a912f16a347}]" +2025-06-03T23:37:03.132Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b181b3e3-ef8c-9eca-d046-ace67869b333 Address:b181b3e3-ef8c-9eca-d046-ace67869b333} {Suffrage:Voter ID:server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d Address:26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d} {Suffrage:Voter ID:server-117aa9d2-21ca-eba3-1d12-6a912f16a347 Address:117aa9d2-21ca-eba3-1d12-6a912f16a347}]" +2025-06-03T23:37:03.132Z [INFO] server-117aa9d2-21ca-eba3-1d12-6a912f16a347: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b181b3e3-ef8c-9eca-d046-ace67869b333 Address:b181b3e3-ef8c-9eca-d046-ace67869b333} {Suffrage:Voter ID:server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d Address:26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d} {Suffrage:Voter ID:server-117aa9d2-21ca-eba3-1d12-6a912f16a347 Address:117aa9d2-21ca-eba3-1d12-6a912f16a347}]" +2025-06-03T23:37:03.132Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:03.132Z [INFO] server-b181b3e3-ef8c-9eca-d046-ace67869b333: entering follower state: follower="Node at b181b3e3-ef8c-9eca-d046-ace67869b333 [Follower]" leader-address= leader-id= +2025-06-03T23:37:03.132Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: entering follower state: follower="Node at 26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d [Follower]" leader-address= leader-id= +2025-06-03T23:37:03.132Z [INFO] server-117aa9d2-21ca-eba3-1d12-6a912f16a347: entering follower state: follower="Node at 117aa9d2-21ca-eba3-1d12-6a912f16a347 [Follower]" leader-address= leader-id= +2025-06-03T23:37:03.671Z [WARN] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:03.671Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: entering candidate state: node="Node at 26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d [Candidate]" term=2 +2025-06-03T23:37:03.671Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: election won: term=2 tally=2 +2025-06-03T23:37:03.671Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: entering leader state: leader="Node at 26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d [Leader]" +2025-06-03T23:37:03.671Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: added peer, starting replication: peer=server-b181b3e3-ef8c-9eca-d046-ace67869b333 +2025-06-03T23:37:03.671Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: added peer, starting replication: peer=server-117aa9d2-21ca-eba3-1d12-6a912f16a347 +2025-06-03T23:37:03.671Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: pipelining replication: peer="{Voter server-117aa9d2-21ca-eba3-1d12-6a912f16a347 117aa9d2-21ca-eba3-1d12-6a912f16a347}" +2025-06-03T23:37:03.671Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: pipelining replication: peer="{Voter server-b181b3e3-ef8c-9eca-d046-ace67869b333 b181b3e3-ef8c-9eca-d046-ace67869b333}" +2025-06-03T23:37:04.679Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:03.671879631 +0000 UTC m=+58.993634521 (1 nodes), 539.50948ms from start of poll, 539.71892ms from cluster start. Timeout at 2025-06-03 23:37:04.679365191 +0000 UTC m=+60.001120081, 1.00748556s after stability +2025-06-03T23:37:04.679Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: starting snapshot up to: index=12 +2025-06-03T23:37:04.679Z [INFO] creating new snapshot: path=/tmp/raft2857532934/snapshots/2-12-1748993824679.tmp +2025-06-03T23:37:04.680Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: snapshot complete up to: index=12 +2025-06-03T23:37:04.680Z [INFO] creating new snapshot: path=/tmp/raft2857532934/snapshots/2-113-1748993824680.tmp +2025-06-03T23:37:04.680Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: copied to local snapshot: bytes=71 +2025-06-03T23:37:04.681Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: snapshot restore progress: id=2-113-1748993824680 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:04.681Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: removing all old logs from log store +2025-06-03T23:37:04.681Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: compacting logs: from=1 to=22 +2025-06-03T23:37:04.681Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: restored user snapshot: index=113 +2025-06-03T23:37:04.681Z [ERROR] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: failed to get log: index=22 error="log not found" +2025-06-03T23:37:04.681Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: aborting pipeline replication: peer="{Voter server-117aa9d2-21ca-eba3-1d12-6a912f16a347 117aa9d2-21ca-eba3-1d12-6a912f16a347}" +2025-06-03T23:37:04.681Z [ERROR] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: failed to get log: index=22 error="log not found" +2025-06-03T23:37:04.681Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: aborting pipeline replication: peer="{Voter server-b181b3e3-ef8c-9eca-d046-ace67869b333 b181b3e3-ef8c-9eca-d046-ace67869b333}" +2025-06-03T23:37:04.691Z [ERROR] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: failed to get log: index=22 error="log not found" +2025-06-03T23:37:04.691Z [INFO] creating new snapshot: path=/tmp/raft2208577990/snapshots/2-113-1748993824691.tmp +2025-06-03T23:37:04.691Z [INFO] server-b181b3e3-ef8c-9eca-d046-ace67869b333: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:04.691Z [INFO] server-b181b3e3-ef8c-9eca-d046-ace67869b333: copied to local snapshot: bytes=71 +2025-06-03T23:37:04.692Z [INFO] server-b181b3e3-ef8c-9eca-d046-ace67869b333: snapshot restore progress: id=2-113-1748993824691 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:04.692Z [INFO] server-b181b3e3-ef8c-9eca-d046-ace67869b333: removing all old logs from log store +2025-06-03T23:37:04.692Z [INFO] server-b181b3e3-ef8c-9eca-d046-ace67869b333: compacting logs: from=1 to=22 +2025-06-03T23:37:04.692Z [INFO] server-b181b3e3-ef8c-9eca-d046-ace67869b333: Installed remote snapshot +2025-06-03T23:37:04.692Z [WARN] server-b181b3e3-ef8c-9eca-d046-ace67869b333: skipping application of old log: index=22 +2025-06-03T23:37:04.692Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: pipelining replication: peer="{Voter server-b181b3e3-ef8c-9eca-d046-ace67869b333 b181b3e3-ef8c-9eca-d046-ace67869b333}" +2025-06-03T23:37:04.692Z [ERROR] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: failed to get log: index=22 error="log not found" +2025-06-03T23:37:04.692Z [INFO] creating new snapshot: path=/tmp/raft1074500470/snapshots/2-113-1748993824692.tmp +2025-06-03T23:37:04.692Z [INFO] server-117aa9d2-21ca-eba3-1d12-6a912f16a347: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:04.692Z [INFO] server-117aa9d2-21ca-eba3-1d12-6a912f16a347: copied to local snapshot: bytes=71 +2025-06-03T23:37:04.692Z [INFO] server-117aa9d2-21ca-eba3-1d12-6a912f16a347: snapshot restore progress: id=2-113-1748993824692 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:04.692Z [INFO] server-117aa9d2-21ca-eba3-1d12-6a912f16a347: removing all old logs from log store +2025-06-03T23:37:04.692Z [INFO] server-117aa9d2-21ca-eba3-1d12-6a912f16a347: compacting logs: from=1 to=22 +2025-06-03T23:37:04.692Z [INFO] server-117aa9d2-21ca-eba3-1d12-6a912f16a347: Installed remote snapshot +2025-06-03T23:37:04.692Z [WARN] server-117aa9d2-21ca-eba3-1d12-6a912f16a347: skipping application of old log: index=22 +2025-06-03T23:37:04.692Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: pipelining replication: peer="{Voter server-117aa9d2-21ca-eba3-1d12-6a912f16a347 117aa9d2-21ca-eba3-1d12-6a912f16a347}" +2025-06-03T23:37:04.692Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:05.703Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:04.692673711 +0000 UTC m=+60.014428561 (2 nodes), 480ns from start of poll, 1.56051296s from cluster start. Timeout at 2025-06-03 23:37:05.703409711 +0000 UTC m=+61.025164561, 1.010736s after stability +2025-06-03T23:37:05.723Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: aborting pipeline replication: peer="{Voter server-117aa9d2-21ca-eba3-1d12-6a912f16a347 117aa9d2-21ca-eba3-1d12-6a912f16a347}" +2025-06-03T23:37:05.723Z [INFO] server-26dffa0c-c45c-ebe5-7e4c-50bd62cc2f8d: aborting pipeline replication: peer="{Voter server-b181b3e3-ef8c-9eca-d046-ace67869b333 b181b3e3-ef8c-9eca-d046-ace67869b333}" === RUN TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true -2025-06-03T23:34:07.052Z [INFO] server-dbe494bb-0acd-677f-a33a-715f45386ded: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dbe494bb-0acd-677f-a33a-715f45386ded Address:dbe494bb-0acd-677f-a33a-715f45386ded} {Suffrage:Voter ID:server-95327c4a-a31a-1077-958a-b1d64580a138 Address:95327c4a-a31a-1077-958a-b1d64580a138} {Suffrage:Voter ID:server-f1343547-c087-9a00-cb54-fd3f0fa08c5a Address:f1343547-c087-9a00-cb54-fd3f0fa08c5a}]" -2025-06-03T23:34:07.052Z [INFO] server-95327c4a-a31a-1077-958a-b1d64580a138: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dbe494bb-0acd-677f-a33a-715f45386ded Address:dbe494bb-0acd-677f-a33a-715f45386ded} {Suffrage:Voter ID:server-95327c4a-a31a-1077-958a-b1d64580a138 Address:95327c4a-a31a-1077-958a-b1d64580a138} {Suffrage:Voter ID:server-f1343547-c087-9a00-cb54-fd3f0fa08c5a Address:f1343547-c087-9a00-cb54-fd3f0fa08c5a}]" -2025-06-03T23:34:07.052Z [INFO] server-95327c4a-a31a-1077-958a-b1d64580a138: entering follower state: follower="Node at 95327c4a-a31a-1077-958a-b1d64580a138 [Follower]" leader-address= leader-id= -2025-06-03T23:34:07.052Z [INFO] server-dbe494bb-0acd-677f-a33a-715f45386ded: entering follower state: follower="Node at dbe494bb-0acd-677f-a33a-715f45386ded [Follower]" leader-address= leader-id= -2025-06-03T23:34:07.052Z [INFO] server-f1343547-c087-9a00-cb54-fd3f0fa08c5a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dbe494bb-0acd-677f-a33a-715f45386ded Address:dbe494bb-0acd-677f-a33a-715f45386ded} {Suffrage:Voter ID:server-95327c4a-a31a-1077-958a-b1d64580a138 Address:95327c4a-a31a-1077-958a-b1d64580a138} {Suffrage:Voter ID:server-f1343547-c087-9a00-cb54-fd3f0fa08c5a Address:f1343547-c087-9a00-cb54-fd3f0fa08c5a}]" -2025-06-03T23:34:07.052Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:07.052Z [INFO] server-f1343547-c087-9a00-cb54-fd3f0fa08c5a: entering follower state: follower="Node at f1343547-c087-9a00-cb54-fd3f0fa08c5a [Follower]" leader-address= leader-id= -2025-06-03T23:34:07.563Z [WARN] server-dbe494bb-0acd-677f-a33a-715f45386ded: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:07.563Z [INFO] server-dbe494bb-0acd-677f-a33a-715f45386ded: entering candidate state: node="Node at dbe494bb-0acd-677f-a33a-715f45386ded [Candidate]" term=2 -2025-06-03T23:34:07.563Z [INFO] server-dbe494bb-0acd-677f-a33a-715f45386ded: election won: term=2 tally=2 -2025-06-03T23:34:07.563Z [INFO] server-dbe494bb-0acd-677f-a33a-715f45386ded: entering leader state: leader="Node at dbe494bb-0acd-677f-a33a-715f45386ded [Leader]" -2025-06-03T23:34:07.563Z [INFO] server-dbe494bb-0acd-677f-a33a-715f45386ded: added peer, starting replication: peer=server-95327c4a-a31a-1077-958a-b1d64580a138 -2025-06-03T23:34:07.563Z [INFO] server-dbe494bb-0acd-677f-a33a-715f45386ded: added peer, starting replication: peer=server-f1343547-c087-9a00-cb54-fd3f0fa08c5a -2025-06-03T23:34:07.563Z [INFO] server-dbe494bb-0acd-677f-a33a-715f45386ded: pipelining replication: peer="{Voter server-f1343547-c087-9a00-cb54-fd3f0fa08c5a f1343547-c087-9a00-cb54-fd3f0fa08c5a}" -2025-06-03T23:34:07.564Z [INFO] server-dbe494bb-0acd-677f-a33a-715f45386ded: pipelining replication: peer="{Voter server-95327c4a-a31a-1077-958a-b1d64580a138 95327c4a-a31a-1077-958a-b1d64580a138}" -2025-06-03T23:34:08.569Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:07.564149911 +0000 UTC m=+62.584969281 (1 nodes), 511.37288ms from start of poll, 512.03804ms from cluster start. Timeout at 2025-06-03 23:34:08.569158351 +0000 UTC m=+63.589977721, 1.00500844s after stability -2025-06-03T23:34:08.569Z [INFO] server-dbe494bb-0acd-677f-a33a-715f45386ded: starting snapshot up to: index=12 -2025-06-03T23:34:08.569Z [INFO] creating new snapshot: path=/tmp/raft4151178935/snapshots/2-12-1748993648569.tmp -2025-06-03T23:34:08.570Z [INFO] server-dbe494bb-0acd-677f-a33a-715f45386ded: snapshot complete up to: index=12 -2025-06-03T23:34:08.572Z [INFO] server-c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33 Address:c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33} {Suffrage:Voter ID:server-8559895d-58b8-5f7f-6d13-3236dc31b4d7 Address:8559895d-58b8-5f7f-6d13-3236dc31b4d7} {Suffrage:Voter ID:server-2909e79c-7124-1ab9-e9fe-497da28a3d16 Address:2909e79c-7124-1ab9-e9fe-497da28a3d16}]" -2025-06-03T23:34:08.572Z [INFO] server-8559895d-58b8-5f7f-6d13-3236dc31b4d7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33 Address:c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33} {Suffrage:Voter ID:server-8559895d-58b8-5f7f-6d13-3236dc31b4d7 Address:8559895d-58b8-5f7f-6d13-3236dc31b4d7} {Suffrage:Voter ID:server-2909e79c-7124-1ab9-e9fe-497da28a3d16 Address:2909e79c-7124-1ab9-e9fe-497da28a3d16}]" -2025-06-03T23:34:08.572Z [INFO] server-c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33: entering follower state: follower="Node at c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33 [Follower]" leader-address= leader-id= -2025-06-03T23:34:08.572Z [INFO] server-8559895d-58b8-5f7f-6d13-3236dc31b4d7: entering follower state: follower="Node at 8559895d-58b8-5f7f-6d13-3236dc31b4d7 [Follower]" leader-address= leader-id= -2025-06-03T23:34:08.572Z [INFO] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33 Address:c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33} {Suffrage:Voter ID:server-8559895d-58b8-5f7f-6d13-3236dc31b4d7 Address:8559895d-58b8-5f7f-6d13-3236dc31b4d7} {Suffrage:Voter ID:server-2909e79c-7124-1ab9-e9fe-497da28a3d16 Address:2909e79c-7124-1ab9-e9fe-497da28a3d16}]" -2025-06-03T23:34:08.572Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:08.572Z [INFO] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: entering follower state: follower="Node at 2909e79c-7124-1ab9-e9fe-497da28a3d16 [Follower]" leader-address= leader-id= -2025-06-03T23:34:09.149Z [WARN] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:09.149Z [INFO] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: entering candidate state: node="Node at 2909e79c-7124-1ab9-e9fe-497da28a3d16 [Candidate]" term=2 -2025-06-03T23:34:09.149Z [INFO] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: election won: term=2 tally=2 -2025-06-03T23:34:09.149Z [INFO] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: entering leader state: leader="Node at 2909e79c-7124-1ab9-e9fe-497da28a3d16 [Leader]" -2025-06-03T23:34:09.149Z [INFO] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: added peer, starting replication: peer=server-c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33 -2025-06-03T23:34:09.149Z [INFO] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: added peer, starting replication: peer=server-8559895d-58b8-5f7f-6d13-3236dc31b4d7 -2025-06-03T23:34:09.149Z [INFO] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: pipelining replication: peer="{Voter server-c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33 c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33}" -2025-06-03T23:34:09.149Z [INFO] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: pipelining replication: peer="{Voter server-8559895d-58b8-5f7f-6d13-3236dc31b4d7 8559895d-58b8-5f7f-6d13-3236dc31b4d7}" -2025-06-03T23:34:10.155Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:09.149719711 +0000 UTC m=+64.170539081 (1 nodes), 577.02344ms from start of poll, 577.64952ms from cluster start. Timeout at 2025-06-03 23:34:10.155178351 +0000 UTC m=+65.175997761, 1.00545868s after stability -2025-06-03T23:34:10.155Z [INFO] creating new snapshot: path=/tmp/raft3213405769/snapshots/2-113-1748993650155.tmp -2025-06-03T23:34:10.156Z [INFO] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: copied to local snapshot: bytes=71 -2025-06-03T23:34:10.156Z [INFO] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: snapshot restore progress: id=2-113-1748993650155 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:10.156Z [INFO] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: restored user snapshot: index=113 -2025-06-03T23:34:10.156Z [ERROR] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: failed to get log: index=3 error="log not found" -2025-06-03T23:34:10.156Z [INFO] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: aborting pipeline replication: peer="{Voter server-8559895d-58b8-5f7f-6d13-3236dc31b4d7 8559895d-58b8-5f7f-6d13-3236dc31b4d7}" -2025-06-03T23:34:10.156Z [ERROR] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: failed to get log: index=3 error="log not found" -2025-06-03T23:34:10.156Z [INFO] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: aborting pipeline replication: peer="{Voter server-c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33 c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33}" -2025-06-03T23:34:10.163Z [ERROR] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: failed to get log: index=3 error="log not found" -2025-06-03T23:34:10.164Z [ERROR] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: failed to get log: index=3 error="log not found" -2025-06-03T23:34:10.164Z [INFO] creating new snapshot: path=/tmp/raft1552509210/snapshots/2-113-1748993650164.tmp -2025-06-03T23:34:10.164Z [INFO] creating new snapshot: path=/tmp/raft770516331/snapshots/2-113-1748993650164.tmp -2025-06-03T23:34:10.164Z [INFO] server-8559895d-58b8-5f7f-6d13-3236dc31b4d7: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:10.164Z [INFO] server-c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:10.164Z [INFO] server-8559895d-58b8-5f7f-6d13-3236dc31b4d7: copied to local snapshot: bytes=71 -2025-06-03T23:34:10.165Z [INFO] server-c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33: copied to local snapshot: bytes=71 -2025-06-03T23:34:10.165Z [INFO] server-8559895d-58b8-5f7f-6d13-3236dc31b4d7: snapshot restore progress: id=2-113-1748993650164 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:10.165Z [INFO] server-8559895d-58b8-5f7f-6d13-3236dc31b4d7: Installed remote snapshot -2025-06-03T23:34:10.165Z [INFO] server-c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33: snapshot restore progress: id=2-113-1748993650164 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:10.165Z [INFO] server-c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33: Installed remote snapshot -2025-06-03T23:34:10.165Z [INFO] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: pipelining replication: peer="{Voter server-8559895d-58b8-5f7f-6d13-3236dc31b4d7 8559895d-58b8-5f7f-6d13-3236dc31b4d7}" -2025-06-03T23:34:10.165Z [INFO] server-2909e79c-7124-1ab9-e9fe-497da28a3d16: pipelining replication: peer="{Voter server-c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33 c75a9a1c-52eb-c06d-0ee3-e4fa1b8cab33}" -2025-06-03T23:34:10.165Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:11.170Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:10.165498871 +0000 UTC m=+65.186318241 (2 nodes), 1.4µs from start of poll, 1.59342868s from cluster start. Timeout at 2025-06-03 23:34:11.170522751 +0000 UTC m=+66.191342121, 1.00502388s after stability -2025-06-03T23:34:11.182Z [INFO] server-dbe494bb-0acd-677f-a33a-715f45386ded: aborting pipeline replication: peer="{Voter server-95327c4a-a31a-1077-958a-b1d64580a138 95327c4a-a31a-1077-958a-b1d64580a138}" -2025-06-03T23:34:11.182Z [INFO] server-dbe494bb-0acd-677f-a33a-715f45386ded: aborting pipeline replication: peer="{Voter server-f1343547-c087-9a00-cb54-fd3f0fa08c5a f1343547-c087-9a00-cb54-fd3f0fa08c5a}" +2025-06-03T23:37:05.724Z [INFO] server-6678ea24-0eeb-b1ec-47d7-948766eb1439: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6678ea24-0eeb-b1ec-47d7-948766eb1439 Address:6678ea24-0eeb-b1ec-47d7-948766eb1439} {Suffrage:Voter ID:server-14434b59-7053-e299-ad10-3994cd0176e9 Address:14434b59-7053-e299-ad10-3994cd0176e9} {Suffrage:Voter ID:server-e5edd036-e6b3-32f3-1f34-0284d2b42163 Address:e5edd036-e6b3-32f3-1f34-0284d2b42163}]" +2025-06-03T23:37:05.724Z [INFO] server-14434b59-7053-e299-ad10-3994cd0176e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6678ea24-0eeb-b1ec-47d7-948766eb1439 Address:6678ea24-0eeb-b1ec-47d7-948766eb1439} {Suffrage:Voter ID:server-14434b59-7053-e299-ad10-3994cd0176e9 Address:14434b59-7053-e299-ad10-3994cd0176e9} {Suffrage:Voter ID:server-e5edd036-e6b3-32f3-1f34-0284d2b42163 Address:e5edd036-e6b3-32f3-1f34-0284d2b42163}]" +2025-06-03T23:37:05.724Z [INFO] server-e5edd036-e6b3-32f3-1f34-0284d2b42163: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6678ea24-0eeb-b1ec-47d7-948766eb1439 Address:6678ea24-0eeb-b1ec-47d7-948766eb1439} {Suffrage:Voter ID:server-14434b59-7053-e299-ad10-3994cd0176e9 Address:14434b59-7053-e299-ad10-3994cd0176e9} {Suffrage:Voter ID:server-e5edd036-e6b3-32f3-1f34-0284d2b42163 Address:e5edd036-e6b3-32f3-1f34-0284d2b42163}]" +2025-06-03T23:37:05.724Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:05.724Z [INFO] server-6678ea24-0eeb-b1ec-47d7-948766eb1439: entering follower state: follower="Node at 6678ea24-0eeb-b1ec-47d7-948766eb1439 [Follower]" leader-address= leader-id= +2025-06-03T23:37:05.724Z [INFO] server-14434b59-7053-e299-ad10-3994cd0176e9: entering follower state: follower="Node at 14434b59-7053-e299-ad10-3994cd0176e9 [Follower]" leader-address= leader-id= +2025-06-03T23:37:05.724Z [INFO] server-e5edd036-e6b3-32f3-1f34-0284d2b42163: entering follower state: follower="Node at e5edd036-e6b3-32f3-1f34-0284d2b42163 [Follower]" leader-address= leader-id= +2025-06-03T23:37:06.420Z [WARN] server-6678ea24-0eeb-b1ec-47d7-948766eb1439: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:06.420Z [INFO] server-6678ea24-0eeb-b1ec-47d7-948766eb1439: entering candidate state: node="Node at 6678ea24-0eeb-b1ec-47d7-948766eb1439 [Candidate]" term=2 +2025-06-03T23:37:06.420Z [INFO] server-6678ea24-0eeb-b1ec-47d7-948766eb1439: election won: term=2 tally=2 +2025-06-03T23:37:06.420Z [INFO] server-6678ea24-0eeb-b1ec-47d7-948766eb1439: entering leader state: leader="Node at 6678ea24-0eeb-b1ec-47d7-948766eb1439 [Leader]" +2025-06-03T23:37:06.420Z [INFO] server-6678ea24-0eeb-b1ec-47d7-948766eb1439: added peer, starting replication: peer=server-14434b59-7053-e299-ad10-3994cd0176e9 +2025-06-03T23:37:06.420Z [INFO] server-6678ea24-0eeb-b1ec-47d7-948766eb1439: added peer, starting replication: peer=server-e5edd036-e6b3-32f3-1f34-0284d2b42163 +2025-06-03T23:37:06.420Z [INFO] server-6678ea24-0eeb-b1ec-47d7-948766eb1439: pipelining replication: peer="{Voter server-e5edd036-e6b3-32f3-1f34-0284d2b42163 e5edd036-e6b3-32f3-1f34-0284d2b42163}" +2025-06-03T23:37:06.420Z [INFO] server-6678ea24-0eeb-b1ec-47d7-948766eb1439: pipelining replication: peer="{Voter server-14434b59-7053-e299-ad10-3994cd0176e9 14434b59-7053-e299-ad10-3994cd0176e9}" +2025-06-03T23:37:07.427Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:06.420358111 +0000 UTC m=+61.742113001 (1 nodes), 696.09136ms from start of poll, 696.28352ms from cluster start. Timeout at 2025-06-03 23:37:07.427723671 +0000 UTC m=+62.749478561, 1.00736556s after stability +2025-06-03T23:37:07.428Z [INFO] server-6678ea24-0eeb-b1ec-47d7-948766eb1439: starting snapshot up to: index=12 +2025-06-03T23:37:07.428Z [INFO] creating new snapshot: path=/tmp/raft3808758560/snapshots/2-12-1748993827428.tmp +2025-06-03T23:37:07.428Z [INFO] server-6678ea24-0eeb-b1ec-47d7-948766eb1439: snapshot complete up to: index=12 +2025-06-03T23:37:07.429Z [INFO] server-3a19e5f7-9dd7-b6a8-5047-2225a9146d1a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3a19e5f7-9dd7-b6a8-5047-2225a9146d1a Address:3a19e5f7-9dd7-b6a8-5047-2225a9146d1a} {Suffrage:Voter ID:server-d1d552c0-0d67-36d9-4949-e989c5a76aae Address:d1d552c0-0d67-36d9-4949-e989c5a76aae} {Suffrage:Voter ID:server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e Address:59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e}]" +2025-06-03T23:37:07.429Z [INFO] server-d1d552c0-0d67-36d9-4949-e989c5a76aae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3a19e5f7-9dd7-b6a8-5047-2225a9146d1a Address:3a19e5f7-9dd7-b6a8-5047-2225a9146d1a} {Suffrage:Voter ID:server-d1d552c0-0d67-36d9-4949-e989c5a76aae Address:d1d552c0-0d67-36d9-4949-e989c5a76aae} {Suffrage:Voter ID:server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e Address:59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e}]" +2025-06-03T23:37:07.429Z [INFO] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3a19e5f7-9dd7-b6a8-5047-2225a9146d1a Address:3a19e5f7-9dd7-b6a8-5047-2225a9146d1a} {Suffrage:Voter ID:server-d1d552c0-0d67-36d9-4949-e989c5a76aae Address:d1d552c0-0d67-36d9-4949-e989c5a76aae} {Suffrage:Voter ID:server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e Address:59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e}]" +2025-06-03T23:37:07.429Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:07.429Z [INFO] server-3a19e5f7-9dd7-b6a8-5047-2225a9146d1a: entering follower state: follower="Node at 3a19e5f7-9dd7-b6a8-5047-2225a9146d1a [Follower]" leader-address= leader-id= +2025-06-03T23:37:07.429Z [INFO] server-d1d552c0-0d67-36d9-4949-e989c5a76aae: entering follower state: follower="Node at d1d552c0-0d67-36d9-4949-e989c5a76aae [Follower]" leader-address= leader-id= +2025-06-03T23:37:07.429Z [INFO] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: entering follower state: follower="Node at 59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e [Follower]" leader-address= leader-id= +2025-06-03T23:37:07.947Z [WARN] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:07.947Z [INFO] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: entering candidate state: node="Node at 59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e [Candidate]" term=2 +2025-06-03T23:37:07.947Z [INFO] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: election won: term=2 tally=2 +2025-06-03T23:37:07.947Z [INFO] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: entering leader state: leader="Node at 59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e [Leader]" +2025-06-03T23:37:07.947Z [INFO] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: added peer, starting replication: peer=server-3a19e5f7-9dd7-b6a8-5047-2225a9146d1a +2025-06-03T23:37:07.947Z [INFO] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: added peer, starting replication: peer=server-d1d552c0-0d67-36d9-4949-e989c5a76aae +2025-06-03T23:37:07.947Z [INFO] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: pipelining replication: peer="{Voter server-d1d552c0-0d67-36d9-4949-e989c5a76aae d1d552c0-0d67-36d9-4949-e989c5a76aae}" +2025-06-03T23:37:07.947Z [INFO] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: pipelining replication: peer="{Voter server-3a19e5f7-9dd7-b6a8-5047-2225a9146d1a 3a19e5f7-9dd7-b6a8-5047-2225a9146d1a}" +2025-06-03T23:37:08.955Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:07.947878231 +0000 UTC m=+63.269633081 (1 nodes), 518.55388ms from start of poll, 518.7436ms from cluster start. Timeout at 2025-06-03 23:37:08.955474951 +0000 UTC m=+64.277229841, 1.00759676s after stability +2025-06-03T23:37:08.958Z [INFO] creating new snapshot: path=/tmp/raft1259906959/snapshots/2-113-1748993828958.tmp +2025-06-03T23:37:08.958Z [INFO] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: copied to local snapshot: bytes=71 +2025-06-03T23:37:08.958Z [INFO] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: snapshot restore progress: id=2-113-1748993828958 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:08.959Z [INFO] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: restored user snapshot: index=113 +2025-06-03T23:37:08.959Z [ERROR] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: failed to get log: index=3 error="log not found" +2025-06-03T23:37:08.959Z [INFO] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: aborting pipeline replication: peer="{Voter server-d1d552c0-0d67-36d9-4949-e989c5a76aae d1d552c0-0d67-36d9-4949-e989c5a76aae}" +2025-06-03T23:37:08.959Z [ERROR] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: failed to get log: index=3 error="log not found" +2025-06-03T23:37:08.959Z [INFO] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: aborting pipeline replication: peer="{Voter server-3a19e5f7-9dd7-b6a8-5047-2225a9146d1a 3a19e5f7-9dd7-b6a8-5047-2225a9146d1a}" +2025-06-03T23:37:08.967Z [ERROR] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: failed to get log: index=3 error="log not found" +2025-06-03T23:37:08.967Z [ERROR] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: failed to get log: index=3 error="log not found" +2025-06-03T23:37:08.967Z [INFO] creating new snapshot: path=/tmp/raft1143269742/snapshots/2-113-1748993828967.tmp +2025-06-03T23:37:08.967Z [INFO] server-d1d552c0-0d67-36d9-4949-e989c5a76aae: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:08.968Z [INFO] server-d1d552c0-0d67-36d9-4949-e989c5a76aae: copied to local snapshot: bytes=71 +2025-06-03T23:37:08.968Z [INFO] creating new snapshot: path=/tmp/raft1186463630/snapshots/2-113-1748993828968.tmp +2025-06-03T23:37:08.968Z [INFO] server-3a19e5f7-9dd7-b6a8-5047-2225a9146d1a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:08.968Z [INFO] server-3a19e5f7-9dd7-b6a8-5047-2225a9146d1a: copied to local snapshot: bytes=71 +2025-06-03T23:37:08.968Z [INFO] server-3a19e5f7-9dd7-b6a8-5047-2225a9146d1a: snapshot restore progress: id=2-113-1748993828968 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:08.968Z [INFO] server-3a19e5f7-9dd7-b6a8-5047-2225a9146d1a: Installed remote snapshot +2025-06-03T23:37:08.968Z [INFO] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: pipelining replication: peer="{Voter server-3a19e5f7-9dd7-b6a8-5047-2225a9146d1a 3a19e5f7-9dd7-b6a8-5047-2225a9146d1a}" +2025-06-03T23:37:08.968Z [INFO] server-d1d552c0-0d67-36d9-4949-e989c5a76aae: snapshot restore progress: id=2-113-1748993828967 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:08.968Z [INFO] server-d1d552c0-0d67-36d9-4949-e989c5a76aae: Installed remote snapshot +2025-06-03T23:37:08.968Z [INFO] server-59e86d2a-ef4f-7c9e-cdc0-846b7f04e42e: pipelining replication: peer="{Voter server-d1d552c0-0d67-36d9-4949-e989c5a76aae d1d552c0-0d67-36d9-4949-e989c5a76aae}" +2025-06-03T23:37:08.968Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:09.974Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:08.968857551 +0000 UTC m=+64.290612441 (2 nodes), 520ns from start of poll, 1.53972296s from cluster start. Timeout at 2025-06-03 23:37:09.974774631 +0000 UTC m=+65.296529561, 1.00591712s after stability +2025-06-03T23:37:09.986Z [INFO] server-6678ea24-0eeb-b1ec-47d7-948766eb1439: aborting pipeline replication: peer="{Voter server-e5edd036-e6b3-32f3-1f34-0284d2b42163 e5edd036-e6b3-32f3-1f34-0284d2b42163}" +2025-06-03T23:37:09.986Z [INFO] server-6678ea24-0eeb-b1ec-47d7-948766eb1439: aborting pipeline replication: peer="{Voter server-14434b59-7053-e299-ad10-3994cd0176e9 14434b59-7053-e299-ad10-3994cd0176e9}" === RUN TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true -2025-06-03T23:34:11.183Z [INFO] server-dea72d8d-1f3d-e571-e358-220b94978928: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dea72d8d-1f3d-e571-e358-220b94978928 Address:dea72d8d-1f3d-e571-e358-220b94978928} {Suffrage:Voter ID:server-5aa0b8a2-07a4-a461-bb82-2a427bccabd5 Address:5aa0b8a2-07a4-a461-bb82-2a427bccabd5} {Suffrage:Voter ID:server-8f720788-a3d9-3bcb-ed74-3086b6d9cba3 Address:8f720788-a3d9-3bcb-ed74-3086b6d9cba3}]" -2025-06-03T23:34:11.183Z [INFO] server-dea72d8d-1f3d-e571-e358-220b94978928: entering follower state: follower="Node at dea72d8d-1f3d-e571-e358-220b94978928 [Follower]" leader-address= leader-id= -2025-06-03T23:34:11.183Z [INFO] server-5aa0b8a2-07a4-a461-bb82-2a427bccabd5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dea72d8d-1f3d-e571-e358-220b94978928 Address:dea72d8d-1f3d-e571-e358-220b94978928} {Suffrage:Voter ID:server-5aa0b8a2-07a4-a461-bb82-2a427bccabd5 Address:5aa0b8a2-07a4-a461-bb82-2a427bccabd5} {Suffrage:Voter ID:server-8f720788-a3d9-3bcb-ed74-3086b6d9cba3 Address:8f720788-a3d9-3bcb-ed74-3086b6d9cba3}]" -2025-06-03T23:34:11.184Z [INFO] server-5aa0b8a2-07a4-a461-bb82-2a427bccabd5: entering follower state: follower="Node at 5aa0b8a2-07a4-a461-bb82-2a427bccabd5 [Follower]" leader-address= leader-id= -2025-06-03T23:34:11.184Z [INFO] server-8f720788-a3d9-3bcb-ed74-3086b6d9cba3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dea72d8d-1f3d-e571-e358-220b94978928 Address:dea72d8d-1f3d-e571-e358-220b94978928} {Suffrage:Voter ID:server-5aa0b8a2-07a4-a461-bb82-2a427bccabd5 Address:5aa0b8a2-07a4-a461-bb82-2a427bccabd5} {Suffrage:Voter ID:server-8f720788-a3d9-3bcb-ed74-3086b6d9cba3 Address:8f720788-a3d9-3bcb-ed74-3086b6d9cba3}]" -2025-06-03T23:34:11.184Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:11.184Z [INFO] server-8f720788-a3d9-3bcb-ed74-3086b6d9cba3: entering follower state: follower="Node at 8f720788-a3d9-3bcb-ed74-3086b6d9cba3 [Follower]" leader-address= leader-id= -2025-06-03T23:34:11.988Z [WARN] server-8f720788-a3d9-3bcb-ed74-3086b6d9cba3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:11.988Z [INFO] server-8f720788-a3d9-3bcb-ed74-3086b6d9cba3: entering candidate state: node="Node at 8f720788-a3d9-3bcb-ed74-3086b6d9cba3 [Candidate]" term=2 -2025-06-03T23:34:11.988Z [INFO] server-8f720788-a3d9-3bcb-ed74-3086b6d9cba3: election won: term=2 tally=2 -2025-06-03T23:34:11.988Z [INFO] server-8f720788-a3d9-3bcb-ed74-3086b6d9cba3: entering leader state: leader="Node at 8f720788-a3d9-3bcb-ed74-3086b6d9cba3 [Leader]" -2025-06-03T23:34:11.988Z [INFO] server-8f720788-a3d9-3bcb-ed74-3086b6d9cba3: added peer, starting replication: peer=server-dea72d8d-1f3d-e571-e358-220b94978928 -2025-06-03T23:34:11.988Z [INFO] server-8f720788-a3d9-3bcb-ed74-3086b6d9cba3: added peer, starting replication: peer=server-5aa0b8a2-07a4-a461-bb82-2a427bccabd5 -2025-06-03T23:34:11.988Z [INFO] server-8f720788-a3d9-3bcb-ed74-3086b6d9cba3: pipelining replication: peer="{Voter server-5aa0b8a2-07a4-a461-bb82-2a427bccabd5 5aa0b8a2-07a4-a461-bb82-2a427bccabd5}" -2025-06-03T23:34:11.988Z [INFO] server-8f720788-a3d9-3bcb-ed74-3086b6d9cba3: pipelining replication: peer="{Voter server-dea72d8d-1f3d-e571-e358-220b94978928 dea72d8d-1f3d-e571-e358-220b94978928}" -2025-06-03T23:34:12.994Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:11.988580951 +0000 UTC m=+67.009400321 (1 nodes), 804.37536ms from start of poll, 804.91116ms from cluster start. Timeout at 2025-06-03 23:34:12.994591031 +0000 UTC m=+68.015410401, 1.00601008s after stability -2025-06-03T23:34:12.995Z [INFO] server-8f720788-a3d9-3bcb-ed74-3086b6d9cba3: starting snapshot up to: index=12 -2025-06-03T23:34:12.995Z [INFO] creating new snapshot: path=/tmp/raft2658113724/snapshots/2-12-1748993652995.tmp -2025-06-03T23:34:12.995Z [INFO] server-8f720788-a3d9-3bcb-ed74-3086b6d9cba3: snapshot complete up to: index=12 -2025-06-03T23:34:12.996Z [INFO] server-91d99953-db50-deaf-d0e6-463fcf51d20b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-91d99953-db50-deaf-d0e6-463fcf51d20b Address:91d99953-db50-deaf-d0e6-463fcf51d20b} {Suffrage:Voter ID:server-43589ba9-4223-6436-249c-338f179e1931 Address:43589ba9-4223-6436-249c-338f179e1931} {Suffrage:Voter ID:server-b2fbd50a-22af-ab85-5044-77d64f6ac249 Address:b2fbd50a-22af-ab85-5044-77d64f6ac249}]" -2025-06-03T23:34:12.997Z [INFO] server-43589ba9-4223-6436-249c-338f179e1931: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-91d99953-db50-deaf-d0e6-463fcf51d20b Address:91d99953-db50-deaf-d0e6-463fcf51d20b} {Suffrage:Voter ID:server-43589ba9-4223-6436-249c-338f179e1931 Address:43589ba9-4223-6436-249c-338f179e1931} {Suffrage:Voter ID:server-b2fbd50a-22af-ab85-5044-77d64f6ac249 Address:b2fbd50a-22af-ab85-5044-77d64f6ac249}]" -2025-06-03T23:34:12.997Z [INFO] server-43589ba9-4223-6436-249c-338f179e1931: entering follower state: follower="Node at 43589ba9-4223-6436-249c-338f179e1931 [Follower]" leader-address= leader-id= -2025-06-03T23:34:12.997Z [INFO] server-b2fbd50a-22af-ab85-5044-77d64f6ac249: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-91d99953-db50-deaf-d0e6-463fcf51d20b Address:91d99953-db50-deaf-d0e6-463fcf51d20b} {Suffrage:Voter ID:server-43589ba9-4223-6436-249c-338f179e1931 Address:43589ba9-4223-6436-249c-338f179e1931} {Suffrage:Voter ID:server-b2fbd50a-22af-ab85-5044-77d64f6ac249 Address:b2fbd50a-22af-ab85-5044-77d64f6ac249}]" -2025-06-03T23:34:12.997Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:12.997Z [INFO] server-91d99953-db50-deaf-d0e6-463fcf51d20b: entering follower state: follower="Node at 91d99953-db50-deaf-d0e6-463fcf51d20b [Follower]" leader-address= leader-id= -2025-06-03T23:34:12.997Z [INFO] server-b2fbd50a-22af-ab85-5044-77d64f6ac249: entering follower state: follower="Node at b2fbd50a-22af-ab85-5044-77d64f6ac249 [Follower]" leader-address= leader-id= -2025-06-03T23:34:13.618Z [WARN] server-43589ba9-4223-6436-249c-338f179e1931: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:13.618Z [INFO] server-43589ba9-4223-6436-249c-338f179e1931: entering candidate state: node="Node at 43589ba9-4223-6436-249c-338f179e1931 [Candidate]" term=2 -2025-06-03T23:34:13.619Z [INFO] server-43589ba9-4223-6436-249c-338f179e1931: election won: term=2 tally=2 -2025-06-03T23:34:13.619Z [INFO] server-43589ba9-4223-6436-249c-338f179e1931: entering leader state: leader="Node at 43589ba9-4223-6436-249c-338f179e1931 [Leader]" -2025-06-03T23:34:13.619Z [INFO] server-43589ba9-4223-6436-249c-338f179e1931: added peer, starting replication: peer=server-91d99953-db50-deaf-d0e6-463fcf51d20b -2025-06-03T23:34:13.619Z [INFO] server-43589ba9-4223-6436-249c-338f179e1931: added peer, starting replication: peer=server-b2fbd50a-22af-ab85-5044-77d64f6ac249 -2025-06-03T23:34:13.619Z [INFO] server-43589ba9-4223-6436-249c-338f179e1931: pipelining replication: peer="{Voter server-b2fbd50a-22af-ab85-5044-77d64f6ac249 b2fbd50a-22af-ab85-5044-77d64f6ac249}" -2025-06-03T23:34:13.619Z [INFO] server-43589ba9-4223-6436-249c-338f179e1931: pipelining replication: peer="{Voter server-91d99953-db50-deaf-d0e6-463fcf51d20b 91d99953-db50-deaf-d0e6-463fcf51d20b}" -2025-06-03T23:34:14.625Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:13.619220231 +0000 UTC m=+68.640039601 (1 nodes), 621.96704ms from start of poll, 622.45188ms from cluster start. Timeout at 2025-06-03 23:34:14.625273951 +0000 UTC m=+69.646093321, 1.00605372s after stability -2025-06-03T23:34:14.625Z [INFO] creating new snapshot: path=/tmp/raft3413885566/snapshots/2-113-1748993654625.tmp -2025-06-03T23:34:14.626Z [INFO] server-43589ba9-4223-6436-249c-338f179e1931: copied to local snapshot: bytes=71 -2025-06-03T23:34:14.634Z [INFO] server-43589ba9-4223-6436-249c-338f179e1931: snapshot restore progress: id=2-113-1748993654625 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:14.634Z [INFO] server-43589ba9-4223-6436-249c-338f179e1931: removing all old logs from log store -2025-06-03T23:34:14.634Z [INFO] server-43589ba9-4223-6436-249c-338f179e1931: compacting logs: from=1 to=2 -2025-06-03T23:34:14.635Z [INFO] server-43589ba9-4223-6436-249c-338f179e1931: restored user snapshot: index=113 -2025-06-03T23:34:14.635Z [ERROR] server-43589ba9-4223-6436-249c-338f179e1931: failed to get log: index=2 error="log not found" -2025-06-03T23:34:14.635Z [INFO] server-43589ba9-4223-6436-249c-338f179e1931: aborting pipeline replication: peer="{Voter server-b2fbd50a-22af-ab85-5044-77d64f6ac249 b2fbd50a-22af-ab85-5044-77d64f6ac249}" -2025-06-03T23:34:14.635Z [ERROR] server-43589ba9-4223-6436-249c-338f179e1931: failed to get log: index=2 error="log not found" -2025-06-03T23:34:14.635Z [INFO] server-43589ba9-4223-6436-249c-338f179e1931: aborting pipeline replication: peer="{Voter server-91d99953-db50-deaf-d0e6-463fcf51d20b 91d99953-db50-deaf-d0e6-463fcf51d20b}" -2025-06-03T23:34:14.643Z [ERROR] server-43589ba9-4223-6436-249c-338f179e1931: failed to get log: index=2 error="log not found" -2025-06-03T23:34:14.643Z [INFO] creating new snapshot: path=/tmp/raft1804512857/snapshots/2-113-1748993654643.tmp -2025-06-03T23:34:14.643Z [INFO] server-b2fbd50a-22af-ab85-5044-77d64f6ac249: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:14.644Z [INFO] server-b2fbd50a-22af-ab85-5044-77d64f6ac249: copied to local snapshot: bytes=71 -2025-06-03T23:34:14.644Z [INFO] server-b2fbd50a-22af-ab85-5044-77d64f6ac249: snapshot restore progress: id=2-113-1748993654643 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:14.644Z [INFO] server-b2fbd50a-22af-ab85-5044-77d64f6ac249: removing all old logs from log store -2025-06-03T23:34:14.644Z [INFO] server-b2fbd50a-22af-ab85-5044-77d64f6ac249: compacting logs: from=1 to=2 -2025-06-03T23:34:14.644Z [INFO] server-b2fbd50a-22af-ab85-5044-77d64f6ac249: Installed remote snapshot -2025-06-03T23:34:14.644Z [INFO] server-43589ba9-4223-6436-249c-338f179e1931: pipelining replication: peer="{Voter server-b2fbd50a-22af-ab85-5044-77d64f6ac249 b2fbd50a-22af-ab85-5044-77d64f6ac249}" -2025-06-03T23:34:14.644Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:14.644Z [ERROR] server-43589ba9-4223-6436-249c-338f179e1931: failed to get log: index=2 error="log not found" -2025-06-03T23:34:14.645Z [INFO] creating new snapshot: path=/tmp/raft879260169/snapshots/2-113-1748993654645.tmp -2025-06-03T23:34:14.645Z [INFO] server-91d99953-db50-deaf-d0e6-463fcf51d20b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:14.645Z [INFO] server-91d99953-db50-deaf-d0e6-463fcf51d20b: copied to local snapshot: bytes=71 -2025-06-03T23:34:14.645Z [INFO] server-91d99953-db50-deaf-d0e6-463fcf51d20b: snapshot restore progress: id=2-113-1748993654645 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:14.645Z [INFO] server-91d99953-db50-deaf-d0e6-463fcf51d20b: removing all old logs from log store -2025-06-03T23:34:14.645Z [INFO] server-91d99953-db50-deaf-d0e6-463fcf51d20b: compacting logs: from=1 to=2 -2025-06-03T23:34:14.645Z [INFO] server-91d99953-db50-deaf-d0e6-463fcf51d20b: Installed remote snapshot -2025-06-03T23:34:14.646Z [INFO] server-43589ba9-4223-6436-249c-338f179e1931: pipelining replication: peer="{Voter server-91d99953-db50-deaf-d0e6-463fcf51d20b 91d99953-db50-deaf-d0e6-463fcf51d20b}" -2025-06-03T23:34:15.649Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:14.644721711 +0000 UTC m=+69.665541081 (2 nodes), 1.08µs from start of poll, 1.64795336s from cluster start. Timeout at 2025-06-03 23:34:15.649829871 +0000 UTC m=+70.670649241, 1.00510816s after stability -2025-06-03T23:34:15.662Z [INFO] server-8f720788-a3d9-3bcb-ed74-3086b6d9cba3: aborting pipeline replication: peer="{Voter server-5aa0b8a2-07a4-a461-bb82-2a427bccabd5 5aa0b8a2-07a4-a461-bb82-2a427bccabd5}" -2025-06-03T23:34:15.662Z [INFO] server-8f720788-a3d9-3bcb-ed74-3086b6d9cba3: aborting pipeline replication: peer="{Voter server-dea72d8d-1f3d-e571-e358-220b94978928 dea72d8d-1f3d-e571-e358-220b94978928}" +2025-06-03T23:37:09.987Z [INFO] server-ab55a806-fb1e-2956-99ba-c4e5bd2c8625: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ab55a806-fb1e-2956-99ba-c4e5bd2c8625 Address:ab55a806-fb1e-2956-99ba-c4e5bd2c8625} {Suffrage:Voter ID:server-eb96da49-488b-7697-c785-875ba4b703ed Address:eb96da49-488b-7697-c785-875ba4b703ed} {Suffrage:Voter ID:server-f81da1ce-24e7-69f5-3f2f-3c27dad8d8e3 Address:f81da1ce-24e7-69f5-3f2f-3c27dad8d8e3}]" +2025-06-03T23:37:09.987Z [INFO] server-ab55a806-fb1e-2956-99ba-c4e5bd2c8625: entering follower state: follower="Node at ab55a806-fb1e-2956-99ba-c4e5bd2c8625 [Follower]" leader-address= leader-id= +2025-06-03T23:37:09.987Z [INFO] server-eb96da49-488b-7697-c785-875ba4b703ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ab55a806-fb1e-2956-99ba-c4e5bd2c8625 Address:ab55a806-fb1e-2956-99ba-c4e5bd2c8625} {Suffrage:Voter ID:server-eb96da49-488b-7697-c785-875ba4b703ed Address:eb96da49-488b-7697-c785-875ba4b703ed} {Suffrage:Voter ID:server-f81da1ce-24e7-69f5-3f2f-3c27dad8d8e3 Address:f81da1ce-24e7-69f5-3f2f-3c27dad8d8e3}]" +2025-06-03T23:37:09.987Z [INFO] server-eb96da49-488b-7697-c785-875ba4b703ed: entering follower state: follower="Node at eb96da49-488b-7697-c785-875ba4b703ed [Follower]" leader-address= leader-id= +2025-06-03T23:37:09.987Z [INFO] server-f81da1ce-24e7-69f5-3f2f-3c27dad8d8e3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ab55a806-fb1e-2956-99ba-c4e5bd2c8625 Address:ab55a806-fb1e-2956-99ba-c4e5bd2c8625} {Suffrage:Voter ID:server-eb96da49-488b-7697-c785-875ba4b703ed Address:eb96da49-488b-7697-c785-875ba4b703ed} {Suffrage:Voter ID:server-f81da1ce-24e7-69f5-3f2f-3c27dad8d8e3 Address:f81da1ce-24e7-69f5-3f2f-3c27dad8d8e3}]" +2025-06-03T23:37:09.987Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:09.987Z [INFO] server-f81da1ce-24e7-69f5-3f2f-3c27dad8d8e3: entering follower state: follower="Node at f81da1ce-24e7-69f5-3f2f-3c27dad8d8e3 [Follower]" leader-address= leader-id= +2025-06-03T23:37:10.554Z [WARN] server-ab55a806-fb1e-2956-99ba-c4e5bd2c8625: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:10.554Z [INFO] server-ab55a806-fb1e-2956-99ba-c4e5bd2c8625: entering candidate state: node="Node at ab55a806-fb1e-2956-99ba-c4e5bd2c8625 [Candidate]" term=2 +2025-06-03T23:37:10.554Z [INFO] server-ab55a806-fb1e-2956-99ba-c4e5bd2c8625: election won: term=2 tally=2 +2025-06-03T23:37:10.554Z [INFO] server-ab55a806-fb1e-2956-99ba-c4e5bd2c8625: entering leader state: leader="Node at ab55a806-fb1e-2956-99ba-c4e5bd2c8625 [Leader]" +2025-06-03T23:37:10.554Z [INFO] server-ab55a806-fb1e-2956-99ba-c4e5bd2c8625: added peer, starting replication: peer=server-eb96da49-488b-7697-c785-875ba4b703ed +2025-06-03T23:37:10.554Z [INFO] server-ab55a806-fb1e-2956-99ba-c4e5bd2c8625: added peer, starting replication: peer=server-f81da1ce-24e7-69f5-3f2f-3c27dad8d8e3 +2025-06-03T23:37:10.555Z [INFO] server-ab55a806-fb1e-2956-99ba-c4e5bd2c8625: pipelining replication: peer="{Voter server-eb96da49-488b-7697-c785-875ba4b703ed eb96da49-488b-7697-c785-875ba4b703ed}" +2025-06-03T23:37:10.555Z [INFO] server-ab55a806-fb1e-2956-99ba-c4e5bd2c8625: pipelining replication: peer="{Voter server-f81da1ce-24e7-69f5-3f2f-3c27dad8d8e3 f81da1ce-24e7-69f5-3f2f-3c27dad8d8e3}" +2025-06-03T23:37:11.560Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:10.554926151 +0000 UTC m=+65.876681121 (1 nodes), 567.57404ms from start of poll, 568.0662ms from cluster start. Timeout at 2025-06-03 23:37:11.559949191 +0000 UTC m=+66.881704121, 1.005023s after stability +2025-06-03T23:37:11.560Z [INFO] server-ab55a806-fb1e-2956-99ba-c4e5bd2c8625: starting snapshot up to: index=12 +2025-06-03T23:37:11.560Z [INFO] creating new snapshot: path=/tmp/raft3468501649/snapshots/2-12-1748993831560.tmp +2025-06-03T23:37:11.561Z [INFO] server-ab55a806-fb1e-2956-99ba-c4e5bd2c8625: snapshot complete up to: index=12 +2025-06-03T23:37:11.562Z [INFO] server-52e611d7-ecba-abd7-9d91-d611b22d30b4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52e611d7-ecba-abd7-9d91-d611b22d30b4 Address:52e611d7-ecba-abd7-9d91-d611b22d30b4} {Suffrage:Voter ID:server-53c865bc-ee24-934e-922e-f2729608c6a4 Address:53c865bc-ee24-934e-922e-f2729608c6a4} {Suffrage:Voter ID:server-9ce2561a-bf83-5e04-a00a-9b07785afd3a Address:9ce2561a-bf83-5e04-a00a-9b07785afd3a}]" +2025-06-03T23:37:11.562Z [INFO] server-53c865bc-ee24-934e-922e-f2729608c6a4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52e611d7-ecba-abd7-9d91-d611b22d30b4 Address:52e611d7-ecba-abd7-9d91-d611b22d30b4} {Suffrage:Voter ID:server-53c865bc-ee24-934e-922e-f2729608c6a4 Address:53c865bc-ee24-934e-922e-f2729608c6a4} {Suffrage:Voter ID:server-9ce2561a-bf83-5e04-a00a-9b07785afd3a Address:9ce2561a-bf83-5e04-a00a-9b07785afd3a}]" +2025-06-03T23:37:11.562Z [INFO] server-52e611d7-ecba-abd7-9d91-d611b22d30b4: entering follower state: follower="Node at 52e611d7-ecba-abd7-9d91-d611b22d30b4 [Follower]" leader-address= leader-id= +2025-06-03T23:37:11.562Z [INFO] server-53c865bc-ee24-934e-922e-f2729608c6a4: entering follower state: follower="Node at 53c865bc-ee24-934e-922e-f2729608c6a4 [Follower]" leader-address= leader-id= +2025-06-03T23:37:11.562Z [INFO] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52e611d7-ecba-abd7-9d91-d611b22d30b4 Address:52e611d7-ecba-abd7-9d91-d611b22d30b4} {Suffrage:Voter ID:server-53c865bc-ee24-934e-922e-f2729608c6a4 Address:53c865bc-ee24-934e-922e-f2729608c6a4} {Suffrage:Voter ID:server-9ce2561a-bf83-5e04-a00a-9b07785afd3a Address:9ce2561a-bf83-5e04-a00a-9b07785afd3a}]" +2025-06-03T23:37:11.562Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:11.562Z [INFO] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: entering follower state: follower="Node at 9ce2561a-bf83-5e04-a00a-9b07785afd3a [Follower]" leader-address= leader-id= +2025-06-03T23:37:12.142Z [WARN] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:12.142Z [INFO] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: entering candidate state: node="Node at 9ce2561a-bf83-5e04-a00a-9b07785afd3a [Candidate]" term=2 +2025-06-03T23:37:12.142Z [INFO] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: election won: term=2 tally=2 +2025-06-03T23:37:12.142Z [INFO] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: entering leader state: leader="Node at 9ce2561a-bf83-5e04-a00a-9b07785afd3a [Leader]" +2025-06-03T23:37:12.142Z [INFO] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: added peer, starting replication: peer=server-52e611d7-ecba-abd7-9d91-d611b22d30b4 +2025-06-03T23:37:12.142Z [INFO] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: added peer, starting replication: peer=server-53c865bc-ee24-934e-922e-f2729608c6a4 +2025-06-03T23:37:12.143Z [INFO] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: pipelining replication: peer="{Voter server-53c865bc-ee24-934e-922e-f2729608c6a4 53c865bc-ee24-934e-922e-f2729608c6a4}" +2025-06-03T23:37:12.143Z [INFO] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: pipelining replication: peer="{Voter server-52e611d7-ecba-abd7-9d91-d611b22d30b4 52e611d7-ecba-abd7-9d91-d611b22d30b4}" +2025-06-03T23:37:13.149Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:12.142950471 +0000 UTC m=+67.464705401 (1 nodes), 580.44332ms from start of poll, 580.91632ms from cluster start. Timeout at 2025-06-03 23:37:13.148983631 +0000 UTC m=+68.470738601, 1.0060332s after stability +2025-06-03T23:37:13.149Z [INFO] creating new snapshot: path=/tmp/raft1269832463/snapshots/2-113-1748993833149.tmp +2025-06-03T23:37:13.149Z [INFO] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: copied to local snapshot: bytes=71 +2025-06-03T23:37:13.150Z [INFO] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: snapshot restore progress: id=2-113-1748993833149 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:13.150Z [INFO] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: removing all old logs from log store +2025-06-03T23:37:13.150Z [INFO] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: compacting logs: from=1 to=2 +2025-06-03T23:37:13.150Z [INFO] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: restored user snapshot: index=113 +2025-06-03T23:37:13.150Z [ERROR] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: failed to get log: index=2 error="log not found" +2025-06-03T23:37:13.150Z [INFO] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: aborting pipeline replication: peer="{Voter server-53c865bc-ee24-934e-922e-f2729608c6a4 53c865bc-ee24-934e-922e-f2729608c6a4}" +2025-06-03T23:37:13.150Z [ERROR] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: failed to get log: index=2 error="log not found" +2025-06-03T23:37:13.150Z [INFO] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: aborting pipeline replication: peer="{Voter server-52e611d7-ecba-abd7-9d91-d611b22d30b4 52e611d7-ecba-abd7-9d91-d611b22d30b4}" +2025-06-03T23:37:13.157Z [ERROR] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: failed to get log: index=2 error="log not found" +2025-06-03T23:37:13.157Z [INFO] creating new snapshot: path=/tmp/raft491855921/snapshots/2-113-1748993833157.tmp +2025-06-03T23:37:13.158Z [INFO] server-52e611d7-ecba-abd7-9d91-d611b22d30b4: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:13.158Z [INFO] server-52e611d7-ecba-abd7-9d91-d611b22d30b4: copied to local snapshot: bytes=71 +2025-06-03T23:37:13.158Z [INFO] server-52e611d7-ecba-abd7-9d91-d611b22d30b4: snapshot restore progress: id=2-113-1748993833157 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:13.158Z [INFO] server-52e611d7-ecba-abd7-9d91-d611b22d30b4: removing all old logs from log store +2025-06-03T23:37:13.158Z [INFO] server-52e611d7-ecba-abd7-9d91-d611b22d30b4: compacting logs: from=1 to=2 +2025-06-03T23:37:13.158Z [INFO] server-52e611d7-ecba-abd7-9d91-d611b22d30b4: Installed remote snapshot +2025-06-03T23:37:13.158Z [WARN] server-52e611d7-ecba-abd7-9d91-d611b22d30b4: skipping application of old log: index=113 +2025-06-03T23:37:13.158Z [INFO] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: pipelining replication: peer="{Voter server-52e611d7-ecba-abd7-9d91-d611b22d30b4 52e611d7-ecba-abd7-9d91-d611b22d30b4}" +2025-06-03T23:37:13.159Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:13.161Z [ERROR] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: failed to get log: index=2 error="log not found" +2025-06-03T23:37:13.161Z [INFO] creating new snapshot: path=/tmp/raft1692379042/snapshots/2-113-1748993833161.tmp +2025-06-03T23:37:13.161Z [INFO] server-53c865bc-ee24-934e-922e-f2729608c6a4: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:13.162Z [INFO] server-53c865bc-ee24-934e-922e-f2729608c6a4: copied to local snapshot: bytes=71 +2025-06-03T23:37:13.162Z [INFO] server-53c865bc-ee24-934e-922e-f2729608c6a4: snapshot restore progress: id=2-113-1748993833161 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:13.162Z [INFO] server-53c865bc-ee24-934e-922e-f2729608c6a4: removing all old logs from log store +2025-06-03T23:37:13.162Z [INFO] server-53c865bc-ee24-934e-922e-f2729608c6a4: compacting logs: from=1 to=2 +2025-06-03T23:37:13.162Z [INFO] server-53c865bc-ee24-934e-922e-f2729608c6a4: Installed remote snapshot +2025-06-03T23:37:13.162Z [INFO] server-9ce2561a-bf83-5e04-a00a-9b07785afd3a: pipelining replication: peer="{Voter server-53c865bc-ee24-934e-922e-f2729608c6a4 53c865bc-ee24-934e-922e-f2729608c6a4}" +2025-06-03T23:37:14.164Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:13.159050231 +0000 UTC m=+68.480805161 (2 nodes), 1.08µs from start of poll, 1.59701608s from cluster start. Timeout at 2025-06-03 23:37:14.164389351 +0000 UTC m=+69.486144321, 1.00533916s after stability +2025-06-03T23:37:14.175Z [INFO] server-ab55a806-fb1e-2956-99ba-c4e5bd2c8625: aborting pipeline replication: peer="{Voter server-f81da1ce-24e7-69f5-3f2f-3c27dad8d8e3 f81da1ce-24e7-69f5-3f2f-3c27dad8d8e3}" +2025-06-03T23:37:14.176Z [INFO] server-ab55a806-fb1e-2956-99ba-c4e5bd2c8625: aborting pipeline replication: peer="{Voter server-eb96da49-488b-7697-c785-875ba4b703ed eb96da49-488b-7697-c785-875ba4b703ed}" === RUN TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false -2025-06-03T23:34:15.663Z [INFO] server-e3d6d055-b8aa-8b1c-7bd2-8510a90e1868: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e3d6d055-b8aa-8b1c-7bd2-8510a90e1868 Address:e3d6d055-b8aa-8b1c-7bd2-8510a90e1868} {Suffrage:Voter ID:server-b183a77f-5dcb-b58d-343e-dd8f11072688 Address:b183a77f-5dcb-b58d-343e-dd8f11072688} {Suffrage:Voter ID:server-17666d3b-7654-d458-0593-92d90b7bda9b Address:17666d3b-7654-d458-0593-92d90b7bda9b}]" -2025-06-03T23:34:15.663Z [INFO] server-e3d6d055-b8aa-8b1c-7bd2-8510a90e1868: entering follower state: follower="Node at e3d6d055-b8aa-8b1c-7bd2-8510a90e1868 [Follower]" leader-address= leader-id= -2025-06-03T23:34:15.663Z [INFO] server-b183a77f-5dcb-b58d-343e-dd8f11072688: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e3d6d055-b8aa-8b1c-7bd2-8510a90e1868 Address:e3d6d055-b8aa-8b1c-7bd2-8510a90e1868} {Suffrage:Voter ID:server-b183a77f-5dcb-b58d-343e-dd8f11072688 Address:b183a77f-5dcb-b58d-343e-dd8f11072688} {Suffrage:Voter ID:server-17666d3b-7654-d458-0593-92d90b7bda9b Address:17666d3b-7654-d458-0593-92d90b7bda9b}]" -2025-06-03T23:34:15.663Z [INFO] server-b183a77f-5dcb-b58d-343e-dd8f11072688: entering follower state: follower="Node at b183a77f-5dcb-b58d-343e-dd8f11072688 [Follower]" leader-address= leader-id= -2025-06-03T23:34:15.663Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e3d6d055-b8aa-8b1c-7bd2-8510a90e1868 Address:e3d6d055-b8aa-8b1c-7bd2-8510a90e1868} {Suffrage:Voter ID:server-b183a77f-5dcb-b58d-343e-dd8f11072688 Address:b183a77f-5dcb-b58d-343e-dd8f11072688} {Suffrage:Voter ID:server-17666d3b-7654-d458-0593-92d90b7bda9b Address:17666d3b-7654-d458-0593-92d90b7bda9b}]" -2025-06-03T23:34:15.663Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:15.663Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: entering follower state: follower="Node at 17666d3b-7654-d458-0593-92d90b7bda9b [Follower]" leader-address= leader-id= -2025-06-03T23:34:16.184Z [WARN] server-17666d3b-7654-d458-0593-92d90b7bda9b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:16.184Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: entering candidate state: node="Node at 17666d3b-7654-d458-0593-92d90b7bda9b [Candidate]" term=2 -2025-06-03T23:34:16.185Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: election won: term=2 tally=2 -2025-06-03T23:34:16.185Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: entering leader state: leader="Node at 17666d3b-7654-d458-0593-92d90b7bda9b [Leader]" -2025-06-03T23:34:16.185Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: added peer, starting replication: peer=server-e3d6d055-b8aa-8b1c-7bd2-8510a90e1868 -2025-06-03T23:34:16.185Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: added peer, starting replication: peer=server-b183a77f-5dcb-b58d-343e-dd8f11072688 -2025-06-03T23:34:16.185Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: pipelining replication: peer="{Voter server-e3d6d055-b8aa-8b1c-7bd2-8510a90e1868 e3d6d055-b8aa-8b1c-7bd2-8510a90e1868}" -2025-06-03T23:34:16.185Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: pipelining replication: peer="{Voter server-b183a77f-5dcb-b58d-343e-dd8f11072688 b183a77f-5dcb-b58d-343e-dd8f11072688}" -2025-06-03T23:34:17.190Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:16.185200191 +0000 UTC m=+71.206019561 (1 nodes), 521.53892ms from start of poll, 522.169ms from cluster start. Timeout at 2025-06-03 23:34:17.190663391 +0000 UTC m=+72.211482801, 1.00546324s after stability -2025-06-03T23:34:17.191Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: starting snapshot up to: index=12 -2025-06-03T23:34:17.191Z [INFO] creating new snapshot: path=/tmp/raft3228500909/snapshots/2-12-1748993657191.tmp -2025-06-03T23:34:17.191Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: snapshot complete up to: index=12 -2025-06-03T23:34:17.192Z [INFO] creating new snapshot: path=/tmp/raft3228500909/snapshots/2-1013-1748993657192.tmp -2025-06-03T23:34:17.193Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: copied to local snapshot: bytes=71 -2025-06-03T23:34:17.193Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: snapshot restore progress: id=2-1013-1748993657192 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:17.193Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: restored user snapshot: index=1013 -2025-06-03T23:34:17.193Z [ERROR] server-17666d3b-7654-d458-0593-92d90b7bda9b: failed to get log: index=23 error="log not found" -2025-06-03T23:34:17.193Z [ERROR] server-17666d3b-7654-d458-0593-92d90b7bda9b: failed to get log: index=23 error="log not found" -2025-06-03T23:34:17.193Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: aborting pipeline replication: peer="{Voter server-e3d6d055-b8aa-8b1c-7bd2-8510a90e1868 e3d6d055-b8aa-8b1c-7bd2-8510a90e1868}" -2025-06-03T23:34:17.193Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: aborting pipeline replication: peer="{Voter server-b183a77f-5dcb-b58d-343e-dd8f11072688 b183a77f-5dcb-b58d-343e-dd8f11072688}" -2025-06-03T23:34:17.199Z [ERROR] server-17666d3b-7654-d458-0593-92d90b7bda9b: failed to get log: index=23 error="log not found" -2025-06-03T23:34:17.200Z [INFO] creating new snapshot: path=/tmp/raft2402999037/snapshots/2-1013-1748993657200.tmp -2025-06-03T23:34:17.200Z [ERROR] server-17666d3b-7654-d458-0593-92d90b7bda9b: failed to get log: index=23 error="log not found" -2025-06-03T23:34:17.200Z [INFO] server-e3d6d055-b8aa-8b1c-7bd2-8510a90e1868: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:17.200Z [INFO] creating new snapshot: path=/tmp/raft829962170/snapshots/2-1013-1748993657200.tmp -2025-06-03T23:34:17.200Z [INFO] server-e3d6d055-b8aa-8b1c-7bd2-8510a90e1868: copied to local snapshot: bytes=71 -2025-06-03T23:34:17.200Z [INFO] server-b183a77f-5dcb-b58d-343e-dd8f11072688: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:17.200Z [INFO] server-e3d6d055-b8aa-8b1c-7bd2-8510a90e1868: snapshot restore progress: id=2-1013-1748993657200 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:17.200Z [INFO] server-e3d6d055-b8aa-8b1c-7bd2-8510a90e1868: Installed remote snapshot -2025-06-03T23:34:17.200Z [WARN] server-e3d6d055-b8aa-8b1c-7bd2-8510a90e1868: skipping application of old log: index=22 -2025-06-03T23:34:17.200Z [INFO] server-b183a77f-5dcb-b58d-343e-dd8f11072688: copied to local snapshot: bytes=71 -2025-06-03T23:34:17.200Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:17.200Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: pipelining replication: peer="{Voter server-e3d6d055-b8aa-8b1c-7bd2-8510a90e1868 e3d6d055-b8aa-8b1c-7bd2-8510a90e1868}" -2025-06-03T23:34:17.201Z [INFO] server-b183a77f-5dcb-b58d-343e-dd8f11072688: snapshot restore progress: id=2-1013-1748993657200 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:17.201Z [INFO] server-b183a77f-5dcb-b58d-343e-dd8f11072688: Installed remote snapshot -2025-06-03T23:34:17.201Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: pipelining replication: peer="{Voter server-b183a77f-5dcb-b58d-343e-dd8f11072688 b183a77f-5dcb-b58d-343e-dd8f11072688}" -2025-06-03T23:34:18.206Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:17.200972191 +0000 UTC m=+72.221791561 (2 nodes), 1.12µs from start of poll, 1.537941s from cluster start. Timeout at 2025-06-03 23:34:18.206447031 +0000 UTC m=+73.227266401, 1.00547484s after stability -2025-06-03T23:34:18.223Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: aborting pipeline replication: peer="{Voter server-b183a77f-5dcb-b58d-343e-dd8f11072688 b183a77f-5dcb-b58d-343e-dd8f11072688}" -2025-06-03T23:34:18.223Z [INFO] server-17666d3b-7654-d458-0593-92d90b7bda9b: aborting pipeline replication: peer="{Voter server-e3d6d055-b8aa-8b1c-7bd2-8510a90e1868 e3d6d055-b8aa-8b1c-7bd2-8510a90e1868}" +2025-06-03T23:37:14.176Z [INFO] server-113fca3d-fac0-bb6e-c0cb-4b5677fe63d9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-113fca3d-fac0-bb6e-c0cb-4b5677fe63d9 Address:113fca3d-fac0-bb6e-c0cb-4b5677fe63d9} {Suffrage:Voter ID:server-80538240-9705-ecf3-4731-e0c0f7fc2eab Address:80538240-9705-ecf3-4731-e0c0f7fc2eab} {Suffrage:Voter ID:server-e1b507ac-f03d-79e4-1316-57abf68c949f Address:e1b507ac-f03d-79e4-1316-57abf68c949f}]" +2025-06-03T23:37:14.177Z [INFO] server-113fca3d-fac0-bb6e-c0cb-4b5677fe63d9: entering follower state: follower="Node at 113fca3d-fac0-bb6e-c0cb-4b5677fe63d9 [Follower]" leader-address= leader-id= +2025-06-03T23:37:14.177Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-113fca3d-fac0-bb6e-c0cb-4b5677fe63d9 Address:113fca3d-fac0-bb6e-c0cb-4b5677fe63d9} {Suffrage:Voter ID:server-80538240-9705-ecf3-4731-e0c0f7fc2eab Address:80538240-9705-ecf3-4731-e0c0f7fc2eab} {Suffrage:Voter ID:server-e1b507ac-f03d-79e4-1316-57abf68c949f Address:e1b507ac-f03d-79e4-1316-57abf68c949f}]" +2025-06-03T23:37:14.177Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: entering follower state: follower="Node at 80538240-9705-ecf3-4731-e0c0f7fc2eab [Follower]" leader-address= leader-id= +2025-06-03T23:37:14.177Z [INFO] server-e1b507ac-f03d-79e4-1316-57abf68c949f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-113fca3d-fac0-bb6e-c0cb-4b5677fe63d9 Address:113fca3d-fac0-bb6e-c0cb-4b5677fe63d9} {Suffrage:Voter ID:server-80538240-9705-ecf3-4731-e0c0f7fc2eab Address:80538240-9705-ecf3-4731-e0c0f7fc2eab} {Suffrage:Voter ID:server-e1b507ac-f03d-79e4-1316-57abf68c949f Address:e1b507ac-f03d-79e4-1316-57abf68c949f}]" +2025-06-03T23:37:14.177Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:14.177Z [INFO] server-e1b507ac-f03d-79e4-1316-57abf68c949f: entering follower state: follower="Node at e1b507ac-f03d-79e4-1316-57abf68c949f [Follower]" leader-address= leader-id= +2025-06-03T23:37:14.769Z [WARN] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:14.769Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: entering candidate state: node="Node at 80538240-9705-ecf3-4731-e0c0f7fc2eab [Candidate]" term=2 +2025-06-03T23:37:14.769Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: election won: term=2 tally=2 +2025-06-03T23:37:14.770Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: entering leader state: leader="Node at 80538240-9705-ecf3-4731-e0c0f7fc2eab [Leader]" +2025-06-03T23:37:14.770Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: added peer, starting replication: peer=server-113fca3d-fac0-bb6e-c0cb-4b5677fe63d9 +2025-06-03T23:37:14.770Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: added peer, starting replication: peer=server-e1b507ac-f03d-79e4-1316-57abf68c949f +2025-06-03T23:37:14.770Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: pipelining replication: peer="{Voter server-e1b507ac-f03d-79e4-1316-57abf68c949f e1b507ac-f03d-79e4-1316-57abf68c949f}" +2025-06-03T23:37:14.770Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: pipelining replication: peer="{Voter server-113fca3d-fac0-bb6e-c0cb-4b5677fe63d9 113fca3d-fac0-bb6e-c0cb-4b5677fe63d9}" +2025-06-03T23:37:15.775Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:14.770122751 +0000 UTC m=+70.091877681 (1 nodes), 592.83688ms from start of poll, 593.31168ms from cluster start. Timeout at 2025-06-03 23:37:15.775198031 +0000 UTC m=+71.096952961, 1.00507528s after stability +2025-06-03T23:37:15.775Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: starting snapshot up to: index=12 +2025-06-03T23:37:15.775Z [INFO] creating new snapshot: path=/tmp/raft2182811896/snapshots/2-12-1748993835775.tmp +2025-06-03T23:37:15.776Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: snapshot complete up to: index=12 +2025-06-03T23:37:15.777Z [INFO] creating new snapshot: path=/tmp/raft2182811896/snapshots/2-1013-1748993835777.tmp +2025-06-03T23:37:15.777Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: copied to local snapshot: bytes=71 +2025-06-03T23:37:15.777Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: snapshot restore progress: id=2-1013-1748993835777 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:15.777Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: restored user snapshot: index=1013 +2025-06-03T23:37:15.778Z [ERROR] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: failed to get log: index=23 error="log not found" +2025-06-03T23:37:15.778Z [ERROR] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: failed to get log: index=23 error="log not found" +2025-06-03T23:37:15.778Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: aborting pipeline replication: peer="{Voter server-e1b507ac-f03d-79e4-1316-57abf68c949f e1b507ac-f03d-79e4-1316-57abf68c949f}" +2025-06-03T23:37:15.778Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: aborting pipeline replication: peer="{Voter server-113fca3d-fac0-bb6e-c0cb-4b5677fe63d9 113fca3d-fac0-bb6e-c0cb-4b5677fe63d9}" +2025-06-03T23:37:15.786Z [ERROR] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: failed to get log: index=23 error="log not found" +2025-06-03T23:37:15.787Z [INFO] creating new snapshot: path=/tmp/raft2007322761/snapshots/2-1013-1748993835787.tmp +2025-06-03T23:37:15.787Z [ERROR] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: failed to get log: index=23 error="log not found" +2025-06-03T23:37:15.787Z [INFO] server-113fca3d-fac0-bb6e-c0cb-4b5677fe63d9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:15.787Z [INFO] creating new snapshot: path=/tmp/raft3284962282/snapshots/2-1013-1748993835787.tmp +2025-06-03T23:37:15.787Z [INFO] server-113fca3d-fac0-bb6e-c0cb-4b5677fe63d9: copied to local snapshot: bytes=71 +2025-06-03T23:37:15.788Z [INFO] server-e1b507ac-f03d-79e4-1316-57abf68c949f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:15.788Z [INFO] server-113fca3d-fac0-bb6e-c0cb-4b5677fe63d9: snapshot restore progress: id=2-1013-1748993835787 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:15.788Z [INFO] server-113fca3d-fac0-bb6e-c0cb-4b5677fe63d9: Installed remote snapshot +2025-06-03T23:37:15.788Z [WARN] server-113fca3d-fac0-bb6e-c0cb-4b5677fe63d9: skipping application of old log: index=22 +2025-06-03T23:37:15.788Z [INFO] server-e1b507ac-f03d-79e4-1316-57abf68c949f: copied to local snapshot: bytes=71 +2025-06-03T23:37:15.788Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: pipelining replication: peer="{Voter server-113fca3d-fac0-bb6e-c0cb-4b5677fe63d9 113fca3d-fac0-bb6e-c0cb-4b5677fe63d9}" +2025-06-03T23:37:15.788Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:15.789Z [INFO] server-e1b507ac-f03d-79e4-1316-57abf68c949f: snapshot restore progress: id=2-1013-1748993835787 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:15.789Z [INFO] server-e1b507ac-f03d-79e4-1316-57abf68c949f: Installed remote snapshot +2025-06-03T23:37:15.789Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: pipelining replication: peer="{Voter server-e1b507ac-f03d-79e4-1316-57abf68c949f e1b507ac-f03d-79e4-1316-57abf68c949f}" +2025-06-03T23:37:16.794Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:15.788991751 +0000 UTC m=+71.110746681 (2 nodes), 1.24µs from start of poll, 1.61218068s from cluster start. Timeout at 2025-06-03 23:37:16.794689271 +0000 UTC m=+72.116444201, 1.00569752s after stability +2025-06-03T23:37:16.806Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: aborting pipeline replication: peer="{Voter server-e1b507ac-f03d-79e4-1316-57abf68c949f e1b507ac-f03d-79e4-1316-57abf68c949f}" +2025-06-03T23:37:16.806Z [INFO] server-80538240-9705-ecf3-4731-e0c0f7fc2eab: aborting pipeline replication: peer="{Voter server-113fca3d-fac0-bb6e-c0cb-4b5677fe63d9 113fca3d-fac0-bb6e-c0cb-4b5677fe63d9}" === RUN TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false -2025-06-03T23:34:18.224Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31 Address:2136e7f4-c8ac-b289-b9a8-989d2ddd5c31} {Suffrage:Voter ID:server-28c1025a-2db0-c89b-1184-67a56d301812 Address:28c1025a-2db0-c89b-1184-67a56d301812} {Suffrage:Voter ID:server-7a4f8c61-20cc-5d2c-7874-d400e5055937 Address:7a4f8c61-20cc-5d2c-7874-d400e5055937}]" -2025-06-03T23:34:18.224Z [INFO] server-28c1025a-2db0-c89b-1184-67a56d301812: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31 Address:2136e7f4-c8ac-b289-b9a8-989d2ddd5c31} {Suffrage:Voter ID:server-28c1025a-2db0-c89b-1184-67a56d301812 Address:28c1025a-2db0-c89b-1184-67a56d301812} {Suffrage:Voter ID:server-7a4f8c61-20cc-5d2c-7874-d400e5055937 Address:7a4f8c61-20cc-5d2c-7874-d400e5055937}]" -2025-06-03T23:34:18.224Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: entering follower state: follower="Node at 2136e7f4-c8ac-b289-b9a8-989d2ddd5c31 [Follower]" leader-address= leader-id= -2025-06-03T23:34:18.224Z [INFO] server-28c1025a-2db0-c89b-1184-67a56d301812: entering follower state: follower="Node at 28c1025a-2db0-c89b-1184-67a56d301812 [Follower]" leader-address= leader-id= -2025-06-03T23:34:18.225Z [INFO] server-7a4f8c61-20cc-5d2c-7874-d400e5055937: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31 Address:2136e7f4-c8ac-b289-b9a8-989d2ddd5c31} {Suffrage:Voter ID:server-28c1025a-2db0-c89b-1184-67a56d301812 Address:28c1025a-2db0-c89b-1184-67a56d301812} {Suffrage:Voter ID:server-7a4f8c61-20cc-5d2c-7874-d400e5055937 Address:7a4f8c61-20cc-5d2c-7874-d400e5055937}]" -2025-06-03T23:34:18.225Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:18.225Z [INFO] server-7a4f8c61-20cc-5d2c-7874-d400e5055937: entering follower state: follower="Node at 7a4f8c61-20cc-5d2c-7874-d400e5055937 [Follower]" leader-address= leader-id= -2025-06-03T23:34:18.965Z [WARN] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:18.965Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: entering candidate state: node="Node at 2136e7f4-c8ac-b289-b9a8-989d2ddd5c31 [Candidate]" term=2 -2025-06-03T23:34:18.965Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: election won: term=2 tally=2 -2025-06-03T23:34:18.965Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: entering leader state: leader="Node at 2136e7f4-c8ac-b289-b9a8-989d2ddd5c31 [Leader]" -2025-06-03T23:34:18.966Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: added peer, starting replication: peer=server-28c1025a-2db0-c89b-1184-67a56d301812 -2025-06-03T23:34:18.966Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: added peer, starting replication: peer=server-7a4f8c61-20cc-5d2c-7874-d400e5055937 -2025-06-03T23:34:18.966Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: pipelining replication: peer="{Voter server-7a4f8c61-20cc-5d2c-7874-d400e5055937 7a4f8c61-20cc-5d2c-7874-d400e5055937}" -2025-06-03T23:34:18.966Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: pipelining replication: peer="{Voter server-28c1025a-2db0-c89b-1184-67a56d301812 28c1025a-2db0-c89b-1184-67a56d301812}" -2025-06-03T23:34:19.971Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:18.965999351 +0000 UTC m=+73.986818721 (1 nodes), 740.90396ms from start of poll, 741.38024ms from cluster start. Timeout at 2025-06-03 23:34:19.971571791 +0000 UTC m=+74.992391161, 1.00557244s after stability -2025-06-03T23:34:19.972Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: starting snapshot up to: index=12 -2025-06-03T23:34:19.972Z [INFO] creating new snapshot: path=/tmp/raft3971555158/snapshots/2-12-1748993659972.tmp -2025-06-03T23:34:19.972Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: snapshot complete up to: index=12 -2025-06-03T23:34:19.973Z [INFO] creating new snapshot: path=/tmp/raft3971555158/snapshots/2-1013-1748993659973.tmp -2025-06-03T23:34:19.974Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: copied to local snapshot: bytes=71 -2025-06-03T23:34:19.974Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: snapshot restore progress: id=2-1013-1748993659973 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:19.974Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: removing all old logs from log store -2025-06-03T23:34:19.974Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: compacting logs: from=1 to=22 -2025-06-03T23:34:19.974Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: restored user snapshot: index=1013 -2025-06-03T23:34:19.974Z [ERROR] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: failed to get log: index=22 error="log not found" -2025-06-03T23:34:19.974Z [ERROR] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: failed to get log: index=22 error="log not found" -2025-06-03T23:34:19.974Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: aborting pipeline replication: peer="{Voter server-28c1025a-2db0-c89b-1184-67a56d301812 28c1025a-2db0-c89b-1184-67a56d301812}" -2025-06-03T23:34:19.974Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: aborting pipeline replication: peer="{Voter server-7a4f8c61-20cc-5d2c-7874-d400e5055937 7a4f8c61-20cc-5d2c-7874-d400e5055937}" -2025-06-03T23:34:19.980Z [ERROR] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: failed to get log: index=22 error="log not found" -2025-06-03T23:34:19.981Z [INFO] creating new snapshot: path=/tmp/raft3288240951/snapshots/2-1013-1748993659981.tmp -2025-06-03T23:34:19.981Z [INFO] server-28c1025a-2db0-c89b-1184-67a56d301812: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:19.981Z [INFO] server-28c1025a-2db0-c89b-1184-67a56d301812: copied to local snapshot: bytes=71 -2025-06-03T23:34:19.982Z [INFO] server-28c1025a-2db0-c89b-1184-67a56d301812: snapshot restore progress: id=2-1013-1748993659981 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:19.982Z [INFO] server-28c1025a-2db0-c89b-1184-67a56d301812: removing all old logs from log store -2025-06-03T23:34:19.982Z [INFO] server-28c1025a-2db0-c89b-1184-67a56d301812: compacting logs: from=1 to=22 -2025-06-03T23:34:19.982Z [INFO] server-28c1025a-2db0-c89b-1184-67a56d301812: Installed remote snapshot -2025-06-03T23:34:19.982Z [WARN] server-28c1025a-2db0-c89b-1184-67a56d301812: skipping application of old log: index=22 -2025-06-03T23:34:19.982Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: pipelining replication: peer="{Voter server-28c1025a-2db0-c89b-1184-67a56d301812 28c1025a-2db0-c89b-1184-67a56d301812}" -2025-06-03T23:34:19.982Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:19.983Z [ERROR] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: failed to get log: index=22 error="log not found" -2025-06-03T23:34:19.984Z [INFO] creating new snapshot: path=/tmp/raft2440446208/snapshots/2-1013-1748993659984.tmp -2025-06-03T23:34:19.984Z [INFO] server-7a4f8c61-20cc-5d2c-7874-d400e5055937: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:19.985Z [INFO] server-7a4f8c61-20cc-5d2c-7874-d400e5055937: copied to local snapshot: bytes=71 -2025-06-03T23:34:19.985Z [INFO] server-7a4f8c61-20cc-5d2c-7874-d400e5055937: snapshot restore progress: id=2-1013-1748993659984 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:19.985Z [INFO] server-7a4f8c61-20cc-5d2c-7874-d400e5055937: removing all old logs from log store -2025-06-03T23:34:19.985Z [INFO] server-7a4f8c61-20cc-5d2c-7874-d400e5055937: compacting logs: from=1 to=22 -2025-06-03T23:34:19.985Z [INFO] server-7a4f8c61-20cc-5d2c-7874-d400e5055937: Installed remote snapshot -2025-06-03T23:34:19.985Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: pipelining replication: peer="{Voter server-7a4f8c61-20cc-5d2c-7874-d400e5055937 7a4f8c61-20cc-5d2c-7874-d400e5055937}" -2025-06-03T23:34:20.988Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:19.982536311 +0000 UTC m=+75.003355641 (2 nodes), 800ns from start of poll, 1.75791716s from cluster start. Timeout at 2025-06-03 23:34:20.988518951 +0000 UTC m=+76.009338321, 1.00598268s after stability -2025-06-03T23:34:21.003Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: aborting pipeline replication: peer="{Voter server-28c1025a-2db0-c89b-1184-67a56d301812 28c1025a-2db0-c89b-1184-67a56d301812}" -2025-06-03T23:34:21.003Z [INFO] server-2136e7f4-c8ac-b289-b9a8-989d2ddd5c31: aborting pipeline replication: peer="{Voter server-7a4f8c61-20cc-5d2c-7874-d400e5055937 7a4f8c61-20cc-5d2c-7874-d400e5055937}" +2025-06-03T23:37:16.807Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-94da09d5-f729-64ab-c3eb-f5520edab365 Address:94da09d5-f729-64ab-c3eb-f5520edab365} {Suffrage:Voter ID:server-6b7e3d81-f169-13e6-d075-3a71ecb42d47 Address:6b7e3d81-f169-13e6-d075-3a71ecb42d47} {Suffrage:Voter ID:server-774a6355-f5a6-5c7a-184b-c636d2447891 Address:774a6355-f5a6-5c7a-184b-c636d2447891}]" +2025-06-03T23:37:16.807Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: entering follower state: follower="Node at 94da09d5-f729-64ab-c3eb-f5520edab365 [Follower]" leader-address= leader-id= +2025-06-03T23:37:16.807Z [INFO] server-6b7e3d81-f169-13e6-d075-3a71ecb42d47: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-94da09d5-f729-64ab-c3eb-f5520edab365 Address:94da09d5-f729-64ab-c3eb-f5520edab365} {Suffrage:Voter ID:server-6b7e3d81-f169-13e6-d075-3a71ecb42d47 Address:6b7e3d81-f169-13e6-d075-3a71ecb42d47} {Suffrage:Voter ID:server-774a6355-f5a6-5c7a-184b-c636d2447891 Address:774a6355-f5a6-5c7a-184b-c636d2447891}]" +2025-06-03T23:37:16.807Z [INFO] server-6b7e3d81-f169-13e6-d075-3a71ecb42d47: entering follower state: follower="Node at 6b7e3d81-f169-13e6-d075-3a71ecb42d47 [Follower]" leader-address= leader-id= +2025-06-03T23:37:16.808Z [INFO] server-774a6355-f5a6-5c7a-184b-c636d2447891: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-94da09d5-f729-64ab-c3eb-f5520edab365 Address:94da09d5-f729-64ab-c3eb-f5520edab365} {Suffrage:Voter ID:server-6b7e3d81-f169-13e6-d075-3a71ecb42d47 Address:6b7e3d81-f169-13e6-d075-3a71ecb42d47} {Suffrage:Voter ID:server-774a6355-f5a6-5c7a-184b-c636d2447891 Address:774a6355-f5a6-5c7a-184b-c636d2447891}]" +2025-06-03T23:37:16.808Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:16.808Z [INFO] server-774a6355-f5a6-5c7a-184b-c636d2447891: entering follower state: follower="Node at 774a6355-f5a6-5c7a-184b-c636d2447891 [Follower]" leader-address= leader-id= +2025-06-03T23:37:17.428Z [WARN] server-94da09d5-f729-64ab-c3eb-f5520edab365: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:17.428Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: entering candidate state: node="Node at 94da09d5-f729-64ab-c3eb-f5520edab365 [Candidate]" term=2 +2025-06-03T23:37:17.428Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: election won: term=2 tally=2 +2025-06-03T23:37:17.428Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: entering leader state: leader="Node at 94da09d5-f729-64ab-c3eb-f5520edab365 [Leader]" +2025-06-03T23:37:17.428Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: added peer, starting replication: peer=server-6b7e3d81-f169-13e6-d075-3a71ecb42d47 +2025-06-03T23:37:17.428Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: added peer, starting replication: peer=server-774a6355-f5a6-5c7a-184b-c636d2447891 +2025-06-03T23:37:17.429Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: pipelining replication: peer="{Voter server-6b7e3d81-f169-13e6-d075-3a71ecb42d47 6b7e3d81-f169-13e6-d075-3a71ecb42d47}" +2025-06-03T23:37:17.429Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: pipelining replication: peer="{Voter server-774a6355-f5a6-5c7a-184b-c636d2447891 774a6355-f5a6-5c7a-184b-c636d2447891}" +2025-06-03T23:37:18.433Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:17.428852631 +0000 UTC m=+72.750607561 (1 nodes), 620.64068ms from start of poll, 621.23976ms from cluster start. Timeout at 2025-06-03 23:37:18.433899031 +0000 UTC m=+73.755654001, 1.00504644s after stability +2025-06-03T23:37:18.434Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: starting snapshot up to: index=12 +2025-06-03T23:37:18.434Z [INFO] creating new snapshot: path=/tmp/raft1541803753/snapshots/2-12-1748993838434.tmp +2025-06-03T23:37:18.435Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: snapshot complete up to: index=12 +2025-06-03T23:37:18.435Z [INFO] creating new snapshot: path=/tmp/raft1541803753/snapshots/2-1013-1748993838435.tmp +2025-06-03T23:37:18.436Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: copied to local snapshot: bytes=71 +2025-06-03T23:37:18.437Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: snapshot restore progress: id=2-1013-1748993838435 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:18.437Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: removing all old logs from log store +2025-06-03T23:37:18.437Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: compacting logs: from=1 to=22 +2025-06-03T23:37:18.437Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: restored user snapshot: index=1013 +2025-06-03T23:37:18.437Z [ERROR] server-94da09d5-f729-64ab-c3eb-f5520edab365: failed to get log: index=22 error="log not found" +2025-06-03T23:37:18.437Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: aborting pipeline replication: peer="{Voter server-6b7e3d81-f169-13e6-d075-3a71ecb42d47 6b7e3d81-f169-13e6-d075-3a71ecb42d47}" +2025-06-03T23:37:18.437Z [ERROR] server-94da09d5-f729-64ab-c3eb-f5520edab365: failed to get log: index=22 error="log not found" +2025-06-03T23:37:18.437Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: aborting pipeline replication: peer="{Voter server-774a6355-f5a6-5c7a-184b-c636d2447891 774a6355-f5a6-5c7a-184b-c636d2447891}" +2025-06-03T23:37:18.444Z [ERROR] server-94da09d5-f729-64ab-c3eb-f5520edab365: failed to get log: index=22 error="log not found" +2025-06-03T23:37:18.445Z [INFO] creating new snapshot: path=/tmp/raft1262300149/snapshots/2-1013-1748993838445.tmp +2025-06-03T23:37:18.445Z [INFO] server-6b7e3d81-f169-13e6-d075-3a71ecb42d47: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:18.445Z [INFO] server-6b7e3d81-f169-13e6-d075-3a71ecb42d47: copied to local snapshot: bytes=71 +2025-06-03T23:37:18.445Z [INFO] server-6b7e3d81-f169-13e6-d075-3a71ecb42d47: snapshot restore progress: id=2-1013-1748993838445 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:18.445Z [INFO] server-6b7e3d81-f169-13e6-d075-3a71ecb42d47: removing all old logs from log store +2025-06-03T23:37:18.445Z [INFO] server-6b7e3d81-f169-13e6-d075-3a71ecb42d47: compacting logs: from=1 to=22 +2025-06-03T23:37:18.445Z [INFO] server-6b7e3d81-f169-13e6-d075-3a71ecb42d47: Installed remote snapshot +2025-06-03T23:37:18.445Z [WARN] server-6b7e3d81-f169-13e6-d075-3a71ecb42d47: skipping application of old log: index=22 +2025-06-03T23:37:18.446Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: pipelining replication: peer="{Voter server-6b7e3d81-f169-13e6-d075-3a71ecb42d47 6b7e3d81-f169-13e6-d075-3a71ecb42d47}" +2025-06-03T23:37:18.446Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:18.448Z [ERROR] server-94da09d5-f729-64ab-c3eb-f5520edab365: failed to get log: index=22 error="log not found" +2025-06-03T23:37:18.448Z [INFO] creating new snapshot: path=/tmp/raft1189665668/snapshots/2-1013-1748993838448.tmp +2025-06-03T23:37:18.448Z [INFO] server-774a6355-f5a6-5c7a-184b-c636d2447891: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:18.449Z [INFO] server-774a6355-f5a6-5c7a-184b-c636d2447891: copied to local snapshot: bytes=71 +2025-06-03T23:37:18.449Z [INFO] server-774a6355-f5a6-5c7a-184b-c636d2447891: snapshot restore progress: id=2-1013-1748993838448 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:18.449Z [INFO] server-774a6355-f5a6-5c7a-184b-c636d2447891: removing all old logs from log store +2025-06-03T23:37:18.449Z [INFO] server-774a6355-f5a6-5c7a-184b-c636d2447891: compacting logs: from=1 to=22 +2025-06-03T23:37:18.449Z [INFO] server-774a6355-f5a6-5c7a-184b-c636d2447891: Installed remote snapshot +2025-06-03T23:37:18.449Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: pipelining replication: peer="{Voter server-774a6355-f5a6-5c7a-184b-c636d2447891 774a6355-f5a6-5c7a-184b-c636d2447891}" +2025-06-03T23:37:19.451Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:18.446097471 +0000 UTC m=+73.767852401 (2 nodes), 1.28µs from start of poll, 1.6384846s from cluster start. Timeout at 2025-06-03 23:37:19.451607351 +0000 UTC m=+74.773362281, 1.00550988s after stability +2025-06-03T23:37:19.463Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: aborting pipeline replication: peer="{Voter server-774a6355-f5a6-5c7a-184b-c636d2447891 774a6355-f5a6-5c7a-184b-c636d2447891}" +2025-06-03T23:37:19.463Z [INFO] server-94da09d5-f729-64ab-c3eb-f5520edab365: aborting pipeline replication: peer="{Voter server-6b7e3d81-f169-13e6-d075-3a71ecb42d47 6b7e3d81-f169-13e6-d075-3a71ecb42d47}" === RUN TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true -2025-06-03T23:34:21.004Z [INFO] server-130d7054-bf3c-d9cb-7622-b31c429eef84: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-130d7054-bf3c-d9cb-7622-b31c429eef84 Address:130d7054-bf3c-d9cb-7622-b31c429eef84} {Suffrage:Voter ID:server-32fe2812-6c0b-a915-ba34-fd53606482d2 Address:32fe2812-6c0b-a915-ba34-fd53606482d2} {Suffrage:Voter ID:server-92a584d8-e237-a22c-6a55-e8f17b768ca3 Address:92a584d8-e237-a22c-6a55-e8f17b768ca3}]" -2025-06-03T23:34:21.004Z [INFO] server-130d7054-bf3c-d9cb-7622-b31c429eef84: entering follower state: follower="Node at 130d7054-bf3c-d9cb-7622-b31c429eef84 [Follower]" leader-address= leader-id= -2025-06-03T23:34:21.004Z [INFO] server-32fe2812-6c0b-a915-ba34-fd53606482d2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-130d7054-bf3c-d9cb-7622-b31c429eef84 Address:130d7054-bf3c-d9cb-7622-b31c429eef84} {Suffrage:Voter ID:server-32fe2812-6c0b-a915-ba34-fd53606482d2 Address:32fe2812-6c0b-a915-ba34-fd53606482d2} {Suffrage:Voter ID:server-92a584d8-e237-a22c-6a55-e8f17b768ca3 Address:92a584d8-e237-a22c-6a55-e8f17b768ca3}]" -2025-06-03T23:34:21.004Z [INFO] server-32fe2812-6c0b-a915-ba34-fd53606482d2: entering follower state: follower="Node at 32fe2812-6c0b-a915-ba34-fd53606482d2 [Follower]" leader-address= leader-id= -2025-06-03T23:34:21.004Z [INFO] server-92a584d8-e237-a22c-6a55-e8f17b768ca3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-130d7054-bf3c-d9cb-7622-b31c429eef84 Address:130d7054-bf3c-d9cb-7622-b31c429eef84} {Suffrage:Voter ID:server-32fe2812-6c0b-a915-ba34-fd53606482d2 Address:32fe2812-6c0b-a915-ba34-fd53606482d2} {Suffrage:Voter ID:server-92a584d8-e237-a22c-6a55-e8f17b768ca3 Address:92a584d8-e237-a22c-6a55-e8f17b768ca3}]" -2025-06-03T23:34:21.004Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:21.004Z [INFO] server-92a584d8-e237-a22c-6a55-e8f17b768ca3: entering follower state: follower="Node at 92a584d8-e237-a22c-6a55-e8f17b768ca3 [Follower]" leader-address= leader-id= -2025-06-03T23:34:21.569Z [WARN] server-130d7054-bf3c-d9cb-7622-b31c429eef84: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:21.569Z [INFO] server-130d7054-bf3c-d9cb-7622-b31c429eef84: entering candidate state: node="Node at 130d7054-bf3c-d9cb-7622-b31c429eef84 [Candidate]" term=2 -2025-06-03T23:34:21.570Z [INFO] server-130d7054-bf3c-d9cb-7622-b31c429eef84: election won: term=2 tally=2 -2025-06-03T23:34:21.570Z [INFO] server-130d7054-bf3c-d9cb-7622-b31c429eef84: entering leader state: leader="Node at 130d7054-bf3c-d9cb-7622-b31c429eef84 [Leader]" -2025-06-03T23:34:21.570Z [INFO] server-130d7054-bf3c-d9cb-7622-b31c429eef84: added peer, starting replication: peer=server-32fe2812-6c0b-a915-ba34-fd53606482d2 -2025-06-03T23:34:21.570Z [INFO] server-130d7054-bf3c-d9cb-7622-b31c429eef84: added peer, starting replication: peer=server-92a584d8-e237-a22c-6a55-e8f17b768ca3 -2025-06-03T23:34:21.570Z [INFO] server-130d7054-bf3c-d9cb-7622-b31c429eef84: pipelining replication: peer="{Voter server-92a584d8-e237-a22c-6a55-e8f17b768ca3 92a584d8-e237-a22c-6a55-e8f17b768ca3}" -2025-06-03T23:34:21.570Z [INFO] server-130d7054-bf3c-d9cb-7622-b31c429eef84: pipelining replication: peer="{Voter server-32fe2812-6c0b-a915-ba34-fd53606482d2 32fe2812-6c0b-a915-ba34-fd53606482d2}" -2025-06-03T23:34:22.575Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:21.570091111 +0000 UTC m=+76.590910481 (1 nodes), 565.37848ms from start of poll, 566.01084ms from cluster start. Timeout at 2025-06-03 23:34:22.575779031 +0000 UTC m=+77.596598401, 1.00568792s after stability -2025-06-03T23:34:22.576Z [INFO] server-130d7054-bf3c-d9cb-7622-b31c429eef84: starting snapshot up to: index=12 -2025-06-03T23:34:22.576Z [INFO] creating new snapshot: path=/tmp/raft3771071474/snapshots/2-12-1748993662576.tmp -2025-06-03T23:34:22.577Z [INFO] server-130d7054-bf3c-d9cb-7622-b31c429eef84: snapshot complete up to: index=12 -2025-06-03T23:34:22.578Z [INFO] server-68715001-1f59-b740-1700-a18d8fc03e78: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68715001-1f59-b740-1700-a18d8fc03e78 Address:68715001-1f59-b740-1700-a18d8fc03e78} {Suffrage:Voter ID:server-20322ac7-0f51-2c27-7651-4de94392a37d Address:20322ac7-0f51-2c27-7651-4de94392a37d} {Suffrage:Voter ID:server-ffc601b9-02b4-e420-6b41-fd77a284880e Address:ffc601b9-02b4-e420-6b41-fd77a284880e}]" -2025-06-03T23:34:22.578Z [INFO] server-68715001-1f59-b740-1700-a18d8fc03e78: entering follower state: follower="Node at 68715001-1f59-b740-1700-a18d8fc03e78 [Follower]" leader-address= leader-id= -2025-06-03T23:34:22.578Z [INFO] server-20322ac7-0f51-2c27-7651-4de94392a37d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68715001-1f59-b740-1700-a18d8fc03e78 Address:68715001-1f59-b740-1700-a18d8fc03e78} {Suffrage:Voter ID:server-20322ac7-0f51-2c27-7651-4de94392a37d Address:20322ac7-0f51-2c27-7651-4de94392a37d} {Suffrage:Voter ID:server-ffc601b9-02b4-e420-6b41-fd77a284880e Address:ffc601b9-02b4-e420-6b41-fd77a284880e}]" -2025-06-03T23:34:22.578Z [INFO] server-20322ac7-0f51-2c27-7651-4de94392a37d: entering follower state: follower="Node at 20322ac7-0f51-2c27-7651-4de94392a37d [Follower]" leader-address= leader-id= -2025-06-03T23:34:22.578Z [INFO] server-ffc601b9-02b4-e420-6b41-fd77a284880e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68715001-1f59-b740-1700-a18d8fc03e78 Address:68715001-1f59-b740-1700-a18d8fc03e78} {Suffrage:Voter ID:server-20322ac7-0f51-2c27-7651-4de94392a37d Address:20322ac7-0f51-2c27-7651-4de94392a37d} {Suffrage:Voter ID:server-ffc601b9-02b4-e420-6b41-fd77a284880e Address:ffc601b9-02b4-e420-6b41-fd77a284880e}]" -2025-06-03T23:34:22.578Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:22.578Z [INFO] server-ffc601b9-02b4-e420-6b41-fd77a284880e: entering follower state: follower="Node at ffc601b9-02b4-e420-6b41-fd77a284880e [Follower]" leader-address= leader-id= -2025-06-03T23:34:23.176Z [WARN] server-68715001-1f59-b740-1700-a18d8fc03e78: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:23.176Z [INFO] server-68715001-1f59-b740-1700-a18d8fc03e78: entering candidate state: node="Node at 68715001-1f59-b740-1700-a18d8fc03e78 [Candidate]" term=2 -2025-06-03T23:34:23.177Z [INFO] server-68715001-1f59-b740-1700-a18d8fc03e78: election won: term=2 tally=2 -2025-06-03T23:34:23.177Z [INFO] server-68715001-1f59-b740-1700-a18d8fc03e78: entering leader state: leader="Node at 68715001-1f59-b740-1700-a18d8fc03e78 [Leader]" -2025-06-03T23:34:23.177Z [INFO] server-68715001-1f59-b740-1700-a18d8fc03e78: added peer, starting replication: peer=server-20322ac7-0f51-2c27-7651-4de94392a37d -2025-06-03T23:34:23.177Z [INFO] server-68715001-1f59-b740-1700-a18d8fc03e78: added peer, starting replication: peer=server-ffc601b9-02b4-e420-6b41-fd77a284880e -2025-06-03T23:34:23.177Z [INFO] server-68715001-1f59-b740-1700-a18d8fc03e78: pipelining replication: peer="{Voter server-20322ac7-0f51-2c27-7651-4de94392a37d 20322ac7-0f51-2c27-7651-4de94392a37d}" -2025-06-03T23:34:23.177Z [INFO] server-68715001-1f59-b740-1700-a18d8fc03e78: pipelining replication: peer="{Voter server-ffc601b9-02b4-e420-6b41-fd77a284880e ffc601b9-02b4-e420-6b41-fd77a284880e}" -2025-06-03T23:34:24.182Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:23.177117111 +0000 UTC m=+78.197936521 (1 nodes), 598.39784ms from start of poll, 598.9884ms from cluster start. Timeout at 2025-06-03 23:34:24.182502471 +0000 UTC m=+79.203321841, 1.00538532s after stability -2025-06-03T23:34:24.182Z [INFO] creating new snapshot: path=/tmp/raft3812908825/snapshots/2-1013-1748993664182.tmp -2025-06-03T23:34:24.183Z [INFO] server-68715001-1f59-b740-1700-a18d8fc03e78: copied to local snapshot: bytes=71 -2025-06-03T23:34:24.183Z [INFO] server-68715001-1f59-b740-1700-a18d8fc03e78: snapshot restore progress: id=2-1013-1748993664182 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:24.183Z [INFO] server-68715001-1f59-b740-1700-a18d8fc03e78: restored user snapshot: index=1013 -2025-06-03T23:34:24.184Z [ERROR] server-68715001-1f59-b740-1700-a18d8fc03e78: failed to get log: index=3 error="log not found" -2025-06-03T23:34:24.184Z [INFO] server-68715001-1f59-b740-1700-a18d8fc03e78: aborting pipeline replication: peer="{Voter server-20322ac7-0f51-2c27-7651-4de94392a37d 20322ac7-0f51-2c27-7651-4de94392a37d}" -2025-06-03T23:34:24.184Z [ERROR] server-68715001-1f59-b740-1700-a18d8fc03e78: failed to get log: index=3 error="log not found" -2025-06-03T23:34:24.184Z [INFO] server-68715001-1f59-b740-1700-a18d8fc03e78: aborting pipeline replication: peer="{Voter server-ffc601b9-02b4-e420-6b41-fd77a284880e ffc601b9-02b4-e420-6b41-fd77a284880e}" -2025-06-03T23:34:24.190Z [ERROR] server-68715001-1f59-b740-1700-a18d8fc03e78: failed to get log: index=3 error="log not found" -2025-06-03T23:34:24.191Z [INFO] creating new snapshot: path=/tmp/raft4223188703/snapshots/2-1013-1748993664191.tmp -2025-06-03T23:34:24.192Z [INFO] server-20322ac7-0f51-2c27-7651-4de94392a37d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:24.192Z [INFO] server-20322ac7-0f51-2c27-7651-4de94392a37d: copied to local snapshot: bytes=71 -2025-06-03T23:34:24.193Z [INFO] server-20322ac7-0f51-2c27-7651-4de94392a37d: snapshot restore progress: id=2-1013-1748993664191 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:24.193Z [INFO] server-20322ac7-0f51-2c27-7651-4de94392a37d: Installed remote snapshot -2025-06-03T23:34:24.193Z [ERROR] server-68715001-1f59-b740-1700-a18d8fc03e78: failed to get log: index=3 error="log not found" -2025-06-03T23:34:24.193Z [INFO] server-68715001-1f59-b740-1700-a18d8fc03e78: pipelining replication: peer="{Voter server-20322ac7-0f51-2c27-7651-4de94392a37d 20322ac7-0f51-2c27-7651-4de94392a37d}" -2025-06-03T23:34:24.193Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:24.194Z [INFO] creating new snapshot: path=/tmp/raft1325720631/snapshots/2-1013-1748993664194.tmp -2025-06-03T23:34:24.194Z [INFO] server-ffc601b9-02b4-e420-6b41-fd77a284880e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:24.194Z [INFO] server-ffc601b9-02b4-e420-6b41-fd77a284880e: copied to local snapshot: bytes=71 -2025-06-03T23:34:24.195Z [INFO] server-ffc601b9-02b4-e420-6b41-fd77a284880e: snapshot restore progress: id=2-1013-1748993664194 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:24.195Z [INFO] server-ffc601b9-02b4-e420-6b41-fd77a284880e: Installed remote snapshot -2025-06-03T23:34:24.195Z [INFO] server-68715001-1f59-b740-1700-a18d8fc03e78: pipelining replication: peer="{Voter server-ffc601b9-02b4-e420-6b41-fd77a284880e ffc601b9-02b4-e420-6b41-fd77a284880e}" -2025-06-03T23:34:25.199Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:24.193647391 +0000 UTC m=+79.214466761 (2 nodes), 1.44µs from start of poll, 1.61551864s from cluster start. Timeout at 2025-06-03 23:34:25.198953631 +0000 UTC m=+80.219773001, 1.00530624s after stability -2025-06-03T23:34:25.211Z [INFO] server-130d7054-bf3c-d9cb-7622-b31c429eef84: aborting pipeline replication: peer="{Voter server-92a584d8-e237-a22c-6a55-e8f17b768ca3 92a584d8-e237-a22c-6a55-e8f17b768ca3}" -2025-06-03T23:34:25.211Z [INFO] server-130d7054-bf3c-d9cb-7622-b31c429eef84: aborting pipeline replication: peer="{Voter server-32fe2812-6c0b-a915-ba34-fd53606482d2 32fe2812-6c0b-a915-ba34-fd53606482d2}" +2025-06-03T23:37:19.464Z [INFO] server-7a10c7af-27a3-6017-b32c-5a5c56eabd95: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a10c7af-27a3-6017-b32c-5a5c56eabd95 Address:7a10c7af-27a3-6017-b32c-5a5c56eabd95} {Suffrage:Voter ID:server-05c9d5f2-0de1-608c-a299-6ec1869317b6 Address:05c9d5f2-0de1-608c-a299-6ec1869317b6} {Suffrage:Voter ID:server-a7f2cb6d-1afc-ee79-e622-036deefba5af Address:a7f2cb6d-1afc-ee79-e622-036deefba5af}]" +2025-06-03T23:37:19.464Z [INFO] server-7a10c7af-27a3-6017-b32c-5a5c56eabd95: entering follower state: follower="Node at 7a10c7af-27a3-6017-b32c-5a5c56eabd95 [Follower]" leader-address= leader-id= +2025-06-03T23:37:19.464Z [INFO] server-05c9d5f2-0de1-608c-a299-6ec1869317b6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a10c7af-27a3-6017-b32c-5a5c56eabd95 Address:7a10c7af-27a3-6017-b32c-5a5c56eabd95} {Suffrage:Voter ID:server-05c9d5f2-0de1-608c-a299-6ec1869317b6 Address:05c9d5f2-0de1-608c-a299-6ec1869317b6} {Suffrage:Voter ID:server-a7f2cb6d-1afc-ee79-e622-036deefba5af Address:a7f2cb6d-1afc-ee79-e622-036deefba5af}]" +2025-06-03T23:37:19.464Z [INFO] server-05c9d5f2-0de1-608c-a299-6ec1869317b6: entering follower state: follower="Node at 05c9d5f2-0de1-608c-a299-6ec1869317b6 [Follower]" leader-address= leader-id= +2025-06-03T23:37:19.464Z [INFO] server-a7f2cb6d-1afc-ee79-e622-036deefba5af: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a10c7af-27a3-6017-b32c-5a5c56eabd95 Address:7a10c7af-27a3-6017-b32c-5a5c56eabd95} {Suffrage:Voter ID:server-05c9d5f2-0de1-608c-a299-6ec1869317b6 Address:05c9d5f2-0de1-608c-a299-6ec1869317b6} {Suffrage:Voter ID:server-a7f2cb6d-1afc-ee79-e622-036deefba5af Address:a7f2cb6d-1afc-ee79-e622-036deefba5af}]" +2025-06-03T23:37:19.464Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:19.464Z [INFO] server-a7f2cb6d-1afc-ee79-e622-036deefba5af: entering follower state: follower="Node at a7f2cb6d-1afc-ee79-e622-036deefba5af [Follower]" leader-address= leader-id= +2025-06-03T23:37:20.272Z [WARN] server-a7f2cb6d-1afc-ee79-e622-036deefba5af: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:20.272Z [INFO] server-a7f2cb6d-1afc-ee79-e622-036deefba5af: entering candidate state: node="Node at a7f2cb6d-1afc-ee79-e622-036deefba5af [Candidate]" term=2 +2025-06-03T23:37:20.272Z [INFO] server-a7f2cb6d-1afc-ee79-e622-036deefba5af: election won: term=2 tally=2 +2025-06-03T23:37:20.272Z [INFO] server-a7f2cb6d-1afc-ee79-e622-036deefba5af: entering leader state: leader="Node at a7f2cb6d-1afc-ee79-e622-036deefba5af [Leader]" +2025-06-03T23:37:20.272Z [INFO] server-a7f2cb6d-1afc-ee79-e622-036deefba5af: added peer, starting replication: peer=server-7a10c7af-27a3-6017-b32c-5a5c56eabd95 +2025-06-03T23:37:20.272Z [INFO] server-a7f2cb6d-1afc-ee79-e622-036deefba5af: added peer, starting replication: peer=server-05c9d5f2-0de1-608c-a299-6ec1869317b6 +2025-06-03T23:37:20.273Z [INFO] server-a7f2cb6d-1afc-ee79-e622-036deefba5af: pipelining replication: peer="{Voter server-05c9d5f2-0de1-608c-a299-6ec1869317b6 05c9d5f2-0de1-608c-a299-6ec1869317b6}" +2025-06-03T23:37:20.273Z [INFO] server-a7f2cb6d-1afc-ee79-e622-036deefba5af: pipelining replication: peer="{Voter server-7a10c7af-27a3-6017-b32c-5a5c56eabd95 7a10c7af-27a3-6017-b32c-5a5c56eabd95}" +2025-06-03T23:37:21.279Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:20.273002231 +0000 UTC m=+75.594757201 (1 nodes), 808.19692ms from start of poll, 808.81588ms from cluster start. Timeout at 2025-06-03 23:37:21.279013991 +0000 UTC m=+76.600768921, 1.00601172s after stability +2025-06-03T23:37:21.279Z [INFO] server-a7f2cb6d-1afc-ee79-e622-036deefba5af: starting snapshot up to: index=12 +2025-06-03T23:37:21.279Z [INFO] creating new snapshot: path=/tmp/raft783328824/snapshots/2-12-1748993841279.tmp +2025-06-03T23:37:21.280Z [INFO] server-a7f2cb6d-1afc-ee79-e622-036deefba5af: snapshot complete up to: index=12 +2025-06-03T23:37:21.281Z [INFO] server-08fe382b-8739-3173-4433-ca3a729caa39: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08fe382b-8739-3173-4433-ca3a729caa39 Address:08fe382b-8739-3173-4433-ca3a729caa39} {Suffrage:Voter ID:server-b0b93c89-4543-d7e1-9855-808462c593ae Address:b0b93c89-4543-d7e1-9855-808462c593ae} {Suffrage:Voter ID:server-6dd065d0-96f8-49b3-5ba6-63ab4a964037 Address:6dd065d0-96f8-49b3-5ba6-63ab4a964037}]" +2025-06-03T23:37:21.281Z [INFO] server-08fe382b-8739-3173-4433-ca3a729caa39: entering follower state: follower="Node at 08fe382b-8739-3173-4433-ca3a729caa39 [Follower]" leader-address= leader-id= +2025-06-03T23:37:21.281Z [INFO] server-b0b93c89-4543-d7e1-9855-808462c593ae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08fe382b-8739-3173-4433-ca3a729caa39 Address:08fe382b-8739-3173-4433-ca3a729caa39} {Suffrage:Voter ID:server-b0b93c89-4543-d7e1-9855-808462c593ae Address:b0b93c89-4543-d7e1-9855-808462c593ae} {Suffrage:Voter ID:server-6dd065d0-96f8-49b3-5ba6-63ab4a964037 Address:6dd065d0-96f8-49b3-5ba6-63ab4a964037}]" +2025-06-03T23:37:21.281Z [INFO] server-b0b93c89-4543-d7e1-9855-808462c593ae: entering follower state: follower="Node at b0b93c89-4543-d7e1-9855-808462c593ae [Follower]" leader-address= leader-id= +2025-06-03T23:37:21.281Z [INFO] server-6dd065d0-96f8-49b3-5ba6-63ab4a964037: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08fe382b-8739-3173-4433-ca3a729caa39 Address:08fe382b-8739-3173-4433-ca3a729caa39} {Suffrage:Voter ID:server-b0b93c89-4543-d7e1-9855-808462c593ae Address:b0b93c89-4543-d7e1-9855-808462c593ae} {Suffrage:Voter ID:server-6dd065d0-96f8-49b3-5ba6-63ab4a964037 Address:6dd065d0-96f8-49b3-5ba6-63ab4a964037}]" +2025-06-03T23:37:21.281Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:21.281Z [INFO] server-6dd065d0-96f8-49b3-5ba6-63ab4a964037: entering follower state: follower="Node at 6dd065d0-96f8-49b3-5ba6-63ab4a964037 [Follower]" leader-address= leader-id= +2025-06-03T23:37:21.900Z [WARN] server-08fe382b-8739-3173-4433-ca3a729caa39: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:21.901Z [INFO] server-08fe382b-8739-3173-4433-ca3a729caa39: entering candidate state: node="Node at 08fe382b-8739-3173-4433-ca3a729caa39 [Candidate]" term=2 +2025-06-03T23:37:21.901Z [INFO] server-08fe382b-8739-3173-4433-ca3a729caa39: election won: term=2 tally=2 +2025-06-03T23:37:21.901Z [INFO] server-08fe382b-8739-3173-4433-ca3a729caa39: entering leader state: leader="Node at 08fe382b-8739-3173-4433-ca3a729caa39 [Leader]" +2025-06-03T23:37:21.901Z [INFO] server-08fe382b-8739-3173-4433-ca3a729caa39: added peer, starting replication: peer=server-b0b93c89-4543-d7e1-9855-808462c593ae +2025-06-03T23:37:21.901Z [INFO] server-08fe382b-8739-3173-4433-ca3a729caa39: added peer, starting replication: peer=server-6dd065d0-96f8-49b3-5ba6-63ab4a964037 +2025-06-03T23:37:21.901Z [INFO] server-08fe382b-8739-3173-4433-ca3a729caa39: pipelining replication: peer="{Voter server-6dd065d0-96f8-49b3-5ba6-63ab4a964037 6dd065d0-96f8-49b3-5ba6-63ab4a964037}" +2025-06-03T23:37:21.901Z [INFO] server-08fe382b-8739-3173-4433-ca3a729caa39: pipelining replication: peer="{Voter server-b0b93c89-4543-d7e1-9855-808462c593ae b0b93c89-4543-d7e1-9855-808462c593ae}" +2025-06-03T23:37:22.906Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:21.901190631 +0000 UTC m=+77.222945561 (1 nodes), 619.6988ms from start of poll, 620.19252ms from cluster start. Timeout at 2025-06-03 23:37:22.906870991 +0000 UTC m=+78.228625921, 1.00568036s after stability +2025-06-03T23:37:22.907Z [INFO] creating new snapshot: path=/tmp/raft3197255700/snapshots/2-1013-1748993842907.tmp +2025-06-03T23:37:22.907Z [INFO] server-08fe382b-8739-3173-4433-ca3a729caa39: copied to local snapshot: bytes=71 +2025-06-03T23:37:22.908Z [INFO] server-08fe382b-8739-3173-4433-ca3a729caa39: snapshot restore progress: id=2-1013-1748993842907 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:22.908Z [INFO] server-08fe382b-8739-3173-4433-ca3a729caa39: restored user snapshot: index=1013 +2025-06-03T23:37:22.908Z [ERROR] server-08fe382b-8739-3173-4433-ca3a729caa39: failed to get log: index=3 error="log not found" +2025-06-03T23:37:22.908Z [ERROR] server-08fe382b-8739-3173-4433-ca3a729caa39: failed to get log: index=3 error="log not found" +2025-06-03T23:37:22.908Z [INFO] server-08fe382b-8739-3173-4433-ca3a729caa39: aborting pipeline replication: peer="{Voter server-b0b93c89-4543-d7e1-9855-808462c593ae b0b93c89-4543-d7e1-9855-808462c593ae}" +2025-06-03T23:37:22.908Z [INFO] server-08fe382b-8739-3173-4433-ca3a729caa39: aborting pipeline replication: peer="{Voter server-6dd065d0-96f8-49b3-5ba6-63ab4a964037 6dd065d0-96f8-49b3-5ba6-63ab4a964037}" +2025-06-03T23:37:22.914Z [ERROR] server-08fe382b-8739-3173-4433-ca3a729caa39: failed to get log: index=3 error="log not found" +2025-06-03T23:37:22.914Z [INFO] creating new snapshot: path=/tmp/raft777281508/snapshots/2-1013-1748993842914.tmp +2025-06-03T23:37:22.915Z [ERROR] server-08fe382b-8739-3173-4433-ca3a729caa39: failed to get log: index=3 error="log not found" +2025-06-03T23:37:22.915Z [INFO] creating new snapshot: path=/tmp/raft515245678/snapshots/2-1013-1748993842915.tmp +2025-06-03T23:37:22.916Z [INFO] server-b0b93c89-4543-d7e1-9855-808462c593ae: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:22.916Z [INFO] server-6dd065d0-96f8-49b3-5ba6-63ab4a964037: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:22.916Z [INFO] server-b0b93c89-4543-d7e1-9855-808462c593ae: copied to local snapshot: bytes=71 +2025-06-03T23:37:22.916Z [INFO] server-6dd065d0-96f8-49b3-5ba6-63ab4a964037: copied to local snapshot: bytes=71 +2025-06-03T23:37:22.917Z [INFO] server-b0b93c89-4543-d7e1-9855-808462c593ae: snapshot restore progress: id=2-1013-1748993842914 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:22.917Z [INFO] server-6dd065d0-96f8-49b3-5ba6-63ab4a964037: snapshot restore progress: id=2-1013-1748993842915 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:22.917Z [INFO] server-b0b93c89-4543-d7e1-9855-808462c593ae: Installed remote snapshot +2025-06-03T23:37:22.917Z [INFO] server-6dd065d0-96f8-49b3-5ba6-63ab4a964037: Installed remote snapshot +2025-06-03T23:37:22.917Z [INFO] server-08fe382b-8739-3173-4433-ca3a729caa39: pipelining replication: peer="{Voter server-b0b93c89-4543-d7e1-9855-808462c593ae b0b93c89-4543-d7e1-9855-808462c593ae}" +2025-06-03T23:37:22.917Z [INFO] server-08fe382b-8739-3173-4433-ca3a729caa39: pipelining replication: peer="{Voter server-6dd065d0-96f8-49b3-5ba6-63ab4a964037 6dd065d0-96f8-49b3-5ba6-63ab4a964037}" +2025-06-03T23:37:22.917Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:23.922Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:22.917813471 +0000 UTC m=+78.239568401 (2 nodes), 2.36µs from start of poll, 1.63681536s from cluster start. Timeout at 2025-06-03 23:37:23.922821711 +0000 UTC m=+79.244576641, 1.00500824s after stability +2025-06-03T23:37:23.934Z [INFO] server-a7f2cb6d-1afc-ee79-e622-036deefba5af: aborting pipeline replication: peer="{Voter server-05c9d5f2-0de1-608c-a299-6ec1869317b6 05c9d5f2-0de1-608c-a299-6ec1869317b6}" +2025-06-03T23:37:23.934Z [INFO] server-a7f2cb6d-1afc-ee79-e622-036deefba5af: aborting pipeline replication: peer="{Voter server-7a10c7af-27a3-6017-b32c-5a5c56eabd95 7a10c7af-27a3-6017-b32c-5a5c56eabd95}" === RUN TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true -2025-06-03T23:34:25.212Z [INFO] server-211852a0-37b5-f76c-2d56-38898e43afbd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-211852a0-37b5-f76c-2d56-38898e43afbd Address:211852a0-37b5-f76c-2d56-38898e43afbd} {Suffrage:Voter ID:server-c86feb72-baf0-ddb3-7a50-40be5dc42c8a Address:c86feb72-baf0-ddb3-7a50-40be5dc42c8a} {Suffrage:Voter ID:server-0f2af742-6952-7b35-5423-3f0ae862c7d8 Address:0f2af742-6952-7b35-5423-3f0ae862c7d8}]" -2025-06-03T23:34:25.212Z [INFO] server-211852a0-37b5-f76c-2d56-38898e43afbd: entering follower state: follower="Node at 211852a0-37b5-f76c-2d56-38898e43afbd [Follower]" leader-address= leader-id= -2025-06-03T23:34:25.213Z [INFO] server-c86feb72-baf0-ddb3-7a50-40be5dc42c8a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-211852a0-37b5-f76c-2d56-38898e43afbd Address:211852a0-37b5-f76c-2d56-38898e43afbd} {Suffrage:Voter ID:server-c86feb72-baf0-ddb3-7a50-40be5dc42c8a Address:c86feb72-baf0-ddb3-7a50-40be5dc42c8a} {Suffrage:Voter ID:server-0f2af742-6952-7b35-5423-3f0ae862c7d8 Address:0f2af742-6952-7b35-5423-3f0ae862c7d8}]" -2025-06-03T23:34:25.213Z [INFO] server-c86feb72-baf0-ddb3-7a50-40be5dc42c8a: entering follower state: follower="Node at c86feb72-baf0-ddb3-7a50-40be5dc42c8a [Follower]" leader-address= leader-id= -2025-06-03T23:34:25.213Z [INFO] server-0f2af742-6952-7b35-5423-3f0ae862c7d8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-211852a0-37b5-f76c-2d56-38898e43afbd Address:211852a0-37b5-f76c-2d56-38898e43afbd} {Suffrage:Voter ID:server-c86feb72-baf0-ddb3-7a50-40be5dc42c8a Address:c86feb72-baf0-ddb3-7a50-40be5dc42c8a} {Suffrage:Voter ID:server-0f2af742-6952-7b35-5423-3f0ae862c7d8 Address:0f2af742-6952-7b35-5423-3f0ae862c7d8}]" -2025-06-03T23:34:25.213Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:25.213Z [INFO] server-0f2af742-6952-7b35-5423-3f0ae862c7d8: entering follower state: follower="Node at 0f2af742-6952-7b35-5423-3f0ae862c7d8 [Follower]" leader-address= leader-id= -2025-06-03T23:34:25.734Z [WARN] server-0f2af742-6952-7b35-5423-3f0ae862c7d8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:25.734Z [INFO] server-0f2af742-6952-7b35-5423-3f0ae862c7d8: entering candidate state: node="Node at 0f2af742-6952-7b35-5423-3f0ae862c7d8 [Candidate]" term=2 -2025-06-03T23:34:25.734Z [INFO] server-0f2af742-6952-7b35-5423-3f0ae862c7d8: election won: term=2 tally=2 -2025-06-03T23:34:25.734Z [INFO] server-0f2af742-6952-7b35-5423-3f0ae862c7d8: entering leader state: leader="Node at 0f2af742-6952-7b35-5423-3f0ae862c7d8 [Leader]" -2025-06-03T23:34:25.734Z [INFO] server-0f2af742-6952-7b35-5423-3f0ae862c7d8: added peer, starting replication: peer=server-211852a0-37b5-f76c-2d56-38898e43afbd -2025-06-03T23:34:25.734Z [INFO] server-0f2af742-6952-7b35-5423-3f0ae862c7d8: added peer, starting replication: peer=server-c86feb72-baf0-ddb3-7a50-40be5dc42c8a -2025-06-03T23:34:25.735Z [INFO] server-0f2af742-6952-7b35-5423-3f0ae862c7d8: pipelining replication: peer="{Voter server-c86feb72-baf0-ddb3-7a50-40be5dc42c8a c86feb72-baf0-ddb3-7a50-40be5dc42c8a}" -2025-06-03T23:34:25.735Z [INFO] server-0f2af742-6952-7b35-5423-3f0ae862c7d8: pipelining replication: peer="{Voter server-211852a0-37b5-f76c-2d56-38898e43afbd 211852a0-37b5-f76c-2d56-38898e43afbd}" -2025-06-03T23:34:26.740Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:25.734892151 +0000 UTC m=+80.755711521 (1 nodes), 521.62392ms from start of poll, 522.2972ms from cluster start. Timeout at 2025-06-03 23:34:26.740889511 +0000 UTC m=+81.761708881, 1.00599736s after stability -2025-06-03T23:34:26.742Z [INFO] server-0f2af742-6952-7b35-5423-3f0ae862c7d8: starting snapshot up to: index=12 -2025-06-03T23:34:26.742Z [INFO] creating new snapshot: path=/tmp/raft3327685587/snapshots/2-12-1748993666742.tmp -2025-06-03T23:34:26.744Z [INFO] server-0f2af742-6952-7b35-5423-3f0ae862c7d8: snapshot complete up to: index=12 -2025-06-03T23:34:26.745Z [INFO] server-fad2a483-ad24-90be-3e40-600804ab4c62: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fad2a483-ad24-90be-3e40-600804ab4c62 Address:fad2a483-ad24-90be-3e40-600804ab4c62} {Suffrage:Voter ID:server-626918ca-22fd-22b0-5082-944c390b3d38 Address:626918ca-22fd-22b0-5082-944c390b3d38} {Suffrage:Voter ID:server-283dc8e9-4785-2fe1-d4fb-e0f117209894 Address:283dc8e9-4785-2fe1-d4fb-e0f117209894}]" -2025-06-03T23:34:26.745Z [INFO] server-fad2a483-ad24-90be-3e40-600804ab4c62: entering follower state: follower="Node at fad2a483-ad24-90be-3e40-600804ab4c62 [Follower]" leader-address= leader-id= -2025-06-03T23:34:26.745Z [INFO] server-626918ca-22fd-22b0-5082-944c390b3d38: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fad2a483-ad24-90be-3e40-600804ab4c62 Address:fad2a483-ad24-90be-3e40-600804ab4c62} {Suffrage:Voter ID:server-626918ca-22fd-22b0-5082-944c390b3d38 Address:626918ca-22fd-22b0-5082-944c390b3d38} {Suffrage:Voter ID:server-283dc8e9-4785-2fe1-d4fb-e0f117209894 Address:283dc8e9-4785-2fe1-d4fb-e0f117209894}]" -2025-06-03T23:34:26.745Z [INFO] server-626918ca-22fd-22b0-5082-944c390b3d38: entering follower state: follower="Node at 626918ca-22fd-22b0-5082-944c390b3d38 [Follower]" leader-address= leader-id= -2025-06-03T23:34:26.745Z [INFO] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fad2a483-ad24-90be-3e40-600804ab4c62 Address:fad2a483-ad24-90be-3e40-600804ab4c62} {Suffrage:Voter ID:server-626918ca-22fd-22b0-5082-944c390b3d38 Address:626918ca-22fd-22b0-5082-944c390b3d38} {Suffrage:Voter ID:server-283dc8e9-4785-2fe1-d4fb-e0f117209894 Address:283dc8e9-4785-2fe1-d4fb-e0f117209894}]" -2025-06-03T23:34:26.745Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:26.745Z [INFO] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: entering follower state: follower="Node at 283dc8e9-4785-2fe1-d4fb-e0f117209894 [Follower]" leader-address= leader-id= -2025-06-03T23:34:27.411Z [WARN] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:27.411Z [INFO] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: entering candidate state: node="Node at 283dc8e9-4785-2fe1-d4fb-e0f117209894 [Candidate]" term=2 -2025-06-03T23:34:27.411Z [INFO] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: election won: term=2 tally=2 -2025-06-03T23:34:27.411Z [INFO] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: entering leader state: leader="Node at 283dc8e9-4785-2fe1-d4fb-e0f117209894 [Leader]" -2025-06-03T23:34:27.411Z [INFO] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: added peer, starting replication: peer=server-fad2a483-ad24-90be-3e40-600804ab4c62 -2025-06-03T23:34:27.411Z [INFO] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: added peer, starting replication: peer=server-626918ca-22fd-22b0-5082-944c390b3d38 -2025-06-03T23:34:27.412Z [INFO] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: pipelining replication: peer="{Voter server-fad2a483-ad24-90be-3e40-600804ab4c62 fad2a483-ad24-90be-3e40-600804ab4c62}" -2025-06-03T23:34:27.412Z [INFO] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: pipelining replication: peer="{Voter server-626918ca-22fd-22b0-5082-944c390b3d38 626918ca-22fd-22b0-5082-944c390b3d38}" -2025-06-03T23:34:28.417Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:27.412003711 +0000 UTC m=+82.432823081 (1 nodes), 666.446ms from start of poll, 666.8984ms from cluster start. Timeout at 2025-06-03 23:34:28.417095031 +0000 UTC m=+83.437914481, 1.0050914s after stability -2025-06-03T23:34:28.417Z [INFO] creating new snapshot: path=/tmp/raft2428187032/snapshots/2-1013-1748993668417.tmp -2025-06-03T23:34:28.418Z [INFO] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: copied to local snapshot: bytes=71 -2025-06-03T23:34:28.419Z [INFO] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: snapshot restore progress: id=2-1013-1748993668417 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:28.419Z [INFO] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: removing all old logs from log store -2025-06-03T23:34:28.419Z [INFO] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: compacting logs: from=1 to=2 -2025-06-03T23:34:28.419Z [INFO] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: restored user snapshot: index=1013 -2025-06-03T23:34:28.419Z [ERROR] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: failed to get log: index=2 error="log not found" -2025-06-03T23:34:28.419Z [ERROR] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: failed to get log: index=2 error="log not found" -2025-06-03T23:34:28.419Z [INFO] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: aborting pipeline replication: peer="{Voter server-626918ca-22fd-22b0-5082-944c390b3d38 626918ca-22fd-22b0-5082-944c390b3d38}" -2025-06-03T23:34:28.419Z [INFO] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: aborting pipeline replication: peer="{Voter server-fad2a483-ad24-90be-3e40-600804ab4c62 fad2a483-ad24-90be-3e40-600804ab4c62}" -2025-06-03T23:34:28.430Z [ERROR] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: failed to get log: index=2 error="log not found" -2025-06-03T23:34:28.430Z [ERROR] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: failed to get log: index=2 error="log not found" -2025-06-03T23:34:28.430Z [INFO] creating new snapshot: path=/tmp/raft4018983311/snapshots/2-1013-1748993668430.tmp -2025-06-03T23:34:28.430Z [INFO] creating new snapshot: path=/tmp/raft597018769/snapshots/2-1013-1748993668430.tmp -2025-06-03T23:34:28.431Z [INFO] server-626918ca-22fd-22b0-5082-944c390b3d38: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:28.431Z [INFO] server-fad2a483-ad24-90be-3e40-600804ab4c62: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:28.431Z [INFO] server-626918ca-22fd-22b0-5082-944c390b3d38: copied to local snapshot: bytes=71 -2025-06-03T23:34:28.431Z [INFO] server-fad2a483-ad24-90be-3e40-600804ab4c62: copied to local snapshot: bytes=71 -2025-06-03T23:34:28.431Z [INFO] server-626918ca-22fd-22b0-5082-944c390b3d38: snapshot restore progress: id=2-1013-1748993668430 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:28.431Z [INFO] server-fad2a483-ad24-90be-3e40-600804ab4c62: snapshot restore progress: id=2-1013-1748993668430 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:28.431Z [INFO] server-626918ca-22fd-22b0-5082-944c390b3d38: removing all old logs from log store -2025-06-03T23:34:28.431Z [INFO] server-626918ca-22fd-22b0-5082-944c390b3d38: compacting logs: from=1 to=2 -2025-06-03T23:34:28.431Z [INFO] server-626918ca-22fd-22b0-5082-944c390b3d38: Installed remote snapshot -2025-06-03T23:34:28.431Z [INFO] server-fad2a483-ad24-90be-3e40-600804ab4c62: removing all old logs from log store -2025-06-03T23:34:28.431Z [INFO] server-fad2a483-ad24-90be-3e40-600804ab4c62: compacting logs: from=1 to=2 -2025-06-03T23:34:28.431Z [INFO] server-fad2a483-ad24-90be-3e40-600804ab4c62: Installed remote snapshot -2025-06-03T23:34:28.431Z [INFO] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: pipelining replication: peer="{Voter server-626918ca-22fd-22b0-5082-944c390b3d38 626918ca-22fd-22b0-5082-944c390b3d38}" -2025-06-03T23:34:28.431Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:28.431Z [INFO] server-283dc8e9-4785-2fe1-d4fb-e0f117209894: pipelining replication: peer="{Voter server-fad2a483-ad24-90be-3e40-600804ab4c62 fad2a483-ad24-90be-3e40-600804ab4c62}" -2025-06-03T23:34:29.437Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:28.431940031 +0000 UTC m=+83.452759401 (2 nodes), 1.12µs from start of poll, 1.68683472s from cluster start. Timeout at 2025-06-03 23:34:29.437814231 +0000 UTC m=+84.458633641, 1.00587424s after stability -2025-06-03T23:34:29.449Z [INFO] server-0f2af742-6952-7b35-5423-3f0ae862c7d8: aborting pipeline replication: peer="{Voter server-c86feb72-baf0-ddb3-7a50-40be5dc42c8a c86feb72-baf0-ddb3-7a50-40be5dc42c8a}" -2025-06-03T23:34:29.449Z [INFO] server-0f2af742-6952-7b35-5423-3f0ae862c7d8: aborting pipeline replication: peer="{Voter server-211852a0-37b5-f76c-2d56-38898e43afbd 211852a0-37b5-f76c-2d56-38898e43afbd}" +2025-06-03T23:37:23.936Z [INFO] server-c20c57be-d95d-8181-1496-799ae25451dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c20c57be-d95d-8181-1496-799ae25451dd Address:c20c57be-d95d-8181-1496-799ae25451dd} {Suffrage:Voter ID:server-8a03aca5-292c-576b-570a-8bada7f65827 Address:8a03aca5-292c-576b-570a-8bada7f65827} {Suffrage:Voter ID:server-0b7e18c1-859c-7473-7820-249bb5b43bce Address:0b7e18c1-859c-7473-7820-249bb5b43bce}]" +2025-06-03T23:37:23.936Z [INFO] server-c20c57be-d95d-8181-1496-799ae25451dd: entering follower state: follower="Node at c20c57be-d95d-8181-1496-799ae25451dd [Follower]" leader-address= leader-id= +2025-06-03T23:37:23.936Z [INFO] server-8a03aca5-292c-576b-570a-8bada7f65827: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c20c57be-d95d-8181-1496-799ae25451dd Address:c20c57be-d95d-8181-1496-799ae25451dd} {Suffrage:Voter ID:server-8a03aca5-292c-576b-570a-8bada7f65827 Address:8a03aca5-292c-576b-570a-8bada7f65827} {Suffrage:Voter ID:server-0b7e18c1-859c-7473-7820-249bb5b43bce Address:0b7e18c1-859c-7473-7820-249bb5b43bce}]" +2025-06-03T23:37:23.936Z [INFO] server-8a03aca5-292c-576b-570a-8bada7f65827: entering follower state: follower="Node at 8a03aca5-292c-576b-570a-8bada7f65827 [Follower]" leader-address= leader-id= +2025-06-03T23:37:23.936Z [INFO] server-0b7e18c1-859c-7473-7820-249bb5b43bce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c20c57be-d95d-8181-1496-799ae25451dd Address:c20c57be-d95d-8181-1496-799ae25451dd} {Suffrage:Voter ID:server-8a03aca5-292c-576b-570a-8bada7f65827 Address:8a03aca5-292c-576b-570a-8bada7f65827} {Suffrage:Voter ID:server-0b7e18c1-859c-7473-7820-249bb5b43bce Address:0b7e18c1-859c-7473-7820-249bb5b43bce}]" +2025-06-03T23:37:23.936Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:23.936Z [INFO] server-0b7e18c1-859c-7473-7820-249bb5b43bce: entering follower state: follower="Node at 0b7e18c1-859c-7473-7820-249bb5b43bce [Follower]" leader-address= leader-id= +2025-06-03T23:37:24.458Z [WARN] server-c20c57be-d95d-8181-1496-799ae25451dd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:24.458Z [INFO] server-c20c57be-d95d-8181-1496-799ae25451dd: entering candidate state: node="Node at c20c57be-d95d-8181-1496-799ae25451dd [Candidate]" term=2 +2025-06-03T23:37:24.458Z [INFO] server-c20c57be-d95d-8181-1496-799ae25451dd: election won: term=2 tally=2 +2025-06-03T23:37:24.458Z [INFO] server-c20c57be-d95d-8181-1496-799ae25451dd: entering leader state: leader="Node at c20c57be-d95d-8181-1496-799ae25451dd [Leader]" +2025-06-03T23:37:24.458Z [INFO] server-c20c57be-d95d-8181-1496-799ae25451dd: added peer, starting replication: peer=server-8a03aca5-292c-576b-570a-8bada7f65827 +2025-06-03T23:37:24.458Z [INFO] server-c20c57be-d95d-8181-1496-799ae25451dd: added peer, starting replication: peer=server-0b7e18c1-859c-7473-7820-249bb5b43bce +2025-06-03T23:37:24.458Z [INFO] server-c20c57be-d95d-8181-1496-799ae25451dd: pipelining replication: peer="{Voter server-8a03aca5-292c-576b-570a-8bada7f65827 8a03aca5-292c-576b-570a-8bada7f65827}" +2025-06-03T23:37:24.458Z [INFO] server-c20c57be-d95d-8181-1496-799ae25451dd: pipelining replication: peer="{Voter server-0b7e18c1-859c-7473-7820-249bb5b43bce 0b7e18c1-859c-7473-7820-249bb5b43bce}" +2025-06-03T23:37:25.463Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:24.458348111 +0000 UTC m=+79.780103041 (1 nodes), 521.96444ms from start of poll, 522.43336ms from cluster start. Timeout at 2025-06-03 23:37:25.463784231 +0000 UTC m=+80.785539161, 1.00543612s after stability +2025-06-03T23:37:25.464Z [INFO] server-c20c57be-d95d-8181-1496-799ae25451dd: starting snapshot up to: index=12 +2025-06-03T23:37:25.464Z [INFO] creating new snapshot: path=/tmp/raft3128464589/snapshots/2-12-1748993845464.tmp +2025-06-03T23:37:25.464Z [INFO] server-c20c57be-d95d-8181-1496-799ae25451dd: snapshot complete up to: index=12 +2025-06-03T23:37:25.465Z [INFO] server-9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c Address:9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c} {Suffrage:Voter ID:server-5bd688af-73d6-6be8-8f63-f26a09bbef82 Address:5bd688af-73d6-6be8-8f63-f26a09bbef82} {Suffrage:Voter ID:server-b86fbcd2-9bf2-78a8-acd3-09e233dec624 Address:b86fbcd2-9bf2-78a8-acd3-09e233dec624}]" +2025-06-03T23:37:25.466Z [INFO] server-9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c: entering follower state: follower="Node at 9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c [Follower]" leader-address= leader-id= +2025-06-03T23:37:25.466Z [INFO] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c Address:9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c} {Suffrage:Voter ID:server-5bd688af-73d6-6be8-8f63-f26a09bbef82 Address:5bd688af-73d6-6be8-8f63-f26a09bbef82} {Suffrage:Voter ID:server-b86fbcd2-9bf2-78a8-acd3-09e233dec624 Address:b86fbcd2-9bf2-78a8-acd3-09e233dec624}]" +2025-06-03T23:37:25.466Z [INFO] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: entering follower state: follower="Node at 5bd688af-73d6-6be8-8f63-f26a09bbef82 [Follower]" leader-address= leader-id= +2025-06-03T23:37:25.466Z [INFO] server-b86fbcd2-9bf2-78a8-acd3-09e233dec624: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c Address:9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c} {Suffrage:Voter ID:server-5bd688af-73d6-6be8-8f63-f26a09bbef82 Address:5bd688af-73d6-6be8-8f63-f26a09bbef82} {Suffrage:Voter ID:server-b86fbcd2-9bf2-78a8-acd3-09e233dec624 Address:b86fbcd2-9bf2-78a8-acd3-09e233dec624}]" +2025-06-03T23:37:25.466Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:25.466Z [INFO] server-b86fbcd2-9bf2-78a8-acd3-09e233dec624: entering follower state: follower="Node at b86fbcd2-9bf2-78a8-acd3-09e233dec624 [Follower]" leader-address= leader-id= +2025-06-03T23:37:26.034Z [WARN] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:26.034Z [INFO] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: entering candidate state: node="Node at 5bd688af-73d6-6be8-8f63-f26a09bbef82 [Candidate]" term=2 +2025-06-03T23:37:26.035Z [INFO] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: election won: term=2 tally=2 +2025-06-03T23:37:26.035Z [INFO] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: entering leader state: leader="Node at 5bd688af-73d6-6be8-8f63-f26a09bbef82 [Leader]" +2025-06-03T23:37:26.035Z [INFO] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: added peer, starting replication: peer=server-9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c +2025-06-03T23:37:26.035Z [INFO] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: added peer, starting replication: peer=server-b86fbcd2-9bf2-78a8-acd3-09e233dec624 +2025-06-03T23:37:26.035Z [INFO] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: pipelining replication: peer="{Voter server-b86fbcd2-9bf2-78a8-acd3-09e233dec624 b86fbcd2-9bf2-78a8-acd3-09e233dec624}" +2025-06-03T23:37:26.035Z [INFO] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: pipelining replication: peer="{Voter server-9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c 9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c}" +2025-06-03T23:37:27.040Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:26.035107231 +0000 UTC m=+81.356862161 (1 nodes), 568.77296ms from start of poll, 569.28124ms from cluster start. Timeout at 2025-06-03 23:37:27.040638591 +0000 UTC m=+82.362393521, 1.00553136s after stability +2025-06-03T23:37:27.040Z [INFO] creating new snapshot: path=/tmp/raft3573897431/snapshots/2-1013-1748993847040.tmp +2025-06-03T23:37:27.041Z [INFO] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: copied to local snapshot: bytes=71 +2025-06-03T23:37:27.041Z [INFO] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: snapshot restore progress: id=2-1013-1748993847040 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:27.041Z [INFO] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: removing all old logs from log store +2025-06-03T23:37:27.041Z [INFO] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: compacting logs: from=1 to=2 +2025-06-03T23:37:27.041Z [INFO] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: restored user snapshot: index=1013 +2025-06-03T23:37:27.041Z [ERROR] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: failed to get log: index=2 error="log not found" +2025-06-03T23:37:27.041Z [INFO] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: aborting pipeline replication: peer="{Voter server-b86fbcd2-9bf2-78a8-acd3-09e233dec624 b86fbcd2-9bf2-78a8-acd3-09e233dec624}" +2025-06-03T23:37:27.041Z [ERROR] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: failed to get log: index=2 error="log not found" +2025-06-03T23:37:27.041Z [INFO] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: aborting pipeline replication: peer="{Voter server-9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c 9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c}" +2025-06-03T23:37:27.048Z [ERROR] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: failed to get log: index=2 error="log not found" +2025-06-03T23:37:27.048Z [INFO] creating new snapshot: path=/tmp/raft642673724/snapshots/2-1013-1748993847048.tmp +2025-06-03T23:37:27.049Z [INFO] server-b86fbcd2-9bf2-78a8-acd3-09e233dec624: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:27.049Z [INFO] server-b86fbcd2-9bf2-78a8-acd3-09e233dec624: copied to local snapshot: bytes=71 +2025-06-03T23:37:27.049Z [INFO] server-b86fbcd2-9bf2-78a8-acd3-09e233dec624: snapshot restore progress: id=2-1013-1748993847048 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:27.049Z [INFO] server-b86fbcd2-9bf2-78a8-acd3-09e233dec624: removing all old logs from log store +2025-06-03T23:37:27.049Z [INFO] server-b86fbcd2-9bf2-78a8-acd3-09e233dec624: compacting logs: from=1 to=2 +2025-06-03T23:37:27.049Z [INFO] server-b86fbcd2-9bf2-78a8-acd3-09e233dec624: Installed remote snapshot +2025-06-03T23:37:27.049Z [INFO] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: pipelining replication: peer="{Voter server-b86fbcd2-9bf2-78a8-acd3-09e233dec624 b86fbcd2-9bf2-78a8-acd3-09e233dec624}" +2025-06-03T23:37:27.049Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:27.052Z [ERROR] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: failed to get log: index=2 error="log not found" +2025-06-03T23:37:27.052Z [INFO] creating new snapshot: path=/tmp/raft240762024/snapshots/2-1013-1748993847052.tmp +2025-06-03T23:37:27.052Z [INFO] server-9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:27.052Z [INFO] server-9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c: copied to local snapshot: bytes=71 +2025-06-03T23:37:27.053Z [INFO] server-9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c: snapshot restore progress: id=2-1013-1748993847052 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:27.053Z [INFO] server-9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c: removing all old logs from log store +2025-06-03T23:37:27.053Z [INFO] server-9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c: compacting logs: from=1 to=2 +2025-06-03T23:37:27.053Z [INFO] server-9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c: Installed remote snapshot +2025-06-03T23:37:27.053Z [INFO] server-5bd688af-73d6-6be8-8f63-f26a09bbef82: pipelining replication: peer="{Voter server-9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c 9b5eebb4-ec76-18e6-5546-d16bdb5a1a6c}" +2025-06-03T23:37:28.055Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:27.049900671 +0000 UTC m=+82.371655601 (2 nodes), 1.64µs from start of poll, 1.58407468s from cluster start. Timeout at 2025-06-03 23:37:28.055214791 +0000 UTC m=+83.376969721, 1.00531412s after stability +2025-06-03T23:37:28.066Z [INFO] server-c20c57be-d95d-8181-1496-799ae25451dd: aborting pipeline replication: peer="{Voter server-0b7e18c1-859c-7473-7820-249bb5b43bce 0b7e18c1-859c-7473-7820-249bb5b43bce}" +2025-06-03T23:37:28.066Z [INFO] server-c20c57be-d95d-8181-1496-799ae25451dd: aborting pipeline replication: peer="{Voter server-8a03aca5-292c-576b-570a-8bada7f65827 8a03aca5-292c-576b-570a-8bada7f65827}" === RUN TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false -2025-06-03T23:34:29.450Z [INFO] server-3989d0ef-ac1c-5194-ffa7-2b9b4c18486d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3989d0ef-ac1c-5194-ffa7-2b9b4c18486d Address:3989d0ef-ac1c-5194-ffa7-2b9b4c18486d} {Suffrage:Voter ID:server-b8609cae-5bfb-30c8-16ad-8a977d21f9fa Address:b8609cae-5bfb-30c8-16ad-8a977d21f9fa} {Suffrage:Voter ID:server-f8d5815e-8c37-e7de-9d8d-9f915a593186 Address:f8d5815e-8c37-e7de-9d8d-9f915a593186}]" -2025-06-03T23:34:29.450Z [INFO] server-3989d0ef-ac1c-5194-ffa7-2b9b4c18486d: entering follower state: follower="Node at 3989d0ef-ac1c-5194-ffa7-2b9b4c18486d [Follower]" leader-address= leader-id= -2025-06-03T23:34:29.450Z [INFO] server-b8609cae-5bfb-30c8-16ad-8a977d21f9fa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3989d0ef-ac1c-5194-ffa7-2b9b4c18486d Address:3989d0ef-ac1c-5194-ffa7-2b9b4c18486d} {Suffrage:Voter ID:server-b8609cae-5bfb-30c8-16ad-8a977d21f9fa Address:b8609cae-5bfb-30c8-16ad-8a977d21f9fa} {Suffrage:Voter ID:server-f8d5815e-8c37-e7de-9d8d-9f915a593186 Address:f8d5815e-8c37-e7de-9d8d-9f915a593186}]" -2025-06-03T23:34:29.450Z [INFO] server-b8609cae-5bfb-30c8-16ad-8a977d21f9fa: entering follower state: follower="Node at b8609cae-5bfb-30c8-16ad-8a977d21f9fa [Follower]" leader-address= leader-id= -2025-06-03T23:34:29.450Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3989d0ef-ac1c-5194-ffa7-2b9b4c18486d Address:3989d0ef-ac1c-5194-ffa7-2b9b4c18486d} {Suffrage:Voter ID:server-b8609cae-5bfb-30c8-16ad-8a977d21f9fa Address:b8609cae-5bfb-30c8-16ad-8a977d21f9fa} {Suffrage:Voter ID:server-f8d5815e-8c37-e7de-9d8d-9f915a593186 Address:f8d5815e-8c37-e7de-9d8d-9f915a593186}]" -2025-06-03T23:34:29.450Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:29.450Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: entering follower state: follower="Node at f8d5815e-8c37-e7de-9d8d-9f915a593186 [Follower]" leader-address= leader-id= -2025-06-03T23:34:29.981Z [WARN] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:29.981Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: entering candidate state: node="Node at f8d5815e-8c37-e7de-9d8d-9f915a593186 [Candidate]" term=2 -2025-06-03T23:34:29.982Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: election won: term=2 tally=2 -2025-06-03T23:34:29.982Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: entering leader state: leader="Node at f8d5815e-8c37-e7de-9d8d-9f915a593186 [Leader]" -2025-06-03T23:34:29.982Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: added peer, starting replication: peer=server-3989d0ef-ac1c-5194-ffa7-2b9b4c18486d -2025-06-03T23:34:29.982Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: added peer, starting replication: peer=server-b8609cae-5bfb-30c8-16ad-8a977d21f9fa -2025-06-03T23:34:29.982Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: pipelining replication: peer="{Voter server-b8609cae-5bfb-30c8-16ad-8a977d21f9fa b8609cae-5bfb-30c8-16ad-8a977d21f9fa}" -2025-06-03T23:34:29.982Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: pipelining replication: peer="{Voter server-3989d0ef-ac1c-5194-ffa7-2b9b4c18486d 3989d0ef-ac1c-5194-ffa7-2b9b4c18486d}" -2025-06-03T23:34:30.988Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:29.982057991 +0000 UTC m=+85.002877361 (1 nodes), 531.39604ms from start of poll, 531.92528ms from cluster start. Timeout at 2025-06-03 23:34:30.988144711 +0000 UTC m=+86.008964081, 1.00608672s after stability -2025-06-03T23:34:30.988Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: starting snapshot up to: index=12 -2025-06-03T23:34:30.988Z [INFO] creating new snapshot: path=/tmp/raft2671263857/snapshots/2-12-1748993670988.tmp -2025-06-03T23:34:30.989Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: snapshot complete up to: index=12 -2025-06-03T23:34:30.990Z [INFO] creating new snapshot: path=/tmp/raft2671263857/snapshots/2-10013-1748993670990.tmp -2025-06-03T23:34:30.990Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: copied to local snapshot: bytes=71 -2025-06-03T23:34:30.991Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: snapshot restore progress: id=2-10013-1748993670990 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:30.991Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: restored user snapshot: index=10013 -2025-06-03T23:34:30.991Z [ERROR] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: failed to get log: index=23 error="log not found" -2025-06-03T23:34:30.991Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: aborting pipeline replication: peer="{Voter server-3989d0ef-ac1c-5194-ffa7-2b9b4c18486d 3989d0ef-ac1c-5194-ffa7-2b9b4c18486d}" -2025-06-03T23:34:30.991Z [ERROR] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: failed to get log: index=23 error="log not found" -2025-06-03T23:34:30.991Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: aborting pipeline replication: peer="{Voter server-b8609cae-5bfb-30c8-16ad-8a977d21f9fa b8609cae-5bfb-30c8-16ad-8a977d21f9fa}" -2025-06-03T23:34:30.997Z [ERROR] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: failed to get log: index=23 error="log not found" -2025-06-03T23:34:30.998Z [INFO] creating new snapshot: path=/tmp/raft2589870041/snapshots/2-10013-1748993670998.tmp -2025-06-03T23:34:30.998Z [INFO] server-3989d0ef-ac1c-5194-ffa7-2b9b4c18486d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:30.998Z [INFO] server-3989d0ef-ac1c-5194-ffa7-2b9b4c18486d: copied to local snapshot: bytes=71 -2025-06-03T23:34:30.999Z [ERROR] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: failed to get log: index=23 error="log not found" -2025-06-03T23:34:30.999Z [INFO] server-3989d0ef-ac1c-5194-ffa7-2b9b4c18486d: snapshot restore progress: id=2-10013-1748993670998 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:30.999Z [INFO] server-3989d0ef-ac1c-5194-ffa7-2b9b4c18486d: Installed remote snapshot -2025-06-03T23:34:30.999Z [WARN] server-3989d0ef-ac1c-5194-ffa7-2b9b4c18486d: skipping application of old log: index=22 -2025-06-03T23:34:30.999Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: pipelining replication: peer="{Voter server-3989d0ef-ac1c-5194-ffa7-2b9b4c18486d 3989d0ef-ac1c-5194-ffa7-2b9b4c18486d}" -2025-06-03T23:34:30.999Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:30.999Z [INFO] creating new snapshot: path=/tmp/raft1469339055/snapshots/2-10013-1748993670999.tmp -2025-06-03T23:34:31.000Z [INFO] server-b8609cae-5bfb-30c8-16ad-8a977d21f9fa: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:31.001Z [INFO] server-b8609cae-5bfb-30c8-16ad-8a977d21f9fa: copied to local snapshot: bytes=71 -2025-06-03T23:34:31.002Z [INFO] server-b8609cae-5bfb-30c8-16ad-8a977d21f9fa: snapshot restore progress: id=2-10013-1748993670999 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:31.003Z [INFO] server-b8609cae-5bfb-30c8-16ad-8a977d21f9fa: Installed remote snapshot -2025-06-03T23:34:31.003Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: pipelining replication: peer="{Voter server-b8609cae-5bfb-30c8-16ad-8a977d21f9fa b8609cae-5bfb-30c8-16ad-8a977d21f9fa}" -2025-06-03T23:34:32.005Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:30.999499751 +0000 UTC m=+86.020319121 (2 nodes), 1.28µs from start of poll, 1.54936704s from cluster start. Timeout at 2025-06-03 23:34:32.005459591 +0000 UTC m=+87.026278961, 1.00595984s after stability -2025-06-03T23:34:32.016Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: aborting pipeline replication: peer="{Voter server-b8609cae-5bfb-30c8-16ad-8a977d21f9fa b8609cae-5bfb-30c8-16ad-8a977d21f9fa}" -2025-06-03T23:34:32.016Z [INFO] server-f8d5815e-8c37-e7de-9d8d-9f915a593186: aborting pipeline replication: peer="{Voter server-3989d0ef-ac1c-5194-ffa7-2b9b4c18486d 3989d0ef-ac1c-5194-ffa7-2b9b4c18486d}" +2025-06-03T23:37:28.067Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6df11825-b6bb-4be4-3c0d-ba177a36efc0 Address:6df11825-b6bb-4be4-3c0d-ba177a36efc0} {Suffrage:Voter ID:server-26283a3f-610f-7984-3224-97b8e1a582db Address:26283a3f-610f-7984-3224-97b8e1a582db} {Suffrage:Voter ID:server-d1a63c28-0a19-b51a-1fd7-984e6dbd51bb Address:d1a63c28-0a19-b51a-1fd7-984e6dbd51bb}]" +2025-06-03T23:37:28.068Z [INFO] server-26283a3f-610f-7984-3224-97b8e1a582db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6df11825-b6bb-4be4-3c0d-ba177a36efc0 Address:6df11825-b6bb-4be4-3c0d-ba177a36efc0} {Suffrage:Voter ID:server-26283a3f-610f-7984-3224-97b8e1a582db Address:26283a3f-610f-7984-3224-97b8e1a582db} {Suffrage:Voter ID:server-d1a63c28-0a19-b51a-1fd7-984e6dbd51bb Address:d1a63c28-0a19-b51a-1fd7-984e6dbd51bb}]" +2025-06-03T23:37:28.068Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: entering follower state: follower="Node at 6df11825-b6bb-4be4-3c0d-ba177a36efc0 [Follower]" leader-address= leader-id= +2025-06-03T23:37:28.068Z [INFO] server-26283a3f-610f-7984-3224-97b8e1a582db: entering follower state: follower="Node at 26283a3f-610f-7984-3224-97b8e1a582db [Follower]" leader-address= leader-id= +2025-06-03T23:37:28.068Z [INFO] server-d1a63c28-0a19-b51a-1fd7-984e6dbd51bb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6df11825-b6bb-4be4-3c0d-ba177a36efc0 Address:6df11825-b6bb-4be4-3c0d-ba177a36efc0} {Suffrage:Voter ID:server-26283a3f-610f-7984-3224-97b8e1a582db Address:26283a3f-610f-7984-3224-97b8e1a582db} {Suffrage:Voter ID:server-d1a63c28-0a19-b51a-1fd7-984e6dbd51bb Address:d1a63c28-0a19-b51a-1fd7-984e6dbd51bb}]" +2025-06-03T23:37:28.068Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:28.068Z [INFO] server-d1a63c28-0a19-b51a-1fd7-984e6dbd51bb: entering follower state: follower="Node at d1a63c28-0a19-b51a-1fd7-984e6dbd51bb [Follower]" leader-address= leader-id= +2025-06-03T23:37:28.702Z [WARN] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:28.702Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: entering candidate state: node="Node at 6df11825-b6bb-4be4-3c0d-ba177a36efc0 [Candidate]" term=2 +2025-06-03T23:37:28.702Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: election won: term=2 tally=2 +2025-06-03T23:37:28.702Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: entering leader state: leader="Node at 6df11825-b6bb-4be4-3c0d-ba177a36efc0 [Leader]" +2025-06-03T23:37:28.702Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: added peer, starting replication: peer=server-26283a3f-610f-7984-3224-97b8e1a582db +2025-06-03T23:37:28.702Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: added peer, starting replication: peer=server-d1a63c28-0a19-b51a-1fd7-984e6dbd51bb +2025-06-03T23:37:28.702Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: pipelining replication: peer="{Voter server-26283a3f-610f-7984-3224-97b8e1a582db 26283a3f-610f-7984-3224-97b8e1a582db}" +2025-06-03T23:37:28.702Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: pipelining replication: peer="{Voter server-d1a63c28-0a19-b51a-1fd7-984e6dbd51bb d1a63c28-0a19-b51a-1fd7-984e6dbd51bb}" +2025-06-03T23:37:29.707Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:28.702582711 +0000 UTC m=+84.024337641 (1 nodes), 634.31696ms from start of poll, 634.7834ms from cluster start. Timeout at 2025-06-03 23:37:29.707611031 +0000 UTC m=+85.029365961, 1.00502832s after stability +2025-06-03T23:37:29.708Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: starting snapshot up to: index=12 +2025-06-03T23:37:29.708Z [INFO] creating new snapshot: path=/tmp/raft1390985374/snapshots/2-12-1748993849708.tmp +2025-06-03T23:37:29.708Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: snapshot complete up to: index=12 +2025-06-03T23:37:29.709Z [INFO] creating new snapshot: path=/tmp/raft1390985374/snapshots/2-10013-1748993849709.tmp +2025-06-03T23:37:29.710Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: copied to local snapshot: bytes=71 +2025-06-03T23:37:29.710Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: snapshot restore progress: id=2-10013-1748993849709 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:29.710Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: restored user snapshot: index=10013 +2025-06-03T23:37:29.710Z [ERROR] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: failed to get log: index=23 error="log not found" +2025-06-03T23:37:29.710Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: aborting pipeline replication: peer="{Voter server-d1a63c28-0a19-b51a-1fd7-984e6dbd51bb d1a63c28-0a19-b51a-1fd7-984e6dbd51bb}" +2025-06-03T23:37:29.710Z [ERROR] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: failed to get log: index=23 error="log not found" +2025-06-03T23:37:29.710Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: aborting pipeline replication: peer="{Voter server-26283a3f-610f-7984-3224-97b8e1a582db 26283a3f-610f-7984-3224-97b8e1a582db}" +2025-06-03T23:37:29.717Z [ERROR] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: failed to get log: index=23 error="log not found" +2025-06-03T23:37:29.718Z [INFO] creating new snapshot: path=/tmp/raft1779236425/snapshots/2-10013-1748993849718.tmp +2025-06-03T23:37:29.718Z [ERROR] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: failed to get log: index=23 error="log not found" +2025-06-03T23:37:29.718Z [INFO] server-d1a63c28-0a19-b51a-1fd7-984e6dbd51bb: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:29.718Z [INFO] server-d1a63c28-0a19-b51a-1fd7-984e6dbd51bb: copied to local snapshot: bytes=71 +2025-06-03T23:37:29.718Z [INFO] creating new snapshot: path=/tmp/raft2900348952/snapshots/2-10013-1748993849718.tmp +2025-06-03T23:37:29.718Z [INFO] server-26283a3f-610f-7984-3224-97b8e1a582db: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:29.719Z [INFO] server-d1a63c28-0a19-b51a-1fd7-984e6dbd51bb: snapshot restore progress: id=2-10013-1748993849718 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:29.719Z [INFO] server-d1a63c28-0a19-b51a-1fd7-984e6dbd51bb: Installed remote snapshot +2025-06-03T23:37:29.719Z [INFO] server-26283a3f-610f-7984-3224-97b8e1a582db: copied to local snapshot: bytes=71 +2025-06-03T23:37:29.719Z [WARN] server-d1a63c28-0a19-b51a-1fd7-984e6dbd51bb: skipping application of old log: index=22 +2025-06-03T23:37:29.719Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: pipelining replication: peer="{Voter server-d1a63c28-0a19-b51a-1fd7-984e6dbd51bb d1a63c28-0a19-b51a-1fd7-984e6dbd51bb}" +2025-06-03T23:37:29.719Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:29.719Z [INFO] server-26283a3f-610f-7984-3224-97b8e1a582db: snapshot restore progress: id=2-10013-1748993849718 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:29.719Z [INFO] server-26283a3f-610f-7984-3224-97b8e1a582db: Installed remote snapshot +2025-06-03T23:37:29.719Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: pipelining replication: peer="{Voter server-26283a3f-610f-7984-3224-97b8e1a582db 26283a3f-610f-7984-3224-97b8e1a582db}" +2025-06-03T23:37:30.725Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:29.719443071 +0000 UTC m=+85.041198001 (2 nodes), 1µs from start of poll, 1.65164376s from cluster start. Timeout at 2025-06-03 23:37:30.725344031 +0000 UTC m=+86.047099041, 1.00590104s after stability +2025-06-03T23:37:30.736Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: aborting pipeline replication: peer="{Voter server-d1a63c28-0a19-b51a-1fd7-984e6dbd51bb d1a63c28-0a19-b51a-1fd7-984e6dbd51bb}" +2025-06-03T23:37:30.736Z [INFO] server-6df11825-b6bb-4be4-3c0d-ba177a36efc0: aborting pipeline replication: peer="{Voter server-26283a3f-610f-7984-3224-97b8e1a582db 26283a3f-610f-7984-3224-97b8e1a582db}" === RUN TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false -2025-06-03T23:34:32.018Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-37d4f49d-e80c-3758-b200-a4e0ab7e451f Address:37d4f49d-e80c-3758-b200-a4e0ab7e451f} {Suffrage:Voter ID:server-0076249b-bd58-abb0-73e7-853546c5a119 Address:0076249b-bd58-abb0-73e7-853546c5a119} {Suffrage:Voter ID:server-12dd1b07-07f5-e1a2-3e3c-db51539f5634 Address:12dd1b07-07f5-e1a2-3e3c-db51539f5634}]" -2025-06-03T23:34:32.018Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: entering follower state: follower="Node at 37d4f49d-e80c-3758-b200-a4e0ab7e451f [Follower]" leader-address= leader-id= -2025-06-03T23:34:32.018Z [INFO] server-0076249b-bd58-abb0-73e7-853546c5a119: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-37d4f49d-e80c-3758-b200-a4e0ab7e451f Address:37d4f49d-e80c-3758-b200-a4e0ab7e451f} {Suffrage:Voter ID:server-0076249b-bd58-abb0-73e7-853546c5a119 Address:0076249b-bd58-abb0-73e7-853546c5a119} {Suffrage:Voter ID:server-12dd1b07-07f5-e1a2-3e3c-db51539f5634 Address:12dd1b07-07f5-e1a2-3e3c-db51539f5634}]" -2025-06-03T23:34:32.018Z [INFO] server-0076249b-bd58-abb0-73e7-853546c5a119: entering follower state: follower="Node at 0076249b-bd58-abb0-73e7-853546c5a119 [Follower]" leader-address= leader-id= -2025-06-03T23:34:32.018Z [INFO] server-12dd1b07-07f5-e1a2-3e3c-db51539f5634: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-37d4f49d-e80c-3758-b200-a4e0ab7e451f Address:37d4f49d-e80c-3758-b200-a4e0ab7e451f} {Suffrage:Voter ID:server-0076249b-bd58-abb0-73e7-853546c5a119 Address:0076249b-bd58-abb0-73e7-853546c5a119} {Suffrage:Voter ID:server-12dd1b07-07f5-e1a2-3e3c-db51539f5634 Address:12dd1b07-07f5-e1a2-3e3c-db51539f5634}]" -2025-06-03T23:34:32.018Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:32.018Z [INFO] server-12dd1b07-07f5-e1a2-3e3c-db51539f5634: entering follower state: follower="Node at 12dd1b07-07f5-e1a2-3e3c-db51539f5634 [Follower]" leader-address= leader-id= -2025-06-03T23:34:32.563Z [WARN] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:32.563Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: entering candidate state: node="Node at 37d4f49d-e80c-3758-b200-a4e0ab7e451f [Candidate]" term=2 -2025-06-03T23:34:32.564Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: election won: term=2 tally=2 -2025-06-03T23:34:32.564Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: entering leader state: leader="Node at 37d4f49d-e80c-3758-b200-a4e0ab7e451f [Leader]" -2025-06-03T23:34:32.564Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: added peer, starting replication: peer=server-0076249b-bd58-abb0-73e7-853546c5a119 -2025-06-03T23:34:32.564Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: added peer, starting replication: peer=server-12dd1b07-07f5-e1a2-3e3c-db51539f5634 -2025-06-03T23:34:32.564Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: pipelining replication: peer="{Voter server-0076249b-bd58-abb0-73e7-853546c5a119 0076249b-bd58-abb0-73e7-853546c5a119}" -2025-06-03T23:34:32.565Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: pipelining replication: peer="{Voter server-12dd1b07-07f5-e1a2-3e3c-db51539f5634 12dd1b07-07f5-e1a2-3e3c-db51539f5634}" -2025-06-03T23:34:33.570Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:32.564635431 +0000 UTC m=+87.585454801 (1 nodes), 546.10836ms from start of poll, 546.67092ms from cluster start. Timeout at 2025-06-03 23:34:33.570653911 +0000 UTC m=+88.591473281, 1.00601848s after stability -2025-06-03T23:34:33.571Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: starting snapshot up to: index=12 -2025-06-03T23:34:33.571Z [INFO] creating new snapshot: path=/tmp/raft747766909/snapshots/2-12-1748993673571.tmp -2025-06-03T23:34:33.572Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: snapshot complete up to: index=12 -2025-06-03T23:34:33.572Z [INFO] creating new snapshot: path=/tmp/raft747766909/snapshots/2-10013-1748993673572.tmp -2025-06-03T23:34:33.573Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: copied to local snapshot: bytes=71 -2025-06-03T23:34:33.573Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: snapshot restore progress: id=2-10013-1748993673572 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:33.573Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: removing all old logs from log store -2025-06-03T23:34:33.573Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: compacting logs: from=1 to=22 -2025-06-03T23:34:33.573Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: restored user snapshot: index=10013 -2025-06-03T23:34:33.573Z [ERROR] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: failed to get log: index=22 error="log not found" -2025-06-03T23:34:33.573Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: aborting pipeline replication: peer="{Voter server-12dd1b07-07f5-e1a2-3e3c-db51539f5634 12dd1b07-07f5-e1a2-3e3c-db51539f5634}" -2025-06-03T23:34:33.573Z [ERROR] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: failed to get log: index=22 error="log not found" -2025-06-03T23:34:33.573Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: aborting pipeline replication: peer="{Voter server-0076249b-bd58-abb0-73e7-853546c5a119 0076249b-bd58-abb0-73e7-853546c5a119}" -2025-06-03T23:34:33.581Z [ERROR] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: failed to get log: index=22 error="log not found" -2025-06-03T23:34:33.582Z [INFO] creating new snapshot: path=/tmp/raft3714868557/snapshots/2-10013-1748993673582.tmp -2025-06-03T23:34:33.582Z [INFO] server-0076249b-bd58-abb0-73e7-853546c5a119: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:33.583Z [INFO] server-0076249b-bd58-abb0-73e7-853546c5a119: copied to local snapshot: bytes=71 -2025-06-03T23:34:33.583Z [ERROR] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: failed to get log: index=22 error="log not found" -2025-06-03T23:34:33.583Z [INFO] server-0076249b-bd58-abb0-73e7-853546c5a119: snapshot restore progress: id=2-10013-1748993673582 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:33.583Z [INFO] server-0076249b-bd58-abb0-73e7-853546c5a119: removing all old logs from log store -2025-06-03T23:34:33.583Z [INFO] server-0076249b-bd58-abb0-73e7-853546c5a119: compacting logs: from=1 to=22 -2025-06-03T23:34:33.583Z [INFO] server-0076249b-bd58-abb0-73e7-853546c5a119: Installed remote snapshot -2025-06-03T23:34:33.583Z [WARN] server-0076249b-bd58-abb0-73e7-853546c5a119: skipping application of old log: index=22 -2025-06-03T23:34:33.583Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: pipelining replication: peer="{Voter server-0076249b-bd58-abb0-73e7-853546c5a119 0076249b-bd58-abb0-73e7-853546c5a119}" -2025-06-03T23:34:33.583Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:33.583Z [INFO] creating new snapshot: path=/tmp/raft736469090/snapshots/2-10013-1748993673583.tmp -2025-06-03T23:34:33.584Z [INFO] server-12dd1b07-07f5-e1a2-3e3c-db51539f5634: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:33.584Z [INFO] server-12dd1b07-07f5-e1a2-3e3c-db51539f5634: copied to local snapshot: bytes=71 -2025-06-03T23:34:33.584Z [INFO] server-12dd1b07-07f5-e1a2-3e3c-db51539f5634: snapshot restore progress: id=2-10013-1748993673583 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:33.584Z [INFO] server-12dd1b07-07f5-e1a2-3e3c-db51539f5634: removing all old logs from log store -2025-06-03T23:34:33.584Z [INFO] server-12dd1b07-07f5-e1a2-3e3c-db51539f5634: compacting logs: from=1 to=22 -2025-06-03T23:34:33.584Z [INFO] server-12dd1b07-07f5-e1a2-3e3c-db51539f5634: Installed remote snapshot -2025-06-03T23:34:33.584Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: pipelining replication: peer="{Voter server-12dd1b07-07f5-e1a2-3e3c-db51539f5634 12dd1b07-07f5-e1a2-3e3c-db51539f5634}" -2025-06-03T23:34:34.589Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:33.583708431 +0000 UTC m=+88.604527801 (2 nodes), 1.24µs from start of poll, 1.56574392s from cluster start. Timeout at 2025-06-03 23:34:34.589371551 +0000 UTC m=+89.610190921, 1.00566312s after stability -2025-06-03T23:34:34.601Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: aborting pipeline replication: peer="{Voter server-12dd1b07-07f5-e1a2-3e3c-db51539f5634 12dd1b07-07f5-e1a2-3e3c-db51539f5634}" -2025-06-03T23:34:34.601Z [INFO] server-37d4f49d-e80c-3758-b200-a4e0ab7e451f: aborting pipeline replication: peer="{Voter server-0076249b-bd58-abb0-73e7-853546c5a119 0076249b-bd58-abb0-73e7-853546c5a119}" +2025-06-03T23:37:30.737Z [INFO] server-45f8a65c-d5a5-7976-32e8-e67b417ac040: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45f8a65c-d5a5-7976-32e8-e67b417ac040 Address:45f8a65c-d5a5-7976-32e8-e67b417ac040} {Suffrage:Voter ID:server-78f36eef-3a19-619d-d495-ffc452c2ae16 Address:78f36eef-3a19-619d-d495-ffc452c2ae16} {Suffrage:Voter ID:server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208 Address:d0f9eb3d-a5bc-76c9-819b-19df58fc8208}]" +2025-06-03T23:37:30.737Z [INFO] server-78f36eef-3a19-619d-d495-ffc452c2ae16: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45f8a65c-d5a5-7976-32e8-e67b417ac040 Address:45f8a65c-d5a5-7976-32e8-e67b417ac040} {Suffrage:Voter ID:server-78f36eef-3a19-619d-d495-ffc452c2ae16 Address:78f36eef-3a19-619d-d495-ffc452c2ae16} {Suffrage:Voter ID:server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208 Address:d0f9eb3d-a5bc-76c9-819b-19df58fc8208}]" +2025-06-03T23:37:30.737Z [INFO] server-45f8a65c-d5a5-7976-32e8-e67b417ac040: entering follower state: follower="Node at 45f8a65c-d5a5-7976-32e8-e67b417ac040 [Follower]" leader-address= leader-id= +2025-06-03T23:37:30.737Z [INFO] server-78f36eef-3a19-619d-d495-ffc452c2ae16: entering follower state: follower="Node at 78f36eef-3a19-619d-d495-ffc452c2ae16 [Follower]" leader-address= leader-id= +2025-06-03T23:37:30.737Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45f8a65c-d5a5-7976-32e8-e67b417ac040 Address:45f8a65c-d5a5-7976-32e8-e67b417ac040} {Suffrage:Voter ID:server-78f36eef-3a19-619d-d495-ffc452c2ae16 Address:78f36eef-3a19-619d-d495-ffc452c2ae16} {Suffrage:Voter ID:server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208 Address:d0f9eb3d-a5bc-76c9-819b-19df58fc8208}]" +2025-06-03T23:37:30.737Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:30.737Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: entering follower state: follower="Node at d0f9eb3d-a5bc-76c9-819b-19df58fc8208 [Follower]" leader-address= leader-id= +2025-06-03T23:37:31.319Z [WARN] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:31.319Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: entering candidate state: node="Node at d0f9eb3d-a5bc-76c9-819b-19df58fc8208 [Candidate]" term=2 +2025-06-03T23:37:31.319Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: election won: term=2 tally=2 +2025-06-03T23:37:31.319Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: entering leader state: leader="Node at d0f9eb3d-a5bc-76c9-819b-19df58fc8208 [Leader]" +2025-06-03T23:37:31.319Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: added peer, starting replication: peer=server-45f8a65c-d5a5-7976-32e8-e67b417ac040 +2025-06-03T23:37:31.319Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: added peer, starting replication: peer=server-78f36eef-3a19-619d-d495-ffc452c2ae16 +2025-06-03T23:37:31.319Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: pipelining replication: peer="{Voter server-78f36eef-3a19-619d-d495-ffc452c2ae16 78f36eef-3a19-619d-d495-ffc452c2ae16}" +2025-06-03T23:37:31.319Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: pipelining replication: peer="{Voter server-45f8a65c-d5a5-7976-32e8-e67b417ac040 45f8a65c-d5a5-7976-32e8-e67b417ac040}" +2025-06-03T23:37:32.325Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:31.319770311 +0000 UTC m=+86.641525241 (1 nodes), 581.80204ms from start of poll, 582.26744ms from cluster start. Timeout at 2025-06-03 23:37:32.325912591 +0000 UTC m=+87.647667521, 1.00614228s after stability +2025-06-03T23:37:32.326Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: starting snapshot up to: index=12 +2025-06-03T23:37:32.326Z [INFO] creating new snapshot: path=/tmp/raft3866824112/snapshots/2-12-1748993852326.tmp +2025-06-03T23:37:32.327Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: snapshot complete up to: index=12 +2025-06-03T23:37:32.328Z [INFO] creating new snapshot: path=/tmp/raft3866824112/snapshots/2-10013-1748993852328.tmp +2025-06-03T23:37:32.330Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: copied to local snapshot: bytes=71 +2025-06-03T23:37:32.331Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: snapshot restore progress: id=2-10013-1748993852328 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:32.331Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: removing all old logs from log store +2025-06-03T23:37:32.331Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: compacting logs: from=1 to=22 +2025-06-03T23:37:32.331Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: restored user snapshot: index=10013 +2025-06-03T23:37:32.331Z [ERROR] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: failed to get log: index=22 error="log not found" +2025-06-03T23:37:32.331Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: aborting pipeline replication: peer="{Voter server-78f36eef-3a19-619d-d495-ffc452c2ae16 78f36eef-3a19-619d-d495-ffc452c2ae16}" +2025-06-03T23:37:32.331Z [ERROR] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: failed to get log: index=22 error="log not found" +2025-06-03T23:37:32.331Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: aborting pipeline replication: peer="{Voter server-45f8a65c-d5a5-7976-32e8-e67b417ac040 45f8a65c-d5a5-7976-32e8-e67b417ac040}" +2025-06-03T23:37:32.340Z [ERROR] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: failed to get log: index=22 error="log not found" +2025-06-03T23:37:32.340Z [INFO] creating new snapshot: path=/tmp/raft2901688230/snapshots/2-10013-1748993852340.tmp +2025-06-03T23:37:32.340Z [INFO] server-78f36eef-3a19-619d-d495-ffc452c2ae16: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:32.340Z [ERROR] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: failed to get log: index=22 error="log not found" +2025-06-03T23:37:32.341Z [INFO] server-78f36eef-3a19-619d-d495-ffc452c2ae16: copied to local snapshot: bytes=71 +2025-06-03T23:37:32.341Z [INFO] server-78f36eef-3a19-619d-d495-ffc452c2ae16: snapshot restore progress: id=2-10013-1748993852340 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:32.341Z [INFO] server-78f36eef-3a19-619d-d495-ffc452c2ae16: removing all old logs from log store +2025-06-03T23:37:32.341Z [INFO] server-78f36eef-3a19-619d-d495-ffc452c2ae16: compacting logs: from=1 to=22 +2025-06-03T23:37:32.341Z [INFO] server-78f36eef-3a19-619d-d495-ffc452c2ae16: Installed remote snapshot +2025-06-03T23:37:32.341Z [INFO] creating new snapshot: path=/tmp/raft1142819352/snapshots/2-10013-1748993852341.tmp +2025-06-03T23:37:32.341Z [WARN] server-78f36eef-3a19-619d-d495-ffc452c2ae16: skipping application of old log: index=22 +2025-06-03T23:37:32.341Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: pipelining replication: peer="{Voter server-78f36eef-3a19-619d-d495-ffc452c2ae16 78f36eef-3a19-619d-d495-ffc452c2ae16}" +2025-06-03T23:37:32.341Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:32.341Z [INFO] server-45f8a65c-d5a5-7976-32e8-e67b417ac040: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:32.341Z [INFO] server-45f8a65c-d5a5-7976-32e8-e67b417ac040: copied to local snapshot: bytes=71 +2025-06-03T23:37:32.342Z [INFO] server-45f8a65c-d5a5-7976-32e8-e67b417ac040: snapshot restore progress: id=2-10013-1748993852341 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:32.342Z [INFO] server-45f8a65c-d5a5-7976-32e8-e67b417ac040: removing all old logs from log store +2025-06-03T23:37:32.342Z [INFO] server-45f8a65c-d5a5-7976-32e8-e67b417ac040: compacting logs: from=1 to=22 +2025-06-03T23:37:32.342Z [INFO] server-45f8a65c-d5a5-7976-32e8-e67b417ac040: Installed remote snapshot +2025-06-03T23:37:32.342Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: pipelining replication: peer="{Voter server-45f8a65c-d5a5-7976-32e8-e67b417ac040 45f8a65c-d5a5-7976-32e8-e67b417ac040}" +2025-06-03T23:37:33.346Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:32.341586391 +0000 UTC m=+87.663341321 (2 nodes), 1µs from start of poll, 1.60408352s from cluster start. Timeout at 2025-06-03 23:37:33.346645431 +0000 UTC m=+88.668400401, 1.00505908s after stability +2025-06-03T23:37:33.359Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: aborting pipeline replication: peer="{Voter server-78f36eef-3a19-619d-d495-ffc452c2ae16 78f36eef-3a19-619d-d495-ffc452c2ae16}" +2025-06-03T23:37:33.359Z [INFO] server-d0f9eb3d-a5bc-76c9-819b-19df58fc8208: aborting pipeline replication: peer="{Voter server-45f8a65c-d5a5-7976-32e8-e67b417ac040 45f8a65c-d5a5-7976-32e8-e67b417ac040}" === RUN TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true -2025-06-03T23:34:34.602Z [INFO] server-c8da18c4-eaeb-0450-dc22-7d53506013a8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c8da18c4-eaeb-0450-dc22-7d53506013a8 Address:c8da18c4-eaeb-0450-dc22-7d53506013a8} {Suffrage:Voter ID:server-373d2734-59a3-32e7-970b-87e8dd284846 Address:373d2734-59a3-32e7-970b-87e8dd284846} {Suffrage:Voter ID:server-3ee28449-98fc-112f-ce72-1d1d101660dd Address:3ee28449-98fc-112f-ce72-1d1d101660dd}]" -2025-06-03T23:34:34.603Z [INFO] server-c8da18c4-eaeb-0450-dc22-7d53506013a8: entering follower state: follower="Node at c8da18c4-eaeb-0450-dc22-7d53506013a8 [Follower]" leader-address= leader-id= -2025-06-03T23:34:34.603Z [INFO] server-373d2734-59a3-32e7-970b-87e8dd284846: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c8da18c4-eaeb-0450-dc22-7d53506013a8 Address:c8da18c4-eaeb-0450-dc22-7d53506013a8} {Suffrage:Voter ID:server-373d2734-59a3-32e7-970b-87e8dd284846 Address:373d2734-59a3-32e7-970b-87e8dd284846} {Suffrage:Voter ID:server-3ee28449-98fc-112f-ce72-1d1d101660dd Address:3ee28449-98fc-112f-ce72-1d1d101660dd}]" -2025-06-03T23:34:34.603Z [INFO] server-373d2734-59a3-32e7-970b-87e8dd284846: entering follower state: follower="Node at 373d2734-59a3-32e7-970b-87e8dd284846 [Follower]" leader-address= leader-id= -2025-06-03T23:34:34.603Z [INFO] server-3ee28449-98fc-112f-ce72-1d1d101660dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c8da18c4-eaeb-0450-dc22-7d53506013a8 Address:c8da18c4-eaeb-0450-dc22-7d53506013a8} {Suffrage:Voter ID:server-373d2734-59a3-32e7-970b-87e8dd284846 Address:373d2734-59a3-32e7-970b-87e8dd284846} {Suffrage:Voter ID:server-3ee28449-98fc-112f-ce72-1d1d101660dd Address:3ee28449-98fc-112f-ce72-1d1d101660dd}]" -2025-06-03T23:34:34.603Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:34.603Z [INFO] server-3ee28449-98fc-112f-ce72-1d1d101660dd: entering follower state: follower="Node at 3ee28449-98fc-112f-ce72-1d1d101660dd [Follower]" leader-address= leader-id= -2025-06-03T23:34:35.364Z [WARN] server-373d2734-59a3-32e7-970b-87e8dd284846: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:35.364Z [INFO] server-373d2734-59a3-32e7-970b-87e8dd284846: entering candidate state: node="Node at 373d2734-59a3-32e7-970b-87e8dd284846 [Candidate]" term=2 -2025-06-03T23:34:35.364Z [INFO] server-373d2734-59a3-32e7-970b-87e8dd284846: election won: term=2 tally=2 -2025-06-03T23:34:35.364Z [INFO] server-373d2734-59a3-32e7-970b-87e8dd284846: entering leader state: leader="Node at 373d2734-59a3-32e7-970b-87e8dd284846 [Leader]" -2025-06-03T23:34:35.364Z [INFO] server-373d2734-59a3-32e7-970b-87e8dd284846: added peer, starting replication: peer=server-c8da18c4-eaeb-0450-dc22-7d53506013a8 -2025-06-03T23:34:35.364Z [INFO] server-373d2734-59a3-32e7-970b-87e8dd284846: added peer, starting replication: peer=server-3ee28449-98fc-112f-ce72-1d1d101660dd -2025-06-03T23:34:35.364Z [INFO] server-373d2734-59a3-32e7-970b-87e8dd284846: pipelining replication: peer="{Voter server-c8da18c4-eaeb-0450-dc22-7d53506013a8 c8da18c4-eaeb-0450-dc22-7d53506013a8}" -2025-06-03T23:34:35.364Z [INFO] server-373d2734-59a3-32e7-970b-87e8dd284846: pipelining replication: peer="{Voter server-3ee28449-98fc-112f-ce72-1d1d101660dd 3ee28449-98fc-112f-ce72-1d1d101660dd}" -2025-06-03T23:34:36.369Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:35.364665391 +0000 UTC m=+90.385484761 (1 nodes), 761.43324ms from start of poll, 761.927ms from cluster start. Timeout at 2025-06-03 23:34:36.369701471 +0000 UTC m=+91.390520841, 1.00503608s after stability -2025-06-03T23:34:36.371Z [INFO] server-373d2734-59a3-32e7-970b-87e8dd284846: starting snapshot up to: index=12 -2025-06-03T23:34:36.372Z [INFO] creating new snapshot: path=/tmp/raft436029407/snapshots/2-12-1748993676371.tmp -2025-06-03T23:34:36.372Z [INFO] server-373d2734-59a3-32e7-970b-87e8dd284846: snapshot complete up to: index=12 -2025-06-03T23:34:36.374Z [INFO] server-67a0b0e1-b080-143d-638b-91757a235618: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67a0b0e1-b080-143d-638b-91757a235618 Address:67a0b0e1-b080-143d-638b-91757a235618} {Suffrage:Voter ID:server-e7832e07-29e4-5c0a-effb-0e5031c45c9e Address:e7832e07-29e4-5c0a-effb-0e5031c45c9e} {Suffrage:Voter ID:server-ed0b570b-6c75-5b66-ff35-be62067995e0 Address:ed0b570b-6c75-5b66-ff35-be62067995e0}]" -2025-06-03T23:34:36.374Z [INFO] server-e7832e07-29e4-5c0a-effb-0e5031c45c9e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67a0b0e1-b080-143d-638b-91757a235618 Address:67a0b0e1-b080-143d-638b-91757a235618} {Suffrage:Voter ID:server-e7832e07-29e4-5c0a-effb-0e5031c45c9e Address:e7832e07-29e4-5c0a-effb-0e5031c45c9e} {Suffrage:Voter ID:server-ed0b570b-6c75-5b66-ff35-be62067995e0 Address:ed0b570b-6c75-5b66-ff35-be62067995e0}]" -2025-06-03T23:34:36.374Z [INFO] server-ed0b570b-6c75-5b66-ff35-be62067995e0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67a0b0e1-b080-143d-638b-91757a235618 Address:67a0b0e1-b080-143d-638b-91757a235618} {Suffrage:Voter ID:server-e7832e07-29e4-5c0a-effb-0e5031c45c9e Address:e7832e07-29e4-5c0a-effb-0e5031c45c9e} {Suffrage:Voter ID:server-ed0b570b-6c75-5b66-ff35-be62067995e0 Address:ed0b570b-6c75-5b66-ff35-be62067995e0}]" -2025-06-03T23:34:36.374Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:36.374Z [INFO] server-67a0b0e1-b080-143d-638b-91757a235618: entering follower state: follower="Node at 67a0b0e1-b080-143d-638b-91757a235618 [Follower]" leader-address= leader-id= -2025-06-03T23:34:36.374Z [INFO] server-e7832e07-29e4-5c0a-effb-0e5031c45c9e: entering follower state: follower="Node at e7832e07-29e4-5c0a-effb-0e5031c45c9e [Follower]" leader-address= leader-id= -2025-06-03T23:34:36.374Z [INFO] server-ed0b570b-6c75-5b66-ff35-be62067995e0: entering follower state: follower="Node at ed0b570b-6c75-5b66-ff35-be62067995e0 [Follower]" leader-address= leader-id= -2025-06-03T23:34:37.149Z [WARN] server-ed0b570b-6c75-5b66-ff35-be62067995e0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:37.149Z [INFO] server-ed0b570b-6c75-5b66-ff35-be62067995e0: entering candidate state: node="Node at ed0b570b-6c75-5b66-ff35-be62067995e0 [Candidate]" term=2 -2025-06-03T23:34:37.149Z [INFO] server-ed0b570b-6c75-5b66-ff35-be62067995e0: election won: term=2 tally=2 -2025-06-03T23:34:37.149Z [INFO] server-ed0b570b-6c75-5b66-ff35-be62067995e0: entering leader state: leader="Node at ed0b570b-6c75-5b66-ff35-be62067995e0 [Leader]" -2025-06-03T23:34:37.149Z [INFO] server-ed0b570b-6c75-5b66-ff35-be62067995e0: added peer, starting replication: peer=server-67a0b0e1-b080-143d-638b-91757a235618 -2025-06-03T23:34:37.149Z [INFO] server-ed0b570b-6c75-5b66-ff35-be62067995e0: added peer, starting replication: peer=server-e7832e07-29e4-5c0a-effb-0e5031c45c9e -2025-06-03T23:34:37.149Z [INFO] server-ed0b570b-6c75-5b66-ff35-be62067995e0: pipelining replication: peer="{Voter server-67a0b0e1-b080-143d-638b-91757a235618 67a0b0e1-b080-143d-638b-91757a235618}" -2025-06-03T23:34:37.149Z [INFO] server-ed0b570b-6c75-5b66-ff35-be62067995e0: pipelining replication: peer="{Voter server-e7832e07-29e4-5c0a-effb-0e5031c45c9e e7832e07-29e4-5c0a-effb-0e5031c45c9e}" -2025-06-03T23:34:38.154Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:37.149462791 +0000 UTC m=+92.170282201 (1 nodes), 775.1158ms from start of poll, 775.60752ms from cluster start. Timeout at 2025-06-03 23:34:38.154631991 +0000 UTC m=+93.175451361, 1.00516916s after stability -2025-06-03T23:34:38.155Z [INFO] creating new snapshot: path=/tmp/raft1350143364/snapshots/2-10013-1748993678155.tmp -2025-06-03T23:34:38.155Z [INFO] server-ed0b570b-6c75-5b66-ff35-be62067995e0: copied to local snapshot: bytes=71 -2025-06-03T23:34:38.155Z [INFO] server-ed0b570b-6c75-5b66-ff35-be62067995e0: snapshot restore progress: id=2-10013-1748993678155 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:38.155Z [INFO] server-ed0b570b-6c75-5b66-ff35-be62067995e0: restored user snapshot: index=10013 -2025-06-03T23:34:38.155Z [ERROR] server-ed0b570b-6c75-5b66-ff35-be62067995e0: failed to get log: index=3 error="log not found" -2025-06-03T23:34:38.155Z [INFO] server-ed0b570b-6c75-5b66-ff35-be62067995e0: aborting pipeline replication: peer="{Voter server-e7832e07-29e4-5c0a-effb-0e5031c45c9e e7832e07-29e4-5c0a-effb-0e5031c45c9e}" -2025-06-03T23:34:38.155Z [ERROR] server-ed0b570b-6c75-5b66-ff35-be62067995e0: failed to get log: index=3 error="log not found" -2025-06-03T23:34:38.155Z [INFO] server-ed0b570b-6c75-5b66-ff35-be62067995e0: aborting pipeline replication: peer="{Voter server-67a0b0e1-b080-143d-638b-91757a235618 67a0b0e1-b080-143d-638b-91757a235618}" -2025-06-03T23:34:38.161Z [ERROR] server-ed0b570b-6c75-5b66-ff35-be62067995e0: failed to get log: index=3 error="log not found" -2025-06-03T23:34:38.161Z [INFO] creating new snapshot: path=/tmp/raft3796095899/snapshots/2-10013-1748993678161.tmp -2025-06-03T23:34:38.162Z [INFO] server-e7832e07-29e4-5c0a-effb-0e5031c45c9e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:38.162Z [INFO] server-e7832e07-29e4-5c0a-effb-0e5031c45c9e: copied to local snapshot: bytes=71 -2025-06-03T23:34:38.162Z [ERROR] server-ed0b570b-6c75-5b66-ff35-be62067995e0: failed to get log: index=3 error="log not found" -2025-06-03T23:34:38.162Z [INFO] server-e7832e07-29e4-5c0a-effb-0e5031c45c9e: snapshot restore progress: id=2-10013-1748993678161 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:38.162Z [INFO] server-e7832e07-29e4-5c0a-effb-0e5031c45c9e: Installed remote snapshot -2025-06-03T23:34:38.162Z [INFO] server-ed0b570b-6c75-5b66-ff35-be62067995e0: pipelining replication: peer="{Voter server-e7832e07-29e4-5c0a-effb-0e5031c45c9e e7832e07-29e4-5c0a-effb-0e5031c45c9e}" -2025-06-03T23:34:38.162Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:38.163Z [INFO] creating new snapshot: path=/tmp/raft3616023600/snapshots/2-10013-1748993678163.tmp -2025-06-03T23:34:38.163Z [INFO] server-67a0b0e1-b080-143d-638b-91757a235618: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:38.163Z [INFO] server-67a0b0e1-b080-143d-638b-91757a235618: copied to local snapshot: bytes=71 -2025-06-03T23:34:38.163Z [INFO] server-67a0b0e1-b080-143d-638b-91757a235618: snapshot restore progress: id=2-10013-1748993678163 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:38.163Z [INFO] server-67a0b0e1-b080-143d-638b-91757a235618: Installed remote snapshot -2025-06-03T23:34:38.164Z [INFO] server-ed0b570b-6c75-5b66-ff35-be62067995e0: pipelining replication: peer="{Voter server-67a0b0e1-b080-143d-638b-91757a235618 67a0b0e1-b080-143d-638b-91757a235618}" -2025-06-03T23:34:39.168Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:38.163030031 +0000 UTC m=+93.183849401 (2 nodes), 1.2µs from start of poll, 1.78917472s from cluster start. Timeout at 2025-06-03 23:34:39.168050511 +0000 UTC m=+94.188869881, 1.00502048s after stability -2025-06-03T23:34:39.179Z [INFO] server-373d2734-59a3-32e7-970b-87e8dd284846: aborting pipeline replication: peer="{Voter server-3ee28449-98fc-112f-ce72-1d1d101660dd 3ee28449-98fc-112f-ce72-1d1d101660dd}" -2025-06-03T23:34:39.179Z [INFO] server-373d2734-59a3-32e7-970b-87e8dd284846: aborting pipeline replication: peer="{Voter server-c8da18c4-eaeb-0450-dc22-7d53506013a8 c8da18c4-eaeb-0450-dc22-7d53506013a8}" +2025-06-03T23:37:33.360Z [INFO] server-7b16a951-f70f-318e-1a66-e5860c13af70: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b16a951-f70f-318e-1a66-e5860c13af70 Address:7b16a951-f70f-318e-1a66-e5860c13af70} {Suffrage:Voter ID:server-0d8703fe-8cb1-c332-cd63-bba33c807354 Address:0d8703fe-8cb1-c332-cd63-bba33c807354} {Suffrage:Voter ID:server-65749557-d8ee-9ca9-3933-00b5fa950581 Address:65749557-d8ee-9ca9-3933-00b5fa950581}]" +2025-06-03T23:37:33.360Z [INFO] server-7b16a951-f70f-318e-1a66-e5860c13af70: entering follower state: follower="Node at 7b16a951-f70f-318e-1a66-e5860c13af70 [Follower]" leader-address= leader-id= +2025-06-03T23:37:33.360Z [INFO] server-0d8703fe-8cb1-c332-cd63-bba33c807354: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b16a951-f70f-318e-1a66-e5860c13af70 Address:7b16a951-f70f-318e-1a66-e5860c13af70} {Suffrage:Voter ID:server-0d8703fe-8cb1-c332-cd63-bba33c807354 Address:0d8703fe-8cb1-c332-cd63-bba33c807354} {Suffrage:Voter ID:server-65749557-d8ee-9ca9-3933-00b5fa950581 Address:65749557-d8ee-9ca9-3933-00b5fa950581}]" +2025-06-03T23:37:33.360Z [INFO] server-0d8703fe-8cb1-c332-cd63-bba33c807354: entering follower state: follower="Node at 0d8703fe-8cb1-c332-cd63-bba33c807354 [Follower]" leader-address= leader-id= +2025-06-03T23:37:33.360Z [INFO] server-65749557-d8ee-9ca9-3933-00b5fa950581: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b16a951-f70f-318e-1a66-e5860c13af70 Address:7b16a951-f70f-318e-1a66-e5860c13af70} {Suffrage:Voter ID:server-0d8703fe-8cb1-c332-cd63-bba33c807354 Address:0d8703fe-8cb1-c332-cd63-bba33c807354} {Suffrage:Voter ID:server-65749557-d8ee-9ca9-3933-00b5fa950581 Address:65749557-d8ee-9ca9-3933-00b5fa950581}]" +2025-06-03T23:37:33.360Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:33.360Z [INFO] server-65749557-d8ee-9ca9-3933-00b5fa950581: entering follower state: follower="Node at 65749557-d8ee-9ca9-3933-00b5fa950581 [Follower]" leader-address= leader-id= +2025-06-03T23:37:33.869Z [WARN] server-65749557-d8ee-9ca9-3933-00b5fa950581: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:33.870Z [INFO] server-65749557-d8ee-9ca9-3933-00b5fa950581: entering candidate state: node="Node at 65749557-d8ee-9ca9-3933-00b5fa950581 [Candidate]" term=2 +2025-06-03T23:37:33.870Z [INFO] server-65749557-d8ee-9ca9-3933-00b5fa950581: election won: term=2 tally=2 +2025-06-03T23:37:33.870Z [INFO] server-65749557-d8ee-9ca9-3933-00b5fa950581: entering leader state: leader="Node at 65749557-d8ee-9ca9-3933-00b5fa950581 [Leader]" +2025-06-03T23:37:33.870Z [INFO] server-65749557-d8ee-9ca9-3933-00b5fa950581: added peer, starting replication: peer=server-7b16a951-f70f-318e-1a66-e5860c13af70 +2025-06-03T23:37:33.870Z [INFO] server-65749557-d8ee-9ca9-3933-00b5fa950581: added peer, starting replication: peer=server-0d8703fe-8cb1-c332-cd63-bba33c807354 +2025-06-03T23:37:33.870Z [INFO] server-65749557-d8ee-9ca9-3933-00b5fa950581: pipelining replication: peer="{Voter server-7b16a951-f70f-318e-1a66-e5860c13af70 7b16a951-f70f-318e-1a66-e5860c13af70}" +2025-06-03T23:37:33.870Z [INFO] server-65749557-d8ee-9ca9-3933-00b5fa950581: pipelining replication: peer="{Voter server-0d8703fe-8cb1-c332-cd63-bba33c807354 0d8703fe-8cb1-c332-cd63-bba33c807354}" +2025-06-03T23:37:34.875Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:33.870236191 +0000 UTC m=+89.191991121 (1 nodes), 509.74652ms from start of poll, 510.1822ms from cluster start. Timeout at 2025-06-03 23:37:34.875870551 +0000 UTC m=+90.197625481, 1.00563436s after stability +2025-06-03T23:37:34.876Z [INFO] server-65749557-d8ee-9ca9-3933-00b5fa950581: starting snapshot up to: index=12 +2025-06-03T23:37:34.876Z [INFO] creating new snapshot: path=/tmp/raft4038391252/snapshots/2-12-1748993854876.tmp +2025-06-03T23:37:34.877Z [INFO] server-65749557-d8ee-9ca9-3933-00b5fa950581: snapshot complete up to: index=12 +2025-06-03T23:37:34.878Z [INFO] server-5afc2faa-570b-13cc-2969-29c6df32cb64: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5afc2faa-570b-13cc-2969-29c6df32cb64 Address:5afc2faa-570b-13cc-2969-29c6df32cb64} {Suffrage:Voter ID:server-d187386a-6e3a-6f98-138c-9a07dadeae02 Address:d187386a-6e3a-6f98-138c-9a07dadeae02} {Suffrage:Voter ID:server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361 Address:b013cec5-aef2-b4b0-d92b-dcdf98d7f361}]" +2025-06-03T23:37:34.878Z [INFO] server-5afc2faa-570b-13cc-2969-29c6df32cb64: entering follower state: follower="Node at 5afc2faa-570b-13cc-2969-29c6df32cb64 [Follower]" leader-address= leader-id= +2025-06-03T23:37:34.878Z [INFO] server-d187386a-6e3a-6f98-138c-9a07dadeae02: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5afc2faa-570b-13cc-2969-29c6df32cb64 Address:5afc2faa-570b-13cc-2969-29c6df32cb64} {Suffrage:Voter ID:server-d187386a-6e3a-6f98-138c-9a07dadeae02 Address:d187386a-6e3a-6f98-138c-9a07dadeae02} {Suffrage:Voter ID:server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361 Address:b013cec5-aef2-b4b0-d92b-dcdf98d7f361}]" +2025-06-03T23:37:34.878Z [INFO] server-d187386a-6e3a-6f98-138c-9a07dadeae02: entering follower state: follower="Node at d187386a-6e3a-6f98-138c-9a07dadeae02 [Follower]" leader-address= leader-id= +2025-06-03T23:37:34.878Z [INFO] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5afc2faa-570b-13cc-2969-29c6df32cb64 Address:5afc2faa-570b-13cc-2969-29c6df32cb64} {Suffrage:Voter ID:server-d187386a-6e3a-6f98-138c-9a07dadeae02 Address:d187386a-6e3a-6f98-138c-9a07dadeae02} {Suffrage:Voter ID:server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361 Address:b013cec5-aef2-b4b0-d92b-dcdf98d7f361}]" +2025-06-03T23:37:34.878Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:34.878Z [INFO] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: entering follower state: follower="Node at b013cec5-aef2-b4b0-d92b-dcdf98d7f361 [Follower]" leader-address= leader-id= +2025-06-03T23:37:35.535Z [WARN] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:35.535Z [INFO] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: entering candidate state: node="Node at b013cec5-aef2-b4b0-d92b-dcdf98d7f361 [Candidate]" term=2 +2025-06-03T23:37:35.536Z [INFO] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: election won: term=2 tally=2 +2025-06-03T23:37:35.536Z [INFO] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: entering leader state: leader="Node at b013cec5-aef2-b4b0-d92b-dcdf98d7f361 [Leader]" +2025-06-03T23:37:35.536Z [INFO] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: added peer, starting replication: peer=server-5afc2faa-570b-13cc-2969-29c6df32cb64 +2025-06-03T23:37:35.536Z [INFO] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: added peer, starting replication: peer=server-d187386a-6e3a-6f98-138c-9a07dadeae02 +2025-06-03T23:37:35.536Z [INFO] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: pipelining replication: peer="{Voter server-5afc2faa-570b-13cc-2969-29c6df32cb64 5afc2faa-570b-13cc-2969-29c6df32cb64}" +2025-06-03T23:37:35.536Z [INFO] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: pipelining replication: peer="{Voter server-d187386a-6e3a-6f98-138c-9a07dadeae02 d187386a-6e3a-6f98-138c-9a07dadeae02}" +2025-06-03T23:37:36.542Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:35.536395231 +0000 UTC m=+90.858150161 (1 nodes), 657.55696ms from start of poll, 658.07024ms from cluster start. Timeout at 2025-06-03 23:37:36.541989271 +0000 UTC m=+91.863744201, 1.00559404s after stability +2025-06-03T23:37:36.542Z [INFO] creating new snapshot: path=/tmp/raft699254755/snapshots/2-10013-1748993856542.tmp +2025-06-03T23:37:36.542Z [INFO] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: copied to local snapshot: bytes=71 +2025-06-03T23:37:36.542Z [INFO] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: snapshot restore progress: id=2-10013-1748993856542 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:36.543Z [INFO] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: restored user snapshot: index=10013 +2025-06-03T23:37:36.543Z [ERROR] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: failed to get log: index=3 error="log not found" +2025-06-03T23:37:36.543Z [INFO] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: aborting pipeline replication: peer="{Voter server-d187386a-6e3a-6f98-138c-9a07dadeae02 d187386a-6e3a-6f98-138c-9a07dadeae02}" +2025-06-03T23:37:36.543Z [ERROR] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: failed to get log: index=3 error="log not found" +2025-06-03T23:37:36.543Z [INFO] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: aborting pipeline replication: peer="{Voter server-5afc2faa-570b-13cc-2969-29c6df32cb64 5afc2faa-570b-13cc-2969-29c6df32cb64}" +2025-06-03T23:37:36.551Z [ERROR] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: failed to get log: index=3 error="log not found" +2025-06-03T23:37:36.552Z [INFO] creating new snapshot: path=/tmp/raft212987362/snapshots/2-10013-1748993856552.tmp +2025-06-03T23:37:36.552Z [INFO] server-d187386a-6e3a-6f98-138c-9a07dadeae02: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:36.552Z [INFO] server-d187386a-6e3a-6f98-138c-9a07dadeae02: copied to local snapshot: bytes=71 +2025-06-03T23:37:36.552Z [ERROR] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: failed to get log: index=3 error="log not found" +2025-06-03T23:37:36.553Z [INFO] server-d187386a-6e3a-6f98-138c-9a07dadeae02: snapshot restore progress: id=2-10013-1748993856552 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:36.553Z [INFO] creating new snapshot: path=/tmp/raft2917221557/snapshots/2-10013-1748993856553.tmp +2025-06-03T23:37:36.553Z [INFO] server-d187386a-6e3a-6f98-138c-9a07dadeae02: Installed remote snapshot +2025-06-03T23:37:36.553Z [INFO] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: pipelining replication: peer="{Voter server-d187386a-6e3a-6f98-138c-9a07dadeae02 d187386a-6e3a-6f98-138c-9a07dadeae02}" +2025-06-03T23:37:36.553Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:36.553Z [INFO] server-5afc2faa-570b-13cc-2969-29c6df32cb64: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:36.554Z [INFO] server-5afc2faa-570b-13cc-2969-29c6df32cb64: copied to local snapshot: bytes=71 +2025-06-03T23:37:36.554Z [INFO] server-5afc2faa-570b-13cc-2969-29c6df32cb64: snapshot restore progress: id=2-10013-1748993856553 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:36.554Z [INFO] server-5afc2faa-570b-13cc-2969-29c6df32cb64: Installed remote snapshot +2025-06-03T23:37:36.554Z [INFO] server-b013cec5-aef2-b4b0-d92b-dcdf98d7f361: pipelining replication: peer="{Voter server-5afc2faa-570b-13cc-2969-29c6df32cb64 5afc2faa-570b-13cc-2969-29c6df32cb64}" +2025-06-03T23:37:37.558Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:36.553155751 +0000 UTC m=+91.874910641 (2 nodes), 560ns from start of poll, 1.67483072s from cluster start. Timeout at 2025-06-03 23:37:37.558758591 +0000 UTC m=+92.880513521, 1.00560288s after stability +2025-06-03T23:37:37.570Z [INFO] server-65749557-d8ee-9ca9-3933-00b5fa950581: aborting pipeline replication: peer="{Voter server-0d8703fe-8cb1-c332-cd63-bba33c807354 0d8703fe-8cb1-c332-cd63-bba33c807354}" +2025-06-03T23:37:37.570Z [INFO] server-65749557-d8ee-9ca9-3933-00b5fa950581: aborting pipeline replication: peer="{Voter server-7b16a951-f70f-318e-1a66-e5860c13af70 7b16a951-f70f-318e-1a66-e5860c13af70}" === RUN TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true -2025-06-03T23:34:39.180Z [INFO] server-f74a98d0-ff2b-d28e-7da4-021f9725d855: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f74a98d0-ff2b-d28e-7da4-021f9725d855 Address:f74a98d0-ff2b-d28e-7da4-021f9725d855} {Suffrage:Voter ID:server-adc70873-96aa-159d-7bd0-5de7d801dbe9 Address:adc70873-96aa-159d-7bd0-5de7d801dbe9} {Suffrage:Voter ID:server-1acbaf23-958d-5e58-fb98-db43f6dfc565 Address:1acbaf23-958d-5e58-fb98-db43f6dfc565}]" -2025-06-03T23:34:39.180Z [INFO] server-adc70873-96aa-159d-7bd0-5de7d801dbe9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f74a98d0-ff2b-d28e-7da4-021f9725d855 Address:f74a98d0-ff2b-d28e-7da4-021f9725d855} {Suffrage:Voter ID:server-adc70873-96aa-159d-7bd0-5de7d801dbe9 Address:adc70873-96aa-159d-7bd0-5de7d801dbe9} {Suffrage:Voter ID:server-1acbaf23-958d-5e58-fb98-db43f6dfc565 Address:1acbaf23-958d-5e58-fb98-db43f6dfc565}]" -2025-06-03T23:34:39.180Z [INFO] server-f74a98d0-ff2b-d28e-7da4-021f9725d855: entering follower state: follower="Node at f74a98d0-ff2b-d28e-7da4-021f9725d855 [Follower]" leader-address= leader-id= -2025-06-03T23:34:39.180Z [INFO] server-adc70873-96aa-159d-7bd0-5de7d801dbe9: entering follower state: follower="Node at adc70873-96aa-159d-7bd0-5de7d801dbe9 [Follower]" leader-address= leader-id= -2025-06-03T23:34:39.180Z [INFO] server-1acbaf23-958d-5e58-fb98-db43f6dfc565: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f74a98d0-ff2b-d28e-7da4-021f9725d855 Address:f74a98d0-ff2b-d28e-7da4-021f9725d855} {Suffrage:Voter ID:server-adc70873-96aa-159d-7bd0-5de7d801dbe9 Address:adc70873-96aa-159d-7bd0-5de7d801dbe9} {Suffrage:Voter ID:server-1acbaf23-958d-5e58-fb98-db43f6dfc565 Address:1acbaf23-958d-5e58-fb98-db43f6dfc565}]" -2025-06-03T23:34:39.180Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:39.180Z [INFO] server-1acbaf23-958d-5e58-fb98-db43f6dfc565: entering follower state: follower="Node at 1acbaf23-958d-5e58-fb98-db43f6dfc565 [Follower]" leader-address= leader-id= -2025-06-03T23:34:39.713Z [WARN] server-adc70873-96aa-159d-7bd0-5de7d801dbe9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:39.713Z [INFO] server-adc70873-96aa-159d-7bd0-5de7d801dbe9: entering candidate state: node="Node at adc70873-96aa-159d-7bd0-5de7d801dbe9 [Candidate]" term=2 -2025-06-03T23:34:39.713Z [INFO] server-adc70873-96aa-159d-7bd0-5de7d801dbe9: election won: term=2 tally=2 -2025-06-03T23:34:39.713Z [INFO] server-adc70873-96aa-159d-7bd0-5de7d801dbe9: entering leader state: leader="Node at adc70873-96aa-159d-7bd0-5de7d801dbe9 [Leader]" -2025-06-03T23:34:39.713Z [INFO] server-adc70873-96aa-159d-7bd0-5de7d801dbe9: added peer, starting replication: peer=server-f74a98d0-ff2b-d28e-7da4-021f9725d855 -2025-06-03T23:34:39.713Z [INFO] server-adc70873-96aa-159d-7bd0-5de7d801dbe9: added peer, starting replication: peer=server-1acbaf23-958d-5e58-fb98-db43f6dfc565 -2025-06-03T23:34:39.714Z [INFO] server-adc70873-96aa-159d-7bd0-5de7d801dbe9: pipelining replication: peer="{Voter server-1acbaf23-958d-5e58-fb98-db43f6dfc565 1acbaf23-958d-5e58-fb98-db43f6dfc565}" -2025-06-03T23:34:39.714Z [INFO] server-adc70873-96aa-159d-7bd0-5de7d801dbe9: pipelining replication: peer="{Voter server-f74a98d0-ff2b-d28e-7da4-021f9725d855 f74a98d0-ff2b-d28e-7da4-021f9725d855}" -2025-06-03T23:34:40.719Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:39.713367591 +0000 UTC m=+94.734186961 (1 nodes), 532.53004ms from start of poll, 533.11ms from cluster start. Timeout at 2025-06-03 23:34:40.719097671 +0000 UTC m=+95.739917081, 1.00573012s after stability -2025-06-03T23:34:40.719Z [INFO] server-adc70873-96aa-159d-7bd0-5de7d801dbe9: starting snapshot up to: index=12 -2025-06-03T23:34:40.719Z [INFO] creating new snapshot: path=/tmp/raft938802679/snapshots/2-12-1748993680719.tmp -2025-06-03T23:34:40.720Z [INFO] server-adc70873-96aa-159d-7bd0-5de7d801dbe9: snapshot complete up to: index=12 -2025-06-03T23:34:40.722Z [INFO] server-a009c017-fc9a-fc47-834b-4b35e0ea34aa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a009c017-fc9a-fc47-834b-4b35e0ea34aa Address:a009c017-fc9a-fc47-834b-4b35e0ea34aa} {Suffrage:Voter ID:server-f4607698-7cf4-27b1-caf7-728dc64245ac Address:f4607698-7cf4-27b1-caf7-728dc64245ac} {Suffrage:Voter ID:server-c4d6d558-84db-1df0-48c5-8ce93e8df8b9 Address:c4d6d558-84db-1df0-48c5-8ce93e8df8b9}]" -2025-06-03T23:34:40.722Z [INFO] server-a009c017-fc9a-fc47-834b-4b35e0ea34aa: entering follower state: follower="Node at a009c017-fc9a-fc47-834b-4b35e0ea34aa [Follower]" leader-address= leader-id= -2025-06-03T23:34:40.722Z [INFO] server-f4607698-7cf4-27b1-caf7-728dc64245ac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a009c017-fc9a-fc47-834b-4b35e0ea34aa Address:a009c017-fc9a-fc47-834b-4b35e0ea34aa} {Suffrage:Voter ID:server-f4607698-7cf4-27b1-caf7-728dc64245ac Address:f4607698-7cf4-27b1-caf7-728dc64245ac} {Suffrage:Voter ID:server-c4d6d558-84db-1df0-48c5-8ce93e8df8b9 Address:c4d6d558-84db-1df0-48c5-8ce93e8df8b9}]" -2025-06-03T23:34:40.722Z [INFO] server-f4607698-7cf4-27b1-caf7-728dc64245ac: entering follower state: follower="Node at f4607698-7cf4-27b1-caf7-728dc64245ac [Follower]" leader-address= leader-id= -2025-06-03T23:34:40.723Z [INFO] server-c4d6d558-84db-1df0-48c5-8ce93e8df8b9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a009c017-fc9a-fc47-834b-4b35e0ea34aa Address:a009c017-fc9a-fc47-834b-4b35e0ea34aa} {Suffrage:Voter ID:server-f4607698-7cf4-27b1-caf7-728dc64245ac Address:f4607698-7cf4-27b1-caf7-728dc64245ac} {Suffrage:Voter ID:server-c4d6d558-84db-1df0-48c5-8ce93e8df8b9 Address:c4d6d558-84db-1df0-48c5-8ce93e8df8b9}]" -2025-06-03T23:34:40.723Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:40.723Z [INFO] server-c4d6d558-84db-1df0-48c5-8ce93e8df8b9: entering follower state: follower="Node at c4d6d558-84db-1df0-48c5-8ce93e8df8b9 [Follower]" leader-address= leader-id= -2025-06-03T23:34:41.266Z [WARN] server-f4607698-7cf4-27b1-caf7-728dc64245ac: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:41.266Z [INFO] server-f4607698-7cf4-27b1-caf7-728dc64245ac: entering candidate state: node="Node at f4607698-7cf4-27b1-caf7-728dc64245ac [Candidate]" term=2 -2025-06-03T23:34:41.266Z [INFO] server-f4607698-7cf4-27b1-caf7-728dc64245ac: election won: term=2 tally=2 -2025-06-03T23:34:41.266Z [INFO] server-f4607698-7cf4-27b1-caf7-728dc64245ac: entering leader state: leader="Node at f4607698-7cf4-27b1-caf7-728dc64245ac [Leader]" -2025-06-03T23:34:41.266Z [INFO] server-f4607698-7cf4-27b1-caf7-728dc64245ac: added peer, starting replication: peer=server-a009c017-fc9a-fc47-834b-4b35e0ea34aa -2025-06-03T23:34:41.266Z [INFO] server-f4607698-7cf4-27b1-caf7-728dc64245ac: added peer, starting replication: peer=server-c4d6d558-84db-1df0-48c5-8ce93e8df8b9 -2025-06-03T23:34:41.266Z [INFO] server-f4607698-7cf4-27b1-caf7-728dc64245ac: pipelining replication: peer="{Voter server-c4d6d558-84db-1df0-48c5-8ce93e8df8b9 c4d6d558-84db-1df0-48c5-8ce93e8df8b9}" -2025-06-03T23:34:41.267Z [INFO] server-f4607698-7cf4-27b1-caf7-728dc64245ac: pipelining replication: peer="{Voter server-a009c017-fc9a-fc47-834b-4b35e0ea34aa a009c017-fc9a-fc47-834b-4b35e0ea34aa}" -2025-06-03T23:34:42.272Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:41.266820831 +0000 UTC m=+96.287640201 (1 nodes), 543.72904ms from start of poll, 544.61132ms from cluster start. Timeout at 2025-06-03 23:34:42.272285111 +0000 UTC m=+97.293104481, 1.00546428s after stability -2025-06-03T23:34:42.272Z [INFO] creating new snapshot: path=/tmp/raft4108880913/snapshots/2-10013-1748993682272.tmp -2025-06-03T23:34:42.273Z [INFO] server-f4607698-7cf4-27b1-caf7-728dc64245ac: copied to local snapshot: bytes=71 -2025-06-03T23:34:42.273Z [INFO] server-f4607698-7cf4-27b1-caf7-728dc64245ac: snapshot restore progress: id=2-10013-1748993682272 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:42.273Z [INFO] server-f4607698-7cf4-27b1-caf7-728dc64245ac: removing all old logs from log store -2025-06-03T23:34:42.273Z [INFO] server-f4607698-7cf4-27b1-caf7-728dc64245ac: compacting logs: from=1 to=2 -2025-06-03T23:34:42.273Z [INFO] server-f4607698-7cf4-27b1-caf7-728dc64245ac: restored user snapshot: index=10013 -2025-06-03T23:34:42.273Z [ERROR] server-f4607698-7cf4-27b1-caf7-728dc64245ac: failed to get log: index=2 error="log not found" -2025-06-03T23:34:42.273Z [INFO] server-f4607698-7cf4-27b1-caf7-728dc64245ac: aborting pipeline replication: peer="{Voter server-c4d6d558-84db-1df0-48c5-8ce93e8df8b9 c4d6d558-84db-1df0-48c5-8ce93e8df8b9}" -2025-06-03T23:34:42.273Z [ERROR] server-f4607698-7cf4-27b1-caf7-728dc64245ac: failed to get log: index=2 error="log not found" -2025-06-03T23:34:42.273Z [INFO] server-f4607698-7cf4-27b1-caf7-728dc64245ac: aborting pipeline replication: peer="{Voter server-a009c017-fc9a-fc47-834b-4b35e0ea34aa a009c017-fc9a-fc47-834b-4b35e0ea34aa}" -2025-06-03T23:34:42.281Z [ERROR] server-f4607698-7cf4-27b1-caf7-728dc64245ac: failed to get log: index=2 error="log not found" -2025-06-03T23:34:42.281Z [ERROR] server-f4607698-7cf4-27b1-caf7-728dc64245ac: failed to get log: index=2 error="log not found" -2025-06-03T23:34:42.281Z [INFO] creating new snapshot: path=/tmp/raft195094389/snapshots/2-10013-1748993682281.tmp -2025-06-03T23:34:42.282Z [INFO] server-a009c017-fc9a-fc47-834b-4b35e0ea34aa: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:42.282Z [INFO] creating new snapshot: path=/tmp/raft95679318/snapshots/2-10013-1748993682282.tmp -2025-06-03T23:34:42.282Z [INFO] server-c4d6d558-84db-1df0-48c5-8ce93e8df8b9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:42.282Z [INFO] server-a009c017-fc9a-fc47-834b-4b35e0ea34aa: copied to local snapshot: bytes=71 -2025-06-03T23:34:42.283Z [INFO] server-c4d6d558-84db-1df0-48c5-8ce93e8df8b9: copied to local snapshot: bytes=71 -2025-06-03T23:34:42.283Z [INFO] server-a009c017-fc9a-fc47-834b-4b35e0ea34aa: snapshot restore progress: id=2-10013-1748993682281 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:42.283Z [INFO] server-a009c017-fc9a-fc47-834b-4b35e0ea34aa: removing all old logs from log store -2025-06-03T23:34:42.283Z [INFO] server-a009c017-fc9a-fc47-834b-4b35e0ea34aa: compacting logs: from=1 to=2 -2025-06-03T23:34:42.283Z [INFO] server-a009c017-fc9a-fc47-834b-4b35e0ea34aa: Installed remote snapshot -2025-06-03T23:34:42.283Z [INFO] server-f4607698-7cf4-27b1-caf7-728dc64245ac: pipelining replication: peer="{Voter server-a009c017-fc9a-fc47-834b-4b35e0ea34aa a009c017-fc9a-fc47-834b-4b35e0ea34aa}" -2025-06-03T23:34:42.283Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:42.283Z [INFO] server-c4d6d558-84db-1df0-48c5-8ce93e8df8b9: snapshot restore progress: id=2-10013-1748993682282 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-06-03T23:34:42.283Z [INFO] server-c4d6d558-84db-1df0-48c5-8ce93e8df8b9: removing all old logs from log store -2025-06-03T23:34:42.283Z [INFO] server-c4d6d558-84db-1df0-48c5-8ce93e8df8b9: compacting logs: from=1 to=2 -2025-06-03T23:34:42.284Z [INFO] server-c4d6d558-84db-1df0-48c5-8ce93e8df8b9: Installed remote snapshot -2025-06-03T23:34:42.284Z [INFO] server-f4607698-7cf4-27b1-caf7-728dc64245ac: pipelining replication: peer="{Voter server-c4d6d558-84db-1df0-48c5-8ce93e8df8b9 c4d6d558-84db-1df0-48c5-8ce93e8df8b9}" -2025-06-03T23:34:43.289Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:42.283928991 +0000 UTC m=+97.304748321 (2 nodes), 1.6µs from start of poll, 1.56171944s from cluster start. Timeout at 2025-06-03 23:34:43.289095511 +0000 UTC m=+98.309914881, 1.00516656s after stability -2025-06-03T23:34:43.299Z [INFO] server-adc70873-96aa-159d-7bd0-5de7d801dbe9: aborting pipeline replication: peer="{Voter server-1acbaf23-958d-5e58-fb98-db43f6dfc565 1acbaf23-958d-5e58-fb98-db43f6dfc565}" -2025-06-03T23:34:43.299Z [INFO] server-adc70873-96aa-159d-7bd0-5de7d801dbe9: aborting pipeline replication: peer="{Voter server-f74a98d0-ff2b-d28e-7da4-021f9725d855 f74a98d0-ff2b-d28e-7da4-021f9725d855}" ---- PASS: TestRaft_UserRestore (84.31s) - --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false (3.04s) - --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false (2.57s) +2025-06-03T23:37:37.571Z [INFO] server-0dcb554e-f05e-4db0-9841-65af0badf3fc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0dcb554e-f05e-4db0-9841-65af0badf3fc Address:0dcb554e-f05e-4db0-9841-65af0badf3fc} {Suffrage:Voter ID:server-30eaaff0-4511-0d40-22da-c8ee3e0391e7 Address:30eaaff0-4511-0d40-22da-c8ee3e0391e7} {Suffrage:Voter ID:server-8a434c56-e11b-3dae-89c5-33f85a410ca8 Address:8a434c56-e11b-3dae-89c5-33f85a410ca8}]" +2025-06-03T23:37:37.571Z [INFO] server-0dcb554e-f05e-4db0-9841-65af0badf3fc: entering follower state: follower="Node at 0dcb554e-f05e-4db0-9841-65af0badf3fc [Follower]" leader-address= leader-id= +2025-06-03T23:37:37.571Z [INFO] server-30eaaff0-4511-0d40-22da-c8ee3e0391e7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0dcb554e-f05e-4db0-9841-65af0badf3fc Address:0dcb554e-f05e-4db0-9841-65af0badf3fc} {Suffrage:Voter ID:server-30eaaff0-4511-0d40-22da-c8ee3e0391e7 Address:30eaaff0-4511-0d40-22da-c8ee3e0391e7} {Suffrage:Voter ID:server-8a434c56-e11b-3dae-89c5-33f85a410ca8 Address:8a434c56-e11b-3dae-89c5-33f85a410ca8}]" +2025-06-03T23:37:37.571Z [INFO] server-30eaaff0-4511-0d40-22da-c8ee3e0391e7: entering follower state: follower="Node at 30eaaff0-4511-0d40-22da-c8ee3e0391e7 [Follower]" leader-address= leader-id= +2025-06-03T23:37:37.572Z [INFO] server-8a434c56-e11b-3dae-89c5-33f85a410ca8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0dcb554e-f05e-4db0-9841-65af0badf3fc Address:0dcb554e-f05e-4db0-9841-65af0badf3fc} {Suffrage:Voter ID:server-30eaaff0-4511-0d40-22da-c8ee3e0391e7 Address:30eaaff0-4511-0d40-22da-c8ee3e0391e7} {Suffrage:Voter ID:server-8a434c56-e11b-3dae-89c5-33f85a410ca8 Address:8a434c56-e11b-3dae-89c5-33f85a410ca8}]" +2025-06-03T23:37:37.572Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:37.572Z [INFO] server-8a434c56-e11b-3dae-89c5-33f85a410ca8: entering follower state: follower="Node at 8a434c56-e11b-3dae-89c5-33f85a410ca8 [Follower]" leader-address= leader-id= +2025-06-03T23:37:38.127Z [WARN] server-30eaaff0-4511-0d40-22da-c8ee3e0391e7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:38.127Z [INFO] server-30eaaff0-4511-0d40-22da-c8ee3e0391e7: entering candidate state: node="Node at 30eaaff0-4511-0d40-22da-c8ee3e0391e7 [Candidate]" term=2 +2025-06-03T23:37:38.127Z [INFO] server-30eaaff0-4511-0d40-22da-c8ee3e0391e7: election won: term=2 tally=2 +2025-06-03T23:37:38.127Z [INFO] server-30eaaff0-4511-0d40-22da-c8ee3e0391e7: entering leader state: leader="Node at 30eaaff0-4511-0d40-22da-c8ee3e0391e7 [Leader]" +2025-06-03T23:37:38.127Z [INFO] server-30eaaff0-4511-0d40-22da-c8ee3e0391e7: added peer, starting replication: peer=server-0dcb554e-f05e-4db0-9841-65af0badf3fc +2025-06-03T23:37:38.127Z [INFO] server-30eaaff0-4511-0d40-22da-c8ee3e0391e7: added peer, starting replication: peer=server-8a434c56-e11b-3dae-89c5-33f85a410ca8 +2025-06-03T23:37:38.127Z [INFO] server-30eaaff0-4511-0d40-22da-c8ee3e0391e7: pipelining replication: peer="{Voter server-8a434c56-e11b-3dae-89c5-33f85a410ca8 8a434c56-e11b-3dae-89c5-33f85a410ca8}" +2025-06-03T23:37:38.127Z [INFO] server-30eaaff0-4511-0d40-22da-c8ee3e0391e7: pipelining replication: peer="{Voter server-0dcb554e-f05e-4db0-9841-65af0badf3fc 0dcb554e-f05e-4db0-9841-65af0badf3fc}" +2025-06-03T23:37:39.133Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:38.127794631 +0000 UTC m=+93.449549561 (1 nodes), 555.7194ms from start of poll, 556.20304ms from cluster start. Timeout at 2025-06-03 23:37:39.133418751 +0000 UTC m=+94.455173641, 1.00562408s after stability +2025-06-03T23:37:39.134Z [INFO] server-30eaaff0-4511-0d40-22da-c8ee3e0391e7: starting snapshot up to: index=12 +2025-06-03T23:37:39.134Z [INFO] creating new snapshot: path=/tmp/raft2718795745/snapshots/2-12-1748993859134.tmp +2025-06-03T23:37:39.136Z [INFO] server-30eaaff0-4511-0d40-22da-c8ee3e0391e7: snapshot complete up to: index=12 +2025-06-03T23:37:39.139Z [INFO] server-0089010b-84e8-34f2-793d-11b1be546199: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0089010b-84e8-34f2-793d-11b1be546199 Address:0089010b-84e8-34f2-793d-11b1be546199} {Suffrage:Voter ID:server-816a4101-5536-0b6a-3258-3205d6cbbbb0 Address:816a4101-5536-0b6a-3258-3205d6cbbbb0} {Suffrage:Voter ID:server-3bb93bbf-0514-0902-b5fe-24b7775b5aae Address:3bb93bbf-0514-0902-b5fe-24b7775b5aae}]" +2025-06-03T23:37:39.139Z [INFO] server-0089010b-84e8-34f2-793d-11b1be546199: entering follower state: follower="Node at 0089010b-84e8-34f2-793d-11b1be546199 [Follower]" leader-address= leader-id= +2025-06-03T23:37:39.139Z [INFO] server-816a4101-5536-0b6a-3258-3205d6cbbbb0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0089010b-84e8-34f2-793d-11b1be546199 Address:0089010b-84e8-34f2-793d-11b1be546199} {Suffrage:Voter ID:server-816a4101-5536-0b6a-3258-3205d6cbbbb0 Address:816a4101-5536-0b6a-3258-3205d6cbbbb0} {Suffrage:Voter ID:server-3bb93bbf-0514-0902-b5fe-24b7775b5aae Address:3bb93bbf-0514-0902-b5fe-24b7775b5aae}]" +2025-06-03T23:37:39.139Z [INFO] server-816a4101-5536-0b6a-3258-3205d6cbbbb0: entering follower state: follower="Node at 816a4101-5536-0b6a-3258-3205d6cbbbb0 [Follower]" leader-address= leader-id= +2025-06-03T23:37:39.139Z [INFO] server-3bb93bbf-0514-0902-b5fe-24b7775b5aae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0089010b-84e8-34f2-793d-11b1be546199 Address:0089010b-84e8-34f2-793d-11b1be546199} {Suffrage:Voter ID:server-816a4101-5536-0b6a-3258-3205d6cbbbb0 Address:816a4101-5536-0b6a-3258-3205d6cbbbb0} {Suffrage:Voter ID:server-3bb93bbf-0514-0902-b5fe-24b7775b5aae Address:3bb93bbf-0514-0902-b5fe-24b7775b5aae}]" +2025-06-03T23:37:39.139Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:39.139Z [INFO] server-3bb93bbf-0514-0902-b5fe-24b7775b5aae: entering follower state: follower="Node at 3bb93bbf-0514-0902-b5fe-24b7775b5aae [Follower]" leader-address= leader-id= +2025-06-03T23:37:39.719Z [WARN] server-0089010b-84e8-34f2-793d-11b1be546199: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:39.719Z [INFO] server-0089010b-84e8-34f2-793d-11b1be546199: entering candidate state: node="Node at 0089010b-84e8-34f2-793d-11b1be546199 [Candidate]" term=2 +2025-06-03T23:37:39.719Z [INFO] server-0089010b-84e8-34f2-793d-11b1be546199: election won: term=2 tally=2 +2025-06-03T23:37:39.719Z [INFO] server-0089010b-84e8-34f2-793d-11b1be546199: entering leader state: leader="Node at 0089010b-84e8-34f2-793d-11b1be546199 [Leader]" +2025-06-03T23:37:39.719Z [INFO] server-0089010b-84e8-34f2-793d-11b1be546199: added peer, starting replication: peer=server-816a4101-5536-0b6a-3258-3205d6cbbbb0 +2025-06-03T23:37:39.719Z [INFO] server-0089010b-84e8-34f2-793d-11b1be546199: added peer, starting replication: peer=server-3bb93bbf-0514-0902-b5fe-24b7775b5aae +2025-06-03T23:37:39.719Z [INFO] server-0089010b-84e8-34f2-793d-11b1be546199: pipelining replication: peer="{Voter server-816a4101-5536-0b6a-3258-3205d6cbbbb0 816a4101-5536-0b6a-3258-3205d6cbbbb0}" +2025-06-03T23:37:39.719Z [INFO] server-0089010b-84e8-34f2-793d-11b1be546199: pipelining replication: peer="{Voter server-3bb93bbf-0514-0902-b5fe-24b7775b5aae 3bb93bbf-0514-0902-b5fe-24b7775b5aae}" +2025-06-03T23:37:40.724Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:39.719289711 +0000 UTC m=+95.041044641 (1 nodes), 579.60632ms from start of poll, 580.09264ms from cluster start. Timeout at 2025-06-03 23:37:40.724704271 +0000 UTC m=+96.046459201, 1.00541456s after stability +2025-06-03T23:37:40.725Z [INFO] creating new snapshot: path=/tmp/raft432645760/snapshots/2-10013-1748993860725.tmp +2025-06-03T23:37:40.725Z [INFO] server-0089010b-84e8-34f2-793d-11b1be546199: copied to local snapshot: bytes=71 +2025-06-03T23:37:40.725Z [INFO] server-0089010b-84e8-34f2-793d-11b1be546199: snapshot restore progress: id=2-10013-1748993860725 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:40.726Z [INFO] server-0089010b-84e8-34f2-793d-11b1be546199: removing all old logs from log store +2025-06-03T23:37:40.726Z [INFO] server-0089010b-84e8-34f2-793d-11b1be546199: compacting logs: from=1 to=2 +2025-06-03T23:37:40.726Z [INFO] server-0089010b-84e8-34f2-793d-11b1be546199: restored user snapshot: index=10013 +2025-06-03T23:37:40.726Z [ERROR] server-0089010b-84e8-34f2-793d-11b1be546199: failed to get log: index=2 error="log not found" +2025-06-03T23:37:40.726Z [INFO] server-0089010b-84e8-34f2-793d-11b1be546199: aborting pipeline replication: peer="{Voter server-3bb93bbf-0514-0902-b5fe-24b7775b5aae 3bb93bbf-0514-0902-b5fe-24b7775b5aae}" +2025-06-03T23:37:40.726Z [ERROR] server-0089010b-84e8-34f2-793d-11b1be546199: failed to get log: index=2 error="log not found" +2025-06-03T23:37:40.726Z [INFO] server-0089010b-84e8-34f2-793d-11b1be546199: aborting pipeline replication: peer="{Voter server-816a4101-5536-0b6a-3258-3205d6cbbbb0 816a4101-5536-0b6a-3258-3205d6cbbbb0}" +2025-06-03T23:37:40.733Z [ERROR] server-0089010b-84e8-34f2-793d-11b1be546199: failed to get log: index=2 error="log not found" +2025-06-03T23:37:40.733Z [ERROR] server-0089010b-84e8-34f2-793d-11b1be546199: failed to get log: index=2 error="log not found" +2025-06-03T23:37:40.733Z [INFO] creating new snapshot: path=/tmp/raft567839582/snapshots/2-10013-1748993860733.tmp +2025-06-03T23:37:40.734Z [INFO] creating new snapshot: path=/tmp/raft3185075285/snapshots/2-10013-1748993860734.tmp +2025-06-03T23:37:40.734Z [INFO] server-3bb93bbf-0514-0902-b5fe-24b7775b5aae: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:40.734Z [INFO] server-816a4101-5536-0b6a-3258-3205d6cbbbb0: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:40.734Z [INFO] server-3bb93bbf-0514-0902-b5fe-24b7775b5aae: copied to local snapshot: bytes=71 +2025-06-03T23:37:40.734Z [INFO] server-816a4101-5536-0b6a-3258-3205d6cbbbb0: copied to local snapshot: bytes=71 +2025-06-03T23:37:40.734Z [INFO] server-3bb93bbf-0514-0902-b5fe-24b7775b5aae: snapshot restore progress: id=2-10013-1748993860733 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:40.734Z [INFO] server-3bb93bbf-0514-0902-b5fe-24b7775b5aae: removing all old logs from log store +2025-06-03T23:37:40.734Z [INFO] server-3bb93bbf-0514-0902-b5fe-24b7775b5aae: compacting logs: from=1 to=2 +2025-06-03T23:37:40.734Z [INFO] server-3bb93bbf-0514-0902-b5fe-24b7775b5aae: Installed remote snapshot +2025-06-03T23:37:40.735Z [INFO] server-0089010b-84e8-34f2-793d-11b1be546199: pipelining replication: peer="{Voter server-3bb93bbf-0514-0902-b5fe-24b7775b5aae 3bb93bbf-0514-0902-b5fe-24b7775b5aae}" +2025-06-03T23:37:40.735Z [INFO] server-816a4101-5536-0b6a-3258-3205d6cbbbb0: snapshot restore progress: id=2-10013-1748993860734 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-03T23:37:40.735Z [INFO] server-816a4101-5536-0b6a-3258-3205d6cbbbb0: removing all old logs from log store +2025-06-03T23:37:40.735Z [INFO] server-816a4101-5536-0b6a-3258-3205d6cbbbb0: compacting logs: from=1 to=2 +2025-06-03T23:37:40.735Z [INFO] server-816a4101-5536-0b6a-3258-3205d6cbbbb0: Installed remote snapshot +2025-06-03T23:37:40.735Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:40.735Z [INFO] server-0089010b-84e8-34f2-793d-11b1be546199: pipelining replication: peer="{Voter server-816a4101-5536-0b6a-3258-3205d6cbbbb0 816a4101-5536-0b6a-3258-3205d6cbbbb0}" +2025-06-03T23:37:41.741Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:40.735207871 +0000 UTC m=+96.056962801 (2 nodes), 1.36µs from start of poll, 1.5960108s from cluster start. Timeout at 2025-06-03 23:37:41.741003791 +0000 UTC m=+97.062758721, 1.00579592s after stability +2025-06-03T23:37:41.757Z [INFO] server-30eaaff0-4511-0d40-22da-c8ee3e0391e7: aborting pipeline replication: peer="{Voter server-8a434c56-e11b-3dae-89c5-33f85a410ca8 8a434c56-e11b-3dae-89c5-33f85a410ca8}" +2025-06-03T23:37:41.757Z [INFO] server-30eaaff0-4511-0d40-22da-c8ee3e0391e7: aborting pipeline replication: peer="{Voter server-0dcb554e-f05e-4db0-9841-65af0badf3fc 0dcb554e-f05e-4db0-9841-65af0badf3fc}" +--- PASS: TestRaft_UserRestore (84.02s) + --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false (2.99s) + --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false (2.64s) --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true (4.34s) --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true (4.23s) - --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false (2.67s) - --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false (2.89s) - --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true (4.45s) - --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true (4.43s) - --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false (2.80s) - --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false (2.56s) - --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true (4.24s) - --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true (4.21s) - --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false (2.95s) - --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false (2.68s) - --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true (4.13s) - --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true (4.48s) - --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false (2.56s) - --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false (2.78s) - --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true (4.21s) - --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true (4.24s) - --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false (2.57s) - --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false (2.58s) - --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true (4.58s) - --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true (4.12s) + --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false (2.87s) + --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false (2.65s) + --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true (4.43s) + --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true (4.10s) + --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false (2.73s) + --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false (2.68s) + --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true (4.31s) + --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true (4.72s) + --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false (2.70s) + --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false (2.59s) + --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true (4.26s) + --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true (4.19s) + --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false (2.63s) + --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false (2.66s) + --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true (4.47s) + --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true (4.13s) + --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false (2.67s) + --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false (2.62s) + --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true (4.21s) + --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true (4.19s) === RUN TestRaft_SendSnapshotFollower -2025-06-03T23:34:43.300Z [INFO] server-75d86dce-5ad7-2d4e-1c83-89dc019c948c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-75d86dce-5ad7-2d4e-1c83-89dc019c948c Address:75d86dce-5ad7-2d4e-1c83-89dc019c948c} {Suffrage:Voter ID:server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8 Address:5a956bc2-b120-daaf-5c52-b122ec9c0fa8} {Suffrage:Voter ID:server-2e25b6d0-d60f-030a-a9e5-d510ef811ac9 Address:2e25b6d0-d60f-030a-a9e5-d510ef811ac9}]" -2025-06-03T23:34:43.301Z [INFO] server-75d86dce-5ad7-2d4e-1c83-89dc019c948c: entering follower state: follower="Node at 75d86dce-5ad7-2d4e-1c83-89dc019c948c [Follower]" leader-address= leader-id= -2025-06-03T23:34:43.301Z [INFO] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-75d86dce-5ad7-2d4e-1c83-89dc019c948c Address:75d86dce-5ad7-2d4e-1c83-89dc019c948c} {Suffrage:Voter ID:server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8 Address:5a956bc2-b120-daaf-5c52-b122ec9c0fa8} {Suffrage:Voter ID:server-2e25b6d0-d60f-030a-a9e5-d510ef811ac9 Address:2e25b6d0-d60f-030a-a9e5-d510ef811ac9}]" -2025-06-03T23:34:43.301Z [INFO] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: entering follower state: follower="Node at 5a956bc2-b120-daaf-5c52-b122ec9c0fa8 [Follower]" leader-address= leader-id= -2025-06-03T23:34:43.301Z [INFO] server-2e25b6d0-d60f-030a-a9e5-d510ef811ac9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-75d86dce-5ad7-2d4e-1c83-89dc019c948c Address:75d86dce-5ad7-2d4e-1c83-89dc019c948c} {Suffrage:Voter ID:server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8 Address:5a956bc2-b120-daaf-5c52-b122ec9c0fa8} {Suffrage:Voter ID:server-2e25b6d0-d60f-030a-a9e5-d510ef811ac9 Address:2e25b6d0-d60f-030a-a9e5-d510ef811ac9}]" -2025-06-03T23:34:43.301Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:43.301Z [INFO] server-2e25b6d0-d60f-030a-a9e5-d510ef811ac9: entering follower state: follower="Node at 2e25b6d0-d60f-030a-a9e5-d510ef811ac9 [Follower]" leader-address= leader-id= -2025-06-03T23:34:43.374Z [WARN] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:43.374Z [INFO] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: entering candidate state: node="Node at 5a956bc2-b120-daaf-5c52-b122ec9c0fa8 [Candidate]" term=2 -2025-06-03T23:34:43.374Z [INFO] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: election won: term=2 tally=2 -2025-06-03T23:34:43.374Z [INFO] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: entering leader state: leader="Node at 5a956bc2-b120-daaf-5c52-b122ec9c0fa8 [Leader]" -2025-06-03T23:34:43.374Z [INFO] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: added peer, starting replication: peer=server-75d86dce-5ad7-2d4e-1c83-89dc019c948c -2025-06-03T23:34:43.374Z [INFO] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: added peer, starting replication: peer=server-2e25b6d0-d60f-030a-a9e5-d510ef811ac9 -2025-06-03T23:34:43.374Z [INFO] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: pipelining replication: peer="{Voter server-75d86dce-5ad7-2d4e-1c83-89dc019c948c 75d86dce-5ad7-2d4e-1c83-89dc019c948c}" -2025-06-03T23:34:43.374Z [INFO] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: pipelining replication: peer="{Voter server-2e25b6d0-d60f-030a-a9e5-d510ef811ac9 2e25b6d0-d60f-030a-a9e5-d510ef811ac9}" -2025-06-03T23:34:43.480Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:43.374451351 +0000 UTC m=+98.395270721 (2 nodes), 73.16228ms from start of poll, 73.63068ms from cluster start. Timeout at 2025-06-03 23:34:43.480042031 +0000 UTC m=+98.500861401, 105.59068ms after stability -2025-06-03T23:34:43.480Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:43.585Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:43.480198511 +0000 UTC m=+98.501017921 (1 nodes), 3.28µs from start of poll, 179.37788ms from cluster start. Timeout at 2025-06-03 23:34:43.585857271 +0000 UTC m=+98.606676641, 105.65872ms after stability -2025-06-03T23:34:43.586Z [ERROR] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: failed to pipeline appendEntries: peer="{Voter server-75d86dce-5ad7-2d4e-1c83-89dc019c948c 75d86dce-5ad7-2d4e-1c83-89dc019c948c}" error="append pipeline closed" -2025-06-03T23:34:43.586Z [INFO] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: aborting pipeline replication: peer="{Voter server-75d86dce-5ad7-2d4e-1c83-89dc019c948c 75d86dce-5ad7-2d4e-1c83-89dc019c948c}" -2025-06-03T23:34:43.586Z [ERROR] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: failed to appendEntries to: peer="{Voter server-75d86dce-5ad7-2d4e-1c83-89dc019c948c 75d86dce-5ad7-2d4e-1c83-89dc019c948c}" error="failed to connect to peer: 75d86dce-5ad7-2d4e-1c83-89dc019c948c" -2025-06-03T23:34:43.591Z [ERROR] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: failed to heartbeat to: peer=75d86dce-5ad7-2d4e-1c83-89dc019c948c backoff time=10ms error="failed to connect to peer: 75d86dce-5ad7-2d4e-1c83-89dc019c948c" +2025-06-03T23:37:41.759Z [INFO] server-869fb176-2716-623b-3d03-ebdef3fe7cd3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-869fb176-2716-623b-3d03-ebdef3fe7cd3 Address:869fb176-2716-623b-3d03-ebdef3fe7cd3} {Suffrage:Voter ID:server-e3b349c2-acdf-4387-0674-a1029252036d Address:e3b349c2-acdf-4387-0674-a1029252036d} {Suffrage:Voter ID:server-d9597d09-e438-6449-9732-70f8d1542cc1 Address:d9597d09-e438-6449-9732-70f8d1542cc1}]" +2025-06-03T23:37:41.759Z [INFO] server-869fb176-2716-623b-3d03-ebdef3fe7cd3: entering follower state: follower="Node at 869fb176-2716-623b-3d03-ebdef3fe7cd3 [Follower]" leader-address= leader-id= +2025-06-03T23:37:41.759Z [INFO] server-e3b349c2-acdf-4387-0674-a1029252036d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-869fb176-2716-623b-3d03-ebdef3fe7cd3 Address:869fb176-2716-623b-3d03-ebdef3fe7cd3} {Suffrage:Voter ID:server-e3b349c2-acdf-4387-0674-a1029252036d Address:e3b349c2-acdf-4387-0674-a1029252036d} {Suffrage:Voter ID:server-d9597d09-e438-6449-9732-70f8d1542cc1 Address:d9597d09-e438-6449-9732-70f8d1542cc1}]" +2025-06-03T23:37:41.759Z [INFO] server-e3b349c2-acdf-4387-0674-a1029252036d: entering follower state: follower="Node at e3b349c2-acdf-4387-0674-a1029252036d [Follower]" leader-address= leader-id= +2025-06-03T23:37:41.759Z [INFO] server-d9597d09-e438-6449-9732-70f8d1542cc1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-869fb176-2716-623b-3d03-ebdef3fe7cd3 Address:869fb176-2716-623b-3d03-ebdef3fe7cd3} {Suffrage:Voter ID:server-e3b349c2-acdf-4387-0674-a1029252036d Address:e3b349c2-acdf-4387-0674-a1029252036d} {Suffrage:Voter ID:server-d9597d09-e438-6449-9732-70f8d1542cc1 Address:d9597d09-e438-6449-9732-70f8d1542cc1}]" +2025-06-03T23:37:41.759Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:41.759Z [INFO] server-d9597d09-e438-6449-9732-70f8d1542cc1: entering follower state: follower="Node at d9597d09-e438-6449-9732-70f8d1542cc1 [Follower]" leader-address= leader-id= +2025-06-03T23:37:41.812Z [WARN] server-e3b349c2-acdf-4387-0674-a1029252036d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:41.812Z [INFO] server-e3b349c2-acdf-4387-0674-a1029252036d: entering candidate state: node="Node at e3b349c2-acdf-4387-0674-a1029252036d [Candidate]" term=2 +2025-06-03T23:37:41.812Z [INFO] server-e3b349c2-acdf-4387-0674-a1029252036d: election won: term=2 tally=2 +2025-06-03T23:37:41.812Z [INFO] server-e3b349c2-acdf-4387-0674-a1029252036d: entering leader state: leader="Node at e3b349c2-acdf-4387-0674-a1029252036d [Leader]" +2025-06-03T23:37:41.812Z [INFO] server-e3b349c2-acdf-4387-0674-a1029252036d: added peer, starting replication: peer=server-869fb176-2716-623b-3d03-ebdef3fe7cd3 +2025-06-03T23:37:41.813Z [INFO] server-e3b349c2-acdf-4387-0674-a1029252036d: added peer, starting replication: peer=server-d9597d09-e438-6449-9732-70f8d1542cc1 +2025-06-03T23:37:41.813Z [INFO] server-e3b349c2-acdf-4387-0674-a1029252036d: pipelining replication: peer="{Voter server-869fb176-2716-623b-3d03-ebdef3fe7cd3 869fb176-2716-623b-3d03-ebdef3fe7cd3}" +2025-06-03T23:37:41.813Z [INFO] server-e3b349c2-acdf-4387-0674-a1029252036d: pipelining replication: peer="{Voter server-d9597d09-e438-6449-9732-70f8d1542cc1 d9597d09-e438-6449-9732-70f8d1542cc1}" +2025-06-03T23:37:41.919Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:41.813010391 +0000 UTC m=+97.134765321 (2 nodes), 53.46464ms from start of poll, 54.167ms from cluster start. Timeout at 2025-06-03 23:37:41.918999471 +0000 UTC m=+97.240754401, 105.98908ms after stability +2025-06-03T23:37:41.919Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:42.024Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:41.919141951 +0000 UTC m=+97.240896881 (1 nodes), 2.48µs from start of poll, 160.29856ms from cluster start. Timeout at 2025-06-03 23:37:42.024378471 +0000 UTC m=+97.346133401, 105.23652ms after stability +2025-06-03T23:37:42.024Z [ERROR] server-e3b349c2-acdf-4387-0674-a1029252036d: failed to heartbeat to: peer=869fb176-2716-623b-3d03-ebdef3fe7cd3 backoff time=10ms error="failed to connect to peer: 869fb176-2716-623b-3d03-ebdef3fe7cd3" +2025-06-03T23:37:42.024Z [ERROR] server-e3b349c2-acdf-4387-0674-a1029252036d: failed to pipeline appendEntries: peer="{Voter server-869fb176-2716-623b-3d03-ebdef3fe7cd3 869fb176-2716-623b-3d03-ebdef3fe7cd3}" error="append pipeline closed" +2025-06-03T23:37:42.024Z [INFO] server-e3b349c2-acdf-4387-0674-a1029252036d: aborting pipeline replication: peer="{Voter server-869fb176-2716-623b-3d03-ebdef3fe7cd3 869fb176-2716-623b-3d03-ebdef3fe7cd3}" +2025-06-03T23:37:42.024Z [ERROR] server-e3b349c2-acdf-4387-0674-a1029252036d: failed to appendEntries to: peer="{Voter server-869fb176-2716-623b-3d03-ebdef3fe7cd3 869fb176-2716-623b-3d03-ebdef3fe7cd3}" error="failed to connect to peer: 869fb176-2716-623b-3d03-ebdef3fe7cd3" raft_test.go:1627: [INFO] Finished apply without behind follower -2025-06-03T23:34:43.592Z [ERROR] server-75d86dce-5ad7-2d4e-1c83-89dc019c948c: failed to take snapshot: error="nothing new to snapshot" -2025-06-03T23:34:43.592Z [INFO] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: starting snapshot up to: index=102 -2025-06-03T23:34:43.592Z [INFO] creating new snapshot: path=/tmp/raft3557238938/snapshots/2-102-1748993683592.tmp -2025-06-03T23:34:43.593Z [INFO] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: compacting logs: from=1 to=92 -2025-06-03T23:34:43.593Z [INFO] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: snapshot complete up to: index=102 -2025-06-03T23:34:43.593Z [INFO] server-2e25b6d0-d60f-030a-a9e5-d510ef811ac9: starting snapshot up to: index=97 -2025-06-03T23:34:43.593Z [INFO] creating new snapshot: path=/tmp/raft403022527/snapshots/2-97-1748993683593.tmp -2025-06-03T23:34:43.593Z [INFO] server-2e25b6d0-d60f-030a-a9e5-d510ef811ac9: compacting logs: from=1 to=92 -2025-06-03T23:34:43.593Z [INFO] server-2e25b6d0-d60f-030a-a9e5-d510ef811ac9: snapshot complete up to: index=97 -2025-06-03T23:34:43.596Z [ERROR] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: failed to get log: index=2 error="log not found" -2025-06-03T23:34:43.597Z [INFO] creating new snapshot: path=/tmp/raft4182915421/snapshots/2-102-1748993683597.tmp -2025-06-03T23:34:43.597Z [INFO] server-75d86dce-5ad7-2d4e-1c83-89dc019c948c: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% -2025-06-03T23:34:43.597Z [INFO] server-75d86dce-5ad7-2d4e-1c83-89dc019c948c: copied to local snapshot: bytes=693 -2025-06-03T23:34:43.597Z [INFO] server-75d86dce-5ad7-2d4e-1c83-89dc019c948c: snapshot restore progress: id=2-102-1748993683597 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2025-06-03T23:34:43.597Z [INFO] server-75d86dce-5ad7-2d4e-1c83-89dc019c948c: Installed remote snapshot -2025-06-03T23:34:43.597Z [WARN] server-75d86dce-5ad7-2d4e-1c83-89dc019c948c: skipping application of old log: index=102 -2025-06-03T23:34:43.598Z [INFO] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: pipelining replication: peer="{Voter server-75d86dce-5ad7-2d4e-1c83-89dc019c948c 75d86dce-5ad7-2d4e-1c83-89dc019c948c}" -2025-06-03T23:34:43.604Z [INFO] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: aborting pipeline replication: peer="{Voter server-2e25b6d0-d60f-030a-a9e5-d510ef811ac9 2e25b6d0-d60f-030a-a9e5-d510ef811ac9}" -2025-06-03T23:34:43.604Z [INFO] server-5a956bc2-b120-daaf-5c52-b122ec9c0fa8: aborting pipeline replication: peer="{Voter server-75d86dce-5ad7-2d4e-1c83-89dc019c948c 75d86dce-5ad7-2d4e-1c83-89dc019c948c}" ---- PASS: TestRaft_SendSnapshotFollower (0.30s) +2025-06-03T23:37:42.028Z [ERROR] server-869fb176-2716-623b-3d03-ebdef3fe7cd3: failed to take snapshot: error="nothing new to snapshot" +2025-06-03T23:37:42.028Z [INFO] server-e3b349c2-acdf-4387-0674-a1029252036d: starting snapshot up to: index=102 +2025-06-03T23:37:42.028Z [INFO] creating new snapshot: path=/tmp/raft3891251557/snapshots/2-102-1748993862028.tmp +2025-06-03T23:37:42.029Z [INFO] server-e3b349c2-acdf-4387-0674-a1029252036d: compacting logs: from=1 to=92 +2025-06-03T23:37:42.029Z [INFO] server-e3b349c2-acdf-4387-0674-a1029252036d: snapshot complete up to: index=102 +2025-06-03T23:37:42.029Z [INFO] server-d9597d09-e438-6449-9732-70f8d1542cc1: starting snapshot up to: index=101 +2025-06-03T23:37:42.029Z [INFO] creating new snapshot: path=/tmp/raft146877487/snapshots/2-101-1748993862029.tmp +2025-06-03T23:37:42.029Z [INFO] server-d9597d09-e438-6449-9732-70f8d1542cc1: compacting logs: from=1 to=92 +2025-06-03T23:37:42.029Z [INFO] server-d9597d09-e438-6449-9732-70f8d1542cc1: snapshot complete up to: index=101 +2025-06-03T23:37:42.035Z [ERROR] server-e3b349c2-acdf-4387-0674-a1029252036d: failed to get log: index=2 error="log not found" +2025-06-03T23:37:42.036Z [INFO] creating new snapshot: path=/tmp/raft2239828669/snapshots/2-102-1748993862036.tmp +2025-06-03T23:37:42.036Z [INFO] server-869fb176-2716-623b-3d03-ebdef3fe7cd3: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% +2025-06-03T23:37:42.036Z [INFO] server-869fb176-2716-623b-3d03-ebdef3fe7cd3: copied to local snapshot: bytes=693 +2025-06-03T23:37:42.037Z [INFO] server-869fb176-2716-623b-3d03-ebdef3fe7cd3: snapshot restore progress: id=2-102-1748993862036 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-06-03T23:37:42.037Z [INFO] server-869fb176-2716-623b-3d03-ebdef3fe7cd3: Installed remote snapshot +2025-06-03T23:37:42.037Z [WARN] server-869fb176-2716-623b-3d03-ebdef3fe7cd3: skipping application of old log: index=102 +2025-06-03T23:37:42.037Z [INFO] server-e3b349c2-acdf-4387-0674-a1029252036d: pipelining replication: peer="{Voter server-869fb176-2716-623b-3d03-ebdef3fe7cd3 869fb176-2716-623b-3d03-ebdef3fe7cd3}" +2025-06-03T23:37:42.040Z [INFO] server-e3b349c2-acdf-4387-0674-a1029252036d: aborting pipeline replication: peer="{Voter server-d9597d09-e438-6449-9732-70f8d1542cc1 d9597d09-e438-6449-9732-70f8d1542cc1}" +2025-06-03T23:37:42.040Z [INFO] server-e3b349c2-acdf-4387-0674-a1029252036d: aborting pipeline replication: peer="{Voter server-869fb176-2716-623b-3d03-ebdef3fe7cd3 869fb176-2716-623b-3d03-ebdef3fe7cd3}" +--- PASS: TestRaft_SendSnapshotFollower (0.28s) === RUN TestRaft_SendSnapshotAndLogsFollower -2025-06-03T23:34:43.605Z [INFO] server-4fa1d3ad-a25d-eb6e-bac6-288c7647dec2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4fa1d3ad-a25d-eb6e-bac6-288c7647dec2 Address:4fa1d3ad-a25d-eb6e-bac6-288c7647dec2} {Suffrage:Voter ID:server-9210acae-917b-2da5-4a9c-4babfd7173ce Address:9210acae-917b-2da5-4a9c-4babfd7173ce} {Suffrage:Voter ID:server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d Address:4b1f1bb6-5411-d885-26bd-9cccf125bb1d}]" -2025-06-03T23:34:43.605Z [INFO] server-4fa1d3ad-a25d-eb6e-bac6-288c7647dec2: entering follower state: follower="Node at 4fa1d3ad-a25d-eb6e-bac6-288c7647dec2 [Follower]" leader-address= leader-id= -2025-06-03T23:34:43.605Z [INFO] server-9210acae-917b-2da5-4a9c-4babfd7173ce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4fa1d3ad-a25d-eb6e-bac6-288c7647dec2 Address:4fa1d3ad-a25d-eb6e-bac6-288c7647dec2} {Suffrage:Voter ID:server-9210acae-917b-2da5-4a9c-4babfd7173ce Address:9210acae-917b-2da5-4a9c-4babfd7173ce} {Suffrage:Voter ID:server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d Address:4b1f1bb6-5411-d885-26bd-9cccf125bb1d}]" -2025-06-03T23:34:43.605Z [INFO] server-9210acae-917b-2da5-4a9c-4babfd7173ce: entering follower state: follower="Node at 9210acae-917b-2da5-4a9c-4babfd7173ce [Follower]" leader-address= leader-id= -2025-06-03T23:34:43.605Z [INFO] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4fa1d3ad-a25d-eb6e-bac6-288c7647dec2 Address:4fa1d3ad-a25d-eb6e-bac6-288c7647dec2} {Suffrage:Voter ID:server-9210acae-917b-2da5-4a9c-4babfd7173ce Address:9210acae-917b-2da5-4a9c-4babfd7173ce} {Suffrage:Voter ID:server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d Address:4b1f1bb6-5411-d885-26bd-9cccf125bb1d}]" -2025-06-03T23:34:43.605Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:43.605Z [INFO] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: entering follower state: follower="Node at 4b1f1bb6-5411-d885-26bd-9cccf125bb1d [Follower]" leader-address= leader-id= -2025-06-03T23:34:43.672Z [WARN] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:43.672Z [INFO] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: entering candidate state: node="Node at 4b1f1bb6-5411-d885-26bd-9cccf125bb1d [Candidate]" term=2 -2025-06-03T23:34:43.672Z [INFO] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: election won: term=2 tally=2 -2025-06-03T23:34:43.672Z [INFO] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: entering leader state: leader="Node at 4b1f1bb6-5411-d885-26bd-9cccf125bb1d [Leader]" -2025-06-03T23:34:43.672Z [INFO] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: added peer, starting replication: peer=server-4fa1d3ad-a25d-eb6e-bac6-288c7647dec2 -2025-06-03T23:34:43.672Z [INFO] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: added peer, starting replication: peer=server-9210acae-917b-2da5-4a9c-4babfd7173ce -2025-06-03T23:34:43.672Z [INFO] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: pipelining replication: peer="{Voter server-9210acae-917b-2da5-4a9c-4babfd7173ce 9210acae-917b-2da5-4a9c-4babfd7173ce}" -2025-06-03T23:34:43.672Z [INFO] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: pipelining replication: peer="{Voter server-4fa1d3ad-a25d-eb6e-bac6-288c7647dec2 4fa1d3ad-a25d-eb6e-bac6-288c7647dec2}" -2025-06-03T23:34:43.778Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:43.672760151 +0000 UTC m=+98.693579561 (2 nodes), 67.03608ms from start of poll, 67.5014ms from cluster start. Timeout at 2025-06-03 23:34:43.778781111 +0000 UTC m=+98.799600521, 106.02096ms after stability -2025-06-03T23:34:43.778Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:43.885Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:43.778913111 +0000 UTC m=+98.799732521 (1 nodes), 3.28µs from start of poll, 173.65436ms from cluster start. Timeout at 2025-06-03 23:34:43.884986711 +0000 UTC m=+98.905806081, 106.07356ms after stability -2025-06-03T23:34:43.885Z [ERROR] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: failed to heartbeat to: peer=4fa1d3ad-a25d-eb6e-bac6-288c7647dec2 backoff time=10ms error="failed to connect to peer: 4fa1d3ad-a25d-eb6e-bac6-288c7647dec2" -2025-06-03T23:34:43.885Z [ERROR] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: failed to pipeline appendEntries: peer="{Voter server-4fa1d3ad-a25d-eb6e-bac6-288c7647dec2 4fa1d3ad-a25d-eb6e-bac6-288c7647dec2}" error="append pipeline closed" -2025-06-03T23:34:43.885Z [INFO] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: aborting pipeline replication: peer="{Voter server-4fa1d3ad-a25d-eb6e-bac6-288c7647dec2 4fa1d3ad-a25d-eb6e-bac6-288c7647dec2}" -2025-06-03T23:34:43.885Z [ERROR] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: failed to appendEntries to: peer="{Voter server-4fa1d3ad-a25d-eb6e-bac6-288c7647dec2 4fa1d3ad-a25d-eb6e-bac6-288c7647dec2}" error="failed to connect to peer: 4fa1d3ad-a25d-eb6e-bac6-288c7647dec2" +2025-06-03T23:37:42.041Z [INFO] server-da100fd5-0d0e-f388-274a-967f635ca2a4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-da100fd5-0d0e-f388-274a-967f635ca2a4 Address:da100fd5-0d0e-f388-274a-967f635ca2a4} {Suffrage:Voter ID:server-cf2d648c-d330-2f27-b7c2-3c653821f44e Address:cf2d648c-d330-2f27-b7c2-3c653821f44e} {Suffrage:Voter ID:server-74dd4f55-e479-b236-c540-b32e314deae7 Address:74dd4f55-e479-b236-c540-b32e314deae7}]" +2025-06-03T23:37:42.041Z [INFO] server-da100fd5-0d0e-f388-274a-967f635ca2a4: entering follower state: follower="Node at da100fd5-0d0e-f388-274a-967f635ca2a4 [Follower]" leader-address= leader-id= +2025-06-03T23:37:42.041Z [INFO] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-da100fd5-0d0e-f388-274a-967f635ca2a4 Address:da100fd5-0d0e-f388-274a-967f635ca2a4} {Suffrage:Voter ID:server-cf2d648c-d330-2f27-b7c2-3c653821f44e Address:cf2d648c-d330-2f27-b7c2-3c653821f44e} {Suffrage:Voter ID:server-74dd4f55-e479-b236-c540-b32e314deae7 Address:74dd4f55-e479-b236-c540-b32e314deae7}]" +2025-06-03T23:37:42.041Z [INFO] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: entering follower state: follower="Node at cf2d648c-d330-2f27-b7c2-3c653821f44e [Follower]" leader-address= leader-id= +2025-06-03T23:37:42.042Z [INFO] server-74dd4f55-e479-b236-c540-b32e314deae7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-da100fd5-0d0e-f388-274a-967f635ca2a4 Address:da100fd5-0d0e-f388-274a-967f635ca2a4} {Suffrage:Voter ID:server-cf2d648c-d330-2f27-b7c2-3c653821f44e Address:cf2d648c-d330-2f27-b7c2-3c653821f44e} {Suffrage:Voter ID:server-74dd4f55-e479-b236-c540-b32e314deae7 Address:74dd4f55-e479-b236-c540-b32e314deae7}]" +2025-06-03T23:37:42.042Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:42.042Z [INFO] server-74dd4f55-e479-b236-c540-b32e314deae7: entering follower state: follower="Node at 74dd4f55-e479-b236-c540-b32e314deae7 [Follower]" leader-address= leader-id= +2025-06-03T23:37:42.106Z [WARN] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:42.106Z [INFO] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: entering candidate state: node="Node at cf2d648c-d330-2f27-b7c2-3c653821f44e [Candidate]" term=2 +2025-06-03T23:37:42.107Z [INFO] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: election won: term=2 tally=2 +2025-06-03T23:37:42.107Z [INFO] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: entering leader state: leader="Node at cf2d648c-d330-2f27-b7c2-3c653821f44e [Leader]" +2025-06-03T23:37:42.107Z [INFO] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: added peer, starting replication: peer=server-da100fd5-0d0e-f388-274a-967f635ca2a4 +2025-06-03T23:37:42.107Z [INFO] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: added peer, starting replication: peer=server-74dd4f55-e479-b236-c540-b32e314deae7 +2025-06-03T23:37:42.107Z [INFO] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: pipelining replication: peer="{Voter server-da100fd5-0d0e-f388-274a-967f635ca2a4 da100fd5-0d0e-f388-274a-967f635ca2a4}" +2025-06-03T23:37:42.107Z [INFO] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: pipelining replication: peer="{Voter server-74dd4f55-e479-b236-c540-b32e314deae7 74dd4f55-e479-b236-c540-b32e314deae7}" +2025-06-03T23:37:42.212Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:42.107048671 +0000 UTC m=+97.428803601 (2 nodes), 64.8908ms from start of poll, 65.50876ms from cluster start. Timeout at 2025-06-03 23:37:42.212289911 +0000 UTC m=+97.534044801, 105.2412ms after stability +2025-06-03T23:37:42.212Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:42.317Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:42.212407391 +0000 UTC m=+97.534162321 (1 nodes), 2.76µs from start of poll, 170.86748ms from cluster start. Timeout at 2025-06-03 23:37:42.317818231 +0000 UTC m=+97.639573161, 105.41084ms after stability +2025-06-03T23:37:42.318Z [ERROR] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: failed to pipeline appendEntries: peer="{Voter server-da100fd5-0d0e-f388-274a-967f635ca2a4 da100fd5-0d0e-f388-274a-967f635ca2a4}" error="append pipeline closed" +2025-06-03T23:37:42.318Z [INFO] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: aborting pipeline replication: peer="{Voter server-da100fd5-0d0e-f388-274a-967f635ca2a4 da100fd5-0d0e-f388-274a-967f635ca2a4}" +2025-06-03T23:37:42.318Z [ERROR] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: failed to appendEntries to: peer="{Voter server-da100fd5-0d0e-f388-274a-967f635ca2a4 da100fd5-0d0e-f388-274a-967f635ca2a4}" error="failed to connect to peer: da100fd5-0d0e-f388-274a-967f635ca2a4" raft_test.go:1669: [INFO] Finished apply without behind follower -2025-06-03T23:34:43.891Z [ERROR] server-4fa1d3ad-a25d-eb6e-bac6-288c7647dec2: failed to take snapshot: error="nothing new to snapshot" -2025-06-03T23:34:43.891Z [INFO] server-9210acae-917b-2da5-4a9c-4babfd7173ce: starting snapshot up to: index=97 -2025-06-03T23:34:43.891Z [INFO] creating new snapshot: path=/tmp/raft1523943134/snapshots/2-97-1748993683891.tmp -2025-06-03T23:34:43.891Z [INFO] server-9210acae-917b-2da5-4a9c-4babfd7173ce: compacting logs: from=1 to=92 -2025-06-03T23:34:43.891Z [INFO] server-9210acae-917b-2da5-4a9c-4babfd7173ce: snapshot complete up to: index=97 -2025-06-03T23:34:43.891Z [INFO] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: starting snapshot up to: index=102 -2025-06-03T23:34:43.891Z [INFO] creating new snapshot: path=/tmp/raft50231737/snapshots/2-102-1748993683891.tmp -2025-06-03T23:34:43.892Z [INFO] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: compacting logs: from=1 to=92 -2025-06-03T23:34:43.892Z [INFO] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: snapshot complete up to: index=102 -2025-06-03T23:34:43.895Z [ERROR] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: failed to get log: index=2 error="log not found" +2025-06-03T23:37:42.322Z [ERROR] server-da100fd5-0d0e-f388-274a-967f635ca2a4: failed to take snapshot: error="nothing new to snapshot" +2025-06-03T23:37:42.322Z [INFO] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: starting snapshot up to: index=102 +2025-06-03T23:37:42.322Z [INFO] creating new snapshot: path=/tmp/raft1385814176/snapshots/2-102-1748993862322.tmp +2025-06-03T23:37:42.322Z [INFO] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: compacting logs: from=1 to=92 +2025-06-03T23:37:42.322Z [INFO] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: snapshot complete up to: index=102 +2025-06-03T23:37:42.322Z [INFO] server-74dd4f55-e479-b236-c540-b32e314deae7: starting snapshot up to: index=101 +2025-06-03T23:37:42.322Z [INFO] creating new snapshot: path=/tmp/raft3200071923/snapshots/2-101-1748993862322.tmp +2025-06-03T23:37:42.323Z [INFO] server-74dd4f55-e479-b236-c540-b32e314deae7: compacting logs: from=1 to=92 +2025-06-03T23:37:42.323Z [INFO] server-74dd4f55-e479-b236-c540-b32e314deae7: snapshot complete up to: index=101 +2025-06-03T23:37:42.323Z [ERROR] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: failed to heartbeat to: peer=da100fd5-0d0e-f388-274a-967f635ca2a4 backoff time=10ms error="failed to connect to peer: da100fd5-0d0e-f388-274a-967f635ca2a4" +2025-06-03T23:37:42.328Z [ERROR] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: failed to get log: index=2 error="log not found" +2025-06-03T23:37:42.329Z [ERROR] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: failed to install snapshot: id=2-102-1748993862322 error="failed to connect to peer: da100fd5-0d0e-f388-274a-967f635ca2a4" +2025-06-03T23:37:42.329Z [ERROR] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: failed to send snapshot to: peer="{Voter server-da100fd5-0d0e-f388-274a-967f635ca2a4 da100fd5-0d0e-f388-274a-967f635ca2a4}" error="failed to connect to peer: da100fd5-0d0e-f388-274a-967f635ca2a4" raft_test.go:1690: [INFO] Finished apply without behind follower -2025-06-03T23:34:43.896Z [ERROR] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: failed to install snapshot: id=2-102-1748993683891 error="failed to connect to peer: 4fa1d3ad-a25d-eb6e-bac6-288c7647dec2" -2025-06-03T23:34:43.896Z [ERROR] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: failed to send snapshot to: peer="{Voter server-4fa1d3ad-a25d-eb6e-bac6-288c7647dec2 4fa1d3ad-a25d-eb6e-bac6-288c7647dec2}" error="failed to connect to peer: 4fa1d3ad-a25d-eb6e-bac6-288c7647dec2" -2025-06-03T23:34:43.907Z [ERROR] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: failed to get log: index=2 error="log not found" -2025-06-03T23:34:43.907Z [INFO] creating new snapshot: path=/tmp/raft2012936130/snapshots/2-102-1748993683907.tmp -2025-06-03T23:34:43.907Z [INFO] server-4fa1d3ad-a25d-eb6e-bac6-288c7647dec2: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% -2025-06-03T23:34:43.908Z [INFO] server-4fa1d3ad-a25d-eb6e-bac6-288c7647dec2: copied to local snapshot: bytes=693 -2025-06-03T23:34:43.908Z [INFO] server-4fa1d3ad-a25d-eb6e-bac6-288c7647dec2: snapshot restore progress: id=2-102-1748993683907 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2025-06-03T23:34:43.908Z [INFO] server-4fa1d3ad-a25d-eb6e-bac6-288c7647dec2: Installed remote snapshot -2025-06-03T23:34:43.908Z [INFO] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: pipelining replication: peer="{Voter server-4fa1d3ad-a25d-eb6e-bac6-288c7647dec2 4fa1d3ad-a25d-eb6e-bac6-288c7647dec2}" -2025-06-03T23:34:43.911Z [INFO] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: aborting pipeline replication: peer="{Voter server-9210acae-917b-2da5-4a9c-4babfd7173ce 9210acae-917b-2da5-4a9c-4babfd7173ce}" -2025-06-03T23:34:43.911Z [INFO] server-4b1f1bb6-5411-d885-26bd-9cccf125bb1d: aborting pipeline replication: peer="{Voter server-4fa1d3ad-a25d-eb6e-bac6-288c7647dec2 4fa1d3ad-a25d-eb6e-bac6-288c7647dec2}" +2025-06-03T23:37:42.339Z [ERROR] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: failed to get log: index=2 error="log not found" +2025-06-03T23:37:42.339Z [INFO] creating new snapshot: path=/tmp/raft1965185416/snapshots/2-102-1748993862339.tmp +2025-06-03T23:37:42.340Z [INFO] server-da100fd5-0d0e-f388-274a-967f635ca2a4: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% +2025-06-03T23:37:42.340Z [INFO] server-da100fd5-0d0e-f388-274a-967f635ca2a4: copied to local snapshot: bytes=693 +2025-06-03T23:37:42.340Z [INFO] server-da100fd5-0d0e-f388-274a-967f635ca2a4: snapshot restore progress: id=2-102-1748993862339 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-06-03T23:37:42.340Z [INFO] server-da100fd5-0d0e-f388-274a-967f635ca2a4: Installed remote snapshot +2025-06-03T23:37:42.341Z [INFO] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: pipelining replication: peer="{Voter server-da100fd5-0d0e-f388-274a-967f635ca2a4 da100fd5-0d0e-f388-274a-967f635ca2a4}" +2025-06-03T23:37:42.345Z [INFO] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: aborting pipeline replication: peer="{Voter server-74dd4f55-e479-b236-c540-b32e314deae7 74dd4f55-e479-b236-c540-b32e314deae7}" +2025-06-03T23:37:42.345Z [INFO] server-cf2d648c-d330-2f27-b7c2-3c653821f44e: aborting pipeline replication: peer="{Voter server-da100fd5-0d0e-f388-274a-967f635ca2a4 da100fd5-0d0e-f388-274a-967f635ca2a4}" --- PASS: TestRaft_SendSnapshotAndLogsFollower (0.31s) === RUN TestRaft_ReJoinFollower -2025-06-03T23:34:43.913Z [INFO] server-2f8b6ac1-2030-595f-d7d0-063ae8e9f541: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2f8b6ac1-2030-595f-d7d0-063ae8e9f541 Address:2f8b6ac1-2030-595f-d7d0-063ae8e9f541} {Suffrage:Voter ID:server-d9bd2e70-ee5a-2aa5-ae0e-aa91a5d82058 Address:d9bd2e70-ee5a-2aa5-ae0e-aa91a5d82058} {Suffrage:Voter ID:server-5e526f52-5131-212f-23ce-681c65dfff65 Address:5e526f52-5131-212f-23ce-681c65dfff65}]" -2025-06-03T23:34:43.913Z [INFO] server-d9bd2e70-ee5a-2aa5-ae0e-aa91a5d82058: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2f8b6ac1-2030-595f-d7d0-063ae8e9f541 Address:2f8b6ac1-2030-595f-d7d0-063ae8e9f541} {Suffrage:Voter ID:server-d9bd2e70-ee5a-2aa5-ae0e-aa91a5d82058 Address:d9bd2e70-ee5a-2aa5-ae0e-aa91a5d82058} {Suffrage:Voter ID:server-5e526f52-5131-212f-23ce-681c65dfff65 Address:5e526f52-5131-212f-23ce-681c65dfff65}]" -2025-06-03T23:34:43.913Z [INFO] server-2f8b6ac1-2030-595f-d7d0-063ae8e9f541: entering follower state: follower="Node at 2f8b6ac1-2030-595f-d7d0-063ae8e9f541 [Follower]" leader-address= leader-id= -2025-06-03T23:34:43.913Z [INFO] server-d9bd2e70-ee5a-2aa5-ae0e-aa91a5d82058: entering follower state: follower="Node at d9bd2e70-ee5a-2aa5-ae0e-aa91a5d82058 [Follower]" leader-address= leader-id= -2025-06-03T23:34:43.913Z [INFO] server-5e526f52-5131-212f-23ce-681c65dfff65: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2f8b6ac1-2030-595f-d7d0-063ae8e9f541 Address:2f8b6ac1-2030-595f-d7d0-063ae8e9f541} {Suffrage:Voter ID:server-d9bd2e70-ee5a-2aa5-ae0e-aa91a5d82058 Address:d9bd2e70-ee5a-2aa5-ae0e-aa91a5d82058} {Suffrage:Voter ID:server-5e526f52-5131-212f-23ce-681c65dfff65 Address:5e526f52-5131-212f-23ce-681c65dfff65}]" -2025-06-03T23:34:43.913Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:43.913Z [INFO] server-5e526f52-5131-212f-23ce-681c65dfff65: entering follower state: follower="Node at 5e526f52-5131-212f-23ce-681c65dfff65 [Follower]" leader-address= leader-id= -2025-06-03T23:34:43.969Z [WARN] server-5e526f52-5131-212f-23ce-681c65dfff65: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:43.969Z [INFO] server-5e526f52-5131-212f-23ce-681c65dfff65: entering candidate state: node="Node at 5e526f52-5131-212f-23ce-681c65dfff65 [Candidate]" term=2 -2025-06-03T23:34:43.970Z [INFO] server-5e526f52-5131-212f-23ce-681c65dfff65: election won: term=2 tally=2 -2025-06-03T23:34:43.970Z [INFO] server-5e526f52-5131-212f-23ce-681c65dfff65: entering leader state: leader="Node at 5e526f52-5131-212f-23ce-681c65dfff65 [Leader]" -2025-06-03T23:34:43.970Z [INFO] server-5e526f52-5131-212f-23ce-681c65dfff65: added peer, starting replication: peer=server-2f8b6ac1-2030-595f-d7d0-063ae8e9f541 -2025-06-03T23:34:43.970Z [INFO] server-5e526f52-5131-212f-23ce-681c65dfff65: added peer, starting replication: peer=server-d9bd2e70-ee5a-2aa5-ae0e-aa91a5d82058 -2025-06-03T23:34:43.970Z [INFO] server-5e526f52-5131-212f-23ce-681c65dfff65: pipelining replication: peer="{Voter server-d9bd2e70-ee5a-2aa5-ae0e-aa91a5d82058 d9bd2e70-ee5a-2aa5-ae0e-aa91a5d82058}" -2025-06-03T23:34:43.970Z [INFO] server-5e526f52-5131-212f-23ce-681c65dfff65: pipelining replication: peer="{Voter server-2f8b6ac1-2030-595f-d7d0-063ae8e9f541 2f8b6ac1-2030-595f-d7d0-063ae8e9f541}" -2025-06-03T23:34:44.076Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:43.970072431 +0000 UTC m=+98.990891801 (1 nodes), 56.6638ms from start of poll, 57.15928ms from cluster start. Timeout at 2025-06-03 23:34:44.076104431 +0000 UTC m=+99.096923801, 106.032ms after stability -2025-06-03T23:34:44.081Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:44.186Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:44.081397191 +0000 UTC m=+99.102216561 (2 nodes), 3.72µs from start of poll, 168.48404ms from cluster start. Timeout at 2025-06-03 23:34:44.186945711 +0000 UTC m=+99.207765081, 105.54852ms after stability -2025-06-03T23:34:44.187Z [INFO] server-5e526f52-5131-212f-23ce-681c65dfff65: updating configuration: command=RemoveServer server-id=server-2f8b6ac1-2030-595f-d7d0-063ae8e9f541 server-addr= servers="[{Suffrage:Voter ID:server-d9bd2e70-ee5a-2aa5-ae0e-aa91a5d82058 Address:d9bd2e70-ee5a-2aa5-ae0e-aa91a5d82058} {Suffrage:Voter ID:server-5e526f52-5131-212f-23ce-681c65dfff65 Address:5e526f52-5131-212f-23ce-681c65dfff65}]" -2025-06-03T23:34:44.187Z [INFO] server-5e526f52-5131-212f-23ce-681c65dfff65: removed peer, stopping replication: peer=server-2f8b6ac1-2030-595f-d7d0-063ae8e9f541 last-index=3 -2025-06-03T23:34:44.187Z [INFO] server-5e526f52-5131-212f-23ce-681c65dfff65: aborting pipeline replication: peer="{Voter server-2f8b6ac1-2030-595f-d7d0-063ae8e9f541 2f8b6ac1-2030-595f-d7d0-063ae8e9f541}" -2025-06-03T23:34:44.263Z [WARN] server-2f8b6ac1-2030-595f-d7d0-063ae8e9f541: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2025-06-03T23:34:44.292Z [INFO] server-5e526f52-5131-212f-23ce-681c65dfff65: updating configuration: command=AddVoter server-id=server-2f8b6ac1-2030-595f-d7d0-063ae8e9f541 server-addr=2f8b6ac1-2030-595f-d7d0-063ae8e9f541 servers="[{Suffrage:Voter ID:server-d9bd2e70-ee5a-2aa5-ae0e-aa91a5d82058 Address:d9bd2e70-ee5a-2aa5-ae0e-aa91a5d82058} {Suffrage:Voter ID:server-5e526f52-5131-212f-23ce-681c65dfff65 Address:5e526f52-5131-212f-23ce-681c65dfff65} {Suffrage:Voter ID:server-2f8b6ac1-2030-595f-d7d0-063ae8e9f541 Address:2f8b6ac1-2030-595f-d7d0-063ae8e9f541}]" -2025-06-03T23:34:44.293Z [INFO] server-5e526f52-5131-212f-23ce-681c65dfff65: added peer, starting replication: peer=server-2f8b6ac1-2030-595f-d7d0-063ae8e9f541 -2025-06-03T23:34:44.293Z [WARN] server-2f8b6ac1-2030-595f-d7d0-063ae8e9f541: failed to get previous log: previous-index=4 last-index=3 error="log not found" -2025-06-03T23:34:44.293Z [WARN] server-5e526f52-5131-212f-23ce-681c65dfff65: appendEntries rejected, sending older logs: peer="{Voter server-2f8b6ac1-2030-595f-d7d0-063ae8e9f541 2f8b6ac1-2030-595f-d7d0-063ae8e9f541}" next=4 -2025-06-03T23:34:44.293Z [INFO] server-5e526f52-5131-212f-23ce-681c65dfff65: pipelining replication: peer="{Voter server-2f8b6ac1-2030-595f-d7d0-063ae8e9f541 2f8b6ac1-2030-595f-d7d0-063ae8e9f541}" -2025-06-03T23:34:44.293Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:44.399Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:44.293327071 +0000 UTC m=+99.314146441 (1 nodes), 1.16µs from start of poll, 380.41392ms from cluster start. Timeout at 2025-06-03 23:34:44.398987471 +0000 UTC m=+99.419806881, 105.66044ms after stability -2025-06-03T23:34:44.399Z [INFO] server-5e526f52-5131-212f-23ce-681c65dfff65: aborting pipeline replication: peer="{Voter server-d9bd2e70-ee5a-2aa5-ae0e-aa91a5d82058 d9bd2e70-ee5a-2aa5-ae0e-aa91a5d82058}" -2025-06-03T23:34:44.399Z [INFO] server-5e526f52-5131-212f-23ce-681c65dfff65: aborting pipeline replication: peer="{Voter server-2f8b6ac1-2030-595f-d7d0-063ae8e9f541 2f8b6ac1-2030-595f-d7d0-063ae8e9f541}" +2025-06-03T23:37:42.346Z [INFO] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000 Address:fc54ca5c-ffc7-8b8e-5329-8f85d1458000} {Suffrage:Voter ID:server-99ee3dde-22c5-65fa-34c8-2874e5047dfb Address:99ee3dde-22c5-65fa-34c8-2874e5047dfb} {Suffrage:Voter ID:server-9c6faae2-20de-9028-5ffc-e87258bccee6 Address:9c6faae2-20de-9028-5ffc-e87258bccee6}]" +2025-06-03T23:37:42.346Z [INFO] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: entering follower state: follower="Node at fc54ca5c-ffc7-8b8e-5329-8f85d1458000 [Follower]" leader-address= leader-id= +2025-06-03T23:37:42.347Z [INFO] server-99ee3dde-22c5-65fa-34c8-2874e5047dfb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000 Address:fc54ca5c-ffc7-8b8e-5329-8f85d1458000} {Suffrage:Voter ID:server-99ee3dde-22c5-65fa-34c8-2874e5047dfb Address:99ee3dde-22c5-65fa-34c8-2874e5047dfb} {Suffrage:Voter ID:server-9c6faae2-20de-9028-5ffc-e87258bccee6 Address:9c6faae2-20de-9028-5ffc-e87258bccee6}]" +2025-06-03T23:37:42.347Z [INFO] server-99ee3dde-22c5-65fa-34c8-2874e5047dfb: entering follower state: follower="Node at 99ee3dde-22c5-65fa-34c8-2874e5047dfb [Follower]" leader-address= leader-id= +2025-06-03T23:37:42.347Z [INFO] server-9c6faae2-20de-9028-5ffc-e87258bccee6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000 Address:fc54ca5c-ffc7-8b8e-5329-8f85d1458000} {Suffrage:Voter ID:server-99ee3dde-22c5-65fa-34c8-2874e5047dfb Address:99ee3dde-22c5-65fa-34c8-2874e5047dfb} {Suffrage:Voter ID:server-9c6faae2-20de-9028-5ffc-e87258bccee6 Address:9c6faae2-20de-9028-5ffc-e87258bccee6}]" +2025-06-03T23:37:42.347Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:42.347Z [INFO] server-9c6faae2-20de-9028-5ffc-e87258bccee6: entering follower state: follower="Node at 9c6faae2-20de-9028-5ffc-e87258bccee6 [Follower]" leader-address= leader-id= +2025-06-03T23:37:42.407Z [WARN] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:42.407Z [INFO] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: entering candidate state: node="Node at fc54ca5c-ffc7-8b8e-5329-8f85d1458000 [Candidate]" term=2 +2025-06-03T23:37:42.407Z [INFO] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: election won: term=2 tally=2 +2025-06-03T23:37:42.407Z [INFO] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: entering leader state: leader="Node at fc54ca5c-ffc7-8b8e-5329-8f85d1458000 [Leader]" +2025-06-03T23:37:42.407Z [INFO] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: added peer, starting replication: peer=server-99ee3dde-22c5-65fa-34c8-2874e5047dfb +2025-06-03T23:37:42.407Z [INFO] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: added peer, starting replication: peer=server-9c6faae2-20de-9028-5ffc-e87258bccee6 +2025-06-03T23:37:42.407Z [INFO] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: pipelining replication: peer="{Voter server-99ee3dde-22c5-65fa-34c8-2874e5047dfb 99ee3dde-22c5-65fa-34c8-2874e5047dfb}" +2025-06-03T23:37:42.407Z [INFO] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: pipelining replication: peer="{Voter server-9c6faae2-20de-9028-5ffc-e87258bccee6 9c6faae2-20de-9028-5ffc-e87258bccee6}" +2025-06-03T23:37:42.513Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:42.407701911 +0000 UTC m=+97.729456881 (1 nodes), 60.52668ms from start of poll, 60.97316ms from cluster start. Timeout at 2025-06-03 23:37:42.513548911 +0000 UTC m=+97.835303841, 105.84696ms after stability +2025-06-03T23:37:42.518Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:42.625Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:42.518881911 +0000 UTC m=+97.840636841 (2 nodes), 3.32µs from start of poll, 172.15312ms from cluster start. Timeout at 2025-06-03 23:37:42.625018791 +0000 UTC m=+97.946773721, 106.13688ms after stability +2025-06-03T23:37:42.625Z [INFO] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: updating configuration: command=RemoveServer server-id=server-99ee3dde-22c5-65fa-34c8-2874e5047dfb server-addr= servers="[{Suffrage:Voter ID:server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000 Address:fc54ca5c-ffc7-8b8e-5329-8f85d1458000} {Suffrage:Voter ID:server-9c6faae2-20de-9028-5ffc-e87258bccee6 Address:9c6faae2-20de-9028-5ffc-e87258bccee6}]" +2025-06-03T23:37:42.625Z [INFO] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: removed peer, stopping replication: peer=server-99ee3dde-22c5-65fa-34c8-2874e5047dfb last-index=3 +2025-06-03T23:37:42.625Z [INFO] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: aborting pipeline replication: peer="{Voter server-99ee3dde-22c5-65fa-34c8-2874e5047dfb 99ee3dde-22c5-65fa-34c8-2874e5047dfb}" +2025-06-03T23:37:42.709Z [WARN] server-99ee3dde-22c5-65fa-34c8-2874e5047dfb: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-06-03T23:37:42.731Z [INFO] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: updating configuration: command=AddVoter server-id=server-99ee3dde-22c5-65fa-34c8-2874e5047dfb server-addr=99ee3dde-22c5-65fa-34c8-2874e5047dfb servers="[{Suffrage:Voter ID:server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000 Address:fc54ca5c-ffc7-8b8e-5329-8f85d1458000} {Suffrage:Voter ID:server-9c6faae2-20de-9028-5ffc-e87258bccee6 Address:9c6faae2-20de-9028-5ffc-e87258bccee6} {Suffrage:Voter ID:server-99ee3dde-22c5-65fa-34c8-2874e5047dfb Address:99ee3dde-22c5-65fa-34c8-2874e5047dfb}]" +2025-06-03T23:37:42.731Z [INFO] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: added peer, starting replication: peer=server-99ee3dde-22c5-65fa-34c8-2874e5047dfb +2025-06-03T23:37:42.731Z [WARN] server-99ee3dde-22c5-65fa-34c8-2874e5047dfb: failed to get previous log: previous-index=4 last-index=3 error="log not found" +2025-06-03T23:37:42.731Z [WARN] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: appendEntries rejected, sending older logs: peer="{Voter server-99ee3dde-22c5-65fa-34c8-2874e5047dfb 99ee3dde-22c5-65fa-34c8-2874e5047dfb}" next=4 +2025-06-03T23:37:42.731Z [INFO] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: pipelining replication: peer="{Voter server-99ee3dde-22c5-65fa-34c8-2874e5047dfb 99ee3dde-22c5-65fa-34c8-2874e5047dfb}" +2025-06-03T23:37:42.731Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:42.837Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:42.731647271 +0000 UTC m=+98.053402201 (1 nodes), 960ns from start of poll, 384.91848ms from cluster start. Timeout at 2025-06-03 23:37:42.837728591 +0000 UTC m=+98.159483521, 106.08132ms after stability +2025-06-03T23:37:42.837Z [INFO] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: aborting pipeline replication: peer="{Voter server-9c6faae2-20de-9028-5ffc-e87258bccee6 9c6faae2-20de-9028-5ffc-e87258bccee6}" +2025-06-03T23:37:42.837Z [INFO] server-fc54ca5c-ffc7-8b8e-5329-8f85d1458000: aborting pipeline replication: peer="{Voter server-99ee3dde-22c5-65fa-34c8-2874e5047dfb 99ee3dde-22c5-65fa-34c8-2874e5047dfb}" --- PASS: TestRaft_ReJoinFollower (0.49s) === RUN TestRaft_LeaderLeaseExpire -2025-06-03T23:34:44.399Z [INFO] server-51501511-805a-45ae-d40c-aeda1c455a60: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-51501511-805a-45ae-d40c-aeda1c455a60 Address:51501511-805a-45ae-d40c-aeda1c455a60} {Suffrage:Voter ID:server-722ef9aa-143d-cd84-c575-8bb37beb4a71 Address:722ef9aa-143d-cd84-c575-8bb37beb4a71}]" -2025-06-03T23:34:44.400Z [INFO] server-51501511-805a-45ae-d40c-aeda1c455a60: entering follower state: follower="Node at 51501511-805a-45ae-d40c-aeda1c455a60 [Follower]" leader-address= leader-id= -2025-06-03T23:34:44.400Z [INFO] server-722ef9aa-143d-cd84-c575-8bb37beb4a71: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-51501511-805a-45ae-d40c-aeda1c455a60 Address:51501511-805a-45ae-d40c-aeda1c455a60} {Suffrage:Voter ID:server-722ef9aa-143d-cd84-c575-8bb37beb4a71 Address:722ef9aa-143d-cd84-c575-8bb37beb4a71}]" -2025-06-03T23:34:44.400Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:44.400Z [INFO] server-722ef9aa-143d-cd84-c575-8bb37beb4a71: entering follower state: follower="Node at 722ef9aa-143d-cd84-c575-8bb37beb4a71 [Follower]" leader-address= leader-id= -2025-06-03T23:34:44.491Z [WARN] server-51501511-805a-45ae-d40c-aeda1c455a60: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:44.491Z [INFO] server-51501511-805a-45ae-d40c-aeda1c455a60: entering candidate state: node="Node at 51501511-805a-45ae-d40c-aeda1c455a60 [Candidate]" term=2 -2025-06-03T23:34:44.491Z [INFO] server-51501511-805a-45ae-d40c-aeda1c455a60: election won: term=2 tally=2 -2025-06-03T23:34:44.491Z [INFO] server-51501511-805a-45ae-d40c-aeda1c455a60: entering leader state: leader="Node at 51501511-805a-45ae-d40c-aeda1c455a60 [Leader]" -2025-06-03T23:34:44.491Z [INFO] server-51501511-805a-45ae-d40c-aeda1c455a60: added peer, starting replication: peer=server-722ef9aa-143d-cd84-c575-8bb37beb4a71 -2025-06-03T23:34:44.491Z [INFO] server-51501511-805a-45ae-d40c-aeda1c455a60: pipelining replication: peer="{Voter server-722ef9aa-143d-cd84-c575-8bb37beb4a71 722ef9aa-143d-cd84-c575-8bb37beb4a71}" -2025-06-03T23:34:44.596Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:44.491598671 +0000 UTC m=+99.512418081 (1 nodes), 91.41092ms from start of poll, 91.78212ms from cluster start. Timeout at 2025-06-03 23:34:44.596769351 +0000 UTC m=+99.617588721, 105.17064ms after stability -2025-06-03T23:34:44.602Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:44.707Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:44.602238071 +0000 UTC m=+99.623057441 (1 nodes), 2.92µs from start of poll, 202.42148ms from cluster start. Timeout at 2025-06-03 23:34:44.707864391 +0000 UTC m=+99.728683761, 105.62632ms after stability - raft_test.go:1799: [INFO] Disconnecting Node at 722ef9aa-143d-cd84-c575-8bb37beb4a71 [Follower] -2025-06-03T23:34:44.709Z [ERROR] server-51501511-805a-45ae-d40c-aeda1c455a60: failed to heartbeat to: peer=722ef9aa-143d-cd84-c575-8bb37beb4a71 backoff time=10ms error="failed to connect to peer: 722ef9aa-143d-cd84-c575-8bb37beb4a71" -2025-06-03T23:34:44.711Z [ERROR] server-51501511-805a-45ae-d40c-aeda1c455a60: failed to pipeline appendEntries: peer="{Voter server-722ef9aa-143d-cd84-c575-8bb37beb4a71 722ef9aa-143d-cd84-c575-8bb37beb4a71}" error="append pipeline closed" -2025-06-03T23:34:44.711Z [INFO] server-51501511-805a-45ae-d40c-aeda1c455a60: aborting pipeline replication: peer="{Voter server-722ef9aa-143d-cd84-c575-8bb37beb4a71 722ef9aa-143d-cd84-c575-8bb37beb4a71}" -2025-06-03T23:34:44.720Z [ERROR] server-51501511-805a-45ae-d40c-aeda1c455a60: failed to appendEntries to: peer="{Voter server-722ef9aa-143d-cd84-c575-8bb37beb4a71 722ef9aa-143d-cd84-c575-8bb37beb4a71}" error="failed to connect to peer: 722ef9aa-143d-cd84-c575-8bb37beb4a71" -2025-06-03T23:34:44.727Z [ERROR] server-51501511-805a-45ae-d40c-aeda1c455a60: failed to heartbeat to: peer=722ef9aa-143d-cd84-c575-8bb37beb4a71 backoff time=10ms error="failed to connect to peer: 722ef9aa-143d-cd84-c575-8bb37beb4a71" -2025-06-03T23:34:44.740Z [ERROR] server-51501511-805a-45ae-d40c-aeda1c455a60: failed to appendEntries to: peer="{Voter server-722ef9aa-143d-cd84-c575-8bb37beb4a71 722ef9aa-143d-cd84-c575-8bb37beb4a71}" error="failed to connect to peer: 722ef9aa-143d-cd84-c575-8bb37beb4a71" -2025-06-03T23:34:44.748Z [ERROR] server-51501511-805a-45ae-d40c-aeda1c455a60: failed to heartbeat to: peer=722ef9aa-143d-cd84-c575-8bb37beb4a71 backoff time=10ms error="failed to connect to peer: 722ef9aa-143d-cd84-c575-8bb37beb4a71" -2025-06-03T23:34:44.753Z [WARN] server-51501511-805a-45ae-d40c-aeda1c455a60: failed to contact: server-id=server-722ef9aa-143d-cd84-c575-8bb37beb4a71 time=50.93048ms -2025-06-03T23:34:44.753Z [WARN] server-51501511-805a-45ae-d40c-aeda1c455a60: failed to contact quorum of nodes, stepping down -2025-06-03T23:34:44.753Z [INFO] server-51501511-805a-45ae-d40c-aeda1c455a60: entering follower state: follower="Node at 51501511-805a-45ae-d40c-aeda1c455a60 [Follower]" leader-address= leader-id= -2025-06-03T23:34:44.753Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:44.756Z [ERROR] server-51501511-805a-45ae-d40c-aeda1c455a60: failed to appendEntries to: peer="{Voter server-722ef9aa-143d-cd84-c575-8bb37beb4a71 722ef9aa-143d-cd84-c575-8bb37beb4a71}" error="failed to connect to peer: 722ef9aa-143d-cd84-c575-8bb37beb4a71" -2025-06-03T23:34:44.843Z [WARN] server-51501511-805a-45ae-d40c-aeda1c455a60: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:44.843Z [INFO] server-51501511-805a-45ae-d40c-aeda1c455a60: entering candidate state: node="Node at 51501511-805a-45ae-d40c-aeda1c455a60 [Candidate]" term=3 -2025-06-03T23:34:44.843Z [ERROR] server-51501511-805a-45ae-d40c-aeda1c455a60: failed to make requestVote RPC: target="{Voter server-722ef9aa-143d-cd84-c575-8bb37beb4a71 722ef9aa-143d-cd84-c575-8bb37beb4a71}" error="failed to connect to peer: 722ef9aa-143d-cd84-c575-8bb37beb4a71" term=3 -2025-06-03T23:34:44.849Z [WARN] server-722ef9aa-143d-cd84-c575-8bb37beb4a71: heartbeat timeout reached, starting election: last-leader-addr=51501511-805a-45ae-d40c-aeda1c455a60 last-leader-id=server-51501511-805a-45ae-d40c-aeda1c455a60 -2025-06-03T23:34:44.849Z [INFO] server-722ef9aa-143d-cd84-c575-8bb37beb4a71: entering candidate state: node="Node at 722ef9aa-143d-cd84-c575-8bb37beb4a71 [Candidate]" term=3 -2025-06-03T23:34:44.849Z [ERROR] server-722ef9aa-143d-cd84-c575-8bb37beb4a71: failed to make requestVote RPC: target="{Voter server-51501511-805a-45ae-d40c-aeda1c455a60 51501511-805a-45ae-d40c-aeda1c455a60}" error="failed to connect to peer: 51501511-805a-45ae-d40c-aeda1c455a60" term=3 -2025-06-03T23:34:44.899Z [WARN] server-722ef9aa-143d-cd84-c575-8bb37beb4a71: Election timeout reached, restarting election -2025-06-03T23:34:44.899Z [INFO] server-722ef9aa-143d-cd84-c575-8bb37beb4a71: entering candidate state: node="Node at 722ef9aa-143d-cd84-c575-8bb37beb4a71 [Candidate]" term=4 -2025-06-03T23:34:44.900Z [ERROR] server-722ef9aa-143d-cd84-c575-8bb37beb4a71: failed to make requestVote RPC: target="{Voter server-51501511-805a-45ae-d40c-aeda1c455a60 51501511-805a-45ae-d40c-aeda1c455a60}" error="failed to connect to peer: 51501511-805a-45ae-d40c-aeda1c455a60" term=4 -2025-06-03T23:34:44.931Z [WARN] server-51501511-805a-45ae-d40c-aeda1c455a60: Election timeout reached, restarting election -2025-06-03T23:34:44.931Z [INFO] server-51501511-805a-45ae-d40c-aeda1c455a60: entering candidate state: node="Node at 51501511-805a-45ae-d40c-aeda1c455a60 [Candidate]" term=4 -2025-06-03T23:34:44.931Z [ERROR] server-51501511-805a-45ae-d40c-aeda1c455a60: failed to make requestVote RPC: target="{Voter server-722ef9aa-143d-cd84-c575-8bb37beb4a71 722ef9aa-143d-cd84-c575-8bb37beb4a71}" error="failed to connect to peer: 722ef9aa-143d-cd84-c575-8bb37beb4a71" term=4 -2025-06-03T23:34:44.955Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:44.849816551 +0000 UTC m=+99.870635921 (0 nodes), 96.22328ms from start of poll, 449.99996ms from cluster start. Timeout at 2025-06-03 23:34:44.955003671 +0000 UTC m=+99.975823041, 105.18712ms after stability -2025-06-03T23:34:44.983Z [WARN] server-722ef9aa-143d-cd84-c575-8bb37beb4a71: Election timeout reached, restarting election -2025-06-03T23:34:44.983Z [INFO] server-722ef9aa-143d-cd84-c575-8bb37beb4a71: entering candidate state: node="Node at 722ef9aa-143d-cd84-c575-8bb37beb4a71 [Candidate]" term=5 -2025-06-03T23:34:44.983Z [ERROR] server-722ef9aa-143d-cd84-c575-8bb37beb4a71: failed to make requestVote RPC: target="{Voter server-51501511-805a-45ae-d40c-aeda1c455a60 51501511-805a-45ae-d40c-aeda1c455a60}" error="failed to connect to peer: 51501511-805a-45ae-d40c-aeda1c455a60" term=5 -2025-06-03T23:34:45.000Z [WARN] server-51501511-805a-45ae-d40c-aeda1c455a60: Election timeout reached, restarting election -2025-06-03T23:34:45.000Z [INFO] server-51501511-805a-45ae-d40c-aeda1c455a60: entering candidate state: node="Node at 51501511-805a-45ae-d40c-aeda1c455a60 [Candidate]" term=5 -2025-06-03T23:34:45.000Z [ERROR] server-51501511-805a-45ae-d40c-aeda1c455a60: failed to make requestVote RPC: target="{Voter server-722ef9aa-143d-cd84-c575-8bb37beb4a71 722ef9aa-143d-cd84-c575-8bb37beb4a71}" error="failed to connect to peer: 722ef9aa-143d-cd84-c575-8bb37beb4a71" term=5 ---- PASS: TestRaft_LeaderLeaseExpire (0.66s) +2025-06-03T23:37:42.838Z [INFO] server-45efa180-39eb-6dac-2369-23a1dee89c9f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45efa180-39eb-6dac-2369-23a1dee89c9f Address:45efa180-39eb-6dac-2369-23a1dee89c9f} {Suffrage:Voter ID:server-67cf94b4-06c6-bae8-ca57-02fcea551a10 Address:67cf94b4-06c6-bae8-ca57-02fcea551a10}]" +2025-06-03T23:37:42.838Z [INFO] server-45efa180-39eb-6dac-2369-23a1dee89c9f: entering follower state: follower="Node at 45efa180-39eb-6dac-2369-23a1dee89c9f [Follower]" leader-address= leader-id= +2025-06-03T23:37:42.838Z [INFO] server-67cf94b4-06c6-bae8-ca57-02fcea551a10: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45efa180-39eb-6dac-2369-23a1dee89c9f Address:45efa180-39eb-6dac-2369-23a1dee89c9f} {Suffrage:Voter ID:server-67cf94b4-06c6-bae8-ca57-02fcea551a10 Address:67cf94b4-06c6-bae8-ca57-02fcea551a10}]" +2025-06-03T23:37:42.838Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:42.838Z [INFO] server-67cf94b4-06c6-bae8-ca57-02fcea551a10: entering follower state: follower="Node at 67cf94b4-06c6-bae8-ca57-02fcea551a10 [Follower]" leader-address= leader-id= +2025-06-03T23:37:42.894Z [WARN] server-45efa180-39eb-6dac-2369-23a1dee89c9f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:42.894Z [INFO] server-45efa180-39eb-6dac-2369-23a1dee89c9f: entering candidate state: node="Node at 45efa180-39eb-6dac-2369-23a1dee89c9f [Candidate]" term=2 +2025-06-03T23:37:42.894Z [INFO] server-45efa180-39eb-6dac-2369-23a1dee89c9f: election won: term=2 tally=2 +2025-06-03T23:37:42.894Z [INFO] server-45efa180-39eb-6dac-2369-23a1dee89c9f: entering leader state: leader="Node at 45efa180-39eb-6dac-2369-23a1dee89c9f [Leader]" +2025-06-03T23:37:42.894Z [INFO] server-45efa180-39eb-6dac-2369-23a1dee89c9f: added peer, starting replication: peer=server-67cf94b4-06c6-bae8-ca57-02fcea551a10 +2025-06-03T23:37:42.895Z [INFO] server-45efa180-39eb-6dac-2369-23a1dee89c9f: pipelining replication: peer="{Voter server-67cf94b4-06c6-bae8-ca57-02fcea551a10 67cf94b4-06c6-bae8-ca57-02fcea551a10}" +2025-06-03T23:37:42.999Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:42.894857911 +0000 UTC m=+98.216612841 (1 nodes), 56.06408ms from start of poll, 56.38068ms from cluster start. Timeout at 2025-06-03 23:37:42.999918551 +0000 UTC m=+98.321673481, 105.06064ms after stability +2025-06-03T23:37:43.005Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:43.110Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:43.005086791 +0000 UTC m=+98.326841721 (1 nodes), 3.52µs from start of poll, 166.60956ms from cluster start. Timeout at 2025-06-03 23:37:43.110761951 +0000 UTC m=+98.432516881, 105.67516ms after stability + raft_test.go:1799: [INFO] Disconnecting Node at 67cf94b4-06c6-bae8-ca57-02fcea551a10 [Follower] +2025-06-03T23:37:43.112Z [ERROR] server-45efa180-39eb-6dac-2369-23a1dee89c9f: failed to pipeline appendEntries: peer="{Voter server-67cf94b4-06c6-bae8-ca57-02fcea551a10 67cf94b4-06c6-bae8-ca57-02fcea551a10}" error="append pipeline closed" +2025-06-03T23:37:43.112Z [INFO] server-45efa180-39eb-6dac-2369-23a1dee89c9f: aborting pipeline replication: peer="{Voter server-67cf94b4-06c6-bae8-ca57-02fcea551a10 67cf94b4-06c6-bae8-ca57-02fcea551a10}" +2025-06-03T23:37:43.115Z [ERROR] server-45efa180-39eb-6dac-2369-23a1dee89c9f: failed to heartbeat to: peer=67cf94b4-06c6-bae8-ca57-02fcea551a10 backoff time=10ms error="failed to connect to peer: 67cf94b4-06c6-bae8-ca57-02fcea551a10" +2025-06-03T23:37:43.119Z [ERROR] server-45efa180-39eb-6dac-2369-23a1dee89c9f: failed to appendEntries to: peer="{Voter server-67cf94b4-06c6-bae8-ca57-02fcea551a10 67cf94b4-06c6-bae8-ca57-02fcea551a10}" error="failed to connect to peer: 67cf94b4-06c6-bae8-ca57-02fcea551a10" +2025-06-03T23:37:43.133Z [ERROR] server-45efa180-39eb-6dac-2369-23a1dee89c9f: failed to heartbeat to: peer=67cf94b4-06c6-bae8-ca57-02fcea551a10 backoff time=10ms error="failed to connect to peer: 67cf94b4-06c6-bae8-ca57-02fcea551a10" +2025-06-03T23:37:43.134Z [ERROR] server-45efa180-39eb-6dac-2369-23a1dee89c9f: failed to appendEntries to: peer="{Voter server-67cf94b4-06c6-bae8-ca57-02fcea551a10 67cf94b4-06c6-bae8-ca57-02fcea551a10}" error="failed to connect to peer: 67cf94b4-06c6-bae8-ca57-02fcea551a10" +2025-06-03T23:37:43.153Z [ERROR] server-45efa180-39eb-6dac-2369-23a1dee89c9f: failed to appendEntries to: peer="{Voter server-67cf94b4-06c6-bae8-ca57-02fcea551a10 67cf94b4-06c6-bae8-ca57-02fcea551a10}" error="failed to connect to peer: 67cf94b4-06c6-bae8-ca57-02fcea551a10" +2025-06-03T23:37:43.153Z [ERROR] server-45efa180-39eb-6dac-2369-23a1dee89c9f: failed to heartbeat to: peer=67cf94b4-06c6-bae8-ca57-02fcea551a10 backoff time=10ms error="failed to connect to peer: 67cf94b4-06c6-bae8-ca57-02fcea551a10" +2025-06-03T23:37:43.157Z [WARN] server-45efa180-39eb-6dac-2369-23a1dee89c9f: failed to contact: server-id=server-67cf94b4-06c6-bae8-ca57-02fcea551a10 time=50.81852ms +2025-06-03T23:37:43.157Z [WARN] server-45efa180-39eb-6dac-2369-23a1dee89c9f: failed to contact quorum of nodes, stepping down +2025-06-03T23:37:43.157Z [INFO] server-45efa180-39eb-6dac-2369-23a1dee89c9f: entering follower state: follower="Node at 45efa180-39eb-6dac-2369-23a1dee89c9f [Follower]" leader-address= leader-id= +2025-06-03T23:37:43.157Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:43.214Z [WARN] server-67cf94b4-06c6-bae8-ca57-02fcea551a10: heartbeat timeout reached, starting election: last-leader-addr=45efa180-39eb-6dac-2369-23a1dee89c9f last-leader-id=server-45efa180-39eb-6dac-2369-23a1dee89c9f +2025-06-03T23:37:43.214Z [INFO] server-67cf94b4-06c6-bae8-ca57-02fcea551a10: entering candidate state: node="Node at 67cf94b4-06c6-bae8-ca57-02fcea551a10 [Candidate]" term=3 +2025-06-03T23:37:43.214Z [ERROR] server-67cf94b4-06c6-bae8-ca57-02fcea551a10: failed to make requestVote RPC: target="{Voter server-45efa180-39eb-6dac-2369-23a1dee89c9f 45efa180-39eb-6dac-2369-23a1dee89c9f}" error="failed to connect to peer: 45efa180-39eb-6dac-2369-23a1dee89c9f" term=3 +2025-06-03T23:37:43.229Z [WARN] server-45efa180-39eb-6dac-2369-23a1dee89c9f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:43.229Z [INFO] server-45efa180-39eb-6dac-2369-23a1dee89c9f: entering candidate state: node="Node at 45efa180-39eb-6dac-2369-23a1dee89c9f [Candidate]" term=3 +2025-06-03T23:37:43.229Z [ERROR] server-45efa180-39eb-6dac-2369-23a1dee89c9f: failed to make requestVote RPC: target="{Voter server-67cf94b4-06c6-bae8-ca57-02fcea551a10 67cf94b4-06c6-bae8-ca57-02fcea551a10}" error="failed to connect to peer: 67cf94b4-06c6-bae8-ca57-02fcea551a10" term=3 +2025-06-03T23:37:43.291Z [WARN] server-67cf94b4-06c6-bae8-ca57-02fcea551a10: Election timeout reached, restarting election +2025-06-03T23:37:43.291Z [INFO] server-67cf94b4-06c6-bae8-ca57-02fcea551a10: entering candidate state: node="Node at 67cf94b4-06c6-bae8-ca57-02fcea551a10 [Candidate]" term=4 +2025-06-03T23:37:43.291Z [ERROR] server-67cf94b4-06c6-bae8-ca57-02fcea551a10: failed to make requestVote RPC: target="{Voter server-45efa180-39eb-6dac-2369-23a1dee89c9f 45efa180-39eb-6dac-2369-23a1dee89c9f}" error="failed to connect to peer: 45efa180-39eb-6dac-2369-23a1dee89c9f" term=4 +2025-06-03T23:37:43.315Z [WARN] server-45efa180-39eb-6dac-2369-23a1dee89c9f: Election timeout reached, restarting election +2025-06-03T23:37:43.315Z [INFO] server-45efa180-39eb-6dac-2369-23a1dee89c9f: entering candidate state: node="Node at 45efa180-39eb-6dac-2369-23a1dee89c9f [Candidate]" term=4 +2025-06-03T23:37:43.315Z [ERROR] server-45efa180-39eb-6dac-2369-23a1dee89c9f: failed to make requestVote RPC: target="{Voter server-67cf94b4-06c6-bae8-ca57-02fcea551a10 67cf94b4-06c6-bae8-ca57-02fcea551a10}" error="failed to connect to peer: 67cf94b4-06c6-bae8-ca57-02fcea551a10" term=4 +2025-06-03T23:37:43.334Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:43.229632871 +0000 UTC m=+98.551387801 (0 nodes), 71.62736ms from start of poll, 391.15564ms from cluster start. Timeout at 2025-06-03 23:37:43.334635951 +0000 UTC m=+98.656390881, 105.00308ms after stability +2025-06-03T23:37:43.345Z [WARN] server-67cf94b4-06c6-bae8-ca57-02fcea551a10: Election timeout reached, restarting election +2025-06-03T23:37:43.345Z [INFO] server-67cf94b4-06c6-bae8-ca57-02fcea551a10: entering candidate state: node="Node at 67cf94b4-06c6-bae8-ca57-02fcea551a10 [Candidate]" term=5 +2025-06-03T23:37:43.345Z [ERROR] server-67cf94b4-06c6-bae8-ca57-02fcea551a10: failed to make requestVote RPC: target="{Voter server-45efa180-39eb-6dac-2369-23a1dee89c9f 45efa180-39eb-6dac-2369-23a1dee89c9f}" error="failed to connect to peer: 45efa180-39eb-6dac-2369-23a1dee89c9f" term=5 +2025-06-03T23:37:43.396Z [WARN] server-45efa180-39eb-6dac-2369-23a1dee89c9f: Election timeout reached, restarting election +2025-06-03T23:37:43.396Z [INFO] server-45efa180-39eb-6dac-2369-23a1dee89c9f: entering candidate state: node="Node at 45efa180-39eb-6dac-2369-23a1dee89c9f [Candidate]" term=5 +2025-06-03T23:37:43.396Z [ERROR] server-45efa180-39eb-6dac-2369-23a1dee89c9f: failed to make requestVote RPC: target="{Voter server-67cf94b4-06c6-bae8-ca57-02fcea551a10 67cf94b4-06c6-bae8-ca57-02fcea551a10}" error="failed to connect to peer: 67cf94b4-06c6-bae8-ca57-02fcea551a10" term=5 +2025-06-03T23:37:43.398Z [WARN] server-67cf94b4-06c6-bae8-ca57-02fcea551a10: Election timeout reached, restarting election +2025-06-03T23:37:43.398Z [INFO] server-67cf94b4-06c6-bae8-ca57-02fcea551a10: entering candidate state: node="Node at 67cf94b4-06c6-bae8-ca57-02fcea551a10 [Candidate]" term=6 +2025-06-03T23:37:43.398Z [ERROR] server-67cf94b4-06c6-bae8-ca57-02fcea551a10: failed to make requestVote RPC: target="{Voter server-45efa180-39eb-6dac-2369-23a1dee89c9f 45efa180-39eb-6dac-2369-23a1dee89c9f}" error="failed to connect to peer: 45efa180-39eb-6dac-2369-23a1dee89c9f" term=6 +--- PASS: TestRaft_LeaderLeaseExpire (0.60s) === RUN TestRaft_Barrier -2025-06-03T23:34:45.061Z [INFO] server-1ad6ed4b-d0ad-bd0d-70ae-85fcabcf0d99: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ad6ed4b-d0ad-bd0d-70ae-85fcabcf0d99 Address:1ad6ed4b-d0ad-bd0d-70ae-85fcabcf0d99} {Suffrage:Voter ID:server-8d59790e-6438-d125-4415-6aa93c246c74 Address:8d59790e-6438-d125-4415-6aa93c246c74} {Suffrage:Voter ID:server-581a480b-52fb-2d29-e4e2-290978cafc1d Address:581a480b-52fb-2d29-e4e2-290978cafc1d}]" -2025-06-03T23:34:45.061Z [INFO] server-1ad6ed4b-d0ad-bd0d-70ae-85fcabcf0d99: entering follower state: follower="Node at 1ad6ed4b-d0ad-bd0d-70ae-85fcabcf0d99 [Follower]" leader-address= leader-id= -2025-06-03T23:34:45.062Z [INFO] server-8d59790e-6438-d125-4415-6aa93c246c74: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ad6ed4b-d0ad-bd0d-70ae-85fcabcf0d99 Address:1ad6ed4b-d0ad-bd0d-70ae-85fcabcf0d99} {Suffrage:Voter ID:server-8d59790e-6438-d125-4415-6aa93c246c74 Address:8d59790e-6438-d125-4415-6aa93c246c74} {Suffrage:Voter ID:server-581a480b-52fb-2d29-e4e2-290978cafc1d Address:581a480b-52fb-2d29-e4e2-290978cafc1d}]" -2025-06-03T23:34:45.062Z [INFO] server-8d59790e-6438-d125-4415-6aa93c246c74: entering follower state: follower="Node at 8d59790e-6438-d125-4415-6aa93c246c74 [Follower]" leader-address= leader-id= -2025-06-03T23:34:45.062Z [INFO] server-581a480b-52fb-2d29-e4e2-290978cafc1d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ad6ed4b-d0ad-bd0d-70ae-85fcabcf0d99 Address:1ad6ed4b-d0ad-bd0d-70ae-85fcabcf0d99} {Suffrage:Voter ID:server-8d59790e-6438-d125-4415-6aa93c246c74 Address:8d59790e-6438-d125-4415-6aa93c246c74} {Suffrage:Voter ID:server-581a480b-52fb-2d29-e4e2-290978cafc1d Address:581a480b-52fb-2d29-e4e2-290978cafc1d}]" -2025-06-03T23:34:45.062Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:45.062Z [INFO] server-581a480b-52fb-2d29-e4e2-290978cafc1d: entering follower state: follower="Node at 581a480b-52fb-2d29-e4e2-290978cafc1d [Follower]" leader-address= leader-id= -2025-06-03T23:34:45.148Z [WARN] server-581a480b-52fb-2d29-e4e2-290978cafc1d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:45.148Z [INFO] server-581a480b-52fb-2d29-e4e2-290978cafc1d: entering candidate state: node="Node at 581a480b-52fb-2d29-e4e2-290978cafc1d [Candidate]" term=2 -2025-06-03T23:34:45.148Z [INFO] server-581a480b-52fb-2d29-e4e2-290978cafc1d: election won: term=2 tally=2 -2025-06-03T23:34:45.148Z [INFO] server-581a480b-52fb-2d29-e4e2-290978cafc1d: entering leader state: leader="Node at 581a480b-52fb-2d29-e4e2-290978cafc1d [Leader]" -2025-06-03T23:34:45.149Z [INFO] server-581a480b-52fb-2d29-e4e2-290978cafc1d: added peer, starting replication: peer=server-1ad6ed4b-d0ad-bd0d-70ae-85fcabcf0d99 -2025-06-03T23:34:45.149Z [INFO] server-581a480b-52fb-2d29-e4e2-290978cafc1d: added peer, starting replication: peer=server-8d59790e-6438-d125-4415-6aa93c246c74 -2025-06-03T23:34:45.149Z [INFO] server-581a480b-52fb-2d29-e4e2-290978cafc1d: pipelining replication: peer="{Voter server-8d59790e-6438-d125-4415-6aa93c246c74 8d59790e-6438-d125-4415-6aa93c246c74}" -2025-06-03T23:34:45.149Z [INFO] server-581a480b-52fb-2d29-e4e2-290978cafc1d: pipelining replication: peer="{Voter server-1ad6ed4b-d0ad-bd0d-70ae-85fcabcf0d99 1ad6ed4b-d0ad-bd0d-70ae-85fcabcf0d99}" -2025-06-03T23:34:45.255Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:45.149009431 +0000 UTC m=+100.169828841 (1 nodes), 86.83688ms from start of poll, 87.32744ms from cluster start. Timeout at 2025-06-03 23:34:45.255051831 +0000 UTC m=+100.275871241, 106.0424ms after stability -2025-06-03T23:34:45.270Z [INFO] server-581a480b-52fb-2d29-e4e2-290978cafc1d: aborting pipeline replication: peer="{Voter server-1ad6ed4b-d0ad-bd0d-70ae-85fcabcf0d99 1ad6ed4b-d0ad-bd0d-70ae-85fcabcf0d99}" -2025-06-03T23:34:45.271Z [INFO] server-581a480b-52fb-2d29-e4e2-290978cafc1d: aborting pipeline replication: peer="{Voter server-8d59790e-6438-d125-4415-6aa93c246c74 8d59790e-6438-d125-4415-6aa93c246c74}" ---- PASS: TestRaft_Barrier (0.21s) +2025-06-03T23:37:43.440Z [INFO] server-f76fa422-fbf5-5995-106d-b9019511a181: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f76fa422-fbf5-5995-106d-b9019511a181 Address:f76fa422-fbf5-5995-106d-b9019511a181} {Suffrage:Voter ID:server-ae83e55b-0c71-9d70-ea65-c4efd74c9ea4 Address:ae83e55b-0c71-9d70-ea65-c4efd74c9ea4} {Suffrage:Voter ID:server-4f0734e6-1fb0-a4df-7d92-7d90314ea40c Address:4f0734e6-1fb0-a4df-7d92-7d90314ea40c}]" +2025-06-03T23:37:43.440Z [INFO] server-f76fa422-fbf5-5995-106d-b9019511a181: entering follower state: follower="Node at f76fa422-fbf5-5995-106d-b9019511a181 [Follower]" leader-address= leader-id= +2025-06-03T23:37:43.440Z [INFO] server-ae83e55b-0c71-9d70-ea65-c4efd74c9ea4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f76fa422-fbf5-5995-106d-b9019511a181 Address:f76fa422-fbf5-5995-106d-b9019511a181} {Suffrage:Voter ID:server-ae83e55b-0c71-9d70-ea65-c4efd74c9ea4 Address:ae83e55b-0c71-9d70-ea65-c4efd74c9ea4} {Suffrage:Voter ID:server-4f0734e6-1fb0-a4df-7d92-7d90314ea40c Address:4f0734e6-1fb0-a4df-7d92-7d90314ea40c}]" +2025-06-03T23:37:43.440Z [INFO] server-ae83e55b-0c71-9d70-ea65-c4efd74c9ea4: entering follower state: follower="Node at ae83e55b-0c71-9d70-ea65-c4efd74c9ea4 [Follower]" leader-address= leader-id= +2025-06-03T23:37:43.441Z [INFO] server-4f0734e6-1fb0-a4df-7d92-7d90314ea40c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f76fa422-fbf5-5995-106d-b9019511a181 Address:f76fa422-fbf5-5995-106d-b9019511a181} {Suffrage:Voter ID:server-ae83e55b-0c71-9d70-ea65-c4efd74c9ea4 Address:ae83e55b-0c71-9d70-ea65-c4efd74c9ea4} {Suffrage:Voter ID:server-4f0734e6-1fb0-a4df-7d92-7d90314ea40c Address:4f0734e6-1fb0-a4df-7d92-7d90314ea40c}]" +2025-06-03T23:37:43.441Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:43.441Z [INFO] server-4f0734e6-1fb0-a4df-7d92-7d90314ea40c: entering follower state: follower="Node at 4f0734e6-1fb0-a4df-7d92-7d90314ea40c [Follower]" leader-address= leader-id= +2025-06-03T23:37:43.494Z [WARN] server-4f0734e6-1fb0-a4df-7d92-7d90314ea40c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:43.494Z [INFO] server-4f0734e6-1fb0-a4df-7d92-7d90314ea40c: entering candidate state: node="Node at 4f0734e6-1fb0-a4df-7d92-7d90314ea40c [Candidate]" term=2 +2025-06-03T23:37:43.494Z [INFO] server-4f0734e6-1fb0-a4df-7d92-7d90314ea40c: election won: term=2 tally=2 +2025-06-03T23:37:43.494Z [INFO] server-4f0734e6-1fb0-a4df-7d92-7d90314ea40c: entering leader state: leader="Node at 4f0734e6-1fb0-a4df-7d92-7d90314ea40c [Leader]" +2025-06-03T23:37:43.494Z [INFO] server-4f0734e6-1fb0-a4df-7d92-7d90314ea40c: added peer, starting replication: peer=server-f76fa422-fbf5-5995-106d-b9019511a181 +2025-06-03T23:37:43.494Z [INFO] server-4f0734e6-1fb0-a4df-7d92-7d90314ea40c: added peer, starting replication: peer=server-ae83e55b-0c71-9d70-ea65-c4efd74c9ea4 +2025-06-03T23:37:43.494Z [INFO] server-4f0734e6-1fb0-a4df-7d92-7d90314ea40c: pipelining replication: peer="{Voter server-f76fa422-fbf5-5995-106d-b9019511a181 f76fa422-fbf5-5995-106d-b9019511a181}" +2025-06-03T23:37:43.494Z [INFO] server-4f0734e6-1fb0-a4df-7d92-7d90314ea40c: pipelining replication: peer="{Voter server-ae83e55b-0c71-9d70-ea65-c4efd74c9ea4 ae83e55b-0c71-9d70-ea65-c4efd74c9ea4}" +2025-06-03T23:37:43.600Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:43.494309871 +0000 UTC m=+98.816064801 (1 nodes), 53.19784ms from start of poll, 53.80756ms from cluster start. Timeout at 2025-06-03 23:37:43.600031671 +0000 UTC m=+98.921786601, 105.7218ms after stability +2025-06-03T23:37:43.607Z [INFO] server-4f0734e6-1fb0-a4df-7d92-7d90314ea40c: aborting pipeline replication: peer="{Voter server-ae83e55b-0c71-9d70-ea65-c4efd74c9ea4 ae83e55b-0c71-9d70-ea65-c4efd74c9ea4}" +2025-06-03T23:37:43.607Z [INFO] server-4f0734e6-1fb0-a4df-7d92-7d90314ea40c: aborting pipeline replication: peer="{Voter server-f76fa422-fbf5-5995-106d-b9019511a181 f76fa422-fbf5-5995-106d-b9019511a181}" +--- PASS: TestRaft_Barrier (0.17s) === RUN TestRaft_VerifyLeader -2025-06-03T23:34:45.271Z [INFO] server-5cdd5ac7-2ada-125e-8ea3-32d82ff187dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5cdd5ac7-2ada-125e-8ea3-32d82ff187dd Address:5cdd5ac7-2ada-125e-8ea3-32d82ff187dd} {Suffrage:Voter ID:server-674148ee-fff3-0d90-574e-db6eef8d88cb Address:674148ee-fff3-0d90-574e-db6eef8d88cb} {Suffrage:Voter ID:server-9cadb71a-8687-e84c-0ce3-220479ad3afb Address:9cadb71a-8687-e84c-0ce3-220479ad3afb}]" -2025-06-03T23:34:45.272Z [INFO] server-5cdd5ac7-2ada-125e-8ea3-32d82ff187dd: entering follower state: follower="Node at 5cdd5ac7-2ada-125e-8ea3-32d82ff187dd [Follower]" leader-address= leader-id= -2025-06-03T23:34:45.272Z [INFO] server-674148ee-fff3-0d90-574e-db6eef8d88cb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5cdd5ac7-2ada-125e-8ea3-32d82ff187dd Address:5cdd5ac7-2ada-125e-8ea3-32d82ff187dd} {Suffrage:Voter ID:server-674148ee-fff3-0d90-574e-db6eef8d88cb Address:674148ee-fff3-0d90-574e-db6eef8d88cb} {Suffrage:Voter ID:server-9cadb71a-8687-e84c-0ce3-220479ad3afb Address:9cadb71a-8687-e84c-0ce3-220479ad3afb}]" -2025-06-03T23:34:45.272Z [INFO] server-674148ee-fff3-0d90-574e-db6eef8d88cb: entering follower state: follower="Node at 674148ee-fff3-0d90-574e-db6eef8d88cb [Follower]" leader-address= leader-id= -2025-06-03T23:34:45.272Z [INFO] server-9cadb71a-8687-e84c-0ce3-220479ad3afb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5cdd5ac7-2ada-125e-8ea3-32d82ff187dd Address:5cdd5ac7-2ada-125e-8ea3-32d82ff187dd} {Suffrage:Voter ID:server-674148ee-fff3-0d90-574e-db6eef8d88cb Address:674148ee-fff3-0d90-574e-db6eef8d88cb} {Suffrage:Voter ID:server-9cadb71a-8687-e84c-0ce3-220479ad3afb Address:9cadb71a-8687-e84c-0ce3-220479ad3afb}]" -2025-06-03T23:34:45.272Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:45.272Z [INFO] server-9cadb71a-8687-e84c-0ce3-220479ad3afb: entering follower state: follower="Node at 9cadb71a-8687-e84c-0ce3-220479ad3afb [Follower]" leader-address= leader-id= -2025-06-03T23:34:45.338Z [WARN] server-674148ee-fff3-0d90-574e-db6eef8d88cb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:45.338Z [INFO] server-674148ee-fff3-0d90-574e-db6eef8d88cb: entering candidate state: node="Node at 674148ee-fff3-0d90-574e-db6eef8d88cb [Candidate]" term=2 -2025-06-03T23:34:45.338Z [INFO] server-674148ee-fff3-0d90-574e-db6eef8d88cb: election won: term=2 tally=2 -2025-06-03T23:34:45.338Z [INFO] server-674148ee-fff3-0d90-574e-db6eef8d88cb: entering leader state: leader="Node at 674148ee-fff3-0d90-574e-db6eef8d88cb [Leader]" -2025-06-03T23:34:45.338Z [INFO] server-674148ee-fff3-0d90-574e-db6eef8d88cb: added peer, starting replication: peer=server-5cdd5ac7-2ada-125e-8ea3-32d82ff187dd -2025-06-03T23:34:45.338Z [INFO] server-674148ee-fff3-0d90-574e-db6eef8d88cb: added peer, starting replication: peer=server-9cadb71a-8687-e84c-0ce3-220479ad3afb -2025-06-03T23:34:45.338Z [INFO] server-674148ee-fff3-0d90-574e-db6eef8d88cb: pipelining replication: peer="{Voter server-9cadb71a-8687-e84c-0ce3-220479ad3afb 9cadb71a-8687-e84c-0ce3-220479ad3afb}" -2025-06-03T23:34:45.338Z [INFO] server-674148ee-fff3-0d90-574e-db6eef8d88cb: pipelining replication: peer="{Voter server-5cdd5ac7-2ada-125e-8ea3-32d82ff187dd 5cdd5ac7-2ada-125e-8ea3-32d82ff187dd}" -2025-06-03T23:34:45.444Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:45.338591231 +0000 UTC m=+100.359410641 (1 nodes), 66.29872ms from start of poll, 66.77912ms from cluster start. Timeout at 2025-06-03 23:34:45.444113591 +0000 UTC m=+100.464932961, 105.52232ms after stability -2025-06-03T23:34:45.444Z [INFO] server-674148ee-fff3-0d90-574e-db6eef8d88cb: aborting pipeline replication: peer="{Voter server-9cadb71a-8687-e84c-0ce3-220479ad3afb 9cadb71a-8687-e84c-0ce3-220479ad3afb}" -2025-06-03T23:34:45.444Z [INFO] server-674148ee-fff3-0d90-574e-db6eef8d88cb: aborting pipeline replication: peer="{Voter server-5cdd5ac7-2ada-125e-8ea3-32d82ff187dd 5cdd5ac7-2ada-125e-8ea3-32d82ff187dd}" +2025-06-03T23:37:43.608Z [INFO] server-fc8c22be-7282-d899-3e21-64be18d810ff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc8c22be-7282-d899-3e21-64be18d810ff Address:fc8c22be-7282-d899-3e21-64be18d810ff} {Suffrage:Voter ID:server-131343fb-5b72-6cc6-c669-73d5b89f5711 Address:131343fb-5b72-6cc6-c669-73d5b89f5711} {Suffrage:Voter ID:server-662cca3a-f3f9-71c7-97b6-804b6026b665 Address:662cca3a-f3f9-71c7-97b6-804b6026b665}]" +2025-06-03T23:37:43.608Z [INFO] server-fc8c22be-7282-d899-3e21-64be18d810ff: entering follower state: follower="Node at fc8c22be-7282-d899-3e21-64be18d810ff [Follower]" leader-address= leader-id= +2025-06-03T23:37:43.608Z [INFO] server-131343fb-5b72-6cc6-c669-73d5b89f5711: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc8c22be-7282-d899-3e21-64be18d810ff Address:fc8c22be-7282-d899-3e21-64be18d810ff} {Suffrage:Voter ID:server-131343fb-5b72-6cc6-c669-73d5b89f5711 Address:131343fb-5b72-6cc6-c669-73d5b89f5711} {Suffrage:Voter ID:server-662cca3a-f3f9-71c7-97b6-804b6026b665 Address:662cca3a-f3f9-71c7-97b6-804b6026b665}]" +2025-06-03T23:37:43.608Z [INFO] server-131343fb-5b72-6cc6-c669-73d5b89f5711: entering follower state: follower="Node at 131343fb-5b72-6cc6-c669-73d5b89f5711 [Follower]" leader-address= leader-id= +2025-06-03T23:37:43.608Z [INFO] server-662cca3a-f3f9-71c7-97b6-804b6026b665: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc8c22be-7282-d899-3e21-64be18d810ff Address:fc8c22be-7282-d899-3e21-64be18d810ff} {Suffrage:Voter ID:server-131343fb-5b72-6cc6-c669-73d5b89f5711 Address:131343fb-5b72-6cc6-c669-73d5b89f5711} {Suffrage:Voter ID:server-662cca3a-f3f9-71c7-97b6-804b6026b665 Address:662cca3a-f3f9-71c7-97b6-804b6026b665}]" +2025-06-03T23:37:43.608Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:43.608Z [INFO] server-662cca3a-f3f9-71c7-97b6-804b6026b665: entering follower state: follower="Node at 662cca3a-f3f9-71c7-97b6-804b6026b665 [Follower]" leader-address= leader-id= +2025-06-03T23:37:43.674Z [WARN] server-662cca3a-f3f9-71c7-97b6-804b6026b665: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:43.674Z [INFO] server-662cca3a-f3f9-71c7-97b6-804b6026b665: entering candidate state: node="Node at 662cca3a-f3f9-71c7-97b6-804b6026b665 [Candidate]" term=2 +2025-06-03T23:37:43.674Z [INFO] server-662cca3a-f3f9-71c7-97b6-804b6026b665: election won: term=2 tally=2 +2025-06-03T23:37:43.674Z [INFO] server-662cca3a-f3f9-71c7-97b6-804b6026b665: entering leader state: leader="Node at 662cca3a-f3f9-71c7-97b6-804b6026b665 [Leader]" +2025-06-03T23:37:43.674Z [INFO] server-662cca3a-f3f9-71c7-97b6-804b6026b665: added peer, starting replication: peer=server-fc8c22be-7282-d899-3e21-64be18d810ff +2025-06-03T23:37:43.674Z [INFO] server-662cca3a-f3f9-71c7-97b6-804b6026b665: added peer, starting replication: peer=server-131343fb-5b72-6cc6-c669-73d5b89f5711 +2025-06-03T23:37:43.675Z [INFO] server-662cca3a-f3f9-71c7-97b6-804b6026b665: pipelining replication: peer="{Voter server-131343fb-5b72-6cc6-c669-73d5b89f5711 131343fb-5b72-6cc6-c669-73d5b89f5711}" +2025-06-03T23:37:43.675Z [INFO] server-662cca3a-f3f9-71c7-97b6-804b6026b665: pipelining replication: peer="{Voter server-fc8c22be-7282-d899-3e21-64be18d810ff fc8c22be-7282-d899-3e21-64be18d810ff}" +2025-06-03T23:37:43.782Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:43.674990951 +0000 UTC m=+98.996745881 (1 nodes), 66.36384ms from start of poll, 66.81756ms from cluster start. Timeout at 2025-06-03 23:37:43.782232151 +0000 UTC m=+99.103987081, 107.2412ms after stability +2025-06-03T23:37:43.782Z [INFO] server-662cca3a-f3f9-71c7-97b6-804b6026b665: aborting pipeline replication: peer="{Voter server-131343fb-5b72-6cc6-c669-73d5b89f5711 131343fb-5b72-6cc6-c669-73d5b89f5711}" +2025-06-03T23:37:43.782Z [INFO] server-662cca3a-f3f9-71c7-97b6-804b6026b665: aborting pipeline replication: peer="{Voter server-fc8c22be-7282-d899-3e21-64be18d810ff fc8c22be-7282-d899-3e21-64be18d810ff}" --- PASS: TestRaft_VerifyLeader (0.17s) === RUN TestRaft_VerifyLeader_Single -2025-06-03T23:34:45.445Z [INFO] server-0c48178a-85c1-9387-3331-595125af9fef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c48178a-85c1-9387-3331-595125af9fef Address:0c48178a-85c1-9387-3331-595125af9fef}]" -2025-06-03T23:34:45.445Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:45.445Z [INFO] server-0c48178a-85c1-9387-3331-595125af9fef: entering follower state: follower="Node at 0c48178a-85c1-9387-3331-595125af9fef [Follower]" leader-address= leader-id= -2025-06-03T23:34:45.517Z [WARN] server-0c48178a-85c1-9387-3331-595125af9fef: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:45.517Z [INFO] server-0c48178a-85c1-9387-3331-595125af9fef: entering candidate state: node="Node at 0c48178a-85c1-9387-3331-595125af9fef [Candidate]" term=2 -2025-06-03T23:34:45.517Z [INFO] server-0c48178a-85c1-9387-3331-595125af9fef: election won: term=2 tally=1 -2025-06-03T23:34:45.517Z [INFO] server-0c48178a-85c1-9387-3331-595125af9fef: entering leader state: leader="Node at 0c48178a-85c1-9387-3331-595125af9fef [Leader]" -2025-06-03T23:34:45.623Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:45.517324351 +0000 UTC m=+100.538143761 (1 nodes), 72.10504ms from start of poll, 72.32352ms from cluster start. Timeout at 2025-06-03 23:34:45.623090071 +0000 UTC m=+100.643909441, 105.76568ms after stability ---- PASS: TestRaft_VerifyLeader_Single (0.18s) +2025-06-03T23:37:43.783Z [INFO] server-96edf9f0-f2d4-c0e7-9234-081fff66ade0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-96edf9f0-f2d4-c0e7-9234-081fff66ade0 Address:96edf9f0-f2d4-c0e7-9234-081fff66ade0}]" +2025-06-03T23:37:43.783Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:43.783Z [INFO] server-96edf9f0-f2d4-c0e7-9234-081fff66ade0: entering follower state: follower="Node at 96edf9f0-f2d4-c0e7-9234-081fff66ade0 [Follower]" leader-address= leader-id= +2025-06-03T23:37:43.850Z [WARN] server-96edf9f0-f2d4-c0e7-9234-081fff66ade0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:43.850Z [INFO] server-96edf9f0-f2d4-c0e7-9234-081fff66ade0: entering candidate state: node="Node at 96edf9f0-f2d4-c0e7-9234-081fff66ade0 [Candidate]" term=2 +2025-06-03T23:37:43.850Z [INFO] server-96edf9f0-f2d4-c0e7-9234-081fff66ade0: election won: term=2 tally=1 +2025-06-03T23:37:43.850Z [INFO] server-96edf9f0-f2d4-c0e7-9234-081fff66ade0: entering leader state: leader="Node at 96edf9f0-f2d4-c0e7-9234-081fff66ade0 [Leader]" +2025-06-03T23:37:43.956Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:43.850410311 +0000 UTC m=+99.172165241 (1 nodes), 67.07384ms from start of poll, 67.2984ms from cluster start. Timeout at 2025-06-03 23:37:43.956059071 +0000 UTC m=+99.277814041, 105.6488ms after stability +--- PASS: TestRaft_VerifyLeader_Single (0.17s) === RUN TestRaft_VerifyLeader_Fail -2025-06-03T23:34:45.623Z [INFO] server-0681e8b8-37dc-82ed-dd9a-d3a7d3a6cd89: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0681e8b8-37dc-82ed-dd9a-d3a7d3a6cd89 Address:0681e8b8-37dc-82ed-dd9a-d3a7d3a6cd89} {Suffrage:Voter ID:server-4e7c6ec7-c49c-6e64-3538-e881447e56a0 Address:4e7c6ec7-c49c-6e64-3538-e881447e56a0}]" -2025-06-03T23:34:45.623Z [INFO] server-0681e8b8-37dc-82ed-dd9a-d3a7d3a6cd89: entering follower state: follower="Node at 0681e8b8-37dc-82ed-dd9a-d3a7d3a6cd89 [Follower]" leader-address= leader-id= -2025-06-03T23:34:45.624Z [INFO] server-4e7c6ec7-c49c-6e64-3538-e881447e56a0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0681e8b8-37dc-82ed-dd9a-d3a7d3a6cd89 Address:0681e8b8-37dc-82ed-dd9a-d3a7d3a6cd89} {Suffrage:Voter ID:server-4e7c6ec7-c49c-6e64-3538-e881447e56a0 Address:4e7c6ec7-c49c-6e64-3538-e881447e56a0}]" -2025-06-03T23:34:45.624Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:45.624Z [INFO] server-4e7c6ec7-c49c-6e64-3538-e881447e56a0: entering follower state: follower="Node at 4e7c6ec7-c49c-6e64-3538-e881447e56a0 [Follower]" leader-address= leader-id= -2025-06-03T23:34:45.687Z [WARN] server-4e7c6ec7-c49c-6e64-3538-e881447e56a0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:45.687Z [INFO] server-4e7c6ec7-c49c-6e64-3538-e881447e56a0: entering candidate state: node="Node at 4e7c6ec7-c49c-6e64-3538-e881447e56a0 [Candidate]" term=2 -2025-06-03T23:34:45.687Z [INFO] server-4e7c6ec7-c49c-6e64-3538-e881447e56a0: election won: term=2 tally=2 -2025-06-03T23:34:45.687Z [INFO] server-4e7c6ec7-c49c-6e64-3538-e881447e56a0: entering leader state: leader="Node at 4e7c6ec7-c49c-6e64-3538-e881447e56a0 [Leader]" -2025-06-03T23:34:45.687Z [INFO] server-4e7c6ec7-c49c-6e64-3538-e881447e56a0: added peer, starting replication: peer=server-0681e8b8-37dc-82ed-dd9a-d3a7d3a6cd89 -2025-06-03T23:34:45.687Z [INFO] server-4e7c6ec7-c49c-6e64-3538-e881447e56a0: pipelining replication: peer="{Voter server-0681e8b8-37dc-82ed-dd9a-d3a7d3a6cd89 0681e8b8-37dc-82ed-dd9a-d3a7d3a6cd89}" -2025-06-03T23:34:45.793Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:45.687868671 +0000 UTC m=+100.708688041 (1 nodes), 63.76488ms from start of poll, 64.14584ms from cluster start. Timeout at 2025-06-03 23:34:45.793339951 +0000 UTC m=+100.814159361, 105.47132ms after stability -2025-06-03T23:34:45.793Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:45.899Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:45.793461591 +0000 UTC m=+100.814281001 (1 nodes), 3.2µs from start of poll, 169.7388ms from cluster start. Timeout at 2025-06-03 23:34:45.899378871 +0000 UTC m=+100.920198241, 105.91724ms after stability -2025-06-03T23:34:45.900Z [ERROR] server-4e7c6ec7-c49c-6e64-3538-e881447e56a0: peer has newer term, stopping replication: peer="{Voter server-0681e8b8-37dc-82ed-dd9a-d3a7d3a6cd89 0681e8b8-37dc-82ed-dd9a-d3a7d3a6cd89}" -2025-06-03T23:34:45.900Z [INFO] server-4e7c6ec7-c49c-6e64-3538-e881447e56a0: aborting pipeline replication: peer="{Voter server-0681e8b8-37dc-82ed-dd9a-d3a7d3a6cd89 0681e8b8-37dc-82ed-dd9a-d3a7d3a6cd89}" -2025-06-03T23:34:45.900Z [INFO] server-4e7c6ec7-c49c-6e64-3538-e881447e56a0: entering follower state: follower="Node at 4e7c6ec7-c49c-6e64-3538-e881447e56a0 [Follower]" leader-address= leader-id= ---- PASS: TestRaft_VerifyLeader_Fail (0.28s) +2025-06-03T23:37:43.957Z [INFO] server-95936c10-6191-f917-2d3e-ed28af0cb656: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-95936c10-6191-f917-2d3e-ed28af0cb656 Address:95936c10-6191-f917-2d3e-ed28af0cb656} {Suffrage:Voter ID:server-5f5b518f-ef08-a17c-b9e8-ac233d050525 Address:5f5b518f-ef08-a17c-b9e8-ac233d050525}]" +2025-06-03T23:37:43.957Z [INFO] server-95936c10-6191-f917-2d3e-ed28af0cb656: entering follower state: follower="Node at 95936c10-6191-f917-2d3e-ed28af0cb656 [Follower]" leader-address= leader-id= +2025-06-03T23:37:43.957Z [INFO] server-5f5b518f-ef08-a17c-b9e8-ac233d050525: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-95936c10-6191-f917-2d3e-ed28af0cb656 Address:95936c10-6191-f917-2d3e-ed28af0cb656} {Suffrage:Voter ID:server-5f5b518f-ef08-a17c-b9e8-ac233d050525 Address:5f5b518f-ef08-a17c-b9e8-ac233d050525}]" +2025-06-03T23:37:43.957Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:43.957Z [INFO] server-5f5b518f-ef08-a17c-b9e8-ac233d050525: entering follower state: follower="Node at 5f5b518f-ef08-a17c-b9e8-ac233d050525 [Follower]" leader-address= leader-id= +2025-06-03T23:37:44.042Z [WARN] server-5f5b518f-ef08-a17c-b9e8-ac233d050525: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:44.043Z [INFO] server-5f5b518f-ef08-a17c-b9e8-ac233d050525: entering candidate state: node="Node at 5f5b518f-ef08-a17c-b9e8-ac233d050525 [Candidate]" term=2 +2025-06-03T23:37:44.043Z [INFO] server-5f5b518f-ef08-a17c-b9e8-ac233d050525: election won: term=2 tally=2 +2025-06-03T23:37:44.043Z [INFO] server-5f5b518f-ef08-a17c-b9e8-ac233d050525: entering leader state: leader="Node at 5f5b518f-ef08-a17c-b9e8-ac233d050525 [Leader]" +2025-06-03T23:37:44.043Z [INFO] server-5f5b518f-ef08-a17c-b9e8-ac233d050525: added peer, starting replication: peer=server-95936c10-6191-f917-2d3e-ed28af0cb656 +2025-06-03T23:37:44.043Z [INFO] server-5f5b518f-ef08-a17c-b9e8-ac233d050525: pipelining replication: peer="{Voter server-95936c10-6191-f917-2d3e-ed28af0cb656 95936c10-6191-f917-2d3e-ed28af0cb656}" +2025-06-03T23:37:44.148Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:44.043125991 +0000 UTC m=+99.364880921 (1 nodes), 85.41868ms from start of poll, 86.06864ms from cluster start. Timeout at 2025-06-03 23:37:44.148482231 +0000 UTC m=+99.470237161, 105.35624ms after stability +2025-06-03T23:37:44.148Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:44.254Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:44.148607471 +0000 UTC m=+99.470362401 (1 nodes), 2.52µs from start of poll, 191.55012ms from cluster start. Timeout at 2025-06-03 23:37:44.254462351 +0000 UTC m=+99.576217281, 105.85488ms after stability +2025-06-03T23:37:44.257Z [ERROR] server-5f5b518f-ef08-a17c-b9e8-ac233d050525: peer has newer term, stopping replication: peer="{Voter server-95936c10-6191-f917-2d3e-ed28af0cb656 95936c10-6191-f917-2d3e-ed28af0cb656}" +2025-06-03T23:37:44.257Z [INFO] server-5f5b518f-ef08-a17c-b9e8-ac233d050525: aborting pipeline replication: peer="{Voter server-95936c10-6191-f917-2d3e-ed28af0cb656 95936c10-6191-f917-2d3e-ed28af0cb656}" +2025-06-03T23:37:44.257Z [INFO] server-5f5b518f-ef08-a17c-b9e8-ac233d050525: entering follower state: follower="Node at 5f5b518f-ef08-a17c-b9e8-ac233d050525 [Follower]" leader-address= leader-id= +--- PASS: TestRaft_VerifyLeader_Fail (0.30s) === RUN TestRaft_VerifyLeader_PartialConnect -2025-06-03T23:34:45.901Z [INFO] server-efd8fa43-0087-7aab-78d2-b1f84db3abfc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-efd8fa43-0087-7aab-78d2-b1f84db3abfc Address:efd8fa43-0087-7aab-78d2-b1f84db3abfc} {Suffrage:Voter ID:server-2c06ddeb-aa06-0215-99df-d665153ed692 Address:2c06ddeb-aa06-0215-99df-d665153ed692} {Suffrage:Voter ID:server-a946f4a5-68a2-6b3d-3619-af746b18d25a Address:a946f4a5-68a2-6b3d-3619-af746b18d25a}]" -2025-06-03T23:34:45.901Z [INFO] server-efd8fa43-0087-7aab-78d2-b1f84db3abfc: entering follower state: follower="Node at efd8fa43-0087-7aab-78d2-b1f84db3abfc [Follower]" leader-address= leader-id= -2025-06-03T23:34:45.901Z [INFO] server-2c06ddeb-aa06-0215-99df-d665153ed692: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-efd8fa43-0087-7aab-78d2-b1f84db3abfc Address:efd8fa43-0087-7aab-78d2-b1f84db3abfc} {Suffrage:Voter ID:server-2c06ddeb-aa06-0215-99df-d665153ed692 Address:2c06ddeb-aa06-0215-99df-d665153ed692} {Suffrage:Voter ID:server-a946f4a5-68a2-6b3d-3619-af746b18d25a Address:a946f4a5-68a2-6b3d-3619-af746b18d25a}]" -2025-06-03T23:34:45.901Z [INFO] server-2c06ddeb-aa06-0215-99df-d665153ed692: entering follower state: follower="Node at 2c06ddeb-aa06-0215-99df-d665153ed692 [Follower]" leader-address= leader-id= -2025-06-03T23:34:45.901Z [INFO] server-a946f4a5-68a2-6b3d-3619-af746b18d25a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-efd8fa43-0087-7aab-78d2-b1f84db3abfc Address:efd8fa43-0087-7aab-78d2-b1f84db3abfc} {Suffrage:Voter ID:server-2c06ddeb-aa06-0215-99df-d665153ed692 Address:2c06ddeb-aa06-0215-99df-d665153ed692} {Suffrage:Voter ID:server-a946f4a5-68a2-6b3d-3619-af746b18d25a Address:a946f4a5-68a2-6b3d-3619-af746b18d25a}]" -2025-06-03T23:34:45.901Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:45.901Z [INFO] server-a946f4a5-68a2-6b3d-3619-af746b18d25a: entering follower state: follower="Node at a946f4a5-68a2-6b3d-3619-af746b18d25a [Follower]" leader-address= leader-id= -2025-06-03T23:34:45.957Z [WARN] server-efd8fa43-0087-7aab-78d2-b1f84db3abfc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:45.957Z [INFO] server-efd8fa43-0087-7aab-78d2-b1f84db3abfc: entering candidate state: node="Node at efd8fa43-0087-7aab-78d2-b1f84db3abfc [Candidate]" term=2 -2025-06-03T23:34:45.957Z [INFO] server-efd8fa43-0087-7aab-78d2-b1f84db3abfc: election won: term=2 tally=2 -2025-06-03T23:34:45.957Z [INFO] server-efd8fa43-0087-7aab-78d2-b1f84db3abfc: entering leader state: leader="Node at efd8fa43-0087-7aab-78d2-b1f84db3abfc [Leader]" -2025-06-03T23:34:45.957Z [INFO] server-efd8fa43-0087-7aab-78d2-b1f84db3abfc: added peer, starting replication: peer=server-2c06ddeb-aa06-0215-99df-d665153ed692 -2025-06-03T23:34:45.957Z [INFO] server-efd8fa43-0087-7aab-78d2-b1f84db3abfc: added peer, starting replication: peer=server-a946f4a5-68a2-6b3d-3619-af746b18d25a -2025-06-03T23:34:45.957Z [INFO] server-efd8fa43-0087-7aab-78d2-b1f84db3abfc: pipelining replication: peer="{Voter server-2c06ddeb-aa06-0215-99df-d665153ed692 2c06ddeb-aa06-0215-99df-d665153ed692}" -2025-06-03T23:34:45.957Z [INFO] server-efd8fa43-0087-7aab-78d2-b1f84db3abfc: pipelining replication: peer="{Voter server-a946f4a5-68a2-6b3d-3619-af746b18d25a a946f4a5-68a2-6b3d-3619-af746b18d25a}" -2025-06-03T23:34:46.062Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:45.957549311 +0000 UTC m=+100.978368681 (1 nodes), 55.55716ms from start of poll, 56.03316ms from cluster start. Timeout at 2025-06-03 23:34:46.062712191 +0000 UTC m=+101.083531561, 105.16288ms after stability -2025-06-03T23:34:46.068Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:46.173Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:46.068261151 +0000 UTC m=+101.089080521 (2 nodes), 3.6µs from start of poll, 166.745ms from cluster start. Timeout at 2025-06-03 23:34:46.173395871 +0000 UTC m=+101.194215241, 105.13472ms after stability - raft_test.go:1968: [INFO] Disconnecting Node at 2c06ddeb-aa06-0215-99df-d665153ed692 [Follower] -2025-06-03T23:34:46.173Z [ERROR] server-efd8fa43-0087-7aab-78d2-b1f84db3abfc: failed to heartbeat to: peer=2c06ddeb-aa06-0215-99df-d665153ed692 backoff time=10ms error="failed to connect to peer: 2c06ddeb-aa06-0215-99df-d665153ed692" -2025-06-03T23:34:46.173Z [INFO] server-efd8fa43-0087-7aab-78d2-b1f84db3abfc: aborting pipeline replication: peer="{Voter server-2c06ddeb-aa06-0215-99df-d665153ed692 2c06ddeb-aa06-0215-99df-d665153ed692}" -2025-06-03T23:34:46.173Z [INFO] server-efd8fa43-0087-7aab-78d2-b1f84db3abfc: aborting pipeline replication: peer="{Voter server-a946f4a5-68a2-6b3d-3619-af746b18d25a a946f4a5-68a2-6b3d-3619-af746b18d25a}" ---- PASS: TestRaft_VerifyLeader_PartialConnect (0.27s) +2025-06-03T23:37:44.258Z [INFO] server-c6423207-f2cb-4919-5f96-c982a79fc9ff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c6423207-f2cb-4919-5f96-c982a79fc9ff Address:c6423207-f2cb-4919-5f96-c982a79fc9ff} {Suffrage:Voter ID:server-952e83c0-9c9f-2d47-f3ab-1be978d777c2 Address:952e83c0-9c9f-2d47-f3ab-1be978d777c2} {Suffrage:Voter ID:server-52497579-f91f-d0bf-9ee8-6e04dde2b796 Address:52497579-f91f-d0bf-9ee8-6e04dde2b796}]" +2025-06-03T23:37:44.258Z [INFO] server-c6423207-f2cb-4919-5f96-c982a79fc9ff: entering follower state: follower="Node at c6423207-f2cb-4919-5f96-c982a79fc9ff [Follower]" leader-address= leader-id= +2025-06-03T23:37:44.258Z [INFO] server-952e83c0-9c9f-2d47-f3ab-1be978d777c2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c6423207-f2cb-4919-5f96-c982a79fc9ff Address:c6423207-f2cb-4919-5f96-c982a79fc9ff} {Suffrage:Voter ID:server-952e83c0-9c9f-2d47-f3ab-1be978d777c2 Address:952e83c0-9c9f-2d47-f3ab-1be978d777c2} {Suffrage:Voter ID:server-52497579-f91f-d0bf-9ee8-6e04dde2b796 Address:52497579-f91f-d0bf-9ee8-6e04dde2b796}]" +2025-06-03T23:37:44.258Z [INFO] server-952e83c0-9c9f-2d47-f3ab-1be978d777c2: entering follower state: follower="Node at 952e83c0-9c9f-2d47-f3ab-1be978d777c2 [Follower]" leader-address= leader-id= +2025-06-03T23:37:44.258Z [INFO] server-52497579-f91f-d0bf-9ee8-6e04dde2b796: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c6423207-f2cb-4919-5f96-c982a79fc9ff Address:c6423207-f2cb-4919-5f96-c982a79fc9ff} {Suffrage:Voter ID:server-952e83c0-9c9f-2d47-f3ab-1be978d777c2 Address:952e83c0-9c9f-2d47-f3ab-1be978d777c2} {Suffrage:Voter ID:server-52497579-f91f-d0bf-9ee8-6e04dde2b796 Address:52497579-f91f-d0bf-9ee8-6e04dde2b796}]" +2025-06-03T23:37:44.258Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:44.258Z [INFO] server-52497579-f91f-d0bf-9ee8-6e04dde2b796: entering follower state: follower="Node at 52497579-f91f-d0bf-9ee8-6e04dde2b796 [Follower]" leader-address= leader-id= +2025-06-03T23:37:44.324Z [WARN] server-c6423207-f2cb-4919-5f96-c982a79fc9ff: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:44.324Z [INFO] server-c6423207-f2cb-4919-5f96-c982a79fc9ff: entering candidate state: node="Node at c6423207-f2cb-4919-5f96-c982a79fc9ff [Candidate]" term=2 +2025-06-03T23:37:44.324Z [INFO] server-c6423207-f2cb-4919-5f96-c982a79fc9ff: election won: term=2 tally=2 +2025-06-03T23:37:44.324Z [INFO] server-c6423207-f2cb-4919-5f96-c982a79fc9ff: entering leader state: leader="Node at c6423207-f2cb-4919-5f96-c982a79fc9ff [Leader]" +2025-06-03T23:37:44.324Z [INFO] server-c6423207-f2cb-4919-5f96-c982a79fc9ff: added peer, starting replication: peer=server-952e83c0-9c9f-2d47-f3ab-1be978d777c2 +2025-06-03T23:37:44.324Z [INFO] server-c6423207-f2cb-4919-5f96-c982a79fc9ff: added peer, starting replication: peer=server-52497579-f91f-d0bf-9ee8-6e04dde2b796 +2025-06-03T23:37:44.325Z [INFO] server-c6423207-f2cb-4919-5f96-c982a79fc9ff: pipelining replication: peer="{Voter server-52497579-f91f-d0bf-9ee8-6e04dde2b796 52497579-f91f-d0bf-9ee8-6e04dde2b796}" +2025-06-03T23:37:44.325Z [INFO] server-c6423207-f2cb-4919-5f96-c982a79fc9ff: pipelining replication: peer="{Voter server-952e83c0-9c9f-2d47-f3ab-1be978d777c2 952e83c0-9c9f-2d47-f3ab-1be978d777c2}" +2025-06-03T23:37:44.430Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:44.324940031 +0000 UTC m=+99.646694961 (1 nodes), 66.20372ms from start of poll, 66.70556ms from cluster start. Timeout at 2025-06-03 23:37:44.430485791 +0000 UTC m=+99.752240761, 105.5458ms after stability +2025-06-03T23:37:44.435Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:44.541Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:44.435627591 +0000 UTC m=+99.757382521 (2 nodes), 1.48µs from start of poll, 177.39312ms from cluster start. Timeout at 2025-06-03 23:37:44.541111991 +0000 UTC m=+99.862866921, 105.4844ms after stability + raft_test.go:1968: [INFO] Disconnecting Node at 952e83c0-9c9f-2d47-f3ab-1be978d777c2 [Follower] +2025-06-03T23:37:44.541Z [ERROR] server-c6423207-f2cb-4919-5f96-c982a79fc9ff: failed to heartbeat to: peer=952e83c0-9c9f-2d47-f3ab-1be978d777c2 backoff time=10ms error="failed to connect to peer: 952e83c0-9c9f-2d47-f3ab-1be978d777c2" +2025-06-03T23:37:44.541Z [INFO] server-c6423207-f2cb-4919-5f96-c982a79fc9ff: aborting pipeline replication: peer="{Voter server-52497579-f91f-d0bf-9ee8-6e04dde2b796 52497579-f91f-d0bf-9ee8-6e04dde2b796}" +2025-06-03T23:37:44.541Z [INFO] server-c6423207-f2cb-4919-5f96-c982a79fc9ff: aborting pipeline replication: peer="{Voter server-952e83c0-9c9f-2d47-f3ab-1be978d777c2 952e83c0-9c9f-2d47-f3ab-1be978d777c2}" +--- PASS: TestRaft_VerifyLeader_PartialConnect (0.28s) === RUN TestRaft_NotifyCh -2025-06-03T23:34:46.174Z [INFO] server-8c960552-b646-9200-7b77-dc49d8567793: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8c960552-b646-9200-7b77-dc49d8567793 Address:8c960552-b646-9200-7b77-dc49d8567793}]" -2025-06-03T23:34:46.174Z [INFO] server-8c960552-b646-9200-7b77-dc49d8567793: entering follower state: follower="Node at 8c960552-b646-9200-7b77-dc49d8567793 [Follower]" leader-address= leader-id= -2025-06-03T23:34:46.274Z [WARN] server-8c960552-b646-9200-7b77-dc49d8567793: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:46.274Z [INFO] server-8c960552-b646-9200-7b77-dc49d8567793: entering candidate state: node="Node at 8c960552-b646-9200-7b77-dc49d8567793 [Candidate]" term=2 -2025-06-03T23:34:46.274Z [INFO] server-8c960552-b646-9200-7b77-dc49d8567793: election won: term=2 tally=1 -2025-06-03T23:34:46.274Z [INFO] server-8c960552-b646-9200-7b77-dc49d8567793: entering leader state: leader="Node at 8c960552-b646-9200-7b77-dc49d8567793 [Leader]" ---- PASS: TestRaft_NotifyCh (0.10s) +2025-06-03T23:37:44.542Z [INFO] server-a2bbd0f6-8f16-b1dd-3aba-264ea65be22b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a2bbd0f6-8f16-b1dd-3aba-264ea65be22b Address:a2bbd0f6-8f16-b1dd-3aba-264ea65be22b}]" +2025-06-03T23:37:44.542Z [INFO] server-a2bbd0f6-8f16-b1dd-3aba-264ea65be22b: entering follower state: follower="Node at a2bbd0f6-8f16-b1dd-3aba-264ea65be22b [Follower]" leader-address= leader-id= +2025-06-03T23:37:44.612Z [WARN] server-a2bbd0f6-8f16-b1dd-3aba-264ea65be22b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:44.612Z [INFO] server-a2bbd0f6-8f16-b1dd-3aba-264ea65be22b: entering candidate state: node="Node at a2bbd0f6-8f16-b1dd-3aba-264ea65be22b [Candidate]" term=2 +2025-06-03T23:37:44.612Z [INFO] server-a2bbd0f6-8f16-b1dd-3aba-264ea65be22b: election won: term=2 tally=1 +2025-06-03T23:37:44.612Z [INFO] server-a2bbd0f6-8f16-b1dd-3aba-264ea65be22b: entering leader state: leader="Node at a2bbd0f6-8f16-b1dd-3aba-264ea65be22b [Leader]" +--- PASS: TestRaft_NotifyCh (0.07s) === RUN TestRaft_AppendEntry -2025-06-03T23:34:46.275Z [INFO] server-3fdd9aa7-3dda-a7fb-b83f-6f32fbcf4eb9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3fdd9aa7-3dda-a7fb-b83f-6f32fbcf4eb9 Address:3fdd9aa7-3dda-a7fb-b83f-6f32fbcf4eb9} {Suffrage:Voter ID:server-2df852e0-cccb-22e6-2a4d-286b115a0220 Address:2df852e0-cccb-22e6-2a4d-286b115a0220} {Suffrage:Voter ID:server-735ef37d-c646-aebd-75ef-9d2958f649b5 Address:735ef37d-c646-aebd-75ef-9d2958f649b5}]" -2025-06-03T23:34:46.275Z [INFO] server-3fdd9aa7-3dda-a7fb-b83f-6f32fbcf4eb9: entering follower state: follower="Node at 3fdd9aa7-3dda-a7fb-b83f-6f32fbcf4eb9 [Follower]" leader-address= leader-id= -2025-06-03T23:34:46.275Z [INFO] server-2df852e0-cccb-22e6-2a4d-286b115a0220: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3fdd9aa7-3dda-a7fb-b83f-6f32fbcf4eb9 Address:3fdd9aa7-3dda-a7fb-b83f-6f32fbcf4eb9} {Suffrage:Voter ID:server-2df852e0-cccb-22e6-2a4d-286b115a0220 Address:2df852e0-cccb-22e6-2a4d-286b115a0220} {Suffrage:Voter ID:server-735ef37d-c646-aebd-75ef-9d2958f649b5 Address:735ef37d-c646-aebd-75ef-9d2958f649b5}]" -2025-06-03T23:34:46.275Z [INFO] server-2df852e0-cccb-22e6-2a4d-286b115a0220: entering follower state: follower="Node at 2df852e0-cccb-22e6-2a4d-286b115a0220 [Follower]" leader-address= leader-id= -2025-06-03T23:34:46.275Z [INFO] server-735ef37d-c646-aebd-75ef-9d2958f649b5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3fdd9aa7-3dda-a7fb-b83f-6f32fbcf4eb9 Address:3fdd9aa7-3dda-a7fb-b83f-6f32fbcf4eb9} {Suffrage:Voter ID:server-2df852e0-cccb-22e6-2a4d-286b115a0220 Address:2df852e0-cccb-22e6-2a4d-286b115a0220} {Suffrage:Voter ID:server-735ef37d-c646-aebd-75ef-9d2958f649b5 Address:735ef37d-c646-aebd-75ef-9d2958f649b5}]" -2025-06-03T23:34:46.275Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:46.275Z [INFO] server-735ef37d-c646-aebd-75ef-9d2958f649b5: entering follower state: follower="Node at 735ef37d-c646-aebd-75ef-9d2958f649b5 [Follower]" leader-address= leader-id= -2025-06-03T23:34:46.349Z [WARN] server-2df852e0-cccb-22e6-2a4d-286b115a0220: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:46.349Z [INFO] server-2df852e0-cccb-22e6-2a4d-286b115a0220: entering candidate state: node="Node at 2df852e0-cccb-22e6-2a4d-286b115a0220 [Candidate]" term=2 -2025-06-03T23:34:46.349Z [INFO] server-2df852e0-cccb-22e6-2a4d-286b115a0220: election won: term=2 tally=2 -2025-06-03T23:34:46.349Z [INFO] server-2df852e0-cccb-22e6-2a4d-286b115a0220: entering leader state: leader="Node at 2df852e0-cccb-22e6-2a4d-286b115a0220 [Leader]" -2025-06-03T23:34:46.349Z [INFO] server-2df852e0-cccb-22e6-2a4d-286b115a0220: added peer, starting replication: peer=server-3fdd9aa7-3dda-a7fb-b83f-6f32fbcf4eb9 -2025-06-03T23:34:46.349Z [INFO] server-2df852e0-cccb-22e6-2a4d-286b115a0220: added peer, starting replication: peer=server-735ef37d-c646-aebd-75ef-9d2958f649b5 -2025-06-03T23:34:46.349Z [INFO] server-2df852e0-cccb-22e6-2a4d-286b115a0220: pipelining replication: peer="{Voter server-735ef37d-c646-aebd-75ef-9d2958f649b5 735ef37d-c646-aebd-75ef-9d2958f649b5}" -2025-06-03T23:34:46.349Z [INFO] server-2df852e0-cccb-22e6-2a4d-286b115a0220: pipelining replication: peer="{Voter server-3fdd9aa7-3dda-a7fb-b83f-6f32fbcf4eb9 3fdd9aa7-3dda-a7fb-b83f-6f32fbcf4eb9}" -2025-06-03T23:34:46.455Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:46.349746591 +0000 UTC m=+101.370565961 (2 nodes), 73.92904ms from start of poll, 74.39048ms from cluster start. Timeout at 2025-06-03 23:34:46.455723471 +0000 UTC m=+101.476542881, 105.97692ms after stability -2025-06-03T23:34:46.455Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:46.561Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:46.455872351 +0000 UTC m=+101.476691721 (1 nodes), 3.24µs from start of poll, 180.51624ms from cluster start. Timeout at 2025-06-03 23:34:46.561844871 +0000 UTC m=+101.582664241, 105.97252ms after stability -2025-06-03T23:34:46.561Z [WARN] server-3fdd9aa7-3dda-a7fb-b83f-6f32fbcf4eb9: clearing log suffix: from=1 to=2 -2025-06-03T23:34:46.561Z [WARN] server-3fdd9aa7-3dda-a7fb-b83f-6f32fbcf4eb9: skipping application of old log: index=1 -2025-06-03T23:34:46.562Z [WARN] server-3fdd9aa7-3dda-a7fb-b83f-6f32fbcf4eb9: skipping application of old log: index=1 -2025-06-03T23:34:46.562Z [INFO] server-2df852e0-cccb-22e6-2a4d-286b115a0220: aborting pipeline replication: peer="{Voter server-735ef37d-c646-aebd-75ef-9d2958f649b5 735ef37d-c646-aebd-75ef-9d2958f649b5}" -2025-06-03T23:34:46.562Z [INFO] server-2df852e0-cccb-22e6-2a4d-286b115a0220: aborting pipeline replication: peer="{Voter server-3fdd9aa7-3dda-a7fb-b83f-6f32fbcf4eb9 3fdd9aa7-3dda-a7fb-b83f-6f32fbcf4eb9}" ---- PASS: TestRaft_AppendEntry (0.29s) +2025-06-03T23:37:44.613Z [INFO] server-034b330f-5b75-605f-42e5-7d12903aaef4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-034b330f-5b75-605f-42e5-7d12903aaef4 Address:034b330f-5b75-605f-42e5-7d12903aaef4} {Suffrage:Voter ID:server-99ec51f8-fc52-3f4b-3e9d-45f8369d026b Address:99ec51f8-fc52-3f4b-3e9d-45f8369d026b} {Suffrage:Voter ID:server-0caff0b1-0659-776c-45a0-47cef239716f Address:0caff0b1-0659-776c-45a0-47cef239716f}]" +2025-06-03T23:37:44.613Z [INFO] server-034b330f-5b75-605f-42e5-7d12903aaef4: entering follower state: follower="Node at 034b330f-5b75-605f-42e5-7d12903aaef4 [Follower]" leader-address= leader-id= +2025-06-03T23:37:44.613Z [INFO] server-99ec51f8-fc52-3f4b-3e9d-45f8369d026b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-034b330f-5b75-605f-42e5-7d12903aaef4 Address:034b330f-5b75-605f-42e5-7d12903aaef4} {Suffrage:Voter ID:server-99ec51f8-fc52-3f4b-3e9d-45f8369d026b Address:99ec51f8-fc52-3f4b-3e9d-45f8369d026b} {Suffrage:Voter ID:server-0caff0b1-0659-776c-45a0-47cef239716f Address:0caff0b1-0659-776c-45a0-47cef239716f}]" +2025-06-03T23:37:44.613Z [INFO] server-99ec51f8-fc52-3f4b-3e9d-45f8369d026b: entering follower state: follower="Node at 99ec51f8-fc52-3f4b-3e9d-45f8369d026b [Follower]" leader-address= leader-id= +2025-06-03T23:37:44.613Z [INFO] server-0caff0b1-0659-776c-45a0-47cef239716f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-034b330f-5b75-605f-42e5-7d12903aaef4 Address:034b330f-5b75-605f-42e5-7d12903aaef4} {Suffrage:Voter ID:server-99ec51f8-fc52-3f4b-3e9d-45f8369d026b Address:99ec51f8-fc52-3f4b-3e9d-45f8369d026b} {Suffrage:Voter ID:server-0caff0b1-0659-776c-45a0-47cef239716f Address:0caff0b1-0659-776c-45a0-47cef239716f}]" +2025-06-03T23:37:44.613Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:44.613Z [INFO] server-0caff0b1-0659-776c-45a0-47cef239716f: entering follower state: follower="Node at 0caff0b1-0659-776c-45a0-47cef239716f [Follower]" leader-address= leader-id= +2025-06-03T23:37:44.673Z [WARN] server-0caff0b1-0659-776c-45a0-47cef239716f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:44.673Z [INFO] server-0caff0b1-0659-776c-45a0-47cef239716f: entering candidate state: node="Node at 0caff0b1-0659-776c-45a0-47cef239716f [Candidate]" term=2 +2025-06-03T23:37:44.673Z [INFO] server-0caff0b1-0659-776c-45a0-47cef239716f: election won: term=2 tally=2 +2025-06-03T23:37:44.673Z [INFO] server-0caff0b1-0659-776c-45a0-47cef239716f: entering leader state: leader="Node at 0caff0b1-0659-776c-45a0-47cef239716f [Leader]" +2025-06-03T23:37:44.673Z [INFO] server-0caff0b1-0659-776c-45a0-47cef239716f: added peer, starting replication: peer=server-034b330f-5b75-605f-42e5-7d12903aaef4 +2025-06-03T23:37:44.673Z [INFO] server-0caff0b1-0659-776c-45a0-47cef239716f: added peer, starting replication: peer=server-99ec51f8-fc52-3f4b-3e9d-45f8369d026b +2025-06-03T23:37:44.673Z [INFO] server-0caff0b1-0659-776c-45a0-47cef239716f: pipelining replication: peer="{Voter server-99ec51f8-fc52-3f4b-3e9d-45f8369d026b 99ec51f8-fc52-3f4b-3e9d-45f8369d026b}" +2025-06-03T23:37:44.673Z [INFO] server-0caff0b1-0659-776c-45a0-47cef239716f: pipelining replication: peer="{Voter server-034b330f-5b75-605f-42e5-7d12903aaef4 034b330f-5b75-605f-42e5-7d12903aaef4}" +2025-06-03T23:37:44.779Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:44.673313991 +0000 UTC m=+99.995069001 (2 nodes), 60.00708ms from start of poll, 60.45772ms from cluster start. Timeout at 2025-06-03 23:37:44.779290071 +0000 UTC m=+100.101045001, 105.976ms after stability +2025-06-03T23:37:44.779Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:44.884Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:44.779467511 +0000 UTC m=+100.101222441 (1 nodes), 1.04µs from start of poll, 166.61116ms from cluster start. Timeout at 2025-06-03 23:37:44.884471111 +0000 UTC m=+100.206226041, 105.0036ms after stability +2025-06-03T23:37:44.884Z [WARN] server-034b330f-5b75-605f-42e5-7d12903aaef4: clearing log suffix: from=1 to=2 +2025-06-03T23:37:44.884Z [WARN] server-034b330f-5b75-605f-42e5-7d12903aaef4: skipping application of old log: index=1 +2025-06-03T23:37:44.884Z [WARN] server-034b330f-5b75-605f-42e5-7d12903aaef4: skipping application of old log: index=1 +2025-06-03T23:37:44.884Z [INFO] server-0caff0b1-0659-776c-45a0-47cef239716f: aborting pipeline replication: peer="{Voter server-034b330f-5b75-605f-42e5-7d12903aaef4 034b330f-5b75-605f-42e5-7d12903aaef4}" +2025-06-03T23:37:44.884Z [INFO] server-0caff0b1-0659-776c-45a0-47cef239716f: aborting pipeline replication: peer="{Voter server-99ec51f8-fc52-3f4b-3e9d-45f8369d026b 99ec51f8-fc52-3f4b-3e9d-45f8369d026b}" +--- PASS: TestRaft_AppendEntry (0.27s) === RUN TestRaft_VotingGrant_WhenLeaderAvailable -2025-06-03T23:34:46.563Z [INFO] server-219f6233-ba38-6bcc-1f82-83b3cb7323aa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-219f6233-ba38-6bcc-1f82-83b3cb7323aa Address:219f6233-ba38-6bcc-1f82-83b3cb7323aa} {Suffrage:Voter ID:server-72db11ed-4930-7068-78ac-f2a868eecb27 Address:72db11ed-4930-7068-78ac-f2a868eecb27} {Suffrage:Voter ID:server-a618bce8-dab7-8915-1711-c162f00236f4 Address:a618bce8-dab7-8915-1711-c162f00236f4}]" -2025-06-03T23:34:46.563Z [INFO] server-219f6233-ba38-6bcc-1f82-83b3cb7323aa: entering follower state: follower="Node at 219f6233-ba38-6bcc-1f82-83b3cb7323aa [Follower]" leader-address= leader-id= -2025-06-03T23:34:46.563Z [INFO] server-72db11ed-4930-7068-78ac-f2a868eecb27: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-219f6233-ba38-6bcc-1f82-83b3cb7323aa Address:219f6233-ba38-6bcc-1f82-83b3cb7323aa} {Suffrage:Voter ID:server-72db11ed-4930-7068-78ac-f2a868eecb27 Address:72db11ed-4930-7068-78ac-f2a868eecb27} {Suffrage:Voter ID:server-a618bce8-dab7-8915-1711-c162f00236f4 Address:a618bce8-dab7-8915-1711-c162f00236f4}]" -2025-06-03T23:34:46.563Z [INFO] server-72db11ed-4930-7068-78ac-f2a868eecb27: entering follower state: follower="Node at 72db11ed-4930-7068-78ac-f2a868eecb27 [Follower]" leader-address= leader-id= -2025-06-03T23:34:46.563Z [INFO] server-a618bce8-dab7-8915-1711-c162f00236f4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-219f6233-ba38-6bcc-1f82-83b3cb7323aa Address:219f6233-ba38-6bcc-1f82-83b3cb7323aa} {Suffrage:Voter ID:server-72db11ed-4930-7068-78ac-f2a868eecb27 Address:72db11ed-4930-7068-78ac-f2a868eecb27} {Suffrage:Voter ID:server-a618bce8-dab7-8915-1711-c162f00236f4 Address:a618bce8-dab7-8915-1711-c162f00236f4}]" -2025-06-03T23:34:46.563Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:46.563Z [INFO] server-a618bce8-dab7-8915-1711-c162f00236f4: entering follower state: follower="Node at a618bce8-dab7-8915-1711-c162f00236f4 [Follower]" leader-address= leader-id= -2025-06-03T23:34:46.615Z [WARN] server-72db11ed-4930-7068-78ac-f2a868eecb27: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:46.615Z [INFO] server-72db11ed-4930-7068-78ac-f2a868eecb27: entering candidate state: node="Node at 72db11ed-4930-7068-78ac-f2a868eecb27 [Candidate]" term=2 -2025-06-03T23:34:46.615Z [INFO] server-72db11ed-4930-7068-78ac-f2a868eecb27: election won: term=2 tally=2 -2025-06-03T23:34:46.615Z [INFO] server-72db11ed-4930-7068-78ac-f2a868eecb27: entering leader state: leader="Node at 72db11ed-4930-7068-78ac-f2a868eecb27 [Leader]" -2025-06-03T23:34:46.615Z [INFO] server-72db11ed-4930-7068-78ac-f2a868eecb27: added peer, starting replication: peer=server-219f6233-ba38-6bcc-1f82-83b3cb7323aa -2025-06-03T23:34:46.615Z [INFO] server-72db11ed-4930-7068-78ac-f2a868eecb27: added peer, starting replication: peer=server-a618bce8-dab7-8915-1711-c162f00236f4 -2025-06-03T23:34:46.615Z [INFO] server-72db11ed-4930-7068-78ac-f2a868eecb27: pipelining replication: peer="{Voter server-219f6233-ba38-6bcc-1f82-83b3cb7323aa 219f6233-ba38-6bcc-1f82-83b3cb7323aa}" -2025-06-03T23:34:46.615Z [INFO] server-72db11ed-4930-7068-78ac-f2a868eecb27: pipelining replication: peer="{Voter server-a618bce8-dab7-8915-1711-c162f00236f4 a618bce8-dab7-8915-1711-c162f00236f4}" -2025-06-03T23:34:46.720Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:46.615457551 +0000 UTC m=+101.636276921 (2 nodes), 51.88924ms from start of poll, 52.38512ms from cluster start. Timeout at 2025-06-03 23:34:46.720518551 +0000 UTC m=+101.741337921, 105.061ms after stability -2025-06-03T23:34:46.720Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:46.825Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:46.720649911 +0000 UTC m=+101.741469281 (1 nodes), 3.24µs from start of poll, 157.57748ms from cluster start. Timeout at 2025-06-03 23:34:46.825715791 +0000 UTC m=+101.846535161, 105.06588ms after stability -2025-06-03T23:34:46.825Z [WARN] server-a618bce8-dab7-8915-1711-c162f00236f4: rejecting vote request since we have a leader: from=219f6233-ba38-6bcc-1f82-83b3cb7323aa leader=72db11ed-4930-7068-78ac-f2a868eecb27 leader-id=server-72db11ed-4930-7068-78ac-f2a868eecb27 -2025-06-03T23:34:46.825Z [INFO] server-72db11ed-4930-7068-78ac-f2a868eecb27: aborting pipeline replication: peer="{Voter server-219f6233-ba38-6bcc-1f82-83b3cb7323aa 219f6233-ba38-6bcc-1f82-83b3cb7323aa}" -2025-06-03T23:34:46.825Z [INFO] server-72db11ed-4930-7068-78ac-f2a868eecb27: aborting pipeline replication: peer="{Voter server-a618bce8-dab7-8915-1711-c162f00236f4 a618bce8-dab7-8915-1711-c162f00236f4}" ---- PASS: TestRaft_VotingGrant_WhenLeaderAvailable (0.26s) +2025-06-03T23:37:44.885Z [INFO] server-41e8cdf4-046f-1ae9-d001-0a62c76bc03a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-41e8cdf4-046f-1ae9-d001-0a62c76bc03a Address:41e8cdf4-046f-1ae9-d001-0a62c76bc03a} {Suffrage:Voter ID:server-9470458e-23d0-852f-32f7-b79eb12f8114 Address:9470458e-23d0-852f-32f7-b79eb12f8114} {Suffrage:Voter ID:server-9b804acd-2a8f-4ffb-10fa-78ceb814b40e Address:9b804acd-2a8f-4ffb-10fa-78ceb814b40e}]" +2025-06-03T23:37:44.885Z [INFO] server-41e8cdf4-046f-1ae9-d001-0a62c76bc03a: entering follower state: follower="Node at 41e8cdf4-046f-1ae9-d001-0a62c76bc03a [Follower]" leader-address= leader-id= +2025-06-03T23:37:44.885Z [INFO] server-9470458e-23d0-852f-32f7-b79eb12f8114: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-41e8cdf4-046f-1ae9-d001-0a62c76bc03a Address:41e8cdf4-046f-1ae9-d001-0a62c76bc03a} {Suffrage:Voter ID:server-9470458e-23d0-852f-32f7-b79eb12f8114 Address:9470458e-23d0-852f-32f7-b79eb12f8114} {Suffrage:Voter ID:server-9b804acd-2a8f-4ffb-10fa-78ceb814b40e Address:9b804acd-2a8f-4ffb-10fa-78ceb814b40e}]" +2025-06-03T23:37:44.885Z [INFO] server-9470458e-23d0-852f-32f7-b79eb12f8114: entering follower state: follower="Node at 9470458e-23d0-852f-32f7-b79eb12f8114 [Follower]" leader-address= leader-id= +2025-06-03T23:37:44.885Z [INFO] server-9b804acd-2a8f-4ffb-10fa-78ceb814b40e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-41e8cdf4-046f-1ae9-d001-0a62c76bc03a Address:41e8cdf4-046f-1ae9-d001-0a62c76bc03a} {Suffrage:Voter ID:server-9470458e-23d0-852f-32f7-b79eb12f8114 Address:9470458e-23d0-852f-32f7-b79eb12f8114} {Suffrage:Voter ID:server-9b804acd-2a8f-4ffb-10fa-78ceb814b40e Address:9b804acd-2a8f-4ffb-10fa-78ceb814b40e}]" +2025-06-03T23:37:44.885Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:44.885Z [INFO] server-9b804acd-2a8f-4ffb-10fa-78ceb814b40e: entering follower state: follower="Node at 9b804acd-2a8f-4ffb-10fa-78ceb814b40e [Follower]" leader-address= leader-id= +2025-06-03T23:37:44.943Z [WARN] server-9470458e-23d0-852f-32f7-b79eb12f8114: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:44.943Z [INFO] server-9470458e-23d0-852f-32f7-b79eb12f8114: entering candidate state: node="Node at 9470458e-23d0-852f-32f7-b79eb12f8114 [Candidate]" term=2 +2025-06-03T23:37:44.943Z [INFO] server-9470458e-23d0-852f-32f7-b79eb12f8114: election won: term=2 tally=2 +2025-06-03T23:37:44.943Z [INFO] server-9470458e-23d0-852f-32f7-b79eb12f8114: entering leader state: leader="Node at 9470458e-23d0-852f-32f7-b79eb12f8114 [Leader]" +2025-06-03T23:37:44.943Z [INFO] server-9470458e-23d0-852f-32f7-b79eb12f8114: added peer, starting replication: peer=server-41e8cdf4-046f-1ae9-d001-0a62c76bc03a +2025-06-03T23:37:44.943Z [INFO] server-9470458e-23d0-852f-32f7-b79eb12f8114: added peer, starting replication: peer=server-9b804acd-2a8f-4ffb-10fa-78ceb814b40e +2025-06-03T23:37:44.943Z [INFO] server-9470458e-23d0-852f-32f7-b79eb12f8114: pipelining replication: peer="{Voter server-9b804acd-2a8f-4ffb-10fa-78ceb814b40e 9b804acd-2a8f-4ffb-10fa-78ceb814b40e}" +2025-06-03T23:37:44.943Z [INFO] server-9470458e-23d0-852f-32f7-b79eb12f8114: pipelining replication: peer="{Voter server-41e8cdf4-046f-1ae9-d001-0a62c76bc03a 41e8cdf4-046f-1ae9-d001-0a62c76bc03a}" +2025-06-03T23:37:45.048Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:44.943476471 +0000 UTC m=+100.265231401 (2 nodes), 57.48956ms from start of poll, 57.95532ms from cluster start. Timeout at 2025-06-03 23:37:45.048488831 +0000 UTC m=+100.370243801, 105.0124ms after stability +2025-06-03T23:37:45.048Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:45.154Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:45.048626671 +0000 UTC m=+100.370381641 (1 nodes), 3.48µs from start of poll, 163.10556ms from cluster start. Timeout at 2025-06-03 23:37:45.153984071 +0000 UTC m=+100.475739001, 105.35736ms after stability +2025-06-03T23:37:45.154Z [WARN] server-9b804acd-2a8f-4ffb-10fa-78ceb814b40e: rejecting vote request since we have a leader: from=41e8cdf4-046f-1ae9-d001-0a62c76bc03a leader=9470458e-23d0-852f-32f7-b79eb12f8114 leader-id=server-9470458e-23d0-852f-32f7-b79eb12f8114 +2025-06-03T23:37:45.154Z [INFO] server-9470458e-23d0-852f-32f7-b79eb12f8114: aborting pipeline replication: peer="{Voter server-9b804acd-2a8f-4ffb-10fa-78ceb814b40e 9b804acd-2a8f-4ffb-10fa-78ceb814b40e}" +2025-06-03T23:37:45.154Z [INFO] server-9470458e-23d0-852f-32f7-b79eb12f8114: aborting pipeline replication: peer="{Voter server-41e8cdf4-046f-1ae9-d001-0a62c76bc03a 41e8cdf4-046f-1ae9-d001-0a62c76bc03a}" +--- PASS: TestRaft_VotingGrant_WhenLeaderAvailable (0.27s) === RUN TestRaft_ProtocolVersion_RejectRPC -2025-06-03T23:34:46.826Z [INFO] server-79546ffb-e872-0309-af31-386b36f8e426: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-79546ffb-e872-0309-af31-386b36f8e426 Address:79546ffb-e872-0309-af31-386b36f8e426} {Suffrage:Voter ID:server-f82f7bf8-67bb-93d4-68fa-02c265ad0aa3 Address:f82f7bf8-67bb-93d4-68fa-02c265ad0aa3} {Suffrage:Voter ID:server-10612a6a-299a-4898-0582-61ded40ebd0c Address:10612a6a-299a-4898-0582-61ded40ebd0c}]" -2025-06-03T23:34:46.827Z [INFO] server-79546ffb-e872-0309-af31-386b36f8e426: entering follower state: follower="Node at 79546ffb-e872-0309-af31-386b36f8e426 [Follower]" leader-address= leader-id= -2025-06-03T23:34:46.827Z [INFO] server-f82f7bf8-67bb-93d4-68fa-02c265ad0aa3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-79546ffb-e872-0309-af31-386b36f8e426 Address:79546ffb-e872-0309-af31-386b36f8e426} {Suffrage:Voter ID:server-f82f7bf8-67bb-93d4-68fa-02c265ad0aa3 Address:f82f7bf8-67bb-93d4-68fa-02c265ad0aa3} {Suffrage:Voter ID:server-10612a6a-299a-4898-0582-61ded40ebd0c Address:10612a6a-299a-4898-0582-61ded40ebd0c}]" -2025-06-03T23:34:46.827Z [INFO] server-f82f7bf8-67bb-93d4-68fa-02c265ad0aa3: entering follower state: follower="Node at f82f7bf8-67bb-93d4-68fa-02c265ad0aa3 [Follower]" leader-address= leader-id= -2025-06-03T23:34:46.827Z [INFO] server-10612a6a-299a-4898-0582-61ded40ebd0c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-79546ffb-e872-0309-af31-386b36f8e426 Address:79546ffb-e872-0309-af31-386b36f8e426} {Suffrage:Voter ID:server-f82f7bf8-67bb-93d4-68fa-02c265ad0aa3 Address:f82f7bf8-67bb-93d4-68fa-02c265ad0aa3} {Suffrage:Voter ID:server-10612a6a-299a-4898-0582-61ded40ebd0c Address:10612a6a-299a-4898-0582-61ded40ebd0c}]" -2025-06-03T23:34:46.827Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:46.827Z [INFO] server-10612a6a-299a-4898-0582-61ded40ebd0c: entering follower state: follower="Node at 10612a6a-299a-4898-0582-61ded40ebd0c [Follower]" leader-address= leader-id= -2025-06-03T23:34:46.889Z [WARN] server-f82f7bf8-67bb-93d4-68fa-02c265ad0aa3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:46.889Z [INFO] server-f82f7bf8-67bb-93d4-68fa-02c265ad0aa3: entering candidate state: node="Node at f82f7bf8-67bb-93d4-68fa-02c265ad0aa3 [Candidate]" term=2 -2025-06-03T23:34:46.889Z [INFO] server-f82f7bf8-67bb-93d4-68fa-02c265ad0aa3: election won: term=2 tally=2 -2025-06-03T23:34:46.889Z [INFO] server-f82f7bf8-67bb-93d4-68fa-02c265ad0aa3: entering leader state: leader="Node at f82f7bf8-67bb-93d4-68fa-02c265ad0aa3 [Leader]" -2025-06-03T23:34:46.889Z [INFO] server-f82f7bf8-67bb-93d4-68fa-02c265ad0aa3: added peer, starting replication: peer=server-79546ffb-e872-0309-af31-386b36f8e426 -2025-06-03T23:34:46.889Z [INFO] server-f82f7bf8-67bb-93d4-68fa-02c265ad0aa3: added peer, starting replication: peer=server-10612a6a-299a-4898-0582-61ded40ebd0c -2025-06-03T23:34:46.889Z [INFO] server-f82f7bf8-67bb-93d4-68fa-02c265ad0aa3: pipelining replication: peer="{Voter server-79546ffb-e872-0309-af31-386b36f8e426 79546ffb-e872-0309-af31-386b36f8e426}" -2025-06-03T23:34:46.889Z [INFO] server-f82f7bf8-67bb-93d4-68fa-02c265ad0aa3: pipelining replication: peer="{Voter server-10612a6a-299a-4898-0582-61ded40ebd0c 10612a6a-299a-4898-0582-61ded40ebd0c}" -2025-06-03T23:34:46.994Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:46.889633231 +0000 UTC m=+101.910452601 (2 nodes), 62.40172ms from start of poll, 62.85612ms from cluster start. Timeout at 2025-06-03 23:34:46.994939751 +0000 UTC m=+102.015759121, 105.30652ms after stability -2025-06-03T23:34:46.995Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:47.100Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:46.995057391 +0000 UTC m=+102.015876721 (1 nodes), 2.68µs from start of poll, 168.28024ms from cluster start. Timeout at 2025-06-03 23:34:47.100319471 +0000 UTC m=+102.121138841, 105.26212ms after stability -2025-06-03T23:34:47.100Z [INFO] server-f82f7bf8-67bb-93d4-68fa-02c265ad0aa3: aborting pipeline replication: peer="{Voter server-10612a6a-299a-4898-0582-61ded40ebd0c 10612a6a-299a-4898-0582-61ded40ebd0c}" -2025-06-03T23:34:47.100Z [INFO] server-f82f7bf8-67bb-93d4-68fa-02c265ad0aa3: aborting pipeline replication: peer="{Voter server-79546ffb-e872-0309-af31-386b36f8e426 79546ffb-e872-0309-af31-386b36f8e426}" ---- PASS: TestRaft_ProtocolVersion_RejectRPC (0.27s) +2025-06-03T23:37:45.155Z [INFO] server-3faf13be-513d-1daf-847c-2df61490403d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3faf13be-513d-1daf-847c-2df61490403d Address:3faf13be-513d-1daf-847c-2df61490403d} {Suffrage:Voter ID:server-b256fdfa-703e-8096-401a-555eaef48063 Address:b256fdfa-703e-8096-401a-555eaef48063} {Suffrage:Voter ID:server-cd7f7bec-a1d0-c8fc-8ec3-6ceb5dff41c0 Address:cd7f7bec-a1d0-c8fc-8ec3-6ceb5dff41c0}]" +2025-06-03T23:37:45.155Z [INFO] server-3faf13be-513d-1daf-847c-2df61490403d: entering follower state: follower="Node at 3faf13be-513d-1daf-847c-2df61490403d [Follower]" leader-address= leader-id= +2025-06-03T23:37:45.155Z [INFO] server-b256fdfa-703e-8096-401a-555eaef48063: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3faf13be-513d-1daf-847c-2df61490403d Address:3faf13be-513d-1daf-847c-2df61490403d} {Suffrage:Voter ID:server-b256fdfa-703e-8096-401a-555eaef48063 Address:b256fdfa-703e-8096-401a-555eaef48063} {Suffrage:Voter ID:server-cd7f7bec-a1d0-c8fc-8ec3-6ceb5dff41c0 Address:cd7f7bec-a1d0-c8fc-8ec3-6ceb5dff41c0}]" +2025-06-03T23:37:45.155Z [INFO] server-b256fdfa-703e-8096-401a-555eaef48063: entering follower state: follower="Node at b256fdfa-703e-8096-401a-555eaef48063 [Follower]" leader-address= leader-id= +2025-06-03T23:37:45.155Z [INFO] server-cd7f7bec-a1d0-c8fc-8ec3-6ceb5dff41c0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3faf13be-513d-1daf-847c-2df61490403d Address:3faf13be-513d-1daf-847c-2df61490403d} {Suffrage:Voter ID:server-b256fdfa-703e-8096-401a-555eaef48063 Address:b256fdfa-703e-8096-401a-555eaef48063} {Suffrage:Voter ID:server-cd7f7bec-a1d0-c8fc-8ec3-6ceb5dff41c0 Address:cd7f7bec-a1d0-c8fc-8ec3-6ceb5dff41c0}]" +2025-06-03T23:37:45.155Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:45.155Z [INFO] server-cd7f7bec-a1d0-c8fc-8ec3-6ceb5dff41c0: entering follower state: follower="Node at cd7f7bec-a1d0-c8fc-8ec3-6ceb5dff41c0 [Follower]" leader-address= leader-id= +2025-06-03T23:37:45.230Z [WARN] server-3faf13be-513d-1daf-847c-2df61490403d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:45.230Z [INFO] server-3faf13be-513d-1daf-847c-2df61490403d: entering candidate state: node="Node at 3faf13be-513d-1daf-847c-2df61490403d [Candidate]" term=2 +2025-06-03T23:37:45.230Z [INFO] server-3faf13be-513d-1daf-847c-2df61490403d: election won: term=2 tally=2 +2025-06-03T23:37:45.230Z [INFO] server-3faf13be-513d-1daf-847c-2df61490403d: entering leader state: leader="Node at 3faf13be-513d-1daf-847c-2df61490403d [Leader]" +2025-06-03T23:37:45.230Z [INFO] server-3faf13be-513d-1daf-847c-2df61490403d: added peer, starting replication: peer=server-b256fdfa-703e-8096-401a-555eaef48063 +2025-06-03T23:37:45.230Z [INFO] server-3faf13be-513d-1daf-847c-2df61490403d: added peer, starting replication: peer=server-cd7f7bec-a1d0-c8fc-8ec3-6ceb5dff41c0 +2025-06-03T23:37:45.230Z [INFO] server-3faf13be-513d-1daf-847c-2df61490403d: pipelining replication: peer="{Voter server-cd7f7bec-a1d0-c8fc-8ec3-6ceb5dff41c0 cd7f7bec-a1d0-c8fc-8ec3-6ceb5dff41c0}" +2025-06-03T23:37:45.231Z [INFO] server-3faf13be-513d-1daf-847c-2df61490403d: pipelining replication: peer="{Voter server-b256fdfa-703e-8096-401a-555eaef48063 b256fdfa-703e-8096-401a-555eaef48063}" +2025-06-03T23:37:45.335Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:45.230822151 +0000 UTC m=+100.552577081 (2 nodes), 75.33132ms from start of poll, 75.82184ms from cluster start. Timeout at 2025-06-03 23:37:45.335871911 +0000 UTC m=+100.657626841, 105.04976ms after stability +2025-06-03T23:37:45.335Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:45.441Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:45.335969071 +0000 UTC m=+100.657724001 (1 nodes), 2.76µs from start of poll, 180.96876ms from cluster start. Timeout at 2025-06-03 23:37:45.441125591 +0000 UTC m=+100.762880521, 105.15652ms after stability +2025-06-03T23:37:45.441Z [INFO] server-3faf13be-513d-1daf-847c-2df61490403d: aborting pipeline replication: peer="{Voter server-cd7f7bec-a1d0-c8fc-8ec3-6ceb5dff41c0 cd7f7bec-a1d0-c8fc-8ec3-6ceb5dff41c0}" +2025-06-03T23:37:45.441Z [INFO] server-3faf13be-513d-1daf-847c-2df61490403d: aborting pipeline replication: peer="{Voter server-b256fdfa-703e-8096-401a-555eaef48063 b256fdfa-703e-8096-401a-555eaef48063}" +--- PASS: TestRaft_ProtocolVersion_RejectRPC (0.29s) === RUN TestRaft_ProtocolVersion_Upgrade_1_2 raft_test.go:2151: Skipping unreliable test --- SKIP: TestRaft_ProtocolVersion_Upgrade_1_2 (0.00s) @@ -3523,267 +3583,267 @@ raft_test.go:2196: Skipping unreliable test --- SKIP: TestRaft_ProtocolVersion_Upgrade_2_3 (0.00s) === RUN TestRaft_LeaderID_Propagated -2025-06-03T23:34:47.101Z [INFO] server-95bbfc8c-f93c-4d29-1728-820aea8a17e8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-95bbfc8c-f93c-4d29-1728-820aea8a17e8 Address:95bbfc8c-f93c-4d29-1728-820aea8a17e8} {Suffrage:Voter ID:server-b7e03214-ddec-3100-6212-7c6e7f261468 Address:b7e03214-ddec-3100-6212-7c6e7f261468} {Suffrage:Voter ID:server-9e5991aa-7fa2-e2eb-3a44-dbdd502dca8f Address:9e5991aa-7fa2-e2eb-3a44-dbdd502dca8f}]" -2025-06-03T23:34:47.101Z [INFO] server-95bbfc8c-f93c-4d29-1728-820aea8a17e8: entering follower state: follower="Node at 95bbfc8c-f93c-4d29-1728-820aea8a17e8 [Follower]" leader-address= leader-id= -2025-06-03T23:34:47.101Z [INFO] server-b7e03214-ddec-3100-6212-7c6e7f261468: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-95bbfc8c-f93c-4d29-1728-820aea8a17e8 Address:95bbfc8c-f93c-4d29-1728-820aea8a17e8} {Suffrage:Voter ID:server-b7e03214-ddec-3100-6212-7c6e7f261468 Address:b7e03214-ddec-3100-6212-7c6e7f261468} {Suffrage:Voter ID:server-9e5991aa-7fa2-e2eb-3a44-dbdd502dca8f Address:9e5991aa-7fa2-e2eb-3a44-dbdd502dca8f}]" -2025-06-03T23:34:47.101Z [INFO] server-b7e03214-ddec-3100-6212-7c6e7f261468: entering follower state: follower="Node at b7e03214-ddec-3100-6212-7c6e7f261468 [Follower]" leader-address= leader-id= -2025-06-03T23:34:47.101Z [INFO] server-9e5991aa-7fa2-e2eb-3a44-dbdd502dca8f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-95bbfc8c-f93c-4d29-1728-820aea8a17e8 Address:95bbfc8c-f93c-4d29-1728-820aea8a17e8} {Suffrage:Voter ID:server-b7e03214-ddec-3100-6212-7c6e7f261468 Address:b7e03214-ddec-3100-6212-7c6e7f261468} {Suffrage:Voter ID:server-9e5991aa-7fa2-e2eb-3a44-dbdd502dca8f Address:9e5991aa-7fa2-e2eb-3a44-dbdd502dca8f}]" -2025-06-03T23:34:47.101Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:47.101Z [INFO] server-9e5991aa-7fa2-e2eb-3a44-dbdd502dca8f: entering follower state: follower="Node at 9e5991aa-7fa2-e2eb-3a44-dbdd502dca8f [Follower]" leader-address= leader-id= -2025-06-03T23:34:47.161Z [WARN] server-b7e03214-ddec-3100-6212-7c6e7f261468: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:47.162Z [INFO] server-b7e03214-ddec-3100-6212-7c6e7f261468: entering candidate state: node="Node at b7e03214-ddec-3100-6212-7c6e7f261468 [Candidate]" term=2 -2025-06-03T23:34:47.162Z [INFO] server-b7e03214-ddec-3100-6212-7c6e7f261468: election won: term=2 tally=2 -2025-06-03T23:34:47.162Z [INFO] server-b7e03214-ddec-3100-6212-7c6e7f261468: entering leader state: leader="Node at b7e03214-ddec-3100-6212-7c6e7f261468 [Leader]" -2025-06-03T23:34:47.162Z [INFO] server-b7e03214-ddec-3100-6212-7c6e7f261468: added peer, starting replication: peer=server-95bbfc8c-f93c-4d29-1728-820aea8a17e8 -2025-06-03T23:34:47.162Z [INFO] server-b7e03214-ddec-3100-6212-7c6e7f261468: added peer, starting replication: peer=server-9e5991aa-7fa2-e2eb-3a44-dbdd502dca8f -2025-06-03T23:34:47.162Z [INFO] server-b7e03214-ddec-3100-6212-7c6e7f261468: pipelining replication: peer="{Voter server-95bbfc8c-f93c-4d29-1728-820aea8a17e8 95bbfc8c-f93c-4d29-1728-820aea8a17e8}" -2025-06-03T23:34:47.162Z [INFO] server-b7e03214-ddec-3100-6212-7c6e7f261468: pipelining replication: peer="{Voter server-9e5991aa-7fa2-e2eb-3a44-dbdd502dca8f 9e5991aa-7fa2-e2eb-3a44-dbdd502dca8f}" -2025-06-03T23:34:47.267Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:47.162187671 +0000 UTC m=+102.183007041 (1 nodes), 60.22924ms from start of poll, 60.69912ms from cluster start. Timeout at 2025-06-03 23:34:47.267855831 +0000 UTC m=+102.288675201, 105.66816ms after stability -2025-06-03T23:34:47.267Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:47.373Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:47.267987471 +0000 UTC m=+102.288806841 (1 nodes), 1.04µs from start of poll, 166.49892ms from cluster start. Timeout at 2025-06-03 23:34:47.373370911 +0000 UTC m=+102.394190281, 105.38344ms after stability -2025-06-03T23:34:47.373Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:47.479Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:47.373641511 +0000 UTC m=+102.394460881 (1 nodes), 1.12µs from start of poll, 272.15296ms from cluster start. Timeout at 2025-06-03 23:34:47.479461671 +0000 UTC m=+102.500281041, 105.82016ms after stability -2025-06-03T23:34:47.479Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:47.584Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:47.479744111 +0000 UTC m=+102.500563441 (1 nodes), 1.36µs from start of poll, 378.25552ms from cluster start. Timeout at 2025-06-03 23:34:47.584771871 +0000 UTC m=+102.605591241, 105.0278ms after stability -2025-06-03T23:34:47.585Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:47.691Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:47.585133791 +0000 UTC m=+102.605953161 (1 nodes), 1.08µs from start of poll, 483.64524ms from cluster start. Timeout at 2025-06-03 23:34:47.691134151 +0000 UTC m=+102.711953521, 106.00036ms after stability -2025-06-03T23:34:47.691Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:47.796Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:47.691597271 +0000 UTC m=+102.712416641 (1 nodes), 1.12µs from start of poll, 590.10872ms from cluster start. Timeout at 2025-06-03 23:34:47.796672791 +0000 UTC m=+102.817492161, 105.07552ms after stability -2025-06-03T23:34:47.901Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:48.007Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:47.902041311 +0000 UTC m=+102.922860721 (1 nodes), 1.08µs from start of poll, 800.5528ms from cluster start. Timeout at 2025-06-03 23:34:48.007753071 +0000 UTC m=+103.028572441, 105.71172ms after stability -2025-06-03T23:34:48.007Z [INFO] server-b7e03214-ddec-3100-6212-7c6e7f261468: aborting pipeline replication: peer="{Voter server-9e5991aa-7fa2-e2eb-3a44-dbdd502dca8f 9e5991aa-7fa2-e2eb-3a44-dbdd502dca8f}" -2025-06-03T23:34:48.007Z [INFO] server-b7e03214-ddec-3100-6212-7c6e7f261468: aborting pipeline replication: peer="{Voter server-95bbfc8c-f93c-4d29-1728-820aea8a17e8 95bbfc8c-f93c-4d29-1728-820aea8a17e8}" +2025-06-03T23:37:45.442Z [INFO] server-ec12d822-46fa-4d27-92d5-f222799c71cd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec12d822-46fa-4d27-92d5-f222799c71cd Address:ec12d822-46fa-4d27-92d5-f222799c71cd} {Suffrage:Voter ID:server-12380fce-8a20-bc1b-14d7-c4789d6e4d95 Address:12380fce-8a20-bc1b-14d7-c4789d6e4d95} {Suffrage:Voter ID:server-7e1375ab-de69-9d4a-6118-92f22c01b70f Address:7e1375ab-de69-9d4a-6118-92f22c01b70f}]" +2025-06-03T23:37:45.442Z [INFO] server-ec12d822-46fa-4d27-92d5-f222799c71cd: entering follower state: follower="Node at ec12d822-46fa-4d27-92d5-f222799c71cd [Follower]" leader-address= leader-id= +2025-06-03T23:37:45.442Z [INFO] server-12380fce-8a20-bc1b-14d7-c4789d6e4d95: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec12d822-46fa-4d27-92d5-f222799c71cd Address:ec12d822-46fa-4d27-92d5-f222799c71cd} {Suffrage:Voter ID:server-12380fce-8a20-bc1b-14d7-c4789d6e4d95 Address:12380fce-8a20-bc1b-14d7-c4789d6e4d95} {Suffrage:Voter ID:server-7e1375ab-de69-9d4a-6118-92f22c01b70f Address:7e1375ab-de69-9d4a-6118-92f22c01b70f}]" +2025-06-03T23:37:45.442Z [INFO] server-12380fce-8a20-bc1b-14d7-c4789d6e4d95: entering follower state: follower="Node at 12380fce-8a20-bc1b-14d7-c4789d6e4d95 [Follower]" leader-address= leader-id= +2025-06-03T23:37:45.442Z [INFO] server-7e1375ab-de69-9d4a-6118-92f22c01b70f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec12d822-46fa-4d27-92d5-f222799c71cd Address:ec12d822-46fa-4d27-92d5-f222799c71cd} {Suffrage:Voter ID:server-12380fce-8a20-bc1b-14d7-c4789d6e4d95 Address:12380fce-8a20-bc1b-14d7-c4789d6e4d95} {Suffrage:Voter ID:server-7e1375ab-de69-9d4a-6118-92f22c01b70f Address:7e1375ab-de69-9d4a-6118-92f22c01b70f}]" +2025-06-03T23:37:45.442Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:45.442Z [INFO] server-7e1375ab-de69-9d4a-6118-92f22c01b70f: entering follower state: follower="Node at 7e1375ab-de69-9d4a-6118-92f22c01b70f [Follower]" leader-address= leader-id= +2025-06-03T23:37:45.508Z [WARN] server-12380fce-8a20-bc1b-14d7-c4789d6e4d95: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:45.508Z [INFO] server-12380fce-8a20-bc1b-14d7-c4789d6e4d95: entering candidate state: node="Node at 12380fce-8a20-bc1b-14d7-c4789d6e4d95 [Candidate]" term=2 +2025-06-03T23:37:45.508Z [INFO] server-12380fce-8a20-bc1b-14d7-c4789d6e4d95: election won: term=2 tally=2 +2025-06-03T23:37:45.508Z [INFO] server-12380fce-8a20-bc1b-14d7-c4789d6e4d95: entering leader state: leader="Node at 12380fce-8a20-bc1b-14d7-c4789d6e4d95 [Leader]" +2025-06-03T23:37:45.508Z [INFO] server-12380fce-8a20-bc1b-14d7-c4789d6e4d95: added peer, starting replication: peer=server-ec12d822-46fa-4d27-92d5-f222799c71cd +2025-06-03T23:37:45.508Z [INFO] server-12380fce-8a20-bc1b-14d7-c4789d6e4d95: added peer, starting replication: peer=server-7e1375ab-de69-9d4a-6118-92f22c01b70f +2025-06-03T23:37:45.508Z [INFO] server-12380fce-8a20-bc1b-14d7-c4789d6e4d95: pipelining replication: peer="{Voter server-7e1375ab-de69-9d4a-6118-92f22c01b70f 7e1375ab-de69-9d4a-6118-92f22c01b70f}" +2025-06-03T23:37:45.508Z [INFO] server-12380fce-8a20-bc1b-14d7-c4789d6e4d95: pipelining replication: peer="{Voter server-ec12d822-46fa-4d27-92d5-f222799c71cd ec12d822-46fa-4d27-92d5-f222799c71cd}" +2025-06-03T23:37:45.613Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:45.508619111 +0000 UTC m=+100.830374041 (1 nodes), 65.7258ms from start of poll, 66.189ms from cluster start. Timeout at 2025-06-03 23:37:45.613675471 +0000 UTC m=+100.935430361, 105.05632ms after stability +2025-06-03T23:37:45.613Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:45.719Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:45.613818111 +0000 UTC m=+100.935573041 (1 nodes), 2.12µs from start of poll, 171.388ms from cluster start. Timeout at 2025-06-03 23:37:45.719877791 +0000 UTC m=+101.041632721, 106.05968ms after stability +2025-06-03T23:37:45.720Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:45.826Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:45.720150911 +0000 UTC m=+101.041905841 (1 nodes), 760ns from start of poll, 277.7208ms from cluster start. Timeout at 2025-06-03 23:37:45.826118631 +0000 UTC m=+101.147873561, 105.96772ms after stability +2025-06-03T23:37:45.826Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:45.931Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:45.826397431 +0000 UTC m=+101.148152361 (1 nodes), 1.16µs from start of poll, 383.96732ms from cluster start. Timeout at 2025-06-03 23:37:45.931492711 +0000 UTC m=+101.253247641, 105.09528ms after stability +2025-06-03T23:37:45.931Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:46.037Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:45.931781711 +0000 UTC m=+101.253536641 (1 nodes), 920ns from start of poll, 489.3516ms from cluster start. Timeout at 2025-06-03 23:37:46.037518551 +0000 UTC m=+101.359273481, 105.73684ms after stability +2025-06-03T23:37:46.037Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:46.143Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:46.037828751 +0000 UTC m=+101.359583681 (1 nodes), 920ns from start of poll, 595.39864ms from cluster start. Timeout at 2025-06-03 23:37:46.143029511 +0000 UTC m=+101.464784441, 105.20076ms after stability +2025-06-03T23:37:46.248Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:46.354Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:46.248406391 +0000 UTC m=+101.570161321 (1 nodes), 2.92µs from start of poll, 805.97628ms from cluster start. Timeout at 2025-06-03 23:37:46.354245031 +0000 UTC m=+101.675999961, 105.83864ms after stability +2025-06-03T23:37:46.354Z [INFO] server-12380fce-8a20-bc1b-14d7-c4789d6e4d95: aborting pipeline replication: peer="{Voter server-ec12d822-46fa-4d27-92d5-f222799c71cd ec12d822-46fa-4d27-92d5-f222799c71cd}" +2025-06-03T23:37:46.354Z [INFO] server-12380fce-8a20-bc1b-14d7-c4789d6e4d95: aborting pipeline replication: peer="{Voter server-7e1375ab-de69-9d4a-6118-92f22c01b70f 7e1375ab-de69-9d4a-6118-92f22c01b70f}" --- PASS: TestRaft_LeaderID_Propagated (0.91s) === RUN TestRaft_LeadershipTransferInProgress --- PASS: TestRaft_LeadershipTransferInProgress (0.00s) === RUN TestRaft_LeadershipTransferPickServer --- PASS: TestRaft_LeadershipTransferPickServer (0.00s) === RUN TestRaft_LeadershipTransfer -2025-06-03T23:34:48.009Z [INFO] server-bc9216b4-95b6-db06-0206-e8401a798ece: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc9216b4-95b6-db06-0206-e8401a798ece Address:bc9216b4-95b6-db06-0206-e8401a798ece} {Suffrage:Voter ID:server-64a1d689-2362-dc0d-6dd2-35819c465392 Address:64a1d689-2362-dc0d-6dd2-35819c465392} {Suffrage:Voter ID:server-9f103c58-619d-eb24-8807-fd53c6e0c803 Address:9f103c58-619d-eb24-8807-fd53c6e0c803}]" -2025-06-03T23:34:48.009Z [INFO] server-bc9216b4-95b6-db06-0206-e8401a798ece: entering follower state: follower="Node at bc9216b4-95b6-db06-0206-e8401a798ece [Follower]" leader-address= leader-id= -2025-06-03T23:34:48.009Z [INFO] server-64a1d689-2362-dc0d-6dd2-35819c465392: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc9216b4-95b6-db06-0206-e8401a798ece Address:bc9216b4-95b6-db06-0206-e8401a798ece} {Suffrage:Voter ID:server-64a1d689-2362-dc0d-6dd2-35819c465392 Address:64a1d689-2362-dc0d-6dd2-35819c465392} {Suffrage:Voter ID:server-9f103c58-619d-eb24-8807-fd53c6e0c803 Address:9f103c58-619d-eb24-8807-fd53c6e0c803}]" -2025-06-03T23:34:48.009Z [INFO] server-64a1d689-2362-dc0d-6dd2-35819c465392: entering follower state: follower="Node at 64a1d689-2362-dc0d-6dd2-35819c465392 [Follower]" leader-address= leader-id= -2025-06-03T23:34:48.009Z [INFO] server-9f103c58-619d-eb24-8807-fd53c6e0c803: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc9216b4-95b6-db06-0206-e8401a798ece Address:bc9216b4-95b6-db06-0206-e8401a798ece} {Suffrage:Voter ID:server-64a1d689-2362-dc0d-6dd2-35819c465392 Address:64a1d689-2362-dc0d-6dd2-35819c465392} {Suffrage:Voter ID:server-9f103c58-619d-eb24-8807-fd53c6e0c803 Address:9f103c58-619d-eb24-8807-fd53c6e0c803}]" -2025-06-03T23:34:48.009Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:48.009Z [INFO] server-9f103c58-619d-eb24-8807-fd53c6e0c803: entering follower state: follower="Node at 9f103c58-619d-eb24-8807-fd53c6e0c803 [Follower]" leader-address= leader-id= -2025-06-03T23:34:48.084Z [WARN] server-bc9216b4-95b6-db06-0206-e8401a798ece: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:48.084Z [INFO] server-bc9216b4-95b6-db06-0206-e8401a798ece: entering candidate state: node="Node at bc9216b4-95b6-db06-0206-e8401a798ece [Candidate]" term=2 -2025-06-03T23:34:48.084Z [INFO] server-bc9216b4-95b6-db06-0206-e8401a798ece: election won: term=2 tally=2 -2025-06-03T23:34:48.084Z [INFO] server-bc9216b4-95b6-db06-0206-e8401a798ece: entering leader state: leader="Node at bc9216b4-95b6-db06-0206-e8401a798ece [Leader]" -2025-06-03T23:34:48.084Z [INFO] server-bc9216b4-95b6-db06-0206-e8401a798ece: added peer, starting replication: peer=server-64a1d689-2362-dc0d-6dd2-35819c465392 -2025-06-03T23:34:48.084Z [INFO] server-bc9216b4-95b6-db06-0206-e8401a798ece: added peer, starting replication: peer=server-9f103c58-619d-eb24-8807-fd53c6e0c803 -2025-06-03T23:34:48.084Z [INFO] server-bc9216b4-95b6-db06-0206-e8401a798ece: pipelining replication: peer="{Voter server-64a1d689-2362-dc0d-6dd2-35819c465392 64a1d689-2362-dc0d-6dd2-35819c465392}" -2025-06-03T23:34:48.084Z [INFO] server-bc9216b4-95b6-db06-0206-e8401a798ece: pipelining replication: peer="{Voter server-9f103c58-619d-eb24-8807-fd53c6e0c803 9f103c58-619d-eb24-8807-fd53c6e0c803}" -2025-06-03T23:34:48.190Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:48.084706751 +0000 UTC m=+103.105526121 (1 nodes), 75.03096ms from start of poll, 75.68496ms from cluster start. Timeout at 2025-06-03 23:34:48.190380991 +0000 UTC m=+103.211200361, 105.67424ms after stability -2025-06-03T23:34:48.190Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:48.295Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:48.190508631 +0000 UTC m=+103.211328001 (1 nodes), 2.6µs from start of poll, 181.48684ms from cluster start. Timeout at 2025-06-03 23:34:48.295566031 +0000 UTC m=+103.316385401, 105.0574ms after stability -2025-06-03T23:34:48.295Z [INFO] server-64a1d689-2362-dc0d-6dd2-35819c465392: entering candidate state: node="Node at 64a1d689-2362-dc0d-6dd2-35819c465392 [Candidate]" term=3 -2025-06-03T23:34:48.295Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:48.295Z [INFO] server-64a1d689-2362-dc0d-6dd2-35819c465392: election won: term=3 tally=2 -2025-06-03T23:34:48.295Z [INFO] server-bc9216b4-95b6-db06-0206-e8401a798ece: entering follower state: follower="Node at bc9216b4-95b6-db06-0206-e8401a798ece [Follower]" leader-address= leader-id= -2025-06-03T23:34:48.295Z [INFO] server-64a1d689-2362-dc0d-6dd2-35819c465392: entering leader state: leader="Node at 64a1d689-2362-dc0d-6dd2-35819c465392 [Leader]" -2025-06-03T23:34:48.295Z [INFO] server-bc9216b4-95b6-db06-0206-e8401a798ece: aborting pipeline replication: peer="{Voter server-9f103c58-619d-eb24-8807-fd53c6e0c803 9f103c58-619d-eb24-8807-fd53c6e0c803}" -2025-06-03T23:34:48.295Z [INFO] server-64a1d689-2362-dc0d-6dd2-35819c465392: added peer, starting replication: peer=server-bc9216b4-95b6-db06-0206-e8401a798ece -2025-06-03T23:34:48.295Z [INFO] server-64a1d689-2362-dc0d-6dd2-35819c465392: added peer, starting replication: peer=server-9f103c58-619d-eb24-8807-fd53c6e0c803 -2025-06-03T23:34:48.295Z [INFO] server-bc9216b4-95b6-db06-0206-e8401a798ece: aborting pipeline replication: peer="{Voter server-64a1d689-2362-dc0d-6dd2-35819c465392 64a1d689-2362-dc0d-6dd2-35819c465392}" -2025-06-03T23:34:48.296Z [INFO] server-64a1d689-2362-dc0d-6dd2-35819c465392: pipelining replication: peer="{Voter server-9f103c58-619d-eb24-8807-fd53c6e0c803 9f103c58-619d-eb24-8807-fd53c6e0c803}" -2025-06-03T23:34:48.296Z [INFO] server-64a1d689-2362-dc0d-6dd2-35819c465392: pipelining replication: peer="{Voter server-bc9216b4-95b6-db06-0206-e8401a798ece bc9216b4-95b6-db06-0206-e8401a798ece}" -2025-06-03T23:34:48.401Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:48.295953351 +0000 UTC m=+103.316772721 (1 nodes), 92.36µs from start of poll, 286.93156ms from cluster start. Timeout at 2025-06-03 23:34:48.401160831 +0000 UTC m=+103.421980201, 105.20748ms after stability -2025-06-03T23:34:48.401Z [INFO] server-64a1d689-2362-dc0d-6dd2-35819c465392: aborting pipeline replication: peer="{Voter server-9f103c58-619d-eb24-8807-fd53c6e0c803 9f103c58-619d-eb24-8807-fd53c6e0c803}" -2025-06-03T23:34:48.401Z [INFO] server-64a1d689-2362-dc0d-6dd2-35819c465392: aborting pipeline replication: peer="{Voter server-bc9216b4-95b6-db06-0206-e8401a798ece bc9216b4-95b6-db06-0206-e8401a798ece}" ---- PASS: TestRaft_LeadershipTransfer (0.39s) +2025-06-03T23:37:46.355Z [INFO] server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b Address:c23337fe-711c-2aa8-0ab1-6f1bb90c572b} {Suffrage:Voter ID:server-f150e749-5f33-7fa0-31d4-18e1539bfe4c Address:f150e749-5f33-7fa0-31d4-18e1539bfe4c} {Suffrage:Voter ID:server-372a35bb-6366-59f0-a23e-2b59084ab356 Address:372a35bb-6366-59f0-a23e-2b59084ab356}]" +2025-06-03T23:37:46.355Z [INFO] server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b: entering follower state: follower="Node at c23337fe-711c-2aa8-0ab1-6f1bb90c572b [Follower]" leader-address= leader-id= +2025-06-03T23:37:46.355Z [INFO] server-f150e749-5f33-7fa0-31d4-18e1539bfe4c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b Address:c23337fe-711c-2aa8-0ab1-6f1bb90c572b} {Suffrage:Voter ID:server-f150e749-5f33-7fa0-31d4-18e1539bfe4c Address:f150e749-5f33-7fa0-31d4-18e1539bfe4c} {Suffrage:Voter ID:server-372a35bb-6366-59f0-a23e-2b59084ab356 Address:372a35bb-6366-59f0-a23e-2b59084ab356}]" +2025-06-03T23:37:46.355Z [INFO] server-f150e749-5f33-7fa0-31d4-18e1539bfe4c: entering follower state: follower="Node at f150e749-5f33-7fa0-31d4-18e1539bfe4c [Follower]" leader-address= leader-id= +2025-06-03T23:37:46.355Z [INFO] server-372a35bb-6366-59f0-a23e-2b59084ab356: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b Address:c23337fe-711c-2aa8-0ab1-6f1bb90c572b} {Suffrage:Voter ID:server-f150e749-5f33-7fa0-31d4-18e1539bfe4c Address:f150e749-5f33-7fa0-31d4-18e1539bfe4c} {Suffrage:Voter ID:server-372a35bb-6366-59f0-a23e-2b59084ab356 Address:372a35bb-6366-59f0-a23e-2b59084ab356}]" +2025-06-03T23:37:46.355Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:46.355Z [INFO] server-372a35bb-6366-59f0-a23e-2b59084ab356: entering follower state: follower="Node at 372a35bb-6366-59f0-a23e-2b59084ab356 [Follower]" leader-address= leader-id= +2025-06-03T23:37:46.440Z [WARN] server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:46.440Z [INFO] server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b: entering candidate state: node="Node at c23337fe-711c-2aa8-0ab1-6f1bb90c572b [Candidate]" term=2 +2025-06-03T23:37:46.440Z [INFO] server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b: election won: term=2 tally=2 +2025-06-03T23:37:46.440Z [INFO] server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b: entering leader state: leader="Node at c23337fe-711c-2aa8-0ab1-6f1bb90c572b [Leader]" +2025-06-03T23:37:46.441Z [INFO] server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b: added peer, starting replication: peer=server-f150e749-5f33-7fa0-31d4-18e1539bfe4c +2025-06-03T23:37:46.441Z [INFO] server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b: added peer, starting replication: peer=server-372a35bb-6366-59f0-a23e-2b59084ab356 +2025-06-03T23:37:46.441Z [INFO] server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b: pipelining replication: peer="{Voter server-f150e749-5f33-7fa0-31d4-18e1539bfe4c f150e749-5f33-7fa0-31d4-18e1539bfe4c}" +2025-06-03T23:37:46.441Z [INFO] server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b: pipelining replication: peer="{Voter server-372a35bb-6366-59f0-a23e-2b59084ab356 372a35bb-6366-59f0-a23e-2b59084ab356}" +2025-06-03T23:37:46.546Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:46.441048031 +0000 UTC m=+101.762802961 (1 nodes), 85.13604ms from start of poll, 85.7686ms from cluster start. Timeout at 2025-06-03 23:37:46.546869591 +0000 UTC m=+101.868624521, 105.82156ms after stability +2025-06-03T23:37:46.546Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:46.652Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:46.546984831 +0000 UTC m=+101.868739761 (1 nodes), 1µs from start of poll, 191.7054ms from cluster start. Timeout at 2025-06-03 23:37:46.652417351 +0000 UTC m=+101.974172281, 105.43252ms after stability +2025-06-03T23:37:46.652Z [INFO] server-f150e749-5f33-7fa0-31d4-18e1539bfe4c: entering candidate state: node="Node at f150e749-5f33-7fa0-31d4-18e1539bfe4c [Candidate]" term=3 +2025-06-03T23:37:46.652Z [INFO] server-f150e749-5f33-7fa0-31d4-18e1539bfe4c: election won: term=3 tally=2 +2025-06-03T23:37:46.652Z [INFO] server-f150e749-5f33-7fa0-31d4-18e1539bfe4c: entering leader state: leader="Node at f150e749-5f33-7fa0-31d4-18e1539bfe4c [Leader]" +2025-06-03T23:37:46.652Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:46.652Z [INFO] server-f150e749-5f33-7fa0-31d4-18e1539bfe4c: added peer, starting replication: peer=server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b +2025-06-03T23:37:46.652Z [INFO] server-f150e749-5f33-7fa0-31d4-18e1539bfe4c: added peer, starting replication: peer=server-372a35bb-6366-59f0-a23e-2b59084ab356 +2025-06-03T23:37:46.652Z [INFO] server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b: entering follower state: follower="Node at c23337fe-711c-2aa8-0ab1-6f1bb90c572b [Follower]" leader-address= leader-id= +2025-06-03T23:37:46.652Z [INFO] server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b: aborting pipeline replication: peer="{Voter server-372a35bb-6366-59f0-a23e-2b59084ab356 372a35bb-6366-59f0-a23e-2b59084ab356}" +2025-06-03T23:37:46.652Z [INFO] server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b: aborting pipeline replication: peer="{Voter server-f150e749-5f33-7fa0-31d4-18e1539bfe4c f150e749-5f33-7fa0-31d4-18e1539bfe4c}" +2025-06-03T23:37:46.652Z [INFO] server-f150e749-5f33-7fa0-31d4-18e1539bfe4c: pipelining replication: peer="{Voter server-372a35bb-6366-59f0-a23e-2b59084ab356 372a35bb-6366-59f0-a23e-2b59084ab356}" +2025-06-03T23:37:46.652Z [INFO] server-f150e749-5f33-7fa0-31d4-18e1539bfe4c: pipelining replication: peer="{Voter server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b c23337fe-711c-2aa8-0ab1-6f1bb90c572b}" +2025-06-03T23:37:46.757Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:46.652773951 +0000 UTC m=+101.974528881 (1 nodes), 47.92µs from start of poll, 297.49452ms from cluster start. Timeout at 2025-06-03 23:37:46.757850591 +0000 UTC m=+102.079605521, 105.07664ms after stability +2025-06-03T23:37:46.758Z [INFO] server-f150e749-5f33-7fa0-31d4-18e1539bfe4c: aborting pipeline replication: peer="{Voter server-372a35bb-6366-59f0-a23e-2b59084ab356 372a35bb-6366-59f0-a23e-2b59084ab356}" +2025-06-03T23:37:46.758Z [INFO] server-f150e749-5f33-7fa0-31d4-18e1539bfe4c: aborting pipeline replication: peer="{Voter server-c23337fe-711c-2aa8-0ab1-6f1bb90c572b c23337fe-711c-2aa8-0ab1-6f1bb90c572b}" +--- PASS: TestRaft_LeadershipTransfer (0.40s) === RUN TestRaft_LeadershipTransferWithOneNode -2025-06-03T23:34:48.402Z [INFO] server-c5ce9b6b-e31a-e656-9dd1-5fb76f11b013: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c5ce9b6b-e31a-e656-9dd1-5fb76f11b013 Address:c5ce9b6b-e31a-e656-9dd1-5fb76f11b013}]" -2025-06-03T23:34:48.402Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:48.402Z [INFO] server-c5ce9b6b-e31a-e656-9dd1-5fb76f11b013: entering follower state: follower="Node at c5ce9b6b-e31a-e656-9dd1-5fb76f11b013 [Follower]" leader-address= leader-id= -2025-06-03T23:34:48.499Z [WARN] server-c5ce9b6b-e31a-e656-9dd1-5fb76f11b013: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:48.499Z [INFO] server-c5ce9b6b-e31a-e656-9dd1-5fb76f11b013: entering candidate state: node="Node at c5ce9b6b-e31a-e656-9dd1-5fb76f11b013 [Candidate]" term=2 -2025-06-03T23:34:48.499Z [INFO] server-c5ce9b6b-e31a-e656-9dd1-5fb76f11b013: election won: term=2 tally=1 -2025-06-03T23:34:48.499Z [INFO] server-c5ce9b6b-e31a-e656-9dd1-5fb76f11b013: entering leader state: leader="Node at c5ce9b6b-e31a-e656-9dd1-5fb76f11b013 [Leader]" -2025-06-03T23:34:48.604Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:48.499245911 +0000 UTC m=+103.520065281 (1 nodes), 97.01544ms from start of poll, 97.2402ms from cluster start. Timeout at 2025-06-03 23:34:48.604388751 +0000 UTC m=+103.625208081, 105.1428ms after stability ---- PASS: TestRaft_LeadershipTransferWithOneNode (0.20s) +2025-06-03T23:37:46.759Z [INFO] server-673d1950-6dc9-741b-ca97-719a8e1212c2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-673d1950-6dc9-741b-ca97-719a8e1212c2 Address:673d1950-6dc9-741b-ca97-719a8e1212c2}]" +2025-06-03T23:37:46.759Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:46.759Z [INFO] server-673d1950-6dc9-741b-ca97-719a8e1212c2: entering follower state: follower="Node at 673d1950-6dc9-741b-ca97-719a8e1212c2 [Follower]" leader-address= leader-id= +2025-06-03T23:37:46.828Z [WARN] server-673d1950-6dc9-741b-ca97-719a8e1212c2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:46.828Z [INFO] server-673d1950-6dc9-741b-ca97-719a8e1212c2: entering candidate state: node="Node at 673d1950-6dc9-741b-ca97-719a8e1212c2 [Candidate]" term=2 +2025-06-03T23:37:46.828Z [INFO] server-673d1950-6dc9-741b-ca97-719a8e1212c2: election won: term=2 tally=1 +2025-06-03T23:37:46.828Z [INFO] server-673d1950-6dc9-741b-ca97-719a8e1212c2: entering leader state: leader="Node at 673d1950-6dc9-741b-ca97-719a8e1212c2 [Leader]" +2025-06-03T23:37:46.933Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:46.828608351 +0000 UTC m=+102.150363241 (1 nodes), 69.1794ms from start of poll, 69.42992ms from cluster start. Timeout at 2025-06-03 23:37:46.933796031 +0000 UTC m=+102.255550961, 105.18772ms after stability +--- PASS: TestRaft_LeadershipTransferWithOneNode (0.18s) === RUN TestRaft_LeadershipTransferWithSevenNodes -2025-06-03T23:34:48.605Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52d9d361-6d81-e571-dbfa-ca361cff89b4 Address:52d9d361-6d81-e571-dbfa-ca361cff89b4} {Suffrage:Voter ID:server-1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe Address:1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe} {Suffrage:Voter ID:server-8e3d6f72-ce25-5db6-b067-d85c4483118b Address:8e3d6f72-ce25-5db6-b067-d85c4483118b} {Suffrage:Voter ID:server-1ae65fd2-edfc-579a-d47f-c641627deb0b Address:1ae65fd2-edfc-579a-d47f-c641627deb0b} {Suffrage:Voter ID:server-c1167219-9a87-ed91-d149-27d033e4907b Address:c1167219-9a87-ed91-d149-27d033e4907b} {Suffrage:Voter ID:server-9d57e135-7d0f-1d60-9222-413307ce1e4c Address:9d57e135-7d0f-1d60-9222-413307ce1e4c} {Suffrage:Voter ID:server-3d43f04f-42bd-cb22-8617-ffe76a5870b7 Address:3d43f04f-42bd-cb22-8617-ffe76a5870b7}]" -2025-06-03T23:34:48.605Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: entering follower state: follower="Node at 52d9d361-6d81-e571-dbfa-ca361cff89b4 [Follower]" leader-address= leader-id= -2025-06-03T23:34:48.605Z [INFO] server-1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52d9d361-6d81-e571-dbfa-ca361cff89b4 Address:52d9d361-6d81-e571-dbfa-ca361cff89b4} {Suffrage:Voter ID:server-1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe Address:1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe} {Suffrage:Voter ID:server-8e3d6f72-ce25-5db6-b067-d85c4483118b Address:8e3d6f72-ce25-5db6-b067-d85c4483118b} {Suffrage:Voter ID:server-1ae65fd2-edfc-579a-d47f-c641627deb0b Address:1ae65fd2-edfc-579a-d47f-c641627deb0b} {Suffrage:Voter ID:server-c1167219-9a87-ed91-d149-27d033e4907b Address:c1167219-9a87-ed91-d149-27d033e4907b} {Suffrage:Voter ID:server-9d57e135-7d0f-1d60-9222-413307ce1e4c Address:9d57e135-7d0f-1d60-9222-413307ce1e4c} {Suffrage:Voter ID:server-3d43f04f-42bd-cb22-8617-ffe76a5870b7 Address:3d43f04f-42bd-cb22-8617-ffe76a5870b7}]" -2025-06-03T23:34:48.605Z [INFO] server-1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe: entering follower state: follower="Node at 1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe [Follower]" leader-address= leader-id= -2025-06-03T23:34:48.605Z [INFO] server-8e3d6f72-ce25-5db6-b067-d85c4483118b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52d9d361-6d81-e571-dbfa-ca361cff89b4 Address:52d9d361-6d81-e571-dbfa-ca361cff89b4} {Suffrage:Voter ID:server-1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe Address:1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe} {Suffrage:Voter ID:server-8e3d6f72-ce25-5db6-b067-d85c4483118b Address:8e3d6f72-ce25-5db6-b067-d85c4483118b} {Suffrage:Voter ID:server-1ae65fd2-edfc-579a-d47f-c641627deb0b Address:1ae65fd2-edfc-579a-d47f-c641627deb0b} {Suffrage:Voter ID:server-c1167219-9a87-ed91-d149-27d033e4907b Address:c1167219-9a87-ed91-d149-27d033e4907b} {Suffrage:Voter ID:server-9d57e135-7d0f-1d60-9222-413307ce1e4c Address:9d57e135-7d0f-1d60-9222-413307ce1e4c} {Suffrage:Voter ID:server-3d43f04f-42bd-cb22-8617-ffe76a5870b7 Address:3d43f04f-42bd-cb22-8617-ffe76a5870b7}]" -2025-06-03T23:34:48.606Z [INFO] server-8e3d6f72-ce25-5db6-b067-d85c4483118b: entering follower state: follower="Node at 8e3d6f72-ce25-5db6-b067-d85c4483118b [Follower]" leader-address= leader-id= -2025-06-03T23:34:48.606Z [INFO] server-1ae65fd2-edfc-579a-d47f-c641627deb0b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52d9d361-6d81-e571-dbfa-ca361cff89b4 Address:52d9d361-6d81-e571-dbfa-ca361cff89b4} {Suffrage:Voter ID:server-1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe Address:1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe} {Suffrage:Voter ID:server-8e3d6f72-ce25-5db6-b067-d85c4483118b Address:8e3d6f72-ce25-5db6-b067-d85c4483118b} {Suffrage:Voter ID:server-1ae65fd2-edfc-579a-d47f-c641627deb0b Address:1ae65fd2-edfc-579a-d47f-c641627deb0b} {Suffrage:Voter ID:server-c1167219-9a87-ed91-d149-27d033e4907b Address:c1167219-9a87-ed91-d149-27d033e4907b} {Suffrage:Voter ID:server-9d57e135-7d0f-1d60-9222-413307ce1e4c Address:9d57e135-7d0f-1d60-9222-413307ce1e4c} {Suffrage:Voter ID:server-3d43f04f-42bd-cb22-8617-ffe76a5870b7 Address:3d43f04f-42bd-cb22-8617-ffe76a5870b7}]" -2025-06-03T23:34:48.606Z [INFO] server-1ae65fd2-edfc-579a-d47f-c641627deb0b: entering follower state: follower="Node at 1ae65fd2-edfc-579a-d47f-c641627deb0b [Follower]" leader-address= leader-id= -2025-06-03T23:34:48.606Z [INFO] server-c1167219-9a87-ed91-d149-27d033e4907b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52d9d361-6d81-e571-dbfa-ca361cff89b4 Address:52d9d361-6d81-e571-dbfa-ca361cff89b4} {Suffrage:Voter ID:server-1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe Address:1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe} {Suffrage:Voter ID:server-8e3d6f72-ce25-5db6-b067-d85c4483118b Address:8e3d6f72-ce25-5db6-b067-d85c4483118b} {Suffrage:Voter ID:server-1ae65fd2-edfc-579a-d47f-c641627deb0b Address:1ae65fd2-edfc-579a-d47f-c641627deb0b} {Suffrage:Voter ID:server-c1167219-9a87-ed91-d149-27d033e4907b Address:c1167219-9a87-ed91-d149-27d033e4907b} {Suffrage:Voter ID:server-9d57e135-7d0f-1d60-9222-413307ce1e4c Address:9d57e135-7d0f-1d60-9222-413307ce1e4c} {Suffrage:Voter ID:server-3d43f04f-42bd-cb22-8617-ffe76a5870b7 Address:3d43f04f-42bd-cb22-8617-ffe76a5870b7}]" -2025-06-03T23:34:48.606Z [INFO] server-c1167219-9a87-ed91-d149-27d033e4907b: entering follower state: follower="Node at c1167219-9a87-ed91-d149-27d033e4907b [Follower]" leader-address= leader-id= -2025-06-03T23:34:48.606Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52d9d361-6d81-e571-dbfa-ca361cff89b4 Address:52d9d361-6d81-e571-dbfa-ca361cff89b4} {Suffrage:Voter ID:server-1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe Address:1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe} {Suffrage:Voter ID:server-8e3d6f72-ce25-5db6-b067-d85c4483118b Address:8e3d6f72-ce25-5db6-b067-d85c4483118b} {Suffrage:Voter ID:server-1ae65fd2-edfc-579a-d47f-c641627deb0b Address:1ae65fd2-edfc-579a-d47f-c641627deb0b} {Suffrage:Voter ID:server-c1167219-9a87-ed91-d149-27d033e4907b Address:c1167219-9a87-ed91-d149-27d033e4907b} {Suffrage:Voter ID:server-9d57e135-7d0f-1d60-9222-413307ce1e4c Address:9d57e135-7d0f-1d60-9222-413307ce1e4c} {Suffrage:Voter ID:server-3d43f04f-42bd-cb22-8617-ffe76a5870b7 Address:3d43f04f-42bd-cb22-8617-ffe76a5870b7}]" -2025-06-03T23:34:48.606Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: entering follower state: follower="Node at 9d57e135-7d0f-1d60-9222-413307ce1e4c [Follower]" leader-address= leader-id= -2025-06-03T23:34:48.606Z [INFO] server-3d43f04f-42bd-cb22-8617-ffe76a5870b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52d9d361-6d81-e571-dbfa-ca361cff89b4 Address:52d9d361-6d81-e571-dbfa-ca361cff89b4} {Suffrage:Voter ID:server-1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe Address:1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe} {Suffrage:Voter ID:server-8e3d6f72-ce25-5db6-b067-d85c4483118b Address:8e3d6f72-ce25-5db6-b067-d85c4483118b} {Suffrage:Voter ID:server-1ae65fd2-edfc-579a-d47f-c641627deb0b Address:1ae65fd2-edfc-579a-d47f-c641627deb0b} {Suffrage:Voter ID:server-c1167219-9a87-ed91-d149-27d033e4907b Address:c1167219-9a87-ed91-d149-27d033e4907b} {Suffrage:Voter ID:server-9d57e135-7d0f-1d60-9222-413307ce1e4c Address:9d57e135-7d0f-1d60-9222-413307ce1e4c} {Suffrage:Voter ID:server-3d43f04f-42bd-cb22-8617-ffe76a5870b7 Address:3d43f04f-42bd-cb22-8617-ffe76a5870b7}]" -2025-06-03T23:34:48.606Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:48.606Z [INFO] server-3d43f04f-42bd-cb22-8617-ffe76a5870b7: entering follower state: follower="Node at 3d43f04f-42bd-cb22-8617-ffe76a5870b7 [Follower]" leader-address= leader-id= -2025-06-03T23:34:48.678Z [WARN] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:48.678Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: entering candidate state: node="Node at 9d57e135-7d0f-1d60-9222-413307ce1e4c [Candidate]" term=2 -2025-06-03T23:34:48.678Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: election won: term=2 tally=4 -2025-06-03T23:34:48.678Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: entering leader state: leader="Node at 9d57e135-7d0f-1d60-9222-413307ce1e4c [Leader]" -2025-06-03T23:34:48.678Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: added peer, starting replication: peer=server-52d9d361-6d81-e571-dbfa-ca361cff89b4 -2025-06-03T23:34:48.678Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: added peer, starting replication: peer=server-1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe -2025-06-03T23:34:48.679Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: added peer, starting replication: peer=server-8e3d6f72-ce25-5db6-b067-d85c4483118b -2025-06-03T23:34:48.679Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: added peer, starting replication: peer=server-1ae65fd2-edfc-579a-d47f-c641627deb0b -2025-06-03T23:34:48.679Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: added peer, starting replication: peer=server-c1167219-9a87-ed91-d149-27d033e4907b -2025-06-03T23:34:48.679Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: added peer, starting replication: peer=server-3d43f04f-42bd-cb22-8617-ffe76a5870b7 -2025-06-03T23:34:48.679Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: pipelining replication: peer="{Voter server-1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe 1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe}" -2025-06-03T23:34:48.679Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: pipelining replication: peer="{Voter server-8e3d6f72-ce25-5db6-b067-d85c4483118b 8e3d6f72-ce25-5db6-b067-d85c4483118b}" -2025-06-03T23:34:48.679Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: pipelining replication: peer="{Voter server-52d9d361-6d81-e571-dbfa-ca361cff89b4 52d9d361-6d81-e571-dbfa-ca361cff89b4}" -2025-06-03T23:34:48.679Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: pipelining replication: peer="{Voter server-c1167219-9a87-ed91-d149-27d033e4907b c1167219-9a87-ed91-d149-27d033e4907b}" -2025-06-03T23:34:48.679Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: pipelining replication: peer="{Voter server-3d43f04f-42bd-cb22-8617-ffe76a5870b7 3d43f04f-42bd-cb22-8617-ffe76a5870b7}" -2025-06-03T23:34:48.679Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: pipelining replication: peer="{Voter server-1ae65fd2-edfc-579a-d47f-c641627deb0b 1ae65fd2-edfc-579a-d47f-c641627deb0b}" -2025-06-03T23:34:48.784Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:48.678986911 +0000 UTC m=+103.699806281 (1 nodes), 72.26664ms from start of poll, 73.5408ms from cluster start. Timeout at 2025-06-03 23:34:48.784841871 +0000 UTC m=+103.805661321, 105.85504ms after stability -2025-06-03T23:34:48.784Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:48.890Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:48.784956191 +0000 UTC m=+103.805775561 (6 nodes), 4.36µs from start of poll, 179.51008ms from cluster start. Timeout at 2025-06-03 23:34:48.889983351 +0000 UTC m=+103.910802721, 105.02716ms after stability -2025-06-03T23:34:48.890Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:48.995Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:48.890086191 +0000 UTC m=+103.910905561 (1 nodes), 2.72µs from start of poll, 284.64008ms from cluster start. Timeout at 2025-06-03 23:34:48.995092591 +0000 UTC m=+104.015911961, 105.0064ms after stability -2025-06-03T23:34:48.995Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: entering candidate state: node="Node at 52d9d361-6d81-e571-dbfa-ca361cff89b4 [Candidate]" term=3 -2025-06-03T23:34:48.995Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:48.995Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: election won: term=3 tally=4 -2025-06-03T23:34:48.995Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: entering leader state: leader="Node at 52d9d361-6d81-e571-dbfa-ca361cff89b4 [Leader]" -2025-06-03T23:34:48.995Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: added peer, starting replication: peer=server-1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe -2025-06-03T23:34:48.995Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: entering follower state: follower="Node at 9d57e135-7d0f-1d60-9222-413307ce1e4c [Follower]" leader-address= leader-id= -2025-06-03T23:34:48.995Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: added peer, starting replication: peer=server-8e3d6f72-ce25-5db6-b067-d85c4483118b -2025-06-03T23:34:48.995Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: added peer, starting replication: peer=server-1ae65fd2-edfc-579a-d47f-c641627deb0b -2025-06-03T23:34:48.995Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: added peer, starting replication: peer=server-c1167219-9a87-ed91-d149-27d033e4907b -2025-06-03T23:34:48.995Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: added peer, starting replication: peer=server-9d57e135-7d0f-1d60-9222-413307ce1e4c -2025-06-03T23:34:48.995Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: added peer, starting replication: peer=server-3d43f04f-42bd-cb22-8617-ffe76a5870b7 -2025-06-03T23:34:48.995Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: aborting pipeline replication: peer="{Voter server-8e3d6f72-ce25-5db6-b067-d85c4483118b 8e3d6f72-ce25-5db6-b067-d85c4483118b}" -2025-06-03T23:34:48.995Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: pipelining replication: peer="{Voter server-1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe 1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe}" -2025-06-03T23:34:48.995Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: aborting pipeline replication: peer="{Voter server-52d9d361-6d81-e571-dbfa-ca361cff89b4 52d9d361-6d81-e571-dbfa-ca361cff89b4}" -2025-06-03T23:34:48.995Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: aborting pipeline replication: peer="{Voter server-1ae65fd2-edfc-579a-d47f-c641627deb0b 1ae65fd2-edfc-579a-d47f-c641627deb0b}" -2025-06-03T23:34:48.995Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: pipelining replication: peer="{Voter server-8e3d6f72-ce25-5db6-b067-d85c4483118b 8e3d6f72-ce25-5db6-b067-d85c4483118b}" -2025-06-03T23:34:48.995Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: aborting pipeline replication: peer="{Voter server-1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe 1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe}" -2025-06-03T23:34:48.995Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: aborting pipeline replication: peer="{Voter server-c1167219-9a87-ed91-d149-27d033e4907b c1167219-9a87-ed91-d149-27d033e4907b}" -2025-06-03T23:34:48.995Z [INFO] server-9d57e135-7d0f-1d60-9222-413307ce1e4c: aborting pipeline replication: peer="{Voter server-3d43f04f-42bd-cb22-8617-ffe76a5870b7 3d43f04f-42bd-cb22-8617-ffe76a5870b7}" -2025-06-03T23:34:48.995Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: pipelining replication: peer="{Voter server-c1167219-9a87-ed91-d149-27d033e4907b c1167219-9a87-ed91-d149-27d033e4907b}" -2025-06-03T23:34:48.995Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: pipelining replication: peer="{Voter server-1ae65fd2-edfc-579a-d47f-c641627deb0b 1ae65fd2-edfc-579a-d47f-c641627deb0b}" -2025-06-03T23:34:48.995Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: pipelining replication: peer="{Voter server-9d57e135-7d0f-1d60-9222-413307ce1e4c 9d57e135-7d0f-1d60-9222-413307ce1e4c}" -2025-06-03T23:34:48.995Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: pipelining replication: peer="{Voter server-3d43f04f-42bd-cb22-8617-ffe76a5870b7 3d43f04f-42bd-cb22-8617-ffe76a5870b7}" -2025-06-03T23:34:49.100Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:48.995503551 +0000 UTC m=+104.016322921 (1 nodes), 117.64µs from start of poll, 390.05744ms from cluster start. Timeout at 2025-06-03 23:34:49.100525911 +0000 UTC m=+104.121345281, 105.02236ms after stability -2025-06-03T23:34:49.100Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: aborting pipeline replication: peer="{Voter server-3d43f04f-42bd-cb22-8617-ffe76a5870b7 3d43f04f-42bd-cb22-8617-ffe76a5870b7}" -2025-06-03T23:34:49.100Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: aborting pipeline replication: peer="{Voter server-1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe 1f3c0d4a-8732-fcd1-8db4-32c2cbabeabe}" -2025-06-03T23:34:49.100Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: aborting pipeline replication: peer="{Voter server-8e3d6f72-ce25-5db6-b067-d85c4483118b 8e3d6f72-ce25-5db6-b067-d85c4483118b}" -2025-06-03T23:34:49.100Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: aborting pipeline replication: peer="{Voter server-9d57e135-7d0f-1d60-9222-413307ce1e4c 9d57e135-7d0f-1d60-9222-413307ce1e4c}" -2025-06-03T23:34:49.100Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: aborting pipeline replication: peer="{Voter server-c1167219-9a87-ed91-d149-27d033e4907b c1167219-9a87-ed91-d149-27d033e4907b}" -2025-06-03T23:34:49.100Z [INFO] server-52d9d361-6d81-e571-dbfa-ca361cff89b4: aborting pipeline replication: peer="{Voter server-1ae65fd2-edfc-579a-d47f-c641627deb0b 1ae65fd2-edfc-579a-d47f-c641627deb0b}" ---- PASS: TestRaft_LeadershipTransferWithSevenNodes (0.50s) +2025-06-03T23:37:46.934Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-afaa54ab-d1a1-6657-988c-4eb789d693cc Address:afaa54ab-d1a1-6657-988c-4eb789d693cc} {Suffrage:Voter ID:server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d Address:30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d} {Suffrage:Voter ID:server-de289290-d8b0-fb12-7732-52b7373037be Address:de289290-d8b0-fb12-7732-52b7373037be} {Suffrage:Voter ID:server-dfc58af6-c022-fa1c-2eb7-3a3e67767837 Address:dfc58af6-c022-fa1c-2eb7-3a3e67767837} {Suffrage:Voter ID:server-6ce33baf-d591-e8bc-6058-c6c292a87adc Address:6ce33baf-d591-e8bc-6058-c6c292a87adc} {Suffrage:Voter ID:server-e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2 Address:e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2} {Suffrage:Voter ID:server-f560f9d7-50db-7838-06fe-d69dcf6b6bae Address:f560f9d7-50db-7838-06fe-d69dcf6b6bae}]" +2025-06-03T23:37:46.934Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: entering follower state: follower="Node at afaa54ab-d1a1-6657-988c-4eb789d693cc [Follower]" leader-address= leader-id= +2025-06-03T23:37:46.935Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-afaa54ab-d1a1-6657-988c-4eb789d693cc Address:afaa54ab-d1a1-6657-988c-4eb789d693cc} {Suffrage:Voter ID:server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d Address:30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d} {Suffrage:Voter ID:server-de289290-d8b0-fb12-7732-52b7373037be Address:de289290-d8b0-fb12-7732-52b7373037be} {Suffrage:Voter ID:server-dfc58af6-c022-fa1c-2eb7-3a3e67767837 Address:dfc58af6-c022-fa1c-2eb7-3a3e67767837} {Suffrage:Voter ID:server-6ce33baf-d591-e8bc-6058-c6c292a87adc Address:6ce33baf-d591-e8bc-6058-c6c292a87adc} {Suffrage:Voter ID:server-e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2 Address:e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2} {Suffrage:Voter ID:server-f560f9d7-50db-7838-06fe-d69dcf6b6bae Address:f560f9d7-50db-7838-06fe-d69dcf6b6bae}]" +2025-06-03T23:37:46.935Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: entering follower state: follower="Node at 30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d [Follower]" leader-address= leader-id= +2025-06-03T23:37:46.935Z [INFO] server-de289290-d8b0-fb12-7732-52b7373037be: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-afaa54ab-d1a1-6657-988c-4eb789d693cc Address:afaa54ab-d1a1-6657-988c-4eb789d693cc} {Suffrage:Voter ID:server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d Address:30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d} {Suffrage:Voter ID:server-de289290-d8b0-fb12-7732-52b7373037be Address:de289290-d8b0-fb12-7732-52b7373037be} {Suffrage:Voter ID:server-dfc58af6-c022-fa1c-2eb7-3a3e67767837 Address:dfc58af6-c022-fa1c-2eb7-3a3e67767837} {Suffrage:Voter ID:server-6ce33baf-d591-e8bc-6058-c6c292a87adc Address:6ce33baf-d591-e8bc-6058-c6c292a87adc} {Suffrage:Voter ID:server-e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2 Address:e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2} {Suffrage:Voter ID:server-f560f9d7-50db-7838-06fe-d69dcf6b6bae Address:f560f9d7-50db-7838-06fe-d69dcf6b6bae}]" +2025-06-03T23:37:46.935Z [INFO] server-de289290-d8b0-fb12-7732-52b7373037be: entering follower state: follower="Node at de289290-d8b0-fb12-7732-52b7373037be [Follower]" leader-address= leader-id= +2025-06-03T23:37:46.935Z [INFO] server-dfc58af6-c022-fa1c-2eb7-3a3e67767837: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-afaa54ab-d1a1-6657-988c-4eb789d693cc Address:afaa54ab-d1a1-6657-988c-4eb789d693cc} {Suffrage:Voter ID:server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d Address:30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d} {Suffrage:Voter ID:server-de289290-d8b0-fb12-7732-52b7373037be Address:de289290-d8b0-fb12-7732-52b7373037be} {Suffrage:Voter ID:server-dfc58af6-c022-fa1c-2eb7-3a3e67767837 Address:dfc58af6-c022-fa1c-2eb7-3a3e67767837} {Suffrage:Voter ID:server-6ce33baf-d591-e8bc-6058-c6c292a87adc Address:6ce33baf-d591-e8bc-6058-c6c292a87adc} {Suffrage:Voter ID:server-e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2 Address:e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2} {Suffrage:Voter ID:server-f560f9d7-50db-7838-06fe-d69dcf6b6bae Address:f560f9d7-50db-7838-06fe-d69dcf6b6bae}]" +2025-06-03T23:37:46.935Z [INFO] server-dfc58af6-c022-fa1c-2eb7-3a3e67767837: entering follower state: follower="Node at dfc58af6-c022-fa1c-2eb7-3a3e67767837 [Follower]" leader-address= leader-id= +2025-06-03T23:37:46.935Z [INFO] server-6ce33baf-d591-e8bc-6058-c6c292a87adc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-afaa54ab-d1a1-6657-988c-4eb789d693cc Address:afaa54ab-d1a1-6657-988c-4eb789d693cc} {Suffrage:Voter ID:server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d Address:30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d} {Suffrage:Voter ID:server-de289290-d8b0-fb12-7732-52b7373037be Address:de289290-d8b0-fb12-7732-52b7373037be} {Suffrage:Voter ID:server-dfc58af6-c022-fa1c-2eb7-3a3e67767837 Address:dfc58af6-c022-fa1c-2eb7-3a3e67767837} {Suffrage:Voter ID:server-6ce33baf-d591-e8bc-6058-c6c292a87adc Address:6ce33baf-d591-e8bc-6058-c6c292a87adc} {Suffrage:Voter ID:server-e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2 Address:e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2} {Suffrage:Voter ID:server-f560f9d7-50db-7838-06fe-d69dcf6b6bae Address:f560f9d7-50db-7838-06fe-d69dcf6b6bae}]" +2025-06-03T23:37:46.935Z [INFO] server-6ce33baf-d591-e8bc-6058-c6c292a87adc: entering follower state: follower="Node at 6ce33baf-d591-e8bc-6058-c6c292a87adc [Follower]" leader-address= leader-id= +2025-06-03T23:37:46.935Z [INFO] server-e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-afaa54ab-d1a1-6657-988c-4eb789d693cc Address:afaa54ab-d1a1-6657-988c-4eb789d693cc} {Suffrage:Voter ID:server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d Address:30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d} {Suffrage:Voter ID:server-de289290-d8b0-fb12-7732-52b7373037be Address:de289290-d8b0-fb12-7732-52b7373037be} {Suffrage:Voter ID:server-dfc58af6-c022-fa1c-2eb7-3a3e67767837 Address:dfc58af6-c022-fa1c-2eb7-3a3e67767837} {Suffrage:Voter ID:server-6ce33baf-d591-e8bc-6058-c6c292a87adc Address:6ce33baf-d591-e8bc-6058-c6c292a87adc} {Suffrage:Voter ID:server-e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2 Address:e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2} {Suffrage:Voter ID:server-f560f9d7-50db-7838-06fe-d69dcf6b6bae Address:f560f9d7-50db-7838-06fe-d69dcf6b6bae}]" +2025-06-03T23:37:46.935Z [INFO] server-e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2: entering follower state: follower="Node at e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2 [Follower]" leader-address= leader-id= +2025-06-03T23:37:46.935Z [INFO] server-f560f9d7-50db-7838-06fe-d69dcf6b6bae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-afaa54ab-d1a1-6657-988c-4eb789d693cc Address:afaa54ab-d1a1-6657-988c-4eb789d693cc} {Suffrage:Voter ID:server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d Address:30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d} {Suffrage:Voter ID:server-de289290-d8b0-fb12-7732-52b7373037be Address:de289290-d8b0-fb12-7732-52b7373037be} {Suffrage:Voter ID:server-dfc58af6-c022-fa1c-2eb7-3a3e67767837 Address:dfc58af6-c022-fa1c-2eb7-3a3e67767837} {Suffrage:Voter ID:server-6ce33baf-d591-e8bc-6058-c6c292a87adc Address:6ce33baf-d591-e8bc-6058-c6c292a87adc} {Suffrage:Voter ID:server-e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2 Address:e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2} {Suffrage:Voter ID:server-f560f9d7-50db-7838-06fe-d69dcf6b6bae Address:f560f9d7-50db-7838-06fe-d69dcf6b6bae}]" +2025-06-03T23:37:46.935Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:46.935Z [INFO] server-f560f9d7-50db-7838-06fe-d69dcf6b6bae: entering follower state: follower="Node at f560f9d7-50db-7838-06fe-d69dcf6b6bae [Follower]" leader-address= leader-id= +2025-06-03T23:37:46.986Z [WARN] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:46.986Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: entering candidate state: node="Node at afaa54ab-d1a1-6657-988c-4eb789d693cc [Candidate]" term=2 +2025-06-03T23:37:46.986Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: election won: term=2 tally=4 +2025-06-03T23:37:46.986Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: entering leader state: leader="Node at afaa54ab-d1a1-6657-988c-4eb789d693cc [Leader]" +2025-06-03T23:37:46.986Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: added peer, starting replication: peer=server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d +2025-06-03T23:37:46.986Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: added peer, starting replication: peer=server-de289290-d8b0-fb12-7732-52b7373037be +2025-06-03T23:37:46.986Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: added peer, starting replication: peer=server-dfc58af6-c022-fa1c-2eb7-3a3e67767837 +2025-06-03T23:37:46.986Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: added peer, starting replication: peer=server-6ce33baf-d591-e8bc-6058-c6c292a87adc +2025-06-03T23:37:46.986Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: added peer, starting replication: peer=server-e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2 +2025-06-03T23:37:46.986Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: added peer, starting replication: peer=server-f560f9d7-50db-7838-06fe-d69dcf6b6bae +2025-06-03T23:37:46.986Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: pipelining replication: peer="{Voter server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d 30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d}" +2025-06-03T23:37:46.986Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: pipelining replication: peer="{Voter server-f560f9d7-50db-7838-06fe-d69dcf6b6bae f560f9d7-50db-7838-06fe-d69dcf6b6bae}" +2025-06-03T23:37:46.986Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: pipelining replication: peer="{Voter server-e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2 e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2}" +2025-06-03T23:37:46.986Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: pipelining replication: peer="{Voter server-6ce33baf-d591-e8bc-6058-c6c292a87adc 6ce33baf-d591-e8bc-6058-c6c292a87adc}" +2025-06-03T23:37:46.986Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: pipelining replication: peer="{Voter server-dfc58af6-c022-fa1c-2eb7-3a3e67767837 dfc58af6-c022-fa1c-2eb7-3a3e67767837}" +2025-06-03T23:37:46.986Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: pipelining replication: peer="{Voter server-de289290-d8b0-fb12-7732-52b7373037be de289290-d8b0-fb12-7732-52b7373037be}" +2025-06-03T23:37:47.091Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:46.986260551 +0000 UTC m=+102.308015481 (1 nodes), 50.35336ms from start of poll, 51.52668ms from cluster start. Timeout at 2025-06-03 23:37:47.091396071 +0000 UTC m=+102.413151001, 105.13552ms after stability +2025-06-03T23:37:47.091Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:47.196Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:47.091497551 +0000 UTC m=+102.413252441 (6 nodes), 2.24µs from start of poll, 156.76364ms from cluster start. Timeout at 2025-06-03 23:37:47.196927911 +0000 UTC m=+102.518682881, 105.43044ms after stability +2025-06-03T23:37:47.196Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:47.302Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:47.197011511 +0000 UTC m=+102.518766441 (1 nodes), 2.6µs from start of poll, 262.27764ms from cluster start. Timeout at 2025-06-03 23:37:47.302034431 +0000 UTC m=+102.623789361, 105.02292ms after stability +2025-06-03T23:37:47.302Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: entering candidate state: node="Node at 30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d [Candidate]" term=3 +2025-06-03T23:37:47.302Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:47.302Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: election won: term=3 tally=4 +2025-06-03T23:37:47.302Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: entering leader state: leader="Node at 30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d [Leader]" +2025-06-03T23:37:47.302Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: added peer, starting replication: peer=server-afaa54ab-d1a1-6657-988c-4eb789d693cc +2025-06-03T23:37:47.302Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: added peer, starting replication: peer=server-de289290-d8b0-fb12-7732-52b7373037be +2025-06-03T23:37:47.302Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: added peer, starting replication: peer=server-dfc58af6-c022-fa1c-2eb7-3a3e67767837 +2025-06-03T23:37:47.302Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: added peer, starting replication: peer=server-6ce33baf-d591-e8bc-6058-c6c292a87adc +2025-06-03T23:37:47.302Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: added peer, starting replication: peer=server-e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2 +2025-06-03T23:37:47.302Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: added peer, starting replication: peer=server-f560f9d7-50db-7838-06fe-d69dcf6b6bae +2025-06-03T23:37:47.302Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: entering follower state: follower="Node at afaa54ab-d1a1-6657-988c-4eb789d693cc [Follower]" leader-address= leader-id= +2025-06-03T23:37:47.302Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: aborting pipeline replication: peer="{Voter server-dfc58af6-c022-fa1c-2eb7-3a3e67767837 dfc58af6-c022-fa1c-2eb7-3a3e67767837}" +2025-06-03T23:37:47.302Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: aborting pipeline replication: peer="{Voter server-f560f9d7-50db-7838-06fe-d69dcf6b6bae f560f9d7-50db-7838-06fe-d69dcf6b6bae}" +2025-06-03T23:37:47.302Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: pipelining replication: peer="{Voter server-afaa54ab-d1a1-6657-988c-4eb789d693cc afaa54ab-d1a1-6657-988c-4eb789d693cc}" +2025-06-03T23:37:47.302Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: aborting pipeline replication: peer="{Voter server-de289290-d8b0-fb12-7732-52b7373037be de289290-d8b0-fb12-7732-52b7373037be}" +2025-06-03T23:37:47.302Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: aborting pipeline replication: peer="{Voter server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d 30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d}" +2025-06-03T23:37:47.302Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: pipelining replication: peer="{Voter server-f560f9d7-50db-7838-06fe-d69dcf6b6bae f560f9d7-50db-7838-06fe-d69dcf6b6bae}" +2025-06-03T23:37:47.302Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: aborting pipeline replication: peer="{Voter server-6ce33baf-d591-e8bc-6058-c6c292a87adc 6ce33baf-d591-e8bc-6058-c6c292a87adc}" +2025-06-03T23:37:47.302Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: pipelining replication: peer="{Voter server-dfc58af6-c022-fa1c-2eb7-3a3e67767837 dfc58af6-c022-fa1c-2eb7-3a3e67767837}" +2025-06-03T23:37:47.302Z [INFO] server-afaa54ab-d1a1-6657-988c-4eb789d693cc: aborting pipeline replication: peer="{Voter server-e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2 e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2}" +2025-06-03T23:37:47.302Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: pipelining replication: peer="{Voter server-6ce33baf-d591-e8bc-6058-c6c292a87adc 6ce33baf-d591-e8bc-6058-c6c292a87adc}" +2025-06-03T23:37:47.302Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: pipelining replication: peer="{Voter server-e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2 e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2}" +2025-06-03T23:37:47.302Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: pipelining replication: peer="{Voter server-de289290-d8b0-fb12-7732-52b7373037be de289290-d8b0-fb12-7732-52b7373037be}" +2025-06-03T23:37:47.407Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:47.302338631 +0000 UTC m=+102.624093561 (1 nodes), 51.72µs from start of poll, 367.60476ms from cluster start. Timeout at 2025-06-03 23:37:47.407350391 +0000 UTC m=+102.729105321, 105.01176ms after stability +2025-06-03T23:37:47.407Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: aborting pipeline replication: peer="{Voter server-dfc58af6-c022-fa1c-2eb7-3a3e67767837 dfc58af6-c022-fa1c-2eb7-3a3e67767837}" +2025-06-03T23:37:47.407Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: aborting pipeline replication: peer="{Voter server-6ce33baf-d591-e8bc-6058-c6c292a87adc 6ce33baf-d591-e8bc-6058-c6c292a87adc}" +2025-06-03T23:37:47.407Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: aborting pipeline replication: peer="{Voter server-f560f9d7-50db-7838-06fe-d69dcf6b6bae f560f9d7-50db-7838-06fe-d69dcf6b6bae}" +2025-06-03T23:37:47.407Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: aborting pipeline replication: peer="{Voter server-de289290-d8b0-fb12-7732-52b7373037be de289290-d8b0-fb12-7732-52b7373037be}" +2025-06-03T23:37:47.407Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: aborting pipeline replication: peer="{Voter server-afaa54ab-d1a1-6657-988c-4eb789d693cc afaa54ab-d1a1-6657-988c-4eb789d693cc}" +2025-06-03T23:37:47.407Z [INFO] server-30dff5f5-eb76-23f5-db6d-0d5bf3bbb01d: aborting pipeline replication: peer="{Voter server-e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2 e19ac24d-5a80-a4db-bf2f-64a4cdc5f4a2}" +--- PASS: TestRaft_LeadershipTransferWithSevenNodes (0.47s) === RUN TestRaft_LeadershipTransferToInvalidID -2025-06-03T23:34:49.102Z [INFO] server-62634064-7f8f-e4d9-2f37-86486b772816: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-62634064-7f8f-e4d9-2f37-86486b772816 Address:62634064-7f8f-e4d9-2f37-86486b772816} {Suffrage:Voter ID:server-bafd6380-79b5-e272-d1bf-65e36c2c88b9 Address:bafd6380-79b5-e272-d1bf-65e36c2c88b9} {Suffrage:Voter ID:server-7c3b47e7-9b25-40ed-cff5-c4bd61ad7f71 Address:7c3b47e7-9b25-40ed-cff5-c4bd61ad7f71}]" -2025-06-03T23:34:49.102Z [INFO] server-62634064-7f8f-e4d9-2f37-86486b772816: entering follower state: follower="Node at 62634064-7f8f-e4d9-2f37-86486b772816 [Follower]" leader-address= leader-id= -2025-06-03T23:34:49.102Z [INFO] server-bafd6380-79b5-e272-d1bf-65e36c2c88b9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-62634064-7f8f-e4d9-2f37-86486b772816 Address:62634064-7f8f-e4d9-2f37-86486b772816} {Suffrage:Voter ID:server-bafd6380-79b5-e272-d1bf-65e36c2c88b9 Address:bafd6380-79b5-e272-d1bf-65e36c2c88b9} {Suffrage:Voter ID:server-7c3b47e7-9b25-40ed-cff5-c4bd61ad7f71 Address:7c3b47e7-9b25-40ed-cff5-c4bd61ad7f71}]" -2025-06-03T23:34:49.102Z [INFO] server-7c3b47e7-9b25-40ed-cff5-c4bd61ad7f71: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-62634064-7f8f-e4d9-2f37-86486b772816 Address:62634064-7f8f-e4d9-2f37-86486b772816} {Suffrage:Voter ID:server-bafd6380-79b5-e272-d1bf-65e36c2c88b9 Address:bafd6380-79b5-e272-d1bf-65e36c2c88b9} {Suffrage:Voter ID:server-7c3b47e7-9b25-40ed-cff5-c4bd61ad7f71 Address:7c3b47e7-9b25-40ed-cff5-c4bd61ad7f71}]" -2025-06-03T23:34:49.102Z [INFO] server-bafd6380-79b5-e272-d1bf-65e36c2c88b9: entering follower state: follower="Node at bafd6380-79b5-e272-d1bf-65e36c2c88b9 [Follower]" leader-address= leader-id= -2025-06-03T23:34:49.102Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:49.102Z [INFO] server-7c3b47e7-9b25-40ed-cff5-c4bd61ad7f71: entering follower state: follower="Node at 7c3b47e7-9b25-40ed-cff5-c4bd61ad7f71 [Follower]" leader-address= leader-id= -2025-06-03T23:34:49.154Z [WARN] server-bafd6380-79b5-e272-d1bf-65e36c2c88b9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:49.154Z [INFO] server-bafd6380-79b5-e272-d1bf-65e36c2c88b9: entering candidate state: node="Node at bafd6380-79b5-e272-d1bf-65e36c2c88b9 [Candidate]" term=2 -2025-06-03T23:34:49.154Z [INFO] server-bafd6380-79b5-e272-d1bf-65e36c2c88b9: election won: term=2 tally=2 -2025-06-03T23:34:49.154Z [INFO] server-bafd6380-79b5-e272-d1bf-65e36c2c88b9: entering leader state: leader="Node at bafd6380-79b5-e272-d1bf-65e36c2c88b9 [Leader]" -2025-06-03T23:34:49.154Z [INFO] server-bafd6380-79b5-e272-d1bf-65e36c2c88b9: added peer, starting replication: peer=server-62634064-7f8f-e4d9-2f37-86486b772816 -2025-06-03T23:34:49.154Z [INFO] server-bafd6380-79b5-e272-d1bf-65e36c2c88b9: added peer, starting replication: peer=server-7c3b47e7-9b25-40ed-cff5-c4bd61ad7f71 -2025-06-03T23:34:49.154Z [INFO] server-bafd6380-79b5-e272-d1bf-65e36c2c88b9: pipelining replication: peer="{Voter server-7c3b47e7-9b25-40ed-cff5-c4bd61ad7f71 7c3b47e7-9b25-40ed-cff5-c4bd61ad7f71}" -2025-06-03T23:34:49.154Z [INFO] server-bafd6380-79b5-e272-d1bf-65e36c2c88b9: pipelining replication: peer="{Voter server-62634064-7f8f-e4d9-2f37-86486b772816 62634064-7f8f-e4d9-2f37-86486b772816}" -2025-06-03T23:34:49.259Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:49.154248511 +0000 UTC m=+104.175067881 (1 nodes), 51.8886ms from start of poll, 52.38288ms from cluster start. Timeout at 2025-06-03 23:34:49.259735791 +0000 UTC m=+104.280555161, 105.48728ms after stability -2025-06-03T23:34:49.259Z [INFO] server-bafd6380-79b5-e272-d1bf-65e36c2c88b9: aborting pipeline replication: peer="{Voter server-7c3b47e7-9b25-40ed-cff5-c4bd61ad7f71 7c3b47e7-9b25-40ed-cff5-c4bd61ad7f71}" -2025-06-03T23:34:49.259Z [INFO] server-bafd6380-79b5-e272-d1bf-65e36c2c88b9: aborting pipeline replication: peer="{Voter server-62634064-7f8f-e4d9-2f37-86486b772816 62634064-7f8f-e4d9-2f37-86486b772816}" +2025-06-03T23:37:47.408Z [INFO] server-cb106894-86df-4179-18cb-cd2fccbc88d5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb106894-86df-4179-18cb-cd2fccbc88d5 Address:cb106894-86df-4179-18cb-cd2fccbc88d5} {Suffrage:Voter ID:server-43349351-608b-2864-c586-a31de9680418 Address:43349351-608b-2864-c586-a31de9680418} {Suffrage:Voter ID:server-7182487a-b59e-6dff-8731-2512212975b9 Address:7182487a-b59e-6dff-8731-2512212975b9}]" +2025-06-03T23:37:47.408Z [INFO] server-43349351-608b-2864-c586-a31de9680418: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb106894-86df-4179-18cb-cd2fccbc88d5 Address:cb106894-86df-4179-18cb-cd2fccbc88d5} {Suffrage:Voter ID:server-43349351-608b-2864-c586-a31de9680418 Address:43349351-608b-2864-c586-a31de9680418} {Suffrage:Voter ID:server-7182487a-b59e-6dff-8731-2512212975b9 Address:7182487a-b59e-6dff-8731-2512212975b9}]" +2025-06-03T23:37:47.409Z [INFO] server-43349351-608b-2864-c586-a31de9680418: entering follower state: follower="Node at 43349351-608b-2864-c586-a31de9680418 [Follower]" leader-address= leader-id= +2025-06-03T23:37:47.408Z [INFO] server-cb106894-86df-4179-18cb-cd2fccbc88d5: entering follower state: follower="Node at cb106894-86df-4179-18cb-cd2fccbc88d5 [Follower]" leader-address= leader-id= +2025-06-03T23:37:47.409Z [INFO] server-7182487a-b59e-6dff-8731-2512212975b9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb106894-86df-4179-18cb-cd2fccbc88d5 Address:cb106894-86df-4179-18cb-cd2fccbc88d5} {Suffrage:Voter ID:server-43349351-608b-2864-c586-a31de9680418 Address:43349351-608b-2864-c586-a31de9680418} {Suffrage:Voter ID:server-7182487a-b59e-6dff-8731-2512212975b9 Address:7182487a-b59e-6dff-8731-2512212975b9}]" +2025-06-03T23:37:47.409Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:47.409Z [INFO] server-7182487a-b59e-6dff-8731-2512212975b9: entering follower state: follower="Node at 7182487a-b59e-6dff-8731-2512212975b9 [Follower]" leader-address= leader-id= +2025-06-03T23:37:47.462Z [WARN] server-cb106894-86df-4179-18cb-cd2fccbc88d5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:47.462Z [INFO] server-cb106894-86df-4179-18cb-cd2fccbc88d5: entering candidate state: node="Node at cb106894-86df-4179-18cb-cd2fccbc88d5 [Candidate]" term=2 +2025-06-03T23:37:47.462Z [INFO] server-cb106894-86df-4179-18cb-cd2fccbc88d5: election won: term=2 tally=2 +2025-06-03T23:37:47.462Z [INFO] server-cb106894-86df-4179-18cb-cd2fccbc88d5: entering leader state: leader="Node at cb106894-86df-4179-18cb-cd2fccbc88d5 [Leader]" +2025-06-03T23:37:47.462Z [INFO] server-cb106894-86df-4179-18cb-cd2fccbc88d5: added peer, starting replication: peer=server-43349351-608b-2864-c586-a31de9680418 +2025-06-03T23:37:47.462Z [INFO] server-cb106894-86df-4179-18cb-cd2fccbc88d5: added peer, starting replication: peer=server-7182487a-b59e-6dff-8731-2512212975b9 +2025-06-03T23:37:47.462Z [INFO] server-cb106894-86df-4179-18cb-cd2fccbc88d5: pipelining replication: peer="{Voter server-7182487a-b59e-6dff-8731-2512212975b9 7182487a-b59e-6dff-8731-2512212975b9}" +2025-06-03T23:37:47.462Z [INFO] server-cb106894-86df-4179-18cb-cd2fccbc88d5: pipelining replication: peer="{Voter server-43349351-608b-2864-c586-a31de9680418 43349351-608b-2864-c586-a31de9680418}" +2025-06-03T23:37:47.568Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:47.462509271 +0000 UTC m=+102.784264201 (1 nodes), 53.37036ms from start of poll, 53.84192ms from cluster start. Timeout at 2025-06-03 23:37:47.568515151 +0000 UTC m=+102.890270041, 106.00584ms after stability +2025-06-03T23:37:47.568Z [INFO] server-cb106894-86df-4179-18cb-cd2fccbc88d5: aborting pipeline replication: peer="{Voter server-7182487a-b59e-6dff-8731-2512212975b9 7182487a-b59e-6dff-8731-2512212975b9}" +2025-06-03T23:37:47.568Z [INFO] server-cb106894-86df-4179-18cb-cd2fccbc88d5: aborting pipeline replication: peer="{Voter server-43349351-608b-2864-c586-a31de9680418 43349351-608b-2864-c586-a31de9680418}" --- PASS: TestRaft_LeadershipTransferToInvalidID (0.16s) === RUN TestRaft_LeadershipTransferToInvalidAddress -2025-06-03T23:34:49.260Z [INFO] server-e8b43014-03bc-93dd-a726-c873e9ef2032: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e8b43014-03bc-93dd-a726-c873e9ef2032 Address:e8b43014-03bc-93dd-a726-c873e9ef2032} {Suffrage:Voter ID:server-869abf05-df91-cd23-0e9e-502fa7843b07 Address:869abf05-df91-cd23-0e9e-502fa7843b07} {Suffrage:Voter ID:server-ca374cd7-d325-a324-176c-9e6bd7d3ef31 Address:ca374cd7-d325-a324-176c-9e6bd7d3ef31}]" -2025-06-03T23:34:49.260Z [INFO] server-e8b43014-03bc-93dd-a726-c873e9ef2032: entering follower state: follower="Node at e8b43014-03bc-93dd-a726-c873e9ef2032 [Follower]" leader-address= leader-id= -2025-06-03T23:34:49.261Z [INFO] server-869abf05-df91-cd23-0e9e-502fa7843b07: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e8b43014-03bc-93dd-a726-c873e9ef2032 Address:e8b43014-03bc-93dd-a726-c873e9ef2032} {Suffrage:Voter ID:server-869abf05-df91-cd23-0e9e-502fa7843b07 Address:869abf05-df91-cd23-0e9e-502fa7843b07} {Suffrage:Voter ID:server-ca374cd7-d325-a324-176c-9e6bd7d3ef31 Address:ca374cd7-d325-a324-176c-9e6bd7d3ef31}]" -2025-06-03T23:34:49.261Z [INFO] server-869abf05-df91-cd23-0e9e-502fa7843b07: entering follower state: follower="Node at 869abf05-df91-cd23-0e9e-502fa7843b07 [Follower]" leader-address= leader-id= -2025-06-03T23:34:49.261Z [INFO] server-ca374cd7-d325-a324-176c-9e6bd7d3ef31: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e8b43014-03bc-93dd-a726-c873e9ef2032 Address:e8b43014-03bc-93dd-a726-c873e9ef2032} {Suffrage:Voter ID:server-869abf05-df91-cd23-0e9e-502fa7843b07 Address:869abf05-df91-cd23-0e9e-502fa7843b07} {Suffrage:Voter ID:server-ca374cd7-d325-a324-176c-9e6bd7d3ef31 Address:ca374cd7-d325-a324-176c-9e6bd7d3ef31}]" -2025-06-03T23:34:49.261Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:49.261Z [INFO] server-ca374cd7-d325-a324-176c-9e6bd7d3ef31: entering follower state: follower="Node at ca374cd7-d325-a324-176c-9e6bd7d3ef31 [Follower]" leader-address= leader-id= -2025-06-03T23:34:49.329Z [WARN] server-ca374cd7-d325-a324-176c-9e6bd7d3ef31: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:49.329Z [INFO] server-ca374cd7-d325-a324-176c-9e6bd7d3ef31: entering candidate state: node="Node at ca374cd7-d325-a324-176c-9e6bd7d3ef31 [Candidate]" term=2 -2025-06-03T23:34:49.329Z [INFO] server-ca374cd7-d325-a324-176c-9e6bd7d3ef31: election won: term=2 tally=2 -2025-06-03T23:34:49.329Z [INFO] server-ca374cd7-d325-a324-176c-9e6bd7d3ef31: entering leader state: leader="Node at ca374cd7-d325-a324-176c-9e6bd7d3ef31 [Leader]" -2025-06-03T23:34:49.329Z [INFO] server-ca374cd7-d325-a324-176c-9e6bd7d3ef31: added peer, starting replication: peer=server-e8b43014-03bc-93dd-a726-c873e9ef2032 -2025-06-03T23:34:49.329Z [INFO] server-ca374cd7-d325-a324-176c-9e6bd7d3ef31: added peer, starting replication: peer=server-869abf05-df91-cd23-0e9e-502fa7843b07 -2025-06-03T23:34:49.329Z [INFO] server-ca374cd7-d325-a324-176c-9e6bd7d3ef31: pipelining replication: peer="{Voter server-869abf05-df91-cd23-0e9e-502fa7843b07 869abf05-df91-cd23-0e9e-502fa7843b07}" -2025-06-03T23:34:49.329Z [INFO] server-ca374cd7-d325-a324-176c-9e6bd7d3ef31: pipelining replication: peer="{Voter server-e8b43014-03bc-93dd-a726-c873e9ef2032 e8b43014-03bc-93dd-a726-c873e9ef2032}" -2025-06-03T23:34:49.435Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:49.329560111 +0000 UTC m=+104.350379481 (2 nodes), 68.17244ms from start of poll, 68.82548ms from cluster start. Timeout at 2025-06-03 23:34:49.435405591 +0000 UTC m=+104.456224961, 105.84548ms after stability -2025-06-03T23:34:49.435Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:49.541Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:49.435537551 +0000 UTC m=+104.456356921 (1 nodes), 3.12µs from start of poll, 174.80292ms from cluster start. Timeout at 2025-06-03 23:34:49.541647751 +0000 UTC m=+104.562467121, 106.1102ms after stability -2025-06-03T23:34:49.541Z [INFO] server-ca374cd7-d325-a324-176c-9e6bd7d3ef31: aborting pipeline replication: peer="{Voter server-e8b43014-03bc-93dd-a726-c873e9ef2032 e8b43014-03bc-93dd-a726-c873e9ef2032}" -2025-06-03T23:34:49.541Z [INFO] server-ca374cd7-d325-a324-176c-9e6bd7d3ef31: aborting pipeline replication: peer="{Voter server-869abf05-df91-cd23-0e9e-502fa7843b07 869abf05-df91-cd23-0e9e-502fa7843b07}" ---- PASS: TestRaft_LeadershipTransferToInvalidAddress (0.28s) +2025-06-03T23:37:47.569Z [INFO] server-daac8d08-6fc3-afd6-e39c-555dd9171a65: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-daac8d08-6fc3-afd6-e39c-555dd9171a65 Address:daac8d08-6fc3-afd6-e39c-555dd9171a65} {Suffrage:Voter ID:server-36719c4c-922e-698f-f05e-1ab01d35ff31 Address:36719c4c-922e-698f-f05e-1ab01d35ff31} {Suffrage:Voter ID:server-6e24bb66-1f18-ece6-5d6f-2f978d112d24 Address:6e24bb66-1f18-ece6-5d6f-2f978d112d24}]" +2025-06-03T23:37:47.569Z [INFO] server-daac8d08-6fc3-afd6-e39c-555dd9171a65: entering follower state: follower="Node at daac8d08-6fc3-afd6-e39c-555dd9171a65 [Follower]" leader-address= leader-id= +2025-06-03T23:37:47.569Z [INFO] server-36719c4c-922e-698f-f05e-1ab01d35ff31: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-daac8d08-6fc3-afd6-e39c-555dd9171a65 Address:daac8d08-6fc3-afd6-e39c-555dd9171a65} {Suffrage:Voter ID:server-36719c4c-922e-698f-f05e-1ab01d35ff31 Address:36719c4c-922e-698f-f05e-1ab01d35ff31} {Suffrage:Voter ID:server-6e24bb66-1f18-ece6-5d6f-2f978d112d24 Address:6e24bb66-1f18-ece6-5d6f-2f978d112d24}]" +2025-06-03T23:37:47.569Z [INFO] server-36719c4c-922e-698f-f05e-1ab01d35ff31: entering follower state: follower="Node at 36719c4c-922e-698f-f05e-1ab01d35ff31 [Follower]" leader-address= leader-id= +2025-06-03T23:37:47.569Z [INFO] server-6e24bb66-1f18-ece6-5d6f-2f978d112d24: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-daac8d08-6fc3-afd6-e39c-555dd9171a65 Address:daac8d08-6fc3-afd6-e39c-555dd9171a65} {Suffrage:Voter ID:server-36719c4c-922e-698f-f05e-1ab01d35ff31 Address:36719c4c-922e-698f-f05e-1ab01d35ff31} {Suffrage:Voter ID:server-6e24bb66-1f18-ece6-5d6f-2f978d112d24 Address:6e24bb66-1f18-ece6-5d6f-2f978d112d24}]" +2025-06-03T23:37:47.569Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:47.569Z [INFO] server-6e24bb66-1f18-ece6-5d6f-2f978d112d24: entering follower state: follower="Node at 6e24bb66-1f18-ece6-5d6f-2f978d112d24 [Follower]" leader-address= leader-id= +2025-06-03T23:37:47.643Z [WARN] server-daac8d08-6fc3-afd6-e39c-555dd9171a65: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:47.643Z [INFO] server-daac8d08-6fc3-afd6-e39c-555dd9171a65: entering candidate state: node="Node at daac8d08-6fc3-afd6-e39c-555dd9171a65 [Candidate]" term=2 +2025-06-03T23:37:47.644Z [INFO] server-daac8d08-6fc3-afd6-e39c-555dd9171a65: election won: term=2 tally=2 +2025-06-03T23:37:47.644Z [INFO] server-daac8d08-6fc3-afd6-e39c-555dd9171a65: entering leader state: leader="Node at daac8d08-6fc3-afd6-e39c-555dd9171a65 [Leader]" +2025-06-03T23:37:47.644Z [INFO] server-daac8d08-6fc3-afd6-e39c-555dd9171a65: added peer, starting replication: peer=server-36719c4c-922e-698f-f05e-1ab01d35ff31 +2025-06-03T23:37:47.644Z [INFO] server-daac8d08-6fc3-afd6-e39c-555dd9171a65: added peer, starting replication: peer=server-6e24bb66-1f18-ece6-5d6f-2f978d112d24 +2025-06-03T23:37:47.644Z [INFO] server-daac8d08-6fc3-afd6-e39c-555dd9171a65: pipelining replication: peer="{Voter server-6e24bb66-1f18-ece6-5d6f-2f978d112d24 6e24bb66-1f18-ece6-5d6f-2f978d112d24}" +2025-06-03T23:37:47.644Z [INFO] server-daac8d08-6fc3-afd6-e39c-555dd9171a65: pipelining replication: peer="{Voter server-36719c4c-922e-698f-f05e-1ab01d35ff31 36719c4c-922e-698f-f05e-1ab01d35ff31}" +2025-06-03T23:37:47.750Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:47.644255911 +0000 UTC m=+102.966010841 (2 nodes), 74.36624ms from start of poll, 74.84284ms from cluster start. Timeout at 2025-06-03 23:37:47.750201191 +0000 UTC m=+103.071956161, 105.94532ms after stability +2025-06-03T23:37:47.750Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:47.855Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:47.750326271 +0000 UTC m=+103.072081241 (1 nodes), 3.28µs from start of poll, 180.91324ms from cluster start. Timeout at 2025-06-03 23:37:47.855907231 +0000 UTC m=+103.177662161, 105.58092ms after stability +2025-06-03T23:37:47.856Z [INFO] server-daac8d08-6fc3-afd6-e39c-555dd9171a65: aborting pipeline replication: peer="{Voter server-6e24bb66-1f18-ece6-5d6f-2f978d112d24 6e24bb66-1f18-ece6-5d6f-2f978d112d24}" +2025-06-03T23:37:47.856Z [INFO] server-daac8d08-6fc3-afd6-e39c-555dd9171a65: aborting pipeline replication: peer="{Voter server-36719c4c-922e-698f-f05e-1ab01d35ff31 36719c4c-922e-698f-f05e-1ab01d35ff31}" +--- PASS: TestRaft_LeadershipTransferToInvalidAddress (0.29s) === RUN TestRaft_LeadershipTransferToBehindServer -2025-06-03T23:34:49.542Z [INFO] server-29028e4c-9eab-7bdd-a3b4-227fb1b8b15b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-29028e4c-9eab-7bdd-a3b4-227fb1b8b15b Address:29028e4c-9eab-7bdd-a3b4-227fb1b8b15b} {Suffrage:Voter ID:server-879e4688-d51e-2d30-bf96-08a6668c3f8a Address:879e4688-d51e-2d30-bf96-08a6668c3f8a} {Suffrage:Voter ID:server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a Address:f5f3677c-bf81-8c37-cf7b-edf2b518bd3a}]" -2025-06-03T23:34:49.542Z [INFO] server-29028e4c-9eab-7bdd-a3b4-227fb1b8b15b: entering follower state: follower="Node at 29028e4c-9eab-7bdd-a3b4-227fb1b8b15b [Follower]" leader-address= leader-id= -2025-06-03T23:34:49.542Z [INFO] server-879e4688-d51e-2d30-bf96-08a6668c3f8a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-29028e4c-9eab-7bdd-a3b4-227fb1b8b15b Address:29028e4c-9eab-7bdd-a3b4-227fb1b8b15b} {Suffrage:Voter ID:server-879e4688-d51e-2d30-bf96-08a6668c3f8a Address:879e4688-d51e-2d30-bf96-08a6668c3f8a} {Suffrage:Voter ID:server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a Address:f5f3677c-bf81-8c37-cf7b-edf2b518bd3a}]" -2025-06-03T23:34:49.542Z [INFO] server-879e4688-d51e-2d30-bf96-08a6668c3f8a: entering follower state: follower="Node at 879e4688-d51e-2d30-bf96-08a6668c3f8a [Follower]" leader-address= leader-id= -2025-06-03T23:34:49.543Z [INFO] server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-29028e4c-9eab-7bdd-a3b4-227fb1b8b15b Address:29028e4c-9eab-7bdd-a3b4-227fb1b8b15b} {Suffrage:Voter ID:server-879e4688-d51e-2d30-bf96-08a6668c3f8a Address:879e4688-d51e-2d30-bf96-08a6668c3f8a} {Suffrage:Voter ID:server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a Address:f5f3677c-bf81-8c37-cf7b-edf2b518bd3a}]" -2025-06-03T23:34:49.543Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:49.543Z [INFO] server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a: entering follower state: follower="Node at f5f3677c-bf81-8c37-cf7b-edf2b518bd3a [Follower]" leader-address= leader-id= -2025-06-03T23:34:49.616Z [WARN] server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:49.616Z [INFO] server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a: entering candidate state: node="Node at f5f3677c-bf81-8c37-cf7b-edf2b518bd3a [Candidate]" term=2 -2025-06-03T23:34:49.616Z [INFO] server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a: election won: term=2 tally=2 -2025-06-03T23:34:49.616Z [INFO] server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a: entering leader state: leader="Node at f5f3677c-bf81-8c37-cf7b-edf2b518bd3a [Leader]" -2025-06-03T23:34:49.616Z [INFO] server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a: added peer, starting replication: peer=server-29028e4c-9eab-7bdd-a3b4-227fb1b8b15b -2025-06-03T23:34:49.616Z [INFO] server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a: added peer, starting replication: peer=server-879e4688-d51e-2d30-bf96-08a6668c3f8a -2025-06-03T23:34:49.616Z [INFO] server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a: pipelining replication: peer="{Voter server-29028e4c-9eab-7bdd-a3b4-227fb1b8b15b 29028e4c-9eab-7bdd-a3b4-227fb1b8b15b}" -2025-06-03T23:34:49.616Z [INFO] server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a: pipelining replication: peer="{Voter server-879e4688-d51e-2d30-bf96-08a6668c3f8a 879e4688-d51e-2d30-bf96-08a6668c3f8a}" -2025-06-03T23:34:49.721Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:49.616428111 +0000 UTC m=+104.637247481 (1 nodes), 73.29836ms from start of poll, 73.79468ms from cluster start. Timeout at 2025-06-03 23:34:49.721515911 +0000 UTC m=+104.742335321, 105.08784ms after stability -2025-06-03T23:34:49.721Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:49.842Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:49.721625511 +0000 UTC m=+104.742444881 (2 nodes), 2.88µs from start of poll, 178.99208ms from cluster start. Timeout at 2025-06-03 23:34:49.842631991 +0000 UTC m=+104.863451361, 121.00648ms after stability -2025-06-03T23:34:49.916Z [INFO] server-29028e4c-9eab-7bdd-a3b4-227fb1b8b15b: entering candidate state: node="Node at 29028e4c-9eab-7bdd-a3b4-227fb1b8b15b [Candidate]" term=3 -2025-06-03T23:34:49.916Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:49.916Z [INFO] server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a: entering follower state: follower="Node at f5f3677c-bf81-8c37-cf7b-edf2b518bd3a [Follower]" leader-address= leader-id= -2025-06-03T23:34:49.916Z [INFO] server-29028e4c-9eab-7bdd-a3b4-227fb1b8b15b: election won: term=3 tally=2 -2025-06-03T23:34:49.916Z [INFO] server-29028e4c-9eab-7bdd-a3b4-227fb1b8b15b: entering leader state: leader="Node at 29028e4c-9eab-7bdd-a3b4-227fb1b8b15b [Leader]" -2025-06-03T23:34:49.916Z [INFO] server-29028e4c-9eab-7bdd-a3b4-227fb1b8b15b: added peer, starting replication: peer=server-879e4688-d51e-2d30-bf96-08a6668c3f8a -2025-06-03T23:34:49.916Z [INFO] server-29028e4c-9eab-7bdd-a3b4-227fb1b8b15b: added peer, starting replication: peer=server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a -2025-06-03T23:34:49.916Z [INFO] server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a: aborting pipeline replication: peer="{Voter server-879e4688-d51e-2d30-bf96-08a6668c3f8a 879e4688-d51e-2d30-bf96-08a6668c3f8a}" -2025-06-03T23:34:49.916Z [INFO] server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a: aborting pipeline replication: peer="{Voter server-29028e4c-9eab-7bdd-a3b4-227fb1b8b15b 29028e4c-9eab-7bdd-a3b4-227fb1b8b15b}" -2025-06-03T23:34:49.916Z [INFO] server-29028e4c-9eab-7bdd-a3b4-227fb1b8b15b: pipelining replication: peer="{Voter server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a f5f3677c-bf81-8c37-cf7b-edf2b518bd3a}" -2025-06-03T23:34:49.916Z [INFO] server-29028e4c-9eab-7bdd-a3b4-227fb1b8b15b: pipelining replication: peer="{Voter server-879e4688-d51e-2d30-bf96-08a6668c3f8a 879e4688-d51e-2d30-bf96-08a6668c3f8a}" -2025-06-03T23:34:50.022Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:49.916868151 +0000 UTC m=+104.937687521 (1 nodes), 191.24µs from start of poll, 374.23472ms from cluster start. Timeout at 2025-06-03 23:34:50.022434991 +0000 UTC m=+105.043254401, 105.56688ms after stability -2025-06-03T23:34:50.022Z [INFO] server-29028e4c-9eab-7bdd-a3b4-227fb1b8b15b: aborting pipeline replication: peer="{Voter server-f5f3677c-bf81-8c37-cf7b-edf2b518bd3a f5f3677c-bf81-8c37-cf7b-edf2b518bd3a}" -2025-06-03T23:34:50.022Z [INFO] server-29028e4c-9eab-7bdd-a3b4-227fb1b8b15b: aborting pipeline replication: peer="{Voter server-879e4688-d51e-2d30-bf96-08a6668c3f8a 879e4688-d51e-2d30-bf96-08a6668c3f8a}" ---- PASS: TestRaft_LeadershipTransferToBehindServer (0.48s) +2025-06-03T23:37:47.857Z [INFO] server-c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd Address:c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd} {Suffrage:Voter ID:server-c554aad0-e5f5-5485-3b22-dc6de829d69e Address:c554aad0-e5f5-5485-3b22-dc6de829d69e} {Suffrage:Voter ID:server-2fcac24e-6065-2c1b-e06a-72fcd69c6144 Address:2fcac24e-6065-2c1b-e06a-72fcd69c6144}]" +2025-06-03T23:37:47.857Z [INFO] server-c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd: entering follower state: follower="Node at c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd [Follower]" leader-address= leader-id= +2025-06-03T23:37:47.857Z [INFO] server-c554aad0-e5f5-5485-3b22-dc6de829d69e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd Address:c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd} {Suffrage:Voter ID:server-c554aad0-e5f5-5485-3b22-dc6de829d69e Address:c554aad0-e5f5-5485-3b22-dc6de829d69e} {Suffrage:Voter ID:server-2fcac24e-6065-2c1b-e06a-72fcd69c6144 Address:2fcac24e-6065-2c1b-e06a-72fcd69c6144}]" +2025-06-03T23:37:47.857Z [INFO] server-c554aad0-e5f5-5485-3b22-dc6de829d69e: entering follower state: follower="Node at c554aad0-e5f5-5485-3b22-dc6de829d69e [Follower]" leader-address= leader-id= +2025-06-03T23:37:47.857Z [INFO] server-2fcac24e-6065-2c1b-e06a-72fcd69c6144: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd Address:c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd} {Suffrage:Voter ID:server-c554aad0-e5f5-5485-3b22-dc6de829d69e Address:c554aad0-e5f5-5485-3b22-dc6de829d69e} {Suffrage:Voter ID:server-2fcac24e-6065-2c1b-e06a-72fcd69c6144 Address:2fcac24e-6065-2c1b-e06a-72fcd69c6144}]" +2025-06-03T23:37:47.857Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:47.857Z [INFO] server-2fcac24e-6065-2c1b-e06a-72fcd69c6144: entering follower state: follower="Node at 2fcac24e-6065-2c1b-e06a-72fcd69c6144 [Follower]" leader-address= leader-id= +2025-06-03T23:37:47.920Z [WARN] server-c554aad0-e5f5-5485-3b22-dc6de829d69e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:47.920Z [INFO] server-c554aad0-e5f5-5485-3b22-dc6de829d69e: entering candidate state: node="Node at c554aad0-e5f5-5485-3b22-dc6de829d69e [Candidate]" term=2 +2025-06-03T23:37:47.920Z [INFO] server-c554aad0-e5f5-5485-3b22-dc6de829d69e: election won: term=2 tally=2 +2025-06-03T23:37:47.920Z [INFO] server-c554aad0-e5f5-5485-3b22-dc6de829d69e: entering leader state: leader="Node at c554aad0-e5f5-5485-3b22-dc6de829d69e [Leader]" +2025-06-03T23:37:47.920Z [INFO] server-c554aad0-e5f5-5485-3b22-dc6de829d69e: added peer, starting replication: peer=server-c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd +2025-06-03T23:37:47.920Z [INFO] server-c554aad0-e5f5-5485-3b22-dc6de829d69e: added peer, starting replication: peer=server-2fcac24e-6065-2c1b-e06a-72fcd69c6144 +2025-06-03T23:37:47.920Z [INFO] server-c554aad0-e5f5-5485-3b22-dc6de829d69e: pipelining replication: peer="{Voter server-2fcac24e-6065-2c1b-e06a-72fcd69c6144 2fcac24e-6065-2c1b-e06a-72fcd69c6144}" +2025-06-03T23:37:47.920Z [INFO] server-c554aad0-e5f5-5485-3b22-dc6de829d69e: pipelining replication: peer="{Voter server-c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd}" +2025-06-03T23:37:48.026Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:47.920802271 +0000 UTC m=+103.242557201 (1 nodes), 63.291ms from start of poll, 63.92548ms from cluster start. Timeout at 2025-06-03 23:37:48.026862191 +0000 UTC m=+103.348617121, 106.05992ms after stability +2025-06-03T23:37:48.026Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:48.132Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:48.026975391 +0000 UTC m=+103.348730321 (2 nodes), 3.44µs from start of poll, 170.0986ms from cluster start. Timeout at 2025-06-03 23:37:48.132301151 +0000 UTC m=+103.454056081, 105.32576ms after stability +2025-06-03T23:37:48.187Z [INFO] server-c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd: entering candidate state: node="Node at c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd [Candidate]" term=3 +2025-06-03T23:37:48.187Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:48.187Z [INFO] server-c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd: election won: term=3 tally=2 +2025-06-03T23:37:48.187Z [INFO] server-c554aad0-e5f5-5485-3b22-dc6de829d69e: entering follower state: follower="Node at c554aad0-e5f5-5485-3b22-dc6de829d69e [Follower]" leader-address= leader-id= +2025-06-03T23:37:48.187Z [INFO] server-c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd: entering leader state: leader="Node at c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd [Leader]" +2025-06-03T23:37:48.187Z [INFO] server-c554aad0-e5f5-5485-3b22-dc6de829d69e: aborting pipeline replication: peer="{Voter server-2fcac24e-6065-2c1b-e06a-72fcd69c6144 2fcac24e-6065-2c1b-e06a-72fcd69c6144}" +2025-06-03T23:37:48.187Z [INFO] server-c554aad0-e5f5-5485-3b22-dc6de829d69e: aborting pipeline replication: peer="{Voter server-c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd}" +2025-06-03T23:37:48.187Z [INFO] server-c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd: added peer, starting replication: peer=server-c554aad0-e5f5-5485-3b22-dc6de829d69e +2025-06-03T23:37:48.187Z [INFO] server-c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd: added peer, starting replication: peer=server-2fcac24e-6065-2c1b-e06a-72fcd69c6144 +2025-06-03T23:37:48.187Z [INFO] server-c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd: pipelining replication: peer="{Voter server-2fcac24e-6065-2c1b-e06a-72fcd69c6144 2fcac24e-6065-2c1b-e06a-72fcd69c6144}" +2025-06-03T23:37:48.187Z [INFO] server-c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd: pipelining replication: peer="{Voter server-c554aad0-e5f5-5485-3b22-dc6de829d69e c554aad0-e5f5-5485-3b22-dc6de829d69e}" +2025-06-03T23:37:48.293Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:48.187543351 +0000 UTC m=+103.509298281 (1 nodes), 79.36µs from start of poll, 330.66656ms from cluster start. Timeout at 2025-06-03 23:37:48.293539671 +0000 UTC m=+103.615294601, 105.99632ms after stability +2025-06-03T23:37:48.293Z [INFO] server-c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd: aborting pipeline replication: peer="{Voter server-2fcac24e-6065-2c1b-e06a-72fcd69c6144 2fcac24e-6065-2c1b-e06a-72fcd69c6144}" +2025-06-03T23:37:48.293Z [INFO] server-c65b28c4-9f8f-d08b-1f12-e1ab2a2e62bd: aborting pipeline replication: peer="{Voter server-c554aad0-e5f5-5485-3b22-dc6de829d69e c554aad0-e5f5-5485-3b22-dc6de829d69e}" +--- PASS: TestRaft_LeadershipTransferToBehindServer (0.44s) === RUN TestRaft_LeadershipTransferToItself -2025-06-03T23:34:50.023Z [INFO] server-259a6842-6308-cbf7-e7db-84bdf7b81223: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-259a6842-6308-cbf7-e7db-84bdf7b81223 Address:259a6842-6308-cbf7-e7db-84bdf7b81223} {Suffrage:Voter ID:server-f5bb397f-06e1-121e-8941-993f0b1516d0 Address:f5bb397f-06e1-121e-8941-993f0b1516d0} {Suffrage:Voter ID:server-a85cb82c-1c8a-fcea-b7de-597bc5ae0627 Address:a85cb82c-1c8a-fcea-b7de-597bc5ae0627}]" -2025-06-03T23:34:50.023Z [INFO] server-259a6842-6308-cbf7-e7db-84bdf7b81223: entering follower state: follower="Node at 259a6842-6308-cbf7-e7db-84bdf7b81223 [Follower]" leader-address= leader-id= -2025-06-03T23:34:50.023Z [INFO] server-f5bb397f-06e1-121e-8941-993f0b1516d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-259a6842-6308-cbf7-e7db-84bdf7b81223 Address:259a6842-6308-cbf7-e7db-84bdf7b81223} {Suffrage:Voter ID:server-f5bb397f-06e1-121e-8941-993f0b1516d0 Address:f5bb397f-06e1-121e-8941-993f0b1516d0} {Suffrage:Voter ID:server-a85cb82c-1c8a-fcea-b7de-597bc5ae0627 Address:a85cb82c-1c8a-fcea-b7de-597bc5ae0627}]" -2025-06-03T23:34:50.023Z [INFO] server-f5bb397f-06e1-121e-8941-993f0b1516d0: entering follower state: follower="Node at f5bb397f-06e1-121e-8941-993f0b1516d0 [Follower]" leader-address= leader-id= -2025-06-03T23:34:50.023Z [INFO] server-a85cb82c-1c8a-fcea-b7de-597bc5ae0627: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-259a6842-6308-cbf7-e7db-84bdf7b81223 Address:259a6842-6308-cbf7-e7db-84bdf7b81223} {Suffrage:Voter ID:server-f5bb397f-06e1-121e-8941-993f0b1516d0 Address:f5bb397f-06e1-121e-8941-993f0b1516d0} {Suffrage:Voter ID:server-a85cb82c-1c8a-fcea-b7de-597bc5ae0627 Address:a85cb82c-1c8a-fcea-b7de-597bc5ae0627}]" -2025-06-03T23:34:50.023Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:50.024Z [INFO] server-a85cb82c-1c8a-fcea-b7de-597bc5ae0627: entering follower state: follower="Node at a85cb82c-1c8a-fcea-b7de-597bc5ae0627 [Follower]" leader-address= leader-id= -2025-06-03T23:34:50.094Z [WARN] server-a85cb82c-1c8a-fcea-b7de-597bc5ae0627: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:50.094Z [INFO] server-a85cb82c-1c8a-fcea-b7de-597bc5ae0627: entering candidate state: node="Node at a85cb82c-1c8a-fcea-b7de-597bc5ae0627 [Candidate]" term=2 -2025-06-03T23:34:50.094Z [INFO] server-a85cb82c-1c8a-fcea-b7de-597bc5ae0627: election won: term=2 tally=2 -2025-06-03T23:34:50.094Z [INFO] server-a85cb82c-1c8a-fcea-b7de-597bc5ae0627: entering leader state: leader="Node at a85cb82c-1c8a-fcea-b7de-597bc5ae0627 [Leader]" -2025-06-03T23:34:50.095Z [INFO] server-a85cb82c-1c8a-fcea-b7de-597bc5ae0627: added peer, starting replication: peer=server-259a6842-6308-cbf7-e7db-84bdf7b81223 -2025-06-03T23:34:50.095Z [INFO] server-a85cb82c-1c8a-fcea-b7de-597bc5ae0627: added peer, starting replication: peer=server-f5bb397f-06e1-121e-8941-993f0b1516d0 -2025-06-03T23:34:50.095Z [INFO] server-a85cb82c-1c8a-fcea-b7de-597bc5ae0627: pipelining replication: peer="{Voter server-f5bb397f-06e1-121e-8941-993f0b1516d0 f5bb397f-06e1-121e-8941-993f0b1516d0}" -2025-06-03T23:34:50.095Z [INFO] server-a85cb82c-1c8a-fcea-b7de-597bc5ae0627: pipelining replication: peer="{Voter server-259a6842-6308-cbf7-e7db-84bdf7b81223 259a6842-6308-cbf7-e7db-84bdf7b81223}" -2025-06-03T23:34:50.201Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:50.095066071 +0000 UTC m=+105.115885441 (1 nodes), 71.05088ms from start of poll, 71.56668ms from cluster start. Timeout at 2025-06-03 23:34:50.201101791 +0000 UTC m=+105.221921161, 106.03572ms after stability -2025-06-03T23:34:50.201Z [INFO] server-a85cb82c-1c8a-fcea-b7de-597bc5ae0627: cannot transfer leadership to itself -2025-06-03T23:34:50.201Z [INFO] server-a85cb82c-1c8a-fcea-b7de-597bc5ae0627: aborting pipeline replication: peer="{Voter server-f5bb397f-06e1-121e-8941-993f0b1516d0 f5bb397f-06e1-121e-8941-993f0b1516d0}" -2025-06-03T23:34:50.201Z [INFO] server-a85cb82c-1c8a-fcea-b7de-597bc5ae0627: aborting pipeline replication: peer="{Voter server-259a6842-6308-cbf7-e7db-84bdf7b81223 259a6842-6308-cbf7-e7db-84bdf7b81223}" ---- PASS: TestRaft_LeadershipTransferToItself (0.18s) +2025-06-03T23:37:48.294Z [INFO] server-224b4c6b-5994-2aef-1085-80764f902012: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-224b4c6b-5994-2aef-1085-80764f902012 Address:224b4c6b-5994-2aef-1085-80764f902012} {Suffrage:Voter ID:server-47de668d-29e7-2bcf-e80d-c4ef594c68a4 Address:47de668d-29e7-2bcf-e80d-c4ef594c68a4} {Suffrage:Voter ID:server-d84d076f-1445-d8c3-3da8-d098f02a5d80 Address:d84d076f-1445-d8c3-3da8-d098f02a5d80}]" +2025-06-03T23:37:48.294Z [INFO] server-47de668d-29e7-2bcf-e80d-c4ef594c68a4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-224b4c6b-5994-2aef-1085-80764f902012 Address:224b4c6b-5994-2aef-1085-80764f902012} {Suffrage:Voter ID:server-47de668d-29e7-2bcf-e80d-c4ef594c68a4 Address:47de668d-29e7-2bcf-e80d-c4ef594c68a4} {Suffrage:Voter ID:server-d84d076f-1445-d8c3-3da8-d098f02a5d80 Address:d84d076f-1445-d8c3-3da8-d098f02a5d80}]" +2025-06-03T23:37:48.294Z [INFO] server-224b4c6b-5994-2aef-1085-80764f902012: entering follower state: follower="Node at 224b4c6b-5994-2aef-1085-80764f902012 [Follower]" leader-address= leader-id= +2025-06-03T23:37:48.294Z [INFO] server-47de668d-29e7-2bcf-e80d-c4ef594c68a4: entering follower state: follower="Node at 47de668d-29e7-2bcf-e80d-c4ef594c68a4 [Follower]" leader-address= leader-id= +2025-06-03T23:37:48.294Z [INFO] server-d84d076f-1445-d8c3-3da8-d098f02a5d80: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-224b4c6b-5994-2aef-1085-80764f902012 Address:224b4c6b-5994-2aef-1085-80764f902012} {Suffrage:Voter ID:server-47de668d-29e7-2bcf-e80d-c4ef594c68a4 Address:47de668d-29e7-2bcf-e80d-c4ef594c68a4} {Suffrage:Voter ID:server-d84d076f-1445-d8c3-3da8-d098f02a5d80 Address:d84d076f-1445-d8c3-3da8-d098f02a5d80}]" +2025-06-03T23:37:48.294Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:48.294Z [INFO] server-d84d076f-1445-d8c3-3da8-d098f02a5d80: entering follower state: follower="Node at d84d076f-1445-d8c3-3da8-d098f02a5d80 [Follower]" leader-address= leader-id= +2025-06-03T23:37:48.349Z [WARN] server-47de668d-29e7-2bcf-e80d-c4ef594c68a4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:48.349Z [INFO] server-47de668d-29e7-2bcf-e80d-c4ef594c68a4: entering candidate state: node="Node at 47de668d-29e7-2bcf-e80d-c4ef594c68a4 [Candidate]" term=2 +2025-06-03T23:37:48.349Z [INFO] server-47de668d-29e7-2bcf-e80d-c4ef594c68a4: election won: term=2 tally=2 +2025-06-03T23:37:48.349Z [INFO] server-47de668d-29e7-2bcf-e80d-c4ef594c68a4: entering leader state: leader="Node at 47de668d-29e7-2bcf-e80d-c4ef594c68a4 [Leader]" +2025-06-03T23:37:48.349Z [INFO] server-47de668d-29e7-2bcf-e80d-c4ef594c68a4: added peer, starting replication: peer=server-224b4c6b-5994-2aef-1085-80764f902012 +2025-06-03T23:37:48.349Z [INFO] server-47de668d-29e7-2bcf-e80d-c4ef594c68a4: added peer, starting replication: peer=server-d84d076f-1445-d8c3-3da8-d098f02a5d80 +2025-06-03T23:37:48.349Z [INFO] server-47de668d-29e7-2bcf-e80d-c4ef594c68a4: pipelining replication: peer="{Voter server-224b4c6b-5994-2aef-1085-80764f902012 224b4c6b-5994-2aef-1085-80764f902012}" +2025-06-03T23:37:48.349Z [INFO] server-47de668d-29e7-2bcf-e80d-c4ef594c68a4: pipelining replication: peer="{Voter server-d84d076f-1445-d8c3-3da8-d098f02a5d80 d84d076f-1445-d8c3-3da8-d098f02a5d80}" +2025-06-03T23:37:48.454Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:48.349677831 +0000 UTC m=+103.671432761 (1 nodes), 55.00952ms from start of poll, 55.39532ms from cluster start. Timeout at 2025-06-03 23:37:48.454770791 +0000 UTC m=+103.776525721, 105.09296ms after stability +2025-06-03T23:37:48.454Z [INFO] server-47de668d-29e7-2bcf-e80d-c4ef594c68a4: cannot transfer leadership to itself +2025-06-03T23:37:48.454Z [INFO] server-47de668d-29e7-2bcf-e80d-c4ef594c68a4: aborting pipeline replication: peer="{Voter server-d84d076f-1445-d8c3-3da8-d098f02a5d80 d84d076f-1445-d8c3-3da8-d098f02a5d80}" +2025-06-03T23:37:48.454Z [INFO] server-47de668d-29e7-2bcf-e80d-c4ef594c68a4: aborting pipeline replication: peer="{Voter server-224b4c6b-5994-2aef-1085-80764f902012 224b4c6b-5994-2aef-1085-80764f902012}" +--- PASS: TestRaft_LeadershipTransferToItself (0.16s) === RUN TestRaft_LeadershipTransferLeaderRejectsClientRequests -2025-06-03T23:34:50.202Z [INFO] server-2c2c20af-7ffd-362a-ec9c-07d2bd23173b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2c2c20af-7ffd-362a-ec9c-07d2bd23173b Address:2c2c20af-7ffd-362a-ec9c-07d2bd23173b} {Suffrage:Voter ID:server-896b2fa0-0e74-36aa-df77-053406376177 Address:896b2fa0-0e74-36aa-df77-053406376177} {Suffrage:Voter ID:server-72faa70f-0a43-72a2-0b1c-060a83960e5f Address:72faa70f-0a43-72a2-0b1c-060a83960e5f}]" -2025-06-03T23:34:50.202Z [INFO] server-2c2c20af-7ffd-362a-ec9c-07d2bd23173b: entering follower state: follower="Node at 2c2c20af-7ffd-362a-ec9c-07d2bd23173b [Follower]" leader-address= leader-id= -2025-06-03T23:34:50.202Z [INFO] server-896b2fa0-0e74-36aa-df77-053406376177: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2c2c20af-7ffd-362a-ec9c-07d2bd23173b Address:2c2c20af-7ffd-362a-ec9c-07d2bd23173b} {Suffrage:Voter ID:server-896b2fa0-0e74-36aa-df77-053406376177 Address:896b2fa0-0e74-36aa-df77-053406376177} {Suffrage:Voter ID:server-72faa70f-0a43-72a2-0b1c-060a83960e5f Address:72faa70f-0a43-72a2-0b1c-060a83960e5f}]" -2025-06-03T23:34:50.202Z [INFO] server-896b2fa0-0e74-36aa-df77-053406376177: entering follower state: follower="Node at 896b2fa0-0e74-36aa-df77-053406376177 [Follower]" leader-address= leader-id= -2025-06-03T23:34:50.202Z [INFO] server-72faa70f-0a43-72a2-0b1c-060a83960e5f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2c2c20af-7ffd-362a-ec9c-07d2bd23173b Address:2c2c20af-7ffd-362a-ec9c-07d2bd23173b} {Suffrage:Voter ID:server-896b2fa0-0e74-36aa-df77-053406376177 Address:896b2fa0-0e74-36aa-df77-053406376177} {Suffrage:Voter ID:server-72faa70f-0a43-72a2-0b1c-060a83960e5f Address:72faa70f-0a43-72a2-0b1c-060a83960e5f}]" -2025-06-03T23:34:50.202Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:50.202Z [INFO] server-72faa70f-0a43-72a2-0b1c-060a83960e5f: entering follower state: follower="Node at 72faa70f-0a43-72a2-0b1c-060a83960e5f [Follower]" leader-address= leader-id= -2025-06-03T23:34:50.257Z [WARN] server-896b2fa0-0e74-36aa-df77-053406376177: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:50.257Z [INFO] server-896b2fa0-0e74-36aa-df77-053406376177: entering candidate state: node="Node at 896b2fa0-0e74-36aa-df77-053406376177 [Candidate]" term=2 -2025-06-03T23:34:50.257Z [INFO] server-896b2fa0-0e74-36aa-df77-053406376177: election won: term=2 tally=2 -2025-06-03T23:34:50.257Z [INFO] server-896b2fa0-0e74-36aa-df77-053406376177: entering leader state: leader="Node at 896b2fa0-0e74-36aa-df77-053406376177 [Leader]" -2025-06-03T23:34:50.257Z [INFO] server-896b2fa0-0e74-36aa-df77-053406376177: added peer, starting replication: peer=server-2c2c20af-7ffd-362a-ec9c-07d2bd23173b -2025-06-03T23:34:50.257Z [INFO] server-896b2fa0-0e74-36aa-df77-053406376177: added peer, starting replication: peer=server-72faa70f-0a43-72a2-0b1c-060a83960e5f -2025-06-03T23:34:50.257Z [INFO] server-896b2fa0-0e74-36aa-df77-053406376177: pipelining replication: peer="{Voter server-2c2c20af-7ffd-362a-ec9c-07d2bd23173b 2c2c20af-7ffd-362a-ec9c-07d2bd23173b}" -2025-06-03T23:34:50.257Z [INFO] server-896b2fa0-0e74-36aa-df77-053406376177: pipelining replication: peer="{Voter server-72faa70f-0a43-72a2-0b1c-060a83960e5f 72faa70f-0a43-72a2-0b1c-060a83960e5f}" -2025-06-03T23:34:50.363Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:50.257714191 +0000 UTC m=+105.278533561 (1 nodes), 55.1124ms from start of poll, 55.64068ms from cluster start. Timeout at 2025-06-03 23:34:50.363869751 +0000 UTC m=+105.384689161, 106.1556ms after stability +2025-06-03T23:37:48.456Z [INFO] server-ad8d7dcc-7725-4718-a725-74ea7a1eb7d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ad8d7dcc-7725-4718-a725-74ea7a1eb7d0 Address:ad8d7dcc-7725-4718-a725-74ea7a1eb7d0} {Suffrage:Voter ID:server-a99b2bb4-2d18-3f07-ae64-f0a1758724a3 Address:a99b2bb4-2d18-3f07-ae64-f0a1758724a3} {Suffrage:Voter ID:server-b686757c-b86b-b563-266c-2b13c3b73765 Address:b686757c-b86b-b563-266c-2b13c3b73765}]" +2025-06-03T23:37:48.456Z [INFO] server-ad8d7dcc-7725-4718-a725-74ea7a1eb7d0: entering follower state: follower="Node at ad8d7dcc-7725-4718-a725-74ea7a1eb7d0 [Follower]" leader-address= leader-id= +2025-06-03T23:37:48.456Z [INFO] server-a99b2bb4-2d18-3f07-ae64-f0a1758724a3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ad8d7dcc-7725-4718-a725-74ea7a1eb7d0 Address:ad8d7dcc-7725-4718-a725-74ea7a1eb7d0} {Suffrage:Voter ID:server-a99b2bb4-2d18-3f07-ae64-f0a1758724a3 Address:a99b2bb4-2d18-3f07-ae64-f0a1758724a3} {Suffrage:Voter ID:server-b686757c-b86b-b563-266c-2b13c3b73765 Address:b686757c-b86b-b563-266c-2b13c3b73765}]" +2025-06-03T23:37:48.456Z [INFO] server-a99b2bb4-2d18-3f07-ae64-f0a1758724a3: entering follower state: follower="Node at a99b2bb4-2d18-3f07-ae64-f0a1758724a3 [Follower]" leader-address= leader-id= +2025-06-03T23:37:48.456Z [INFO] server-b686757c-b86b-b563-266c-2b13c3b73765: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ad8d7dcc-7725-4718-a725-74ea7a1eb7d0 Address:ad8d7dcc-7725-4718-a725-74ea7a1eb7d0} {Suffrage:Voter ID:server-a99b2bb4-2d18-3f07-ae64-f0a1758724a3 Address:a99b2bb4-2d18-3f07-ae64-f0a1758724a3} {Suffrage:Voter ID:server-b686757c-b86b-b563-266c-2b13c3b73765 Address:b686757c-b86b-b563-266c-2b13c3b73765}]" +2025-06-03T23:37:48.456Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:48.456Z [INFO] server-b686757c-b86b-b563-266c-2b13c3b73765: entering follower state: follower="Node at b686757c-b86b-b563-266c-2b13c3b73765 [Follower]" leader-address= leader-id= +2025-06-03T23:37:48.508Z [WARN] server-a99b2bb4-2d18-3f07-ae64-f0a1758724a3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:48.508Z [INFO] server-a99b2bb4-2d18-3f07-ae64-f0a1758724a3: entering candidate state: node="Node at a99b2bb4-2d18-3f07-ae64-f0a1758724a3 [Candidate]" term=2 +2025-06-03T23:37:48.508Z [INFO] server-a99b2bb4-2d18-3f07-ae64-f0a1758724a3: election won: term=2 tally=2 +2025-06-03T23:37:48.508Z [INFO] server-a99b2bb4-2d18-3f07-ae64-f0a1758724a3: entering leader state: leader="Node at a99b2bb4-2d18-3f07-ae64-f0a1758724a3 [Leader]" +2025-06-03T23:37:48.508Z [INFO] server-a99b2bb4-2d18-3f07-ae64-f0a1758724a3: added peer, starting replication: peer=server-ad8d7dcc-7725-4718-a725-74ea7a1eb7d0 +2025-06-03T23:37:48.508Z [INFO] server-a99b2bb4-2d18-3f07-ae64-f0a1758724a3: added peer, starting replication: peer=server-b686757c-b86b-b563-266c-2b13c3b73765 +2025-06-03T23:37:48.508Z [INFO] server-a99b2bb4-2d18-3f07-ae64-f0a1758724a3: pipelining replication: peer="{Voter server-b686757c-b86b-b563-266c-2b13c3b73765 b686757c-b86b-b563-266c-2b13c3b73765}" +2025-06-03T23:37:48.508Z [INFO] server-a99b2bb4-2d18-3f07-ae64-f0a1758724a3: pipelining replication: peer="{Voter server-ad8d7dcc-7725-4718-a725-74ea7a1eb7d0 ad8d7dcc-7725-4718-a725-74ea7a1eb7d0}" +2025-06-03T23:37:48.614Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:48.508593831 +0000 UTC m=+103.830348761 (1 nodes), 52.25492ms from start of poll, 52.73912ms from cluster start. Timeout at 2025-06-03 23:37:48.614088751 +0000 UTC m=+103.935843681, 105.49492ms after stability raft_test.go:2457: waiting on future 0 raft_test.go:2457: waiting on future 1 raft_test.go:2457: waiting on future 2 @@ -3791,359 +3851,339 @@ raft_test.go:2457: waiting on future 4 raft_test.go:2457: waiting on future 5 raft_test.go:2457: waiting on future 6 -2025-06-03T23:34:50.364Z [INFO] server-896b2fa0-0e74-36aa-df77-053406376177: aborting pipeline replication: peer="{Voter server-72faa70f-0a43-72a2-0b1c-060a83960e5f 72faa70f-0a43-72a2-0b1c-060a83960e5f}" -2025-06-03T23:34:50.364Z [INFO] server-896b2fa0-0e74-36aa-df77-053406376177: aborting pipeline replication: peer="{Voter server-2c2c20af-7ffd-362a-ec9c-07d2bd23173b 2c2c20af-7ffd-362a-ec9c-07d2bd23173b}" +2025-06-03T23:37:48.614Z [INFO] server-a99b2bb4-2d18-3f07-ae64-f0a1758724a3: aborting pipeline replication: peer="{Voter server-b686757c-b86b-b563-266c-2b13c3b73765 b686757c-b86b-b563-266c-2b13c3b73765}" +2025-06-03T23:37:48.614Z [INFO] server-a99b2bb4-2d18-3f07-ae64-f0a1758724a3: aborting pipeline replication: peer="{Voter server-ad8d7dcc-7725-4718-a725-74ea7a1eb7d0 ad8d7dcc-7725-4718-a725-74ea7a1eb7d0}" --- PASS: TestRaft_LeadershipTransferLeaderRejectsClientRequests (0.16s) === RUN TestRaft_LeadershipTransferLeaderReplicationTimeout -2025-06-03T23:34:50.365Z [INFO] server-f74fe204-e542-d9b8-6e12-89c000558b24: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f74fe204-e542-d9b8-6e12-89c000558b24 Address:f74fe204-e542-d9b8-6e12-89c000558b24} {Suffrage:Voter ID:server-ea176915-16a2-6b4e-f41a-921a8fe00925 Address:ea176915-16a2-6b4e-f41a-921a8fe00925} {Suffrage:Voter ID:server-a862ba8f-e272-200d-3ade-606c8d57a8be Address:a862ba8f-e272-200d-3ade-606c8d57a8be}]" -2025-06-03T23:34:50.365Z [INFO] server-f74fe204-e542-d9b8-6e12-89c000558b24: entering follower state: follower="Node at f74fe204-e542-d9b8-6e12-89c000558b24 [Follower]" leader-address= leader-id= -2025-06-03T23:34:50.365Z [INFO] server-ea176915-16a2-6b4e-f41a-921a8fe00925: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f74fe204-e542-d9b8-6e12-89c000558b24 Address:f74fe204-e542-d9b8-6e12-89c000558b24} {Suffrage:Voter ID:server-ea176915-16a2-6b4e-f41a-921a8fe00925 Address:ea176915-16a2-6b4e-f41a-921a8fe00925} {Suffrage:Voter ID:server-a862ba8f-e272-200d-3ade-606c8d57a8be Address:a862ba8f-e272-200d-3ade-606c8d57a8be}]" -2025-06-03T23:34:50.365Z [INFO] server-ea176915-16a2-6b4e-f41a-921a8fe00925: entering follower state: follower="Node at ea176915-16a2-6b4e-f41a-921a8fe00925 [Follower]" leader-address= leader-id= -2025-06-03T23:34:50.365Z [INFO] server-a862ba8f-e272-200d-3ade-606c8d57a8be: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f74fe204-e542-d9b8-6e12-89c000558b24 Address:f74fe204-e542-d9b8-6e12-89c000558b24} {Suffrage:Voter ID:server-ea176915-16a2-6b4e-f41a-921a8fe00925 Address:ea176915-16a2-6b4e-f41a-921a8fe00925} {Suffrage:Voter ID:server-a862ba8f-e272-200d-3ade-606c8d57a8be Address:a862ba8f-e272-200d-3ade-606c8d57a8be}]" -2025-06-03T23:34:50.365Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:50.365Z [INFO] server-a862ba8f-e272-200d-3ade-606c8d57a8be: entering follower state: follower="Node at a862ba8f-e272-200d-3ade-606c8d57a8be [Follower]" leader-address= leader-id= -2025-06-03T23:34:50.423Z [WARN] server-f74fe204-e542-d9b8-6e12-89c000558b24: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:50.423Z [INFO] server-f74fe204-e542-d9b8-6e12-89c000558b24: entering candidate state: node="Node at f74fe204-e542-d9b8-6e12-89c000558b24 [Candidate]" term=2 -2025-06-03T23:34:50.423Z [INFO] server-f74fe204-e542-d9b8-6e12-89c000558b24: election won: term=2 tally=2 -2025-06-03T23:34:50.423Z [INFO] server-f74fe204-e542-d9b8-6e12-89c000558b24: entering leader state: leader="Node at f74fe204-e542-d9b8-6e12-89c000558b24 [Leader]" -2025-06-03T23:34:50.423Z [INFO] server-f74fe204-e542-d9b8-6e12-89c000558b24: added peer, starting replication: peer=server-ea176915-16a2-6b4e-f41a-921a8fe00925 -2025-06-03T23:34:50.423Z [INFO] server-f74fe204-e542-d9b8-6e12-89c000558b24: added peer, starting replication: peer=server-a862ba8f-e272-200d-3ade-606c8d57a8be -2025-06-03T23:34:50.423Z [INFO] server-f74fe204-e542-d9b8-6e12-89c000558b24: pipelining replication: peer="{Voter server-a862ba8f-e272-200d-3ade-606c8d57a8be a862ba8f-e272-200d-3ade-606c8d57a8be}" -2025-06-03T23:34:50.423Z [INFO] server-f74fe204-e542-d9b8-6e12-89c000558b24: pipelining replication: peer="{Voter server-ea176915-16a2-6b4e-f41a-921a8fe00925 ea176915-16a2-6b4e-f41a-921a8fe00925}" -2025-06-03T23:34:50.528Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:50.423762111 +0000 UTC m=+105.444581481 (1 nodes), 58.32636ms from start of poll, 58.77568ms from cluster start. Timeout at 2025-06-03 23:34:50.528796071 +0000 UTC m=+105.549615441, 105.03396ms after stability -2025-06-03T23:34:50.528Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:50.634Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:50.528913951 +0000 UTC m=+105.549733321 (2 nodes), 2.28µs from start of poll, 163.92752ms from cluster start. Timeout at 2025-06-03 23:34:50.634414031 +0000 UTC m=+105.655233401, 105.50008ms after stability -2025-06-03T23:34:51.108Z [INFO] server-f74fe204-e542-d9b8-6e12-89c000558b24: aborting pipeline replication: peer="{Voter server-a862ba8f-e272-200d-3ade-606c8d57a8be a862ba8f-e272-200d-3ade-606c8d57a8be}" -2025-06-03T23:34:51.108Z [INFO] server-f74fe204-e542-d9b8-6e12-89c000558b24: aborting pipeline replication: peer="{Voter server-ea176915-16a2-6b4e-f41a-921a8fe00925 ea176915-16a2-6b4e-f41a-921a8fe00925}" -2025-06-03T23:34:51.609Z [ERROR] server-f74fe204-e542-d9b8-6e12-89c000558b24: failed to heartbeat to: peer=ea176915-16a2-6b4e-f41a-921a8fe00925 backoff time=10ms error="command timed out" ---- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (1.25s) +2025-06-03T23:37:48.615Z [INFO] server-391e0b13-6617-2132-83d1-a96b3e24ebf7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-391e0b13-6617-2132-83d1-a96b3e24ebf7 Address:391e0b13-6617-2132-83d1-a96b3e24ebf7} {Suffrage:Voter ID:server-227e3164-b1f3-67a7-7f5a-ea1134db4b5b Address:227e3164-b1f3-67a7-7f5a-ea1134db4b5b} {Suffrage:Voter ID:server-abde4dde-239e-f124-8e01-ab642b639128 Address:abde4dde-239e-f124-8e01-ab642b639128}]" +2025-06-03T23:37:48.615Z [INFO] server-391e0b13-6617-2132-83d1-a96b3e24ebf7: entering follower state: follower="Node at 391e0b13-6617-2132-83d1-a96b3e24ebf7 [Follower]" leader-address= leader-id= +2025-06-03T23:37:48.615Z [INFO] server-227e3164-b1f3-67a7-7f5a-ea1134db4b5b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-391e0b13-6617-2132-83d1-a96b3e24ebf7 Address:391e0b13-6617-2132-83d1-a96b3e24ebf7} {Suffrage:Voter ID:server-227e3164-b1f3-67a7-7f5a-ea1134db4b5b Address:227e3164-b1f3-67a7-7f5a-ea1134db4b5b} {Suffrage:Voter ID:server-abde4dde-239e-f124-8e01-ab642b639128 Address:abde4dde-239e-f124-8e01-ab642b639128}]" +2025-06-03T23:37:48.615Z [INFO] server-227e3164-b1f3-67a7-7f5a-ea1134db4b5b: entering follower state: follower="Node at 227e3164-b1f3-67a7-7f5a-ea1134db4b5b [Follower]" leader-address= leader-id= +2025-06-03T23:37:48.615Z [INFO] server-abde4dde-239e-f124-8e01-ab642b639128: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-391e0b13-6617-2132-83d1-a96b3e24ebf7 Address:391e0b13-6617-2132-83d1-a96b3e24ebf7} {Suffrage:Voter ID:server-227e3164-b1f3-67a7-7f5a-ea1134db4b5b Address:227e3164-b1f3-67a7-7f5a-ea1134db4b5b} {Suffrage:Voter ID:server-abde4dde-239e-f124-8e01-ab642b639128 Address:abde4dde-239e-f124-8e01-ab642b639128}]" +2025-06-03T23:37:48.615Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:48.615Z [INFO] server-abde4dde-239e-f124-8e01-ab642b639128: entering follower state: follower="Node at abde4dde-239e-f124-8e01-ab642b639128 [Follower]" leader-address= leader-id= +2025-06-03T23:37:48.670Z [WARN] server-227e3164-b1f3-67a7-7f5a-ea1134db4b5b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:48.670Z [INFO] server-227e3164-b1f3-67a7-7f5a-ea1134db4b5b: entering candidate state: node="Node at 227e3164-b1f3-67a7-7f5a-ea1134db4b5b [Candidate]" term=2 +2025-06-03T23:37:48.670Z [INFO] server-227e3164-b1f3-67a7-7f5a-ea1134db4b5b: election won: term=2 tally=2 +2025-06-03T23:37:48.670Z [INFO] server-227e3164-b1f3-67a7-7f5a-ea1134db4b5b: entering leader state: leader="Node at 227e3164-b1f3-67a7-7f5a-ea1134db4b5b [Leader]" +2025-06-03T23:37:48.670Z [INFO] server-227e3164-b1f3-67a7-7f5a-ea1134db4b5b: added peer, starting replication: peer=server-391e0b13-6617-2132-83d1-a96b3e24ebf7 +2025-06-03T23:37:48.670Z [INFO] server-227e3164-b1f3-67a7-7f5a-ea1134db4b5b: added peer, starting replication: peer=server-abde4dde-239e-f124-8e01-ab642b639128 +2025-06-03T23:37:48.670Z [INFO] server-227e3164-b1f3-67a7-7f5a-ea1134db4b5b: pipelining replication: peer="{Voter server-abde4dde-239e-f124-8e01-ab642b639128 abde4dde-239e-f124-8e01-ab642b639128}" +2025-06-03T23:37:48.670Z [INFO] server-227e3164-b1f3-67a7-7f5a-ea1134db4b5b: pipelining replication: peer="{Voter server-391e0b13-6617-2132-83d1-a96b3e24ebf7 391e0b13-6617-2132-83d1-a96b3e24ebf7}" +2025-06-03T23:37:48.776Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:48.670533511 +0000 UTC m=+103.992288441 (1 nodes), 54.78788ms from start of poll, 55.30932ms from cluster start. Timeout at 2025-06-03 23:37:48.776338751 +0000 UTC m=+104.098093721, 105.80528ms after stability +2025-06-03T23:37:48.776Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:48.881Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:48.776464191 +0000 UTC m=+104.098219121 (2 nodes), 2.76µs from start of poll, 161.24ms from cluster start. Timeout at 2025-06-03 23:37:48.881470071 +0000 UTC m=+104.203225001, 105.00588ms after stability +2025-06-03T23:37:49.407Z [INFO] server-391e0b13-6617-2132-83d1-a96b3e24ebf7: entering candidate state: node="Node at 391e0b13-6617-2132-83d1-a96b3e24ebf7 [Candidate]" term=3 +2025-06-03T23:37:49.407Z [INFO] server-227e3164-b1f3-67a7-7f5a-ea1134db4b5b: aborting pipeline replication: peer="{Voter server-391e0b13-6617-2132-83d1-a96b3e24ebf7 391e0b13-6617-2132-83d1-a96b3e24ebf7}" +2025-06-03T23:37:49.407Z [INFO] server-227e3164-b1f3-67a7-7f5a-ea1134db4b5b: aborting pipeline replication: peer="{Voter server-abde4dde-239e-f124-8e01-ab642b639128 abde4dde-239e-f124-8e01-ab642b639128}" +2025-06-03T23:37:49.908Z [ERROR] server-391e0b13-6617-2132-83d1-a96b3e24ebf7: failed to make requestVote RPC: target="{Voter server-227e3164-b1f3-67a7-7f5a-ea1134db4b5b 227e3164-b1f3-67a7-7f5a-ea1134db4b5b}" error="command timed out" term=3 +2025-06-03T23:37:49.908Z [ERROR] server-391e0b13-6617-2132-83d1-a96b3e24ebf7: failed to make requestVote RPC: target="{Voter server-abde4dde-239e-f124-8e01-ab642b639128 abde4dde-239e-f124-8e01-ab642b639128}" error="command timed out" term=3 +--- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (1.29s) === RUN TestRaft_LeadershipTransferIgnoresNonvoters -2025-06-03T23:34:51.610Z [INFO] server-71c6a874-0cfe-d6f8-3b02-1899289d4edc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-71c6a874-0cfe-d6f8-3b02-1899289d4edc Address:71c6a874-0cfe-d6f8-3b02-1899289d4edc} {Suffrage:Voter ID:server-bd03b90d-2bcb-1bfc-3b8f-4c17ac197a59 Address:bd03b90d-2bcb-1bfc-3b8f-4c17ac197a59}]" -2025-06-03T23:34:51.610Z [INFO] server-71c6a874-0cfe-d6f8-3b02-1899289d4edc: entering follower state: follower="Node at 71c6a874-0cfe-d6f8-3b02-1899289d4edc [Follower]" leader-address= leader-id= -2025-06-03T23:34:51.610Z [INFO] server-bd03b90d-2bcb-1bfc-3b8f-4c17ac197a59: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-71c6a874-0cfe-d6f8-3b02-1899289d4edc Address:71c6a874-0cfe-d6f8-3b02-1899289d4edc} {Suffrage:Voter ID:server-bd03b90d-2bcb-1bfc-3b8f-4c17ac197a59 Address:bd03b90d-2bcb-1bfc-3b8f-4c17ac197a59}]" -2025-06-03T23:34:51.610Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:51.610Z [INFO] server-bd03b90d-2bcb-1bfc-3b8f-4c17ac197a59: entering follower state: follower="Node at bd03b90d-2bcb-1bfc-3b8f-4c17ac197a59 [Follower]" leader-address= leader-id= -2025-06-03T23:34:51.668Z [WARN] server-71c6a874-0cfe-d6f8-3b02-1899289d4edc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:51.668Z [INFO] server-71c6a874-0cfe-d6f8-3b02-1899289d4edc: entering candidate state: node="Node at 71c6a874-0cfe-d6f8-3b02-1899289d4edc [Candidate]" term=2 -2025-06-03T23:34:51.668Z [INFO] server-71c6a874-0cfe-d6f8-3b02-1899289d4edc: election won: term=2 tally=2 -2025-06-03T23:34:51.668Z [INFO] server-71c6a874-0cfe-d6f8-3b02-1899289d4edc: entering leader state: leader="Node at 71c6a874-0cfe-d6f8-3b02-1899289d4edc [Leader]" -2025-06-03T23:34:51.668Z [INFO] server-71c6a874-0cfe-d6f8-3b02-1899289d4edc: added peer, starting replication: peer=server-bd03b90d-2bcb-1bfc-3b8f-4c17ac197a59 -2025-06-03T23:34:51.668Z [INFO] server-71c6a874-0cfe-d6f8-3b02-1899289d4edc: pipelining replication: peer="{Voter server-bd03b90d-2bcb-1bfc-3b8f-4c17ac197a59 bd03b90d-2bcb-1bfc-3b8f-4c17ac197a59}" -2025-06-03T23:34:51.775Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:51.668233911 +0000 UTC m=+106.689053321 (1 nodes), 57.41516ms from start of poll, 57.82732ms from cluster start. Timeout at 2025-06-03 23:34:51.774954551 +0000 UTC m=+106.795773921, 106.7206ms after stability -2025-06-03T23:34:51.775Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:51.880Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:51.775100231 +0000 UTC m=+106.795919641 (1 nodes), 1.4µs from start of poll, 164.69364ms from cluster start. Timeout at 2025-06-03 23:34:51.880854831 +0000 UTC m=+106.901674201, 105.75456ms after stability -2025-06-03T23:34:51.881Z [INFO] server-71c6a874-0cfe-d6f8-3b02-1899289d4edc: updating configuration: command=DemoteVoter server-id=server-bd03b90d-2bcb-1bfc-3b8f-4c17ac197a59 server-addr= servers="[{Suffrage:Voter ID:server-71c6a874-0cfe-d6f8-3b02-1899289d4edc Address:71c6a874-0cfe-d6f8-3b02-1899289d4edc} {Suffrage:Nonvoter ID:server-bd03b90d-2bcb-1bfc-3b8f-4c17ac197a59 Address:bd03b90d-2bcb-1bfc-3b8f-4c17ac197a59}]" -2025-06-03T23:34:51.881Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:51.986Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:51.881187591 +0000 UTC m=+106.902006961 (1 nodes), 1.12µs from start of poll, 270.78096ms from cluster start. Timeout at 2025-06-03 23:34:51.986691031 +0000 UTC m=+107.007510401, 105.50344ms after stability -2025-06-03T23:34:51.986Z [INFO] server-71c6a874-0cfe-d6f8-3b02-1899289d4edc: aborting pipeline replication: peer="{Voter server-bd03b90d-2bcb-1bfc-3b8f-4c17ac197a59 bd03b90d-2bcb-1bfc-3b8f-4c17ac197a59}" ---- PASS: TestRaft_LeadershipTransferIgnoresNonvoters (0.38s) +2025-06-03T23:37:49.909Z [INFO] server-b5ec98f0-be63-835f-753b-f9ad475e304c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b5ec98f0-be63-835f-753b-f9ad475e304c Address:b5ec98f0-be63-835f-753b-f9ad475e304c} {Suffrage:Voter ID:server-7867fa76-607c-e42d-7fcd-d6782c013253 Address:7867fa76-607c-e42d-7fcd-d6782c013253}]" +2025-06-03T23:37:49.909Z [INFO] server-b5ec98f0-be63-835f-753b-f9ad475e304c: entering follower state: follower="Node at b5ec98f0-be63-835f-753b-f9ad475e304c [Follower]" leader-address= leader-id= +2025-06-03T23:37:49.909Z [INFO] server-7867fa76-607c-e42d-7fcd-d6782c013253: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b5ec98f0-be63-835f-753b-f9ad475e304c Address:b5ec98f0-be63-835f-753b-f9ad475e304c} {Suffrage:Voter ID:server-7867fa76-607c-e42d-7fcd-d6782c013253 Address:7867fa76-607c-e42d-7fcd-d6782c013253}]" +2025-06-03T23:37:49.909Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:49.909Z [INFO] server-7867fa76-607c-e42d-7fcd-d6782c013253: entering follower state: follower="Node at 7867fa76-607c-e42d-7fcd-d6782c013253 [Follower]" leader-address= leader-id= +2025-06-03T23:37:49.984Z [WARN] server-7867fa76-607c-e42d-7fcd-d6782c013253: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:49.984Z [INFO] server-7867fa76-607c-e42d-7fcd-d6782c013253: entering candidate state: node="Node at 7867fa76-607c-e42d-7fcd-d6782c013253 [Candidate]" term=2 +2025-06-03T23:37:49.984Z [INFO] server-7867fa76-607c-e42d-7fcd-d6782c013253: election won: term=2 tally=2 +2025-06-03T23:37:49.984Z [INFO] server-7867fa76-607c-e42d-7fcd-d6782c013253: entering leader state: leader="Node at 7867fa76-607c-e42d-7fcd-d6782c013253 [Leader]" +2025-06-03T23:37:49.984Z [INFO] server-7867fa76-607c-e42d-7fcd-d6782c013253: added peer, starting replication: peer=server-b5ec98f0-be63-835f-753b-f9ad475e304c +2025-06-03T23:37:49.984Z [INFO] server-7867fa76-607c-e42d-7fcd-d6782c013253: pipelining replication: peer="{Voter server-b5ec98f0-be63-835f-753b-f9ad475e304c b5ec98f0-be63-835f-753b-f9ad475e304c}" +2025-06-03T23:37:50.090Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:49.984833151 +0000 UTC m=+105.306588081 (1 nodes), 75.42012ms from start of poll, 75.78392ms from cluster start. Timeout at 2025-06-03 23:37:50.090342231 +0000 UTC m=+105.412097121, 105.50904ms after stability +2025-06-03T23:37:50.090Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:50.196Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:50.090451351 +0000 UTC m=+105.412206241 (1 nodes), 960ns from start of poll, 181.40208ms from cluster start. Timeout at 2025-06-03 23:37:50.196133351 +0000 UTC m=+105.517888281, 105.68204ms after stability +2025-06-03T23:37:50.196Z [INFO] server-7867fa76-607c-e42d-7fcd-d6782c013253: updating configuration: command=DemoteVoter server-id=server-b5ec98f0-be63-835f-753b-f9ad475e304c server-addr= servers="[{Suffrage:Nonvoter ID:server-b5ec98f0-be63-835f-753b-f9ad475e304c Address:b5ec98f0-be63-835f-753b-f9ad475e304c} {Suffrage:Voter ID:server-7867fa76-607c-e42d-7fcd-d6782c013253 Address:7867fa76-607c-e42d-7fcd-d6782c013253}]" +2025-06-03T23:37:50.196Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:50.301Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:50.196399791 +0000 UTC m=+105.518154721 (1 nodes), 1.28µs from start of poll, 287.35056ms from cluster start. Timeout at 2025-06-03 23:37:50.301907431 +0000 UTC m=+105.623662361, 105.50764ms after stability +2025-06-03T23:37:50.302Z [INFO] server-7867fa76-607c-e42d-7fcd-d6782c013253: aborting pipeline replication: peer="{Voter server-b5ec98f0-be63-835f-753b-f9ad475e304c b5ec98f0-be63-835f-753b-f9ad475e304c}" +--- PASS: TestRaft_LeadershipTransferIgnoresNonvoters (0.39s) === RUN TestRaft_LeadershipTransferStopRightAway --- PASS: TestRaft_LeadershipTransferStopRightAway (0.00s) === RUN TestRaft_GetConfigurationNoBootstrap -2025-06-03T23:34:51.987Z [INFO] server-be1ba1ed-5a92-6ab9-bc73-f21df1041eeb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-be1ba1ed-5a92-6ab9-bc73-f21df1041eeb Address:be1ba1ed-5a92-6ab9-bc73-f21df1041eeb} {Suffrage:Voter ID:server-9d1e9124-b4ba-c9dc-7f11-6bf2f307eb30 Address:9d1e9124-b4ba-c9dc-7f11-6bf2f307eb30}]" -2025-06-03T23:34:51.987Z [INFO] server-9d1e9124-b4ba-c9dc-7f11-6bf2f307eb30: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-be1ba1ed-5a92-6ab9-bc73-f21df1041eeb Address:be1ba1ed-5a92-6ab9-bc73-f21df1041eeb} {Suffrage:Voter ID:server-9d1e9124-b4ba-c9dc-7f11-6bf2f307eb30 Address:9d1e9124-b4ba-c9dc-7f11-6bf2f307eb30}]" -2025-06-03T23:34:51.987Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:51.987Z [INFO] server-be1ba1ed-5a92-6ab9-bc73-f21df1041eeb: entering follower state: follower="Node at be1ba1ed-5a92-6ab9-bc73-f21df1041eeb [Follower]" leader-address= leader-id= -2025-06-03T23:34:51.987Z [INFO] server-9d1e9124-b4ba-c9dc-7f11-6bf2f307eb30: entering follower state: follower="Node at 9d1e9124-b4ba-c9dc-7f11-6bf2f307eb30 [Follower]" leader-address= leader-id= -2025-06-03T23:34:52.052Z [WARN] server-be1ba1ed-5a92-6ab9-bc73-f21df1041eeb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:52.052Z [INFO] server-be1ba1ed-5a92-6ab9-bc73-f21df1041eeb: entering candidate state: node="Node at be1ba1ed-5a92-6ab9-bc73-f21df1041eeb [Candidate]" term=2 -2025-06-03T23:34:52.052Z [INFO] server-be1ba1ed-5a92-6ab9-bc73-f21df1041eeb: election won: term=2 tally=2 -2025-06-03T23:34:52.052Z [INFO] server-be1ba1ed-5a92-6ab9-bc73-f21df1041eeb: entering leader state: leader="Node at be1ba1ed-5a92-6ab9-bc73-f21df1041eeb [Leader]" -2025-06-03T23:34:52.052Z [INFO] server-be1ba1ed-5a92-6ab9-bc73-f21df1041eeb: added peer, starting replication: peer=server-9d1e9124-b4ba-c9dc-7f11-6bf2f307eb30 -2025-06-03T23:34:52.052Z [INFO] server-be1ba1ed-5a92-6ab9-bc73-f21df1041eeb: pipelining replication: peer="{Voter server-9d1e9124-b4ba-c9dc-7f11-6bf2f307eb30 9d1e9124-b4ba-c9dc-7f11-6bf2f307eb30}" -2025-06-03T23:34:52.157Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:52.052589951 +0000 UTC m=+107.073409361 (1 nodes), 64.77564ms from start of poll, 65.11708ms from cluster start. Timeout at 2025-06-03 23:34:52.157854751 +0000 UTC m=+107.178674161, 105.2648ms after stability -2025-06-03T23:34:52.157Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:52.264Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:52.157984911 +0000 UTC m=+107.178804281 (1 nodes), 2.84µs from start of poll, 170.512ms from cluster start. Timeout at 2025-06-03 23:34:52.264076631 +0000 UTC m=+107.284896001, 106.09172ms after stability -2025-06-03T23:34:52.275Z [INFO] server-9d1e9124-b4ba-c9dc-7f11-6bf2f307eb30: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-be1ba1ed-5a92-6ab9-bc73-f21df1041eeb Address:be1ba1ed-5a92-6ab9-bc73-f21df1041eeb} {Suffrage:Voter ID:server-9d1e9124-b4ba-c9dc-7f11-6bf2f307eb30 Address:9d1e9124-b4ba-c9dc-7f11-6bf2f307eb30}]" -2025-06-03T23:34:52.275Z [INFO] server-be1ba1ed-5a92-6ab9-bc73-f21df1041eeb: aborting pipeline replication: peer="{Voter server-9d1e9124-b4ba-c9dc-7f11-6bf2f307eb30 9d1e9124-b4ba-c9dc-7f11-6bf2f307eb30}" +2025-06-03T23:37:50.302Z [INFO] server-26f66d9f-eb35-4e09-9c80-404619d3e7c0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-26f66d9f-eb35-4e09-9c80-404619d3e7c0 Address:26f66d9f-eb35-4e09-9c80-404619d3e7c0} {Suffrage:Voter ID:server-5d9d2e65-5a00-fa1a-794b-471cb42f3e10 Address:5d9d2e65-5a00-fa1a-794b-471cb42f3e10}]" +2025-06-03T23:37:50.303Z [INFO] server-26f66d9f-eb35-4e09-9c80-404619d3e7c0: entering follower state: follower="Node at 26f66d9f-eb35-4e09-9c80-404619d3e7c0 [Follower]" leader-address= leader-id= +2025-06-03T23:37:50.303Z [INFO] server-5d9d2e65-5a00-fa1a-794b-471cb42f3e10: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-26f66d9f-eb35-4e09-9c80-404619d3e7c0 Address:26f66d9f-eb35-4e09-9c80-404619d3e7c0} {Suffrage:Voter ID:server-5d9d2e65-5a00-fa1a-794b-471cb42f3e10 Address:5d9d2e65-5a00-fa1a-794b-471cb42f3e10}]" +2025-06-03T23:37:50.303Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:50.303Z [INFO] server-5d9d2e65-5a00-fa1a-794b-471cb42f3e10: entering follower state: follower="Node at 5d9d2e65-5a00-fa1a-794b-471cb42f3e10 [Follower]" leader-address= leader-id= +2025-06-03T23:37:50.370Z [WARN] server-5d9d2e65-5a00-fa1a-794b-471cb42f3e10: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:50.370Z [INFO] server-5d9d2e65-5a00-fa1a-794b-471cb42f3e10: entering candidate state: node="Node at 5d9d2e65-5a00-fa1a-794b-471cb42f3e10 [Candidate]" term=2 +2025-06-03T23:37:50.370Z [INFO] server-5d9d2e65-5a00-fa1a-794b-471cb42f3e10: election won: term=2 tally=2 +2025-06-03T23:37:50.370Z [INFO] server-5d9d2e65-5a00-fa1a-794b-471cb42f3e10: entering leader state: leader="Node at 5d9d2e65-5a00-fa1a-794b-471cb42f3e10 [Leader]" +2025-06-03T23:37:50.370Z [INFO] server-5d9d2e65-5a00-fa1a-794b-471cb42f3e10: added peer, starting replication: peer=server-26f66d9f-eb35-4e09-9c80-404619d3e7c0 +2025-06-03T23:37:50.370Z [INFO] server-5d9d2e65-5a00-fa1a-794b-471cb42f3e10: pipelining replication: peer="{Voter server-26f66d9f-eb35-4e09-9c80-404619d3e7c0 26f66d9f-eb35-4e09-9c80-404619d3e7c0}" +2025-06-03T23:37:50.476Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:50.370655311 +0000 UTC m=+105.692410241 (1 nodes), 67.49516ms from start of poll, 67.86524ms from cluster start. Timeout at 2025-06-03 23:37:50.476725031 +0000 UTC m=+105.798480001, 106.06976ms after stability +2025-06-03T23:37:50.476Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:50.582Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:50.476841991 +0000 UTC m=+105.798596921 (1 nodes), 1µs from start of poll, 174.05192ms from cluster start. Timeout at 2025-06-03 23:37:50.582007911 +0000 UTC m=+105.903762841, 105.16592ms after stability +2025-06-03T23:37:50.594Z [INFO] server-5d9d2e65-5a00-fa1a-794b-471cb42f3e10: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-26f66d9f-eb35-4e09-9c80-404619d3e7c0 Address:26f66d9f-eb35-4e09-9c80-404619d3e7c0} {Suffrage:Voter ID:server-5d9d2e65-5a00-fa1a-794b-471cb42f3e10 Address:5d9d2e65-5a00-fa1a-794b-471cb42f3e10}]" +2025-06-03T23:37:50.594Z [INFO] server-5d9d2e65-5a00-fa1a-794b-471cb42f3e10: aborting pipeline replication: peer="{Voter server-26f66d9f-eb35-4e09-9c80-404619d3e7c0 26f66d9f-eb35-4e09-9c80-404619d3e7c0}" --- PASS: TestRaft_GetConfigurationNoBootstrap (0.29s) === RUN TestRaft_LogStoreIsMonotonic -2025-06-03T23:34:52.275Z [INFO] server-d07ab17f-0f42-7fb2-e16d-2700e6f869b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d07ab17f-0f42-7fb2-e16d-2700e6f869b7 Address:d07ab17f-0f42-7fb2-e16d-2700e6f869b7}]" -2025-06-03T23:34:52.275Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:52.275Z [INFO] server-d07ab17f-0f42-7fb2-e16d-2700e6f869b7: entering follower state: follower="Node at d07ab17f-0f42-7fb2-e16d-2700e6f869b7 [Follower]" leader-address= leader-id= -2025-06-03T23:34:52.360Z [WARN] server-d07ab17f-0f42-7fb2-e16d-2700e6f869b7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:52.360Z [INFO] server-d07ab17f-0f42-7fb2-e16d-2700e6f869b7: entering candidate state: node="Node at d07ab17f-0f42-7fb2-e16d-2700e6f869b7 [Candidate]" term=2 -2025-06-03T23:34:52.360Z [INFO] server-d07ab17f-0f42-7fb2-e16d-2700e6f869b7: election won: term=2 tally=1 -2025-06-03T23:34:52.360Z [INFO] server-d07ab17f-0f42-7fb2-e16d-2700e6f869b7: entering leader state: leader="Node at d07ab17f-0f42-7fb2-e16d-2700e6f869b7 [Leader]" -2025-06-03T23:34:52.465Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:52.360526351 +0000 UTC m=+107.381345721 (1 nodes), 84.64984ms from start of poll, 84.856ms from cluster start. Timeout at 2025-06-03 23:34:52.465545151 +0000 UTC m=+107.486364521, 105.0188ms after stability ---- PASS: TestRaft_LogStoreIsMonotonic (0.19s) +2025-06-03T23:37:50.595Z [INFO] server-17a03fda-4541-c6ce-5fe1-97a0f8a48c01: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-17a03fda-4541-c6ce-5fe1-97a0f8a48c01 Address:17a03fda-4541-c6ce-5fe1-97a0f8a48c01}]" +2025-06-03T23:37:50.595Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:50.595Z [INFO] server-17a03fda-4541-c6ce-5fe1-97a0f8a48c01: entering follower state: follower="Node at 17a03fda-4541-c6ce-5fe1-97a0f8a48c01 [Follower]" leader-address= leader-id= +2025-06-03T23:37:50.666Z [WARN] server-17a03fda-4541-c6ce-5fe1-97a0f8a48c01: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:50.666Z [INFO] server-17a03fda-4541-c6ce-5fe1-97a0f8a48c01: entering candidate state: node="Node at 17a03fda-4541-c6ce-5fe1-97a0f8a48c01 [Candidate]" term=2 +2025-06-03T23:37:50.666Z [INFO] server-17a03fda-4541-c6ce-5fe1-97a0f8a48c01: election won: term=2 tally=1 +2025-06-03T23:37:50.666Z [INFO] server-17a03fda-4541-c6ce-5fe1-97a0f8a48c01: entering leader state: leader="Node at 17a03fda-4541-c6ce-5fe1-97a0f8a48c01 [Leader]" +2025-06-03T23:37:50.771Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:50.666289751 +0000 UTC m=+105.988044681 (1 nodes), 71.02788ms from start of poll, 71.2686ms from cluster start. Timeout at 2025-06-03 23:37:50.771784311 +0000 UTC m=+106.093539241, 105.49456ms after stability +--- PASS: TestRaft_LogStoreIsMonotonic (0.18s) === RUN TestRaft_CacheLogWithStoreError -2025-06-03T23:34:52.466Z [INFO] server-50941f68-2fb6-3dff-eec7-f015c8b2bbb5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-50941f68-2fb6-3dff-eec7-f015c8b2bbb5 Address:50941f68-2fb6-3dff-eec7-f015c8b2bbb5} {Suffrage:Voter ID:server-dd4518c5-3d2d-648c-58b4-e7d47606afb8 Address:dd4518c5-3d2d-648c-58b4-e7d47606afb8}]" -2025-06-03T23:34:52.466Z [INFO] server-50941f68-2fb6-3dff-eec7-f015c8b2bbb5: entering follower state: follower="Node at 50941f68-2fb6-3dff-eec7-f015c8b2bbb5 [Follower]" leader-address= leader-id= -2025-06-03T23:34:52.466Z [INFO] server-dd4518c5-3d2d-648c-58b4-e7d47606afb8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-50941f68-2fb6-3dff-eec7-f015c8b2bbb5 Address:50941f68-2fb6-3dff-eec7-f015c8b2bbb5} {Suffrage:Voter ID:server-dd4518c5-3d2d-648c-58b4-e7d47606afb8 Address:dd4518c5-3d2d-648c-58b4-e7d47606afb8}]" -2025-06-03T23:34:52.466Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:52.466Z [INFO] server-dd4518c5-3d2d-648c-58b4-e7d47606afb8: entering follower state: follower="Node at dd4518c5-3d2d-648c-58b4-e7d47606afb8 [Follower]" leader-address= leader-id= -2025-06-03T23:34:52.522Z [WARN] server-50941f68-2fb6-3dff-eec7-f015c8b2bbb5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:52.522Z [INFO] server-50941f68-2fb6-3dff-eec7-f015c8b2bbb5: entering candidate state: node="Node at 50941f68-2fb6-3dff-eec7-f015c8b2bbb5 [Candidate]" term=2 -2025-06-03T23:34:52.522Z [INFO] server-50941f68-2fb6-3dff-eec7-f015c8b2bbb5: election won: term=2 tally=2 -2025-06-03T23:34:52.522Z [INFO] server-50941f68-2fb6-3dff-eec7-f015c8b2bbb5: entering leader state: leader="Node at 50941f68-2fb6-3dff-eec7-f015c8b2bbb5 [Leader]" -2025-06-03T23:34:52.522Z [INFO] server-50941f68-2fb6-3dff-eec7-f015c8b2bbb5: added peer, starting replication: peer=server-dd4518c5-3d2d-648c-58b4-e7d47606afb8 -2025-06-03T23:34:52.522Z [INFO] server-50941f68-2fb6-3dff-eec7-f015c8b2bbb5: pipelining replication: peer="{Voter server-dd4518c5-3d2d-648c-58b4-e7d47606afb8 dd4518c5-3d2d-648c-58b4-e7d47606afb8}" -2025-06-03T23:34:52.627Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:52.522804431 +0000 UTC m=+107.543623801 (1 nodes), 56.10964ms from start of poll, 56.50412ms from cluster start. Timeout at 2025-06-03 23:34:52.627818031 +0000 UTC m=+107.648637401, 105.0136ms after stability -2025-06-03T23:34:52.627Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:52.733Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:52.627939151 +0000 UTC m=+107.648758521 (1 nodes), 1.48µs from start of poll, 161.63884ms from cluster start. Timeout at 2025-06-03 23:34:52.733632191 +0000 UTC m=+107.754451601, 105.69308ms after stability -2025-06-03T23:34:52.734Z [ERROR] server-dd4518c5-3d2d-648c-58b4-e7d47606afb8: failed to append to logs: error="unable to store logs within log store, err: "some error"" -2025-06-03T23:34:52.734Z [WARN] server-dd4518c5-3d2d-648c-58b4-e7d47606afb8: failed to get previous log: previous-index=9 last-index=7 error="log not found" -2025-06-03T23:34:52.734Z [INFO] server-50941f68-2fb6-3dff-eec7-f015c8b2bbb5: aborting pipeline replication: peer="{Voter server-dd4518c5-3d2d-648c-58b4-e7d47606afb8 dd4518c5-3d2d-648c-58b4-e7d47606afb8}" -2025-06-03T23:34:52.742Z [INFO] server-50941f68-2fb6-3dff-eec7-f015c8b2bbb5: pipelining replication: peer="{Voter server-dd4518c5-3d2d-648c-58b4-e7d47606afb8 dd4518c5-3d2d-648c-58b4-e7d47606afb8}" -2025-06-03T23:34:52.742Z [INFO] server-dd4518c5-3d2d-648c-58b4-e7d47606afb8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-50941f68-2fb6-3dff-eec7-f015c8b2bbb5 Address:50941f68-2fb6-3dff-eec7-f015c8b2bbb5} {Suffrage:Voter ID:server-dd4518c5-3d2d-648c-58b4-e7d47606afb8 Address:dd4518c5-3d2d-648c-58b4-e7d47606afb8}]" -2025-06-03T23:34:52.742Z [INFO] server-50941f68-2fb6-3dff-eec7-f015c8b2bbb5: aborting pipeline replication: peer="{Voter server-dd4518c5-3d2d-648c-58b4-e7d47606afb8 dd4518c5-3d2d-648c-58b4-e7d47606afb8}" +2025-06-03T23:37:50.772Z [INFO] server-583b78f1-ac49-d547-7c7e-0956d4048112: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-583b78f1-ac49-d547-7c7e-0956d4048112 Address:583b78f1-ac49-d547-7c7e-0956d4048112} {Suffrage:Voter ID:server-45d9166a-df69-19df-5768-077cb82cb79e Address:45d9166a-df69-19df-5768-077cb82cb79e}]" +2025-06-03T23:37:50.772Z [INFO] server-45d9166a-df69-19df-5768-077cb82cb79e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-583b78f1-ac49-d547-7c7e-0956d4048112 Address:583b78f1-ac49-d547-7c7e-0956d4048112} {Suffrage:Voter ID:server-45d9166a-df69-19df-5768-077cb82cb79e Address:45d9166a-df69-19df-5768-077cb82cb79e}]" +2025-06-03T23:37:50.772Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:50.772Z [INFO] server-583b78f1-ac49-d547-7c7e-0956d4048112: entering follower state: follower="Node at 583b78f1-ac49-d547-7c7e-0956d4048112 [Follower]" leader-address= leader-id= +2025-06-03T23:37:50.772Z [INFO] server-45d9166a-df69-19df-5768-077cb82cb79e: entering follower state: follower="Node at 45d9166a-df69-19df-5768-077cb82cb79e [Follower]" leader-address= leader-id= +2025-06-03T23:37:50.828Z [WARN] server-45d9166a-df69-19df-5768-077cb82cb79e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:50.828Z [INFO] server-45d9166a-df69-19df-5768-077cb82cb79e: entering candidate state: node="Node at 45d9166a-df69-19df-5768-077cb82cb79e [Candidate]" term=2 +2025-06-03T23:37:50.828Z [INFO] server-45d9166a-df69-19df-5768-077cb82cb79e: election won: term=2 tally=2 +2025-06-03T23:37:50.828Z [INFO] server-45d9166a-df69-19df-5768-077cb82cb79e: entering leader state: leader="Node at 45d9166a-df69-19df-5768-077cb82cb79e [Leader]" +2025-06-03T23:37:50.828Z [INFO] server-45d9166a-df69-19df-5768-077cb82cb79e: added peer, starting replication: peer=server-583b78f1-ac49-d547-7c7e-0956d4048112 +2025-06-03T23:37:50.828Z [INFO] server-45d9166a-df69-19df-5768-077cb82cb79e: pipelining replication: peer="{Voter server-583b78f1-ac49-d547-7c7e-0956d4048112 583b78f1-ac49-d547-7c7e-0956d4048112}" +2025-06-03T23:37:50.935Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:50.828698751 +0000 UTC m=+106.150453681 (1 nodes), 55.79752ms from start of poll, 56.18312ms from cluster start. Timeout at 2025-06-03 23:37:50.935224631 +0000 UTC m=+106.256979561, 106.52588ms after stability +2025-06-03T23:37:50.935Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:51.040Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:50.935427671 +0000 UTC m=+106.257182601 (1 nodes), 2.32µs from start of poll, 162.91204ms from cluster start. Timeout at 2025-06-03 23:37:51.040890191 +0000 UTC m=+106.362645081, 105.46248ms after stability +2025-06-03T23:37:51.041Z [ERROR] server-583b78f1-ac49-d547-7c7e-0956d4048112: failed to append to logs: error="unable to store logs within log store, err: "some error"" +2025-06-03T23:37:51.041Z [WARN] server-583b78f1-ac49-d547-7c7e-0956d4048112: failed to get previous log: previous-index=9 last-index=7 error="log not found" +2025-06-03T23:37:51.041Z [INFO] server-45d9166a-df69-19df-5768-077cb82cb79e: aborting pipeline replication: peer="{Voter server-583b78f1-ac49-d547-7c7e-0956d4048112 583b78f1-ac49-d547-7c7e-0956d4048112}" +2025-06-03T23:37:51.050Z [INFO] server-45d9166a-df69-19df-5768-077cb82cb79e: pipelining replication: peer="{Voter server-583b78f1-ac49-d547-7c7e-0956d4048112 583b78f1-ac49-d547-7c7e-0956d4048112}" +2025-06-03T23:37:51.050Z [INFO] server-583b78f1-ac49-d547-7c7e-0956d4048112: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-583b78f1-ac49-d547-7c7e-0956d4048112 Address:583b78f1-ac49-d547-7c7e-0956d4048112} {Suffrage:Voter ID:server-45d9166a-df69-19df-5768-077cb82cb79e Address:45d9166a-df69-19df-5768-077cb82cb79e}]" +2025-06-03T23:37:51.050Z [INFO] server-45d9166a-df69-19df-5768-077cb82cb79e: aborting pipeline replication: peer="{Voter server-583b78f1-ac49-d547-7c7e-0956d4048112 583b78f1-ac49-d547-7c7e-0956d4048112}" --- PASS: TestRaft_CacheLogWithStoreError (0.28s) === RUN TestRaft_ReloadConfig -2025-06-03T23:34:52.743Z [INFO] server-28967ad0-6ef5-7c7a-ec5c-c6402d869c5e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-28967ad0-6ef5-7c7a-ec5c-c6402d869c5e Address:28967ad0-6ef5-7c7a-ec5c-c6402d869c5e}]" +2025-06-03T23:37:51.051Z [INFO] server-ffa00e65-2dab-1b70-cee8-1a718f229615: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ffa00e65-2dab-1b70-cee8-1a718f229615 Address:ffa00e65-2dab-1b70-cee8-1a718f229615}]" --- PASS: TestRaft_ReloadConfig (0.00s) === RUN TestRaft_ReloadConfigValidates -2025-06-03T23:34:52.743Z [INFO] server-3eb470b9-69ab-c6b9-8a6e-3cfb0fc3396b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3eb470b9-69ab-c6b9-8a6e-3cfb0fc3396b Address:3eb470b9-69ab-c6b9-8a6e-3cfb0fc3396b}]" +2025-06-03T23:37:51.051Z [INFO] server-6ef3dc30-e6cc-f635-d3f0-a6d4fc559317: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ef3dc30-e6cc-f635-d3f0-a6d4fc559317 Address:6ef3dc30-e6cc-f635-d3f0-a6d4fc559317}]" --- PASS: TestRaft_ReloadConfigValidates (0.00s) === RUN TestRaft_InstallSnapshot_InvalidPeers -2025-06-03T23:34:52.743Z [ERROR] failed to install snapshot: error="failed to decode peers: codec.decoder: Only encoded map or array can be decoded into a slice. (valueType: 2)" +2025-06-03T23:37:51.052Z [ERROR] failed to install snapshot: error="failed to decode peers: codec.decoder: Only encoded map or array can be decoded into a slice. (valueType: 2)" --- PASS: TestRaft_InstallSnapshot_InvalidPeers (0.00s) === RUN TestRaft_VoteNotGranted_WhenNodeNotInCluster -2025-06-03T23:34:52.744Z [INFO] server-c06443f7-bced-b107-90b7-1310a3bcc5dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c06443f7-bced-b107-90b7-1310a3bcc5dd Address:c06443f7-bced-b107-90b7-1310a3bcc5dd} {Suffrage:Voter ID:server-28ff8f8b-eaf0-7e31-b916-783b28f55467 Address:28ff8f8b-eaf0-7e31-b916-783b28f55467} {Suffrage:Voter ID:server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b Address:d3adfec1-a5fa-9ef5-d190-192e474ccd9b}]" -2025-06-03T23:34:52.744Z [INFO] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c06443f7-bced-b107-90b7-1310a3bcc5dd Address:c06443f7-bced-b107-90b7-1310a3bcc5dd} {Suffrage:Voter ID:server-28ff8f8b-eaf0-7e31-b916-783b28f55467 Address:28ff8f8b-eaf0-7e31-b916-783b28f55467} {Suffrage:Voter ID:server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b Address:d3adfec1-a5fa-9ef5-d190-192e474ccd9b}]" -2025-06-03T23:34:52.744Z [INFO] server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c06443f7-bced-b107-90b7-1310a3bcc5dd Address:c06443f7-bced-b107-90b7-1310a3bcc5dd} {Suffrage:Voter ID:server-28ff8f8b-eaf0-7e31-b916-783b28f55467 Address:28ff8f8b-eaf0-7e31-b916-783b28f55467} {Suffrage:Voter ID:server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b Address:d3adfec1-a5fa-9ef5-d190-192e474ccd9b}]" -2025-06-03T23:34:52.744Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:52.744Z [INFO] server-c06443f7-bced-b107-90b7-1310a3bcc5dd: entering follower state: follower="Node at c06443f7-bced-b107-90b7-1310a3bcc5dd [Follower]" leader-address= leader-id= -2025-06-03T23:34:52.744Z [INFO] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: entering follower state: follower="Node at 28ff8f8b-eaf0-7e31-b916-783b28f55467 [Follower]" leader-address= leader-id= -2025-06-03T23:34:52.744Z [INFO] server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b: entering follower state: follower="Node at d3adfec1-a5fa-9ef5-d190-192e474ccd9b [Follower]" leader-address= leader-id= -2025-06-03T23:34:52.826Z [WARN] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:52.826Z [INFO] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: entering candidate state: node="Node at 28ff8f8b-eaf0-7e31-b916-783b28f55467 [Candidate]" term=2 -2025-06-03T23:34:52.826Z [WARN] server-c06443f7-bced-b107-90b7-1310a3bcc5dd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:52.826Z [INFO] server-c06443f7-bced-b107-90b7-1310a3bcc5dd: entering candidate state: node="Node at c06443f7-bced-b107-90b7-1310a3bcc5dd [Candidate]" term=2 -2025-06-03T23:34:52.826Z [INFO] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: election won: term=2 tally=2 -2025-06-03T23:34:52.826Z [INFO] server-c06443f7-bced-b107-90b7-1310a3bcc5dd: duplicate requestVote for same term: term=2 -2025-06-03T23:34:52.826Z [INFO] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: entering leader state: leader="Node at 28ff8f8b-eaf0-7e31-b916-783b28f55467 [Leader]" -2025-06-03T23:34:52.826Z [INFO] server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b: duplicate requestVote for same term: term=2 -2025-06-03T23:34:52.826Z [INFO] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: added peer, starting replication: peer=server-c06443f7-bced-b107-90b7-1310a3bcc5dd -2025-06-03T23:34:52.826Z [INFO] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: added peer, starting replication: peer=server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b -2025-06-03T23:34:52.826Z [WARN] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: rejecting vote request since we have a leader: from=c06443f7-bced-b107-90b7-1310a3bcc5dd leader=28ff8f8b-eaf0-7e31-b916-783b28f55467 leader-id=server-28ff8f8b-eaf0-7e31-b916-783b28f55467 -2025-06-03T23:34:52.827Z [INFO] server-c06443f7-bced-b107-90b7-1310a3bcc5dd: entering follower state: follower="Node at c06443f7-bced-b107-90b7-1310a3bcc5dd [Follower]" leader-address=28ff8f8b-eaf0-7e31-b916-783b28f55467 leader-id=server-28ff8f8b-eaf0-7e31-b916-783b28f55467 -2025-06-03T23:34:52.827Z [INFO] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: pipelining replication: peer="{Voter server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b d3adfec1-a5fa-9ef5-d190-192e474ccd9b}" -2025-06-03T23:34:52.827Z [INFO] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: pipelining replication: peer="{Voter server-c06443f7-bced-b107-90b7-1310a3bcc5dd c06443f7-bced-b107-90b7-1310a3bcc5dd}" -2025-06-03T23:34:52.932Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:52.827129191 +0000 UTC m=+107.847948561 (1 nodes), 82.40224ms from start of poll, 82.81104ms from cluster start. Timeout at 2025-06-03 23:34:52.932343311 +0000 UTC m=+107.953162681, 105.21412ms after stability -2025-06-03T23:34:52.937Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:53.042Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:52.937520431 +0000 UTC m=+107.958339841 (2 nodes), 4.24µs from start of poll, 193.20232ms from cluster start. Timeout at 2025-06-03 23:34:53.042928351 +0000 UTC m=+108.063747721, 105.40788ms after stability -2025-06-03T23:34:53.043Z [INFO] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: updating configuration: command=RemoveServer server-id=server-c06443f7-bced-b107-90b7-1310a3bcc5dd server-addr= servers="[{Suffrage:Voter ID:server-28ff8f8b-eaf0-7e31-b916-783b28f55467 Address:28ff8f8b-eaf0-7e31-b916-783b28f55467} {Suffrage:Voter ID:server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b Address:d3adfec1-a5fa-9ef5-d190-192e474ccd9b}]" -2025-06-03T23:34:53.043Z [INFO] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: removed peer, stopping replication: peer=server-c06443f7-bced-b107-90b7-1310a3bcc5dd last-index=3 -2025-06-03T23:34:53.043Z [INFO] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: aborting pipeline replication: peer="{Voter server-c06443f7-bced-b107-90b7-1310a3bcc5dd c06443f7-bced-b107-90b7-1310a3bcc5dd}" -2025-06-03T23:34:53.153Z [ERROR] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: failed to pipeline appendEntries: peer="{Voter server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b d3adfec1-a5fa-9ef5-d190-192e474ccd9b}" error="append pipeline closed" -2025-06-03T23:34:53.153Z [INFO] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: aborting pipeline replication: peer="{Voter server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b d3adfec1-a5fa-9ef5-d190-192e474ccd9b}" -2025-06-03T23:34:53.155Z [ERROR] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: failed to heartbeat to: peer=d3adfec1-a5fa-9ef5-d190-192e474ccd9b backoff time=10ms error="failed to connect to peer: d3adfec1-a5fa-9ef5-d190-192e474ccd9b" -2025-06-03T23:34:53.157Z [WARN] server-c06443f7-bced-b107-90b7-1310a3bcc5dd: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2025-06-03T23:34:53.161Z [ERROR] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: failed to appendEntries to: peer="{Voter server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b d3adfec1-a5fa-9ef5-d190-192e474ccd9b}" error="failed to connect to peer: d3adfec1-a5fa-9ef5-d190-192e474ccd9b" -2025-06-03T23:34:53.176Z [ERROR] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: failed to heartbeat to: peer=d3adfec1-a5fa-9ef5-d190-192e474ccd9b backoff time=10ms error="failed to connect to peer: d3adfec1-a5fa-9ef5-d190-192e474ccd9b" -2025-06-03T23:34:53.181Z [ERROR] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: failed to appendEntries to: peer="{Voter server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b d3adfec1-a5fa-9ef5-d190-192e474ccd9b}" error="failed to connect to peer: d3adfec1-a5fa-9ef5-d190-192e474ccd9b" -2025-06-03T23:34:53.195Z [WARN] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: failed to contact: server-id=server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b time=50.13608ms -2025-06-03T23:34:53.195Z [WARN] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: failed to contact quorum of nodes, stepping down -2025-06-03T23:34:53.195Z [INFO] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: entering follower state: follower="Node at 28ff8f8b-eaf0-7e31-b916-783b28f55467 [Follower]" leader-address= leader-id= -2025-06-03T23:34:53.196Z [ERROR] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: failed to heartbeat to: peer=d3adfec1-a5fa-9ef5-d190-192e474ccd9b backoff time=10ms error="failed to connect to peer: d3adfec1-a5fa-9ef5-d190-192e474ccd9b" -2025-06-03T23:34:53.198Z [ERROR] server-28ff8f8b-eaf0-7e31-b916-783b28f55467: failed to appendEntries to: peer="{Voter server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b d3adfec1-a5fa-9ef5-d190-192e474ccd9b}" error="failed to connect to peer: d3adfec1-a5fa-9ef5-d190-192e474ccd9b" -2025-06-03T23:34:53.250Z [WARN] server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b: heartbeat timeout reached, starting election: last-leader-addr=28ff8f8b-eaf0-7e31-b916-783b28f55467 last-leader-id=server-28ff8f8b-eaf0-7e31-b916-783b28f55467 -2025-06-03T23:34:53.250Z [INFO] server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b: entering candidate state: node="Node at d3adfec1-a5fa-9ef5-d190-192e474ccd9b [Candidate]" term=3 -2025-06-03T23:34:53.250Z [ERROR] server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b: failed to make requestVote RPC: target="{Voter server-28ff8f8b-eaf0-7e31-b916-783b28f55467 28ff8f8b-eaf0-7e31-b916-783b28f55467}" error="failed to connect to peer: 28ff8f8b-eaf0-7e31-b916-783b28f55467" term=3 -2025-06-03T23:34:53.254Z [WARN] server-d3adfec1-a5fa-9ef5-d190-192e474ccd9b: rejecting vote request since node is not in configuration: from=c06443f7-bced-b107-90b7-1310a3bcc5dd ---- PASS: TestRaft_VoteNotGranted_WhenNodeNotInCluster (0.51s) +2025-06-03T23:37:51.052Z [INFO] server-074eb9a1-816d-f111-05d8-9627e3b44546: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-074eb9a1-816d-f111-05d8-9627e3b44546 Address:074eb9a1-816d-f111-05d8-9627e3b44546} {Suffrage:Voter ID:server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b Address:3f1bbcaf-270b-4e3b-088c-372ef06eb16b} {Suffrage:Voter ID:server-1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62 Address:1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62}]" +2025-06-03T23:37:51.052Z [INFO] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-074eb9a1-816d-f111-05d8-9627e3b44546 Address:074eb9a1-816d-f111-05d8-9627e3b44546} {Suffrage:Voter ID:server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b Address:3f1bbcaf-270b-4e3b-088c-372ef06eb16b} {Suffrage:Voter ID:server-1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62 Address:1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62}]" +2025-06-03T23:37:51.052Z [INFO] server-1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-074eb9a1-816d-f111-05d8-9627e3b44546 Address:074eb9a1-816d-f111-05d8-9627e3b44546} {Suffrage:Voter ID:server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b Address:3f1bbcaf-270b-4e3b-088c-372ef06eb16b} {Suffrage:Voter ID:server-1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62 Address:1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62}]" +2025-06-03T23:37:51.052Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:51.052Z [INFO] server-074eb9a1-816d-f111-05d8-9627e3b44546: entering follower state: follower="Node at 074eb9a1-816d-f111-05d8-9627e3b44546 [Follower]" leader-address= leader-id= +2025-06-03T23:37:51.052Z [INFO] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: entering follower state: follower="Node at 3f1bbcaf-270b-4e3b-088c-372ef06eb16b [Follower]" leader-address= leader-id= +2025-06-03T23:37:51.052Z [INFO] server-1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62: entering follower state: follower="Node at 1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62 [Follower]" leader-address= leader-id= +2025-06-03T23:37:51.119Z [WARN] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:51.119Z [INFO] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: entering candidate state: node="Node at 3f1bbcaf-270b-4e3b-088c-372ef06eb16b [Candidate]" term=2 +2025-06-03T23:37:51.119Z [INFO] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: election won: term=2 tally=2 +2025-06-03T23:37:51.119Z [INFO] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: entering leader state: leader="Node at 3f1bbcaf-270b-4e3b-088c-372ef06eb16b [Leader]" +2025-06-03T23:37:51.119Z [INFO] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: added peer, starting replication: peer=server-074eb9a1-816d-f111-05d8-9627e3b44546 +2025-06-03T23:37:51.119Z [INFO] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: added peer, starting replication: peer=server-1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62 +2025-06-03T23:37:51.119Z [INFO] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: pipelining replication: peer="{Voter server-074eb9a1-816d-f111-05d8-9627e3b44546 074eb9a1-816d-f111-05d8-9627e3b44546}" +2025-06-03T23:37:51.119Z [INFO] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: pipelining replication: peer="{Voter server-1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62 1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62}" +2025-06-03T23:37:51.224Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:51.119429751 +0000 UTC m=+106.441184641 (1 nodes), 66.54612ms from start of poll, 66.96732ms from cluster start. Timeout at 2025-06-03 23:37:51.224664471 +0000 UTC m=+106.546419441, 105.2348ms after stability +2025-06-03T23:37:51.229Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:51.336Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:51.229979951 +0000 UTC m=+106.551734881 (2 nodes), 3.88µs from start of poll, 177.51756ms from cluster start. Timeout at 2025-06-03 23:37:51.336109511 +0000 UTC m=+106.657864441, 106.12956ms after stability +2025-06-03T23:37:51.336Z [INFO] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: updating configuration: command=RemoveServer server-id=server-074eb9a1-816d-f111-05d8-9627e3b44546 server-addr= servers="[{Suffrage:Voter ID:server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b Address:3f1bbcaf-270b-4e3b-088c-372ef06eb16b} {Suffrage:Voter ID:server-1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62 Address:1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62}]" +2025-06-03T23:37:51.336Z [INFO] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: removed peer, stopping replication: peer=server-074eb9a1-816d-f111-05d8-9627e3b44546 last-index=3 +2025-06-03T23:37:51.336Z [INFO] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: aborting pipeline replication: peer="{Voter server-074eb9a1-816d-f111-05d8-9627e3b44546 074eb9a1-816d-f111-05d8-9627e3b44546}" +2025-06-03T23:37:51.406Z [WARN] server-074eb9a1-816d-f111-05d8-9627e3b44546: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-06-03T23:37:51.445Z [ERROR] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: failed to heartbeat to: peer=1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62 backoff time=10ms error="failed to connect to peer: 1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62" +2025-06-03T23:37:51.445Z [ERROR] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: failed to pipeline appendEntries: peer="{Voter server-1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62 1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62}" error="append pipeline closed" +2025-06-03T23:37:51.445Z [INFO] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: aborting pipeline replication: peer="{Voter server-1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62 1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62}" +2025-06-03T23:37:51.453Z [ERROR] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: failed to appendEntries to: peer="{Voter server-1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62 1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62}" error="failed to connect to peer: 1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62" +2025-06-03T23:37:51.466Z [ERROR] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: failed to heartbeat to: peer=1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62 backoff time=10ms error="failed to connect to peer: 1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62" +2025-06-03T23:37:51.473Z [ERROR] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: failed to appendEntries to: peer="{Voter server-1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62 1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62}" error="failed to connect to peer: 1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62" +2025-06-03T23:37:51.483Z [ERROR] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: failed to heartbeat to: peer=1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62 backoff time=10ms error="failed to connect to peer: 1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62" +2025-06-03T23:37:51.488Z [WARN] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: failed to contact: server-id=server-1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62 time=50.27768ms +2025-06-03T23:37:51.488Z [WARN] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: failed to contact quorum of nodes, stepping down +2025-06-03T23:37:51.488Z [INFO] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: entering follower state: follower="Node at 3f1bbcaf-270b-4e3b-088c-372ef06eb16b [Follower]" leader-address= leader-id= +2025-06-03T23:37:51.491Z [ERROR] server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b: failed to appendEntries to: peer="{Voter server-1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62 1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62}" error="failed to connect to peer: 1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62" +2025-06-03T23:37:51.511Z [WARN] server-1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62: heartbeat timeout reached, starting election: last-leader-addr=3f1bbcaf-270b-4e3b-088c-372ef06eb16b last-leader-id=server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b +2025-06-03T23:37:51.511Z [INFO] server-1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62: entering candidate state: node="Node at 1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62 [Candidate]" term=3 +2025-06-03T23:37:51.511Z [ERROR] server-1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62: failed to make requestVote RPC: target="{Voter server-3f1bbcaf-270b-4e3b-088c-372ef06eb16b 3f1bbcaf-270b-4e3b-088c-372ef06eb16b}" error="failed to connect to peer: 3f1bbcaf-270b-4e3b-088c-372ef06eb16b" term=3 +2025-06-03T23:37:51.547Z [WARN] server-1c2acdff-9f5b-f7b1-2e02-8d53ef7f2e62: rejecting vote request since node is not in configuration: from=074eb9a1-816d-f111-05d8-9627e3b44546 +--- PASS: TestRaft_VoteNotGranted_WhenNodeNotInCluster (0.50s) === RUN TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin -2025-06-03T23:34:53.255Z [INFO] server-689dc968-07a6-2aa4-022a-2c385f02e179: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-689dc968-07a6-2aa4-022a-2c385f02e179 Address:689dc968-07a6-2aa4-022a-2c385f02e179} {Suffrage:Voter ID:server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 Address:5b0f99ca-83fa-877f-efe9-7b7b34f67ef1} {Suffrage:Voter ID:server-87ea3645-6fc7-8361-a769-3a3b09067a88 Address:87ea3645-6fc7-8361-a769-3a3b09067a88}]" -2025-06-03T23:34:53.255Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-689dc968-07a6-2aa4-022a-2c385f02e179 Address:689dc968-07a6-2aa4-022a-2c385f02e179} {Suffrage:Voter ID:server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 Address:5b0f99ca-83fa-877f-efe9-7b7b34f67ef1} {Suffrage:Voter ID:server-87ea3645-6fc7-8361-a769-3a3b09067a88 Address:87ea3645-6fc7-8361-a769-3a3b09067a88}]" -2025-06-03T23:34:53.256Z [INFO] server-87ea3645-6fc7-8361-a769-3a3b09067a88: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-689dc968-07a6-2aa4-022a-2c385f02e179 Address:689dc968-07a6-2aa4-022a-2c385f02e179} {Suffrage:Voter ID:server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 Address:5b0f99ca-83fa-877f-efe9-7b7b34f67ef1} {Suffrage:Voter ID:server-87ea3645-6fc7-8361-a769-3a3b09067a88 Address:87ea3645-6fc7-8361-a769-3a3b09067a88}]" -2025-06-03T23:34:53.256Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:53.256Z [INFO] server-689dc968-07a6-2aa4-022a-2c385f02e179: entering follower state: follower="Node at 689dc968-07a6-2aa4-022a-2c385f02e179 [Follower]" leader-address= leader-id= -2025-06-03T23:34:53.256Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: entering follower state: follower="Node at 5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 [Follower]" leader-address= leader-id= -2025-06-03T23:34:53.256Z [INFO] server-87ea3645-6fc7-8361-a769-3a3b09067a88: entering follower state: follower="Node at 87ea3645-6fc7-8361-a769-3a3b09067a88 [Follower]" leader-address= leader-id= -2025-06-03T23:34:53.309Z [WARN] server-87ea3645-6fc7-8361-a769-3a3b09067a88: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:53.309Z [INFO] server-87ea3645-6fc7-8361-a769-3a3b09067a88: entering candidate state: node="Node at 87ea3645-6fc7-8361-a769-3a3b09067a88 [Candidate]" term=2 -2025-06-03T23:34:53.309Z [INFO] server-87ea3645-6fc7-8361-a769-3a3b09067a88: election won: term=2 tally=2 -2025-06-03T23:34:53.309Z [INFO] server-87ea3645-6fc7-8361-a769-3a3b09067a88: entering leader state: leader="Node at 87ea3645-6fc7-8361-a769-3a3b09067a88 [Leader]" -2025-06-03T23:34:53.309Z [INFO] server-87ea3645-6fc7-8361-a769-3a3b09067a88: added peer, starting replication: peer=server-689dc968-07a6-2aa4-022a-2c385f02e179 -2025-06-03T23:34:53.309Z [INFO] server-87ea3645-6fc7-8361-a769-3a3b09067a88: added peer, starting replication: peer=server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 -2025-06-03T23:34:53.310Z [INFO] server-87ea3645-6fc7-8361-a769-3a3b09067a88: pipelining replication: peer="{Voter server-689dc968-07a6-2aa4-022a-2c385f02e179 689dc968-07a6-2aa4-022a-2c385f02e179}" -2025-06-03T23:34:53.310Z [INFO] server-87ea3645-6fc7-8361-a769-3a3b09067a88: pipelining replication: peer="{Voter server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 5b0f99ca-83fa-877f-efe9-7b7b34f67ef1}" -2025-06-03T23:34:53.415Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:53.309968111 +0000 UTC m=+108.330787481 (1 nodes), 53.911ms from start of poll, 54.43888ms from cluster start. Timeout at 2025-06-03 23:34:53.415490671 +0000 UTC m=+108.436310041, 105.52256ms after stability -2025-06-03T23:34:53.420Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:53.526Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:53.420962271 +0000 UTC m=+108.441781641 (2 nodes), 4.08µs from start of poll, 165.43304ms from cluster start. Timeout at 2025-06-03 23:34:53.526327871 +0000 UTC m=+108.547147281, 105.36564ms after stability -2025-06-03T23:34:53.529Z [ERROR] server-87ea3645-6fc7-8361-a769-3a3b09067a88: failed to heartbeat to: peer=689dc968-07a6-2aa4-022a-2c385f02e179 backoff time=10ms error="failed to connect to peer: 689dc968-07a6-2aa4-022a-2c385f02e179" -2025-06-03T23:34:53.531Z [ERROR] server-87ea3645-6fc7-8361-a769-3a3b09067a88: failed to pipeline appendEntries: peer="{Voter server-689dc968-07a6-2aa4-022a-2c385f02e179 689dc968-07a6-2aa4-022a-2c385f02e179}" error="append pipeline closed" -2025-06-03T23:34:53.531Z [INFO] server-87ea3645-6fc7-8361-a769-3a3b09067a88: aborting pipeline replication: peer="{Voter server-689dc968-07a6-2aa4-022a-2c385f02e179 689dc968-07a6-2aa4-022a-2c385f02e179}" -2025-06-03T23:34:53.537Z [ERROR] server-87ea3645-6fc7-8361-a769-3a3b09067a88: failed to appendEntries to: peer="{Voter server-689dc968-07a6-2aa4-022a-2c385f02e179 689dc968-07a6-2aa4-022a-2c385f02e179}" error="failed to connect to peer: 689dc968-07a6-2aa4-022a-2c385f02e179" -2025-06-03T23:34:53.554Z [ERROR] server-87ea3645-6fc7-8361-a769-3a3b09067a88: failed to appendEntries to: peer="{Voter server-689dc968-07a6-2aa4-022a-2c385f02e179 689dc968-07a6-2aa4-022a-2c385f02e179}" error="failed to connect to peer: 689dc968-07a6-2aa4-022a-2c385f02e179" -2025-06-03T23:34:53.556Z [ERROR] server-87ea3645-6fc7-8361-a769-3a3b09067a88: failed to heartbeat to: peer=689dc968-07a6-2aa4-022a-2c385f02e179 backoff time=10ms error="failed to connect to peer: 689dc968-07a6-2aa4-022a-2c385f02e179" -2025-06-03T23:34:53.571Z [ERROR] server-87ea3645-6fc7-8361-a769-3a3b09067a88: failed to appendEntries to: peer="{Voter server-689dc968-07a6-2aa4-022a-2c385f02e179 689dc968-07a6-2aa4-022a-2c385f02e179}" error="failed to connect to peer: 689dc968-07a6-2aa4-022a-2c385f02e179" -2025-06-03T23:34:53.573Z [WARN] server-87ea3645-6fc7-8361-a769-3a3b09067a88: failed to contact: server-id=server-689dc968-07a6-2aa4-022a-2c385f02e179 time=50.43392ms -2025-06-03T23:34:53.577Z [ERROR] server-87ea3645-6fc7-8361-a769-3a3b09067a88: failed to heartbeat to: peer=689dc968-07a6-2aa4-022a-2c385f02e179 backoff time=10ms error="failed to connect to peer: 689dc968-07a6-2aa4-022a-2c385f02e179" -2025-06-03T23:34:53.596Z [ERROR] server-87ea3645-6fc7-8361-a769-3a3b09067a88: failed to heartbeat to: peer=689dc968-07a6-2aa4-022a-2c385f02e179 backoff time=20ms error="failed to connect to peer: 689dc968-07a6-2aa4-022a-2c385f02e179" -2025-06-03T23:34:53.601Z [ERROR] server-87ea3645-6fc7-8361-a769-3a3b09067a88: failed to appendEntries to: peer="{Voter server-689dc968-07a6-2aa4-022a-2c385f02e179 689dc968-07a6-2aa4-022a-2c385f02e179}" error="failed to connect to peer: 689dc968-07a6-2aa4-022a-2c385f02e179" -2025-06-03T23:34:53.622Z [WARN] server-87ea3645-6fc7-8361-a769-3a3b09067a88: failed to contact: server-id=server-689dc968-07a6-2aa4-022a-2c385f02e179 time=99.4876ms -2025-06-03T23:34:53.624Z [ERROR] server-87ea3645-6fc7-8361-a769-3a3b09067a88: failed to heartbeat to: peer=689dc968-07a6-2aa4-022a-2c385f02e179 backoff time=25ms error="failed to connect to peer: 689dc968-07a6-2aa4-022a-2c385f02e179" -2025-06-03T23:34:53.632Z [INFO] server-87ea3645-6fc7-8361-a769-3a3b09067a88: updating configuration: command=RemoveServer server-id=server-689dc968-07a6-2aa4-022a-2c385f02e179 server-addr= servers="[{Suffrage:Voter ID:server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 Address:5b0f99ca-83fa-877f-efe9-7b7b34f67ef1} {Suffrage:Voter ID:server-87ea3645-6fc7-8361-a769-3a3b09067a88 Address:87ea3645-6fc7-8361-a769-3a3b09067a88}]" -2025-06-03T23:34:53.632Z [INFO] server-87ea3645-6fc7-8361-a769-3a3b09067a88: removed peer, stopping replication: peer=server-689dc968-07a6-2aa4-022a-2c385f02e179 last-index=3 -2025-06-03T23:34:53.632Z [INFO] server-87ea3645-6fc7-8361-a769-3a3b09067a88: updating configuration: command=AddNonvoter server-id=server-689dc968-07a6-2aa4-022a-2c385f02e179 server-addr=689dc968-07a6-2aa4-022a-2c385f02e179 servers="[{Suffrage:Voter ID:server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 Address:5b0f99ca-83fa-877f-efe9-7b7b34f67ef1} {Suffrage:Voter ID:server-87ea3645-6fc7-8361-a769-3a3b09067a88 Address:87ea3645-6fc7-8361-a769-3a3b09067a88} {Suffrage:Nonvoter ID:server-689dc968-07a6-2aa4-022a-2c385f02e179 Address:689dc968-07a6-2aa4-022a-2c385f02e179}]" -2025-06-03T23:34:53.632Z [INFO] server-87ea3645-6fc7-8361-a769-3a3b09067a88: added peer, starting replication: peer=server-689dc968-07a6-2aa4-022a-2c385f02e179 -2025-06-03T23:34:53.632Z [ERROR] server-87ea3645-6fc7-8361-a769-3a3b09067a88: failed to appendEntries to: peer="{Nonvoter server-689dc968-07a6-2aa4-022a-2c385f02e179 689dc968-07a6-2aa4-022a-2c385f02e179}" error="failed to connect to peer: 689dc968-07a6-2aa4-022a-2c385f02e179" -2025-06-03T23:34:53.636Z [WARN] server-689dc968-07a6-2aa4-022a-2c385f02e179: heartbeat timeout reached, starting election: last-leader-addr=87ea3645-6fc7-8361-a769-3a3b09067a88 last-leader-id=server-87ea3645-6fc7-8361-a769-3a3b09067a88 -2025-06-03T23:34:53.636Z [INFO] server-689dc968-07a6-2aa4-022a-2c385f02e179: entering candidate state: node="Node at 689dc968-07a6-2aa4-022a-2c385f02e179 [Candidate]" term=23 -2025-06-03T23:34:53.636Z [WARN] server-87ea3645-6fc7-8361-a769-3a3b09067a88: rejecting vote request since we have a leader: from=689dc968-07a6-2aa4-022a-2c385f02e179 leader=87ea3645-6fc7-8361-a769-3a3b09067a88 leader-id=server-87ea3645-6fc7-8361-a769-3a3b09067a88 -2025-06-03T23:34:53.636Z [WARN] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: rejecting vote request since we have a leader: from=689dc968-07a6-2aa4-022a-2c385f02e179 leader=87ea3645-6fc7-8361-a769-3a3b09067a88 leader-id=server-87ea3645-6fc7-8361-a769-3a3b09067a88 -2025-06-03T23:34:53.649Z [ERROR] server-87ea3645-6fc7-8361-a769-3a3b09067a88: peer has newer term, stopping replication: peer="{Voter server-689dc968-07a6-2aa4-022a-2c385f02e179 689dc968-07a6-2aa4-022a-2c385f02e179}" -2025-06-03T23:34:53.649Z [INFO] server-87ea3645-6fc7-8361-a769-3a3b09067a88: entering follower state: follower="Node at 87ea3645-6fc7-8361-a769-3a3b09067a88 [Follower]" leader-address= leader-id= -2025-06-03T23:34:53.649Z [INFO] server-87ea3645-6fc7-8361-a769-3a3b09067a88: aborting pipeline replication: peer="{Voter server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 5b0f99ca-83fa-877f-efe9-7b7b34f67ef1}" -2025-06-03T23:34:53.652Z [ERROR] server-87ea3645-6fc7-8361-a769-3a3b09067a88: peer has newer term, stopping replication: peer="{Nonvoter server-689dc968-07a6-2aa4-022a-2c385f02e179 689dc968-07a6-2aa4-022a-2c385f02e179}" -2025-06-03T23:34:53.720Z [WARN] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: heartbeat timeout reached, starting election: last-leader-addr=87ea3645-6fc7-8361-a769-3a3b09067a88 last-leader-id=server-87ea3645-6fc7-8361-a769-3a3b09067a88 -2025-06-03T23:34:53.721Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: entering candidate state: node="Node at 5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 [Candidate]" term=3 -2025-06-03T23:34:53.721Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: election won: term=3 tally=2 -2025-06-03T23:34:53.721Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: entering leader state: leader="Node at 5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 [Leader]" -2025-06-03T23:34:53.721Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: added peer, starting replication: peer=server-87ea3645-6fc7-8361-a769-3a3b09067a88 -2025-06-03T23:34:53.721Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: added peer, starting replication: peer=server-689dc968-07a6-2aa4-022a-2c385f02e179 -2025-06-03T23:34:53.721Z [ERROR] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: peer has newer term, stopping replication: peer="{Nonvoter server-689dc968-07a6-2aa4-022a-2c385f02e179 689dc968-07a6-2aa4-022a-2c385f02e179}" -2025-06-03T23:34:53.721Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: pipelining replication: peer="{Voter server-87ea3645-6fc7-8361-a769-3a3b09067a88 87ea3645-6fc7-8361-a769-3a3b09067a88}" -2025-06-03T23:34:53.721Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: aborting pipeline replication: peer="{Voter server-87ea3645-6fc7-8361-a769-3a3b09067a88 87ea3645-6fc7-8361-a769-3a3b09067a88}" -2025-06-03T23:34:53.721Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: entering follower state: follower="Node at 5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 [Follower]" leader-address= leader-id= -2025-06-03T23:34:53.724Z [WARN] server-689dc968-07a6-2aa4-022a-2c385f02e179: Election timeout reached, restarting election -2025-06-03T23:34:53.724Z [INFO] server-689dc968-07a6-2aa4-022a-2c385f02e179: entering candidate state: node="Node at 689dc968-07a6-2aa4-022a-2c385f02e179 [Candidate]" term=24 -2025-06-03T23:34:53.724Z [WARN] server-87ea3645-6fc7-8361-a769-3a3b09067a88: rejecting vote request since we have a leader: from=689dc968-07a6-2aa4-022a-2c385f02e179 leader=5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 leader-id=server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 -2025-06-03T23:34:53.724Z [WARN] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: rejecting vote request since node is not a voter: from=689dc968-07a6-2aa4-022a-2c385f02e179 -2025-06-03T23:34:53.738Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:53.793Z [WARN] server-689dc968-07a6-2aa4-022a-2c385f02e179: Election timeout reached, restarting election -2025-06-03T23:34:53.793Z [INFO] server-689dc968-07a6-2aa4-022a-2c385f02e179: entering candidate state: node="Node at 689dc968-07a6-2aa4-022a-2c385f02e179 [Candidate]" term=25 -2025-06-03T23:34:53.793Z [WARN] server-87ea3645-6fc7-8361-a769-3a3b09067a88: rejecting vote request since we have a leader: from=689dc968-07a6-2aa4-022a-2c385f02e179 leader=5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 leader-id=server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 -2025-06-03T23:34:53.793Z [WARN] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: rejecting vote request since node is not a voter: from=689dc968-07a6-2aa4-022a-2c385f02e179 -2025-06-03T23:34:53.801Z [WARN] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:53.802Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: entering candidate state: node="Node at 5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 [Candidate]" term=26 -2025-06-03T23:34:53.802Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: election won: term=26 tally=2 -2025-06-03T23:34:53.802Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: entering leader state: leader="Node at 5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 [Leader]" -2025-06-03T23:34:53.802Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: added peer, starting replication: peer=server-87ea3645-6fc7-8361-a769-3a3b09067a88 -2025-06-03T23:34:53.802Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: added peer, starting replication: peer=server-689dc968-07a6-2aa4-022a-2c385f02e179 -2025-06-03T23:34:53.802Z [WARN] server-689dc968-07a6-2aa4-022a-2c385f02e179: failed to get previous log: previous-index=5 last-index=2 error="log not found" -2025-06-03T23:34:53.802Z [INFO] server-689dc968-07a6-2aa4-022a-2c385f02e179: entering follower state: follower="Node at 689dc968-07a6-2aa4-022a-2c385f02e179 [Follower]" leader-address=5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 leader-id=server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 -2025-06-03T23:34:53.802Z [WARN] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: appendEntries rejected, sending older logs: peer="{Nonvoter server-689dc968-07a6-2aa4-022a-2c385f02e179 689dc968-07a6-2aa4-022a-2c385f02e179}" next=3 -2025-06-03T23:34:53.802Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: pipelining replication: peer="{Nonvoter server-689dc968-07a6-2aa4-022a-2c385f02e179 689dc968-07a6-2aa4-022a-2c385f02e179}" -2025-06-03T23:34:53.802Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: pipelining replication: peer="{Voter server-87ea3645-6fc7-8361-a769-3a3b09067a88 87ea3645-6fc7-8361-a769-3a3b09067a88}" -2025-06-03T23:34:53.908Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:53.802630711 +0000 UTC m=+108.823450081 (1 nodes), 63.9604ms from start of poll, 547.10148ms from cluster start. Timeout at 2025-06-03 23:34:53.908380231 +0000 UTC m=+108.929199641, 105.74956ms after stability -2025-06-03T23:34:53.943Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: updating configuration: command=RemoveServer server-id=server-689dc968-07a6-2aa4-022a-2c385f02e179 server-addr= servers="[{Suffrage:Voter ID:server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 Address:5b0f99ca-83fa-877f-efe9-7b7b34f67ef1} {Suffrage:Voter ID:server-87ea3645-6fc7-8361-a769-3a3b09067a88 Address:87ea3645-6fc7-8361-a769-3a3b09067a88}]" -2025-06-03T23:34:53.943Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: removed peer, stopping replication: peer=server-689dc968-07a6-2aa4-022a-2c385f02e179 last-index=107 -2025-06-03T23:34:53.943Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: aborting pipeline replication: peer="{Nonvoter server-689dc968-07a6-2aa4-022a-2c385f02e179 689dc968-07a6-2aa4-022a-2c385f02e179}" -2025-06-03T23:34:53.943Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: updating configuration: command=AddVoter server-id=server-689dc968-07a6-2aa4-022a-2c385f02e179 server-addr=689dc968-07a6-2aa4-022a-2c385f02e179 servers="[{Suffrage:Voter ID:server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1 Address:5b0f99ca-83fa-877f-efe9-7b7b34f67ef1} {Suffrage:Voter ID:server-87ea3645-6fc7-8361-a769-3a3b09067a88 Address:87ea3645-6fc7-8361-a769-3a3b09067a88} {Suffrage:Voter ID:server-689dc968-07a6-2aa4-022a-2c385f02e179 Address:689dc968-07a6-2aa4-022a-2c385f02e179}]" -2025-06-03T23:34:53.943Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: added peer, starting replication: peer=server-689dc968-07a6-2aa4-022a-2c385f02e179 -2025-06-03T23:34:53.943Z [WARN] server-689dc968-07a6-2aa4-022a-2c385f02e179: failed to get previous log: previous-index=108 last-index=107 error="log not found" -2025-06-03T23:34:53.943Z [WARN] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: appendEntries rejected, sending older logs: peer="{Voter server-689dc968-07a6-2aa4-022a-2c385f02e179 689dc968-07a6-2aa4-022a-2c385f02e179}" next=108 -2025-06-03T23:34:53.943Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: pipelining replication: peer="{Voter server-689dc968-07a6-2aa4-022a-2c385f02e179 689dc968-07a6-2aa4-022a-2c385f02e179}" -2025-06-03T23:34:55.024Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:55.130Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:55.024586751 +0000 UTC m=+110.045406121 (1 nodes), 1.48µs from start of poll, 1.76905752s from cluster start. Timeout at 2025-06-03 23:34:55.130194871 +0000 UTC m=+110.151014241, 105.60812ms after stability -2025-06-03T23:34:55.130Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: aborting pipeline replication: peer="{Voter server-689dc968-07a6-2aa4-022a-2c385f02e179 689dc968-07a6-2aa4-022a-2c385f02e179}" -2025-06-03T23:34:55.130Z [INFO] server-5b0f99ca-83fa-877f-efe9-7b7b34f67ef1: aborting pipeline replication: peer="{Voter server-87ea3645-6fc7-8361-a769-3a3b09067a88 87ea3645-6fc7-8361-a769-3a3b09067a88}" ---- PASS: TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin (1.88s) +2025-06-03T23:37:51.548Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-02f3c82e-7627-f9b1-d50c-6e56e30cf428 Address:02f3c82e-7627-f9b1-d50c-6e56e30cf428} {Suffrage:Voter ID:server-3fe3683f-2a41-872d-a20e-6e81b09c73af Address:3fe3683f-2a41-872d-a20e-6e81b09c73af} {Suffrage:Voter ID:server-5b670af8-ea97-55d9-daa8-167698893e33 Address:5b670af8-ea97-55d9-daa8-167698893e33}]" +2025-06-03T23:37:51.548Z [INFO] server-3fe3683f-2a41-872d-a20e-6e81b09c73af: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-02f3c82e-7627-f9b1-d50c-6e56e30cf428 Address:02f3c82e-7627-f9b1-d50c-6e56e30cf428} {Suffrage:Voter ID:server-3fe3683f-2a41-872d-a20e-6e81b09c73af Address:3fe3683f-2a41-872d-a20e-6e81b09c73af} {Suffrage:Voter ID:server-5b670af8-ea97-55d9-daa8-167698893e33 Address:5b670af8-ea97-55d9-daa8-167698893e33}]" +2025-06-03T23:37:51.548Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: entering follower state: follower="Node at 02f3c82e-7627-f9b1-d50c-6e56e30cf428 [Follower]" leader-address= leader-id= +2025-06-03T23:37:51.548Z [INFO] server-3fe3683f-2a41-872d-a20e-6e81b09c73af: entering follower state: follower="Node at 3fe3683f-2a41-872d-a20e-6e81b09c73af [Follower]" leader-address= leader-id= +2025-06-03T23:37:51.548Z [INFO] server-5b670af8-ea97-55d9-daa8-167698893e33: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-02f3c82e-7627-f9b1-d50c-6e56e30cf428 Address:02f3c82e-7627-f9b1-d50c-6e56e30cf428} {Suffrage:Voter ID:server-3fe3683f-2a41-872d-a20e-6e81b09c73af Address:3fe3683f-2a41-872d-a20e-6e81b09c73af} {Suffrage:Voter ID:server-5b670af8-ea97-55d9-daa8-167698893e33 Address:5b670af8-ea97-55d9-daa8-167698893e33}]" +2025-06-03T23:37:51.548Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:51.548Z [INFO] server-5b670af8-ea97-55d9-daa8-167698893e33: entering follower state: follower="Node at 5b670af8-ea97-55d9-daa8-167698893e33 [Follower]" leader-address= leader-id= +2025-06-03T23:37:51.607Z [WARN] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:51.607Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: entering candidate state: node="Node at 02f3c82e-7627-f9b1-d50c-6e56e30cf428 [Candidate]" term=2 +2025-06-03T23:37:51.607Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: election won: term=2 tally=2 +2025-06-03T23:37:51.607Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: entering leader state: leader="Node at 02f3c82e-7627-f9b1-d50c-6e56e30cf428 [Leader]" +2025-06-03T23:37:51.607Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: added peer, starting replication: peer=server-3fe3683f-2a41-872d-a20e-6e81b09c73af +2025-06-03T23:37:51.607Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: added peer, starting replication: peer=server-5b670af8-ea97-55d9-daa8-167698893e33 +2025-06-03T23:37:51.607Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: pipelining replication: peer="{Voter server-3fe3683f-2a41-872d-a20e-6e81b09c73af 3fe3683f-2a41-872d-a20e-6e81b09c73af}" +2025-06-03T23:37:51.607Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: pipelining replication: peer="{Voter server-5b670af8-ea97-55d9-daa8-167698893e33 5b670af8-ea97-55d9-daa8-167698893e33}" +2025-06-03T23:37:51.713Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:51.607728471 +0000 UTC m=+106.929483401 (1 nodes), 59.08152ms from start of poll, 59.58492ms from cluster start. Timeout at 2025-06-03 23:37:51.713639151 +0000 UTC m=+107.035394081, 105.91068ms after stability +2025-06-03T23:37:51.719Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:51.825Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:51.719411111 +0000 UTC m=+107.041166041 (2 nodes), 4.08µs from start of poll, 171.26756ms from cluster start. Timeout at 2025-06-03 23:37:51.825337471 +0000 UTC m=+107.147092401, 105.92636ms after stability +2025-06-03T23:37:51.825Z [ERROR] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: failed to pipeline appendEntries: peer="{Voter server-3fe3683f-2a41-872d-a20e-6e81b09c73af 3fe3683f-2a41-872d-a20e-6e81b09c73af}" error="append pipeline closed" +2025-06-03T23:37:51.825Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: aborting pipeline replication: peer="{Voter server-3fe3683f-2a41-872d-a20e-6e81b09c73af 3fe3683f-2a41-872d-a20e-6e81b09c73af}" +2025-06-03T23:37:51.827Z [ERROR] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: failed to heartbeat to: peer=3fe3683f-2a41-872d-a20e-6e81b09c73af backoff time=10ms error="failed to connect to peer: 3fe3683f-2a41-872d-a20e-6e81b09c73af" +2025-06-03T23:37:51.836Z [ERROR] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: failed to appendEntries to: peer="{Voter server-3fe3683f-2a41-872d-a20e-6e81b09c73af 3fe3683f-2a41-872d-a20e-6e81b09c73af}" error="failed to connect to peer: 3fe3683f-2a41-872d-a20e-6e81b09c73af" +2025-06-03T23:37:51.843Z [ERROR] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: failed to heartbeat to: peer=3fe3683f-2a41-872d-a20e-6e81b09c73af backoff time=10ms error="failed to connect to peer: 3fe3683f-2a41-872d-a20e-6e81b09c73af" +2025-06-03T23:37:51.856Z [ERROR] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: failed to appendEntries to: peer="{Voter server-3fe3683f-2a41-872d-a20e-6e81b09c73af 3fe3683f-2a41-872d-a20e-6e81b09c73af}" error="failed to connect to peer: 3fe3683f-2a41-872d-a20e-6e81b09c73af" +2025-06-03T23:37:51.863Z [ERROR] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: failed to heartbeat to: peer=3fe3683f-2a41-872d-a20e-6e81b09c73af backoff time=10ms error="failed to connect to peer: 3fe3683f-2a41-872d-a20e-6e81b09c73af" +2025-06-03T23:37:51.869Z [WARN] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: failed to contact: server-id=server-3fe3683f-2a41-872d-a20e-6e81b09c73af time=50.45956ms +2025-06-03T23:37:51.875Z [ERROR] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: failed to appendEntries to: peer="{Voter server-3fe3683f-2a41-872d-a20e-6e81b09c73af 3fe3683f-2a41-872d-a20e-6e81b09c73af}" error="failed to connect to peer: 3fe3683f-2a41-872d-a20e-6e81b09c73af" +2025-06-03T23:37:51.879Z [ERROR] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: failed to heartbeat to: peer=3fe3683f-2a41-872d-a20e-6e81b09c73af backoff time=20ms error="failed to connect to peer: 3fe3683f-2a41-872d-a20e-6e81b09c73af" +2025-06-03T23:37:51.906Z [ERROR] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: failed to appendEntries to: peer="{Voter server-3fe3683f-2a41-872d-a20e-6e81b09c73af 3fe3683f-2a41-872d-a20e-6e81b09c73af}" error="failed to connect to peer: 3fe3683f-2a41-872d-a20e-6e81b09c73af" +2025-06-03T23:37:51.908Z [ERROR] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: failed to heartbeat to: peer=3fe3683f-2a41-872d-a20e-6e81b09c73af backoff time=25ms error="failed to connect to peer: 3fe3683f-2a41-872d-a20e-6e81b09c73af" +2025-06-03T23:37:51.910Z [WARN] server-3fe3683f-2a41-872d-a20e-6e81b09c73af: heartbeat timeout reached, starting election: last-leader-addr=02f3c82e-7627-f9b1-d50c-6e56e30cf428 last-leader-id=server-02f3c82e-7627-f9b1-d50c-6e56e30cf428 +2025-06-03T23:37:51.910Z [INFO] server-3fe3683f-2a41-872d-a20e-6e81b09c73af: entering candidate state: node="Node at 3fe3683f-2a41-872d-a20e-6e81b09c73af [Candidate]" term=3 +2025-06-03T23:37:51.910Z [ERROR] server-3fe3683f-2a41-872d-a20e-6e81b09c73af: failed to make requestVote RPC: target="{Voter server-02f3c82e-7627-f9b1-d50c-6e56e30cf428 02f3c82e-7627-f9b1-d50c-6e56e30cf428}" error="failed to connect to peer: 02f3c82e-7627-f9b1-d50c-6e56e30cf428" term=3 +2025-06-03T23:37:51.910Z [ERROR] server-3fe3683f-2a41-872d-a20e-6e81b09c73af: failed to make requestVote RPC: target="{Voter server-5b670af8-ea97-55d9-daa8-167698893e33 5b670af8-ea97-55d9-daa8-167698893e33}" error="failed to connect to peer: 5b670af8-ea97-55d9-daa8-167698893e33" term=3 +2025-06-03T23:37:51.917Z [WARN] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: failed to contact: server-id=server-3fe3683f-2a41-872d-a20e-6e81b09c73af time=98.04604ms +2025-06-03T23:37:51.930Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: updating configuration: command=RemoveServer server-id=server-3fe3683f-2a41-872d-a20e-6e81b09c73af server-addr= servers="[{Suffrage:Voter ID:server-02f3c82e-7627-f9b1-d50c-6e56e30cf428 Address:02f3c82e-7627-f9b1-d50c-6e56e30cf428} {Suffrage:Voter ID:server-5b670af8-ea97-55d9-daa8-167698893e33 Address:5b670af8-ea97-55d9-daa8-167698893e33}]" +2025-06-03T23:37:51.931Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: removed peer, stopping replication: peer=server-3fe3683f-2a41-872d-a20e-6e81b09c73af last-index=3 +2025-06-03T23:37:51.931Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: updating configuration: command=AddNonvoter server-id=server-3fe3683f-2a41-872d-a20e-6e81b09c73af server-addr=3fe3683f-2a41-872d-a20e-6e81b09c73af servers="[{Suffrage:Voter ID:server-02f3c82e-7627-f9b1-d50c-6e56e30cf428 Address:02f3c82e-7627-f9b1-d50c-6e56e30cf428} {Suffrage:Voter ID:server-5b670af8-ea97-55d9-daa8-167698893e33 Address:5b670af8-ea97-55d9-daa8-167698893e33} {Suffrage:Nonvoter ID:server-3fe3683f-2a41-872d-a20e-6e81b09c73af Address:3fe3683f-2a41-872d-a20e-6e81b09c73af}]" +2025-06-03T23:37:51.931Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: added peer, starting replication: peer=server-3fe3683f-2a41-872d-a20e-6e81b09c73af +2025-06-03T23:37:51.931Z [ERROR] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: failed to appendEntries to: peer="{Nonvoter server-3fe3683f-2a41-872d-a20e-6e81b09c73af 3fe3683f-2a41-872d-a20e-6e81b09c73af}" error="failed to connect to peer: 3fe3683f-2a41-872d-a20e-6e81b09c73af" +2025-06-03T23:37:51.950Z [ERROR] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: peer has newer term, stopping replication: peer="{Nonvoter server-3fe3683f-2a41-872d-a20e-6e81b09c73af 3fe3683f-2a41-872d-a20e-6e81b09c73af}" +2025-06-03T23:37:51.950Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: entering follower state: follower="Node at 02f3c82e-7627-f9b1-d50c-6e56e30cf428 [Follower]" leader-address= leader-id= +2025-06-03T23:37:51.950Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: aborting pipeline replication: peer="{Voter server-5b670af8-ea97-55d9-daa8-167698893e33 5b670af8-ea97-55d9-daa8-167698893e33}" +2025-06-03T23:37:51.953Z [ERROR] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: peer has newer term, stopping replication: peer="{Voter server-3fe3683f-2a41-872d-a20e-6e81b09c73af 3fe3683f-2a41-872d-a20e-6e81b09c73af}" +2025-06-03T23:37:51.988Z [WARN] server-3fe3683f-2a41-872d-a20e-6e81b09c73af: Election timeout reached, restarting election +2025-06-03T23:37:51.988Z [INFO] server-3fe3683f-2a41-872d-a20e-6e81b09c73af: entering candidate state: node="Node at 3fe3683f-2a41-872d-a20e-6e81b09c73af [Candidate]" term=23 +2025-06-03T23:37:51.988Z [WARN] server-5b670af8-ea97-55d9-daa8-167698893e33: rejecting vote request since we have a leader: from=3fe3683f-2a41-872d-a20e-6e81b09c73af leader=02f3c82e-7627-f9b1-d50c-6e56e30cf428 leader-id=server-02f3c82e-7627-f9b1-d50c-6e56e30cf428 +2025-06-03T23:37:51.989Z [WARN] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: rejecting vote request since node is not a voter: from=3fe3683f-2a41-872d-a20e-6e81b09c73af +2025-06-03T23:37:52.014Z [WARN] server-5b670af8-ea97-55d9-daa8-167698893e33: heartbeat timeout reached, starting election: last-leader-addr=02f3c82e-7627-f9b1-d50c-6e56e30cf428 last-leader-id=server-02f3c82e-7627-f9b1-d50c-6e56e30cf428 +2025-06-03T23:37:52.014Z [INFO] server-5b670af8-ea97-55d9-daa8-167698893e33: entering candidate state: node="Node at 5b670af8-ea97-55d9-daa8-167698893e33 [Candidate]" term=3 +2025-06-03T23:37:52.014Z [INFO] server-5b670af8-ea97-55d9-daa8-167698893e33: entering follower state: follower="Node at 5b670af8-ea97-55d9-daa8-167698893e33 [Follower]" leader-address= leader-id= +2025-06-03T23:37:52.036Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:52.045Z [WARN] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:52.045Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: entering candidate state: node="Node at 02f3c82e-7627-f9b1-d50c-6e56e30cf428 [Candidate]" term=24 +2025-06-03T23:37:52.045Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: election won: term=24 tally=2 +2025-06-03T23:37:52.045Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: entering leader state: leader="Node at 02f3c82e-7627-f9b1-d50c-6e56e30cf428 [Leader]" +2025-06-03T23:37:52.045Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: added peer, starting replication: peer=server-5b670af8-ea97-55d9-daa8-167698893e33 +2025-06-03T23:37:52.045Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: added peer, starting replication: peer=server-3fe3683f-2a41-872d-a20e-6e81b09c73af +2025-06-03T23:37:52.045Z [WARN] server-3fe3683f-2a41-872d-a20e-6e81b09c73af: failed to get previous log: previous-index=4 last-index=2 error="log not found" +2025-06-03T23:37:52.045Z [INFO] server-3fe3683f-2a41-872d-a20e-6e81b09c73af: entering follower state: follower="Node at 3fe3683f-2a41-872d-a20e-6e81b09c73af [Follower]" leader-address=02f3c82e-7627-f9b1-d50c-6e56e30cf428 leader-id=server-02f3c82e-7627-f9b1-d50c-6e56e30cf428 +2025-06-03T23:37:52.045Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: pipelining replication: peer="{Voter server-5b670af8-ea97-55d9-daa8-167698893e33 5b670af8-ea97-55d9-daa8-167698893e33}" +2025-06-03T23:37:52.045Z [WARN] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: appendEntries rejected, sending older logs: peer="{Nonvoter server-3fe3683f-2a41-872d-a20e-6e81b09c73af 3fe3683f-2a41-872d-a20e-6e81b09c73af}" next=3 +2025-06-03T23:37:52.046Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: pipelining replication: peer="{Nonvoter server-3fe3683f-2a41-872d-a20e-6e81b09c73af 3fe3683f-2a41-872d-a20e-6e81b09c73af}" +2025-06-03T23:37:52.151Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:52.045842831 +0000 UTC m=+107.367597761 (1 nodes), 9.02716ms from start of poll, 497.69928ms from cluster start. Timeout at 2025-06-03 23:37:52.151888991 +0000 UTC m=+107.473643921, 106.04616ms after stability +2025-06-03T23:37:52.182Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: updating configuration: command=RemoveServer server-id=server-3fe3683f-2a41-872d-a20e-6e81b09c73af server-addr= servers="[{Suffrage:Voter ID:server-02f3c82e-7627-f9b1-d50c-6e56e30cf428 Address:02f3c82e-7627-f9b1-d50c-6e56e30cf428} {Suffrage:Voter ID:server-5b670af8-ea97-55d9-daa8-167698893e33 Address:5b670af8-ea97-55d9-daa8-167698893e33}]" +2025-06-03T23:37:52.182Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: removed peer, stopping replication: peer=server-3fe3683f-2a41-872d-a20e-6e81b09c73af last-index=106 +2025-06-03T23:37:52.182Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: aborting pipeline replication: peer="{Nonvoter server-3fe3683f-2a41-872d-a20e-6e81b09c73af 3fe3683f-2a41-872d-a20e-6e81b09c73af}" +2025-06-03T23:37:52.182Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: updating configuration: command=AddVoter server-id=server-3fe3683f-2a41-872d-a20e-6e81b09c73af server-addr=3fe3683f-2a41-872d-a20e-6e81b09c73af servers="[{Suffrage:Voter ID:server-02f3c82e-7627-f9b1-d50c-6e56e30cf428 Address:02f3c82e-7627-f9b1-d50c-6e56e30cf428} {Suffrage:Voter ID:server-5b670af8-ea97-55d9-daa8-167698893e33 Address:5b670af8-ea97-55d9-daa8-167698893e33} {Suffrage:Voter ID:server-3fe3683f-2a41-872d-a20e-6e81b09c73af Address:3fe3683f-2a41-872d-a20e-6e81b09c73af}]" +2025-06-03T23:37:52.182Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: added peer, starting replication: peer=server-3fe3683f-2a41-872d-a20e-6e81b09c73af +2025-06-03T23:37:52.182Z [WARN] server-3fe3683f-2a41-872d-a20e-6e81b09c73af: failed to get previous log: previous-index=107 last-index=106 error="log not found" +2025-06-03T23:37:52.182Z [WARN] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: appendEntries rejected, sending older logs: peer="{Voter server-3fe3683f-2a41-872d-a20e-6e81b09c73af 3fe3683f-2a41-872d-a20e-6e81b09c73af}" next=107 +2025-06-03T23:37:52.183Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: pipelining replication: peer="{Voter server-3fe3683f-2a41-872d-a20e-6e81b09c73af 3fe3683f-2a41-872d-a20e-6e81b09c73af}" +2025-06-03T23:37:53.255Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:53.361Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:53.255654551 +0000 UTC m=+108.577409441 (1 nodes), 1.04µs from start of poll, 1.70751096s from cluster start. Timeout at 2025-06-03 23:37:53.361576791 +0000 UTC m=+108.683331721, 105.92228ms after stability +2025-06-03T23:37:53.361Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: aborting pipeline replication: peer="{Voter server-3fe3683f-2a41-872d-a20e-6e81b09c73af 3fe3683f-2a41-872d-a20e-6e81b09c73af}" +2025-06-03T23:37:53.361Z [INFO] server-02f3c82e-7627-f9b1-d50c-6e56e30cf428: aborting pipeline replication: peer="{Voter server-5b670af8-ea97-55d9-daa8-167698893e33 5b670af8-ea97-55d9-daa8-167698893e33}" +--- PASS: TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin (1.81s) === RUN TestRaft_FollowerRemovalNoElection -2025-06-03T23:34:55.131Z [INFO] server-ff54fc01-0f40-0a48-30ca-8fee6836910b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff54fc01-0f40-0a48-30ca-8fee6836910b Address:ff54fc01-0f40-0a48-30ca-8fee6836910b} {Suffrage:Voter ID:server-7e3cf5f2-7cf5-ff69-10da-c49106e8ed48 Address:7e3cf5f2-7cf5-ff69-10da-c49106e8ed48} {Suffrage:Voter ID:server-6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7 Address:6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7}]" -2025-06-03T23:34:55.131Z [INFO] server-7e3cf5f2-7cf5-ff69-10da-c49106e8ed48: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff54fc01-0f40-0a48-30ca-8fee6836910b Address:ff54fc01-0f40-0a48-30ca-8fee6836910b} {Suffrage:Voter ID:server-7e3cf5f2-7cf5-ff69-10da-c49106e8ed48 Address:7e3cf5f2-7cf5-ff69-10da-c49106e8ed48} {Suffrage:Voter ID:server-6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7 Address:6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7}]" -2025-06-03T23:34:55.131Z [INFO] server-ff54fc01-0f40-0a48-30ca-8fee6836910b: entering follower state: follower="Node at ff54fc01-0f40-0a48-30ca-8fee6836910b [Follower]" leader-address= leader-id= -2025-06-03T23:34:55.131Z [INFO] server-7e3cf5f2-7cf5-ff69-10da-c49106e8ed48: entering follower state: follower="Node at 7e3cf5f2-7cf5-ff69-10da-c49106e8ed48 [Follower]" leader-address= leader-id= -2025-06-03T23:34:55.131Z [INFO] server-6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff54fc01-0f40-0a48-30ca-8fee6836910b Address:ff54fc01-0f40-0a48-30ca-8fee6836910b} {Suffrage:Voter ID:server-7e3cf5f2-7cf5-ff69-10da-c49106e8ed48 Address:7e3cf5f2-7cf5-ff69-10da-c49106e8ed48} {Suffrage:Voter ID:server-6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7 Address:6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7}]" -2025-06-03T23:34:55.131Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:55.131Z [INFO] server-6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7: entering follower state: follower="Node at 6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7 [Follower]" leader-address= leader-id= -2025-06-03T23:34:55.304Z [WARN] server-6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:55.304Z [INFO] server-6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7: entering candidate state: node="Node at 6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7 [Candidate]" term=2 -2025-06-03T23:34:55.304Z [INFO] server-6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7: election won: term=2 tally=2 -2025-06-03T23:34:55.304Z [INFO] server-6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7: entering leader state: leader="Node at 6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7 [Leader]" -2025-06-03T23:34:55.304Z [INFO] server-6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7: added peer, starting replication: peer=server-ff54fc01-0f40-0a48-30ca-8fee6836910b -2025-06-03T23:34:55.304Z [INFO] server-6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7: added peer, starting replication: peer=server-7e3cf5f2-7cf5-ff69-10da-c49106e8ed48 -2025-06-03T23:34:55.305Z [INFO] server-6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7: pipelining replication: peer="{Voter server-7e3cf5f2-7cf5-ff69-10da-c49106e8ed48 7e3cf5f2-7cf5-ff69-10da-c49106e8ed48}" -2025-06-03T23:34:55.305Z [INFO] server-6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7: pipelining replication: peer="{Voter server-ff54fc01-0f40-0a48-30ca-8fee6836910b ff54fc01-0f40-0a48-30ca-8fee6836910b}" -2025-06-03T23:34:55.511Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:55.305038991 +0000 UTC m=+110.325858361 (1 nodes), 173.43068ms from start of poll, 173.89276ms from cluster start. Timeout at 2025-06-03 23:34:55.510925271 +0000 UTC m=+110.531744641, 205.88628ms after stability -2025-06-03T23:34:55.511Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:55.717Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:55.511099511 +0000 UTC m=+110.531918921 (1 nodes), 2.52µs from start of poll, 379.95332ms from cluster start. Timeout at 2025-06-03 23:34:55.717262311 +0000 UTC m=+110.738081681, 206.16276ms after stability -2025-06-03T23:34:55.722Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:55.928Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:55.722646351 +0000 UTC m=+110.743465721 (2 nodes), 4.64µs from start of poll, 591.50012ms from cluster start. Timeout at 2025-06-03 23:34:55.928063191 +0000 UTC m=+110.948882601, 205.41688ms after stability -2025-06-03T23:34:55.928Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:56.133Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:55.928190031 +0000 UTC m=+110.949009441 (1 nodes), 3.12µs from start of poll, 797.04384ms from cluster start. Timeout at 2025-06-03 23:34:56.133265311 +0000 UTC m=+111.154084681, 205.07524ms after stability - raft_test.go:2961: [INFO] restarting Node at ff54fc01-0f40-0a48-30ca-8fee6836910b [Follower] -2025-06-03T23:34:56.133Z [INFO] server-ff54fc01-0f40-0a48-30ca-8fee6836910b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff54fc01-0f40-0a48-30ca-8fee6836910b Address:ff54fc01-0f40-0a48-30ca-8fee6836910b} {Suffrage:Voter ID:server-7e3cf5f2-7cf5-ff69-10da-c49106e8ed48 Address:7e3cf5f2-7cf5-ff69-10da-c49106e8ed48} {Suffrage:Voter ID:server-6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7 Address:6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7}]" -2025-06-03T23:34:56.133Z [INFO] server-ff54fc01-0f40-0a48-30ca-8fee6836910b: entering follower state: follower="Node at ff54fc01-0f40-0a48-30ca-8fee6836910b [Follower]" leader-address= leader-id= -2025-06-03T23:34:56.384Z [INFO] server-6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7: aborting pipeline replication: peer="{Voter server-7e3cf5f2-7cf5-ff69-10da-c49106e8ed48 7e3cf5f2-7cf5-ff69-10da-c49106e8ed48}" -2025-06-03T23:34:56.776Z [ERROR] server-6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7: failed to pipeline appendEntries: peer="{Voter server-ff54fc01-0f40-0a48-30ca-8fee6836910b ff54fc01-0f40-0a48-30ca-8fee6836910b}" error="command enqueue timeout" -2025-06-03T23:34:56.776Z [INFO] server-6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7: aborting pipeline replication: peer="{Voter server-ff54fc01-0f40-0a48-30ca-8fee6836910b ff54fc01-0f40-0a48-30ca-8fee6836910b}" -2025-06-03T23:34:56.888Z [ERROR] server-6434b9bc-6be9-3b90-5d9d-afcc3e1b70b7: failed to heartbeat to: peer=ff54fc01-0f40-0a48-30ca-8fee6836910b backoff time=10ms error="command timed out" ---- PASS: TestRaft_FollowerRemovalNoElection (1.76s) +2025-06-03T23:37:53.362Z [INFO] server-dd43d255-dca0-7bba-9071-d0b33e7c79f5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd43d255-dca0-7bba-9071-d0b33e7c79f5 Address:dd43d255-dca0-7bba-9071-d0b33e7c79f5} {Suffrage:Voter ID:server-09bb642a-77ed-ac78-3a62-a90d18cd20da Address:09bb642a-77ed-ac78-3a62-a90d18cd20da} {Suffrage:Voter ID:server-d8ecbde0-5fd9-4584-9cc5-b12de7e2af57 Address:d8ecbde0-5fd9-4584-9cc5-b12de7e2af57}]" +2025-06-03T23:37:53.362Z [INFO] server-dd43d255-dca0-7bba-9071-d0b33e7c79f5: entering follower state: follower="Node at dd43d255-dca0-7bba-9071-d0b33e7c79f5 [Follower]" leader-address= leader-id= +2025-06-03T23:37:53.362Z [INFO] server-09bb642a-77ed-ac78-3a62-a90d18cd20da: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd43d255-dca0-7bba-9071-d0b33e7c79f5 Address:dd43d255-dca0-7bba-9071-d0b33e7c79f5} {Suffrage:Voter ID:server-09bb642a-77ed-ac78-3a62-a90d18cd20da Address:09bb642a-77ed-ac78-3a62-a90d18cd20da} {Suffrage:Voter ID:server-d8ecbde0-5fd9-4584-9cc5-b12de7e2af57 Address:d8ecbde0-5fd9-4584-9cc5-b12de7e2af57}]" +2025-06-03T23:37:53.362Z [INFO] server-09bb642a-77ed-ac78-3a62-a90d18cd20da: entering follower state: follower="Node at 09bb642a-77ed-ac78-3a62-a90d18cd20da [Follower]" leader-address= leader-id= +2025-06-03T23:37:53.362Z [INFO] server-d8ecbde0-5fd9-4584-9cc5-b12de7e2af57: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd43d255-dca0-7bba-9071-d0b33e7c79f5 Address:dd43d255-dca0-7bba-9071-d0b33e7c79f5} {Suffrage:Voter ID:server-09bb642a-77ed-ac78-3a62-a90d18cd20da Address:09bb642a-77ed-ac78-3a62-a90d18cd20da} {Suffrage:Voter ID:server-d8ecbde0-5fd9-4584-9cc5-b12de7e2af57 Address:d8ecbde0-5fd9-4584-9cc5-b12de7e2af57}]" +2025-06-03T23:37:53.363Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:53.363Z [INFO] server-d8ecbde0-5fd9-4584-9cc5-b12de7e2af57: entering follower state: follower="Node at d8ecbde0-5fd9-4584-9cc5-b12de7e2af57 [Follower]" leader-address= leader-id= +2025-06-03T23:37:53.483Z [WARN] server-09bb642a-77ed-ac78-3a62-a90d18cd20da: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:53.483Z [INFO] server-09bb642a-77ed-ac78-3a62-a90d18cd20da: entering candidate state: node="Node at 09bb642a-77ed-ac78-3a62-a90d18cd20da [Candidate]" term=2 +2025-06-03T23:37:53.483Z [INFO] server-09bb642a-77ed-ac78-3a62-a90d18cd20da: election won: term=2 tally=2 +2025-06-03T23:37:53.483Z [INFO] server-09bb642a-77ed-ac78-3a62-a90d18cd20da: entering leader state: leader="Node at 09bb642a-77ed-ac78-3a62-a90d18cd20da [Leader]" +2025-06-03T23:37:53.483Z [INFO] server-09bb642a-77ed-ac78-3a62-a90d18cd20da: added peer, starting replication: peer=server-dd43d255-dca0-7bba-9071-d0b33e7c79f5 +2025-06-03T23:37:53.483Z [INFO] server-09bb642a-77ed-ac78-3a62-a90d18cd20da: added peer, starting replication: peer=server-d8ecbde0-5fd9-4584-9cc5-b12de7e2af57 +2025-06-03T23:37:53.484Z [INFO] server-09bb642a-77ed-ac78-3a62-a90d18cd20da: pipelining replication: peer="{Voter server-dd43d255-dca0-7bba-9071-d0b33e7c79f5 dd43d255-dca0-7bba-9071-d0b33e7c79f5}" +2025-06-03T23:37:53.484Z [INFO] server-09bb642a-77ed-ac78-3a62-a90d18cd20da: pipelining replication: peer="{Voter server-d8ecbde0-5fd9-4584-9cc5-b12de7e2af57 d8ecbde0-5fd9-4584-9cc5-b12de7e2af57}" +2025-06-03T23:37:53.689Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:53.483967791 +0000 UTC m=+108.805722721 (1 nodes), 120.91716ms from start of poll, 121.3884ms from cluster start. Timeout at 2025-06-03 23:37:53.689050391 +0000 UTC m=+109.010805321, 205.0826ms after stability +2025-06-03T23:37:53.689Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:53.894Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:53.689191911 +0000 UTC m=+109.010946841 (1 nodes), 2.8µs from start of poll, 326.61252ms from cluster start. Timeout at 2025-06-03 23:37:53.894471031 +0000 UTC m=+109.216225961, 205.27912ms after stability +2025-06-03T23:37:53.899Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:54.104Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:53.899623151 +0000 UTC m=+109.221378081 (2 nodes), 3.04µs from start of poll, 537.04376ms from cluster start. Timeout at 2025-06-03 23:37:54.104866711 +0000 UTC m=+109.426621641, 205.24356ms after stability +2025-06-03T23:37:54.104Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:54.310Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:54.104969511 +0000 UTC m=+109.426724441 (1 nodes), 2.88µs from start of poll, 742.39012ms from cluster start. Timeout at 2025-06-03 23:37:54.310035751 +0000 UTC m=+109.631790681, 205.06624ms after stability + raft_test.go:2961: [INFO] restarting Node at dd43d255-dca0-7bba-9071-d0b33e7c79f5 [Follower] +2025-06-03T23:37:54.310Z [INFO] server-dd43d255-dca0-7bba-9071-d0b33e7c79f5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd43d255-dca0-7bba-9071-d0b33e7c79f5 Address:dd43d255-dca0-7bba-9071-d0b33e7c79f5} {Suffrage:Voter ID:server-09bb642a-77ed-ac78-3a62-a90d18cd20da Address:09bb642a-77ed-ac78-3a62-a90d18cd20da} {Suffrage:Voter ID:server-d8ecbde0-5fd9-4584-9cc5-b12de7e2af57 Address:d8ecbde0-5fd9-4584-9cc5-b12de7e2af57}]" +2025-06-03T23:37:54.310Z [INFO] server-dd43d255-dca0-7bba-9071-d0b33e7c79f5: entering follower state: follower="Node at dd43d255-dca0-7bba-9071-d0b33e7c79f5 [Follower]" leader-address= leader-id= +2025-06-03T23:37:54.561Z [INFO] server-09bb642a-77ed-ac78-3a62-a90d18cd20da: aborting pipeline replication: peer="{Voter server-d8ecbde0-5fd9-4584-9cc5-b12de7e2af57 d8ecbde0-5fd9-4584-9cc5-b12de7e2af57}" +2025-06-03T23:37:54.948Z [ERROR] server-09bb642a-77ed-ac78-3a62-a90d18cd20da: failed to pipeline appendEntries: peer="{Voter server-dd43d255-dca0-7bba-9071-d0b33e7c79f5 dd43d255-dca0-7bba-9071-d0b33e7c79f5}" error="command enqueue timeout" +2025-06-03T23:37:54.948Z [INFO] server-09bb642a-77ed-ac78-3a62-a90d18cd20da: aborting pipeline replication: peer="{Voter server-dd43d255-dca0-7bba-9071-d0b33e7c79f5 dd43d255-dca0-7bba-9071-d0b33e7c79f5}" +2025-06-03T23:37:55.073Z [ERROR] server-09bb642a-77ed-ac78-3a62-a90d18cd20da: failed to heartbeat to: peer=dd43d255-dca0-7bba-9071-d0b33e7c79f5 backoff time=10ms error="command timed out" +--- PASS: TestRaft_FollowerRemovalNoElection (1.71s) === RUN TestRaft_VoteWithNoIDNoAddr -2025-06-03T23:34:56.889Z [INFO] server-754fe75c-6a6e-2d32-712f-81e3902c60d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-754fe75c-6a6e-2d32-712f-81e3902c60d0 Address:754fe75c-6a6e-2d32-712f-81e3902c60d0} {Suffrage:Voter ID:server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183 Address:8e4fa05c-3f09-f0c4-93d2-93fd61b5e183} {Suffrage:Voter ID:server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92 Address:03e78feb-d6c2-cac3-e60e-c8dcd521cf92}]" -2025-06-03T23:34:56.889Z [INFO] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-754fe75c-6a6e-2d32-712f-81e3902c60d0 Address:754fe75c-6a6e-2d32-712f-81e3902c60d0} {Suffrage:Voter ID:server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183 Address:8e4fa05c-3f09-f0c4-93d2-93fd61b5e183} {Suffrage:Voter ID:server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92 Address:03e78feb-d6c2-cac3-e60e-c8dcd521cf92}]" -2025-06-03T23:34:56.889Z [INFO] server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-754fe75c-6a6e-2d32-712f-81e3902c60d0 Address:754fe75c-6a6e-2d32-712f-81e3902c60d0} {Suffrage:Voter ID:server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183 Address:8e4fa05c-3f09-f0c4-93d2-93fd61b5e183} {Suffrage:Voter ID:server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92 Address:03e78feb-d6c2-cac3-e60e-c8dcd521cf92}]" -2025-06-03T23:34:56.889Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:56.889Z [INFO] server-754fe75c-6a6e-2d32-712f-81e3902c60d0: entering follower state: follower="Node at 754fe75c-6a6e-2d32-712f-81e3902c60d0 [Follower]" leader-address= leader-id= -2025-06-03T23:34:56.889Z [INFO] server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92: entering follower state: follower="Node at 03e78feb-d6c2-cac3-e60e-c8dcd521cf92 [Follower]" leader-address= leader-id= -2025-06-03T23:34:56.889Z [INFO] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: entering follower state: follower="Node at 8e4fa05c-3f09-f0c4-93d2-93fd61b5e183 [Follower]" leader-address= leader-id= -2025-06-03T23:34:56.953Z [WARN] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-06-03T23:34:56.953Z [INFO] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: entering candidate state: node="Node at 8e4fa05c-3f09-f0c4-93d2-93fd61b5e183 [Candidate]" term=2 -2025-06-03T23:34:56.954Z [INFO] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: election won: term=2 tally=2 -2025-06-03T23:34:56.954Z [INFO] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: entering leader state: leader="Node at 8e4fa05c-3f09-f0c4-93d2-93fd61b5e183 [Leader]" -2025-06-03T23:34:56.954Z [INFO] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: added peer, starting replication: peer=server-754fe75c-6a6e-2d32-712f-81e3902c60d0 -2025-06-03T23:34:56.954Z [INFO] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: added peer, starting replication: peer=server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92 -2025-06-03T23:34:56.954Z [INFO] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: pipelining replication: peer="{Voter server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92 03e78feb-d6c2-cac3-e60e-c8dcd521cf92}" -2025-06-03T23:34:56.954Z [INFO] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: pipelining replication: peer="{Voter server-754fe75c-6a6e-2d32-712f-81e3902c60d0 754fe75c-6a6e-2d32-712f-81e3902c60d0}" -2025-06-03T23:34:57.060Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:56.954169151 +0000 UTC m=+111.974988521 (1 nodes), 64.43068ms from start of poll, 65.294ms from cluster start. Timeout at 2025-06-03 23:34:57.060142031 +0000 UTC m=+112.080961441, 105.97292ms after stability -2025-06-03T23:34:57.060Z [INFO] cluster: starting stability test: raft-state=Leader -2025-06-03T23:34:57.166Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:34:57.060285431 +0000 UTC m=+112.081104801 (1 nodes), 3.32µs from start of poll, 171.41028ms from cluster start. Timeout at 2025-06-03 23:34:57.166026311 +0000 UTC m=+112.186845681, 105.74088ms after stability -2025-06-03T23:34:57.171Z [INFO] cluster: starting stability test: raft-state=Follower -2025-06-03T23:34:57.277Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:34:57.171274791 +0000 UTC m=+112.192094161 (2 nodes), 3.16µs from start of poll, 282.39964ms from cluster start. Timeout at 2025-06-03 23:34:57.277432071 +0000 UTC m=+112.298251441, 106.15728ms after stability -2025-06-03T23:34:57.281Z [ERROR] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to pipeline appendEntries: peer="{Voter server-754fe75c-6a6e-2d32-712f-81e3902c60d0 754fe75c-6a6e-2d32-712f-81e3902c60d0}" error="append pipeline closed" -2025-06-03T23:34:57.281Z [INFO] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: aborting pipeline replication: peer="{Voter server-754fe75c-6a6e-2d32-712f-81e3902c60d0 754fe75c-6a6e-2d32-712f-81e3902c60d0}" -2025-06-03T23:34:57.281Z [ERROR] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to pipeline appendEntries: peer="{Voter server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92 03e78feb-d6c2-cac3-e60e-c8dcd521cf92}" error="append pipeline closed" -2025-06-03T23:34:57.281Z [INFO] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: aborting pipeline replication: peer="{Voter server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92 03e78feb-d6c2-cac3-e60e-c8dcd521cf92}" -2025-06-03T23:34:57.284Z [ERROR] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to heartbeat to: peer=754fe75c-6a6e-2d32-712f-81e3902c60d0 backoff time=10ms error="failed to connect to peer: 754fe75c-6a6e-2d32-712f-81e3902c60d0" -2025-06-03T23:34:57.286Z [ERROR] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to heartbeat to: peer=03e78feb-d6c2-cac3-e60e-c8dcd521cf92 backoff time=10ms error="failed to connect to peer: 03e78feb-d6c2-cac3-e60e-c8dcd521cf92" -2025-06-03T23:34:57.289Z [ERROR] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to appendEntries to: peer="{Voter server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92 03e78feb-d6c2-cac3-e60e-c8dcd521cf92}" error="failed to connect to peer: 03e78feb-d6c2-cac3-e60e-c8dcd521cf92" -2025-06-03T23:34:57.290Z [ERROR] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to appendEntries to: peer="{Voter server-754fe75c-6a6e-2d32-712f-81e3902c60d0 754fe75c-6a6e-2d32-712f-81e3902c60d0}" error="failed to connect to peer: 754fe75c-6a6e-2d32-712f-81e3902c60d0" -2025-06-03T23:34:57.301Z [ERROR] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to heartbeat to: peer=754fe75c-6a6e-2d32-712f-81e3902c60d0 backoff time=10ms error="failed to connect to peer: 754fe75c-6a6e-2d32-712f-81e3902c60d0" -2025-06-03T23:34:57.304Z [ERROR] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to heartbeat to: peer=03e78feb-d6c2-cac3-e60e-c8dcd521cf92 backoff time=10ms error="failed to connect to peer: 03e78feb-d6c2-cac3-e60e-c8dcd521cf92" -2025-06-03T23:34:57.306Z [ERROR] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to appendEntries to: peer="{Voter server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92 03e78feb-d6c2-cac3-e60e-c8dcd521cf92}" error="failed to connect to peer: 03e78feb-d6c2-cac3-e60e-c8dcd521cf92" -2025-06-03T23:34:57.307Z [ERROR] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to appendEntries to: peer="{Voter server-754fe75c-6a6e-2d32-712f-81e3902c60d0 754fe75c-6a6e-2d32-712f-81e3902c60d0}" error="failed to connect to peer: 754fe75c-6a6e-2d32-712f-81e3902c60d0" -2025-06-03T23:34:57.322Z [ERROR] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to heartbeat to: peer=03e78feb-d6c2-cac3-e60e-c8dcd521cf92 backoff time=10ms error="failed to connect to peer: 03e78feb-d6c2-cac3-e60e-c8dcd521cf92" -2025-06-03T23:34:57.322Z [ERROR] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to heartbeat to: peer=754fe75c-6a6e-2d32-712f-81e3902c60d0 backoff time=10ms error="failed to connect to peer: 754fe75c-6a6e-2d32-712f-81e3902c60d0" -2025-06-03T23:34:57.326Z [ERROR] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to appendEntries to: peer="{Voter server-754fe75c-6a6e-2d32-712f-81e3902c60d0 754fe75c-6a6e-2d32-712f-81e3902c60d0}" error="failed to connect to peer: 754fe75c-6a6e-2d32-712f-81e3902c60d0" -2025-06-03T23:34:57.327Z [WARN] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to contact: server-id=server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92 time=50.04548ms -2025-06-03T23:34:57.327Z [ERROR] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to appendEntries to: peer="{Voter server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92 03e78feb-d6c2-cac3-e60e-c8dcd521cf92}" error="failed to connect to peer: 03e78feb-d6c2-cac3-e60e-c8dcd521cf92" -2025-06-03T23:34:57.338Z [WARN] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to contact: server-id=server-754fe75c-6a6e-2d32-712f-81e3902c60d0 time=60.60472ms -2025-06-03T23:34:57.338Z [WARN] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to contact: server-id=server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92 time=60.65604ms -2025-06-03T23:34:57.338Z [WARN] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to contact quorum of nodes, stepping down -2025-06-03T23:34:57.338Z [INFO] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: entering follower state: follower="Node at 8e4fa05c-3f09-f0c4-93d2-93fd61b5e183 [Follower]" leader-address= leader-id= -2025-06-03T23:34:57.339Z [ERROR] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to heartbeat to: peer=754fe75c-6a6e-2d32-712f-81e3902c60d0 backoff time=20ms error="failed to connect to peer: 754fe75c-6a6e-2d32-712f-81e3902c60d0" -2025-06-03T23:34:57.341Z [ERROR] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to heartbeat to: peer=03e78feb-d6c2-cac3-e60e-c8dcd521cf92 backoff time=20ms error="failed to connect to peer: 03e78feb-d6c2-cac3-e60e-c8dcd521cf92" -2025-06-03T23:34:57.352Z [ERROR] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to appendEntries to: peer="{Voter server-754fe75c-6a6e-2d32-712f-81e3902c60d0 754fe75c-6a6e-2d32-712f-81e3902c60d0}" error="failed to connect to peer: 754fe75c-6a6e-2d32-712f-81e3902c60d0" -2025-06-03T23:34:57.355Z [WARN] server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92: heartbeat timeout reached, starting election: last-leader-addr=8e4fa05c-3f09-f0c4-93d2-93fd61b5e183 last-leader-id=server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183 -2025-06-03T23:34:57.355Z [INFO] server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92: entering candidate state: node="Node at 03e78feb-d6c2-cac3-e60e-c8dcd521cf92 [Candidate]" term=3 -2025-06-03T23:34:57.355Z [ERROR] server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92: failed to make requestVote RPC: target="{Voter server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183 8e4fa05c-3f09-f0c4-93d2-93fd61b5e183}" error="failed to connect to peer: 8e4fa05c-3f09-f0c4-93d2-93fd61b5e183" term=3 -2025-06-03T23:34:57.355Z [WARN] server-754fe75c-6a6e-2d32-712f-81e3902c60d0: rejecting vote request since we have a leader: from=03e78feb-d6c2-cac3-e60e-c8dcd521cf92 leader=8e4fa05c-3f09-f0c4-93d2-93fd61b5e183 leader-id=server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183 -2025-06-03T23:34:57.357Z [ERROR] server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183: failed to appendEntries to: peer="{Voter server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92 03e78feb-d6c2-cac3-e60e-c8dcd521cf92}" error="failed to connect to peer: 03e78feb-d6c2-cac3-e60e-c8dcd521cf92" -2025-06-03T23:34:57.375Z [WARN] server-754fe75c-6a6e-2d32-712f-81e3902c60d0: heartbeat timeout reached, starting election: last-leader-addr=8e4fa05c-3f09-f0c4-93d2-93fd61b5e183 last-leader-id=server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183 -2025-06-03T23:34:57.375Z [INFO] server-754fe75c-6a6e-2d32-712f-81e3902c60d0: entering candidate state: node="Node at 754fe75c-6a6e-2d32-712f-81e3902c60d0 [Candidate]" term=3 -2025-06-03T23:34:57.375Z [INFO] server-03e78feb-d6c2-cac3-e60e-c8dcd521cf92: duplicate requestVote for same term: term=3 -2025-06-03T23:34:57.375Z [ERROR] server-754fe75c-6a6e-2d32-712f-81e3902c60d0: failed to make requestVote RPC: target="{Voter server-8e4fa05c-3f09-f0c4-93d2-93fd61b5e183 8e4fa05c-3f09-f0c4-93d2-93fd61b5e183}" error="failed to connect to peer: 8e4fa05c-3f09-f0c4-93d2-93fd61b5e183" term=3 -2025-06-03T23:34:57.382Z [INFO] server-754fe75c-6a6e-2d32-712f-81e3902c60d0: entering follower state: follower="Node at 754fe75c-6a6e-2d32-712f-81e3902c60d0 [Follower]" leader-address= leader-id= ---- PASS: TestRaft_VoteWithNoIDNoAddr (0.49s) +2025-06-03T23:37:55.074Z [INFO] server-1fc834e3-a7b1-eac5-044a-77bb5eb3aad4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1fc834e3-a7b1-eac5-044a-77bb5eb3aad4 Address:1fc834e3-a7b1-eac5-044a-77bb5eb3aad4} {Suffrage:Voter ID:server-9d36fd3a-2b51-e783-05e8-2e8f5e868960 Address:9d36fd3a-2b51-e783-05e8-2e8f5e868960} {Suffrage:Voter ID:server-883f905d-cb9b-6b64-dc4f-906e59e0e728 Address:883f905d-cb9b-6b64-dc4f-906e59e0e728}]" +2025-06-03T23:37:55.074Z [INFO] server-9d36fd3a-2b51-e783-05e8-2e8f5e868960: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1fc834e3-a7b1-eac5-044a-77bb5eb3aad4 Address:1fc834e3-a7b1-eac5-044a-77bb5eb3aad4} {Suffrage:Voter ID:server-9d36fd3a-2b51-e783-05e8-2e8f5e868960 Address:9d36fd3a-2b51-e783-05e8-2e8f5e868960} {Suffrage:Voter ID:server-883f905d-cb9b-6b64-dc4f-906e59e0e728 Address:883f905d-cb9b-6b64-dc4f-906e59e0e728}]" +2025-06-03T23:37:55.074Z [INFO] server-1fc834e3-a7b1-eac5-044a-77bb5eb3aad4: entering follower state: follower="Node at 1fc834e3-a7b1-eac5-044a-77bb5eb3aad4 [Follower]" leader-address= leader-id= +2025-06-03T23:37:55.074Z [INFO] server-9d36fd3a-2b51-e783-05e8-2e8f5e868960: entering follower state: follower="Node at 9d36fd3a-2b51-e783-05e8-2e8f5e868960 [Follower]" leader-address= leader-id= +2025-06-03T23:37:55.074Z [INFO] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1fc834e3-a7b1-eac5-044a-77bb5eb3aad4 Address:1fc834e3-a7b1-eac5-044a-77bb5eb3aad4} {Suffrage:Voter ID:server-9d36fd3a-2b51-e783-05e8-2e8f5e868960 Address:9d36fd3a-2b51-e783-05e8-2e8f5e868960} {Suffrage:Voter ID:server-883f905d-cb9b-6b64-dc4f-906e59e0e728 Address:883f905d-cb9b-6b64-dc4f-906e59e0e728}]" +2025-06-03T23:37:55.074Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:55.074Z [INFO] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: entering follower state: follower="Node at 883f905d-cb9b-6b64-dc4f-906e59e0e728 [Follower]" leader-address= leader-id= +2025-06-03T23:37:55.127Z [WARN] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:55.127Z [INFO] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: entering candidate state: node="Node at 883f905d-cb9b-6b64-dc4f-906e59e0e728 [Candidate]" term=2 +2025-06-03T23:37:55.127Z [INFO] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: election won: term=2 tally=2 +2025-06-03T23:37:55.127Z [INFO] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: entering leader state: leader="Node at 883f905d-cb9b-6b64-dc4f-906e59e0e728 [Leader]" +2025-06-03T23:37:55.127Z [INFO] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: added peer, starting replication: peer=server-1fc834e3-a7b1-eac5-044a-77bb5eb3aad4 +2025-06-03T23:37:55.127Z [INFO] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: added peer, starting replication: peer=server-9d36fd3a-2b51-e783-05e8-2e8f5e868960 +2025-06-03T23:37:55.127Z [INFO] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: pipelining replication: peer="{Voter server-1fc834e3-a7b1-eac5-044a-77bb5eb3aad4 1fc834e3-a7b1-eac5-044a-77bb5eb3aad4}" +2025-06-03T23:37:55.127Z [INFO] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: pipelining replication: peer="{Voter server-9d36fd3a-2b51-e783-05e8-2e8f5e868960 9d36fd3a-2b51-e783-05e8-2e8f5e868960}" +2025-06-03T23:37:55.233Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:55.127842751 +0000 UTC m=+110.449597641 (1 nodes), 53.14752ms from start of poll, 53.62408ms from cluster start. Timeout at 2025-06-03 23:37:55.233320591 +0000 UTC m=+110.555075521, 105.47788ms after stability +2025-06-03T23:37:55.233Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-03T23:37:55.338Z [INFO] cluster: stable state for Leader reached at 2025-06-03 23:37:55.233452271 +0000 UTC m=+110.555207241 (1 nodes), 17.96µs from start of poll, 159.23368ms from cluster start. Timeout at 2025-06-03 23:37:55.338464151 +0000 UTC m=+110.660219081, 105.01184ms after stability +2025-06-03T23:37:55.343Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-03T23:37:55.448Z [INFO] cluster: stable state for Follower reached at 2025-06-03 23:37:55.343632551 +0000 UTC m=+110.665387481 (2 nodes), 2.96µs from start of poll, 269.41392ms from cluster start. Timeout at 2025-06-03 23:37:55.448857031 +0000 UTC m=+110.770611961, 105.22448ms after stability +2025-06-03T23:37:55.452Z [ERROR] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: failed to heartbeat to: peer=1fc834e3-a7b1-eac5-044a-77bb5eb3aad4 backoff time=10ms error="failed to connect to peer: 1fc834e3-a7b1-eac5-044a-77bb5eb3aad4" +2025-06-03T23:37:55.452Z [ERROR] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: failed to pipeline appendEntries: peer="{Voter server-1fc834e3-a7b1-eac5-044a-77bb5eb3aad4 1fc834e3-a7b1-eac5-044a-77bb5eb3aad4}" error="append pipeline closed" +2025-06-03T23:37:55.452Z [INFO] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: aborting pipeline replication: peer="{Voter server-1fc834e3-a7b1-eac5-044a-77bb5eb3aad4 1fc834e3-a7b1-eac5-044a-77bb5eb3aad4}" +2025-06-03T23:37:55.455Z [ERROR] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: failed to pipeline appendEntries: peer="{Voter server-9d36fd3a-2b51-e783-05e8-2e8f5e868960 9d36fd3a-2b51-e783-05e8-2e8f5e868960}" error="append pipeline closed" +2025-06-03T23:37:55.455Z [INFO] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: aborting pipeline replication: peer="{Voter server-9d36fd3a-2b51-e783-05e8-2e8f5e868960 9d36fd3a-2b51-e783-05e8-2e8f5e868960}" +2025-06-03T23:37:55.458Z [ERROR] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: failed to heartbeat to: peer=9d36fd3a-2b51-e783-05e8-2e8f5e868960 backoff time=10ms error="failed to connect to peer: 9d36fd3a-2b51-e783-05e8-2e8f5e868960" +2025-06-03T23:37:55.461Z [ERROR] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: failed to appendEntries to: peer="{Voter server-1fc834e3-a7b1-eac5-044a-77bb5eb3aad4 1fc834e3-a7b1-eac5-044a-77bb5eb3aad4}" error="failed to connect to peer: 1fc834e3-a7b1-eac5-044a-77bb5eb3aad4" +2025-06-03T23:37:55.465Z [ERROR] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: failed to appendEntries to: peer="{Voter server-9d36fd3a-2b51-e783-05e8-2e8f5e868960 9d36fd3a-2b51-e783-05e8-2e8f5e868960}" error="failed to connect to peer: 9d36fd3a-2b51-e783-05e8-2e8f5e868960" +2025-06-03T23:37:55.471Z [ERROR] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: failed to heartbeat to: peer=1fc834e3-a7b1-eac5-044a-77bb5eb3aad4 backoff time=10ms error="failed to connect to peer: 1fc834e3-a7b1-eac5-044a-77bb5eb3aad4" +2025-06-03T23:37:55.473Z [ERROR] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: failed to heartbeat to: peer=9d36fd3a-2b51-e783-05e8-2e8f5e868960 backoff time=10ms error="failed to connect to peer: 9d36fd3a-2b51-e783-05e8-2e8f5e868960" +2025-06-03T23:37:55.482Z [ERROR] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: failed to appendEntries to: peer="{Voter server-1fc834e3-a7b1-eac5-044a-77bb5eb3aad4 1fc834e3-a7b1-eac5-044a-77bb5eb3aad4}" error="failed to connect to peer: 1fc834e3-a7b1-eac5-044a-77bb5eb3aad4" +2025-06-03T23:37:55.484Z [ERROR] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: failed to appendEntries to: peer="{Voter server-9d36fd3a-2b51-e783-05e8-2e8f5e868960 9d36fd3a-2b51-e783-05e8-2e8f5e868960}" error="failed to connect to peer: 9d36fd3a-2b51-e783-05e8-2e8f5e868960" +2025-06-03T23:37:55.492Z [ERROR] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: failed to heartbeat to: peer=1fc834e3-a7b1-eac5-044a-77bb5eb3aad4 backoff time=10ms error="failed to connect to peer: 1fc834e3-a7b1-eac5-044a-77bb5eb3aad4" +2025-06-03T23:37:55.494Z [ERROR] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: failed to heartbeat to: peer=9d36fd3a-2b51-e783-05e8-2e8f5e868960 backoff time=10ms error="failed to connect to peer: 9d36fd3a-2b51-e783-05e8-2e8f5e868960" +2025-06-03T23:37:55.502Z [ERROR] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: failed to appendEntries to: peer="{Voter server-1fc834e3-a7b1-eac5-044a-77bb5eb3aad4 1fc834e3-a7b1-eac5-044a-77bb5eb3aad4}" error="failed to connect to peer: 1fc834e3-a7b1-eac5-044a-77bb5eb3aad4" +2025-06-03T23:37:55.503Z [WARN] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: failed to contact: server-id=server-1fc834e3-a7b1-eac5-044a-77bb5eb3aad4 time=55.405ms +2025-06-03T23:37:55.503Z [WARN] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: failed to contact: server-id=server-9d36fd3a-2b51-e783-05e8-2e8f5e868960 time=54.28448ms +2025-06-03T23:37:55.503Z [WARN] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: failed to contact quorum of nodes, stepping down +2025-06-03T23:37:55.503Z [INFO] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: entering follower state: follower="Node at 883f905d-cb9b-6b64-dc4f-906e59e0e728 [Follower]" leader-address= leader-id= +2025-06-03T23:37:55.503Z [ERROR] server-883f905d-cb9b-6b64-dc4f-906e59e0e728: failed to appendEntries to: peer="{Voter server-9d36fd3a-2b51-e783-05e8-2e8f5e868960 9d36fd3a-2b51-e783-05e8-2e8f5e868960}" error="failed to connect to peer: 9d36fd3a-2b51-e783-05e8-2e8f5e868960" +2025-06-03T23:37:55.564Z [WARN] server-9d36fd3a-2b51-e783-05e8-2e8f5e868960: heartbeat timeout reached, starting election: last-leader-addr=883f905d-cb9b-6b64-dc4f-906e59e0e728 last-leader-id=server-883f905d-cb9b-6b64-dc4f-906e59e0e728 +2025-06-03T23:37:55.564Z [INFO] server-9d36fd3a-2b51-e783-05e8-2e8f5e868960: entering candidate state: node="Node at 9d36fd3a-2b51-e783-05e8-2e8f5e868960 [Candidate]" term=3 +2025-06-03T23:37:55.564Z [ERROR] server-9d36fd3a-2b51-e783-05e8-2e8f5e868960: failed to make requestVote RPC: target="{Voter server-883f905d-cb9b-6b64-dc4f-906e59e0e728 883f905d-cb9b-6b64-dc4f-906e59e0e728}" error="failed to connect to peer: 883f905d-cb9b-6b64-dc4f-906e59e0e728" term=3 +2025-06-03T23:37:55.564Z [WARN] server-1fc834e3-a7b1-eac5-044a-77bb5eb3aad4: rejecting vote request since we have a leader: from=9d36fd3a-2b51-e783-05e8-2e8f5e868960 leader=883f905d-cb9b-6b64-dc4f-906e59e0e728 leader-id=server-883f905d-cb9b-6b64-dc4f-906e59e0e728 +2025-06-03T23:37:55.581Z [WARN] server-1fc834e3-a7b1-eac5-044a-77bb5eb3aad4: heartbeat timeout reached, starting election: last-leader-addr=883f905d-cb9b-6b64-dc4f-906e59e0e728 last-leader-id=server-883f905d-cb9b-6b64-dc4f-906e59e0e728 +2025-06-03T23:37:55.581Z [INFO] server-1fc834e3-a7b1-eac5-044a-77bb5eb3aad4: entering candidate state: node="Node at 1fc834e3-a7b1-eac5-044a-77bb5eb3aad4 [Candidate]" term=3 +2025-06-03T23:37:55.581Z [INFO] server-9d36fd3a-2b51-e783-05e8-2e8f5e868960: duplicate requestVote for same term: term=3 +2025-06-03T23:37:55.581Z [ERROR] server-1fc834e3-a7b1-eac5-044a-77bb5eb3aad4: failed to make requestVote RPC: target="{Voter server-883f905d-cb9b-6b64-dc4f-906e59e0e728 883f905d-cb9b-6b64-dc4f-906e59e0e728}" error="failed to connect to peer: 883f905d-cb9b-6b64-dc4f-906e59e0e728" term=3 +2025-06-03T23:37:55.581Z [INFO] server-1fc834e3-a7b1-eac5-044a-77bb5eb3aad4: entering follower state: follower="Node at 1fc834e3-a7b1-eac5-044a-77bb5eb3aad4 [Follower]" leader-address= leader-id= +--- PASS: TestRaft_VoteWithNoIDNoAddr (0.51s) === RUN TestRaft_runFollower_State_Transition === RUN TestRaft_runFollower_State_Transition/NonVoter -2025-06-03T23:34:57.383Z [INFO] 127.0.0.1:34339 :: starting node: addr=127.0.0.1:34339 -2025-06-03T23:34:57.383Z [INFO] 127.0.0.1:34339 :: initial configuration: index=0 servers=[] -2025-06-03T23:34:57.383Z [INFO] 127.0.0.1:34339 :: entering follower state: follower="Node at 127.0.0.1:34339 [Follower]" leader-address= leader-id= -2025-06-03T23:34:57.482Z [WARN] 127.0.0.1:34339 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-06-03T23:37:55.582Z [INFO] 127.0.0.1:43427 :: starting node: addr=127.0.0.1:43427 +2025-06-03T23:37:55.582Z [INFO] 127.0.0.1:43427 :: initial configuration: index=0 servers=[] +2025-06-03T23:37:55.582Z [INFO] 127.0.0.1:43427 :: entering follower state: follower="Node at 127.0.0.1:43427 [Follower]" leader-address= leader-id= +2025-06-03T23:37:55.637Z [WARN] 127.0.0.1:43427 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election === RUN TestRaft_runFollower_State_Transition/Voter -2025-06-03T23:34:57.535Z [INFO] 127.0.0.1:39689 :: starting node: addr=127.0.0.1:39689 -2025-06-03T23:34:57.535Z [INFO] 127.0.0.1:39689 :: initial configuration: index=0 servers=[] -2025-06-03T23:34:57.535Z [INFO] 127.0.0.1:39689 :: entering follower state: follower="Node at 127.0.0.1:39689 [Follower]" leader-address= leader-id= -2025-06-03T23:34:57.597Z [WARN] 127.0.0.1:39689 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:55.733Z [INFO] 127.0.0.1:40097 :: starting node: addr=127.0.0.1:40097 +2025-06-03T23:37:55.733Z [INFO] 127.0.0.1:40097 :: initial configuration: index=0 servers=[] +2025-06-03T23:37:55.733Z [INFO] 127.0.0.1:40097 :: entering follower state: follower="Node at 127.0.0.1:40097 [Follower]" leader-address= leader-id= +2025-06-03T23:37:55.810Z [WARN] 127.0.0.1:40097 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= === RUN TestRaft_runFollower_State_Transition/Not_in_Config -2025-06-03T23:34:57.686Z [INFO] 127.0.0.1:36247 :: starting node: addr=127.0.0.1:36247 -2025-06-03T23:34:57.687Z [INFO] 127.0.0.1:36247 :: initial configuration: index=0 servers=[] -2025-06-03T23:34:57.687Z [INFO] 127.0.0.1:36247 :: entering follower state: follower="Node at 127.0.0.1:36247 [Follower]" leader-address= leader-id= -2025-06-03T23:34:57.773Z [WARN] 127.0.0.1:36247 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-06-03T23:37:55.883Z [INFO] 127.0.0.1:35497 :: starting node: addr=127.0.0.1:35497 +2025-06-03T23:37:55.883Z [INFO] 127.0.0.1:35497 :: initial configuration: index=0 servers=[] +2025-06-03T23:37:55.884Z [INFO] 127.0.0.1:35497 :: entering follower state: follower="Node at 127.0.0.1:35497 [Follower]" leader-address= leader-id= +2025-06-03T23:37:55.935Z [WARN] 127.0.0.1:35497 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election --- PASS: TestRaft_runFollower_State_Transition (0.45s) --- PASS: TestRaft_runFollower_State_Transition/NonVoter (0.15s) --- PASS: TestRaft_runFollower_State_Transition/Voter (0.15s) --- PASS: TestRaft_runFollower_State_Transition/Not_in_Config (0.15s) === RUN TestRaft_runFollower_ReloadTimeoutConfigs -2025-06-03T23:34:57.837Z [INFO] 127.0.0.1:44595 :: starting node: addr=127.0.0.1:44595 -2025-06-03T23:34:57.837Z [INFO] 127.0.0.1:44595 :: initial configuration: index=0 servers=[] -2025-06-03T23:34:57.837Z [INFO] 127.0.0.1:44595 :: entering follower state: follower="Node at 127.0.0.1:44595 [Follower]" leader-address= leader-id= -2025-06-03T23:34:57.838Z [WARN] 127.0.0.1:44595 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-03T23:37:56.035Z [INFO] 127.0.0.1:43745 :: starting node: addr=127.0.0.1:43745 +2025-06-03T23:37:56.035Z [INFO] 127.0.0.1:43745 :: initial configuration: index=0 servers=[] +2025-06-03T23:37:56.035Z [INFO] 127.0.0.1:43745 :: entering follower state: follower="Node at 127.0.0.1:43745 [Follower]" leader-address= leader-id= +2025-06-03T23:37:56.035Z [WARN] 127.0.0.1:43745 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= --- PASS: TestRaft_runFollower_ReloadTimeoutConfigs (0.15s) === RUN TestSaturationMetric === RUN TestSaturationMetric/without_smoothing @@ -4192,7 +4232,7 @@ === RUN TestOverrideNotifyBool --- PASS: TestOverrideNotifyBool (0.00s) PASS -ok github.com/hashicorp/raft 113.050s +ok github.com/hashicorp/raft 111.527s make[1]: Leaving directory '/build/reproducible-path/golang-github-hashicorp-raft-1.5.0' create-stamp debian/debhelper-build-stamp dh_testroot -O--buildsystem=golang @@ -4220,12 +4260,14 @@ dpkg-buildpackage: info: binary-only upload (no source included) dpkg-genchanges: info: including full source code in upload I: copying local configuration +I: user script /srv/workspace/pbuilder/4129085/tmp/hooks/B01_cleanup starting +I: user script /srv/workspace/pbuilder/4129085/tmp/hooks/B01_cleanup finished I: unmounting dev/ptmx filesystem I: unmounting dev/pts filesystem I: unmounting dev/shm filesystem I: unmounting proc filesystem I: unmounting sys filesystem I: cleaning the build env -I: removing directory /srv/workspace/pbuilder/2735949 and its subdirectories -I: Current time: Tue Jun 3 11:35:03 -12 2025 -I: pbuilder-time-stamp: 1748993703 +I: removing directory /srv/workspace/pbuilder/4129085 and its subdirectories +I: Current time: Wed Jun 4 13:38:02 +14 2025 +I: pbuilder-time-stamp: 1748993882