Diff of the two buildlogs: -- --- b1/build.log 2024-05-12 20:52:05.746261971 +0000 +++ b2/build.log 2024-05-12 20:55:21.741919028 +0000 @@ -1,6 +1,6 @@ I: pbuilder: network access will be disabled during build -I: Current time: Sun May 12 08:46:10 -12 2024 -I: pbuilder-time-stamp: 1715546770 +I: Current time: Sun Jun 15 17:15:08 +14 2025 +I: pbuilder-time-stamp: 1749957308 I: Building the build Environment I: extracting base tarball [/var/cache/pbuilder/unstable-reproducible-base.tgz] I: copying local configuration @@ -28,52 +28,84 @@ dpkg-source: info: applying 0001-Skip-unreliable-tests.patch I: Not using root during the build. I: Installing the build-deps -I: user script /srv/workspace/pbuilder/1949903/tmp/hooks/D02_print_environment starting +I: user script /srv/workspace/pbuilder/2083635/tmp/hooks/D01_modify_environment starting +debug: Running on ionos15-amd64. +I: Changing host+domainname to test build reproducibility +I: Adding a custom variable just for the fun of it... +I: Changing /bin/sh to bash +'/bin/sh' -> '/bin/bash' +lrwxrwxrwx 1 root root 9 Jun 15 03:15 /bin/sh -> /bin/bash +I: Setting pbuilder2's login shell to /bin/bash +I: Setting pbuilder2's GECOS to second user,second room,second work-phone,second home-phone,second other +I: user script /srv/workspace/pbuilder/2083635/tmp/hooks/D01_modify_environment finished +I: user script /srv/workspace/pbuilder/2083635/tmp/hooks/D02_print_environment starting I: set - BUILDDIR='/build/reproducible-path' - BUILDUSERGECOS='first user,first room,first work-phone,first home-phone,first other' - BUILDUSERNAME='pbuilder1' - BUILD_ARCH='amd64' - DEBIAN_FRONTEND='noninteractive' - DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=20 ' - DISTRIBUTION='unstable' - HOME='/root' - HOST_ARCH='amd64' + BASH=/bin/sh + BASHOPTS=checkwinsize:cmdhist:complete_fullquote:extquote:force_fignore:globasciiranges:globskipdots:hostcomplete:interactive_comments:patsub_replacement:progcomp:promptvars:sourcepath + BASH_ALIASES=() + BASH_ARGC=() + BASH_ARGV=() + BASH_CMDS=() + BASH_LINENO=([0]="12" [1]="0") + BASH_LOADABLES_PATH=/usr/local/lib/bash:/usr/lib/bash:/opt/local/lib/bash:/usr/pkg/lib/bash:/opt/pkg/lib/bash:. + BASH_SOURCE=([0]="/tmp/hooks/D02_print_environment" [1]="/tmp/hooks/D02_print_environment") + BASH_VERSINFO=([0]="5" [1]="2" [2]="21" [3]="1" [4]="release" [5]="x86_64-pc-linux-gnu") + BASH_VERSION='5.2.21(1)-release' + BUILDDIR=/build/reproducible-path + BUILDUSERGECOS='second user,second room,second work-phone,second home-phone,second other' + BUILDUSERNAME=pbuilder2 + BUILD_ARCH=amd64 + DEBIAN_FRONTEND=noninteractive + DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=42 ' + DIRSTACK=() + DISTRIBUTION=unstable + EUID=0 + FUNCNAME=([0]="Echo" [1]="main") + GROUPS=() + HOME=/root + HOSTNAME=i-capture-the-hostname + HOSTTYPE=x86_64 + HOST_ARCH=amd64 IFS=' ' - INVOCATION_ID='9b7a4b2edd8340079dc9de0e99bbeafc' - LANG='C' - LANGUAGE='en_US:en' - LC_ALL='C' - MAIL='/var/mail/root' - OPTIND='1' - PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/games' - PBCURRENTCOMMANDLINEOPERATION='build' - PBUILDER_OPERATION='build' - PBUILDER_PKGDATADIR='/usr/share/pbuilder' - PBUILDER_PKGLIBDIR='/usr/lib/pbuilder' - PBUILDER_SYSCONFDIR='/etc' - PPID='1949903' - PS1='# ' - PS2='> ' + INVOCATION_ID=28dcc4dc27874a3c9f6d66fac92a6568 + LANG=C + LANGUAGE=et_EE:et + LC_ALL=C + MACHTYPE=x86_64-pc-linux-gnu + MAIL=/var/mail/root + OPTERR=1 + OPTIND=1 + OSTYPE=linux-gnu + PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path + PBCURRENTCOMMANDLINEOPERATION=build + PBUILDER_OPERATION=build + PBUILDER_PKGDATADIR=/usr/share/pbuilder + PBUILDER_PKGLIBDIR=/usr/lib/pbuilder + PBUILDER_SYSCONFDIR=/etc + PIPESTATUS=([0]="0") + POSIXLY_CORRECT=y + PPID=2083635 PS4='+ ' - PWD='/' - SHELL='/bin/bash' - SHLVL='2' - SUDO_COMMAND='/usr/bin/timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.3XKPPBM3/pbuilderrc_RfEk --distribution unstable --hookdir /etc/pbuilder/first-build-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/unstable-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.3XKPPBM3/b1 --logfile b1/build.log golang-github-hashicorp-raft_1.5.0-1.dsc' - SUDO_GID='110' - SUDO_UID='105' - SUDO_USER='jenkins' - TERM='unknown' - TZ='/usr/share/zoneinfo/Etc/GMT+12' - USER='root' - _='/usr/bin/systemd-run' - http_proxy='http://46.16.76.132:3128' + PWD=/ + SHELL=/bin/bash + SHELLOPTS=braceexpand:errexit:hashall:interactive-comments:posix + SHLVL=3 + SUDO_COMMAND='/usr/bin/timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.3XKPPBM3/pbuilderrc_ialw --distribution unstable --hookdir /etc/pbuilder/rebuild-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/unstable-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.3XKPPBM3/b2 --logfile b2/build.log golang-github-hashicorp-raft_1.5.0-1.dsc' + SUDO_GID=111 + SUDO_UID=106 + SUDO_USER=jenkins + TERM=unknown + TZ=/usr/share/zoneinfo/Etc/GMT-14 + UID=0 + USER=root + _='I: set' + http_proxy=http://213.165.73.152:3128 I: uname -a - Linux ionos1-amd64 6.1.0-21-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.90-1 (2024-05-03) x86_64 GNU/Linux + Linux i-capture-the-hostname 6.6.13+bpo-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.6.13-1~bpo12+1 (2024-02-15) x86_64 GNU/Linux I: ls -l /bin - lrwxrwxrwx 1 root root 7 May 11 07:43 /bin -> usr/bin -I: user script /srv/workspace/pbuilder/1949903/tmp/hooks/D02_print_environment finished + lrwxrwxrwx 1 root root 7 Jun 14 14:05 /bin -> usr/bin +I: user script /srv/workspace/pbuilder/2083635/tmp/hooks/D02_print_environment finished -> Attempting to satisfy build-dependencies -> Creating pbuilder-satisfydepends-dummy package Package: pbuilder-satisfydepends-dummy @@ -224,7 +256,7 @@ Get: 101 http://deb.debian.org/debian unstable/main amd64 golang-gopkg-check.v1-dev all 0.0+git20200902.038fdea-1 [32.1 kB] Get: 102 http://deb.debian.org/debian unstable/main amd64 golang-gopkg-mgo.v2-dev all 2016.08.01-7 [316 kB] Get: 103 http://deb.debian.org/debian unstable/main amd64 golang-github-hashicorp-go-msgpack-dev all 0.5.5-1 [43.3 kB] -Fetched 90.2 MB in 7s (12.4 MB/s) +Fetched 90.2 MB in 1s (116 MB/s) debconf: delaying package configuration, since apt-utils is not installed Selecting previously unselected package sensible-utils. (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 19699 files and directories currently installed.) @@ -651,7 +683,11 @@ Building tag database... -> Finished parsing the build-deps I: Building the package -I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.5.0/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.5.0-1_source.changes +I: user script /srv/workspace/pbuilder/2083635/tmp/hooks/A99_set_merged_usr starting +Not re-configuring usrmerge for unstable +I: user script /srv/workspace/pbuilder/2083635/tmp/hooks/A99_set_merged_usr finished +hostname: Name or service not known +I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.5.0/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.5.0-1_source.changes dpkg-buildpackage: info: source package golang-github-hashicorp-raft dpkg-buildpackage: info: source version 1.5.0-1 dpkg-buildpackage: info: source distribution unstable @@ -669,66 +705,66 @@ dh_autoreconf -O--buildsystem=golang dh_auto_configure -O--buildsystem=golang dh_auto_build -O--buildsystem=golang - cd obj-x86_64-linux-gnu && go install -trimpath -v -p 20 github.com/hashicorp/raft github.com/hashicorp/raft/bench + cd obj-x86_64-linux-gnu && go install -trimpath -v -p 42 github.com/hashicorp/raft github.com/hashicorp/raft/bench +unicode/utf8 +internal/unsafeheader +internal/itoa internal/goarch -internal/goexperiment internal/godebugs -internal/goos -internal/unsafeheader -internal/coverage/rtcov -math/bits +internal/nettrace +internal/cpu +encoding cmp +vendor/golang.org/x/crypto/internal/alias internal/race -internal/itoa +unicode +internal/coverage/rtcov +unicode/utf16 +log/internal +math/bits +runtime/internal/atomic +sync/atomic +container/list crypto/internal/alias +vendor/golang.org/x/crypto/cryptobyte/asn1 +internal/goos +crypto/subtle crypto/internal/boring/sig runtime/internal/syscall -internal/cpu -crypto/subtle -container/list -runtime/internal/atomic -unicode/utf8 -internal/chacha8rand +internal/goexperiment runtime/internal/math -runtime/internal/sys -unicode -encoding -sync/atomic +internal/chacha8rand internal/abi -unicode/utf16 -log/internal -internal/nettrace -vendor/golang.org/x/crypto/cryptobyte/asn1 -vendor/golang.org/x/crypto/internal/alias +runtime/internal/sys slices internal/bytealg math runtime -sync internal/reflectlite +sync internal/testlog -internal/bisect internal/singleflight +internal/bisect runtime/cgo errors -sort internal/godebug io +sort internal/oserror -internal/safefilepath -strconv path github.com/hashicorp/golang-lru/simplelru -math/rand -crypto/internal/nistec/fiat +internal/safefilepath vendor/golang.org/x/net/dns/dnsmessage +strconv +crypto/internal/nistec/fiat internal/intern +math/rand syscall +crypto/internal/randutil +internal/saferio +hash bytes strings -hash -internal/saferio -crypto/internal/randutil hash/crc64 hash/crc32 vendor/golang.org/x/text/transform @@ -737,104 +773,104 @@ net/netip reflect net/http/internal/ascii +bufio github.com/hashicorp/go-immutable-radix html -bufio regexp/syntax +regexp internal/syscall/execenv internal/syscall/unix time -regexp -io/fs context +io/fs internal/poll embed -encoding/binary internal/fmtsort +encoding/binary os -crypto/cipher encoding/base64 -crypto/internal/nistec crypto/internal/edwards25519/field +crypto/cipher crypto/md5 -golang.org/x/sys/unix vendor/golang.org/x/crypto/internal/poly1305 +crypto/internal/nistec +golang.org/x/sys/unix encoding/pem crypto/internal/boring -crypto/des vendor/golang.org/x/crypto/chacha20 +crypto/des crypto/internal/edwards25519 -crypto/aes +crypto/hmac crypto/sha1 crypto/sha512 -crypto/hmac crypto/sha256 -fmt +crypto/aes +vendor/golang.org/x/crypto/hkdf +os/signal path/filepath -vendor/golang.org/x/sys/cpu +fmt internal/sysinfo -os/signal -net io/ioutil -vendor/golang.org/x/crypto/hkdf -vendor/golang.org/x/crypto/chacha20poly1305 +vendor/golang.org/x/sys/cpu +net crypto/ecdh +vendor/golang.org/x/crypto/chacha20poly1305 log -encoding/json encoding/hex net/url -compress/flate +flag mime/quotedprintable -net/http/internal -vendor/golang.org/x/net/http2/hpack -mime go/token +net/http/internal +runtime/trace text/template/parse +mime +vendor/golang.org/x/net/http2/hpack runtime/debug -flag +encoding/json +compress/flate encoding/gob -runtime/trace -math/big vendor/golang.org/x/text/unicode/norm +math/big vendor/golang.org/x/text/unicode/bidi testing compress/gzip -text/template vendor/golang.org/x/text/secure/bidirule +text/template vendor/golang.org/x/net/idna -github.com/mattn/go-isatty -github.com/mattn/go-colorable +html/template +crypto/internal/boring/bbig crypto/rand +crypto/dsa crypto/elliptic -crypto/internal/boring/bbig crypto/internal/bigmod encoding/asn1 -crypto/dsa -github.com/fatih/color +github.com/mattn/go-isatty crypto/ed25519 -github.com/hashicorp/go-hclog +github.com/mattn/go-colorable crypto/rsa -vendor/golang.org/x/crypto/cryptobyte +github.com/fatih/color +github.com/hashicorp/go-hclog crypto/x509/pkix -html/template +vendor/golang.org/x/crypto/cryptobyte crypto/ecdsa -net/textproto vendor/golang.org/x/net/http/httpproxy +net/textproto crypto/x509 vendor/golang.org/x/net/http/httpguts mime/multipart crypto/tls net/http/httptrace net/http -github.com/armon/go-metrics net/rpc +github.com/armon/go-metrics github.com/hashicorp/go-msgpack/codec github.com/hashicorp/raft github.com/hashicorp/raft/bench debian/rules override_dh_auto_test make[1]: Entering directory '/build/reproducible-path/golang-github-hashicorp-raft-1.5.0' dh_auto_test - cd obj-x86_64-linux-gnu && go test -vet=off -v -p 20 github.com/hashicorp/raft github.com/hashicorp/raft/bench + cd obj-x86_64-linux-gnu && go test -vet=off -v -p 42 github.com/hashicorp/raft github.com/hashicorp/raft/bench ? github.com/hashicorp/raft/bench [no test files] === RUN TestCommitment_setVoters --- PASS: TestCommitment_setVoters (0.00s) @@ -877,33 +913,33 @@ === RUN TestFileSnapshotSinkImpl --- PASS: TestFileSnapshotSinkImpl (0.00s) === RUN TestFileSS_CreateSnapshotMissingParentDir -2024-05-12T20:49:48.722Z [INFO] creating new snapshot: path=/tmp/raft423213444/raft631868589/snapshots/3-10-1715546988722.tmp +2025-06-15T03:16:24.290Z [INFO] creating new snapshot: path=/tmp/raft935490689/raft3770680322/snapshots/3-10-1749957384290.tmp --- PASS: TestFileSS_CreateSnapshotMissingParentDir (0.01s) === RUN TestFileSS_CreateSnapshot -2024-05-12T20:49:48.730Z [INFO] creating new snapshot: path=/tmp/raft1191659096/snapshots/3-10-1715546988730.tmp -2024-05-12T20:49:48.732Z [WARN] found temporary snapshot: name=3-10-1715546988730.tmp +2025-06-15T03:16:24.295Z [INFO] creating new snapshot: path=/tmp/raft3066955146/snapshots/3-10-1749957384295.tmp +2025-06-15T03:16:24.297Z [WARN] found temporary snapshot: name=3-10-1749957384295.tmp --- PASS: TestFileSS_CreateSnapshot (0.01s) === RUN TestFileSS_CancelSnapshot -2024-05-12T20:49:48.741Z [INFO] creating new snapshot: path=/tmp/raft1618262377/snapshots/3-10-1715546988741.tmp ---- PASS: TestFileSS_CancelSnapshot (0.01s) +2025-06-15T03:16:24.306Z [INFO] creating new snapshot: path=/tmp/raft803176193/snapshots/3-10-1749957384306.tmp +--- PASS: TestFileSS_CancelSnapshot (0.00s) === RUN TestFileSS_Retention -2024-05-12T20:49:48.746Z [INFO] creating new snapshot: path=/tmp/raft3974593205/snapshots/3-10-1715546988746.tmp -2024-05-12T20:49:48.756Z [INFO] creating new snapshot: path=/tmp/raft3974593205/snapshots/3-11-1715546988756.tmp -2024-05-12T20:49:48.768Z [INFO] creating new snapshot: path=/tmp/raft3974593205/snapshots/3-12-1715546988768.tmp -2024-05-12T20:49:48.781Z [INFO] reaping snapshot: path=/tmp/raft3974593205/snapshots/3-10-1715546988746 -2024-05-12T20:49:48.781Z [INFO] creating new snapshot: path=/tmp/raft3974593205/snapshots/3-13-1715546988781.tmp -2024-05-12T20:49:48.792Z [INFO] reaping snapshot: path=/tmp/raft3974593205/snapshots/3-11-1715546988756 -2024-05-12T20:49:48.793Z [INFO] creating new snapshot: path=/tmp/raft3974593205/snapshots/3-14-1715546988793.tmp -2024-05-12T20:49:48.804Z [INFO] reaping snapshot: path=/tmp/raft3974593205/snapshots/3-12-1715546988768 ---- PASS: TestFileSS_Retention (0.06s) +2025-06-15T03:16:24.311Z [INFO] creating new snapshot: path=/tmp/raft2107785422/snapshots/3-10-1749957384311.tmp +2025-06-15T03:16:24.318Z [INFO] creating new snapshot: path=/tmp/raft2107785422/snapshots/3-11-1749957384318.tmp +2025-06-15T03:16:24.327Z [INFO] creating new snapshot: path=/tmp/raft2107785422/snapshots/3-12-1749957384327.tmp +2025-06-15T03:16:24.335Z [INFO] reaping snapshot: path=/tmp/raft2107785422/snapshots/3-10-1749957384311 +2025-06-15T03:16:24.336Z [INFO] creating new snapshot: path=/tmp/raft2107785422/snapshots/3-13-1749957384336.tmp +2025-06-15T03:16:24.344Z [INFO] reaping snapshot: path=/tmp/raft2107785422/snapshots/3-11-1749957384318 +2025-06-15T03:16:24.344Z [INFO] creating new snapshot: path=/tmp/raft2107785422/snapshots/3-14-1749957384344.tmp +2025-06-15T03:16:24.354Z [INFO] reaping snapshot: path=/tmp/raft2107785422/snapshots/3-12-1749957384327 +--- PASS: TestFileSS_Retention (0.04s) === RUN TestFileSS_BadPerm --- PASS: TestFileSS_BadPerm (0.00s) === RUN TestFileSS_MissingParentDir --- PASS: TestFileSS_MissingParentDir (0.00s) === RUN TestFileSS_Ordering -2024-05-12T20:49:48.806Z [INFO] creating new snapshot: path=/tmp/raft3457936151/snapshots/5-130350-1715546988806.tmp -2024-05-12T20:49:48.818Z [INFO] creating new snapshot: path=/tmp/raft3457936151/snapshots/36-204917-1715546988818.tmp ---- PASS: TestFileSS_Ordering (0.02s) +2025-06-15T03:16:24.355Z [INFO] creating new snapshot: path=/tmp/raft3551689843/snapshots/5-130350-1749957384355.tmp +2025-06-15T03:16:24.360Z [INFO] creating new snapshot: path=/tmp/raft3551689843/snapshots/36-204917-1749957384360.tmp +--- PASS: TestFileSS_Ordering (0.01s) === RUN TestDeferFutureSuccess --- PASS: TestDeferFutureSuccess (0.00s) === RUN TestDeferFutureError @@ -939,32 +975,32 @@ === RUN TestEmitsLogStoreMetrics --- PASS: TestEmitsLogStoreMetrics (0.01s) === RUN TestNetworkTransport_CloseStreams ---- PASS: TestNetworkTransport_CloseStreams (0.11s) +--- PASS: TestNetworkTransport_CloseStreams (0.00s) === RUN TestNetworkTransport_StartStop --- PASS: TestNetworkTransport_StartStop (0.00s) === RUN TestNetworkTransport_Heartbeat_FastPath ---- PASS: TestNetworkTransport_Heartbeat_FastPath (0.01s) +--- PASS: TestNetworkTransport_Heartbeat_FastPath (0.00s) === RUN TestNetworkTransport_AppendEntries --- PASS: TestNetworkTransport_AppendEntries (0.00s) === RUN TestNetworkTransport_AppendEntriesPipeline ---- PASS: TestNetworkTransport_AppendEntriesPipeline (0.02s) +--- PASS: TestNetworkTransport_AppendEntriesPipeline (0.00s) === RUN TestNetworkTransport_AppendEntriesPipeline_CloseStreams --- PASS: TestNetworkTransport_AppendEntriesPipeline_CloseStreams (0.02s) === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=0 +2025-06-15T03:16:24.457Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=1 === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=2 -2024-05-12T20:49:49.071Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" +2025-06-15T03:16:24.510Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=10 -2024-05-12T20:49:49.127Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" ---- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight (0.16s) - --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=0 (0.06s) +--- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight (0.15s) + --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=0 (0.05s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=1 (0.00s) - --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=2 (0.06s) + --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=2 (0.05s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=10 (0.05s) +2025-06-15T03:16:24.561Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_RequestVote -2024-05-12T20:49:49.179Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" ---- PASS: TestNetworkTransport_RequestVote (0.01s) +--- PASS: TestNetworkTransport_RequestVote (0.00s) === RUN TestNetworkTransport_InstallSnapshot --- PASS: TestNetworkTransport_InstallSnapshot (0.00s) === RUN TestNetworkTransport_EncodeDecode @@ -972,29 +1008,29 @@ === RUN TestNetworkTransport_EncodeDecode_AddressProvider --- PASS: TestNetworkTransport_EncodeDecode_AddressProvider (0.00s) === RUN TestNetworkTransport_PooledConn ---- PASS: TestNetworkTransport_PooledConn (0.02s) +--- PASS: TestNetworkTransport_PooledConn (0.00s) === RUN TestNetworkTransport_ListenBackoff - net_transport_test.go:839: countingWriter: 2024-05-12T20:49:49.208Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-15T03:16:24.566Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-12T20:49:49.214Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-15T03:16:24.571Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-12T20:49:49.225Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-15T03:16:24.582Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-12T20:49:49.245Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-15T03:16:24.602Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-12T20:49:49.286Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-15T03:16:24.643Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-12T20:49:49.366Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-15T03:16:24.724Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-12T20:49:49.534Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-15T03:16:24.884Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-12T20:49:49.858Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-15T03:16:25.204Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-12T20:49:50.502Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-15T03:16:25.845Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-12T20:49:51.506Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-15T03:16:26.846Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2024-05-12T20:49:52.510Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-06-15T03:16:27.848Z [ERROR] test: failed to accept connection: error="intentional error in test" --- PASS: TestNetworkTransport_ListenBackoff (4.00s) === RUN TestPeersJSON_BadConfiguration @@ -1004,2573 +1040,2527 @@ === RUN TestPeersJSON_ReadConfigJSON --- PASS: TestPeersJSON_ReadConfigJSON (0.00s) === RUN TestRaft_StartStop -2024-05-12T20:49:53.213Z [INFO] server-0c01f18c-76c6-6289-a82a-a9f9a8408ea0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c01f18c-76c6-6289-a82a-a9f9a8408ea0 Address:0c01f18c-76c6-6289-a82a-a9f9a8408ea0}]" +2025-06-15T03:16:28.568Z [INFO] server-fcfe83d5-f186-ccbe-3dbe-448749c42cbc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fcfe83d5-f186-ccbe-3dbe-448749c42cbc Address:fcfe83d5-f186-ccbe-3dbe-448749c42cbc}]" --- PASS: TestRaft_StartStop (0.00s) === RUN TestRaft_AfterShutdown -2024-05-12T20:49:53.214Z [INFO] server-4ffeb9c4-7301-ae34-51e6-39edae1c4504: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ffeb9c4-7301-ae34-51e6-39edae1c4504 Address:4ffeb9c4-7301-ae34-51e6-39edae1c4504}]" +2025-06-15T03:16:28.569Z [INFO] server-60a64b64-aad2-822c-0024-5765305b45e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60a64b64-aad2-822c-0024-5765305b45e4 Address:60a64b64-aad2-822c-0024-5765305b45e4}]" --- PASS: TestRaft_AfterShutdown (0.00s) === RUN TestRaft_LiveBootstrap -2024-05-12T20:49:53.219Z [INFO] server-56e23daf-f14c-a1ec-dad5-4839e25e046f: initial configuration: index=0 servers=[] -2024-05-12T20:49:53.219Z [INFO] server-68601d57-f247-532e-a73f-bfd83497b953: initial configuration: index=0 servers=[] -2024-05-12T20:49:53.219Z [INFO] server-324f4da5-0b61-3d5f-03fc-785df3c5b954: initial configuration: index=0 servers=[] -2024-05-12T20:49:53.219Z [INFO] server-56e23daf-f14c-a1ec-dad5-4839e25e046f: entering follower state: follower="Node at 56e23daf-f14c-a1ec-dad5-4839e25e046f [Follower]" leader-address= leader-id= -2024-05-12T20:49:53.219Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:49:53.219Z [INFO] server-68601d57-f247-532e-a73f-bfd83497b953: entering follower state: follower="Node at 68601d57-f247-532e-a73f-bfd83497b953 [Follower]" leader-address= leader-id= -2024-05-12T20:49:53.220Z [INFO] server-324f4da5-0b61-3d5f-03fc-785df3c5b954: entering follower state: follower="Node at 324f4da5-0b61-3d5f-03fc-785df3c5b954 [Follower]" leader-address= leader-id= -2024-05-12T20:49:53.278Z [WARN] server-56e23daf-f14c-a1ec-dad5-4839e25e046f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:53.278Z [INFO] server-56e23daf-f14c-a1ec-dad5-4839e25e046f: entering candidate state: node="Node at 56e23daf-f14c-a1ec-dad5-4839e25e046f [Candidate]" term=2 -2024-05-12T20:49:53.279Z [INFO] server-56e23daf-f14c-a1ec-dad5-4839e25e046f: election won: term=2 tally=2 -2024-05-12T20:49:53.279Z [INFO] server-56e23daf-f14c-a1ec-dad5-4839e25e046f: entering leader state: leader="Node at 56e23daf-f14c-a1ec-dad5-4839e25e046f [Leader]" -2024-05-12T20:49:53.279Z [INFO] server-56e23daf-f14c-a1ec-dad5-4839e25e046f: added peer, starting replication: peer=server-68601d57-f247-532e-a73f-bfd83497b953 -2024-05-12T20:49:53.279Z [INFO] server-56e23daf-f14c-a1ec-dad5-4839e25e046f: added peer, starting replication: peer=server-324f4da5-0b61-3d5f-03fc-785df3c5b954 -2024-05-12T20:49:53.279Z [WARN] server-324f4da5-0b61-3d5f-03fc-785df3c5b954: failed to get previous log: previous-index=1 last-index=0 error="log not found" -2024-05-12T20:49:53.279Z [WARN] server-56e23daf-f14c-a1ec-dad5-4839e25e046f: appendEntries rejected, sending older logs: peer="{Voter server-324f4da5-0b61-3d5f-03fc-785df3c5b954 324f4da5-0b61-3d5f-03fc-785df3c5b954}" next=1 -2024-05-12T20:49:53.279Z [INFO] server-56e23daf-f14c-a1ec-dad5-4839e25e046f: pipelining replication: peer="{Voter server-324f4da5-0b61-3d5f-03fc-785df3c5b954 324f4da5-0b61-3d5f-03fc-785df3c5b954}" -2024-05-12T20:49:53.279Z [WARN] server-68601d57-f247-532e-a73f-bfd83497b953: failed to get previous log: previous-index=1 last-index=0 error="log not found" -2024-05-12T20:49:53.279Z [WARN] server-56e23daf-f14c-a1ec-dad5-4839e25e046f: appendEntries rejected, sending older logs: peer="{Voter server-68601d57-f247-532e-a73f-bfd83497b953 68601d57-f247-532e-a73f-bfd83497b953}" next=1 -2024-05-12T20:49:53.279Z [INFO] server-56e23daf-f14c-a1ec-dad5-4839e25e046f: pipelining replication: peer="{Voter server-68601d57-f247-532e-a73f-bfd83497b953 68601d57-f247-532e-a73f-bfd83497b953}" -2024-05-12T20:49:53.386Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:49:53.2798632 +0000 UTC m=+4.560563578 (2 nodes), 59.912187ms from start of poll, 60.484836ms from cluster start. Timeout at 2024-05-12 20:49:53.386790613 +0000 UTC m=+4.667490961, 106.927383ms after stability -2024-05-12T20:49:53.386Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:53.506Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:53.387010107 +0000 UTC m=+4.667710568 (1 nodes), 3.927µs from start of poll, 167.631826ms from cluster start. Timeout at 2024-05-12 20:49:53.506856473 +0000 UTC m=+4.787556853, 119.846285ms after stability -2024-05-12T20:49:53.522Z [INFO] server-56e23daf-f14c-a1ec-dad5-4839e25e046f: aborting pipeline replication: peer="{Voter server-324f4da5-0b61-3d5f-03fc-785df3c5b954 324f4da5-0b61-3d5f-03fc-785df3c5b954}" -2024-05-12T20:49:53.523Z [INFO] server-56e23daf-f14c-a1ec-dad5-4839e25e046f: aborting pipeline replication: peer="{Voter server-68601d57-f247-532e-a73f-bfd83497b953 68601d57-f247-532e-a73f-bfd83497b953}" ---- PASS: TestRaft_LiveBootstrap (0.31s) +2025-06-15T03:16:28.569Z [INFO] server-2ad9d924-8f9d-1c01-d326-bfb3506893e1: initial configuration: index=0 servers=[] +2025-06-15T03:16:28.569Z [INFO] server-facfee4a-df46-d636-a2aa-3000892b69c8: initial configuration: index=0 servers=[] +2025-06-15T03:16:28.569Z [INFO] server-2ad9d924-8f9d-1c01-d326-bfb3506893e1: entering follower state: follower="Node at 2ad9d924-8f9d-1c01-d326-bfb3506893e1 [Follower]" leader-address= leader-id= +2025-06-15T03:16:28.570Z [INFO] server-f1bf79fd-7c76-0ce1-982d-83b1f63348df: initial configuration: index=0 servers=[] +2025-06-15T03:16:28.569Z [INFO] server-facfee4a-df46-d636-a2aa-3000892b69c8: entering follower state: follower="Node at facfee4a-df46-d636-a2aa-3000892b69c8 [Follower]" leader-address= leader-id= +2025-06-15T03:16:28.570Z [INFO] server-f1bf79fd-7c76-0ce1-982d-83b1f63348df: entering follower state: follower="Node at f1bf79fd-7c76-0ce1-982d-83b1f63348df [Follower]" leader-address= leader-id= +2025-06-15T03:16:28.570Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:16:28.633Z [WARN] server-f1bf79fd-7c76-0ce1-982d-83b1f63348df: no known peers, aborting election +2025-06-15T03:16:28.640Z [WARN] server-facfee4a-df46-d636-a2aa-3000892b69c8: no known peers, aborting election +2025-06-15T03:16:28.646Z [WARN] server-2ad9d924-8f9d-1c01-d326-bfb3506893e1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:28.646Z [INFO] server-2ad9d924-8f9d-1c01-d326-bfb3506893e1: entering candidate state: node="Node at 2ad9d924-8f9d-1c01-d326-bfb3506893e1 [Candidate]" term=2 +2025-06-15T03:16:28.646Z [INFO] server-2ad9d924-8f9d-1c01-d326-bfb3506893e1: election won: term=2 tally=2 +2025-06-15T03:16:28.646Z [INFO] server-2ad9d924-8f9d-1c01-d326-bfb3506893e1: entering leader state: leader="Node at 2ad9d924-8f9d-1c01-d326-bfb3506893e1 [Leader]" +2025-06-15T03:16:28.646Z [INFO] server-2ad9d924-8f9d-1c01-d326-bfb3506893e1: added peer, starting replication: peer=server-facfee4a-df46-d636-a2aa-3000892b69c8 +2025-06-15T03:16:28.646Z [INFO] server-2ad9d924-8f9d-1c01-d326-bfb3506893e1: added peer, starting replication: peer=server-f1bf79fd-7c76-0ce1-982d-83b1f63348df +2025-06-15T03:16:28.647Z [WARN] server-f1bf79fd-7c76-0ce1-982d-83b1f63348df: failed to get previous log: previous-index=1 last-index=0 error="log not found" +2025-06-15T03:16:28.647Z [WARN] server-2ad9d924-8f9d-1c01-d326-bfb3506893e1: appendEntries rejected, sending older logs: peer="{Voter server-f1bf79fd-7c76-0ce1-982d-83b1f63348df f1bf79fd-7c76-0ce1-982d-83b1f63348df}" next=1 +2025-06-15T03:16:28.647Z [WARN] server-facfee4a-df46-d636-a2aa-3000892b69c8: failed to get previous log: previous-index=1 last-index=0 error="log not found" +2025-06-15T03:16:28.647Z [INFO] server-2ad9d924-8f9d-1c01-d326-bfb3506893e1: pipelining replication: peer="{Voter server-f1bf79fd-7c76-0ce1-982d-83b1f63348df f1bf79fd-7c76-0ce1-982d-83b1f63348df}" +2025-06-15T03:16:28.647Z [WARN] server-2ad9d924-8f9d-1c01-d326-bfb3506893e1: appendEntries rejected, sending older logs: peer="{Voter server-facfee4a-df46-d636-a2aa-3000892b69c8 facfee4a-df46-d636-a2aa-3000892b69c8}" next=1 +2025-06-15T03:16:28.647Z [INFO] server-2ad9d924-8f9d-1c01-d326-bfb3506893e1: pipelining replication: peer="{Voter server-facfee4a-df46-d636-a2aa-3000892b69c8 facfee4a-df46-d636-a2aa-3000892b69c8}" +2025-06-15T03:16:28.752Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:16:28.646943904 +0000 UTC m=+4.358622076 (2 nodes), 76.820758ms from start of poll, 77.116776ms from cluster start. Timeout at 2025-06-15 03:16:28.752391129 +0000 UTC m=+4.464069308, 105.447232ms after stability +2025-06-15T03:16:28.752Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:28.857Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:28.752467794 +0000 UTC m=+4.464145960 (1 nodes), 834ns from start of poll, 182.64066ms from cluster start. Timeout at 2025-06-15 03:16:28.857617618 +0000 UTC m=+4.569295847, 105.149887ms after stability +2025-06-15T03:16:28.873Z [INFO] server-2ad9d924-8f9d-1c01-d326-bfb3506893e1: aborting pipeline replication: peer="{Voter server-f1bf79fd-7c76-0ce1-982d-83b1f63348df f1bf79fd-7c76-0ce1-982d-83b1f63348df}" +2025-06-15T03:16:28.873Z [INFO] server-2ad9d924-8f9d-1c01-d326-bfb3506893e1: aborting pipeline replication: peer="{Voter server-facfee4a-df46-d636-a2aa-3000892b69c8 facfee4a-df46-d636-a2aa-3000892b69c8}" +--- PASS: TestRaft_LiveBootstrap (0.30s) === RUN TestRaft_LiveBootstrap_From_NonVoter -2024-05-12T20:49:53.524Z [INFO] server-cf4cfe27-69ae-3160-79a3-906a9026a707: initial configuration: index=0 servers=[] -2024-05-12T20:49:53.524Z [INFO] server-cac48b85-5393-b648-c89f-00b901706bfa: initial configuration: index=0 servers=[] -2024-05-12T20:49:53.524Z [INFO] server-cf4cfe27-69ae-3160-79a3-906a9026a707: entering follower state: follower="Node at cf4cfe27-69ae-3160-79a3-906a9026a707 [Follower]" leader-address= leader-id= +2025-06-15T03:16:28.874Z [INFO] server-caaaf964-ef7a-5c23-ce37-bb388d26a3c9: initial configuration: index=0 servers=[] +2025-06-15T03:16:28.874Z [INFO] server-caaaf964-ef7a-5c23-ce37-bb388d26a3c9: entering follower state: follower="Node at caaaf964-ef7a-5c23-ce37-bb388d26a3c9 [Follower]" leader-address= leader-id= +2025-06-15T03:16:28.874Z [INFO] server-33139494-fc78-c1ce-d440-dfe71b18509f: initial configuration: index=0 servers=[] --- PASS: TestRaft_LiveBootstrap_From_NonVoter (0.00s) === RUN TestRaft_RecoverCluster_NoState -2024-05-12T20:49:53.525Z [INFO] server-466c425c-64b8-f314-1f05-87fd952314a3: initial configuration: index=0 servers=[] +2025-06-15T03:16:28.875Z [INFO] server-dd3d294f-43f3-2dbb-2c12-0c9ca25b0107: initial configuration: index=0 servers=[] +2025-06-15T03:16:28.875Z [INFO] server-dd3d294f-43f3-2dbb-2c12-0c9ca25b0107: entering follower state: follower="Node at dd3d294f-43f3-2dbb-2c12-0c9ca25b0107 [Follower]" leader-address= leader-id= --- PASS: TestRaft_RecoverCluster_NoState (0.00s) === RUN TestRaft_RecoverCluster === RUN TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs -2024-05-12T20:49:53.526Z [INFO] server-e8dc8aba-ee0d-37f2-d098-469625a840a5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e8dc8aba-ee0d-37f2-d098-469625a840a5 Address:e8dc8aba-ee0d-37f2-d098-469625a840a5} {Suffrage:Voter ID:server-43d0b008-12a0-4ec0-7125-d08e3eb8d112 Address:43d0b008-12a0-4ec0-7125-d08e3eb8d112} {Suffrage:Voter ID:server-0be446e7-6348-b5b4-f031-fb5eac25e590 Address:0be446e7-6348-b5b4-f031-fb5eac25e590}]" -2024-05-12T20:49:53.526Z [INFO] server-43d0b008-12a0-4ec0-7125-d08e3eb8d112: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e8dc8aba-ee0d-37f2-d098-469625a840a5 Address:e8dc8aba-ee0d-37f2-d098-469625a840a5} {Suffrage:Voter ID:server-43d0b008-12a0-4ec0-7125-d08e3eb8d112 Address:43d0b008-12a0-4ec0-7125-d08e3eb8d112} {Suffrage:Voter ID:server-0be446e7-6348-b5b4-f031-fb5eac25e590 Address:0be446e7-6348-b5b4-f031-fb5eac25e590}]" -2024-05-12T20:49:53.526Z [INFO] server-0be446e7-6348-b5b4-f031-fb5eac25e590: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e8dc8aba-ee0d-37f2-d098-469625a840a5 Address:e8dc8aba-ee0d-37f2-d098-469625a840a5} {Suffrage:Voter ID:server-43d0b008-12a0-4ec0-7125-d08e3eb8d112 Address:43d0b008-12a0-4ec0-7125-d08e3eb8d112} {Suffrage:Voter ID:server-0be446e7-6348-b5b4-f031-fb5eac25e590 Address:0be446e7-6348-b5b4-f031-fb5eac25e590}]" -2024-05-12T20:49:53.526Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:53.526Z [INFO] server-e8dc8aba-ee0d-37f2-d098-469625a840a5: entering follower state: follower="Node at e8dc8aba-ee0d-37f2-d098-469625a840a5 [Follower]" leader-address= leader-id= -2024-05-12T20:49:53.526Z [INFO] server-0be446e7-6348-b5b4-f031-fb5eac25e590: entering follower state: follower="Node at 0be446e7-6348-b5b4-f031-fb5eac25e590 [Follower]" leader-address= leader-id= -2024-05-12T20:49:53.526Z [INFO] server-43d0b008-12a0-4ec0-7125-d08e3eb8d112: entering follower state: follower="Node at 43d0b008-12a0-4ec0-7125-d08e3eb8d112 [Follower]" leader-address= leader-id= -2024-05-12T20:49:53.586Z [WARN] server-0be446e7-6348-b5b4-f031-fb5eac25e590: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:53.586Z [INFO] server-0be446e7-6348-b5b4-f031-fb5eac25e590: entering candidate state: node="Node at 0be446e7-6348-b5b4-f031-fb5eac25e590 [Candidate]" term=2 -2024-05-12T20:49:53.586Z [INFO] server-0be446e7-6348-b5b4-f031-fb5eac25e590: election won: term=2 tally=2 -2024-05-12T20:49:53.586Z [INFO] server-0be446e7-6348-b5b4-f031-fb5eac25e590: entering leader state: leader="Node at 0be446e7-6348-b5b4-f031-fb5eac25e590 [Leader]" -2024-05-12T20:49:53.587Z [INFO] server-0be446e7-6348-b5b4-f031-fb5eac25e590: added peer, starting replication: peer=server-e8dc8aba-ee0d-37f2-d098-469625a840a5 -2024-05-12T20:49:53.587Z [INFO] server-0be446e7-6348-b5b4-f031-fb5eac25e590: added peer, starting replication: peer=server-43d0b008-12a0-4ec0-7125-d08e3eb8d112 -2024-05-12T20:49:53.587Z [INFO] server-0be446e7-6348-b5b4-f031-fb5eac25e590: pipelining replication: peer="{Voter server-43d0b008-12a0-4ec0-7125-d08e3eb8d112 43d0b008-12a0-4ec0-7125-d08e3eb8d112}" -2024-05-12T20:49:53.587Z [INFO] server-0be446e7-6348-b5b4-f031-fb5eac25e590: pipelining replication: peer="{Voter server-e8dc8aba-ee0d-37f2-d098-469625a840a5 e8dc8aba-ee0d-37f2-d098-469625a840a5}" -2024-05-12T20:49:53.698Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:53.587470939 +0000 UTC m=+4.868171344 (1 nodes), 60.799118ms from start of poll, 61.134971ms from cluster start. Timeout at 2024-05-12 20:49:53.698806033 +0000 UTC m=+4.979505928, 111.334584ms after stability -2024-05-12T20:49:53.699Z [INFO] server-0be446e7-6348-b5b4-f031-fb5eac25e590: aborting pipeline replication: peer="{Voter server-43d0b008-12a0-4ec0-7125-d08e3eb8d112 43d0b008-12a0-4ec0-7125-d08e3eb8d112}" -2024-05-12T20:49:53.699Z [INFO] server-0be446e7-6348-b5b4-f031-fb5eac25e590: aborting pipeline replication: peer="{Voter server-e8dc8aba-ee0d-37f2-d098-469625a840a5 e8dc8aba-ee0d-37f2-d098-469625a840a5}" -2024-05-12T20:49:53.699Z [INFO] creating new snapshot: path=/tmp/raft702998711/snapshots/2-2-1715546993699.tmp -2024-05-12T20:49:53.701Z [INFO] server-e8dc8aba-ee0d-37f2-d098-469625a840a5: starting restore from snapshot: id=2-2-1715546993699 last-index=2 last-term=2 size-in-bytes=1 -2024-05-12T20:49:53.701Z [INFO] server-e8dc8aba-ee0d-37f2-d098-469625a840a5: snapshot restore progress: id=2-2-1715546993699 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2024-05-12T20:49:53.701Z [INFO] server-e8dc8aba-ee0d-37f2-d098-469625a840a5: restored from snapshot: id=2-2-1715546993699 last-index=2 last-term=2 size-in-bytes=1 -2024-05-12T20:49:53.701Z [INFO] server-e8dc8aba-ee0d-37f2-d098-469625a840a5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e8dc8aba-ee0d-37f2-d098-469625a840a5 Address:e8dc8aba-ee0d-37f2-d098-469625a840a5} {Suffrage:Voter ID:server-43d0b008-12a0-4ec0-7125-d08e3eb8d112 Address:43d0b008-12a0-4ec0-7125-d08e3eb8d112} {Suffrage:Voter ID:server-0be446e7-6348-b5b4-f031-fb5eac25e590 Address:0be446e7-6348-b5b4-f031-fb5eac25e590}]" -2024-05-12T20:49:53.701Z [INFO] creating new snapshot: path=/tmp/raft2398685516/snapshots/2-2-1715546993701.tmp -2024-05-12T20:49:53.702Z [INFO] server-e8dc8aba-ee0d-37f2-d098-469625a840a5: entering follower state: follower="Node at e8dc8aba-ee0d-37f2-d098-469625a840a5 [Follower]" leader-address= leader-id= -2024-05-12T20:49:53.702Z [INFO] server-43d0b008-12a0-4ec0-7125-d08e3eb8d112: starting restore from snapshot: id=2-2-1715546993701 last-index=2 last-term=2 size-in-bytes=1 -2024-05-12T20:49:53.703Z [INFO] server-43d0b008-12a0-4ec0-7125-d08e3eb8d112: snapshot restore progress: id=2-2-1715546993701 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2024-05-12T20:49:53.703Z [INFO] server-43d0b008-12a0-4ec0-7125-d08e3eb8d112: restored from snapshot: id=2-2-1715546993701 last-index=2 last-term=2 size-in-bytes=1 -2024-05-12T20:49:53.703Z [INFO] server-43d0b008-12a0-4ec0-7125-d08e3eb8d112: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e8dc8aba-ee0d-37f2-d098-469625a840a5 Address:e8dc8aba-ee0d-37f2-d098-469625a840a5} {Suffrage:Voter ID:server-43d0b008-12a0-4ec0-7125-d08e3eb8d112 Address:43d0b008-12a0-4ec0-7125-d08e3eb8d112} {Suffrage:Voter ID:server-0be446e7-6348-b5b4-f031-fb5eac25e590 Address:0be446e7-6348-b5b4-f031-fb5eac25e590}]" -2024-05-12T20:49:53.703Z [INFO] creating new snapshot: path=/tmp/raft1412382366/snapshots/2-2-1715546993703.tmp -2024-05-12T20:49:53.704Z [INFO] server-0be446e7-6348-b5b4-f031-fb5eac25e590: starting restore from snapshot: id=2-2-1715546993703 last-index=2 last-term=2 size-in-bytes=1 -2024-05-12T20:49:53.704Z [INFO] server-0be446e7-6348-b5b4-f031-fb5eac25e590: snapshot restore progress: id=2-2-1715546993703 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2024-05-12T20:49:53.704Z [INFO] server-0be446e7-6348-b5b4-f031-fb5eac25e590: restored from snapshot: id=2-2-1715546993703 last-index=2 last-term=2 size-in-bytes=1 -2024-05-12T20:49:53.704Z [INFO] server-0be446e7-6348-b5b4-f031-fb5eac25e590: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e8dc8aba-ee0d-37f2-d098-469625a840a5 Address:e8dc8aba-ee0d-37f2-d098-469625a840a5} {Suffrage:Voter ID:server-43d0b008-12a0-4ec0-7125-d08e3eb8d112 Address:43d0b008-12a0-4ec0-7125-d08e3eb8d112} {Suffrage:Voter ID:server-0be446e7-6348-b5b4-f031-fb5eac25e590 Address:0be446e7-6348-b5b4-f031-fb5eac25e590}]" -2024-05-12T20:49:53.704Z [INFO] server-43d0b008-12a0-4ec0-7125-d08e3eb8d112: entering follower state: follower="Node at 43d0b008-12a0-4ec0-7125-d08e3eb8d112 [Follower]" leader-address= leader-id= -2024-05-12T20:49:53.704Z [INFO] server-0be446e7-6348-b5b4-f031-fb5eac25e590: entering follower state: follower="Node at 0be446e7-6348-b5b4-f031-fb5eac25e590 [Follower]" leader-address= leader-id= -2024-05-12T20:49:53.766Z [WARN] server-e8dc8aba-ee0d-37f2-d098-469625a840a5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:53.766Z [INFO] server-e8dc8aba-ee0d-37f2-d098-469625a840a5: entering candidate state: node="Node at e8dc8aba-ee0d-37f2-d098-469625a840a5 [Candidate]" term=3 -2024-05-12T20:49:53.767Z [INFO] server-e8dc8aba-ee0d-37f2-d098-469625a840a5: election won: term=3 tally=2 -2024-05-12T20:49:53.767Z [INFO] server-e8dc8aba-ee0d-37f2-d098-469625a840a5: entering leader state: leader="Node at e8dc8aba-ee0d-37f2-d098-469625a840a5 [Leader]" -2024-05-12T20:49:53.767Z [INFO] server-e8dc8aba-ee0d-37f2-d098-469625a840a5: added peer, starting replication: peer=server-43d0b008-12a0-4ec0-7125-d08e3eb8d112 -2024-05-12T20:49:53.767Z [INFO] server-e8dc8aba-ee0d-37f2-d098-469625a840a5: added peer, starting replication: peer=server-0be446e7-6348-b5b4-f031-fb5eac25e590 -2024-05-12T20:49:53.767Z [INFO] server-e8dc8aba-ee0d-37f2-d098-469625a840a5: pipelining replication: peer="{Voter server-0be446e7-6348-b5b4-f031-fb5eac25e590 0be446e7-6348-b5b4-f031-fb5eac25e590}" -2024-05-12T20:49:53.767Z [INFO] server-e8dc8aba-ee0d-37f2-d098-469625a840a5: pipelining replication: peer="{Voter server-43d0b008-12a0-4ec0-7125-d08e3eb8d112 43d0b008-12a0-4ec0-7125-d08e3eb8d112}" -2024-05-12T20:49:54.030Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:54.138Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:54.030971547 +0000 UTC m=+5.311671886 (1 nodes), 41.042µs from start of poll, 504.635513ms from cluster start. Timeout at 2024-05-12 20:49:54.13881538 +0000 UTC m=+5.419515834, 107.843948ms after stability -2024-05-12T20:49:54.139Z [INFO] server-e8dc8aba-ee0d-37f2-d098-469625a840a5: aborting pipeline replication: peer="{Voter server-0be446e7-6348-b5b4-f031-fb5eac25e590 0be446e7-6348-b5b4-f031-fb5eac25e590}" -2024-05-12T20:49:54.139Z [INFO] server-e8dc8aba-ee0d-37f2-d098-469625a840a5: aborting pipeline replication: peer="{Voter server-43d0b008-12a0-4ec0-7125-d08e3eb8d112 43d0b008-12a0-4ec0-7125-d08e3eb8d112}" +2025-06-15T03:16:28.876Z [INFO] server-2c6daae5-9d4b-a83b-5e19-150ab300c776: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2c6daae5-9d4b-a83b-5e19-150ab300c776 Address:2c6daae5-9d4b-a83b-5e19-150ab300c776} {Suffrage:Voter ID:server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546 Address:806452e8-4e15-27a3-7f5c-6ddf8ad4a546} {Suffrage:Voter ID:server-d931d97d-c554-d61f-f8f9-bdbc98b526c0 Address:d931d97d-c554-d61f-f8f9-bdbc98b526c0}]" +2025-06-15T03:16:28.876Z [INFO] server-2c6daae5-9d4b-a83b-5e19-150ab300c776: entering follower state: follower="Node at 2c6daae5-9d4b-a83b-5e19-150ab300c776 [Follower]" leader-address= leader-id= +2025-06-15T03:16:28.876Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2c6daae5-9d4b-a83b-5e19-150ab300c776 Address:2c6daae5-9d4b-a83b-5e19-150ab300c776} {Suffrage:Voter ID:server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546 Address:806452e8-4e15-27a3-7f5c-6ddf8ad4a546} {Suffrage:Voter ID:server-d931d97d-c554-d61f-f8f9-bdbc98b526c0 Address:d931d97d-c554-d61f-f8f9-bdbc98b526c0}]" +2025-06-15T03:16:28.876Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: entering follower state: follower="Node at 806452e8-4e15-27a3-7f5c-6ddf8ad4a546 [Follower]" leader-address= leader-id= +2025-06-15T03:16:28.876Z [INFO] server-d931d97d-c554-d61f-f8f9-bdbc98b526c0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2c6daae5-9d4b-a83b-5e19-150ab300c776 Address:2c6daae5-9d4b-a83b-5e19-150ab300c776} {Suffrage:Voter ID:server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546 Address:806452e8-4e15-27a3-7f5c-6ddf8ad4a546} {Suffrage:Voter ID:server-d931d97d-c554-d61f-f8f9-bdbc98b526c0 Address:d931d97d-c554-d61f-f8f9-bdbc98b526c0}]" +2025-06-15T03:16:28.876Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:28.876Z [INFO] server-d931d97d-c554-d61f-f8f9-bdbc98b526c0: entering follower state: follower="Node at d931d97d-c554-d61f-f8f9-bdbc98b526c0 [Follower]" leader-address= leader-id= +2025-06-15T03:16:28.926Z [WARN] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:28.926Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: entering candidate state: node="Node at 806452e8-4e15-27a3-7f5c-6ddf8ad4a546 [Candidate]" term=2 +2025-06-15T03:16:28.926Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: election won: term=2 tally=2 +2025-06-15T03:16:28.926Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: entering leader state: leader="Node at 806452e8-4e15-27a3-7f5c-6ddf8ad4a546 [Leader]" +2025-06-15T03:16:28.926Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: added peer, starting replication: peer=server-2c6daae5-9d4b-a83b-5e19-150ab300c776 +2025-06-15T03:16:28.927Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: added peer, starting replication: peer=server-d931d97d-c554-d61f-f8f9-bdbc98b526c0 +2025-06-15T03:16:28.927Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: pipelining replication: peer="{Voter server-d931d97d-c554-d61f-f8f9-bdbc98b526c0 d931d97d-c554-d61f-f8f9-bdbc98b526c0}" +2025-06-15T03:16:28.927Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: pipelining replication: peer="{Voter server-2c6daae5-9d4b-a83b-5e19-150ab300c776 2c6daae5-9d4b-a83b-5e19-150ab300c776}" +2025-06-15T03:16:29.032Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:28.927055785 +0000 UTC m=+4.638733947 (1 nodes), 50.611306ms from start of poll, 50.895403ms from cluster start. Timeout at 2025-06-15 03:16:29.032294503 +0000 UTC m=+4.743972681, 105.238734ms after stability +2025-06-15T03:16:29.032Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: aborting pipeline replication: peer="{Voter server-d931d97d-c554-d61f-f8f9-bdbc98b526c0 d931d97d-c554-d61f-f8f9-bdbc98b526c0}" +2025-06-15T03:16:29.032Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: aborting pipeline replication: peer="{Voter server-2c6daae5-9d4b-a83b-5e19-150ab300c776 2c6daae5-9d4b-a83b-5e19-150ab300c776}" +2025-06-15T03:16:29.032Z [INFO] creating new snapshot: path=/tmp/raft3758853895/snapshots/2-2-1749957389032.tmp +2025-06-15T03:16:29.033Z [INFO] server-2c6daae5-9d4b-a83b-5e19-150ab300c776: starting restore from snapshot: id=2-2-1749957389032 last-index=2 last-term=2 size-in-bytes=1 +2025-06-15T03:16:29.033Z [INFO] server-2c6daae5-9d4b-a83b-5e19-150ab300c776: snapshot restore progress: id=2-2-1749957389032 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-06-15T03:16:29.033Z [INFO] server-2c6daae5-9d4b-a83b-5e19-150ab300c776: restored from snapshot: id=2-2-1749957389032 last-index=2 last-term=2 size-in-bytes=1 +2025-06-15T03:16:29.033Z [INFO] server-2c6daae5-9d4b-a83b-5e19-150ab300c776: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2c6daae5-9d4b-a83b-5e19-150ab300c776 Address:2c6daae5-9d4b-a83b-5e19-150ab300c776} {Suffrage:Voter ID:server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546 Address:806452e8-4e15-27a3-7f5c-6ddf8ad4a546} {Suffrage:Voter ID:server-d931d97d-c554-d61f-f8f9-bdbc98b526c0 Address:d931d97d-c554-d61f-f8f9-bdbc98b526c0}]" +2025-06-15T03:16:29.033Z [INFO] creating new snapshot: path=/tmp/raft1940287707/snapshots/2-2-1749957389033.tmp +2025-06-15T03:16:29.033Z [INFO] server-2c6daae5-9d4b-a83b-5e19-150ab300c776: entering follower state: follower="Node at 2c6daae5-9d4b-a83b-5e19-150ab300c776 [Follower]" leader-address= leader-id= +2025-06-15T03:16:29.033Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: starting restore from snapshot: id=2-2-1749957389033 last-index=2 last-term=2 size-in-bytes=1 +2025-06-15T03:16:29.033Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: snapshot restore progress: id=2-2-1749957389033 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-06-15T03:16:29.033Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: restored from snapshot: id=2-2-1749957389033 last-index=2 last-term=2 size-in-bytes=1 +2025-06-15T03:16:29.033Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2c6daae5-9d4b-a83b-5e19-150ab300c776 Address:2c6daae5-9d4b-a83b-5e19-150ab300c776} {Suffrage:Voter ID:server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546 Address:806452e8-4e15-27a3-7f5c-6ddf8ad4a546} {Suffrage:Voter ID:server-d931d97d-c554-d61f-f8f9-bdbc98b526c0 Address:d931d97d-c554-d61f-f8f9-bdbc98b526c0}]" +2025-06-15T03:16:29.033Z [INFO] creating new snapshot: path=/tmp/raft773499949/snapshots/2-2-1749957389033.tmp +2025-06-15T03:16:29.033Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: entering follower state: follower="Node at 806452e8-4e15-27a3-7f5c-6ddf8ad4a546 [Follower]" leader-address= leader-id= +2025-06-15T03:16:29.034Z [INFO] server-d931d97d-c554-d61f-f8f9-bdbc98b526c0: starting restore from snapshot: id=2-2-1749957389033 last-index=2 last-term=2 size-in-bytes=1 +2025-06-15T03:16:29.034Z [INFO] server-d931d97d-c554-d61f-f8f9-bdbc98b526c0: snapshot restore progress: id=2-2-1749957389033 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-06-15T03:16:29.034Z [INFO] server-d931d97d-c554-d61f-f8f9-bdbc98b526c0: restored from snapshot: id=2-2-1749957389033 last-index=2 last-term=2 size-in-bytes=1 +2025-06-15T03:16:29.034Z [INFO] server-d931d97d-c554-d61f-f8f9-bdbc98b526c0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2c6daae5-9d4b-a83b-5e19-150ab300c776 Address:2c6daae5-9d4b-a83b-5e19-150ab300c776} {Suffrage:Voter ID:server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546 Address:806452e8-4e15-27a3-7f5c-6ddf8ad4a546} {Suffrage:Voter ID:server-d931d97d-c554-d61f-f8f9-bdbc98b526c0 Address:d931d97d-c554-d61f-f8f9-bdbc98b526c0}]" +2025-06-15T03:16:29.034Z [INFO] server-d931d97d-c554-d61f-f8f9-bdbc98b526c0: entering follower state: follower="Node at d931d97d-c554-d61f-f8f9-bdbc98b526c0 [Follower]" leader-address= leader-id= +2025-06-15T03:16:29.107Z [WARN] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:29.107Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: entering candidate state: node="Node at 806452e8-4e15-27a3-7f5c-6ddf8ad4a546 [Candidate]" term=3 +2025-06-15T03:16:29.107Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: election won: term=3 tally=2 +2025-06-15T03:16:29.107Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: entering leader state: leader="Node at 806452e8-4e15-27a3-7f5c-6ddf8ad4a546 [Leader]" +2025-06-15T03:16:29.107Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: added peer, starting replication: peer=server-2c6daae5-9d4b-a83b-5e19-150ab300c776 +2025-06-15T03:16:29.107Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: added peer, starting replication: peer=server-d931d97d-c554-d61f-f8f9-bdbc98b526c0 +2025-06-15T03:16:29.107Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: pipelining replication: peer="{Voter server-d931d97d-c554-d61f-f8f9-bdbc98b526c0 d931d97d-c554-d61f-f8f9-bdbc98b526c0}" +2025-06-15T03:16:29.107Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: pipelining replication: peer="{Voter server-2c6daae5-9d4b-a83b-5e19-150ab300c776 2c6daae5-9d4b-a83b-5e19-150ab300c776}" +2025-06-15T03:16:29.350Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:29.455Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:29.35025245 +0000 UTC m=+5.061930666 (1 nodes), 51.73µs from start of poll, 474.092122ms from cluster start. Timeout at 2025-06-15 03:16:29.4558256 +0000 UTC m=+5.167503765, 105.573099ms after stability +2025-06-15T03:16:29.455Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: aborting pipeline replication: peer="{Voter server-d931d97d-c554-d61f-f8f9-bdbc98b526c0 d931d97d-c554-d61f-f8f9-bdbc98b526c0}" +2025-06-15T03:16:29.455Z [INFO] server-806452e8-4e15-27a3-7f5c-6ddf8ad4a546: aborting pipeline replication: peer="{Voter server-2c6daae5-9d4b-a83b-5e19-150ab300c776 2c6daae5-9d4b-a83b-5e19-150ab300c776}" === RUN TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs -2024-05-12T20:49:54.141Z [INFO] server-254a4a21-a64c-4751-6720-3dd63e9e1a0c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-254a4a21-a64c-4751-6720-3dd63e9e1a0c Address:254a4a21-a64c-4751-6720-3dd63e9e1a0c} {Suffrage:Voter ID:server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a Address:8f8f39a7-e892-6a0d-7d95-bf21e36bab6a} {Suffrage:Voter ID:server-98a5fd1a-8384-43a4-e856-599222f20e17 Address:98a5fd1a-8384-43a4-e856-599222f20e17}]" -2024-05-12T20:49:54.141Z [INFO] server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-254a4a21-a64c-4751-6720-3dd63e9e1a0c Address:254a4a21-a64c-4751-6720-3dd63e9e1a0c} {Suffrage:Voter ID:server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a Address:8f8f39a7-e892-6a0d-7d95-bf21e36bab6a} {Suffrage:Voter ID:server-98a5fd1a-8384-43a4-e856-599222f20e17 Address:98a5fd1a-8384-43a4-e856-599222f20e17}]" -2024-05-12T20:49:54.141Z [INFO] server-98a5fd1a-8384-43a4-e856-599222f20e17: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-254a4a21-a64c-4751-6720-3dd63e9e1a0c Address:254a4a21-a64c-4751-6720-3dd63e9e1a0c} {Suffrage:Voter ID:server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a Address:8f8f39a7-e892-6a0d-7d95-bf21e36bab6a} {Suffrage:Voter ID:server-98a5fd1a-8384-43a4-e856-599222f20e17 Address:98a5fd1a-8384-43a4-e856-599222f20e17}]" -2024-05-12T20:49:54.141Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:54.141Z [INFO] server-254a4a21-a64c-4751-6720-3dd63e9e1a0c: entering follower state: follower="Node at 254a4a21-a64c-4751-6720-3dd63e9e1a0c [Follower]" leader-address= leader-id= -2024-05-12T20:49:54.141Z [INFO] server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a: entering follower state: follower="Node at 8f8f39a7-e892-6a0d-7d95-bf21e36bab6a [Follower]" leader-address= leader-id= -2024-05-12T20:49:54.141Z [INFO] server-98a5fd1a-8384-43a4-e856-599222f20e17: entering follower state: follower="Node at 98a5fd1a-8384-43a4-e856-599222f20e17 [Follower]" leader-address= leader-id= -2024-05-12T20:49:54.210Z [WARN] server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:54.210Z [INFO] server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a: entering candidate state: node="Node at 8f8f39a7-e892-6a0d-7d95-bf21e36bab6a [Candidate]" term=2 -2024-05-12T20:49:54.210Z [INFO] server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a: election won: term=2 tally=2 -2024-05-12T20:49:54.210Z [INFO] server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a: entering leader state: leader="Node at 8f8f39a7-e892-6a0d-7d95-bf21e36bab6a [Leader]" -2024-05-12T20:49:54.211Z [INFO] server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a: added peer, starting replication: peer=server-254a4a21-a64c-4751-6720-3dd63e9e1a0c -2024-05-12T20:49:54.211Z [INFO] server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a: added peer, starting replication: peer=server-98a5fd1a-8384-43a4-e856-599222f20e17 -2024-05-12T20:49:54.211Z [INFO] server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a: pipelining replication: peer="{Voter server-98a5fd1a-8384-43a4-e856-599222f20e17 98a5fd1a-8384-43a4-e856-599222f20e17}" -2024-05-12T20:49:54.211Z [INFO] server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a: pipelining replication: peer="{Voter server-254a4a21-a64c-4751-6720-3dd63e9e1a0c 254a4a21-a64c-4751-6720-3dd63e9e1a0c}" -2024-05-12T20:49:54.318Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:54.211459432 +0000 UTC m=+5.492159855 (1 nodes), 69.802011ms from start of poll, 70.205438ms from cluster start. Timeout at 2024-05-12 20:49:54.318764959 +0000 UTC m=+5.599466395, 107.30654ms after stability -2024-05-12T20:49:54.319Z [INFO] server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a: aborting pipeline replication: peer="{Voter server-98a5fd1a-8384-43a4-e856-599222f20e17 98a5fd1a-8384-43a4-e856-599222f20e17}" -2024-05-12T20:49:54.319Z [INFO] server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a: aborting pipeline replication: peer="{Voter server-254a4a21-a64c-4751-6720-3dd63e9e1a0c 254a4a21-a64c-4751-6720-3dd63e9e1a0c}" -2024-05-12T20:49:54.320Z [INFO] creating new snapshot: path=/tmp/raft2883965011/snapshots/2-6-1715546994320.tmp -2024-05-12T20:49:54.321Z [INFO] server-254a4a21-a64c-4751-6720-3dd63e9e1a0c: starting restore from snapshot: id=2-6-1715546994320 last-index=6 last-term=2 size-in-bytes=25 -2024-05-12T20:49:54.322Z [INFO] server-254a4a21-a64c-4751-6720-3dd63e9e1a0c: snapshot restore progress: id=2-6-1715546994320 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2024-05-12T20:49:54.322Z [INFO] server-254a4a21-a64c-4751-6720-3dd63e9e1a0c: restored from snapshot: id=2-6-1715546994320 last-index=6 last-term=2 size-in-bytes=25 -2024-05-12T20:49:54.322Z [INFO] server-254a4a21-a64c-4751-6720-3dd63e9e1a0c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-254a4a21-a64c-4751-6720-3dd63e9e1a0c Address:254a4a21-a64c-4751-6720-3dd63e9e1a0c} {Suffrage:Voter ID:server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a Address:8f8f39a7-e892-6a0d-7d95-bf21e36bab6a} {Suffrage:Voter ID:server-98a5fd1a-8384-43a4-e856-599222f20e17 Address:98a5fd1a-8384-43a4-e856-599222f20e17}]" -2024-05-12T20:49:54.322Z [INFO] creating new snapshot: path=/tmp/raft2319850457/snapshots/2-6-1715546994322.tmp -2024-05-12T20:49:54.322Z [INFO] server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a: starting restore from snapshot: id=2-6-1715546994322 last-index=6 last-term=2 size-in-bytes=25 -2024-05-12T20:49:54.322Z [INFO] server-254a4a21-a64c-4751-6720-3dd63e9e1a0c: entering follower state: follower="Node at 254a4a21-a64c-4751-6720-3dd63e9e1a0c [Follower]" leader-address= leader-id= -2024-05-12T20:49:54.323Z [INFO] server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a: snapshot restore progress: id=2-6-1715546994322 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2024-05-12T20:49:54.323Z [INFO] server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a: restored from snapshot: id=2-6-1715546994322 last-index=6 last-term=2 size-in-bytes=25 -2024-05-12T20:49:54.323Z [INFO] server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-254a4a21-a64c-4751-6720-3dd63e9e1a0c Address:254a4a21-a64c-4751-6720-3dd63e9e1a0c} {Suffrage:Voter ID:server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a Address:8f8f39a7-e892-6a0d-7d95-bf21e36bab6a} {Suffrage:Voter ID:server-98a5fd1a-8384-43a4-e856-599222f20e17 Address:98a5fd1a-8384-43a4-e856-599222f20e17}]" -2024-05-12T20:49:54.323Z [INFO] creating new snapshot: path=/tmp/raft4095152103/snapshots/2-6-1715546994323.tmp -2024-05-12T20:49:54.323Z [INFO] server-98a5fd1a-8384-43a4-e856-599222f20e17: starting restore from snapshot: id=2-6-1715546994323 last-index=6 last-term=2 size-in-bytes=25 -2024-05-12T20:49:54.324Z [INFO] server-98a5fd1a-8384-43a4-e856-599222f20e17: snapshot restore progress: id=2-6-1715546994323 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2024-05-12T20:49:54.324Z [INFO] server-98a5fd1a-8384-43a4-e856-599222f20e17: restored from snapshot: id=2-6-1715546994323 last-index=6 last-term=2 size-in-bytes=25 -2024-05-12T20:49:54.324Z [INFO] server-98a5fd1a-8384-43a4-e856-599222f20e17: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-254a4a21-a64c-4751-6720-3dd63e9e1a0c Address:254a4a21-a64c-4751-6720-3dd63e9e1a0c} {Suffrage:Voter ID:server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a Address:8f8f39a7-e892-6a0d-7d95-bf21e36bab6a} {Suffrage:Voter ID:server-98a5fd1a-8384-43a4-e856-599222f20e17 Address:98a5fd1a-8384-43a4-e856-599222f20e17}]" -2024-05-12T20:49:54.324Z [INFO] server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a: entering follower state: follower="Node at 8f8f39a7-e892-6a0d-7d95-bf21e36bab6a [Follower]" leader-address= leader-id= -2024-05-12T20:49:54.324Z [INFO] server-98a5fd1a-8384-43a4-e856-599222f20e17: entering follower state: follower="Node at 98a5fd1a-8384-43a4-e856-599222f20e17 [Follower]" leader-address= leader-id= -2024-05-12T20:49:54.390Z [WARN] server-254a4a21-a64c-4751-6720-3dd63e9e1a0c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:54.390Z [INFO] server-254a4a21-a64c-4751-6720-3dd63e9e1a0c: entering candidate state: node="Node at 254a4a21-a64c-4751-6720-3dd63e9e1a0c [Candidate]" term=3 -2024-05-12T20:49:54.390Z [INFO] server-254a4a21-a64c-4751-6720-3dd63e9e1a0c: election won: term=3 tally=2 -2024-05-12T20:49:54.390Z [INFO] server-254a4a21-a64c-4751-6720-3dd63e9e1a0c: entering leader state: leader="Node at 254a4a21-a64c-4751-6720-3dd63e9e1a0c [Leader]" -2024-05-12T20:49:54.391Z [INFO] server-254a4a21-a64c-4751-6720-3dd63e9e1a0c: added peer, starting replication: peer=server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a -2024-05-12T20:49:54.391Z [INFO] server-254a4a21-a64c-4751-6720-3dd63e9e1a0c: added peer, starting replication: peer=server-98a5fd1a-8384-43a4-e856-599222f20e17 -2024-05-12T20:49:54.391Z [INFO] server-254a4a21-a64c-4751-6720-3dd63e9e1a0c: pipelining replication: peer="{Voter server-98a5fd1a-8384-43a4-e856-599222f20e17 98a5fd1a-8384-43a4-e856-599222f20e17}" -2024-05-12T20:49:54.391Z [INFO] server-254a4a21-a64c-4751-6720-3dd63e9e1a0c: pipelining replication: peer="{Voter server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a 8f8f39a7-e892-6a0d-7d95-bf21e36bab6a}" -2024-05-12T20:49:54.642Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:54.750Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:54.64296843 +0000 UTC m=+5.923668768 (1 nodes), 44.071µs from start of poll, 501.714351ms from cluster start. Timeout at 2024-05-12 20:49:54.750797061 +0000 UTC m=+6.031497683, 107.828915ms after stability -2024-05-12T20:49:54.751Z [INFO] server-254a4a21-a64c-4751-6720-3dd63e9e1a0c: aborting pipeline replication: peer="{Voter server-98a5fd1a-8384-43a4-e856-599222f20e17 98a5fd1a-8384-43a4-e856-599222f20e17}" -2024-05-12T20:49:54.751Z [INFO] server-254a4a21-a64c-4751-6720-3dd63e9e1a0c: aborting pipeline replication: peer="{Voter server-8f8f39a7-e892-6a0d-7d95-bf21e36bab6a 8f8f39a7-e892-6a0d-7d95-bf21e36bab6a}" +2025-06-15T03:16:29.457Z [INFO] server-b705197e-fe54-1e92-9db9-81234b294024: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b705197e-fe54-1e92-9db9-81234b294024 Address:b705197e-fe54-1e92-9db9-81234b294024} {Suffrage:Voter ID:server-4f21f229-4bde-c33e-00f6-250cd99bd0e9 Address:4f21f229-4bde-c33e-00f6-250cd99bd0e9} {Suffrage:Voter ID:server-ecde819e-a490-7990-1cc9-cff736204d4b Address:ecde819e-a490-7990-1cc9-cff736204d4b}]" +2025-06-15T03:16:29.457Z [INFO] server-b705197e-fe54-1e92-9db9-81234b294024: entering follower state: follower="Node at b705197e-fe54-1e92-9db9-81234b294024 [Follower]" leader-address= leader-id= +2025-06-15T03:16:29.457Z [INFO] server-4f21f229-4bde-c33e-00f6-250cd99bd0e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b705197e-fe54-1e92-9db9-81234b294024 Address:b705197e-fe54-1e92-9db9-81234b294024} {Suffrage:Voter ID:server-4f21f229-4bde-c33e-00f6-250cd99bd0e9 Address:4f21f229-4bde-c33e-00f6-250cd99bd0e9} {Suffrage:Voter ID:server-ecde819e-a490-7990-1cc9-cff736204d4b Address:ecde819e-a490-7990-1cc9-cff736204d4b}]" +2025-06-15T03:16:29.457Z [INFO] server-4f21f229-4bde-c33e-00f6-250cd99bd0e9: entering follower state: follower="Node at 4f21f229-4bde-c33e-00f6-250cd99bd0e9 [Follower]" leader-address= leader-id= +2025-06-15T03:16:29.457Z [INFO] server-ecde819e-a490-7990-1cc9-cff736204d4b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b705197e-fe54-1e92-9db9-81234b294024 Address:b705197e-fe54-1e92-9db9-81234b294024} {Suffrage:Voter ID:server-4f21f229-4bde-c33e-00f6-250cd99bd0e9 Address:4f21f229-4bde-c33e-00f6-250cd99bd0e9} {Suffrage:Voter ID:server-ecde819e-a490-7990-1cc9-cff736204d4b Address:ecde819e-a490-7990-1cc9-cff736204d4b}]" +2025-06-15T03:16:29.457Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:29.457Z [INFO] server-ecde819e-a490-7990-1cc9-cff736204d4b: entering follower state: follower="Node at ecde819e-a490-7990-1cc9-cff736204d4b [Follower]" leader-address= leader-id= +2025-06-15T03:16:29.530Z [WARN] server-ecde819e-a490-7990-1cc9-cff736204d4b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:29.530Z [INFO] server-ecde819e-a490-7990-1cc9-cff736204d4b: entering candidate state: node="Node at ecde819e-a490-7990-1cc9-cff736204d4b [Candidate]" term=2 +2025-06-15T03:16:29.530Z [INFO] server-ecde819e-a490-7990-1cc9-cff736204d4b: election won: term=2 tally=2 +2025-06-15T03:16:29.530Z [INFO] server-ecde819e-a490-7990-1cc9-cff736204d4b: entering leader state: leader="Node at ecde819e-a490-7990-1cc9-cff736204d4b [Leader]" +2025-06-15T03:16:29.530Z [INFO] server-ecde819e-a490-7990-1cc9-cff736204d4b: added peer, starting replication: peer=server-b705197e-fe54-1e92-9db9-81234b294024 +2025-06-15T03:16:29.530Z [INFO] server-ecde819e-a490-7990-1cc9-cff736204d4b: added peer, starting replication: peer=server-4f21f229-4bde-c33e-00f6-250cd99bd0e9 +2025-06-15T03:16:29.530Z [INFO] server-ecde819e-a490-7990-1cc9-cff736204d4b: pipelining replication: peer="{Voter server-b705197e-fe54-1e92-9db9-81234b294024 b705197e-fe54-1e92-9db9-81234b294024}" +2025-06-15T03:16:29.530Z [INFO] server-ecde819e-a490-7990-1cc9-cff736204d4b: pipelining replication: peer="{Voter server-4f21f229-4bde-c33e-00f6-250cd99bd0e9 4f21f229-4bde-c33e-00f6-250cd99bd0e9}" +2025-06-15T03:16:29.635Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:29.530292047 +0000 UTC m=+5.241970231 (1 nodes), 72.815802ms from start of poll, 73.232167ms from cluster start. Timeout at 2025-06-15 03:16:29.635380244 +0000 UTC m=+5.347058409, 105.088178ms after stability +2025-06-15T03:16:29.635Z [INFO] server-ecde819e-a490-7990-1cc9-cff736204d4b: aborting pipeline replication: peer="{Voter server-4f21f229-4bde-c33e-00f6-250cd99bd0e9 4f21f229-4bde-c33e-00f6-250cd99bd0e9}" +2025-06-15T03:16:29.635Z [INFO] server-ecde819e-a490-7990-1cc9-cff736204d4b: aborting pipeline replication: peer="{Voter server-b705197e-fe54-1e92-9db9-81234b294024 b705197e-fe54-1e92-9db9-81234b294024}" +2025-06-15T03:16:29.635Z [INFO] creating new snapshot: path=/tmp/raft3054068618/snapshots/2-6-1749957389635.tmp +2025-06-15T03:16:29.636Z [INFO] server-b705197e-fe54-1e92-9db9-81234b294024: starting restore from snapshot: id=2-6-1749957389635 last-index=6 last-term=2 size-in-bytes=25 +2025-06-15T03:16:29.636Z [INFO] server-b705197e-fe54-1e92-9db9-81234b294024: snapshot restore progress: id=2-6-1749957389635 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-06-15T03:16:29.636Z [INFO] server-b705197e-fe54-1e92-9db9-81234b294024: restored from snapshot: id=2-6-1749957389635 last-index=6 last-term=2 size-in-bytes=25 +2025-06-15T03:16:29.636Z [INFO] server-b705197e-fe54-1e92-9db9-81234b294024: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b705197e-fe54-1e92-9db9-81234b294024 Address:b705197e-fe54-1e92-9db9-81234b294024} {Suffrage:Voter ID:server-4f21f229-4bde-c33e-00f6-250cd99bd0e9 Address:4f21f229-4bde-c33e-00f6-250cd99bd0e9} {Suffrage:Voter ID:server-ecde819e-a490-7990-1cc9-cff736204d4b Address:ecde819e-a490-7990-1cc9-cff736204d4b}]" +2025-06-15T03:16:29.636Z [INFO] server-b705197e-fe54-1e92-9db9-81234b294024: entering follower state: follower="Node at b705197e-fe54-1e92-9db9-81234b294024 [Follower]" leader-address= leader-id= +2025-06-15T03:16:29.636Z [INFO] creating new snapshot: path=/tmp/raft2770767400/snapshots/2-6-1749957389636.tmp +2025-06-15T03:16:29.637Z [INFO] server-4f21f229-4bde-c33e-00f6-250cd99bd0e9: starting restore from snapshot: id=2-6-1749957389636 last-index=6 last-term=2 size-in-bytes=25 +2025-06-15T03:16:29.637Z [INFO] server-4f21f229-4bde-c33e-00f6-250cd99bd0e9: snapshot restore progress: id=2-6-1749957389636 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-06-15T03:16:29.637Z [INFO] server-4f21f229-4bde-c33e-00f6-250cd99bd0e9: restored from snapshot: id=2-6-1749957389636 last-index=6 last-term=2 size-in-bytes=25 +2025-06-15T03:16:29.637Z [INFO] server-4f21f229-4bde-c33e-00f6-250cd99bd0e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b705197e-fe54-1e92-9db9-81234b294024 Address:b705197e-fe54-1e92-9db9-81234b294024} {Suffrage:Voter ID:server-4f21f229-4bde-c33e-00f6-250cd99bd0e9 Address:4f21f229-4bde-c33e-00f6-250cd99bd0e9} {Suffrage:Voter ID:server-ecde819e-a490-7990-1cc9-cff736204d4b Address:ecde819e-a490-7990-1cc9-cff736204d4b}]" +2025-06-15T03:16:29.637Z [INFO] server-4f21f229-4bde-c33e-00f6-250cd99bd0e9: entering follower state: follower="Node at 4f21f229-4bde-c33e-00f6-250cd99bd0e9 [Follower]" leader-address= leader-id= +2025-06-15T03:16:29.637Z [INFO] creating new snapshot: path=/tmp/raft2630086846/snapshots/2-6-1749957389637.tmp +2025-06-15T03:16:29.638Z [INFO] server-ecde819e-a490-7990-1cc9-cff736204d4b: starting restore from snapshot: id=2-6-1749957389637 last-index=6 last-term=2 size-in-bytes=25 +2025-06-15T03:16:29.638Z [INFO] server-ecde819e-a490-7990-1cc9-cff736204d4b: snapshot restore progress: id=2-6-1749957389637 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-06-15T03:16:29.638Z [INFO] server-ecde819e-a490-7990-1cc9-cff736204d4b: restored from snapshot: id=2-6-1749957389637 last-index=6 last-term=2 size-in-bytes=25 +2025-06-15T03:16:29.638Z [INFO] server-ecde819e-a490-7990-1cc9-cff736204d4b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b705197e-fe54-1e92-9db9-81234b294024 Address:b705197e-fe54-1e92-9db9-81234b294024} {Suffrage:Voter ID:server-4f21f229-4bde-c33e-00f6-250cd99bd0e9 Address:4f21f229-4bde-c33e-00f6-250cd99bd0e9} {Suffrage:Voter ID:server-ecde819e-a490-7990-1cc9-cff736204d4b Address:ecde819e-a490-7990-1cc9-cff736204d4b}]" +2025-06-15T03:16:29.638Z [INFO] server-ecde819e-a490-7990-1cc9-cff736204d4b: entering follower state: follower="Node at ecde819e-a490-7990-1cc9-cff736204d4b [Follower]" leader-address= leader-id= +2025-06-15T03:16:29.692Z [WARN] server-4f21f229-4bde-c33e-00f6-250cd99bd0e9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:29.692Z [INFO] server-4f21f229-4bde-c33e-00f6-250cd99bd0e9: entering candidate state: node="Node at 4f21f229-4bde-c33e-00f6-250cd99bd0e9 [Candidate]" term=3 +2025-06-15T03:16:29.692Z [INFO] server-4f21f229-4bde-c33e-00f6-250cd99bd0e9: election won: term=3 tally=2 +2025-06-15T03:16:29.692Z [INFO] server-4f21f229-4bde-c33e-00f6-250cd99bd0e9: entering leader state: leader="Node at 4f21f229-4bde-c33e-00f6-250cd99bd0e9 [Leader]" +2025-06-15T03:16:29.692Z [INFO] server-4f21f229-4bde-c33e-00f6-250cd99bd0e9: added peer, starting replication: peer=server-b705197e-fe54-1e92-9db9-81234b294024 +2025-06-15T03:16:29.692Z [INFO] server-4f21f229-4bde-c33e-00f6-250cd99bd0e9: added peer, starting replication: peer=server-ecde819e-a490-7990-1cc9-cff736204d4b +2025-06-15T03:16:29.692Z [INFO] server-4f21f229-4bde-c33e-00f6-250cd99bd0e9: pipelining replication: peer="{Voter server-ecde819e-a490-7990-1cc9-cff736204d4b ecde819e-a490-7990-1cc9-cff736204d4b}" +2025-06-15T03:16:29.692Z [INFO] server-4f21f229-4bde-c33e-00f6-250cd99bd0e9: pipelining replication: peer="{Voter server-b705197e-fe54-1e92-9db9-81234b294024 b705197e-fe54-1e92-9db9-81234b294024}" +2025-06-15T03:16:29.953Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:30.059Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:29.953682453 +0000 UTC m=+5.665360619 (1 nodes), 18.528µs from start of poll, 496.622555ms from cluster start. Timeout at 2025-06-15 03:16:30.059182447 +0000 UTC m=+5.770860612, 105.499993ms after stability +2025-06-15T03:16:30.059Z [INFO] server-4f21f229-4bde-c33e-00f6-250cd99bd0e9: aborting pipeline replication: peer="{Voter server-ecde819e-a490-7990-1cc9-cff736204d4b ecde819e-a490-7990-1cc9-cff736204d4b}" +2025-06-15T03:16:30.059Z [INFO] server-4f21f229-4bde-c33e-00f6-250cd99bd0e9: aborting pipeline replication: peer="{Voter server-b705197e-fe54-1e92-9db9-81234b294024 b705197e-fe54-1e92-9db9-81234b294024}" === RUN TestRaft_RecoverCluster/snapshot,_with_trailing_logs -2024-05-12T20:49:54.753Z [INFO] server-f776917b-3330-fcf2-0a77-c5196b4a67d4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f776917b-3330-fcf2-0a77-c5196b4a67d4 Address:f776917b-3330-fcf2-0a77-c5196b4a67d4} {Suffrage:Voter ID:server-7a83ed19-09e4-3ea9-6989-11e385c48d1f Address:7a83ed19-09e4-3ea9-6989-11e385c48d1f} {Suffrage:Voter ID:server-e5892d18-fc5d-6995-c813-b747477b0654 Address:e5892d18-fc5d-6995-c813-b747477b0654}]" -2024-05-12T20:49:54.753Z [INFO] server-7a83ed19-09e4-3ea9-6989-11e385c48d1f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f776917b-3330-fcf2-0a77-c5196b4a67d4 Address:f776917b-3330-fcf2-0a77-c5196b4a67d4} {Suffrage:Voter ID:server-7a83ed19-09e4-3ea9-6989-11e385c48d1f Address:7a83ed19-09e4-3ea9-6989-11e385c48d1f} {Suffrage:Voter ID:server-e5892d18-fc5d-6995-c813-b747477b0654 Address:e5892d18-fc5d-6995-c813-b747477b0654}]" -2024-05-12T20:49:54.754Z [INFO] server-e5892d18-fc5d-6995-c813-b747477b0654: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f776917b-3330-fcf2-0a77-c5196b4a67d4 Address:f776917b-3330-fcf2-0a77-c5196b4a67d4} {Suffrage:Voter ID:server-7a83ed19-09e4-3ea9-6989-11e385c48d1f Address:7a83ed19-09e4-3ea9-6989-11e385c48d1f} {Suffrage:Voter ID:server-e5892d18-fc5d-6995-c813-b747477b0654 Address:e5892d18-fc5d-6995-c813-b747477b0654}]" -2024-05-12T20:49:54.754Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:54.754Z [INFO] server-f776917b-3330-fcf2-0a77-c5196b4a67d4: entering follower state: follower="Node at f776917b-3330-fcf2-0a77-c5196b4a67d4 [Follower]" leader-address= leader-id= -2024-05-12T20:49:54.754Z [INFO] server-7a83ed19-09e4-3ea9-6989-11e385c48d1f: entering follower state: follower="Node at 7a83ed19-09e4-3ea9-6989-11e385c48d1f [Follower]" leader-address= leader-id= -2024-05-12T20:49:54.754Z [INFO] server-e5892d18-fc5d-6995-c813-b747477b0654: entering follower state: follower="Node at e5892d18-fc5d-6995-c813-b747477b0654 [Follower]" leader-address= leader-id= -2024-05-12T20:49:54.826Z [WARN] server-7a83ed19-09e4-3ea9-6989-11e385c48d1f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:54.826Z [INFO] server-7a83ed19-09e4-3ea9-6989-11e385c48d1f: entering candidate state: node="Node at 7a83ed19-09e4-3ea9-6989-11e385c48d1f [Candidate]" term=2 -2024-05-12T20:49:54.826Z [INFO] server-7a83ed19-09e4-3ea9-6989-11e385c48d1f: election won: term=2 tally=2 -2024-05-12T20:49:54.826Z [INFO] server-7a83ed19-09e4-3ea9-6989-11e385c48d1f: entering leader state: leader="Node at 7a83ed19-09e4-3ea9-6989-11e385c48d1f [Leader]" -2024-05-12T20:49:54.827Z [INFO] server-7a83ed19-09e4-3ea9-6989-11e385c48d1f: added peer, starting replication: peer=server-f776917b-3330-fcf2-0a77-c5196b4a67d4 -2024-05-12T20:49:54.827Z [INFO] server-7a83ed19-09e4-3ea9-6989-11e385c48d1f: added peer, starting replication: peer=server-e5892d18-fc5d-6995-c813-b747477b0654 -2024-05-12T20:49:54.827Z [INFO] server-7a83ed19-09e4-3ea9-6989-11e385c48d1f: pipelining replication: peer="{Voter server-e5892d18-fc5d-6995-c813-b747477b0654 e5892d18-fc5d-6995-c813-b747477b0654}" -2024-05-12T20:49:54.827Z [INFO] server-7a83ed19-09e4-3ea9-6989-11e385c48d1f: pipelining replication: peer="{Voter server-f776917b-3330-fcf2-0a77-c5196b4a67d4 f776917b-3330-fcf2-0a77-c5196b4a67d4}" -2024-05-12T20:49:54.942Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:54.827442475 +0000 UTC m=+6.108142810 (1 nodes), 73.355052ms from start of poll, 73.890559ms from cluster start. Timeout at 2024-05-12 20:49:54.942875239 +0000 UTC m=+6.223575623, 115.432813ms after stability -2024-05-12T20:49:54.949Z [INFO] server-7a83ed19-09e4-3ea9-6989-11e385c48d1f: aborting pipeline replication: peer="{Voter server-e5892d18-fc5d-6995-c813-b747477b0654 e5892d18-fc5d-6995-c813-b747477b0654}" -2024-05-12T20:49:54.949Z [INFO] server-7a83ed19-09e4-3ea9-6989-11e385c48d1f: aborting pipeline replication: peer="{Voter server-f776917b-3330-fcf2-0a77-c5196b4a67d4 f776917b-3330-fcf2-0a77-c5196b4a67d4}" -2024-05-12T20:49:54.949Z [INFO] creating new snapshot: path=/tmp/raft596842790/snapshots/2-27-1715546994949.tmp -2024-05-12T20:49:54.951Z [INFO] server-f776917b-3330-fcf2-0a77-c5196b4a67d4: starting restore from snapshot: id=2-27-1715546994949 last-index=27 last-term=2 size-in-bytes=168 -2024-05-12T20:49:54.951Z [INFO] server-f776917b-3330-fcf2-0a77-c5196b4a67d4: snapshot restore progress: id=2-27-1715546994949 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2024-05-12T20:49:54.951Z [INFO] server-f776917b-3330-fcf2-0a77-c5196b4a67d4: restored from snapshot: id=2-27-1715546994949 last-index=27 last-term=2 size-in-bytes=168 -2024-05-12T20:49:54.951Z [INFO] server-f776917b-3330-fcf2-0a77-c5196b4a67d4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f776917b-3330-fcf2-0a77-c5196b4a67d4 Address:f776917b-3330-fcf2-0a77-c5196b4a67d4} {Suffrage:Voter ID:server-7a83ed19-09e4-3ea9-6989-11e385c48d1f Address:7a83ed19-09e4-3ea9-6989-11e385c48d1f} {Suffrage:Voter ID:server-e5892d18-fc5d-6995-c813-b747477b0654 Address:e5892d18-fc5d-6995-c813-b747477b0654}]" -2024-05-12T20:49:54.951Z [INFO] creating new snapshot: path=/tmp/raft626237601/snapshots/2-27-1715546994951.tmp -2024-05-12T20:49:54.952Z [INFO] server-7a83ed19-09e4-3ea9-6989-11e385c48d1f: starting restore from snapshot: id=2-27-1715546994951 last-index=27 last-term=2 size-in-bytes=168 -2024-05-12T20:49:54.962Z [INFO] server-f776917b-3330-fcf2-0a77-c5196b4a67d4: entering follower state: follower="Node at f776917b-3330-fcf2-0a77-c5196b4a67d4 [Follower]" leader-address= leader-id= -2024-05-12T20:49:54.963Z [INFO] server-7a83ed19-09e4-3ea9-6989-11e385c48d1f: snapshot restore progress: id=2-27-1715546994951 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2024-05-12T20:49:54.963Z [INFO] server-7a83ed19-09e4-3ea9-6989-11e385c48d1f: restored from snapshot: id=2-27-1715546994951 last-index=27 last-term=2 size-in-bytes=168 -2024-05-12T20:49:54.963Z [INFO] server-7a83ed19-09e4-3ea9-6989-11e385c48d1f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f776917b-3330-fcf2-0a77-c5196b4a67d4 Address:f776917b-3330-fcf2-0a77-c5196b4a67d4} {Suffrage:Voter ID:server-7a83ed19-09e4-3ea9-6989-11e385c48d1f Address:7a83ed19-09e4-3ea9-6989-11e385c48d1f} {Suffrage:Voter ID:server-e5892d18-fc5d-6995-c813-b747477b0654 Address:e5892d18-fc5d-6995-c813-b747477b0654}]" -2024-05-12T20:49:54.963Z [INFO] creating new snapshot: path=/tmp/raft2521542956/snapshots/2-27-1715546994963.tmp -2024-05-12T20:49:54.964Z [INFO] server-e5892d18-fc5d-6995-c813-b747477b0654: starting restore from snapshot: id=2-27-1715546994963 last-index=27 last-term=2 size-in-bytes=168 -2024-05-12T20:49:54.965Z [INFO] server-e5892d18-fc5d-6995-c813-b747477b0654: snapshot restore progress: id=2-27-1715546994963 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2024-05-12T20:49:54.965Z [INFO] server-e5892d18-fc5d-6995-c813-b747477b0654: restored from snapshot: id=2-27-1715546994963 last-index=27 last-term=2 size-in-bytes=168 -2024-05-12T20:49:54.965Z [INFO] server-e5892d18-fc5d-6995-c813-b747477b0654: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f776917b-3330-fcf2-0a77-c5196b4a67d4 Address:f776917b-3330-fcf2-0a77-c5196b4a67d4} {Suffrage:Voter ID:server-7a83ed19-09e4-3ea9-6989-11e385c48d1f Address:7a83ed19-09e4-3ea9-6989-11e385c48d1f} {Suffrage:Voter ID:server-e5892d18-fc5d-6995-c813-b747477b0654 Address:e5892d18-fc5d-6995-c813-b747477b0654}]" -2024-05-12T20:49:54.965Z [INFO] server-7a83ed19-09e4-3ea9-6989-11e385c48d1f: entering follower state: follower="Node at 7a83ed19-09e4-3ea9-6989-11e385c48d1f [Follower]" leader-address= leader-id= -2024-05-12T20:49:54.965Z [INFO] server-e5892d18-fc5d-6995-c813-b747477b0654: entering follower state: follower="Node at e5892d18-fc5d-6995-c813-b747477b0654 [Follower]" leader-address= leader-id= -2024-05-12T20:49:55.026Z [WARN] server-f776917b-3330-fcf2-0a77-c5196b4a67d4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:55.026Z [INFO] server-f776917b-3330-fcf2-0a77-c5196b4a67d4: entering candidate state: node="Node at f776917b-3330-fcf2-0a77-c5196b4a67d4 [Candidate]" term=3 -2024-05-12T20:49:55.027Z [INFO] server-f776917b-3330-fcf2-0a77-c5196b4a67d4: election won: term=3 tally=2 -2024-05-12T20:49:55.027Z [INFO] server-f776917b-3330-fcf2-0a77-c5196b4a67d4: entering leader state: leader="Node at f776917b-3330-fcf2-0a77-c5196b4a67d4 [Leader]" -2024-05-12T20:49:55.027Z [INFO] server-f776917b-3330-fcf2-0a77-c5196b4a67d4: added peer, starting replication: peer=server-7a83ed19-09e4-3ea9-6989-11e385c48d1f -2024-05-12T20:49:55.027Z [INFO] server-f776917b-3330-fcf2-0a77-c5196b4a67d4: added peer, starting replication: peer=server-e5892d18-fc5d-6995-c813-b747477b0654 -2024-05-12T20:49:55.027Z [INFO] server-f776917b-3330-fcf2-0a77-c5196b4a67d4: pipelining replication: peer="{Voter server-e5892d18-fc5d-6995-c813-b747477b0654 e5892d18-fc5d-6995-c813-b747477b0654}" -2024-05-12T20:49:55.027Z [INFO] server-f776917b-3330-fcf2-0a77-c5196b4a67d4: pipelining replication: peer="{Voter server-7a83ed19-09e4-3ea9-6989-11e385c48d1f 7a83ed19-09e4-3ea9-6989-11e385c48d1f}" -2024-05-12T20:49:55.282Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:55.398Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:55.283079085 +0000 UTC m=+6.563779497 (1 nodes), 69.251µs from start of poll, 529.527246ms from cluster start. Timeout at 2024-05-12 20:49:55.398770278 +0000 UTC m=+6.679470846, 115.691349ms after stability -2024-05-12T20:49:55.399Z [INFO] server-f776917b-3330-fcf2-0a77-c5196b4a67d4: aborting pipeline replication: peer="{Voter server-e5892d18-fc5d-6995-c813-b747477b0654 e5892d18-fc5d-6995-c813-b747477b0654}" -2024-05-12T20:49:55.399Z [INFO] server-f776917b-3330-fcf2-0a77-c5196b4a67d4: aborting pipeline replication: peer="{Voter server-7a83ed19-09e4-3ea9-6989-11e385c48d1f 7a83ed19-09e4-3ea9-6989-11e385c48d1f}" -2024-05-12T20:49:55.902Z [ERROR] server-f776917b-3330-fcf2-0a77-c5196b4a67d4: failed to heartbeat to: peer=e5892d18-fc5d-6995-c813-b747477b0654 backoff time=10ms error="command timed out" ---- PASS: TestRaft_RecoverCluster (2.38s) - --- PASS: TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs (0.61s) - --- PASS: TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs (0.61s) - --- PASS: TestRaft_RecoverCluster/snapshot,_with_trailing_logs (1.15s) +2025-06-15T03:16:30.060Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74 Address:4ef1e45e-4347-66aa-58f0-6ac5eded6e74} {Suffrage:Voter ID:server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6 Address:d1c483f8-ba7b-c000-6796-fe6fe82c6ab6} {Suffrage:Voter ID:server-931e191e-3735-6972-78bf-b10bd5897f0b Address:931e191e-3735-6972-78bf-b10bd5897f0b}]" +2025-06-15T03:16:30.060Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: entering follower state: follower="Node at 4ef1e45e-4347-66aa-58f0-6ac5eded6e74 [Follower]" leader-address= leader-id= +2025-06-15T03:16:30.060Z [INFO] server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74 Address:4ef1e45e-4347-66aa-58f0-6ac5eded6e74} {Suffrage:Voter ID:server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6 Address:d1c483f8-ba7b-c000-6796-fe6fe82c6ab6} {Suffrage:Voter ID:server-931e191e-3735-6972-78bf-b10bd5897f0b Address:931e191e-3735-6972-78bf-b10bd5897f0b}]" +2025-06-15T03:16:30.060Z [INFO] server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6: entering follower state: follower="Node at d1c483f8-ba7b-c000-6796-fe6fe82c6ab6 [Follower]" leader-address= leader-id= +2025-06-15T03:16:30.060Z [INFO] server-931e191e-3735-6972-78bf-b10bd5897f0b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74 Address:4ef1e45e-4347-66aa-58f0-6ac5eded6e74} {Suffrage:Voter ID:server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6 Address:d1c483f8-ba7b-c000-6796-fe6fe82c6ab6} {Suffrage:Voter ID:server-931e191e-3735-6972-78bf-b10bd5897f0b Address:931e191e-3735-6972-78bf-b10bd5897f0b}]" +2025-06-15T03:16:30.060Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:30.060Z [INFO] server-931e191e-3735-6972-78bf-b10bd5897f0b: entering follower state: follower="Node at 931e191e-3735-6972-78bf-b10bd5897f0b [Follower]" leader-address= leader-id= +2025-06-15T03:16:30.122Z [WARN] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:30.122Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: entering candidate state: node="Node at 4ef1e45e-4347-66aa-58f0-6ac5eded6e74 [Candidate]" term=2 +2025-06-15T03:16:30.123Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: election won: term=2 tally=2 +2025-06-15T03:16:30.123Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: entering leader state: leader="Node at 4ef1e45e-4347-66aa-58f0-6ac5eded6e74 [Leader]" +2025-06-15T03:16:30.123Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: added peer, starting replication: peer=server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6 +2025-06-15T03:16:30.123Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: added peer, starting replication: peer=server-931e191e-3735-6972-78bf-b10bd5897f0b +2025-06-15T03:16:30.123Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: pipelining replication: peer="{Voter server-931e191e-3735-6972-78bf-b10bd5897f0b 931e191e-3735-6972-78bf-b10bd5897f0b}" +2025-06-15T03:16:30.123Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: pipelining replication: peer="{Voter server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6 d1c483f8-ba7b-c000-6796-fe6fe82c6ab6}" +2025-06-15T03:16:30.228Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:30.123109486 +0000 UTC m=+5.834787648 (1 nodes), 62.305348ms from start of poll, 62.678052ms from cluster start. Timeout at 2025-06-15 03:16:30.2289216 +0000 UTC m=+5.940599761, 105.812113ms after stability +2025-06-15T03:16:30.231Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: aborting pipeline replication: peer="{Voter server-931e191e-3735-6972-78bf-b10bd5897f0b 931e191e-3735-6972-78bf-b10bd5897f0b}" +2025-06-15T03:16:30.231Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: aborting pipeline replication: peer="{Voter server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6 d1c483f8-ba7b-c000-6796-fe6fe82c6ab6}" +2025-06-15T03:16:30.231Z [INFO] creating new snapshot: path=/tmp/raft3933839026/snapshots/2-27-1749957390231.tmp +2025-06-15T03:16:30.232Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: starting restore from snapshot: id=2-27-1749957390231 last-index=27 last-term=2 size-in-bytes=168 +2025-06-15T03:16:30.232Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: snapshot restore progress: id=2-27-1749957390231 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-06-15T03:16:30.232Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: restored from snapshot: id=2-27-1749957390231 last-index=27 last-term=2 size-in-bytes=168 +2025-06-15T03:16:30.232Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74 Address:4ef1e45e-4347-66aa-58f0-6ac5eded6e74} {Suffrage:Voter ID:server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6 Address:d1c483f8-ba7b-c000-6796-fe6fe82c6ab6} {Suffrage:Voter ID:server-931e191e-3735-6972-78bf-b10bd5897f0b Address:931e191e-3735-6972-78bf-b10bd5897f0b}]" +2025-06-15T03:16:30.232Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: entering follower state: follower="Node at 4ef1e45e-4347-66aa-58f0-6ac5eded6e74 [Follower]" leader-address= leader-id= +2025-06-15T03:16:30.232Z [INFO] creating new snapshot: path=/tmp/raft633873020/snapshots/2-27-1749957390232.tmp +2025-06-15T03:16:30.233Z [INFO] server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6: starting restore from snapshot: id=2-27-1749957390232 last-index=27 last-term=2 size-in-bytes=168 +2025-06-15T03:16:30.233Z [INFO] server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6: snapshot restore progress: id=2-27-1749957390232 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-06-15T03:16:30.233Z [INFO] server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6: restored from snapshot: id=2-27-1749957390232 last-index=27 last-term=2 size-in-bytes=168 +2025-06-15T03:16:30.233Z [INFO] server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74 Address:4ef1e45e-4347-66aa-58f0-6ac5eded6e74} {Suffrage:Voter ID:server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6 Address:d1c483f8-ba7b-c000-6796-fe6fe82c6ab6} {Suffrage:Voter ID:server-931e191e-3735-6972-78bf-b10bd5897f0b Address:931e191e-3735-6972-78bf-b10bd5897f0b}]" +2025-06-15T03:16:30.233Z [INFO] server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6: entering follower state: follower="Node at d1c483f8-ba7b-c000-6796-fe6fe82c6ab6 [Follower]" leader-address= leader-id= +2025-06-15T03:16:30.233Z [INFO] creating new snapshot: path=/tmp/raft4083060745/snapshots/2-27-1749957390233.tmp +2025-06-15T03:16:30.234Z [INFO] server-931e191e-3735-6972-78bf-b10bd5897f0b: starting restore from snapshot: id=2-27-1749957390233 last-index=27 last-term=2 size-in-bytes=168 +2025-06-15T03:16:30.234Z [INFO] server-931e191e-3735-6972-78bf-b10bd5897f0b: snapshot restore progress: id=2-27-1749957390233 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-06-15T03:16:30.234Z [INFO] server-931e191e-3735-6972-78bf-b10bd5897f0b: restored from snapshot: id=2-27-1749957390233 last-index=27 last-term=2 size-in-bytes=168 +2025-06-15T03:16:30.234Z [INFO] server-931e191e-3735-6972-78bf-b10bd5897f0b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74 Address:4ef1e45e-4347-66aa-58f0-6ac5eded6e74} {Suffrage:Voter ID:server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6 Address:d1c483f8-ba7b-c000-6796-fe6fe82c6ab6} {Suffrage:Voter ID:server-931e191e-3735-6972-78bf-b10bd5897f0b Address:931e191e-3735-6972-78bf-b10bd5897f0b}]" +2025-06-15T03:16:30.234Z [INFO] server-931e191e-3735-6972-78bf-b10bd5897f0b: entering follower state: follower="Node at 931e191e-3735-6972-78bf-b10bd5897f0b [Follower]" leader-address= leader-id= +2025-06-15T03:16:30.285Z [WARN] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:30.285Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: entering candidate state: node="Node at 4ef1e45e-4347-66aa-58f0-6ac5eded6e74 [Candidate]" term=3 +2025-06-15T03:16:30.285Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: election won: term=3 tally=2 +2025-06-15T03:16:30.285Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: entering leader state: leader="Node at 4ef1e45e-4347-66aa-58f0-6ac5eded6e74 [Leader]" +2025-06-15T03:16:30.285Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: added peer, starting replication: peer=server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6 +2025-06-15T03:16:30.285Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: added peer, starting replication: peer=server-931e191e-3735-6972-78bf-b10bd5897f0b +2025-06-15T03:16:30.285Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: pipelining replication: peer="{Voter server-931e191e-3735-6972-78bf-b10bd5897f0b 931e191e-3735-6972-78bf-b10bd5897f0b}" +2025-06-15T03:16:30.286Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: pipelining replication: peer="{Voter server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6 d1c483f8-ba7b-c000-6796-fe6fe82c6ab6}" +2025-06-15T03:16:30.550Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:30.655Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:30.550769074 +0000 UTC m=+6.262447236 (1 nodes), 33.834µs from start of poll, 490.33764ms from cluster start. Timeout at 2025-06-15 03:16:30.655854395 +0000 UTC m=+6.367532557, 105.085321ms after stability +2025-06-15T03:16:30.655Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: aborting pipeline replication: peer="{Voter server-d1c483f8-ba7b-c000-6796-fe6fe82c6ab6 d1c483f8-ba7b-c000-6796-fe6fe82c6ab6}" +2025-06-15T03:16:30.656Z [INFO] server-4ef1e45e-4347-66aa-58f0-6ac5eded6e74: aborting pipeline replication: peer="{Voter server-931e191e-3735-6972-78bf-b10bd5897f0b 931e191e-3735-6972-78bf-b10bd5897f0b}" +--- PASS: TestRaft_RecoverCluster (1.78s) + --- PASS: TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs (0.58s) + --- PASS: TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs (0.60s) + --- PASS: TestRaft_RecoverCluster/snapshot,_with_trailing_logs (0.60s) === RUN TestRaft_HasExistingState raft_test.go:258: Skipping unreliable test --- SKIP: TestRaft_HasExistingState (0.00s) === RUN TestRaft_SingleNode -2024-05-12T20:49:55.905Z [INFO] server-d3809a67-4771-a0ea-63b8-09e182168a1d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3809a67-4771-a0ea-63b8-09e182168a1d Address:d3809a67-4771-a0ea-63b8-09e182168a1d}]" -2024-05-12T20:49:55.905Z [INFO] server-d3809a67-4771-a0ea-63b8-09e182168a1d: entering follower state: follower="Node at d3809a67-4771-a0ea-63b8-09e182168a1d [Follower]" leader-address= leader-id= -2024-05-12T20:49:55.962Z [WARN] server-d3809a67-4771-a0ea-63b8-09e182168a1d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:55.962Z [INFO] server-d3809a67-4771-a0ea-63b8-09e182168a1d: entering candidate state: node="Node at d3809a67-4771-a0ea-63b8-09e182168a1d [Candidate]" term=2 -2024-05-12T20:49:55.963Z [INFO] server-d3809a67-4771-a0ea-63b8-09e182168a1d: election won: term=2 tally=1 -2024-05-12T20:49:55.963Z [INFO] server-d3809a67-4771-a0ea-63b8-09e182168a1d: entering leader state: leader="Node at d3809a67-4771-a0ea-63b8-09e182168a1d [Leader]" ---- PASS: TestRaft_SingleNode (0.06s) +2025-06-15T03:16:30.657Z [INFO] server-7e63bdf9-fcb0-7b32-d5f0-b49de3f504c7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7e63bdf9-fcb0-7b32-d5f0-b49de3f504c7 Address:7e63bdf9-fcb0-7b32-d5f0-b49de3f504c7}]" +2025-06-15T03:16:30.657Z [INFO] server-7e63bdf9-fcb0-7b32-d5f0-b49de3f504c7: entering follower state: follower="Node at 7e63bdf9-fcb0-7b32-d5f0-b49de3f504c7 [Follower]" leader-address= leader-id= +2025-06-15T03:16:30.738Z [WARN] server-7e63bdf9-fcb0-7b32-d5f0-b49de3f504c7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:30.738Z [INFO] server-7e63bdf9-fcb0-7b32-d5f0-b49de3f504c7: entering candidate state: node="Node at 7e63bdf9-fcb0-7b32-d5f0-b49de3f504c7 [Candidate]" term=2 +2025-06-15T03:16:30.738Z [INFO] server-7e63bdf9-fcb0-7b32-d5f0-b49de3f504c7: election won: term=2 tally=1 +2025-06-15T03:16:30.738Z [INFO] server-7e63bdf9-fcb0-7b32-d5f0-b49de3f504c7: entering leader state: leader="Node at 7e63bdf9-fcb0-7b32-d5f0-b49de3f504c7 [Leader]" +--- PASS: TestRaft_SingleNode (0.08s) === RUN TestRaft_TripleNode -2024-05-12T20:49:55.965Z [INFO] server-dfba48c7-24ab-4507-7673-e1a676d2ecf7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dfba48c7-24ab-4507-7673-e1a676d2ecf7 Address:dfba48c7-24ab-4507-7673-e1a676d2ecf7} {Suffrage:Voter ID:server-602866c0-8753-d539-029b-02f7109610f2 Address:602866c0-8753-d539-029b-02f7109610f2} {Suffrage:Voter ID:server-11a7278c-76d8-4f00-4135-7217a5dc39c8 Address:11a7278c-76d8-4f00-4135-7217a5dc39c8}]" -2024-05-12T20:49:55.965Z [INFO] server-602866c0-8753-d539-029b-02f7109610f2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dfba48c7-24ab-4507-7673-e1a676d2ecf7 Address:dfba48c7-24ab-4507-7673-e1a676d2ecf7} {Suffrage:Voter ID:server-602866c0-8753-d539-029b-02f7109610f2 Address:602866c0-8753-d539-029b-02f7109610f2} {Suffrage:Voter ID:server-11a7278c-76d8-4f00-4135-7217a5dc39c8 Address:11a7278c-76d8-4f00-4135-7217a5dc39c8}]" -2024-05-12T20:49:55.965Z [INFO] server-11a7278c-76d8-4f00-4135-7217a5dc39c8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dfba48c7-24ab-4507-7673-e1a676d2ecf7 Address:dfba48c7-24ab-4507-7673-e1a676d2ecf7} {Suffrage:Voter ID:server-602866c0-8753-d539-029b-02f7109610f2 Address:602866c0-8753-d539-029b-02f7109610f2} {Suffrage:Voter ID:server-11a7278c-76d8-4f00-4135-7217a5dc39c8 Address:11a7278c-76d8-4f00-4135-7217a5dc39c8}]" -2024-05-12T20:49:55.965Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:49:55.965Z [INFO] server-dfba48c7-24ab-4507-7673-e1a676d2ecf7: entering follower state: follower="Node at dfba48c7-24ab-4507-7673-e1a676d2ecf7 [Follower]" leader-address= leader-id= -2024-05-12T20:49:55.965Z [INFO] server-602866c0-8753-d539-029b-02f7109610f2: entering follower state: follower="Node at 602866c0-8753-d539-029b-02f7109610f2 [Follower]" leader-address= leader-id= -2024-05-12T20:49:55.965Z [INFO] server-11a7278c-76d8-4f00-4135-7217a5dc39c8: entering follower state: follower="Node at 11a7278c-76d8-4f00-4135-7217a5dc39c8 [Follower]" leader-address= leader-id= -2024-05-12T20:49:56.050Z [WARN] server-602866c0-8753-d539-029b-02f7109610f2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:56.050Z [INFO] server-602866c0-8753-d539-029b-02f7109610f2: entering candidate state: node="Node at 602866c0-8753-d539-029b-02f7109610f2 [Candidate]" term=2 -2024-05-12T20:49:56.051Z [INFO] server-602866c0-8753-d539-029b-02f7109610f2: election won: term=2 tally=2 -2024-05-12T20:49:56.051Z [INFO] server-602866c0-8753-d539-029b-02f7109610f2: entering leader state: leader="Node at 602866c0-8753-d539-029b-02f7109610f2 [Leader]" -2024-05-12T20:49:56.051Z [INFO] server-602866c0-8753-d539-029b-02f7109610f2: added peer, starting replication: peer=server-dfba48c7-24ab-4507-7673-e1a676d2ecf7 -2024-05-12T20:49:56.051Z [INFO] server-602866c0-8753-d539-029b-02f7109610f2: added peer, starting replication: peer=server-11a7278c-76d8-4f00-4135-7217a5dc39c8 -2024-05-12T20:49:56.051Z [INFO] server-602866c0-8753-d539-029b-02f7109610f2: pipelining replication: peer="{Voter server-11a7278c-76d8-4f00-4135-7217a5dc39c8 11a7278c-76d8-4f00-4135-7217a5dc39c8}" -2024-05-12T20:49:56.051Z [WARN] server-dfba48c7-24ab-4507-7673-e1a676d2ecf7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:56.051Z [INFO] server-dfba48c7-24ab-4507-7673-e1a676d2ecf7: entering candidate state: node="Node at dfba48c7-24ab-4507-7673-e1a676d2ecf7 [Candidate]" term=2 -2024-05-12T20:49:56.051Z [WARN] server-11a7278c-76d8-4f00-4135-7217a5dc39c8: rejecting vote request since we have a leader: from=dfba48c7-24ab-4507-7673-e1a676d2ecf7 leader=602866c0-8753-d539-029b-02f7109610f2 leader-id=server-602866c0-8753-d539-029b-02f7109610f2 -2024-05-12T20:49:56.051Z [INFO] server-dfba48c7-24ab-4507-7673-e1a676d2ecf7: duplicate requestVote for same term: term=2 -2024-05-12T20:49:56.051Z [INFO] server-dfba48c7-24ab-4507-7673-e1a676d2ecf7: entering follower state: follower="Node at dfba48c7-24ab-4507-7673-e1a676d2ecf7 [Follower]" leader-address=602866c0-8753-d539-029b-02f7109610f2 leader-id=server-602866c0-8753-d539-029b-02f7109610f2 -2024-05-12T20:49:56.051Z [INFO] server-602866c0-8753-d539-029b-02f7109610f2: pipelining replication: peer="{Voter server-dfba48c7-24ab-4507-7673-e1a676d2ecf7 dfba48c7-24ab-4507-7673-e1a676d2ecf7}" -2024-05-12T20:49:56.051Z [WARN] server-602866c0-8753-d539-029b-02f7109610f2: rejecting vote request since we have a leader: from=dfba48c7-24ab-4507-7673-e1a676d2ecf7 leader=602866c0-8753-d539-029b-02f7109610f2 leader-id=server-602866c0-8753-d539-029b-02f7109610f2 -2024-05-12T20:49:56.158Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:49:56.052053698 +0000 UTC m=+7.332754124 (2 nodes), 86.70279ms from start of poll, 87.196921ms from cluster start. Timeout at 2024-05-12 20:49:56.158784087 +0000 UTC m=+7.439484687, 106.730563ms after stability -2024-05-12T20:49:56.158Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:56.267Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:56.158979003 +0000 UTC m=+7.439679689 (1 nodes), 3.217µs from start of poll, 194.122486ms from cluster start. Timeout at 2024-05-12 20:49:56.266964287 +0000 UTC m=+7.547664875, 107.985186ms after stability -2024-05-12T20:49:56.285Z [INFO] server-602866c0-8753-d539-029b-02f7109610f2: aborting pipeline replication: peer="{Voter server-11a7278c-76d8-4f00-4135-7217a5dc39c8 11a7278c-76d8-4f00-4135-7217a5dc39c8}" -2024-05-12T20:49:56.290Z [INFO] server-602866c0-8753-d539-029b-02f7109610f2: aborting pipeline replication: peer="{Voter server-dfba48c7-24ab-4507-7673-e1a676d2ecf7 dfba48c7-24ab-4507-7673-e1a676d2ecf7}" -2024-05-12T20:49:56.794Z [ERROR] server-602866c0-8753-d539-029b-02f7109610f2: failed to heartbeat to: peer=dfba48c7-24ab-4507-7673-e1a676d2ecf7 backoff time=10ms error="command timed out" ---- PASS: TestRaft_TripleNode (0.83s) +2025-06-15T03:16:30.739Z [INFO] server-239004ab-6e68-7ab0-aa96-cd4a875d8d5d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-239004ab-6e68-7ab0-aa96-cd4a875d8d5d Address:239004ab-6e68-7ab0-aa96-cd4a875d8d5d} {Suffrage:Voter ID:server-1c6f3cae-26b1-dbac-abbd-772dadf45629 Address:1c6f3cae-26b1-dbac-abbd-772dadf45629} {Suffrage:Voter ID:server-c14884db-5d20-1395-8784-8f46a31efe28 Address:c14884db-5d20-1395-8784-8f46a31efe28}]" +2025-06-15T03:16:30.739Z [INFO] server-239004ab-6e68-7ab0-aa96-cd4a875d8d5d: entering follower state: follower="Node at 239004ab-6e68-7ab0-aa96-cd4a875d8d5d [Follower]" leader-address= leader-id= +2025-06-15T03:16:30.739Z [INFO] server-1c6f3cae-26b1-dbac-abbd-772dadf45629: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-239004ab-6e68-7ab0-aa96-cd4a875d8d5d Address:239004ab-6e68-7ab0-aa96-cd4a875d8d5d} {Suffrage:Voter ID:server-1c6f3cae-26b1-dbac-abbd-772dadf45629 Address:1c6f3cae-26b1-dbac-abbd-772dadf45629} {Suffrage:Voter ID:server-c14884db-5d20-1395-8784-8f46a31efe28 Address:c14884db-5d20-1395-8784-8f46a31efe28}]" +2025-06-15T03:16:30.739Z [INFO] server-1c6f3cae-26b1-dbac-abbd-772dadf45629: entering follower state: follower="Node at 1c6f3cae-26b1-dbac-abbd-772dadf45629 [Follower]" leader-address= leader-id= +2025-06-15T03:16:30.739Z [INFO] server-c14884db-5d20-1395-8784-8f46a31efe28: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-239004ab-6e68-7ab0-aa96-cd4a875d8d5d Address:239004ab-6e68-7ab0-aa96-cd4a875d8d5d} {Suffrage:Voter ID:server-1c6f3cae-26b1-dbac-abbd-772dadf45629 Address:1c6f3cae-26b1-dbac-abbd-772dadf45629} {Suffrage:Voter ID:server-c14884db-5d20-1395-8784-8f46a31efe28 Address:c14884db-5d20-1395-8784-8f46a31efe28}]" +2025-06-15T03:16:30.739Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:16:30.739Z [INFO] server-c14884db-5d20-1395-8784-8f46a31efe28: entering follower state: follower="Node at c14884db-5d20-1395-8784-8f46a31efe28 [Follower]" leader-address= leader-id= +2025-06-15T03:16:30.796Z [WARN] server-239004ab-6e68-7ab0-aa96-cd4a875d8d5d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:30.796Z [INFO] server-239004ab-6e68-7ab0-aa96-cd4a875d8d5d: entering candidate state: node="Node at 239004ab-6e68-7ab0-aa96-cd4a875d8d5d [Candidate]" term=2 +2025-06-15T03:16:30.796Z [INFO] server-239004ab-6e68-7ab0-aa96-cd4a875d8d5d: election won: term=2 tally=2 +2025-06-15T03:16:30.796Z [INFO] server-239004ab-6e68-7ab0-aa96-cd4a875d8d5d: entering leader state: leader="Node at 239004ab-6e68-7ab0-aa96-cd4a875d8d5d [Leader]" +2025-06-15T03:16:30.796Z [INFO] server-239004ab-6e68-7ab0-aa96-cd4a875d8d5d: added peer, starting replication: peer=server-1c6f3cae-26b1-dbac-abbd-772dadf45629 +2025-06-15T03:16:30.796Z [INFO] server-239004ab-6e68-7ab0-aa96-cd4a875d8d5d: added peer, starting replication: peer=server-c14884db-5d20-1395-8784-8f46a31efe28 +2025-06-15T03:16:30.797Z [INFO] server-239004ab-6e68-7ab0-aa96-cd4a875d8d5d: pipelining replication: peer="{Voter server-c14884db-5d20-1395-8784-8f46a31efe28 c14884db-5d20-1395-8784-8f46a31efe28}" +2025-06-15T03:16:30.797Z [INFO] server-239004ab-6e68-7ab0-aa96-cd4a875d8d5d: pipelining replication: peer="{Voter server-1c6f3cae-26b1-dbac-abbd-772dadf45629 1c6f3cae-26b1-dbac-abbd-772dadf45629}" +2025-06-15T03:16:30.902Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:16:30.796938931 +0000 UTC m=+6.508617116 (2 nodes), 57.347508ms from start of poll, 57.744351ms from cluster start. Timeout at 2025-06-15 03:16:30.902526906 +0000 UTC m=+6.614205068, 105.587952ms after stability +2025-06-15T03:16:30.902Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:31.007Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:30.90257853 +0000 UTC m=+6.614256696 (1 nodes), 1.187µs from start of poll, 163.383931ms from cluster start. Timeout at 2025-06-15 03:16:31.007626102 +0000 UTC m=+6.719304267, 105.047571ms after stability +2025-06-15T03:16:31.018Z [INFO] server-239004ab-6e68-7ab0-aa96-cd4a875d8d5d: aborting pipeline replication: peer="{Voter server-c14884db-5d20-1395-8784-8f46a31efe28 c14884db-5d20-1395-8784-8f46a31efe28}" +2025-06-15T03:16:31.018Z [INFO] server-239004ab-6e68-7ab0-aa96-cd4a875d8d5d: aborting pipeline replication: peer="{Voter server-1c6f3cae-26b1-dbac-abbd-772dadf45629 1c6f3cae-26b1-dbac-abbd-772dadf45629}" +--- PASS: TestRaft_TripleNode (0.30s) === RUN TestRaft_LeaderFail -2024-05-12T20:49:56.796Z [INFO] server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a Address:9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a} {Suffrage:Voter ID:server-383659af-df91-c9da-3ecd-0828e07f5adb Address:383659af-df91-c9da-3ecd-0828e07f5adb} {Suffrage:Voter ID:server-54fb8d4a-27ef-288f-25d3-827b4ca2f334 Address:54fb8d4a-27ef-288f-25d3-827b4ca2f334}]" -2024-05-12T20:49:56.796Z [INFO] server-383659af-df91-c9da-3ecd-0828e07f5adb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a Address:9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a} {Suffrage:Voter ID:server-383659af-df91-c9da-3ecd-0828e07f5adb Address:383659af-df91-c9da-3ecd-0828e07f5adb} {Suffrage:Voter ID:server-54fb8d4a-27ef-288f-25d3-827b4ca2f334 Address:54fb8d4a-27ef-288f-25d3-827b4ca2f334}]" -2024-05-12T20:49:56.797Z [INFO] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a Address:9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a} {Suffrage:Voter ID:server-383659af-df91-c9da-3ecd-0828e07f5adb Address:383659af-df91-c9da-3ecd-0828e07f5adb} {Suffrage:Voter ID:server-54fb8d4a-27ef-288f-25d3-827b4ca2f334 Address:54fb8d4a-27ef-288f-25d3-827b4ca2f334}]" -2024-05-12T20:49:56.797Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:49:56.797Z [INFO] server-383659af-df91-c9da-3ecd-0828e07f5adb: entering follower state: follower="Node at 383659af-df91-c9da-3ecd-0828e07f5adb [Follower]" leader-address= leader-id= -2024-05-12T20:49:56.797Z [INFO] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: entering follower state: follower="Node at 54fb8d4a-27ef-288f-25d3-827b4ca2f334 [Follower]" leader-address= leader-id= -2024-05-12T20:49:56.797Z [INFO] server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a: entering follower state: follower="Node at 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a [Follower]" leader-address= leader-id= -2024-05-12T20:49:56.862Z [WARN] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:56.862Z [INFO] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: entering candidate state: node="Node at 54fb8d4a-27ef-288f-25d3-827b4ca2f334 [Candidate]" term=2 -2024-05-12T20:49:56.862Z [INFO] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: election won: term=2 tally=2 -2024-05-12T20:49:56.862Z [INFO] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: entering leader state: leader="Node at 54fb8d4a-27ef-288f-25d3-827b4ca2f334 [Leader]" -2024-05-12T20:49:56.862Z [INFO] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: added peer, starting replication: peer=server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a -2024-05-12T20:49:56.863Z [INFO] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: added peer, starting replication: peer=server-383659af-df91-c9da-3ecd-0828e07f5adb -2024-05-12T20:49:56.863Z [INFO] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: pipelining replication: peer="{Voter server-383659af-df91-c9da-3ecd-0828e07f5adb 383659af-df91-c9da-3ecd-0828e07f5adb}" -2024-05-12T20:49:56.863Z [INFO] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: pipelining replication: peer="{Voter server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a}" -2024-05-12T20:49:56.968Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:49:56.863427222 +0000 UTC m=+8.144127561 (2 nodes), 66.298127ms from start of poll, 66.735874ms from cluster start. Timeout at 2024-05-12 20:49:56.968443887 +0000 UTC m=+8.249144253, 105.016692ms after stability -2024-05-12T20:49:56.968Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:57.075Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:56.968564665 +0000 UTC m=+8.249265063 (1 nodes), 3.966µs from start of poll, 171.873376ms from cluster start. Timeout at 2024-05-12 20:49:57.075259095 +0000 UTC m=+8.355959438, 106.694375ms after stability - raft_test.go:378: [INFO] Disconnecting Node at 54fb8d4a-27ef-288f-25d3-827b4ca2f334 [Leader] -2024-05-12T20:49:57.099Z [ERROR] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to pipeline appendEntries: peer="{Voter server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a}" error="append pipeline closed" -2024-05-12T20:49:57.099Z [INFO] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: aborting pipeline replication: peer="{Voter server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a}" -2024-05-12T20:49:57.100Z [ERROR] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to heartbeat to: peer=9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a backoff time=10ms error="failed to connect to peer: 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a" -2024-05-12T20:49:57.100Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:57.100Z [ERROR] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to pipeline appendEntries: peer="{Voter server-383659af-df91-c9da-3ecd-0828e07f5adb 383659af-df91-c9da-3ecd-0828e07f5adb}" error="append pipeline closed" -2024-05-12T20:49:57.100Z [INFO] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: aborting pipeline replication: peer="{Voter server-383659af-df91-c9da-3ecd-0828e07f5adb 383659af-df91-c9da-3ecd-0828e07f5adb}" -2024-05-12T20:49:57.110Z [ERROR] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to appendEntries to: peer="{Voter server-383659af-df91-c9da-3ecd-0828e07f5adb 383659af-df91-c9da-3ecd-0828e07f5adb}" error="failed to connect to peer: 383659af-df91-c9da-3ecd-0828e07f5adb" -2024-05-12T20:49:57.111Z [ERROR] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to appendEntries to: peer="{Voter server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a}" error="failed to connect to peer: 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a" -2024-05-12T20:49:57.111Z [ERROR] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to heartbeat to: peer=383659af-df91-c9da-3ecd-0828e07f5adb backoff time=10ms error="failed to connect to peer: 383659af-df91-c9da-3ecd-0828e07f5adb" -2024-05-12T20:49:57.122Z [ERROR] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to heartbeat to: peer=9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a backoff time=10ms error="failed to connect to peer: 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a" -2024-05-12T20:49:57.130Z [ERROR] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to appendEntries to: peer="{Voter server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a}" error="failed to connect to peer: 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a" -2024-05-12T20:49:57.134Z [ERROR] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to heartbeat to: peer=383659af-df91-c9da-3ecd-0828e07f5adb backoff time=10ms error="failed to connect to peer: 383659af-df91-c9da-3ecd-0828e07f5adb" -2024-05-12T20:49:57.135Z [ERROR] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to appendEntries to: peer="{Voter server-383659af-df91-c9da-3ecd-0828e07f5adb 383659af-df91-c9da-3ecd-0828e07f5adb}" error="failed to connect to peer: 383659af-df91-c9da-3ecd-0828e07f5adb" -2024-05-12T20:49:57.142Z [WARN] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to contact: server-id=server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a time=51.738707ms -2024-05-12T20:49:57.147Z [ERROR] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to heartbeat to: peer=9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a backoff time=10ms error="failed to connect to peer: 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a" -2024-05-12T20:49:57.154Z [ERROR] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to appendEntries to: peer="{Voter server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a}" error="failed to connect to peer: 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a" -2024-05-12T20:49:57.154Z [WARN] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to contact: server-id=server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a time=63.803886ms -2024-05-12T20:49:57.154Z [WARN] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to contact: server-id=server-383659af-df91-c9da-3ecd-0828e07f5adb time=57.204969ms -2024-05-12T20:49:57.154Z [WARN] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to contact quorum of nodes, stepping down -2024-05-12T20:49:57.154Z [INFO] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: entering follower state: follower="Node at 54fb8d4a-27ef-288f-25d3-827b4ca2f334 [Follower]" leader-address= leader-id= -2024-05-12T20:49:57.158Z [ERROR] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to heartbeat to: peer=383659af-df91-c9da-3ecd-0828e07f5adb backoff time=10ms error="failed to connect to peer: 383659af-df91-c9da-3ecd-0828e07f5adb" -2024-05-12T20:49:57.159Z [ERROR] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to appendEntries to: peer="{Voter server-383659af-df91-c9da-3ecd-0828e07f5adb 383659af-df91-c9da-3ecd-0828e07f5adb}" error="failed to connect to peer: 383659af-df91-c9da-3ecd-0828e07f5adb" -2024-05-12T20:49:57.174Z [WARN] server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a: heartbeat timeout reached, starting election: last-leader-addr=54fb8d4a-27ef-288f-25d3-827b4ca2f334 last-leader-id=server-54fb8d4a-27ef-288f-25d3-827b4ca2f334 -2024-05-12T20:49:57.174Z [INFO] server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a: entering candidate state: node="Node at 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a [Candidate]" term=3 -2024-05-12T20:49:57.174Z [ERROR] server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a: failed to make requestVote RPC: target="{Voter server-54fb8d4a-27ef-288f-25d3-827b4ca2f334 54fb8d4a-27ef-288f-25d3-827b4ca2f334}" error="failed to connect to peer: 54fb8d4a-27ef-288f-25d3-827b4ca2f334" term=3 -2024-05-12T20:49:57.175Z [WARN] server-383659af-df91-c9da-3ecd-0828e07f5adb: rejecting vote request since we have a leader: from=9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a leader=54fb8d4a-27ef-288f-25d3-827b4ca2f334 leader-id=server-54fb8d4a-27ef-288f-25d3-827b4ca2f334 -2024-05-12T20:49:57.194Z [WARN] server-383659af-df91-c9da-3ecd-0828e07f5adb: heartbeat timeout reached, starting election: last-leader-addr=54fb8d4a-27ef-288f-25d3-827b4ca2f334 last-leader-id=server-54fb8d4a-27ef-288f-25d3-827b4ca2f334 -2024-05-12T20:49:57.195Z [INFO] server-383659af-df91-c9da-3ecd-0828e07f5adb: entering candidate state: node="Node at 383659af-df91-c9da-3ecd-0828e07f5adb [Candidate]" term=3 -2024-05-12T20:49:57.195Z [ERROR] server-383659af-df91-c9da-3ecd-0828e07f5adb: failed to make requestVote RPC: target="{Voter server-54fb8d4a-27ef-288f-25d3-827b4ca2f334 54fb8d4a-27ef-288f-25d3-827b4ca2f334}" error="failed to connect to peer: 54fb8d4a-27ef-288f-25d3-827b4ca2f334" term=3 -2024-05-12T20:49:57.195Z [INFO] server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a: duplicate requestVote for same term: term=3 -2024-05-12T20:49:57.231Z [WARN] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:57.231Z [INFO] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: entering candidate state: node="Node at 54fb8d4a-27ef-288f-25d3-827b4ca2f334 [Candidate]" term=3 -2024-05-12T20:49:57.231Z [ERROR] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to make requestVote RPC: target="{Voter server-383659af-df91-c9da-3ecd-0828e07f5adb 383659af-df91-c9da-3ecd-0828e07f5adb}" error="failed to connect to peer: 383659af-df91-c9da-3ecd-0828e07f5adb" term=3 -2024-05-12T20:49:57.231Z [ERROR] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to make requestVote RPC: target="{Voter server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a}" error="failed to connect to peer: 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a" term=3 -2024-05-12T20:49:57.254Z [WARN] server-383659af-df91-c9da-3ecd-0828e07f5adb: Election timeout reached, restarting election -2024-05-12T20:49:57.255Z [INFO] server-383659af-df91-c9da-3ecd-0828e07f5adb: entering candidate state: node="Node at 383659af-df91-c9da-3ecd-0828e07f5adb [Candidate]" term=4 -2024-05-12T20:49:57.255Z [ERROR] server-383659af-df91-c9da-3ecd-0828e07f5adb: failed to make requestVote RPC: target="{Voter server-54fb8d4a-27ef-288f-25d3-827b4ca2f334 54fb8d4a-27ef-288f-25d3-827b4ca2f334}" error="failed to connect to peer: 54fb8d4a-27ef-288f-25d3-827b4ca2f334" term=4 -2024-05-12T20:49:57.255Z [INFO] server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a: entering follower state: follower="Node at 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a [Follower]" leader-address= leader-id= -2024-05-12T20:49:57.255Z [INFO] server-383659af-df91-c9da-3ecd-0828e07f5adb: election won: term=4 tally=2 -2024-05-12T20:49:57.255Z [INFO] server-383659af-df91-c9da-3ecd-0828e07f5adb: entering leader state: leader="Node at 383659af-df91-c9da-3ecd-0828e07f5adb [Leader]" -2024-05-12T20:49:57.255Z [INFO] server-383659af-df91-c9da-3ecd-0828e07f5adb: added peer, starting replication: peer=server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a -2024-05-12T20:49:57.255Z [INFO] server-383659af-df91-c9da-3ecd-0828e07f5adb: added peer, starting replication: peer=server-54fb8d4a-27ef-288f-25d3-827b4ca2f334 -2024-05-12T20:49:57.255Z [ERROR] server-383659af-df91-c9da-3ecd-0828e07f5adb: failed to appendEntries to: peer="{Voter server-54fb8d4a-27ef-288f-25d3-827b4ca2f334 54fb8d4a-27ef-288f-25d3-827b4ca2f334}" error="failed to connect to peer: 54fb8d4a-27ef-288f-25d3-827b4ca2f334" -2024-05-12T20:49:57.255Z [INFO] server-383659af-df91-c9da-3ecd-0828e07f5adb: pipelining replication: peer="{Voter server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a}" -2024-05-12T20:49:57.266Z [ERROR] server-383659af-df91-c9da-3ecd-0828e07f5adb: failed to heartbeat to: peer=54fb8d4a-27ef-288f-25d3-827b4ca2f334 backoff time=10ms error="failed to connect to peer: 54fb8d4a-27ef-288f-25d3-827b4ca2f334" -2024-05-12T20:49:57.278Z [ERROR] server-383659af-df91-c9da-3ecd-0828e07f5adb: failed to appendEntries to: peer="{Voter server-54fb8d4a-27ef-288f-25d3-827b4ca2f334 54fb8d4a-27ef-288f-25d3-827b4ca2f334}" error="failed to connect to peer: 54fb8d4a-27ef-288f-25d3-827b4ca2f334" -2024-05-12T20:49:57.286Z [ERROR] server-383659af-df91-c9da-3ecd-0828e07f5adb: failed to heartbeat to: peer=54fb8d4a-27ef-288f-25d3-827b4ca2f334 backoff time=10ms error="failed to connect to peer: 54fb8d4a-27ef-288f-25d3-827b4ca2f334" -2024-05-12T20:49:57.306Z [WARN] server-383659af-df91-c9da-3ecd-0828e07f5adb: failed to contact: server-id=server-54fb8d4a-27ef-288f-25d3-827b4ca2f334 time=51.469412ms -2024-05-12T20:49:57.306Z [ERROR] server-383659af-df91-c9da-3ecd-0828e07f5adb: failed to appendEntries to: peer="{Voter server-54fb8d4a-27ef-288f-25d3-827b4ca2f334 54fb8d4a-27ef-288f-25d3-827b4ca2f334}" error="failed to connect to peer: 54fb8d4a-27ef-288f-25d3-827b4ca2f334" -2024-05-12T20:49:57.310Z [WARN] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: Election timeout reached, restarting election -2024-05-12T20:49:57.310Z [INFO] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: entering candidate state: node="Node at 54fb8d4a-27ef-288f-25d3-827b4ca2f334 [Candidate]" term=4 -2024-05-12T20:49:57.310Z [ERROR] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to make requestVote RPC: target="{Voter server-383659af-df91-c9da-3ecd-0828e07f5adb 383659af-df91-c9da-3ecd-0828e07f5adb}" error="failed to connect to peer: 383659af-df91-c9da-3ecd-0828e07f5adb" term=4 -2024-05-12T20:49:57.310Z [ERROR] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: failed to make requestVote RPC: target="{Voter server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a}" error="failed to connect to peer: 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a" term=4 -2024-05-12T20:49:57.314Z [ERROR] server-383659af-df91-c9da-3ecd-0828e07f5adb: failed to heartbeat to: peer=54fb8d4a-27ef-288f-25d3-827b4ca2f334 backoff time=10ms error="failed to connect to peer: 54fb8d4a-27ef-288f-25d3-827b4ca2f334" -2024-05-12T20:49:57.334Z [ERROR] server-383659af-df91-c9da-3ecd-0828e07f5adb: failed to heartbeat to: peer=54fb8d4a-27ef-288f-25d3-827b4ca2f334 backoff time=20ms error="failed to connect to peer: 54fb8d4a-27ef-288f-25d3-827b4ca2f334" -2024-05-12T20:49:57.338Z [ERROR] server-383659af-df91-c9da-3ecd-0828e07f5adb: failed to appendEntries to: peer="{Voter server-54fb8d4a-27ef-288f-25d3-827b4ca2f334 54fb8d4a-27ef-288f-25d3-827b4ca2f334}" error="failed to connect to peer: 54fb8d4a-27ef-288f-25d3-827b4ca2f334" -2024-05-12T20:49:57.355Z [WARN] server-383659af-df91-c9da-3ecd-0828e07f5adb: failed to contact: server-id=server-54fb8d4a-27ef-288f-25d3-827b4ca2f334 time=99.717043ms -2024-05-12T20:49:57.362Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:57.255550116 +0000 UTC m=+8.536250467 (1 nodes), 155.073164ms from start of poll, 458.85878ms from cluster start. Timeout at 2024-05-12 20:49:57.362766304 +0000 UTC m=+8.643466771, 107.216304ms after stability - raft_test.go:413: [INFO] Reconnecting Node at 54fb8d4a-27ef-288f-25d3-827b4ca2f334 [Candidate] -2024-05-12T20:49:57.366Z [INFO] server-54fb8d4a-27ef-288f-25d3-827b4ca2f334: entering follower state: follower="Node at 54fb8d4a-27ef-288f-25d3-827b4ca2f334 [Follower]" leader-address=383659af-df91-c9da-3ecd-0828e07f5adb leader-id=server-383659af-df91-c9da-3ecd-0828e07f5adb -2024-05-12T20:49:57.402Z [INFO] server-383659af-df91-c9da-3ecd-0828e07f5adb: pipelining replication: peer="{Voter server-54fb8d4a-27ef-288f-25d3-827b4ca2f334 54fb8d4a-27ef-288f-25d3-827b4ca2f334}" -2024-05-12T20:49:57.411Z [INFO] server-383659af-df91-c9da-3ecd-0828e07f5adb: aborting pipeline replication: peer="{Voter server-9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a 9b46dc41-7f3d-fe9a-cfcd-b4539ee8919a}" -2024-05-12T20:49:57.411Z [INFO] server-383659af-df91-c9da-3ecd-0828e07f5adb: aborting pipeline replication: peer="{Voter server-54fb8d4a-27ef-288f-25d3-827b4ca2f334 54fb8d4a-27ef-288f-25d3-827b4ca2f334}" ---- PASS: TestRaft_LeaderFail (0.62s) +2025-06-15T03:16:31.040Z [INFO] server-87dc065a-6c09-b060-5f0d-b0a93123e0ec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-87dc065a-6c09-b060-5f0d-b0a93123e0ec Address:87dc065a-6c09-b060-5f0d-b0a93123e0ec} {Suffrage:Voter ID:server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 Address:38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2} {Suffrage:Voter ID:server-007f8103-fb5f-d7d7-941c-25eb10389268 Address:007f8103-fb5f-d7d7-941c-25eb10389268}]" +2025-06-15T03:16:31.040Z [INFO] server-87dc065a-6c09-b060-5f0d-b0a93123e0ec: entering follower state: follower="Node at 87dc065a-6c09-b060-5f0d-b0a93123e0ec [Follower]" leader-address= leader-id= +2025-06-15T03:16:31.040Z [INFO] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-87dc065a-6c09-b060-5f0d-b0a93123e0ec Address:87dc065a-6c09-b060-5f0d-b0a93123e0ec} {Suffrage:Voter ID:server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 Address:38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2} {Suffrage:Voter ID:server-007f8103-fb5f-d7d7-941c-25eb10389268 Address:007f8103-fb5f-d7d7-941c-25eb10389268}]" +2025-06-15T03:16:31.041Z [INFO] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: entering follower state: follower="Node at 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 [Follower]" leader-address= leader-id= +2025-06-15T03:16:31.041Z [INFO] server-007f8103-fb5f-d7d7-941c-25eb10389268: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-87dc065a-6c09-b060-5f0d-b0a93123e0ec Address:87dc065a-6c09-b060-5f0d-b0a93123e0ec} {Suffrage:Voter ID:server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 Address:38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2} {Suffrage:Voter ID:server-007f8103-fb5f-d7d7-941c-25eb10389268 Address:007f8103-fb5f-d7d7-941c-25eb10389268}]" +2025-06-15T03:16:31.041Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:16:31.041Z [INFO] server-007f8103-fb5f-d7d7-941c-25eb10389268: entering follower state: follower="Node at 007f8103-fb5f-d7d7-941c-25eb10389268 [Follower]" leader-address= leader-id= +2025-06-15T03:16:31.106Z [WARN] server-007f8103-fb5f-d7d7-941c-25eb10389268: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:31.106Z [INFO] server-007f8103-fb5f-d7d7-941c-25eb10389268: entering candidate state: node="Node at 007f8103-fb5f-d7d7-941c-25eb10389268 [Candidate]" term=2 +2025-06-15T03:16:31.106Z [INFO] server-007f8103-fb5f-d7d7-941c-25eb10389268: election won: term=2 tally=2 +2025-06-15T03:16:31.106Z [INFO] server-007f8103-fb5f-d7d7-941c-25eb10389268: entering leader state: leader="Node at 007f8103-fb5f-d7d7-941c-25eb10389268 [Leader]" +2025-06-15T03:16:31.106Z [INFO] server-007f8103-fb5f-d7d7-941c-25eb10389268: added peer, starting replication: peer=server-87dc065a-6c09-b060-5f0d-b0a93123e0ec +2025-06-15T03:16:31.106Z [INFO] server-007f8103-fb5f-d7d7-941c-25eb10389268: added peer, starting replication: peer=server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 +2025-06-15T03:16:31.106Z [INFO] server-007f8103-fb5f-d7d7-941c-25eb10389268: pipelining replication: peer="{Voter server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2}" +2025-06-15T03:16:31.106Z [INFO] server-007f8103-fb5f-d7d7-941c-25eb10389268: pipelining replication: peer="{Voter server-87dc065a-6c09-b060-5f0d-b0a93123e0ec 87dc065a-6c09-b060-5f0d-b0a93123e0ec}" +2025-06-15T03:16:31.211Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:16:31.106241755 +0000 UTC m=+6.817919931 (2 nodes), 65.109106ms from start of poll, 65.507207ms from cluster start. Timeout at 2025-06-15 03:16:31.211372976 +0000 UTC m=+6.923051146, 105.131215ms after stability +2025-06-15T03:16:31.211Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:31.316Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:31.211436473 +0000 UTC m=+6.923114639 (1 nodes), 650ns from start of poll, 170.701915ms from cluster start. Timeout at 2025-06-15 03:16:31.316694037 +0000 UTC m=+7.028372199, 105.25756ms after stability + raft_test.go:378: [INFO] Disconnecting Node at 007f8103-fb5f-d7d7-941c-25eb10389268 [Leader] +2025-06-15T03:16:31.330Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to pipeline appendEntries: peer="{Voter server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2}" error="append pipeline closed" +2025-06-15T03:16:31.330Z [INFO] server-007f8103-fb5f-d7d7-941c-25eb10389268: aborting pipeline replication: peer="{Voter server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2}" +2025-06-15T03:16:31.330Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to heartbeat to: peer=87dc065a-6c09-b060-5f0d-b0a93123e0ec backoff time=10ms error="failed to connect to peer: 87dc065a-6c09-b060-5f0d-b0a93123e0ec" +2025-06-15T03:16:31.330Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:31.332Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to heartbeat to: peer=38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 backoff time=10ms error="failed to connect to peer: 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2" +2025-06-15T03:16:31.334Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to pipeline appendEntries: peer="{Voter server-87dc065a-6c09-b060-5f0d-b0a93123e0ec 87dc065a-6c09-b060-5f0d-b0a93123e0ec}" error="append pipeline closed" +2025-06-15T03:16:31.334Z [INFO] server-007f8103-fb5f-d7d7-941c-25eb10389268: aborting pipeline replication: peer="{Voter server-87dc065a-6c09-b060-5f0d-b0a93123e0ec 87dc065a-6c09-b060-5f0d-b0a93123e0ec}" +2025-06-15T03:16:31.340Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to appendEntries to: peer="{Voter server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2}" error="failed to connect to peer: 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2" +2025-06-15T03:16:31.341Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to appendEntries to: peer="{Voter server-87dc065a-6c09-b060-5f0d-b0a93123e0ec 87dc065a-6c09-b060-5f0d-b0a93123e0ec}" error="failed to connect to peer: 87dc065a-6c09-b060-5f0d-b0a93123e0ec" +2025-06-15T03:16:31.349Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to heartbeat to: peer=87dc065a-6c09-b060-5f0d-b0a93123e0ec backoff time=10ms error="failed to connect to peer: 87dc065a-6c09-b060-5f0d-b0a93123e0ec" +2025-06-15T03:16:31.352Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to heartbeat to: peer=38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 backoff time=10ms error="failed to connect to peer: 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2" +2025-06-15T03:16:31.359Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to appendEntries to: peer="{Voter server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2}" error="failed to connect to peer: 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2" +2025-06-15T03:16:31.359Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to appendEntries to: peer="{Voter server-87dc065a-6c09-b060-5f0d-b0a93123e0ec 87dc065a-6c09-b060-5f0d-b0a93123e0ec}" error="failed to connect to peer: 87dc065a-6c09-b060-5f0d-b0a93123e0ec" +2025-06-15T03:16:31.369Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to heartbeat to: peer=87dc065a-6c09-b060-5f0d-b0a93123e0ec backoff time=10ms error="failed to connect to peer: 87dc065a-6c09-b060-5f0d-b0a93123e0ec" +2025-06-15T03:16:31.369Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to heartbeat to: peer=38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 backoff time=10ms error="failed to connect to peer: 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2" +2025-06-15T03:16:31.375Z [WARN] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to contact: server-id=server-87dc065a-6c09-b060-5f0d-b0a93123e0ec time=50.207607ms +2025-06-15T03:16:31.376Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to appendEntries to: peer="{Voter server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2}" error="failed to connect to peer: 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2" +2025-06-15T03:16:31.378Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to appendEntries to: peer="{Voter server-87dc065a-6c09-b060-5f0d-b0a93123e0ec 87dc065a-6c09-b060-5f0d-b0a93123e0ec}" error="failed to connect to peer: 87dc065a-6c09-b060-5f0d-b0a93123e0ec" +2025-06-15T03:16:31.386Z [WARN] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to contact: server-id=server-87dc065a-6c09-b060-5f0d-b0a93123e0ec time=60.848698ms +2025-06-15T03:16:31.386Z [WARN] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to contact: server-id=server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 time=58.733045ms +2025-06-15T03:16:31.386Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to heartbeat to: peer=87dc065a-6c09-b060-5f0d-b0a93123e0ec backoff time=20ms error="failed to connect to peer: 87dc065a-6c09-b060-5f0d-b0a93123e0ec" +2025-06-15T03:16:31.386Z [WARN] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to contact quorum of nodes, stepping down +2025-06-15T03:16:31.386Z [INFO] server-007f8103-fb5f-d7d7-941c-25eb10389268: entering follower state: follower="Node at 007f8103-fb5f-d7d7-941c-25eb10389268 [Follower]" leader-address= leader-id= +2025-06-15T03:16:31.389Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to heartbeat to: peer=38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 backoff time=20ms error="failed to connect to peer: 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2" +2025-06-15T03:16:31.406Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to appendEntries to: peer="{Voter server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2}" error="failed to connect to peer: 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2" +2025-06-15T03:16:31.407Z [WARN] server-87dc065a-6c09-b060-5f0d-b0a93123e0ec: heartbeat timeout reached, starting election: last-leader-addr=007f8103-fb5f-d7d7-941c-25eb10389268 last-leader-id=server-007f8103-fb5f-d7d7-941c-25eb10389268 +2025-06-15T03:16:31.407Z [INFO] server-87dc065a-6c09-b060-5f0d-b0a93123e0ec: entering candidate state: node="Node at 87dc065a-6c09-b060-5f0d-b0a93123e0ec [Candidate]" term=3 +2025-06-15T03:16:31.407Z [ERROR] server-87dc065a-6c09-b060-5f0d-b0a93123e0ec: failed to make requestVote RPC: target="{Voter server-007f8103-fb5f-d7d7-941c-25eb10389268 007f8103-fb5f-d7d7-941c-25eb10389268}" error="failed to connect to peer: 007f8103-fb5f-d7d7-941c-25eb10389268" term=3 +2025-06-15T03:16:31.407Z [WARN] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: rejecting vote request since we have a leader: from=87dc065a-6c09-b060-5f0d-b0a93123e0ec leader=007f8103-fb5f-d7d7-941c-25eb10389268 leader-id=server-007f8103-fb5f-d7d7-941c-25eb10389268 +2025-06-15T03:16:31.419Z [WARN] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: heartbeat timeout reached, starting election: last-leader-addr=007f8103-fb5f-d7d7-941c-25eb10389268 last-leader-id=server-007f8103-fb5f-d7d7-941c-25eb10389268 +2025-06-15T03:16:31.419Z [INFO] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: entering candidate state: node="Node at 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 [Candidate]" term=3 +2025-06-15T03:16:31.419Z [ERROR] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: failed to make requestVote RPC: target="{Voter server-007f8103-fb5f-d7d7-941c-25eb10389268 007f8103-fb5f-d7d7-941c-25eb10389268}" error="failed to connect to peer: 007f8103-fb5f-d7d7-941c-25eb10389268" term=3 +2025-06-15T03:16:31.419Z [INFO] server-87dc065a-6c09-b060-5f0d-b0a93123e0ec: duplicate requestVote for same term: term=3 +2025-06-15T03:16:31.467Z [WARN] server-007f8103-fb5f-d7d7-941c-25eb10389268: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:31.467Z [INFO] server-007f8103-fb5f-d7d7-941c-25eb10389268: entering candidate state: node="Node at 007f8103-fb5f-d7d7-941c-25eb10389268 [Candidate]" term=3 +2025-06-15T03:16:31.467Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to make requestVote RPC: target="{Voter server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2}" error="failed to connect to peer: 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2" term=3 +2025-06-15T03:16:31.467Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to make requestVote RPC: target="{Voter server-87dc065a-6c09-b060-5f0d-b0a93123e0ec 87dc065a-6c09-b060-5f0d-b0a93123e0ec}" error="failed to connect to peer: 87dc065a-6c09-b060-5f0d-b0a93123e0ec" term=3 +2025-06-15T03:16:31.480Z [WARN] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: Election timeout reached, restarting election +2025-06-15T03:16:31.480Z [INFO] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: entering candidate state: node="Node at 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 [Candidate]" term=4 +2025-06-15T03:16:31.481Z [ERROR] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: failed to make requestVote RPC: target="{Voter server-007f8103-fb5f-d7d7-941c-25eb10389268 007f8103-fb5f-d7d7-941c-25eb10389268}" error="failed to connect to peer: 007f8103-fb5f-d7d7-941c-25eb10389268" term=4 +2025-06-15T03:16:31.481Z [INFO] server-87dc065a-6c09-b060-5f0d-b0a93123e0ec: entering follower state: follower="Node at 87dc065a-6c09-b060-5f0d-b0a93123e0ec [Follower]" leader-address= leader-id= +2025-06-15T03:16:31.481Z [INFO] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: election won: term=4 tally=2 +2025-06-15T03:16:31.481Z [INFO] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: entering leader state: leader="Node at 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 [Leader]" +2025-06-15T03:16:31.481Z [INFO] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: added peer, starting replication: peer=server-87dc065a-6c09-b060-5f0d-b0a93123e0ec +2025-06-15T03:16:31.481Z [INFO] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: added peer, starting replication: peer=server-007f8103-fb5f-d7d7-941c-25eb10389268 +2025-06-15T03:16:31.481Z [ERROR] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: failed to appendEntries to: peer="{Voter server-007f8103-fb5f-d7d7-941c-25eb10389268 007f8103-fb5f-d7d7-941c-25eb10389268}" error="failed to connect to peer: 007f8103-fb5f-d7d7-941c-25eb10389268" +2025-06-15T03:16:31.481Z [INFO] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: pipelining replication: peer="{Voter server-87dc065a-6c09-b060-5f0d-b0a93123e0ec 87dc065a-6c09-b060-5f0d-b0a93123e0ec}" +2025-06-15T03:16:31.487Z [ERROR] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: failed to heartbeat to: peer=007f8103-fb5f-d7d7-941c-25eb10389268 backoff time=10ms error="failed to connect to peer: 007f8103-fb5f-d7d7-941c-25eb10389268" +2025-06-15T03:16:31.499Z [ERROR] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: failed to appendEntries to: peer="{Voter server-007f8103-fb5f-d7d7-941c-25eb10389268 007f8103-fb5f-d7d7-941c-25eb10389268}" error="failed to connect to peer: 007f8103-fb5f-d7d7-941c-25eb10389268" +2025-06-15T03:16:31.507Z [ERROR] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: failed to heartbeat to: peer=007f8103-fb5f-d7d7-941c-25eb10389268 backoff time=10ms error="failed to connect to peer: 007f8103-fb5f-d7d7-941c-25eb10389268" +2025-06-15T03:16:31.516Z [ERROR] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: failed to appendEntries to: peer="{Voter server-007f8103-fb5f-d7d7-941c-25eb10389268 007f8103-fb5f-d7d7-941c-25eb10389268}" error="failed to connect to peer: 007f8103-fb5f-d7d7-941c-25eb10389268" +2025-06-15T03:16:31.518Z [WARN] server-007f8103-fb5f-d7d7-941c-25eb10389268: Election timeout reached, restarting election +2025-06-15T03:16:31.518Z [INFO] server-007f8103-fb5f-d7d7-941c-25eb10389268: entering candidate state: node="Node at 007f8103-fb5f-d7d7-941c-25eb10389268 [Candidate]" term=4 +2025-06-15T03:16:31.518Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to make requestVote RPC: target="{Voter server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2}" error="failed to connect to peer: 38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2" term=4 +2025-06-15T03:16:31.518Z [ERROR] server-007f8103-fb5f-d7d7-941c-25eb10389268: failed to make requestVote RPC: target="{Voter server-87dc065a-6c09-b060-5f0d-b0a93123e0ec 87dc065a-6c09-b060-5f0d-b0a93123e0ec}" error="failed to connect to peer: 87dc065a-6c09-b060-5f0d-b0a93123e0ec" term=4 +2025-06-15T03:16:31.525Z [ERROR] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: failed to heartbeat to: peer=007f8103-fb5f-d7d7-941c-25eb10389268 backoff time=10ms error="failed to connect to peer: 007f8103-fb5f-d7d7-941c-25eb10389268" +2025-06-15T03:16:31.531Z [WARN] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: failed to contact: server-id=server-007f8103-fb5f-d7d7-941c-25eb10389268 time=50.150088ms +2025-06-15T03:16:31.542Z [ERROR] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: failed to appendEntries to: peer="{Voter server-007f8103-fb5f-d7d7-941c-25eb10389268 007f8103-fb5f-d7d7-941c-25eb10389268}" error="failed to connect to peer: 007f8103-fb5f-d7d7-941c-25eb10389268" +2025-06-15T03:16:31.544Z [ERROR] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: failed to heartbeat to: peer=007f8103-fb5f-d7d7-941c-25eb10389268 backoff time=20ms error="failed to connect to peer: 007f8103-fb5f-d7d7-941c-25eb10389268" +2025-06-15T03:16:31.575Z [ERROR] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: failed to heartbeat to: peer=007f8103-fb5f-d7d7-941c-25eb10389268 backoff time=25ms error="failed to connect to peer: 007f8103-fb5f-d7d7-941c-25eb10389268" +2025-06-15T03:16:31.580Z [WARN] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: failed to contact: server-id=server-007f8103-fb5f-d7d7-941c-25eb10389268 time=99.418491ms +2025-06-15T03:16:31.586Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:31.481170276 +0000 UTC m=+7.192848452 (1 nodes), 150.27508ms from start of poll, 440.435728ms from cluster start. Timeout at 2025-06-15 03:16:31.586799224 +0000 UTC m=+7.298477386, 105.628934ms after stability + raft_test.go:413: [INFO] Reconnecting Node at 007f8103-fb5f-d7d7-941c-25eb10389268 [Candidate] +2025-06-15T03:16:31.588Z [INFO] server-007f8103-fb5f-d7d7-941c-25eb10389268: entering follower state: follower="Node at 007f8103-fb5f-d7d7-941c-25eb10389268 [Follower]" leader-address=38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 leader-id=server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2 +2025-06-15T03:16:31.589Z [INFO] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: pipelining replication: peer="{Voter server-007f8103-fb5f-d7d7-941c-25eb10389268 007f8103-fb5f-d7d7-941c-25eb10389268}" +2025-06-15T03:16:31.599Z [INFO] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: aborting pipeline replication: peer="{Voter server-007f8103-fb5f-d7d7-941c-25eb10389268 007f8103-fb5f-d7d7-941c-25eb10389268}" +2025-06-15T03:16:31.599Z [INFO] server-38ad8bdb-0765-8ea2-f10d-5b86b7b90dc2: aborting pipeline replication: peer="{Voter server-87dc065a-6c09-b060-5f0d-b0a93123e0ec 87dc065a-6c09-b060-5f0d-b0a93123e0ec}" +--- PASS: TestRaft_LeaderFail (0.56s) === RUN TestRaft_BehindFollower -2024-05-12T20:49:57.413Z [INFO] server-5a54eefb-c73d-14db-1f93-a6b622c4cebf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5a54eefb-c73d-14db-1f93-a6b622c4cebf Address:5a54eefb-c73d-14db-1f93-a6b622c4cebf} {Suffrage:Voter ID:server-536f6c2b-f3dd-061d-e5d3-f87c16afce5d Address:536f6c2b-f3dd-061d-e5d3-f87c16afce5d} {Suffrage:Voter ID:server-356c2d0a-813e-f106-5b26-c4f1880ee8f8 Address:356c2d0a-813e-f106-5b26-c4f1880ee8f8}]" -2024-05-12T20:49:57.413Z [INFO] server-536f6c2b-f3dd-061d-e5d3-f87c16afce5d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5a54eefb-c73d-14db-1f93-a6b622c4cebf Address:5a54eefb-c73d-14db-1f93-a6b622c4cebf} {Suffrage:Voter ID:server-536f6c2b-f3dd-061d-e5d3-f87c16afce5d Address:536f6c2b-f3dd-061d-e5d3-f87c16afce5d} {Suffrage:Voter ID:server-356c2d0a-813e-f106-5b26-c4f1880ee8f8 Address:356c2d0a-813e-f106-5b26-c4f1880ee8f8}]" -2024-05-12T20:49:57.413Z [INFO] server-356c2d0a-813e-f106-5b26-c4f1880ee8f8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5a54eefb-c73d-14db-1f93-a6b622c4cebf Address:5a54eefb-c73d-14db-1f93-a6b622c4cebf} {Suffrage:Voter ID:server-536f6c2b-f3dd-061d-e5d3-f87c16afce5d Address:536f6c2b-f3dd-061d-e5d3-f87c16afce5d} {Suffrage:Voter ID:server-356c2d0a-813e-f106-5b26-c4f1880ee8f8 Address:356c2d0a-813e-f106-5b26-c4f1880ee8f8}]" -2024-05-12T20:49:57.413Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:57.413Z [INFO] server-5a54eefb-c73d-14db-1f93-a6b622c4cebf: entering follower state: follower="Node at 5a54eefb-c73d-14db-1f93-a6b622c4cebf [Follower]" leader-address= leader-id= -2024-05-12T20:49:57.413Z [INFO] server-536f6c2b-f3dd-061d-e5d3-f87c16afce5d: entering follower state: follower="Node at 536f6c2b-f3dd-061d-e5d3-f87c16afce5d [Follower]" leader-address= leader-id= -2024-05-12T20:49:57.413Z [INFO] server-356c2d0a-813e-f106-5b26-c4f1880ee8f8: entering follower state: follower="Node at 356c2d0a-813e-f106-5b26-c4f1880ee8f8 [Follower]" leader-address= leader-id= -2024-05-12T20:49:57.474Z [WARN] server-5a54eefb-c73d-14db-1f93-a6b622c4cebf: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:57.474Z [INFO] server-5a54eefb-c73d-14db-1f93-a6b622c4cebf: entering candidate state: node="Node at 5a54eefb-c73d-14db-1f93-a6b622c4cebf [Candidate]" term=2 -2024-05-12T20:49:57.475Z [INFO] server-5a54eefb-c73d-14db-1f93-a6b622c4cebf: election won: term=2 tally=2 -2024-05-12T20:49:57.475Z [INFO] server-5a54eefb-c73d-14db-1f93-a6b622c4cebf: entering leader state: leader="Node at 5a54eefb-c73d-14db-1f93-a6b622c4cebf [Leader]" -2024-05-12T20:49:57.475Z [INFO] server-5a54eefb-c73d-14db-1f93-a6b622c4cebf: added peer, starting replication: peer=server-536f6c2b-f3dd-061d-e5d3-f87c16afce5d -2024-05-12T20:49:57.475Z [INFO] server-5a54eefb-c73d-14db-1f93-a6b622c4cebf: added peer, starting replication: peer=server-356c2d0a-813e-f106-5b26-c4f1880ee8f8 -2024-05-12T20:49:57.475Z [INFO] server-5a54eefb-c73d-14db-1f93-a6b622c4cebf: pipelining replication: peer="{Voter server-356c2d0a-813e-f106-5b26-c4f1880ee8f8 356c2d0a-813e-f106-5b26-c4f1880ee8f8}" -2024-05-12T20:49:57.475Z [INFO] server-5a54eefb-c73d-14db-1f93-a6b622c4cebf: pipelining replication: peer="{Voter server-536f6c2b-f3dd-061d-e5d3-f87c16afce5d 536f6c2b-f3dd-061d-e5d3-f87c16afce5d}" -2024-05-12T20:49:57.582Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:57.475544049 +0000 UTC m=+8.756244421 (1 nodes), 61.903481ms from start of poll, 62.284026ms from cluster start. Timeout at 2024-05-12 20:49:57.582773216 +0000 UTC m=+8.863473736, 107.229315ms after stability -2024-05-12T20:49:57.582Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:49:57.691Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:49:57.5828833 +0000 UTC m=+8.863583612 (2 nodes), 2.629µs from start of poll, 169.623217ms from cluster start. Timeout at 2024-05-12 20:49:57.6908043 +0000 UTC m=+8.971505480, 107.921868ms after stability -2024-05-12T20:49:57.691Z [ERROR] server-5a54eefb-c73d-14db-1f93-a6b622c4cebf: failed to pipeline appendEntries: peer="{Voter server-536f6c2b-f3dd-061d-e5d3-f87c16afce5d 536f6c2b-f3dd-061d-e5d3-f87c16afce5d}" error="append pipeline closed" -2024-05-12T20:49:57.691Z [INFO] server-5a54eefb-c73d-14db-1f93-a6b622c4cebf: aborting pipeline replication: peer="{Voter server-536f6c2b-f3dd-061d-e5d3-f87c16afce5d 536f6c2b-f3dd-061d-e5d3-f87c16afce5d}" -2024-05-12T20:49:57.691Z [ERROR] server-5a54eefb-c73d-14db-1f93-a6b622c4cebf: failed to appendEntries to: peer="{Voter server-536f6c2b-f3dd-061d-e5d3-f87c16afce5d 536f6c2b-f3dd-061d-e5d3-f87c16afce5d}" error="failed to connect to peer: 536f6c2b-f3dd-061d-e5d3-f87c16afce5d" -2024-05-12T20:49:57.706Z [ERROR] server-5a54eefb-c73d-14db-1f93-a6b622c4cebf: failed to heartbeat to: peer=536f6c2b-f3dd-061d-e5d3-f87c16afce5d backoff time=10ms error="failed to connect to peer: 536f6c2b-f3dd-061d-e5d3-f87c16afce5d" -2024-05-12T20:49:57.716Z [ERROR] server-5a54eefb-c73d-14db-1f93-a6b622c4cebf: failed to appendEntries to: peer="{Voter server-536f6c2b-f3dd-061d-e5d3-f87c16afce5d 536f6c2b-f3dd-061d-e5d3-f87c16afce5d}" error="failed to connect to peer: 536f6c2b-f3dd-061d-e5d3-f87c16afce5d" +2025-06-15T03:16:31.601Z [INFO] server-819c2c4d-3326-183a-6370-5c41571f1241: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-819c2c4d-3326-183a-6370-5c41571f1241 Address:819c2c4d-3326-183a-6370-5c41571f1241} {Suffrage:Voter ID:server-fd0c4bb6-0236-54d1-27c9-9e124871be14 Address:fd0c4bb6-0236-54d1-27c9-9e124871be14} {Suffrage:Voter ID:server-9c759bae-2cce-b248-5daf-ab71bc835841 Address:9c759bae-2cce-b248-5daf-ab71bc835841}]" +2025-06-15T03:16:31.601Z [INFO] server-fd0c4bb6-0236-54d1-27c9-9e124871be14: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-819c2c4d-3326-183a-6370-5c41571f1241 Address:819c2c4d-3326-183a-6370-5c41571f1241} {Suffrage:Voter ID:server-fd0c4bb6-0236-54d1-27c9-9e124871be14 Address:fd0c4bb6-0236-54d1-27c9-9e124871be14} {Suffrage:Voter ID:server-9c759bae-2cce-b248-5daf-ab71bc835841 Address:9c759bae-2cce-b248-5daf-ab71bc835841}]" +2025-06-15T03:16:31.601Z [INFO] server-819c2c4d-3326-183a-6370-5c41571f1241: entering follower state: follower="Node at 819c2c4d-3326-183a-6370-5c41571f1241 [Follower]" leader-address= leader-id= +2025-06-15T03:16:31.601Z [INFO] server-fd0c4bb6-0236-54d1-27c9-9e124871be14: entering follower state: follower="Node at fd0c4bb6-0236-54d1-27c9-9e124871be14 [Follower]" leader-address= leader-id= +2025-06-15T03:16:31.601Z [INFO] server-9c759bae-2cce-b248-5daf-ab71bc835841: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-819c2c4d-3326-183a-6370-5c41571f1241 Address:819c2c4d-3326-183a-6370-5c41571f1241} {Suffrage:Voter ID:server-fd0c4bb6-0236-54d1-27c9-9e124871be14 Address:fd0c4bb6-0236-54d1-27c9-9e124871be14} {Suffrage:Voter ID:server-9c759bae-2cce-b248-5daf-ab71bc835841 Address:9c759bae-2cce-b248-5daf-ab71bc835841}]" +2025-06-15T03:16:31.601Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:31.601Z [INFO] server-9c759bae-2cce-b248-5daf-ab71bc835841: entering follower state: follower="Node at 9c759bae-2cce-b248-5daf-ab71bc835841 [Follower]" leader-address= leader-id= +2025-06-15T03:16:31.675Z [WARN] server-819c2c4d-3326-183a-6370-5c41571f1241: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:31.675Z [INFO] server-819c2c4d-3326-183a-6370-5c41571f1241: entering candidate state: node="Node at 819c2c4d-3326-183a-6370-5c41571f1241 [Candidate]" term=2 +2025-06-15T03:16:31.676Z [INFO] server-819c2c4d-3326-183a-6370-5c41571f1241: election won: term=2 tally=2 +2025-06-15T03:16:31.676Z [INFO] server-819c2c4d-3326-183a-6370-5c41571f1241: entering leader state: leader="Node at 819c2c4d-3326-183a-6370-5c41571f1241 [Leader]" +2025-06-15T03:16:31.676Z [INFO] server-819c2c4d-3326-183a-6370-5c41571f1241: added peer, starting replication: peer=server-fd0c4bb6-0236-54d1-27c9-9e124871be14 +2025-06-15T03:16:31.676Z [INFO] server-819c2c4d-3326-183a-6370-5c41571f1241: added peer, starting replication: peer=server-9c759bae-2cce-b248-5daf-ab71bc835841 +2025-06-15T03:16:31.676Z [INFO] server-819c2c4d-3326-183a-6370-5c41571f1241: pipelining replication: peer="{Voter server-9c759bae-2cce-b248-5daf-ab71bc835841 9c759bae-2cce-b248-5daf-ab71bc835841}" +2025-06-15T03:16:31.676Z [INFO] server-819c2c4d-3326-183a-6370-5c41571f1241: pipelining replication: peer="{Voter server-fd0c4bb6-0236-54d1-27c9-9e124871be14 fd0c4bb6-0236-54d1-27c9-9e124871be14}" +2025-06-15T03:16:31.781Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:31.67609307 +0000 UTC m=+7.387771273 (1 nodes), 74.280315ms from start of poll, 74.701567ms from cluster start. Timeout at 2025-06-15 03:16:31.781961927 +0000 UTC m=+7.493640088, 105.868815ms after stability +2025-06-15T03:16:31.782Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:16:31.887Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:16:31.78200976 +0000 UTC m=+7.493687921 (2 nodes), 969ns from start of poll, 180.618215ms from cluster start. Timeout at 2025-06-15 03:16:31.887101702 +0000 UTC m=+7.598779867, 105.091946ms after stability +2025-06-15T03:16:31.887Z [ERROR] server-819c2c4d-3326-183a-6370-5c41571f1241: failed to pipeline appendEntries: peer="{Voter server-fd0c4bb6-0236-54d1-27c9-9e124871be14 fd0c4bb6-0236-54d1-27c9-9e124871be14}" error="append pipeline closed" +2025-06-15T03:16:31.887Z [INFO] server-819c2c4d-3326-183a-6370-5c41571f1241: aborting pipeline replication: peer="{Voter server-fd0c4bb6-0236-54d1-27c9-9e124871be14 fd0c4bb6-0236-54d1-27c9-9e124871be14}" +2025-06-15T03:16:31.887Z [ERROR] server-819c2c4d-3326-183a-6370-5c41571f1241: failed to appendEntries to: peer="{Voter server-fd0c4bb6-0236-54d1-27c9-9e124871be14 fd0c4bb6-0236-54d1-27c9-9e124871be14}" error="failed to connect to peer: fd0c4bb6-0236-54d1-27c9-9e124871be14" raft_test.go:459: [INFO] Finished apply without behind follower -2024-05-12T20:49:57.727Z [INFO] server-5a54eefb-c73d-14db-1f93-a6b622c4cebf: pipelining replication: peer="{Voter server-536f6c2b-f3dd-061d-e5d3-f87c16afce5d 536f6c2b-f3dd-061d-e5d3-f87c16afce5d}" -2024-05-12T20:49:57.730Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:57.838Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:57.730454272 +0000 UTC m=+9.011154582 (1 nodes), 2.041µs from start of poll, 317.194187ms from cluster start. Timeout at 2024-05-12 20:49:57.838773965 +0000 UTC m=+9.119479450, 108.324868ms after stability -2024-05-12T20:49:57.839Z [INFO] server-5a54eefb-c73d-14db-1f93-a6b622c4cebf: aborting pipeline replication: peer="{Voter server-356c2d0a-813e-f106-5b26-c4f1880ee8f8 356c2d0a-813e-f106-5b26-c4f1880ee8f8}" -2024-05-12T20:49:57.839Z [INFO] server-5a54eefb-c73d-14db-1f93-a6b622c4cebf: aborting pipeline replication: peer="{Voter server-536f6c2b-f3dd-061d-e5d3-f87c16afce5d 536f6c2b-f3dd-061d-e5d3-f87c16afce5d}" ---- PASS: TestRaft_BehindFollower (0.43s) +2025-06-15T03:16:31.897Z [INFO] server-819c2c4d-3326-183a-6370-5c41571f1241: pipelining replication: peer="{Voter server-fd0c4bb6-0236-54d1-27c9-9e124871be14 fd0c4bb6-0236-54d1-27c9-9e124871be14}" +2025-06-15T03:16:31.906Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:32.012Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:31.90616268 +0000 UTC m=+7.617840842 (1 nodes), 923ns from start of poll, 304.771136ms from cluster start. Timeout at 2025-06-15 03:16:32.011978942 +0000 UTC m=+7.723657107, 105.816265ms after stability +2025-06-15T03:16:32.012Z [INFO] server-819c2c4d-3326-183a-6370-5c41571f1241: aborting pipeline replication: peer="{Voter server-fd0c4bb6-0236-54d1-27c9-9e124871be14 fd0c4bb6-0236-54d1-27c9-9e124871be14}" +2025-06-15T03:16:32.012Z [INFO] server-819c2c4d-3326-183a-6370-5c41571f1241: aborting pipeline replication: peer="{Voter server-9c759bae-2cce-b248-5daf-ab71bc835841 9c759bae-2cce-b248-5daf-ab71bc835841}" +--- PASS: TestRaft_BehindFollower (0.41s) === RUN TestRaft_ApplyNonLeader -2024-05-12T20:49:57.840Z [INFO] server-9a801732-1940-0fa4-46d0-b796799db9b0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a801732-1940-0fa4-46d0-b796799db9b0 Address:9a801732-1940-0fa4-46d0-b796799db9b0} {Suffrage:Voter ID:server-40f986b9-2cb9-d9f8-5682-9107694a389e Address:40f986b9-2cb9-d9f8-5682-9107694a389e} {Suffrage:Voter ID:server-66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d Address:66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d}]" -2024-05-12T20:49:57.841Z [INFO] server-40f986b9-2cb9-d9f8-5682-9107694a389e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a801732-1940-0fa4-46d0-b796799db9b0 Address:9a801732-1940-0fa4-46d0-b796799db9b0} {Suffrage:Voter ID:server-40f986b9-2cb9-d9f8-5682-9107694a389e Address:40f986b9-2cb9-d9f8-5682-9107694a389e} {Suffrage:Voter ID:server-66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d Address:66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d}]" -2024-05-12T20:49:57.841Z [INFO] server-66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a801732-1940-0fa4-46d0-b796799db9b0 Address:9a801732-1940-0fa4-46d0-b796799db9b0} {Suffrage:Voter ID:server-40f986b9-2cb9-d9f8-5682-9107694a389e Address:40f986b9-2cb9-d9f8-5682-9107694a389e} {Suffrage:Voter ID:server-66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d Address:66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d}]" -2024-05-12T20:49:57.841Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:57.841Z [INFO] server-9a801732-1940-0fa4-46d0-b796799db9b0: entering follower state: follower="Node at 9a801732-1940-0fa4-46d0-b796799db9b0 [Follower]" leader-address= leader-id= -2024-05-12T20:49:57.841Z [INFO] server-40f986b9-2cb9-d9f8-5682-9107694a389e: entering follower state: follower="Node at 40f986b9-2cb9-d9f8-5682-9107694a389e [Follower]" leader-address= leader-id= -2024-05-12T20:49:57.841Z [INFO] server-66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d: entering follower state: follower="Node at 66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d [Follower]" leader-address= leader-id= -2024-05-12T20:49:57.910Z [WARN] server-9a801732-1940-0fa4-46d0-b796799db9b0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:57.910Z [INFO] server-9a801732-1940-0fa4-46d0-b796799db9b0: entering candidate state: node="Node at 9a801732-1940-0fa4-46d0-b796799db9b0 [Candidate]" term=2 -2024-05-12T20:49:57.910Z [WARN] server-66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:57.911Z [INFO] server-66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d: entering candidate state: node="Node at 66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d [Candidate]" term=2 -2024-05-12T20:49:57.911Z [INFO] server-66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d: duplicate requestVote for same term: term=2 -2024-05-12T20:49:57.911Z [INFO] server-9a801732-1940-0fa4-46d0-b796799db9b0: election won: term=2 tally=2 -2024-05-12T20:49:57.911Z [INFO] server-9a801732-1940-0fa4-46d0-b796799db9b0: entering leader state: leader="Node at 9a801732-1940-0fa4-46d0-b796799db9b0 [Leader]" -2024-05-12T20:49:57.911Z [INFO] server-9a801732-1940-0fa4-46d0-b796799db9b0: added peer, starting replication: peer=server-40f986b9-2cb9-d9f8-5682-9107694a389e -2024-05-12T20:49:57.911Z [INFO] server-9a801732-1940-0fa4-46d0-b796799db9b0: added peer, starting replication: peer=server-66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d -2024-05-12T20:49:57.911Z [WARN] server-9a801732-1940-0fa4-46d0-b796799db9b0: rejecting vote request since we have a leader: from=66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d leader=9a801732-1940-0fa4-46d0-b796799db9b0 leader-id=server-9a801732-1940-0fa4-46d0-b796799db9b0 -2024-05-12T20:49:57.911Z [INFO] server-66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d: entering follower state: follower="Node at 66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d [Follower]" leader-address=9a801732-1940-0fa4-46d0-b796799db9b0 leader-id=server-9a801732-1940-0fa4-46d0-b796799db9b0 -2024-05-12T20:49:57.911Z [INFO] server-40f986b9-2cb9-d9f8-5682-9107694a389e: duplicate requestVote for same term: term=2 -2024-05-12T20:49:57.911Z [INFO] server-9a801732-1940-0fa4-46d0-b796799db9b0: pipelining replication: peer="{Voter server-66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d 66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d}" -2024-05-12T20:49:57.911Z [INFO] server-9a801732-1940-0fa4-46d0-b796799db9b0: pipelining replication: peer="{Voter server-40f986b9-2cb9-d9f8-5682-9107694a389e 40f986b9-2cb9-d9f8-5682-9107694a389e}" -2024-05-12T20:49:58.018Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:57.911563398 +0000 UTC m=+9.192263739 (1 nodes), 70.442614ms from start of poll, 70.815431ms from cluster start. Timeout at 2024-05-12 20:49:58.018787942 +0000 UTC m=+9.299492297, 107.228558ms after stability -2024-05-12T20:49:58.018Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:49:58.138Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:49:58.018934219 +0000 UTC m=+9.299634669 (2 nodes), 2.183µs from start of poll, 178.186361ms from cluster start. Timeout at 2024-05-12 20:49:58.138774454 +0000 UTC m=+9.419474349, 119.83968ms after stability -2024-05-12T20:49:58.139Z [INFO] server-9a801732-1940-0fa4-46d0-b796799db9b0: aborting pipeline replication: peer="{Voter server-66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d 66b1cb25-8cbc-cf2c-10d3-c428a6b42a2d}" -2024-05-12T20:49:58.139Z [INFO] server-9a801732-1940-0fa4-46d0-b796799db9b0: aborting pipeline replication: peer="{Voter server-40f986b9-2cb9-d9f8-5682-9107694a389e 40f986b9-2cb9-d9f8-5682-9107694a389e}" ---- PASS: TestRaft_ApplyNonLeader (0.31s) +2025-06-15T03:16:32.013Z [INFO] server-f78ebbb7-0a84-7b67-aafe-72e676c2e155: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f78ebbb7-0a84-7b67-aafe-72e676c2e155 Address:f78ebbb7-0a84-7b67-aafe-72e676c2e155} {Suffrage:Voter ID:server-c145c98d-9b44-fe1f-fe9c-7558f69e58c7 Address:c145c98d-9b44-fe1f-fe9c-7558f69e58c7} {Suffrage:Voter ID:server-e7df1692-dcfc-3be5-a8d8-e824c25119ce Address:e7df1692-dcfc-3be5-a8d8-e824c25119ce}]" +2025-06-15T03:16:32.013Z [INFO] server-f78ebbb7-0a84-7b67-aafe-72e676c2e155: entering follower state: follower="Node at f78ebbb7-0a84-7b67-aafe-72e676c2e155 [Follower]" leader-address= leader-id= +2025-06-15T03:16:32.013Z [INFO] server-c145c98d-9b44-fe1f-fe9c-7558f69e58c7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f78ebbb7-0a84-7b67-aafe-72e676c2e155 Address:f78ebbb7-0a84-7b67-aafe-72e676c2e155} {Suffrage:Voter ID:server-c145c98d-9b44-fe1f-fe9c-7558f69e58c7 Address:c145c98d-9b44-fe1f-fe9c-7558f69e58c7} {Suffrage:Voter ID:server-e7df1692-dcfc-3be5-a8d8-e824c25119ce Address:e7df1692-dcfc-3be5-a8d8-e824c25119ce}]" +2025-06-15T03:16:32.013Z [INFO] server-c145c98d-9b44-fe1f-fe9c-7558f69e58c7: entering follower state: follower="Node at c145c98d-9b44-fe1f-fe9c-7558f69e58c7 [Follower]" leader-address= leader-id= +2025-06-15T03:16:32.013Z [INFO] server-e7df1692-dcfc-3be5-a8d8-e824c25119ce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f78ebbb7-0a84-7b67-aafe-72e676c2e155 Address:f78ebbb7-0a84-7b67-aafe-72e676c2e155} {Suffrage:Voter ID:server-c145c98d-9b44-fe1f-fe9c-7558f69e58c7 Address:c145c98d-9b44-fe1f-fe9c-7558f69e58c7} {Suffrage:Voter ID:server-e7df1692-dcfc-3be5-a8d8-e824c25119ce Address:e7df1692-dcfc-3be5-a8d8-e824c25119ce}]" +2025-06-15T03:16:32.013Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:32.013Z [INFO] server-e7df1692-dcfc-3be5-a8d8-e824c25119ce: entering follower state: follower="Node at e7df1692-dcfc-3be5-a8d8-e824c25119ce [Follower]" leader-address= leader-id= +2025-06-15T03:16:32.066Z [WARN] server-f78ebbb7-0a84-7b67-aafe-72e676c2e155: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:32.066Z [INFO] server-f78ebbb7-0a84-7b67-aafe-72e676c2e155: entering candidate state: node="Node at f78ebbb7-0a84-7b67-aafe-72e676c2e155 [Candidate]" term=2 +2025-06-15T03:16:32.066Z [INFO] server-f78ebbb7-0a84-7b67-aafe-72e676c2e155: election won: term=2 tally=2 +2025-06-15T03:16:32.066Z [INFO] server-f78ebbb7-0a84-7b67-aafe-72e676c2e155: entering leader state: leader="Node at f78ebbb7-0a84-7b67-aafe-72e676c2e155 [Leader]" +2025-06-15T03:16:32.066Z [INFO] server-f78ebbb7-0a84-7b67-aafe-72e676c2e155: added peer, starting replication: peer=server-c145c98d-9b44-fe1f-fe9c-7558f69e58c7 +2025-06-15T03:16:32.066Z [INFO] server-f78ebbb7-0a84-7b67-aafe-72e676c2e155: added peer, starting replication: peer=server-e7df1692-dcfc-3be5-a8d8-e824c25119ce +2025-06-15T03:16:32.066Z [INFO] server-f78ebbb7-0a84-7b67-aafe-72e676c2e155: pipelining replication: peer="{Voter server-e7df1692-dcfc-3be5-a8d8-e824c25119ce e7df1692-dcfc-3be5-a8d8-e824c25119ce}" +2025-06-15T03:16:32.066Z [INFO] server-f78ebbb7-0a84-7b67-aafe-72e676c2e155: pipelining replication: peer="{Voter server-c145c98d-9b44-fe1f-fe9c-7558f69e58c7 c145c98d-9b44-fe1f-fe9c-7558f69e58c7}" +2025-06-15T03:16:32.171Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:32.066828162 +0000 UTC m=+7.778506343 (1 nodes), 53.518245ms from start of poll, 53.86487ms from cluster start. Timeout at 2025-06-15 03:16:32.171878209 +0000 UTC m=+7.883556375, 105.050032ms after stability +2025-06-15T03:16:32.171Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:16:32.277Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:16:32.171931799 +0000 UTC m=+7.883609961 (2 nodes), 1.736µs from start of poll, 158.968488ms from cluster start. Timeout at 2025-06-15 03:16:32.27758202 +0000 UTC m=+7.989260220, 105.650259ms after stability +2025-06-15T03:16:32.277Z [INFO] server-f78ebbb7-0a84-7b67-aafe-72e676c2e155: aborting pipeline replication: peer="{Voter server-c145c98d-9b44-fe1f-fe9c-7558f69e58c7 c145c98d-9b44-fe1f-fe9c-7558f69e58c7}" +2025-06-15T03:16:32.277Z [INFO] server-f78ebbb7-0a84-7b67-aafe-72e676c2e155: aborting pipeline replication: peer="{Voter server-e7df1692-dcfc-3be5-a8d8-e824c25119ce e7df1692-dcfc-3be5-a8d8-e824c25119ce}" +--- PASS: TestRaft_ApplyNonLeader (0.27s) === RUN TestRaft_ApplyConcurrent -2024-05-12T20:49:58.155Z [INFO] server-78f6d530-03e8-5aa5-5024-51f9b6cd4ea0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-78f6d530-03e8-5aa5-5024-51f9b6cd4ea0 Address:78f6d530-03e8-5aa5-5024-51f9b6cd4ea0} {Suffrage:Voter ID:server-84728867-831f-14b3-8586-069c2b9dfae4 Address:84728867-831f-14b3-8586-069c2b9dfae4} {Suffrage:Voter ID:server-6a0af2fe-a77c-91d6-702f-b18842f196f1 Address:6a0af2fe-a77c-91d6-702f-b18842f196f1}]" -2024-05-12T20:49:58.155Z [INFO] server-84728867-831f-14b3-8586-069c2b9dfae4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-78f6d530-03e8-5aa5-5024-51f9b6cd4ea0 Address:78f6d530-03e8-5aa5-5024-51f9b6cd4ea0} {Suffrage:Voter ID:server-84728867-831f-14b3-8586-069c2b9dfae4 Address:84728867-831f-14b3-8586-069c2b9dfae4} {Suffrage:Voter ID:server-6a0af2fe-a77c-91d6-702f-b18842f196f1 Address:6a0af2fe-a77c-91d6-702f-b18842f196f1}]" -2024-05-12T20:49:58.155Z [INFO] server-6a0af2fe-a77c-91d6-702f-b18842f196f1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-78f6d530-03e8-5aa5-5024-51f9b6cd4ea0 Address:78f6d530-03e8-5aa5-5024-51f9b6cd4ea0} {Suffrage:Voter ID:server-84728867-831f-14b3-8586-069c2b9dfae4 Address:84728867-831f-14b3-8586-069c2b9dfae4} {Suffrage:Voter ID:server-6a0af2fe-a77c-91d6-702f-b18842f196f1 Address:6a0af2fe-a77c-91d6-702f-b18842f196f1}]" -2024-05-12T20:49:58.155Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:58.158Z [INFO] server-84728867-831f-14b3-8586-069c2b9dfae4: entering follower state: follower="Node at 84728867-831f-14b3-8586-069c2b9dfae4 [Follower]" leader-address= leader-id= -2024-05-12T20:49:58.158Z [INFO] server-78f6d530-03e8-5aa5-5024-51f9b6cd4ea0: entering follower state: follower="Node at 78f6d530-03e8-5aa5-5024-51f9b6cd4ea0 [Follower]" leader-address= leader-id= -2024-05-12T20:49:58.159Z [INFO] server-6a0af2fe-a77c-91d6-702f-b18842f196f1: entering follower state: follower="Node at 6a0af2fe-a77c-91d6-702f-b18842f196f1 [Follower]" leader-address= leader-id= -2024-05-12T20:49:58.262Z [WARN] server-84728867-831f-14b3-8586-069c2b9dfae4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:58.262Z [INFO] server-84728867-831f-14b3-8586-069c2b9dfae4: entering candidate state: node="Node at 84728867-831f-14b3-8586-069c2b9dfae4 [Candidate]" term=2 -2024-05-12T20:49:58.262Z [INFO] server-84728867-831f-14b3-8586-069c2b9dfae4: election won: term=2 tally=2 -2024-05-12T20:49:58.262Z [INFO] server-84728867-831f-14b3-8586-069c2b9dfae4: entering leader state: leader="Node at 84728867-831f-14b3-8586-069c2b9dfae4 [Leader]" -2024-05-12T20:49:58.263Z [INFO] server-84728867-831f-14b3-8586-069c2b9dfae4: added peer, starting replication: peer=server-78f6d530-03e8-5aa5-5024-51f9b6cd4ea0 -2024-05-12T20:49:58.263Z [INFO] server-84728867-831f-14b3-8586-069c2b9dfae4: added peer, starting replication: peer=server-6a0af2fe-a77c-91d6-702f-b18842f196f1 -2024-05-12T20:49:58.263Z [INFO] server-84728867-831f-14b3-8586-069c2b9dfae4: pipelining replication: peer="{Voter server-6a0af2fe-a77c-91d6-702f-b18842f196f1 6a0af2fe-a77c-91d6-702f-b18842f196f1}" -2024-05-12T20:49:58.263Z [INFO] server-84728867-831f-14b3-8586-069c2b9dfae4: pipelining replication: peer="{Voter server-78f6d530-03e8-5aa5-5024-51f9b6cd4ea0 78f6d530-03e8-5aa5-5024-51f9b6cd4ea0}" -2024-05-12T20:49:58.470Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:58.263296297 +0000 UTC m=+9.543996800 (1 nodes), 107.422956ms from start of poll, 115.031762ms from cluster start. Timeout at 2024-05-12 20:49:58.470773125 +0000 UTC m=+9.751473351, 207.476551ms after stability -2024-05-12T20:49:58.503Z [INFO] server-84728867-831f-14b3-8586-069c2b9dfae4: aborting pipeline replication: peer="{Voter server-6a0af2fe-a77c-91d6-702f-b18842f196f1 6a0af2fe-a77c-91d6-702f-b18842f196f1}" -2024-05-12T20:49:58.503Z [INFO] server-84728867-831f-14b3-8586-069c2b9dfae4: aborting pipeline replication: peer="{Voter server-78f6d530-03e8-5aa5-5024-51f9b6cd4ea0 78f6d530-03e8-5aa5-5024-51f9b6cd4ea0}" ---- PASS: TestRaft_ApplyConcurrent (0.36s) +2025-06-15T03:16:32.279Z [INFO] server-0e044976-f0a1-8fd6-b4dd-94655c133d0c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0e044976-f0a1-8fd6-b4dd-94655c133d0c Address:0e044976-f0a1-8fd6-b4dd-94655c133d0c} {Suffrage:Voter ID:server-d0f03b65-4efa-60b9-08ca-792230849e9e Address:d0f03b65-4efa-60b9-08ca-792230849e9e} {Suffrage:Voter ID:server-4eb62e46-5caf-daed-cffd-c523d972cf6b Address:4eb62e46-5caf-daed-cffd-c523d972cf6b}]" +2025-06-15T03:16:32.279Z [INFO] server-d0f03b65-4efa-60b9-08ca-792230849e9e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0e044976-f0a1-8fd6-b4dd-94655c133d0c Address:0e044976-f0a1-8fd6-b4dd-94655c133d0c} {Suffrage:Voter ID:server-d0f03b65-4efa-60b9-08ca-792230849e9e Address:d0f03b65-4efa-60b9-08ca-792230849e9e} {Suffrage:Voter ID:server-4eb62e46-5caf-daed-cffd-c523d972cf6b Address:4eb62e46-5caf-daed-cffd-c523d972cf6b}]" +2025-06-15T03:16:32.279Z [INFO] server-0e044976-f0a1-8fd6-b4dd-94655c133d0c: entering follower state: follower="Node at 0e044976-f0a1-8fd6-b4dd-94655c133d0c [Follower]" leader-address= leader-id= +2025-06-15T03:16:32.279Z [INFO] server-d0f03b65-4efa-60b9-08ca-792230849e9e: entering follower state: follower="Node at d0f03b65-4efa-60b9-08ca-792230849e9e [Follower]" leader-address= leader-id= +2025-06-15T03:16:32.279Z [INFO] server-4eb62e46-5caf-daed-cffd-c523d972cf6b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0e044976-f0a1-8fd6-b4dd-94655c133d0c Address:0e044976-f0a1-8fd6-b4dd-94655c133d0c} {Suffrage:Voter ID:server-d0f03b65-4efa-60b9-08ca-792230849e9e Address:d0f03b65-4efa-60b9-08ca-792230849e9e} {Suffrage:Voter ID:server-4eb62e46-5caf-daed-cffd-c523d972cf6b Address:4eb62e46-5caf-daed-cffd-c523d972cf6b}]" +2025-06-15T03:16:32.279Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:32.279Z [INFO] server-4eb62e46-5caf-daed-cffd-c523d972cf6b: entering follower state: follower="Node at 4eb62e46-5caf-daed-cffd-c523d972cf6b [Follower]" leader-address= leader-id= +2025-06-15T03:16:32.382Z [WARN] server-4eb62e46-5caf-daed-cffd-c523d972cf6b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:32.382Z [INFO] server-4eb62e46-5caf-daed-cffd-c523d972cf6b: entering candidate state: node="Node at 4eb62e46-5caf-daed-cffd-c523d972cf6b [Candidate]" term=2 +2025-06-15T03:16:32.383Z [INFO] server-4eb62e46-5caf-daed-cffd-c523d972cf6b: election won: term=2 tally=2 +2025-06-15T03:16:32.383Z [INFO] server-4eb62e46-5caf-daed-cffd-c523d972cf6b: entering leader state: leader="Node at 4eb62e46-5caf-daed-cffd-c523d972cf6b [Leader]" +2025-06-15T03:16:32.383Z [INFO] server-4eb62e46-5caf-daed-cffd-c523d972cf6b: added peer, starting replication: peer=server-0e044976-f0a1-8fd6-b4dd-94655c133d0c +2025-06-15T03:16:32.383Z [INFO] server-4eb62e46-5caf-daed-cffd-c523d972cf6b: added peer, starting replication: peer=server-d0f03b65-4efa-60b9-08ca-792230849e9e +2025-06-15T03:16:32.383Z [INFO] server-4eb62e46-5caf-daed-cffd-c523d972cf6b: pipelining replication: peer="{Voter server-d0f03b65-4efa-60b9-08ca-792230849e9e d0f03b65-4efa-60b9-08ca-792230849e9e}" +2025-06-15T03:16:32.383Z [INFO] server-4eb62e46-5caf-daed-cffd-c523d972cf6b: pipelining replication: peer="{Voter server-0e044976-f0a1-8fd6-b4dd-94655c133d0c 0e044976-f0a1-8fd6-b4dd-94655c133d0c}" +2025-06-15T03:16:32.589Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:32.38337731 +0000 UTC m=+8.095055486 (1 nodes), 103.850664ms from start of poll, 104.307727ms from cluster start. Timeout at 2025-06-15 03:16:32.589174326 +0000 UTC m=+8.300852515, 205.797029ms after stability +2025-06-15T03:16:32.613Z [INFO] server-4eb62e46-5caf-daed-cffd-c523d972cf6b: aborting pipeline replication: peer="{Voter server-d0f03b65-4efa-60b9-08ca-792230849e9e d0f03b65-4efa-60b9-08ca-792230849e9e}" +2025-06-15T03:16:32.613Z [INFO] server-4eb62e46-5caf-daed-cffd-c523d972cf6b: aborting pipeline replication: peer="{Voter server-0e044976-f0a1-8fd6-b4dd-94655c133d0c 0e044976-f0a1-8fd6-b4dd-94655c133d0c}" +--- PASS: TestRaft_ApplyConcurrent (0.34s) === RUN TestRaft_ApplyConcurrent_Timeout -2024-05-12T20:49:58.512Z [INFO] server-36bfc8da-2fd3-2c15-a0a5-5169f0e6f4e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-36bfc8da-2fd3-2c15-a0a5-5169f0e6f4e9 Address:36bfc8da-2fd3-2c15-a0a5-5169f0e6f4e9}]" -2024-05-12T20:49:58.512Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:58.512Z [INFO] server-36bfc8da-2fd3-2c15-a0a5-5169f0e6f4e9: entering follower state: follower="Node at 36bfc8da-2fd3-2c15-a0a5-5169f0e6f4e9 [Follower]" leader-address= leader-id= -2024-05-12T20:49:58.678Z [WARN] server-36bfc8da-2fd3-2c15-a0a5-5169f0e6f4e9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:58.678Z [INFO] server-36bfc8da-2fd3-2c15-a0a5-5169f0e6f4e9: entering candidate state: node="Node at 36bfc8da-2fd3-2c15-a0a5-5169f0e6f4e9 [Candidate]" term=2 -2024-05-12T20:49:58.678Z [INFO] server-36bfc8da-2fd3-2c15-a0a5-5169f0e6f4e9: election won: term=2 tally=1 -2024-05-12T20:49:58.678Z [INFO] server-36bfc8da-2fd3-2c15-a0a5-5169f0e6f4e9: entering leader state: leader="Node at 36bfc8da-2fd3-2c15-a0a5-5169f0e6f4e9 [Leader]" -2024-05-12T20:49:58.882Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:58.679059071 +0000 UTC m=+9.959759429 (1 nodes), 166.90109ms from start of poll, 167.125162ms from cluster start. Timeout at 2024-05-12 20:49:58.882755636 +0000 UTC m=+10.163456337, 203.696908ms after stability ---- PASS: TestRaft_ApplyConcurrent_Timeout (0.38s) +2025-06-15T03:16:32.615Z [INFO] server-a18f6d37-27f0-eca5-ac22-21ee28bf1aba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a18f6d37-27f0-eca5-ac22-21ee28bf1aba Address:a18f6d37-27f0-eca5-ac22-21ee28bf1aba}]" +2025-06-15T03:16:32.615Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:32.615Z [INFO] server-a18f6d37-27f0-eca5-ac22-21ee28bf1aba: entering follower state: follower="Node at a18f6d37-27f0-eca5-ac22-21ee28bf1aba [Follower]" leader-address= leader-id= +2025-06-15T03:16:32.802Z [WARN] server-a18f6d37-27f0-eca5-ac22-21ee28bf1aba: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:32.806Z [INFO] server-a18f6d37-27f0-eca5-ac22-21ee28bf1aba: entering candidate state: node="Node at a18f6d37-27f0-eca5-ac22-21ee28bf1aba [Candidate]" term=2 +2025-06-15T03:16:32.806Z [INFO] server-a18f6d37-27f0-eca5-ac22-21ee28bf1aba: election won: term=2 tally=1 +2025-06-15T03:16:32.806Z [INFO] server-a18f6d37-27f0-eca5-ac22-21ee28bf1aba: entering leader state: leader="Node at a18f6d37-27f0-eca5-ac22-21ee28bf1aba [Leader]" +2025-06-15T03:16:33.007Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:32.806265392 +0000 UTC m=+8.517943576 (1 nodes), 190.98616ms from start of poll, 191.188562ms from cluster start. Timeout at 2025-06-15 03:16:33.00726931 +0000 UTC m=+8.718947486, 201.00391ms after stability +--- PASS: TestRaft_ApplyConcurrent_Timeout (0.40s) === RUN TestRaft_JoinNode -2024-05-12T20:49:58.899Z [INFO] server-75590a3f-ee04-db05-952b-916fa9d36a32: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-75590a3f-ee04-db05-952b-916fa9d36a32 Address:75590a3f-ee04-db05-952b-916fa9d36a32} {Suffrage:Voter ID:server-3cafcfca-19df-9a7e-243a-3e18ea55a346 Address:3cafcfca-19df-9a7e-243a-3e18ea55a346}]" -2024-05-12T20:49:58.900Z [INFO] server-3cafcfca-19df-9a7e-243a-3e18ea55a346: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-75590a3f-ee04-db05-952b-916fa9d36a32 Address:75590a3f-ee04-db05-952b-916fa9d36a32} {Suffrage:Voter ID:server-3cafcfca-19df-9a7e-243a-3e18ea55a346 Address:3cafcfca-19df-9a7e-243a-3e18ea55a346}]" -2024-05-12T20:49:58.900Z [INFO] server-9f7f4c65-0ad6-1b7b-58e5-f8b2bda92564: initial configuration: index=0 servers=[] -2024-05-12T20:49:58.900Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:58.902Z [INFO] server-3cafcfca-19df-9a7e-243a-3e18ea55a346: entering follower state: follower="Node at 3cafcfca-19df-9a7e-243a-3e18ea55a346 [Follower]" leader-address= leader-id= -2024-05-12T20:49:58.903Z [INFO] server-75590a3f-ee04-db05-952b-916fa9d36a32: entering follower state: follower="Node at 75590a3f-ee04-db05-952b-916fa9d36a32 [Follower]" leader-address= leader-id= -2024-05-12T20:49:58.903Z [INFO] server-9f7f4c65-0ad6-1b7b-58e5-f8b2bda92564: entering follower state: follower="Node at 9f7f4c65-0ad6-1b7b-58e5-f8b2bda92564 [Follower]" leader-address= leader-id= -2024-05-12T20:49:58.970Z [WARN] server-9f7f4c65-0ad6-1b7b-58e5-f8b2bda92564: no known peers, aborting election -2024-05-12T20:49:58.986Z [WARN] server-75590a3f-ee04-db05-952b-916fa9d36a32: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:58.986Z [INFO] server-75590a3f-ee04-db05-952b-916fa9d36a32: entering candidate state: node="Node at 75590a3f-ee04-db05-952b-916fa9d36a32 [Candidate]" term=2 -2024-05-12T20:49:58.986Z [INFO] server-75590a3f-ee04-db05-952b-916fa9d36a32: election won: term=2 tally=2 -2024-05-12T20:49:58.986Z [INFO] server-75590a3f-ee04-db05-952b-916fa9d36a32: entering leader state: leader="Node at 75590a3f-ee04-db05-952b-916fa9d36a32 [Leader]" -2024-05-12T20:49:58.986Z [INFO] server-75590a3f-ee04-db05-952b-916fa9d36a32: added peer, starting replication: peer=server-3cafcfca-19df-9a7e-243a-3e18ea55a346 -2024-05-12T20:49:58.987Z [INFO] server-75590a3f-ee04-db05-952b-916fa9d36a32: pipelining replication: peer="{Voter server-3cafcfca-19df-9a7e-243a-3e18ea55a346 3cafcfca-19df-9a7e-243a-3e18ea55a346}" -2024-05-12T20:49:59.094Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:58.987205129 +0000 UTC m=+10.267905466 (1 nodes), 86.486598ms from start of poll, 87.642289ms from cluster start. Timeout at 2024-05-12 20:49:59.09476272 +0000 UTC m=+10.375464078, 107.558612ms after stability -2024-05-12T20:49:59.094Z [INFO] server-75590a3f-ee04-db05-952b-916fa9d36a32: updating configuration: command=AddVoter server-id=server-9f7f4c65-0ad6-1b7b-58e5-f8b2bda92564 server-addr=9f7f4c65-0ad6-1b7b-58e5-f8b2bda92564 servers="[{Suffrage:Voter ID:server-75590a3f-ee04-db05-952b-916fa9d36a32 Address:75590a3f-ee04-db05-952b-916fa9d36a32} {Suffrage:Voter ID:server-3cafcfca-19df-9a7e-243a-3e18ea55a346 Address:3cafcfca-19df-9a7e-243a-3e18ea55a346} {Suffrage:Voter ID:server-9f7f4c65-0ad6-1b7b-58e5-f8b2bda92564 Address:9f7f4c65-0ad6-1b7b-58e5-f8b2bda92564}]" -2024-05-12T20:49:59.094Z [INFO] server-75590a3f-ee04-db05-952b-916fa9d36a32: added peer, starting replication: peer=server-9f7f4c65-0ad6-1b7b-58e5-f8b2bda92564 -2024-05-12T20:49:59.095Z [WARN] server-9f7f4c65-0ad6-1b7b-58e5-f8b2bda92564: failed to get previous log: previous-index=3 last-index=0 error="log not found" -2024-05-12T20:49:59.095Z [WARN] server-75590a3f-ee04-db05-952b-916fa9d36a32: appendEntries rejected, sending older logs: peer="{Voter server-9f7f4c65-0ad6-1b7b-58e5-f8b2bda92564 9f7f4c65-0ad6-1b7b-58e5-f8b2bda92564}" next=1 -2024-05-12T20:49:59.095Z [INFO] server-75590a3f-ee04-db05-952b-916fa9d36a32: pipelining replication: peer="{Voter server-9f7f4c65-0ad6-1b7b-58e5-f8b2bda92564 9f7f4c65-0ad6-1b7b-58e5-f8b2bda92564}" -2024-05-12T20:49:59.095Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:59.202Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:59.095475025 +0000 UTC m=+10.376175320 (1 nodes), 1.303µs from start of poll, 195.912143ms from cluster start. Timeout at 2024-05-12 20:49:59.202841972 +0000 UTC m=+10.483542658, 107.367338ms after stability -2024-05-12T20:49:59.203Z [INFO] server-75590a3f-ee04-db05-952b-916fa9d36a32: aborting pipeline replication: peer="{Voter server-9f7f4c65-0ad6-1b7b-58e5-f8b2bda92564 9f7f4c65-0ad6-1b7b-58e5-f8b2bda92564}" -2024-05-12T20:49:59.203Z [INFO] server-75590a3f-ee04-db05-952b-916fa9d36a32: aborting pipeline replication: peer="{Voter server-3cafcfca-19df-9a7e-243a-3e18ea55a346 3cafcfca-19df-9a7e-243a-3e18ea55a346}" ---- PASS: TestRaft_JoinNode (0.31s) +2025-06-15T03:16:33.017Z [INFO] server-fb08b91e-4f69-ecc5-3329-c2f05f3411b9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fb08b91e-4f69-ecc5-3329-c2f05f3411b9 Address:fb08b91e-4f69-ecc5-3329-c2f05f3411b9} {Suffrage:Voter ID:server-61f5799d-2efe-480a-41bb-48eff232e732 Address:61f5799d-2efe-480a-41bb-48eff232e732}]" +2025-06-15T03:16:33.019Z [INFO] server-fb08b91e-4f69-ecc5-3329-c2f05f3411b9: entering follower state: follower="Node at fb08b91e-4f69-ecc5-3329-c2f05f3411b9 [Follower]" leader-address= leader-id= +2025-06-15T03:16:33.020Z [INFO] server-61f5799d-2efe-480a-41bb-48eff232e732: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fb08b91e-4f69-ecc5-3329-c2f05f3411b9 Address:fb08b91e-4f69-ecc5-3329-c2f05f3411b9} {Suffrage:Voter ID:server-61f5799d-2efe-480a-41bb-48eff232e732 Address:61f5799d-2efe-480a-41bb-48eff232e732}]" +2025-06-15T03:16:33.020Z [INFO] server-61f5799d-2efe-480a-41bb-48eff232e732: entering follower state: follower="Node at 61f5799d-2efe-480a-41bb-48eff232e732 [Follower]" leader-address= leader-id= +2025-06-15T03:16:33.020Z [INFO] server-b545dc3c-c784-3dc9-b808-b8a3464ac197: initial configuration: index=0 servers=[] +2025-06-15T03:16:33.020Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:33.020Z [INFO] server-b545dc3c-c784-3dc9-b808-b8a3464ac197: entering follower state: follower="Node at b545dc3c-c784-3dc9-b808-b8a3464ac197 [Follower]" leader-address= leader-id= +2025-06-15T03:16:33.075Z [WARN] server-61f5799d-2efe-480a-41bb-48eff232e732: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:33.075Z [INFO] server-61f5799d-2efe-480a-41bb-48eff232e732: entering candidate state: node="Node at 61f5799d-2efe-480a-41bb-48eff232e732 [Candidate]" term=2 +2025-06-15T03:16:33.075Z [INFO] server-61f5799d-2efe-480a-41bb-48eff232e732: election won: term=2 tally=2 +2025-06-15T03:16:33.075Z [INFO] server-61f5799d-2efe-480a-41bb-48eff232e732: entering leader state: leader="Node at 61f5799d-2efe-480a-41bb-48eff232e732 [Leader]" +2025-06-15T03:16:33.075Z [INFO] server-61f5799d-2efe-480a-41bb-48eff232e732: added peer, starting replication: peer=server-fb08b91e-4f69-ecc5-3329-c2f05f3411b9 +2025-06-15T03:16:33.075Z [INFO] server-61f5799d-2efe-480a-41bb-48eff232e732: pipelining replication: peer="{Voter server-fb08b91e-4f69-ecc5-3329-c2f05f3411b9 fb08b91e-4f69-ecc5-3329-c2f05f3411b9}" +2025-06-15T03:16:33.093Z [WARN] server-b545dc3c-c784-3dc9-b808-b8a3464ac197: no known peers, aborting election +2025-06-15T03:16:33.181Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:33.075483579 +0000 UTC m=+8.787161754 (1 nodes), 54.663061ms from start of poll, 58.105356ms from cluster start. Timeout at 2025-06-15 03:16:33.181164152 +0000 UTC m=+8.892842338, 105.680584ms after stability +2025-06-15T03:16:33.181Z [INFO] server-61f5799d-2efe-480a-41bb-48eff232e732: updating configuration: command=AddVoter server-id=server-b545dc3c-c784-3dc9-b808-b8a3464ac197 server-addr=b545dc3c-c784-3dc9-b808-b8a3464ac197 servers="[{Suffrage:Voter ID:server-fb08b91e-4f69-ecc5-3329-c2f05f3411b9 Address:fb08b91e-4f69-ecc5-3329-c2f05f3411b9} {Suffrage:Voter ID:server-61f5799d-2efe-480a-41bb-48eff232e732 Address:61f5799d-2efe-480a-41bb-48eff232e732} {Suffrage:Voter ID:server-b545dc3c-c784-3dc9-b808-b8a3464ac197 Address:b545dc3c-c784-3dc9-b808-b8a3464ac197}]" +2025-06-15T03:16:33.181Z [INFO] server-61f5799d-2efe-480a-41bb-48eff232e732: added peer, starting replication: peer=server-b545dc3c-c784-3dc9-b808-b8a3464ac197 +2025-06-15T03:16:33.181Z [WARN] server-b545dc3c-c784-3dc9-b808-b8a3464ac197: failed to get previous log: previous-index=3 last-index=0 error="log not found" +2025-06-15T03:16:33.181Z [WARN] server-61f5799d-2efe-480a-41bb-48eff232e732: appendEntries rejected, sending older logs: peer="{Voter server-b545dc3c-c784-3dc9-b808-b8a3464ac197 b545dc3c-c784-3dc9-b808-b8a3464ac197}" next=1 +2025-06-15T03:16:33.181Z [INFO] server-61f5799d-2efe-480a-41bb-48eff232e732: pipelining replication: peer="{Voter server-b545dc3c-c784-3dc9-b808-b8a3464ac197 b545dc3c-c784-3dc9-b808-b8a3464ac197}" +2025-06-15T03:16:33.181Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:33.287Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:33.181772788 +0000 UTC m=+8.893450950 (1 nodes), 885ns from start of poll, 164.394552ms from cluster start. Timeout at 2025-06-15 03:16:33.287813363 +0000 UTC m=+8.999491547, 106.040597ms after stability +2025-06-15T03:16:33.287Z [INFO] server-61f5799d-2efe-480a-41bb-48eff232e732: aborting pipeline replication: peer="{Voter server-b545dc3c-c784-3dc9-b808-b8a3464ac197 b545dc3c-c784-3dc9-b808-b8a3464ac197}" +2025-06-15T03:16:33.288Z [INFO] server-61f5799d-2efe-480a-41bb-48eff232e732: aborting pipeline replication: peer="{Voter server-fb08b91e-4f69-ecc5-3329-c2f05f3411b9 fb08b91e-4f69-ecc5-3329-c2f05f3411b9}" +--- PASS: TestRaft_JoinNode (0.27s) === RUN TestRaft_JoinNode_ConfigStore -2024-05-12T20:49:59.204Z [INFO] server-4ab44649-9000-e0c6-2fd2-72effcd32df0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ab44649-9000-e0c6-2fd2-72effcd32df0 Address:4ab44649-9000-e0c6-2fd2-72effcd32df0}]" -2024-05-12T20:49:59.204Z [INFO] server-4ab44649-9000-e0c6-2fd2-72effcd32df0: entering follower state: follower="Node at 4ab44649-9000-e0c6-2fd2-72effcd32df0 [Follower]" leader-address= leader-id= -2024-05-12T20:49:59.204Z [INFO] server-34a2a277-5396-f6a5-7d57-a0731919b9dd: initial configuration: index=0 servers=[] -2024-05-12T20:49:59.204Z [INFO] server-34a2a277-5396-f6a5-7d57-a0731919b9dd: entering follower state: follower="Node at 34a2a277-5396-f6a5-7d57-a0731919b9dd [Follower]" leader-address= leader-id= -2024-05-12T20:49:59.205Z [INFO] server-0433de85-8f1d-43cc-a24c-8bb74fc405ad: initial configuration: index=0 servers=[] -2024-05-12T20:49:59.205Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:59.205Z [INFO] server-0433de85-8f1d-43cc-a24c-8bb74fc405ad: entering follower state: follower="Node at 0433de85-8f1d-43cc-a24c-8bb74fc405ad [Follower]" leader-address= leader-id= -2024-05-12T20:49:59.258Z [WARN] server-4ab44649-9000-e0c6-2fd2-72effcd32df0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:59.258Z [INFO] server-4ab44649-9000-e0c6-2fd2-72effcd32df0: entering candidate state: node="Node at 4ab44649-9000-e0c6-2fd2-72effcd32df0 [Candidate]" term=2 -2024-05-12T20:49:59.259Z [INFO] server-4ab44649-9000-e0c6-2fd2-72effcd32df0: election won: term=2 tally=1 -2024-05-12T20:49:59.259Z [INFO] server-4ab44649-9000-e0c6-2fd2-72effcd32df0: entering leader state: leader="Node at 4ab44649-9000-e0c6-2fd2-72effcd32df0 [Leader]" -2024-05-12T20:49:59.270Z [WARN] server-34a2a277-5396-f6a5-7d57-a0731919b9dd: no known peers, aborting election -2024-05-12T20:49:59.282Z [WARN] server-0433de85-8f1d-43cc-a24c-8bb74fc405ad: no known peers, aborting election -2024-05-12T20:49:59.365Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:59.259278838 +0000 UTC m=+10.539979261 (1 nodes), 54.115982ms from start of poll, 54.98394ms from cluster start. Timeout at 2024-05-12 20:49:59.365412276 +0000 UTC m=+10.646112683, 106.133422ms after stability -2024-05-12T20:49:59.365Z [INFO] server-4ab44649-9000-e0c6-2fd2-72effcd32df0: updating configuration: command=AddVoter server-id=server-34a2a277-5396-f6a5-7d57-a0731919b9dd server-addr=34a2a277-5396-f6a5-7d57-a0731919b9dd servers="[{Suffrage:Voter ID:server-4ab44649-9000-e0c6-2fd2-72effcd32df0 Address:4ab44649-9000-e0c6-2fd2-72effcd32df0} {Suffrage:Voter ID:server-34a2a277-5396-f6a5-7d57-a0731919b9dd Address:34a2a277-5396-f6a5-7d57-a0731919b9dd}]" -2024-05-12T20:49:59.365Z [INFO] server-4ab44649-9000-e0c6-2fd2-72effcd32df0: added peer, starting replication: peer=server-34a2a277-5396-f6a5-7d57-a0731919b9dd -2024-05-12T20:49:59.365Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:59.365Z [WARN] server-34a2a277-5396-f6a5-7d57-a0731919b9dd: failed to get previous log: previous-index=3 last-index=0 error="log not found" -2024-05-12T20:49:59.366Z [WARN] server-4ab44649-9000-e0c6-2fd2-72effcd32df0: appendEntries rejected, sending older logs: peer="{Voter server-34a2a277-5396-f6a5-7d57-a0731919b9dd 34a2a277-5396-f6a5-7d57-a0731919b9dd}" next=1 -2024-05-12T20:49:59.366Z [INFO] server-4ab44649-9000-e0c6-2fd2-72effcd32df0: pipelining replication: peer="{Voter server-34a2a277-5396-f6a5-7d57-a0731919b9dd 34a2a277-5396-f6a5-7d57-a0731919b9dd}" -2024-05-12T20:49:59.470Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:59.36587557 +0000 UTC m=+10.646575869 (1 nodes), 1.75µs from start of poll, 161.580548ms from cluster start. Timeout at 2024-05-12 20:49:59.470910945 +0000 UTC m=+10.751611317, 105.035448ms after stability -2024-05-12T20:49:59.471Z [INFO] server-4ab44649-9000-e0c6-2fd2-72effcd32df0: updating configuration: command=AddVoter server-id=server-0433de85-8f1d-43cc-a24c-8bb74fc405ad server-addr=0433de85-8f1d-43cc-a24c-8bb74fc405ad servers="[{Suffrage:Voter ID:server-4ab44649-9000-e0c6-2fd2-72effcd32df0 Address:4ab44649-9000-e0c6-2fd2-72effcd32df0} {Suffrage:Voter ID:server-34a2a277-5396-f6a5-7d57-a0731919b9dd Address:34a2a277-5396-f6a5-7d57-a0731919b9dd} {Suffrage:Voter ID:server-0433de85-8f1d-43cc-a24c-8bb74fc405ad Address:0433de85-8f1d-43cc-a24c-8bb74fc405ad}]" -2024-05-12T20:49:59.471Z [INFO] server-4ab44649-9000-e0c6-2fd2-72effcd32df0: added peer, starting replication: peer=server-0433de85-8f1d-43cc-a24c-8bb74fc405ad -2024-05-12T20:49:59.471Z [WARN] server-0433de85-8f1d-43cc-a24c-8bb74fc405ad: failed to get previous log: previous-index=4 last-index=0 error="log not found" -2024-05-12T20:49:59.471Z [WARN] server-4ab44649-9000-e0c6-2fd2-72effcd32df0: appendEntries rejected, sending older logs: peer="{Voter server-0433de85-8f1d-43cc-a24c-8bb74fc405ad 0433de85-8f1d-43cc-a24c-8bb74fc405ad}" next=1 -2024-05-12T20:49:59.471Z [INFO] server-4ab44649-9000-e0c6-2fd2-72effcd32df0: pipelining replication: peer="{Voter server-0433de85-8f1d-43cc-a24c-8bb74fc405ad 0433de85-8f1d-43cc-a24c-8bb74fc405ad}" -2024-05-12T20:49:59.471Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:59.577Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:59.471747209 +0000 UTC m=+10.752447579 (1 nodes), 1.739µs from start of poll, 267.452258ms from cluster start. Timeout at 2024-05-12 20:49:59.577223584 +0000 UTC m=+10.857923999, 105.47642ms after stability -2024-05-12T20:49:59.577Z [INFO] server-4ab44649-9000-e0c6-2fd2-72effcd32df0: aborting pipeline replication: peer="{Voter server-0433de85-8f1d-43cc-a24c-8bb74fc405ad 0433de85-8f1d-43cc-a24c-8bb74fc405ad}" -2024-05-12T20:49:59.577Z [INFO] server-4ab44649-9000-e0c6-2fd2-72effcd32df0: aborting pipeline replication: peer="{Voter server-34a2a277-5396-f6a5-7d57-a0731919b9dd 34a2a277-5396-f6a5-7d57-a0731919b9dd}" ---- PASS: TestRaft_JoinNode_ConfigStore (0.37s) +2025-06-15T03:16:33.289Z [INFO] server-c97ea98e-715c-04ac-3f94-50b31c46906c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c97ea98e-715c-04ac-3f94-50b31c46906c Address:c97ea98e-715c-04ac-3f94-50b31c46906c}]" +2025-06-15T03:16:33.289Z [INFO] server-c90f4d63-9865-5be7-b99a-2b7acb4a560d: initial configuration: index=0 servers=[] +2025-06-15T03:16:33.289Z [INFO] server-5d716974-616d-14c5-4d20-cd5ed2c388de: initial configuration: index=0 servers=[] +2025-06-15T03:16:33.289Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:33.289Z [INFO] server-c97ea98e-715c-04ac-3f94-50b31c46906c: entering follower state: follower="Node at c97ea98e-715c-04ac-3f94-50b31c46906c [Follower]" leader-address= leader-id= +2025-06-15T03:16:33.290Z [INFO] server-c90f4d63-9865-5be7-b99a-2b7acb4a560d: entering follower state: follower="Node at c90f4d63-9865-5be7-b99a-2b7acb4a560d [Follower]" leader-address= leader-id= +2025-06-15T03:16:33.290Z [INFO] server-5d716974-616d-14c5-4d20-cd5ed2c388de: entering follower state: follower="Node at 5d716974-616d-14c5-4d20-cd5ed2c388de [Follower]" leader-address= leader-id= +2025-06-15T03:16:33.345Z [WARN] server-c90f4d63-9865-5be7-b99a-2b7acb4a560d: no known peers, aborting election +2025-06-15T03:16:33.356Z [WARN] server-5d716974-616d-14c5-4d20-cd5ed2c388de: no known peers, aborting election +2025-06-15T03:16:33.386Z [WARN] server-c97ea98e-715c-04ac-3f94-50b31c46906c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:33.386Z [INFO] server-c97ea98e-715c-04ac-3f94-50b31c46906c: entering candidate state: node="Node at c97ea98e-715c-04ac-3f94-50b31c46906c [Candidate]" term=2 +2025-06-15T03:16:33.386Z [INFO] server-c97ea98e-715c-04ac-3f94-50b31c46906c: election won: term=2 tally=1 +2025-06-15T03:16:33.386Z [INFO] server-c97ea98e-715c-04ac-3f94-50b31c46906c: entering leader state: leader="Node at c97ea98e-715c-04ac-3f94-50b31c46906c [Leader]" +2025-06-15T03:16:33.492Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:33.38694459 +0000 UTC m=+9.098622770 (1 nodes), 97.048372ms from start of poll, 97.880404ms from cluster start. Timeout at 2025-06-15 03:16:33.492573565 +0000 UTC m=+9.204251739, 105.628969ms after stability +2025-06-15T03:16:33.492Z [INFO] server-c97ea98e-715c-04ac-3f94-50b31c46906c: updating configuration: command=AddVoter server-id=server-c90f4d63-9865-5be7-b99a-2b7acb4a560d server-addr=c90f4d63-9865-5be7-b99a-2b7acb4a560d servers="[{Suffrage:Voter ID:server-c97ea98e-715c-04ac-3f94-50b31c46906c Address:c97ea98e-715c-04ac-3f94-50b31c46906c} {Suffrage:Voter ID:server-c90f4d63-9865-5be7-b99a-2b7acb4a560d Address:c90f4d63-9865-5be7-b99a-2b7acb4a560d}]" +2025-06-15T03:16:33.492Z [INFO] server-c97ea98e-715c-04ac-3f94-50b31c46906c: added peer, starting replication: peer=server-c90f4d63-9865-5be7-b99a-2b7acb4a560d +2025-06-15T03:16:33.492Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:33.492Z [WARN] server-c90f4d63-9865-5be7-b99a-2b7acb4a560d: failed to get previous log: previous-index=3 last-index=0 error="log not found" +2025-06-15T03:16:33.493Z [WARN] server-c97ea98e-715c-04ac-3f94-50b31c46906c: appendEntries rejected, sending older logs: peer="{Voter server-c90f4d63-9865-5be7-b99a-2b7acb4a560d c90f4d63-9865-5be7-b99a-2b7acb4a560d}" next=1 +2025-06-15T03:16:33.493Z [INFO] server-c97ea98e-715c-04ac-3f94-50b31c46906c: pipelining replication: peer="{Voter server-c90f4d63-9865-5be7-b99a-2b7acb4a560d c90f4d63-9865-5be7-b99a-2b7acb4a560d}" +2025-06-15T03:16:33.598Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:33.492927457 +0000 UTC m=+9.204605634 (1 nodes), 864ns from start of poll, 203.863268ms from cluster start. Timeout at 2025-06-15 03:16:33.598871413 +0000 UTC m=+9.310549606, 105.943972ms after stability +2025-06-15T03:16:33.598Z [INFO] server-c97ea98e-715c-04ac-3f94-50b31c46906c: updating configuration: command=AddVoter server-id=server-5d716974-616d-14c5-4d20-cd5ed2c388de server-addr=5d716974-616d-14c5-4d20-cd5ed2c388de servers="[{Suffrage:Voter ID:server-c97ea98e-715c-04ac-3f94-50b31c46906c Address:c97ea98e-715c-04ac-3f94-50b31c46906c} {Suffrage:Voter ID:server-c90f4d63-9865-5be7-b99a-2b7acb4a560d Address:c90f4d63-9865-5be7-b99a-2b7acb4a560d} {Suffrage:Voter ID:server-5d716974-616d-14c5-4d20-cd5ed2c388de Address:5d716974-616d-14c5-4d20-cd5ed2c388de}]" +2025-06-15T03:16:33.599Z [INFO] server-c97ea98e-715c-04ac-3f94-50b31c46906c: added peer, starting replication: peer=server-5d716974-616d-14c5-4d20-cd5ed2c388de +2025-06-15T03:16:33.599Z [WARN] server-5d716974-616d-14c5-4d20-cd5ed2c388de: failed to get previous log: previous-index=4 last-index=0 error="log not found" +2025-06-15T03:16:33.599Z [WARN] server-c97ea98e-715c-04ac-3f94-50b31c46906c: appendEntries rejected, sending older logs: peer="{Voter server-5d716974-616d-14c5-4d20-cd5ed2c388de 5d716974-616d-14c5-4d20-cd5ed2c388de}" next=1 +2025-06-15T03:16:33.599Z [INFO] server-c97ea98e-715c-04ac-3f94-50b31c46906c: pipelining replication: peer="{Voter server-5d716974-616d-14c5-4d20-cd5ed2c388de 5d716974-616d-14c5-4d20-cd5ed2c388de}" +2025-06-15T03:16:33.599Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:33.705Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:33.599555858 +0000 UTC m=+9.311234027 (1 nodes), 916ns from start of poll, 310.491661ms from cluster start. Timeout at 2025-06-15 03:16:33.705534059 +0000 UTC m=+9.417212249, 105.978222ms after stability +2025-06-15T03:16:33.705Z [INFO] server-c97ea98e-715c-04ac-3f94-50b31c46906c: aborting pipeline replication: peer="{Voter server-5d716974-616d-14c5-4d20-cd5ed2c388de 5d716974-616d-14c5-4d20-cd5ed2c388de}" +2025-06-15T03:16:33.705Z [INFO] server-c97ea98e-715c-04ac-3f94-50b31c46906c: aborting pipeline replication: peer="{Voter server-c90f4d63-9865-5be7-b99a-2b7acb4a560d c90f4d63-9865-5be7-b99a-2b7acb4a560d}" +--- PASS: TestRaft_JoinNode_ConfigStore (0.42s) === RUN TestRaft_RemoveFollower -2024-05-12T20:49:59.579Z [INFO] server-cfd13c23-f0ad-61df-513d-01869b08d27c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cfd13c23-f0ad-61df-513d-01869b08d27c Address:cfd13c23-f0ad-61df-513d-01869b08d27c} {Suffrage:Voter ID:server-6233f2c8-e14a-f975-7b04-aa6ea11e3c9f Address:6233f2c8-e14a-f975-7b04-aa6ea11e3c9f} {Suffrage:Voter ID:server-75c113ab-77c3-2260-9b85-868121da3060 Address:75c113ab-77c3-2260-9b85-868121da3060}]" -2024-05-12T20:49:59.579Z [INFO] server-6233f2c8-e14a-f975-7b04-aa6ea11e3c9f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cfd13c23-f0ad-61df-513d-01869b08d27c Address:cfd13c23-f0ad-61df-513d-01869b08d27c} {Suffrage:Voter ID:server-6233f2c8-e14a-f975-7b04-aa6ea11e3c9f Address:6233f2c8-e14a-f975-7b04-aa6ea11e3c9f} {Suffrage:Voter ID:server-75c113ab-77c3-2260-9b85-868121da3060 Address:75c113ab-77c3-2260-9b85-868121da3060}]" -2024-05-12T20:49:59.579Z [INFO] server-75c113ab-77c3-2260-9b85-868121da3060: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cfd13c23-f0ad-61df-513d-01869b08d27c Address:cfd13c23-f0ad-61df-513d-01869b08d27c} {Suffrage:Voter ID:server-6233f2c8-e14a-f975-7b04-aa6ea11e3c9f Address:6233f2c8-e14a-f975-7b04-aa6ea11e3c9f} {Suffrage:Voter ID:server-75c113ab-77c3-2260-9b85-868121da3060 Address:75c113ab-77c3-2260-9b85-868121da3060}]" -2024-05-12T20:49:59.579Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:59.579Z [INFO] server-cfd13c23-f0ad-61df-513d-01869b08d27c: entering follower state: follower="Node at cfd13c23-f0ad-61df-513d-01869b08d27c [Follower]" leader-address= leader-id= -2024-05-12T20:49:59.579Z [INFO] server-6233f2c8-e14a-f975-7b04-aa6ea11e3c9f: entering follower state: follower="Node at 6233f2c8-e14a-f975-7b04-aa6ea11e3c9f [Follower]" leader-address= leader-id= -2024-05-12T20:49:59.579Z [INFO] server-75c113ab-77c3-2260-9b85-868121da3060: entering follower state: follower="Node at 75c113ab-77c3-2260-9b85-868121da3060 [Follower]" leader-address= leader-id= -2024-05-12T20:49:59.638Z [WARN] server-6233f2c8-e14a-f975-7b04-aa6ea11e3c9f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:49:59.638Z [INFO] server-6233f2c8-e14a-f975-7b04-aa6ea11e3c9f: entering candidate state: node="Node at 6233f2c8-e14a-f975-7b04-aa6ea11e3c9f [Candidate]" term=2 -2024-05-12T20:49:59.639Z [INFO] server-6233f2c8-e14a-f975-7b04-aa6ea11e3c9f: election won: term=2 tally=2 -2024-05-12T20:49:59.639Z [INFO] server-6233f2c8-e14a-f975-7b04-aa6ea11e3c9f: entering leader state: leader="Node at 6233f2c8-e14a-f975-7b04-aa6ea11e3c9f [Leader]" -2024-05-12T20:49:59.639Z [INFO] server-6233f2c8-e14a-f975-7b04-aa6ea11e3c9f: added peer, starting replication: peer=server-cfd13c23-f0ad-61df-513d-01869b08d27c -2024-05-12T20:49:59.639Z [INFO] server-6233f2c8-e14a-f975-7b04-aa6ea11e3c9f: added peer, starting replication: peer=server-75c113ab-77c3-2260-9b85-868121da3060 -2024-05-12T20:49:59.639Z [INFO] server-6233f2c8-e14a-f975-7b04-aa6ea11e3c9f: pipelining replication: peer="{Voter server-75c113ab-77c3-2260-9b85-868121da3060 75c113ab-77c3-2260-9b85-868121da3060}" -2024-05-12T20:49:59.639Z [INFO] server-6233f2c8-e14a-f975-7b04-aa6ea11e3c9f: pipelining replication: peer="{Voter server-cfd13c23-f0ad-61df-513d-01869b08d27c cfd13c23-f0ad-61df-513d-01869b08d27c}" -2024-05-12T20:49:59.746Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:49:59.639323723 +0000 UTC m=+10.920024209 (1 nodes), 59.704477ms from start of poll, 60.123073ms from cluster start. Timeout at 2024-05-12 20:49:59.746768032 +0000 UTC m=+11.027468233, 107.444024ms after stability -2024-05-12T20:49:59.754Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:49:59.862Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:49:59.754803374 +0000 UTC m=+11.035503269 (2 nodes), 8.246µs from start of poll, 175.602133ms from cluster start. Timeout at 2024-05-12 20:49:59.862772038 +0000 UTC m=+11.143472833, 107.969564ms after stability -2024-05-12T20:49:59.862Z [INFO] server-6233f2c8-e14a-f975-7b04-aa6ea11e3c9f: updating configuration: command=RemoveServer server-id=server-cfd13c23-f0ad-61df-513d-01869b08d27c server-addr= servers="[{Suffrage:Voter ID:server-6233f2c8-e14a-f975-7b04-aa6ea11e3c9f Address:6233f2c8-e14a-f975-7b04-aa6ea11e3c9f} {Suffrage:Voter ID:server-75c113ab-77c3-2260-9b85-868121da3060 Address:75c113ab-77c3-2260-9b85-868121da3060}]" -2024-05-12T20:49:59.862Z [INFO] server-6233f2c8-e14a-f975-7b04-aa6ea11e3c9f: removed peer, stopping replication: peer=server-cfd13c23-f0ad-61df-513d-01869b08d27c last-index=3 -2024-05-12T20:49:59.863Z [INFO] server-6233f2c8-e14a-f975-7b04-aa6ea11e3c9f: aborting pipeline replication: peer="{Voter server-cfd13c23-f0ad-61df-513d-01869b08d27c cfd13c23-f0ad-61df-513d-01869b08d27c}" -2024-05-12T20:49:59.970Z [INFO] server-6233f2c8-e14a-f975-7b04-aa6ea11e3c9f: aborting pipeline replication: peer="{Voter server-75c113ab-77c3-2260-9b85-868121da3060 75c113ab-77c3-2260-9b85-868121da3060}" +2025-06-15T03:16:33.707Z [INFO] server-646a6ac5-a704-a207-8399-5cfedf9f7735: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-646a6ac5-a704-a207-8399-5cfedf9f7735 Address:646a6ac5-a704-a207-8399-5cfedf9f7735} {Suffrage:Voter ID:server-7ff63524-1007-a60f-9d88-d7f5dcfe301f Address:7ff63524-1007-a60f-9d88-d7f5dcfe301f} {Suffrage:Voter ID:server-ad7d89ba-ecf8-96e1-cfbd-1986a136efaf Address:ad7d89ba-ecf8-96e1-cfbd-1986a136efaf}]" +2025-06-15T03:16:33.707Z [INFO] server-7ff63524-1007-a60f-9d88-d7f5dcfe301f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-646a6ac5-a704-a207-8399-5cfedf9f7735 Address:646a6ac5-a704-a207-8399-5cfedf9f7735} {Suffrage:Voter ID:server-7ff63524-1007-a60f-9d88-d7f5dcfe301f Address:7ff63524-1007-a60f-9d88-d7f5dcfe301f} {Suffrage:Voter ID:server-ad7d89ba-ecf8-96e1-cfbd-1986a136efaf Address:ad7d89ba-ecf8-96e1-cfbd-1986a136efaf}]" +2025-06-15T03:16:33.707Z [INFO] server-ad7d89ba-ecf8-96e1-cfbd-1986a136efaf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-646a6ac5-a704-a207-8399-5cfedf9f7735 Address:646a6ac5-a704-a207-8399-5cfedf9f7735} {Suffrage:Voter ID:server-7ff63524-1007-a60f-9d88-d7f5dcfe301f Address:7ff63524-1007-a60f-9d88-d7f5dcfe301f} {Suffrage:Voter ID:server-ad7d89ba-ecf8-96e1-cfbd-1986a136efaf Address:ad7d89ba-ecf8-96e1-cfbd-1986a136efaf}]" +2025-06-15T03:16:33.707Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:33.707Z [INFO] server-646a6ac5-a704-a207-8399-5cfedf9f7735: entering follower state: follower="Node at 646a6ac5-a704-a207-8399-5cfedf9f7735 [Follower]" leader-address= leader-id= +2025-06-15T03:16:33.707Z [INFO] server-7ff63524-1007-a60f-9d88-d7f5dcfe301f: entering follower state: follower="Node at 7ff63524-1007-a60f-9d88-d7f5dcfe301f [Follower]" leader-address= leader-id= +2025-06-15T03:16:33.707Z [INFO] server-ad7d89ba-ecf8-96e1-cfbd-1986a136efaf: entering follower state: follower="Node at ad7d89ba-ecf8-96e1-cfbd-1986a136efaf [Follower]" leader-address= leader-id= +2025-06-15T03:16:33.769Z [WARN] server-646a6ac5-a704-a207-8399-5cfedf9f7735: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:33.769Z [INFO] server-646a6ac5-a704-a207-8399-5cfedf9f7735: entering candidate state: node="Node at 646a6ac5-a704-a207-8399-5cfedf9f7735 [Candidate]" term=2 +2025-06-15T03:16:33.769Z [INFO] server-646a6ac5-a704-a207-8399-5cfedf9f7735: election won: term=2 tally=2 +2025-06-15T03:16:33.769Z [INFO] server-646a6ac5-a704-a207-8399-5cfedf9f7735: entering leader state: leader="Node at 646a6ac5-a704-a207-8399-5cfedf9f7735 [Leader]" +2025-06-15T03:16:33.769Z [INFO] server-646a6ac5-a704-a207-8399-5cfedf9f7735: added peer, starting replication: peer=server-7ff63524-1007-a60f-9d88-d7f5dcfe301f +2025-06-15T03:16:33.769Z [INFO] server-646a6ac5-a704-a207-8399-5cfedf9f7735: added peer, starting replication: peer=server-ad7d89ba-ecf8-96e1-cfbd-1986a136efaf +2025-06-15T03:16:33.769Z [INFO] server-646a6ac5-a704-a207-8399-5cfedf9f7735: pipelining replication: peer="{Voter server-ad7d89ba-ecf8-96e1-cfbd-1986a136efaf ad7d89ba-ecf8-96e1-cfbd-1986a136efaf}" +2025-06-15T03:16:33.769Z [INFO] server-646a6ac5-a704-a207-8399-5cfedf9f7735: pipelining replication: peer="{Voter server-7ff63524-1007-a60f-9d88-d7f5dcfe301f 7ff63524-1007-a60f-9d88-d7f5dcfe301f}" +2025-06-15T03:16:33.875Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:33.769913977 +0000 UTC m=+9.481592146 (1 nodes), 62.285871ms from start of poll, 62.712514ms from cluster start. Timeout at 2025-06-15 03:16:33.87498328 +0000 UTC m=+9.586661478, 105.069332ms after stability +2025-06-15T03:16:33.880Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:16:33.986Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:16:33.880776238 +0000 UTC m=+9.592454406 (2 nodes), 3.573µs from start of poll, 173.574774ms from cluster start. Timeout at 2025-06-15 03:16:33.986694009 +0000 UTC m=+9.698372203, 105.917797ms after stability +2025-06-15T03:16:33.986Z [INFO] server-646a6ac5-a704-a207-8399-5cfedf9f7735: updating configuration: command=RemoveServer server-id=server-7ff63524-1007-a60f-9d88-d7f5dcfe301f server-addr= servers="[{Suffrage:Voter ID:server-646a6ac5-a704-a207-8399-5cfedf9f7735 Address:646a6ac5-a704-a207-8399-5cfedf9f7735} {Suffrage:Voter ID:server-ad7d89ba-ecf8-96e1-cfbd-1986a136efaf Address:ad7d89ba-ecf8-96e1-cfbd-1986a136efaf}]" +2025-06-15T03:16:33.986Z [INFO] server-646a6ac5-a704-a207-8399-5cfedf9f7735: removed peer, stopping replication: peer=server-7ff63524-1007-a60f-9d88-d7f5dcfe301f last-index=3 +2025-06-15T03:16:33.986Z [INFO] server-646a6ac5-a704-a207-8399-5cfedf9f7735: aborting pipeline replication: peer="{Voter server-7ff63524-1007-a60f-9d88-d7f5dcfe301f 7ff63524-1007-a60f-9d88-d7f5dcfe301f}" +2025-06-15T03:16:34.081Z [WARN] server-7ff63524-1007-a60f-9d88-d7f5dcfe301f: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-06-15T03:16:34.092Z [INFO] server-646a6ac5-a704-a207-8399-5cfedf9f7735: aborting pipeline replication: peer="{Voter server-ad7d89ba-ecf8-96e1-cfbd-1986a136efaf ad7d89ba-ecf8-96e1-cfbd-1986a136efaf}" --- PASS: TestRaft_RemoveFollower (0.39s) === RUN TestRaft_RemoveLeader -2024-05-12T20:49:59.972Z [INFO] server-385b3c76-1edd-a1b4-483f-be06f769152a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-385b3c76-1edd-a1b4-483f-be06f769152a Address:385b3c76-1edd-a1b4-483f-be06f769152a} {Suffrage:Voter ID:server-a48241a6-dc20-3ff8-9e7c-3c841351d70a Address:a48241a6-dc20-3ff8-9e7c-3c841351d70a} {Suffrage:Voter ID:server-deee0673-e874-b6ee-33b7-a125275e92f9 Address:deee0673-e874-b6ee-33b7-a125275e92f9}]" -2024-05-12T20:49:59.972Z [INFO] server-a48241a6-dc20-3ff8-9e7c-3c841351d70a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-385b3c76-1edd-a1b4-483f-be06f769152a Address:385b3c76-1edd-a1b4-483f-be06f769152a} {Suffrage:Voter ID:server-a48241a6-dc20-3ff8-9e7c-3c841351d70a Address:a48241a6-dc20-3ff8-9e7c-3c841351d70a} {Suffrage:Voter ID:server-deee0673-e874-b6ee-33b7-a125275e92f9 Address:deee0673-e874-b6ee-33b7-a125275e92f9}]" -2024-05-12T20:49:59.972Z [INFO] server-deee0673-e874-b6ee-33b7-a125275e92f9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-385b3c76-1edd-a1b4-483f-be06f769152a Address:385b3c76-1edd-a1b4-483f-be06f769152a} {Suffrage:Voter ID:server-a48241a6-dc20-3ff8-9e7c-3c841351d70a Address:a48241a6-dc20-3ff8-9e7c-3c841351d70a} {Suffrage:Voter ID:server-deee0673-e874-b6ee-33b7-a125275e92f9 Address:deee0673-e874-b6ee-33b7-a125275e92f9}]" -2024-05-12T20:49:59.972Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:49:59.972Z [INFO] server-385b3c76-1edd-a1b4-483f-be06f769152a: entering follower state: follower="Node at 385b3c76-1edd-a1b4-483f-be06f769152a [Follower]" leader-address= leader-id= -2024-05-12T20:49:59.972Z [INFO] server-a48241a6-dc20-3ff8-9e7c-3c841351d70a: entering follower state: follower="Node at a48241a6-dc20-3ff8-9e7c-3c841351d70a [Follower]" leader-address= leader-id= -2024-05-12T20:49:59.973Z [INFO] server-deee0673-e874-b6ee-33b7-a125275e92f9: entering follower state: follower="Node at deee0673-e874-b6ee-33b7-a125275e92f9 [Follower]" leader-address= leader-id= -2024-05-12T20:50:00.046Z [WARN] server-385b3c76-1edd-a1b4-483f-be06f769152a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:00.046Z [INFO] server-385b3c76-1edd-a1b4-483f-be06f769152a: entering candidate state: node="Node at 385b3c76-1edd-a1b4-483f-be06f769152a [Candidate]" term=2 -2024-05-12T20:50:00.047Z [INFO] server-385b3c76-1edd-a1b4-483f-be06f769152a: election won: term=2 tally=2 -2024-05-12T20:50:00.047Z [INFO] server-385b3c76-1edd-a1b4-483f-be06f769152a: entering leader state: leader="Node at 385b3c76-1edd-a1b4-483f-be06f769152a [Leader]" -2024-05-12T20:50:00.047Z [INFO] server-385b3c76-1edd-a1b4-483f-be06f769152a: added peer, starting replication: peer=server-a48241a6-dc20-3ff8-9e7c-3c841351d70a -2024-05-12T20:50:00.047Z [INFO] server-385b3c76-1edd-a1b4-483f-be06f769152a: added peer, starting replication: peer=server-deee0673-e874-b6ee-33b7-a125275e92f9 -2024-05-12T20:50:00.047Z [INFO] server-385b3c76-1edd-a1b4-483f-be06f769152a: pipelining replication: peer="{Voter server-deee0673-e874-b6ee-33b7-a125275e92f9 deee0673-e874-b6ee-33b7-a125275e92f9}" -2024-05-12T20:50:00.047Z [INFO] server-385b3c76-1edd-a1b4-483f-be06f769152a: pipelining replication: peer="{Voter server-a48241a6-dc20-3ff8-9e7c-3c841351d70a a48241a6-dc20-3ff8-9e7c-3c841351d70a}" -2024-05-12T20:50:00.154Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:00.047520034 +0000 UTC m=+11.328220368 (1 nodes), 74.620321ms from start of poll, 74.971444ms from cluster start. Timeout at 2024-05-12 20:50:00.154759018 +0000 UTC m=+11.435459623, 107.239255ms after stability -2024-05-12T20:50:00.162Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:00.270Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:00.162953767 +0000 UTC m=+11.443654794 (2 nodes), 5.035µs from start of poll, 190.40587ms from cluster start. Timeout at 2024-05-12 20:50:00.270763269 +0000 UTC m=+11.551463777, 107.808983ms after stability -2024-05-12T20:50:00.270Z [INFO] server-385b3c76-1edd-a1b4-483f-be06f769152a: updating configuration: command=RemoveServer server-id=server-385b3c76-1edd-a1b4-483f-be06f769152a server-addr= servers="[{Suffrage:Voter ID:server-a48241a6-dc20-3ff8-9e7c-3c841351d70a Address:a48241a6-dc20-3ff8-9e7c-3c841351d70a} {Suffrage:Voter ID:server-deee0673-e874-b6ee-33b7-a125275e92f9 Address:deee0673-e874-b6ee-33b7-a125275e92f9}]" -2024-05-12T20:50:00.271Z [INFO] server-385b3c76-1edd-a1b4-483f-be06f769152a: removed ourself, shutting down -2024-05-12T20:50:00.271Z [INFO] server-385b3c76-1edd-a1b4-483f-be06f769152a: aborting pipeline replication: peer="{Voter server-deee0673-e874-b6ee-33b7-a125275e92f9 deee0673-e874-b6ee-33b7-a125275e92f9}" -2024-05-12T20:50:00.271Z [INFO] server-385b3c76-1edd-a1b4-483f-be06f769152a: aborting pipeline replication: peer="{Voter server-a48241a6-dc20-3ff8-9e7c-3c841351d70a a48241a6-dc20-3ff8-9e7c-3c841351d70a}" -2024-05-12T20:50:00.338Z [WARN] server-a48241a6-dc20-3ff8-9e7c-3c841351d70a: heartbeat timeout reached, starting election: last-leader-addr=385b3c76-1edd-a1b4-483f-be06f769152a last-leader-id=server-385b3c76-1edd-a1b4-483f-be06f769152a -2024-05-12T20:50:00.338Z [INFO] server-a48241a6-dc20-3ff8-9e7c-3c841351d70a: entering candidate state: node="Node at a48241a6-dc20-3ff8-9e7c-3c841351d70a [Candidate]" term=3 -2024-05-12T20:50:00.338Z [WARN] server-deee0673-e874-b6ee-33b7-a125275e92f9: rejecting vote request since we have a leader: from=a48241a6-dc20-3ff8-9e7c-3c841351d70a leader=385b3c76-1edd-a1b4-483f-be06f769152a leader-id=server-385b3c76-1edd-a1b4-483f-be06f769152a -2024-05-12T20:50:00.350Z [WARN] server-deee0673-e874-b6ee-33b7-a125275e92f9: heartbeat timeout reached, starting election: last-leader-addr=385b3c76-1edd-a1b4-483f-be06f769152a last-leader-id=server-385b3c76-1edd-a1b4-483f-be06f769152a -2024-05-12T20:50:00.350Z [INFO] server-deee0673-e874-b6ee-33b7-a125275e92f9: entering candidate state: node="Node at deee0673-e874-b6ee-33b7-a125275e92f9 [Candidate]" term=3 -2024-05-12T20:50:00.350Z [INFO] server-a48241a6-dc20-3ff8-9e7c-3c841351d70a: duplicate requestVote for same term: term=3 -2024-05-12T20:50:00.422Z [WARN] server-deee0673-e874-b6ee-33b7-a125275e92f9: Election timeout reached, restarting election -2024-05-12T20:50:00.422Z [INFO] server-deee0673-e874-b6ee-33b7-a125275e92f9: entering candidate state: node="Node at deee0673-e874-b6ee-33b7-a125275e92f9 [Candidate]" term=4 -2024-05-12T20:50:00.422Z [INFO] server-a48241a6-dc20-3ff8-9e7c-3c841351d70a: entering follower state: follower="Node at a48241a6-dc20-3ff8-9e7c-3c841351d70a [Follower]" leader-address= leader-id= -2024-05-12T20:50:00.422Z [INFO] server-deee0673-e874-b6ee-33b7-a125275e92f9: election won: term=4 tally=2 -2024-05-12T20:50:00.422Z [INFO] server-deee0673-e874-b6ee-33b7-a125275e92f9: entering leader state: leader="Node at deee0673-e874-b6ee-33b7-a125275e92f9 [Leader]" -2024-05-12T20:50:00.422Z [INFO] server-deee0673-e874-b6ee-33b7-a125275e92f9: added peer, starting replication: peer=server-a48241a6-dc20-3ff8-9e7c-3c841351d70a -2024-05-12T20:50:00.423Z [INFO] server-deee0673-e874-b6ee-33b7-a125275e92f9: pipelining replication: peer="{Voter server-a48241a6-dc20-3ff8-9e7c-3c841351d70a a48241a6-dc20-3ff8-9e7c-3c841351d70a}" -2024-05-12T20:50:00.486Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:00.594Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:00.486898841 +0000 UTC m=+11.767599127 (1 nodes), 74.476µs from start of poll, 514.350203ms from cluster start. Timeout at 2024-05-12 20:50:00.59477031 +0000 UTC m=+11.875470205, 107.871078ms after stability -2024-05-12T20:50:00.595Z [INFO] server-deee0673-e874-b6ee-33b7-a125275e92f9: aborting pipeline replication: peer="{Voter server-a48241a6-dc20-3ff8-9e7c-3c841351d70a a48241a6-dc20-3ff8-9e7c-3c841351d70a}" ---- PASS: TestRaft_RemoveLeader (0.63s) +2025-06-15T03:16:34.093Z [INFO] server-8452a01f-5b60-43b0-2abc-0252507ff151: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8452a01f-5b60-43b0-2abc-0252507ff151 Address:8452a01f-5b60-43b0-2abc-0252507ff151} {Suffrage:Voter ID:server-c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708 Address:c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708} {Suffrage:Voter ID:server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32 Address:da6e4d6e-3fba-87c3-c5e3-dc31ce464f32}]" +2025-06-15T03:16:34.094Z [INFO] server-c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8452a01f-5b60-43b0-2abc-0252507ff151 Address:8452a01f-5b60-43b0-2abc-0252507ff151} {Suffrage:Voter ID:server-c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708 Address:c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708} {Suffrage:Voter ID:server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32 Address:da6e4d6e-3fba-87c3-c5e3-dc31ce464f32}]" +2025-06-15T03:16:34.094Z [INFO] server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8452a01f-5b60-43b0-2abc-0252507ff151 Address:8452a01f-5b60-43b0-2abc-0252507ff151} {Suffrage:Voter ID:server-c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708 Address:c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708} {Suffrage:Voter ID:server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32 Address:da6e4d6e-3fba-87c3-c5e3-dc31ce464f32}]" +2025-06-15T03:16:34.094Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:34.094Z [INFO] server-8452a01f-5b60-43b0-2abc-0252507ff151: entering follower state: follower="Node at 8452a01f-5b60-43b0-2abc-0252507ff151 [Follower]" leader-address= leader-id= +2025-06-15T03:16:34.094Z [INFO] server-c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708: entering follower state: follower="Node at c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708 [Follower]" leader-address= leader-id= +2025-06-15T03:16:34.094Z [INFO] server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32: entering follower state: follower="Node at da6e4d6e-3fba-87c3-c5e3-dc31ce464f32 [Follower]" leader-address= leader-id= +2025-06-15T03:16:34.153Z [WARN] server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:34.154Z [INFO] server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32: entering candidate state: node="Node at da6e4d6e-3fba-87c3-c5e3-dc31ce464f32 [Candidate]" term=2 +2025-06-15T03:16:34.154Z [INFO] server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32: election won: term=2 tally=2 +2025-06-15T03:16:34.154Z [INFO] server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32: entering leader state: leader="Node at da6e4d6e-3fba-87c3-c5e3-dc31ce464f32 [Leader]" +2025-06-15T03:16:34.154Z [INFO] server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32: added peer, starting replication: peer=server-8452a01f-5b60-43b0-2abc-0252507ff151 +2025-06-15T03:16:34.154Z [INFO] server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32: added peer, starting replication: peer=server-c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708 +2025-06-15T03:16:34.154Z [INFO] server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32: pipelining replication: peer="{Voter server-c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708 c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708}" +2025-06-15T03:16:34.154Z [INFO] server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32: pipelining replication: peer="{Voter server-8452a01f-5b60-43b0-2abc-0252507ff151 8452a01f-5b60-43b0-2abc-0252507ff151}" +2025-06-15T03:16:34.260Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:34.154610769 +0000 UTC m=+9.866288951 (1 nodes), 60.265552ms from start of poll, 60.827327ms from cluster start. Timeout at 2025-06-15 03:16:34.26014937 +0000 UTC m=+9.971827572, 105.538621ms after stability +2025-06-15T03:16:34.265Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:16:34.372Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:16:34.265911893 +0000 UTC m=+9.977590061 (2 nodes), 2.414µs from start of poll, 172.128437ms from cluster start. Timeout at 2025-06-15 03:16:34.372089042 +0000 UTC m=+10.083767222, 106.177161ms after stability +2025-06-15T03:16:34.372Z [INFO] server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32: updating configuration: command=RemoveServer server-id=server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32 server-addr= servers="[{Suffrage:Voter ID:server-8452a01f-5b60-43b0-2abc-0252507ff151 Address:8452a01f-5b60-43b0-2abc-0252507ff151} {Suffrage:Voter ID:server-c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708 Address:c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708}]" +2025-06-15T03:16:34.372Z [INFO] server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32: removed ourself, shutting down +2025-06-15T03:16:34.372Z [INFO] server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32: aborting pipeline replication: peer="{Voter server-c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708 c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708}" +2025-06-15T03:16:34.372Z [INFO] server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32: aborting pipeline replication: peer="{Voter server-8452a01f-5b60-43b0-2abc-0252507ff151 8452a01f-5b60-43b0-2abc-0252507ff151}" +2025-06-15T03:16:34.483Z [WARN] server-8452a01f-5b60-43b0-2abc-0252507ff151: heartbeat timeout reached, starting election: last-leader-addr=da6e4d6e-3fba-87c3-c5e3-dc31ce464f32 last-leader-id=server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32 +2025-06-15T03:16:34.483Z [INFO] server-8452a01f-5b60-43b0-2abc-0252507ff151: entering candidate state: node="Node at 8452a01f-5b60-43b0-2abc-0252507ff151 [Candidate]" term=3 +2025-06-15T03:16:34.483Z [WARN] server-c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708: rejecting vote request since we have a leader: from=8452a01f-5b60-43b0-2abc-0252507ff151 leader=da6e4d6e-3fba-87c3-c5e3-dc31ce464f32 leader-id=server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32 +2025-06-15T03:16:34.511Z [WARN] server-c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708: heartbeat timeout reached, starting election: last-leader-addr=da6e4d6e-3fba-87c3-c5e3-dc31ce464f32 last-leader-id=server-da6e4d6e-3fba-87c3-c5e3-dc31ce464f32 +2025-06-15T03:16:34.511Z [INFO] server-c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708: entering candidate state: node="Node at c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708 [Candidate]" term=3 +2025-06-15T03:16:34.511Z [INFO] server-8452a01f-5b60-43b0-2abc-0252507ff151: duplicate requestVote for same term: term=3 +2025-06-15T03:16:34.562Z [WARN] server-8452a01f-5b60-43b0-2abc-0252507ff151: Election timeout reached, restarting election +2025-06-15T03:16:34.562Z [INFO] server-8452a01f-5b60-43b0-2abc-0252507ff151: entering candidate state: node="Node at 8452a01f-5b60-43b0-2abc-0252507ff151 [Candidate]" term=4 +2025-06-15T03:16:34.562Z [INFO] server-c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708: entering follower state: follower="Node at c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708 [Follower]" leader-address= leader-id= +2025-06-15T03:16:34.562Z [INFO] server-8452a01f-5b60-43b0-2abc-0252507ff151: election won: term=4 tally=2 +2025-06-15T03:16:34.562Z [INFO] server-8452a01f-5b60-43b0-2abc-0252507ff151: entering leader state: leader="Node at 8452a01f-5b60-43b0-2abc-0252507ff151 [Leader]" +2025-06-15T03:16:34.562Z [INFO] server-8452a01f-5b60-43b0-2abc-0252507ff151: added peer, starting replication: peer=server-c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708 +2025-06-15T03:16:34.562Z [INFO] server-8452a01f-5b60-43b0-2abc-0252507ff151: pipelining replication: peer="{Voter server-c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708 c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708}" +2025-06-15T03:16:34.587Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:34.692Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:34.587665592 +0000 UTC m=+10.299343776 (1 nodes), 46.944µs from start of poll, 493.882152ms from cluster start. Timeout at 2025-06-15 03:16:34.692778536 +0000 UTC m=+10.404456725, 105.112949ms after stability +2025-06-15T03:16:34.692Z [INFO] server-8452a01f-5b60-43b0-2abc-0252507ff151: aborting pipeline replication: peer="{Voter server-c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708 c6d2050c-1468-2d0a-8f2f-ff9d0ecdc708}" +--- PASS: TestRaft_RemoveLeader (0.60s) === RUN TestRaft_RemoveLeader_NoShutdown -2024-05-12T20:50:00.600Z [INFO] server-d4dfea85-48c1-b22e-b937-536f4c85843b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d4dfea85-48c1-b22e-b937-536f4c85843b Address:d4dfea85-48c1-b22e-b937-536f4c85843b} {Suffrage:Voter ID:server-6a975917-711b-becf-59b7-d565b4a5f081 Address:6a975917-711b-becf-59b7-d565b4a5f081} {Suffrage:Voter ID:server-14f5418e-b703-f79d-9955-c2ad8f3bcf59 Address:14f5418e-b703-f79d-9955-c2ad8f3bcf59}]" -2024-05-12T20:50:00.600Z [INFO] server-6a975917-711b-becf-59b7-d565b4a5f081: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d4dfea85-48c1-b22e-b937-536f4c85843b Address:d4dfea85-48c1-b22e-b937-536f4c85843b} {Suffrage:Voter ID:server-6a975917-711b-becf-59b7-d565b4a5f081 Address:6a975917-711b-becf-59b7-d565b4a5f081} {Suffrage:Voter ID:server-14f5418e-b703-f79d-9955-c2ad8f3bcf59 Address:14f5418e-b703-f79d-9955-c2ad8f3bcf59}]" -2024-05-12T20:50:00.600Z [INFO] server-14f5418e-b703-f79d-9955-c2ad8f3bcf59: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d4dfea85-48c1-b22e-b937-536f4c85843b Address:d4dfea85-48c1-b22e-b937-536f4c85843b} {Suffrage:Voter ID:server-6a975917-711b-becf-59b7-d565b4a5f081 Address:6a975917-711b-becf-59b7-d565b4a5f081} {Suffrage:Voter ID:server-14f5418e-b703-f79d-9955-c2ad8f3bcf59 Address:14f5418e-b703-f79d-9955-c2ad8f3bcf59}]" -2024-05-12T20:50:00.600Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:00.600Z [INFO] server-d4dfea85-48c1-b22e-b937-536f4c85843b: entering follower state: follower="Node at d4dfea85-48c1-b22e-b937-536f4c85843b [Follower]" leader-address= leader-id= -2024-05-12T20:50:00.600Z [INFO] server-6a975917-711b-becf-59b7-d565b4a5f081: entering follower state: follower="Node at 6a975917-711b-becf-59b7-d565b4a5f081 [Follower]" leader-address= leader-id= -2024-05-12T20:50:00.600Z [INFO] server-14f5418e-b703-f79d-9955-c2ad8f3bcf59: entering follower state: follower="Node at 14f5418e-b703-f79d-9955-c2ad8f3bcf59 [Follower]" leader-address= leader-id= -2024-05-12T20:50:00.670Z [WARN] server-14f5418e-b703-f79d-9955-c2ad8f3bcf59: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:00.670Z [INFO] server-14f5418e-b703-f79d-9955-c2ad8f3bcf59: entering candidate state: node="Node at 14f5418e-b703-f79d-9955-c2ad8f3bcf59 [Candidate]" term=2 -2024-05-12T20:50:00.671Z [INFO] server-14f5418e-b703-f79d-9955-c2ad8f3bcf59: election won: term=2 tally=2 -2024-05-12T20:50:00.671Z [INFO] server-14f5418e-b703-f79d-9955-c2ad8f3bcf59: entering leader state: leader="Node at 14f5418e-b703-f79d-9955-c2ad8f3bcf59 [Leader]" -2024-05-12T20:50:00.671Z [INFO] server-14f5418e-b703-f79d-9955-c2ad8f3bcf59: added peer, starting replication: peer=server-d4dfea85-48c1-b22e-b937-536f4c85843b -2024-05-12T20:50:00.671Z [INFO] server-14f5418e-b703-f79d-9955-c2ad8f3bcf59: added peer, starting replication: peer=server-6a975917-711b-becf-59b7-d565b4a5f081 -2024-05-12T20:50:00.671Z [INFO] server-14f5418e-b703-f79d-9955-c2ad8f3bcf59: pipelining replication: peer="{Voter server-6a975917-711b-becf-59b7-d565b4a5f081 6a975917-711b-becf-59b7-d565b4a5f081}" -2024-05-12T20:50:00.671Z [INFO] server-14f5418e-b703-f79d-9955-c2ad8f3bcf59: pipelining replication: peer="{Voter server-d4dfea85-48c1-b22e-b937-536f4c85843b d4dfea85-48c1-b22e-b937-536f4c85843b}" -2024-05-12T20:50:00.778Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:00.671748399 +0000 UTC m=+11.952448785 (2 nodes), 71.092434ms from start of poll, 71.502033ms from cluster start. Timeout at 2024-05-12 20:50:00.778753785 +0000 UTC m=+12.059455521, 107.006736ms after stability -2024-05-12T20:50:00.778Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:00.890Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:00.778866735 +0000 UTC m=+12.059566630 (1 nodes), 2.817µs from start of poll, 178.619878ms from cluster start. Timeout at 2024-05-12 20:50:00.89080047 +0000 UTC m=+12.171501184, 111.934554ms after stability -2024-05-12T20:50:00.892Z [INFO] server-14f5418e-b703-f79d-9955-c2ad8f3bcf59: updating configuration: command=RemoveServer server-id=server-14f5418e-b703-f79d-9955-c2ad8f3bcf59 server-addr= servers="[{Suffrage:Voter ID:server-d4dfea85-48c1-b22e-b937-536f4c85843b Address:d4dfea85-48c1-b22e-b937-536f4c85843b} {Suffrage:Voter ID:server-6a975917-711b-becf-59b7-d565b4a5f081 Address:6a975917-711b-becf-59b7-d565b4a5f081}]" -2024-05-12T20:50:00.899Z [INFO] server-14f5418e-b703-f79d-9955-c2ad8f3bcf59: removed ourself, transitioning to follower -2024-05-12T20:50:00.899Z [INFO] server-14f5418e-b703-f79d-9955-c2ad8f3bcf59: entering follower state: follower="Node at 14f5418e-b703-f79d-9955-c2ad8f3bcf59 [Follower]" leader-address= leader-id= -2024-05-12T20:50:00.899Z [INFO] server-14f5418e-b703-f79d-9955-c2ad8f3bcf59: aborting pipeline replication: peer="{Voter server-6a975917-711b-becf-59b7-d565b4a5f081 6a975917-711b-becf-59b7-d565b4a5f081}" -2024-05-12T20:50:00.900Z [INFO] server-14f5418e-b703-f79d-9955-c2ad8f3bcf59: aborting pipeline replication: peer="{Voter server-d4dfea85-48c1-b22e-b937-536f4c85843b d4dfea85-48c1-b22e-b937-536f4c85843b}" -2024-05-12T20:50:00.962Z [WARN] server-d4dfea85-48c1-b22e-b937-536f4c85843b: heartbeat timeout reached, starting election: last-leader-addr=14f5418e-b703-f79d-9955-c2ad8f3bcf59 last-leader-id=server-14f5418e-b703-f79d-9955-c2ad8f3bcf59 -2024-05-12T20:50:00.963Z [INFO] server-d4dfea85-48c1-b22e-b937-536f4c85843b: entering candidate state: node="Node at d4dfea85-48c1-b22e-b937-536f4c85843b [Candidate]" term=3 -2024-05-12T20:50:00.963Z [WARN] server-6a975917-711b-becf-59b7-d565b4a5f081: rejecting vote request since we have a leader: from=d4dfea85-48c1-b22e-b937-536f4c85843b leader=14f5418e-b703-f79d-9955-c2ad8f3bcf59 leader-id=server-14f5418e-b703-f79d-9955-c2ad8f3bcf59 -2024-05-12T20:50:00.970Z [WARN] server-14f5418e-b703-f79d-9955-c2ad8f3bcf59: not part of stable configuration, aborting election -2024-05-12T20:50:00.990Z [WARN] server-6a975917-711b-becf-59b7-d565b4a5f081: heartbeat timeout reached, starting election: last-leader-addr=14f5418e-b703-f79d-9955-c2ad8f3bcf59 last-leader-id=server-14f5418e-b703-f79d-9955-c2ad8f3bcf59 -2024-05-12T20:50:00.990Z [INFO] server-6a975917-711b-becf-59b7-d565b4a5f081: entering candidate state: node="Node at 6a975917-711b-becf-59b7-d565b4a5f081 [Candidate]" term=3 -2024-05-12T20:50:00.991Z [INFO] server-d4dfea85-48c1-b22e-b937-536f4c85843b: duplicate requestVote for same term: term=3 -2024-05-12T20:50:01.006Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:01.046Z [WARN] server-d4dfea85-48c1-b22e-b937-536f4c85843b: Election timeout reached, restarting election -2024-05-12T20:50:01.046Z [INFO] server-d4dfea85-48c1-b22e-b937-536f4c85843b: entering candidate state: node="Node at d4dfea85-48c1-b22e-b937-536f4c85843b [Candidate]" term=4 -2024-05-12T20:50:01.047Z [INFO] server-6a975917-711b-becf-59b7-d565b4a5f081: entering follower state: follower="Node at 6a975917-711b-becf-59b7-d565b4a5f081 [Follower]" leader-address= leader-id= -2024-05-12T20:50:01.047Z [INFO] server-d4dfea85-48c1-b22e-b937-536f4c85843b: election won: term=4 tally=2 -2024-05-12T20:50:01.047Z [INFO] server-d4dfea85-48c1-b22e-b937-536f4c85843b: entering leader state: leader="Node at d4dfea85-48c1-b22e-b937-536f4c85843b [Leader]" -2024-05-12T20:50:01.047Z [INFO] server-d4dfea85-48c1-b22e-b937-536f4c85843b: added peer, starting replication: peer=server-6a975917-711b-becf-59b7-d565b4a5f081 -2024-05-12T20:50:01.047Z [INFO] server-d4dfea85-48c1-b22e-b937-536f4c85843b: pipelining replication: peer="{Voter server-6a975917-711b-becf-59b7-d565b4a5f081 6a975917-711b-becf-59b7-d565b4a5f081}" -2024-05-12T20:50:01.154Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:01.047313568 +0000 UTC m=+12.328013887 (1 nodes), 40.413369ms from start of poll, 447.067135ms from cluster start. Timeout at 2024-05-12 20:50:01.154757253 +0000 UTC m=+12.435458935, 107.445048ms after stability -2024-05-12T20:50:01.262Z [INFO] server-d4dfea85-48c1-b22e-b937-536f4c85843b: aborting pipeline replication: peer="{Voter server-6a975917-711b-becf-59b7-d565b4a5f081 6a975917-711b-becf-59b7-d565b4a5f081}" ---- PASS: TestRaft_RemoveLeader_NoShutdown (0.66s) +2025-06-15T03:16:34.694Z [INFO] server-8619bc77-e506-330f-9705-6343be482e95: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8619bc77-e506-330f-9705-6343be482e95 Address:8619bc77-e506-330f-9705-6343be482e95} {Suffrage:Voter ID:server-af4423d5-ad9a-abe1-aea2-833e2682a2ea Address:af4423d5-ad9a-abe1-aea2-833e2682a2ea} {Suffrage:Voter ID:server-17217a5b-b0ad-7ab7-1707-53010c40a729 Address:17217a5b-b0ad-7ab7-1707-53010c40a729}]" +2025-06-15T03:16:34.694Z [INFO] server-af4423d5-ad9a-abe1-aea2-833e2682a2ea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8619bc77-e506-330f-9705-6343be482e95 Address:8619bc77-e506-330f-9705-6343be482e95} {Suffrage:Voter ID:server-af4423d5-ad9a-abe1-aea2-833e2682a2ea Address:af4423d5-ad9a-abe1-aea2-833e2682a2ea} {Suffrage:Voter ID:server-17217a5b-b0ad-7ab7-1707-53010c40a729 Address:17217a5b-b0ad-7ab7-1707-53010c40a729}]" +2025-06-15T03:16:34.694Z [INFO] server-af4423d5-ad9a-abe1-aea2-833e2682a2ea: entering follower state: follower="Node at af4423d5-ad9a-abe1-aea2-833e2682a2ea [Follower]" leader-address= leader-id= +2025-06-15T03:16:34.694Z [INFO] server-8619bc77-e506-330f-9705-6343be482e95: entering follower state: follower="Node at 8619bc77-e506-330f-9705-6343be482e95 [Follower]" leader-address= leader-id= +2025-06-15T03:16:34.694Z [INFO] server-17217a5b-b0ad-7ab7-1707-53010c40a729: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8619bc77-e506-330f-9705-6343be482e95 Address:8619bc77-e506-330f-9705-6343be482e95} {Suffrage:Voter ID:server-af4423d5-ad9a-abe1-aea2-833e2682a2ea Address:af4423d5-ad9a-abe1-aea2-833e2682a2ea} {Suffrage:Voter ID:server-17217a5b-b0ad-7ab7-1707-53010c40a729 Address:17217a5b-b0ad-7ab7-1707-53010c40a729}]" +2025-06-15T03:16:34.694Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:16:34.694Z [INFO] server-17217a5b-b0ad-7ab7-1707-53010c40a729: entering follower state: follower="Node at 17217a5b-b0ad-7ab7-1707-53010c40a729 [Follower]" leader-address= leader-id= +2025-06-15T03:16:34.758Z [WARN] server-8619bc77-e506-330f-9705-6343be482e95: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:34.758Z [INFO] server-8619bc77-e506-330f-9705-6343be482e95: entering candidate state: node="Node at 8619bc77-e506-330f-9705-6343be482e95 [Candidate]" term=2 +2025-06-15T03:16:34.758Z [INFO] server-8619bc77-e506-330f-9705-6343be482e95: election won: term=2 tally=2 +2025-06-15T03:16:34.758Z [INFO] server-8619bc77-e506-330f-9705-6343be482e95: entering leader state: leader="Node at 8619bc77-e506-330f-9705-6343be482e95 [Leader]" +2025-06-15T03:16:34.758Z [INFO] server-8619bc77-e506-330f-9705-6343be482e95: added peer, starting replication: peer=server-af4423d5-ad9a-abe1-aea2-833e2682a2ea +2025-06-15T03:16:34.758Z [INFO] server-8619bc77-e506-330f-9705-6343be482e95: added peer, starting replication: peer=server-17217a5b-b0ad-7ab7-1707-53010c40a729 +2025-06-15T03:16:34.759Z [INFO] server-8619bc77-e506-330f-9705-6343be482e95: pipelining replication: peer="{Voter server-17217a5b-b0ad-7ab7-1707-53010c40a729 17217a5b-b0ad-7ab7-1707-53010c40a729}" +2025-06-15T03:16:34.759Z [INFO] server-8619bc77-e506-330f-9705-6343be482e95: pipelining replication: peer="{Voter server-af4423d5-ad9a-abe1-aea2-833e2682a2ea af4423d5-ad9a-abe1-aea2-833e2682a2ea}" +2025-06-15T03:16:34.864Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:16:34.759299585 +0000 UTC m=+10.470977767 (2 nodes), 64.444013ms from start of poll, 64.94075ms from cluster start. Timeout at 2025-06-15 03:16:34.864919911 +0000 UTC m=+10.576598088, 105.620321ms after stability +2025-06-15T03:16:34.865Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:34.970Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:34.865027721 +0000 UTC m=+10.576705893 (1 nodes), 2.986µs from start of poll, 170.668876ms from cluster start. Timeout at 2025-06-15 03:16:34.970754752 +0000 UTC m=+10.682432946, 105.727053ms after stability +2025-06-15T03:16:34.978Z [INFO] server-8619bc77-e506-330f-9705-6343be482e95: updating configuration: command=RemoveServer server-id=server-8619bc77-e506-330f-9705-6343be482e95 server-addr= servers="[{Suffrage:Voter ID:server-af4423d5-ad9a-abe1-aea2-833e2682a2ea Address:af4423d5-ad9a-abe1-aea2-833e2682a2ea} {Suffrage:Voter ID:server-17217a5b-b0ad-7ab7-1707-53010c40a729 Address:17217a5b-b0ad-7ab7-1707-53010c40a729}]" +2025-06-15T03:16:34.979Z [INFO] server-8619bc77-e506-330f-9705-6343be482e95: removed ourself, transitioning to follower +2025-06-15T03:16:34.979Z [INFO] server-8619bc77-e506-330f-9705-6343be482e95: entering follower state: follower="Node at 8619bc77-e506-330f-9705-6343be482e95 [Follower]" leader-address= leader-id= +2025-06-15T03:16:34.979Z [INFO] server-8619bc77-e506-330f-9705-6343be482e95: aborting pipeline replication: peer="{Voter server-17217a5b-b0ad-7ab7-1707-53010c40a729 17217a5b-b0ad-7ab7-1707-53010c40a729}" +2025-06-15T03:16:34.979Z [INFO] server-8619bc77-e506-330f-9705-6343be482e95: aborting pipeline replication: peer="{Voter server-af4423d5-ad9a-abe1-aea2-833e2682a2ea af4423d5-ad9a-abe1-aea2-833e2682a2ea}" +2025-06-15T03:16:35.031Z [WARN] server-8619bc77-e506-330f-9705-6343be482e95: not part of stable configuration, aborting election +2025-06-15T03:16:35.038Z [WARN] server-af4423d5-ad9a-abe1-aea2-833e2682a2ea: heartbeat timeout reached, starting election: last-leader-addr=8619bc77-e506-330f-9705-6343be482e95 last-leader-id=server-8619bc77-e506-330f-9705-6343be482e95 +2025-06-15T03:16:35.038Z [INFO] server-af4423d5-ad9a-abe1-aea2-833e2682a2ea: entering candidate state: node="Node at af4423d5-ad9a-abe1-aea2-833e2682a2ea [Candidate]" term=3 +2025-06-15T03:16:35.038Z [WARN] server-17217a5b-b0ad-7ab7-1707-53010c40a729: rejecting vote request since we have a leader: from=af4423d5-ad9a-abe1-aea2-833e2682a2ea leader=8619bc77-e506-330f-9705-6343be482e95 leader-id=server-8619bc77-e506-330f-9705-6343be482e95 +2025-06-15T03:16:35.078Z [WARN] server-17217a5b-b0ad-7ab7-1707-53010c40a729: heartbeat timeout reached, starting election: last-leader-addr=8619bc77-e506-330f-9705-6343be482e95 last-leader-id=server-8619bc77-e506-330f-9705-6343be482e95 +2025-06-15T03:16:35.078Z [INFO] server-17217a5b-b0ad-7ab7-1707-53010c40a729: entering candidate state: node="Node at 17217a5b-b0ad-7ab7-1707-53010c40a729 [Candidate]" term=3 +2025-06-15T03:16:35.078Z [INFO] server-af4423d5-ad9a-abe1-aea2-833e2682a2ea: duplicate requestVote for same term: term=3 +2025-06-15T03:16:35.085Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:35.128Z [WARN] server-af4423d5-ad9a-abe1-aea2-833e2682a2ea: Election timeout reached, restarting election +2025-06-15T03:16:35.128Z [INFO] server-af4423d5-ad9a-abe1-aea2-833e2682a2ea: entering candidate state: node="Node at af4423d5-ad9a-abe1-aea2-833e2682a2ea [Candidate]" term=4 +2025-06-15T03:16:35.128Z [INFO] server-17217a5b-b0ad-7ab7-1707-53010c40a729: entering follower state: follower="Node at 17217a5b-b0ad-7ab7-1707-53010c40a729 [Follower]" leader-address= leader-id= +2025-06-15T03:16:35.128Z [INFO] server-af4423d5-ad9a-abe1-aea2-833e2682a2ea: election won: term=4 tally=2 +2025-06-15T03:16:35.128Z [INFO] server-af4423d5-ad9a-abe1-aea2-833e2682a2ea: entering leader state: leader="Node at af4423d5-ad9a-abe1-aea2-833e2682a2ea [Leader]" +2025-06-15T03:16:35.128Z [INFO] server-af4423d5-ad9a-abe1-aea2-833e2682a2ea: added peer, starting replication: peer=server-17217a5b-b0ad-7ab7-1707-53010c40a729 +2025-06-15T03:16:35.128Z [INFO] server-af4423d5-ad9a-abe1-aea2-833e2682a2ea: pipelining replication: peer="{Voter server-17217a5b-b0ad-7ab7-1707-53010c40a729 17217a5b-b0ad-7ab7-1707-53010c40a729}" +2025-06-15T03:16:35.238Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:35.128703135 +0000 UTC m=+10.840381320 (1 nodes), 43.641383ms from start of poll, 434.344303ms from cluster start. Timeout at 2025-06-15 03:16:35.238088556 +0000 UTC m=+10.949766734, 109.385414ms after stability +2025-06-15T03:16:35.346Z [INFO] server-af4423d5-ad9a-abe1-aea2-833e2682a2ea: aborting pipeline replication: peer="{Voter server-17217a5b-b0ad-7ab7-1707-53010c40a729 17217a5b-b0ad-7ab7-1707-53010c40a729}" +--- PASS: TestRaft_RemoveLeader_NoShutdown (0.65s) === RUN TestRaft_RemoveFollower_SplitCluster -2024-05-12T20:50:01.264Z [INFO] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7 Address:11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7} {Suffrage:Voter ID:server-8a1a9772-2b64-04a2-8159-88293f31f263 Address:8a1a9772-2b64-04a2-8159-88293f31f263} {Suffrage:Voter ID:server-70e97a2a-b98f-09de-9157-f375880bf056 Address:70e97a2a-b98f-09de-9157-f375880bf056} {Suffrage:Voter ID:server-5b3cf53d-120b-9113-8c62-b541e9510d27 Address:5b3cf53d-120b-9113-8c62-b541e9510d27}]" -2024-05-12T20:50:01.265Z [INFO] server-8a1a9772-2b64-04a2-8159-88293f31f263: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7 Address:11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7} {Suffrage:Voter ID:server-8a1a9772-2b64-04a2-8159-88293f31f263 Address:8a1a9772-2b64-04a2-8159-88293f31f263} {Suffrage:Voter ID:server-70e97a2a-b98f-09de-9157-f375880bf056 Address:70e97a2a-b98f-09de-9157-f375880bf056} {Suffrage:Voter ID:server-5b3cf53d-120b-9113-8c62-b541e9510d27 Address:5b3cf53d-120b-9113-8c62-b541e9510d27}]" -2024-05-12T20:50:01.265Z [INFO] server-70e97a2a-b98f-09de-9157-f375880bf056: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7 Address:11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7} {Suffrage:Voter ID:server-8a1a9772-2b64-04a2-8159-88293f31f263 Address:8a1a9772-2b64-04a2-8159-88293f31f263} {Suffrage:Voter ID:server-70e97a2a-b98f-09de-9157-f375880bf056 Address:70e97a2a-b98f-09de-9157-f375880bf056} {Suffrage:Voter ID:server-5b3cf53d-120b-9113-8c62-b541e9510d27 Address:5b3cf53d-120b-9113-8c62-b541e9510d27}]" -2024-05-12T20:50:01.265Z [INFO] server-5b3cf53d-120b-9113-8c62-b541e9510d27: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7 Address:11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7} {Suffrage:Voter ID:server-8a1a9772-2b64-04a2-8159-88293f31f263 Address:8a1a9772-2b64-04a2-8159-88293f31f263} {Suffrage:Voter ID:server-70e97a2a-b98f-09de-9157-f375880bf056 Address:70e97a2a-b98f-09de-9157-f375880bf056} {Suffrage:Voter ID:server-5b3cf53d-120b-9113-8c62-b541e9510d27 Address:5b3cf53d-120b-9113-8c62-b541e9510d27}]" -2024-05-12T20:50:01.265Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:01.265Z [INFO] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: entering follower state: follower="Node at 11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7 [Follower]" leader-address= leader-id= -2024-05-12T20:50:01.265Z [INFO] server-8a1a9772-2b64-04a2-8159-88293f31f263: entering follower state: follower="Node at 8a1a9772-2b64-04a2-8159-88293f31f263 [Follower]" leader-address= leader-id= -2024-05-12T20:50:01.265Z [INFO] server-70e97a2a-b98f-09de-9157-f375880bf056: entering follower state: follower="Node at 70e97a2a-b98f-09de-9157-f375880bf056 [Follower]" leader-address= leader-id= -2024-05-12T20:50:01.265Z [INFO] server-5b3cf53d-120b-9113-8c62-b541e9510d27: entering follower state: follower="Node at 5b3cf53d-120b-9113-8c62-b541e9510d27 [Follower]" leader-address= leader-id= -2024-05-12T20:50:01.326Z [WARN] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:01.326Z [INFO] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: entering candidate state: node="Node at 11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7 [Candidate]" term=2 -2024-05-12T20:50:01.327Z [WARN] server-5b3cf53d-120b-9113-8c62-b541e9510d27: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:01.327Z [INFO] server-5b3cf53d-120b-9113-8c62-b541e9510d27: entering candidate state: node="Node at 5b3cf53d-120b-9113-8c62-b541e9510d27 [Candidate]" term=2 -2024-05-12T20:50:01.327Z [INFO] server-5b3cf53d-120b-9113-8c62-b541e9510d27: duplicate requestVote for same term: term=2 -2024-05-12T20:50:01.327Z [INFO] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: election won: term=2 tally=3 -2024-05-12T20:50:01.327Z [INFO] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: entering leader state: leader="Node at 11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7 [Leader]" -2024-05-12T20:50:01.327Z [INFO] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: added peer, starting replication: peer=server-8a1a9772-2b64-04a2-8159-88293f31f263 -2024-05-12T20:50:01.327Z [INFO] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: added peer, starting replication: peer=server-70e97a2a-b98f-09de-9157-f375880bf056 -2024-05-12T20:50:01.327Z [INFO] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: added peer, starting replication: peer=server-5b3cf53d-120b-9113-8c62-b541e9510d27 -2024-05-12T20:50:01.327Z [INFO] server-5b3cf53d-120b-9113-8c62-b541e9510d27: entering follower state: follower="Node at 5b3cf53d-120b-9113-8c62-b541e9510d27 [Follower]" leader-address=11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7 leader-id=server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7 -2024-05-12T20:50:01.327Z [INFO] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: pipelining replication: peer="{Voter server-5b3cf53d-120b-9113-8c62-b541e9510d27 5b3cf53d-120b-9113-8c62-b541e9510d27}" -2024-05-12T20:50:01.327Z [WARN] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: rejecting vote request since we have a leader: from=5b3cf53d-120b-9113-8c62-b541e9510d27 leader=11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7 leader-id=server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7 -2024-05-12T20:50:01.327Z [INFO] server-8a1a9772-2b64-04a2-8159-88293f31f263: duplicate requestVote for same term: term=2 -2024-05-12T20:50:01.327Z [INFO] server-70e97a2a-b98f-09de-9157-f375880bf056: duplicate requestVote for same term: term=2 -2024-05-12T20:50:01.327Z [INFO] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: pipelining replication: peer="{Voter server-8a1a9772-2b64-04a2-8159-88293f31f263 8a1a9772-2b64-04a2-8159-88293f31f263}" -2024-05-12T20:50:01.327Z [INFO] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: pipelining replication: peer="{Voter server-70e97a2a-b98f-09de-9157-f375880bf056 70e97a2a-b98f-09de-9157-f375880bf056}" -2024-05-12T20:50:01.434Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:01.327705412 +0000 UTC m=+12.608405755 (1 nodes), 62.417256ms from start of poll, 62.952895ms from cluster start. Timeout at 2024-05-12 20:50:01.434740509 +0000 UTC m=+12.715442511, 107.036756ms after stability -2024-05-12T20:50:01.542Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:01.650Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:01.542881112 +0000 UTC m=+12.823581542 (3 nodes), 2.683µs from start of poll, 278.128682ms from cluster start. Timeout at 2024-05-12 20:50:01.65096054 +0000 UTC m=+12.931660969, 108.079427ms after stability -2024-05-12T20:50:01.651Z [INFO] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: updating configuration: command=RemoveServer server-id=server-5b3cf53d-120b-9113-8c62-b541e9510d27 server-addr= servers="[{Suffrage:Voter ID:server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7 Address:11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7} {Suffrage:Voter ID:server-8a1a9772-2b64-04a2-8159-88293f31f263 Address:8a1a9772-2b64-04a2-8159-88293f31f263} {Suffrage:Voter ID:server-70e97a2a-b98f-09de-9157-f375880bf056 Address:70e97a2a-b98f-09de-9157-f375880bf056}]" -2024-05-12T20:50:01.651Z [INFO] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: removed peer, stopping replication: peer=server-5b3cf53d-120b-9113-8c62-b541e9510d27 last-index=3 -2024-05-12T20:50:01.651Z [INFO] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: aborting pipeline replication: peer="{Voter server-5b3cf53d-120b-9113-8c62-b541e9510d27 5b3cf53d-120b-9113-8c62-b541e9510d27}" -2024-05-12T20:50:01.651Z [ERROR] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: failed to pipeline appendEntries: peer="{Voter server-8a1a9772-2b64-04a2-8159-88293f31f263 8a1a9772-2b64-04a2-8159-88293f31f263}" error="append pipeline closed" -2024-05-12T20:50:01.651Z [INFO] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: aborting pipeline replication: peer="{Voter server-8a1a9772-2b64-04a2-8159-88293f31f263 8a1a9772-2b64-04a2-8159-88293f31f263}" -2024-05-12T20:50:01.651Z [ERROR] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: failed to pipeline appendEntries: peer="{Voter server-70e97a2a-b98f-09de-9157-f375880bf056 70e97a2a-b98f-09de-9157-f375880bf056}" error="append pipeline closed" -2024-05-12T20:50:01.651Z [INFO] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: aborting pipeline replication: peer="{Voter server-70e97a2a-b98f-09de-9157-f375880bf056 70e97a2a-b98f-09de-9157-f375880bf056}" -2024-05-12T20:50:01.658Z [ERROR] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: failed to heartbeat to: peer=70e97a2a-b98f-09de-9157-f375880bf056 backoff time=10ms error="failed to connect to peer: 70e97a2a-b98f-09de-9157-f375880bf056" -2024-05-12T20:50:01.658Z [ERROR] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: failed to appendEntries to: peer="{Voter server-70e97a2a-b98f-09de-9157-f375880bf056 70e97a2a-b98f-09de-9157-f375880bf056}" error="failed to connect to peer: 70e97a2a-b98f-09de-9157-f375880bf056" -2024-05-12T20:50:01.662Z [ERROR] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: failed to appendEntries to: peer="{Voter server-8a1a9772-2b64-04a2-8159-88293f31f263 8a1a9772-2b64-04a2-8159-88293f31f263}" error="failed to connect to peer: 8a1a9772-2b64-04a2-8159-88293f31f263" -2024-05-12T20:50:01.670Z [ERROR] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: failed to heartbeat to: peer=8a1a9772-2b64-04a2-8159-88293f31f263 backoff time=10ms error="failed to connect to peer: 8a1a9772-2b64-04a2-8159-88293f31f263" -2024-05-12T20:50:01.678Z [ERROR] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: failed to heartbeat to: peer=70e97a2a-b98f-09de-9157-f375880bf056 backoff time=10ms error="failed to connect to peer: 70e97a2a-b98f-09de-9157-f375880bf056" -2024-05-12T20:50:01.678Z [ERROR] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: failed to appendEntries to: peer="{Voter server-70e97a2a-b98f-09de-9157-f375880bf056 70e97a2a-b98f-09de-9157-f375880bf056}" error="failed to connect to peer: 70e97a2a-b98f-09de-9157-f375880bf056" -2024-05-12T20:50:01.690Z [ERROR] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: failed to appendEntries to: peer="{Voter server-8a1a9772-2b64-04a2-8159-88293f31f263 8a1a9772-2b64-04a2-8159-88293f31f263}" error="failed to connect to peer: 8a1a9772-2b64-04a2-8159-88293f31f263" -2024-05-12T20:50:01.694Z [ERROR] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: failed to heartbeat to: peer=8a1a9772-2b64-04a2-8159-88293f31f263 backoff time=10ms error="failed to connect to peer: 8a1a9772-2b64-04a2-8159-88293f31f263" -2024-05-12T20:50:01.698Z [ERROR] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: failed to appendEntries to: peer="{Voter server-70e97a2a-b98f-09de-9157-f375880bf056 70e97a2a-b98f-09de-9157-f375880bf056}" error="failed to connect to peer: 70e97a2a-b98f-09de-9157-f375880bf056" -2024-05-12T20:50:01.702Z [ERROR] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: failed to heartbeat to: peer=70e97a2a-b98f-09de-9157-f375880bf056 backoff time=10ms error="failed to connect to peer: 70e97a2a-b98f-09de-9157-f375880bf056" -2024-05-12T20:50:01.710Z [WARN] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: failed to contact: server-id=server-8a1a9772-2b64-04a2-8159-88293f31f263 time=59.81493ms -2024-05-12T20:50:01.710Z [WARN] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: failed to contact: server-id=server-70e97a2a-b98f-09de-9157-f375880bf056 time=59.96259ms -2024-05-12T20:50:01.710Z [WARN] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: failed to contact quorum of nodes, stepping down -2024-05-12T20:50:01.710Z [INFO] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: entering follower state: follower="Node at 11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7 [Follower]" leader-address= leader-id= -2024-05-12T20:50:01.710Z [ERROR] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: failed to appendEntries to: peer="{Voter server-70e97a2a-b98f-09de-9157-f375880bf056 70e97a2a-b98f-09de-9157-f375880bf056}" error="failed to connect to peer: 70e97a2a-b98f-09de-9157-f375880bf056" -2024-05-12T20:50:01.710Z [ERROR] server-11fd42f4-2d1d-cb2d-4d18-9ac31d9a5ed7: failed to appendEntries to: peer="{Voter server-8a1a9772-2b64-04a2-8159-88293f31f263 8a1a9772-2b64-04a2-8159-88293f31f263}" error="failed to connect to peer: 8a1a9772-2b64-04a2-8159-88293f31f263" ---- PASS: TestRaft_RemoveFollower_SplitCluster (0.45s) +2025-06-15T03:16:35.348Z [INFO] server-99a1b518-a908-d152-3235-b1916e97d9fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99a1b518-a908-d152-3235-b1916e97d9fb Address:99a1b518-a908-d152-3235-b1916e97d9fb} {Suffrage:Voter ID:server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f Address:e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f} {Suffrage:Voter ID:server-efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1 Address:efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1} {Suffrage:Voter ID:server-a09444a8-2bbe-4831-4cec-efa078fd9198 Address:a09444a8-2bbe-4831-4cec-efa078fd9198}]" +2025-06-15T03:16:35.348Z [INFO] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99a1b518-a908-d152-3235-b1916e97d9fb Address:99a1b518-a908-d152-3235-b1916e97d9fb} {Suffrage:Voter ID:server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f Address:e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f} {Suffrage:Voter ID:server-efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1 Address:efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1} {Suffrage:Voter ID:server-a09444a8-2bbe-4831-4cec-efa078fd9198 Address:a09444a8-2bbe-4831-4cec-efa078fd9198}]" +2025-06-15T03:16:35.348Z [INFO] server-efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99a1b518-a908-d152-3235-b1916e97d9fb Address:99a1b518-a908-d152-3235-b1916e97d9fb} {Suffrage:Voter ID:server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f Address:e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f} {Suffrage:Voter ID:server-efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1 Address:efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1} {Suffrage:Voter ID:server-a09444a8-2bbe-4831-4cec-efa078fd9198 Address:a09444a8-2bbe-4831-4cec-efa078fd9198}]" +2025-06-15T03:16:35.348Z [INFO] server-a09444a8-2bbe-4831-4cec-efa078fd9198: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99a1b518-a908-d152-3235-b1916e97d9fb Address:99a1b518-a908-d152-3235-b1916e97d9fb} {Suffrage:Voter ID:server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f Address:e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f} {Suffrage:Voter ID:server-efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1 Address:efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1} {Suffrage:Voter ID:server-a09444a8-2bbe-4831-4cec-efa078fd9198 Address:a09444a8-2bbe-4831-4cec-efa078fd9198}]" +2025-06-15T03:16:35.348Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:35.348Z [INFO] server-99a1b518-a908-d152-3235-b1916e97d9fb: entering follower state: follower="Node at 99a1b518-a908-d152-3235-b1916e97d9fb [Follower]" leader-address= leader-id= +2025-06-15T03:16:35.348Z [INFO] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: entering follower state: follower="Node at e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f [Follower]" leader-address= leader-id= +2025-06-15T03:16:35.348Z [INFO] server-efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1: entering follower state: follower="Node at efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1 [Follower]" leader-address= leader-id= +2025-06-15T03:16:35.348Z [INFO] server-a09444a8-2bbe-4831-4cec-efa078fd9198: entering follower state: follower="Node at a09444a8-2bbe-4831-4cec-efa078fd9198 [Follower]" leader-address= leader-id= +2025-06-15T03:16:35.399Z [WARN] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:35.400Z [INFO] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: entering candidate state: node="Node at e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f [Candidate]" term=2 +2025-06-15T03:16:35.400Z [INFO] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: election won: term=2 tally=3 +2025-06-15T03:16:35.400Z [INFO] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: entering leader state: leader="Node at e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f [Leader]" +2025-06-15T03:16:35.400Z [INFO] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: added peer, starting replication: peer=server-99a1b518-a908-d152-3235-b1916e97d9fb +2025-06-15T03:16:35.400Z [INFO] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: added peer, starting replication: peer=server-efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1 +2025-06-15T03:16:35.400Z [INFO] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: added peer, starting replication: peer=server-a09444a8-2bbe-4831-4cec-efa078fd9198 +2025-06-15T03:16:35.400Z [INFO] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: pipelining replication: peer="{Voter server-a09444a8-2bbe-4831-4cec-efa078fd9198 a09444a8-2bbe-4831-4cec-efa078fd9198}" +2025-06-15T03:16:35.400Z [INFO] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: pipelining replication: peer="{Voter server-efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1 efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1}" +2025-06-15T03:16:35.400Z [WARN] server-99a1b518-a908-d152-3235-b1916e97d9fb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:35.400Z [INFO] server-99a1b518-a908-d152-3235-b1916e97d9fb: entering candidate state: node="Node at 99a1b518-a908-d152-3235-b1916e97d9fb [Candidate]" term=2 +2025-06-15T03:16:35.400Z [INFO] server-99a1b518-a908-d152-3235-b1916e97d9fb: duplicate requestVote for same term: term=2 +2025-06-15T03:16:35.400Z [INFO] server-99a1b518-a908-d152-3235-b1916e97d9fb: entering follower state: follower="Node at 99a1b518-a908-d152-3235-b1916e97d9fb [Follower]" leader-address=e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f leader-id=server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f +2025-06-15T03:16:35.400Z [INFO] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: pipelining replication: peer="{Voter server-99a1b518-a908-d152-3235-b1916e97d9fb 99a1b518-a908-d152-3235-b1916e97d9fb}" +2025-06-15T03:16:35.400Z [WARN] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: rejecting vote request since we have a leader: from=99a1b518-a908-d152-3235-b1916e97d9fb leader=e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f leader-id=server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f +2025-06-15T03:16:35.400Z [WARN] server-efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1: rejecting vote request since we have a leader: from=99a1b518-a908-d152-3235-b1916e97d9fb leader=e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f leader-id=server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f +2025-06-15T03:16:35.400Z [WARN] server-a09444a8-2bbe-4831-4cec-efa078fd9198: rejecting vote request since we have a leader: from=99a1b518-a908-d152-3235-b1916e97d9fb leader=e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f leader-id=server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f +2025-06-15T03:16:35.506Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:35.400925129 +0000 UTC m=+11.112603301 (1 nodes), 52.285403ms from start of poll, 52.880957ms from cluster start. Timeout at 2025-06-15 03:16:35.506186826 +0000 UTC m=+11.217865017, 105.261716ms after stability +2025-06-15T03:16:35.611Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:16:35.722Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:16:35.611672494 +0000 UTC m=+11.323350688 (3 nodes), 3.526µs from start of poll, 263.628344ms from cluster start. Timeout at 2025-06-15 03:16:35.722183176 +0000 UTC m=+11.433861371, 110.510683ms after stability +2025-06-15T03:16:35.722Z [INFO] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: updating configuration: command=RemoveServer server-id=server-a09444a8-2bbe-4831-4cec-efa078fd9198 server-addr= servers="[{Suffrage:Voter ID:server-99a1b518-a908-d152-3235-b1916e97d9fb Address:99a1b518-a908-d152-3235-b1916e97d9fb} {Suffrage:Voter ID:server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f Address:e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f} {Suffrage:Voter ID:server-efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1 Address:efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1}]" +2025-06-15T03:16:35.722Z [INFO] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: removed peer, stopping replication: peer=server-a09444a8-2bbe-4831-4cec-efa078fd9198 last-index=3 +2025-06-15T03:16:35.722Z [ERROR] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: failed to pipeline appendEntries: peer="{Voter server-99a1b518-a908-d152-3235-b1916e97d9fb 99a1b518-a908-d152-3235-b1916e97d9fb}" error="append pipeline closed" +2025-06-15T03:16:35.722Z [ERROR] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: failed to pipeline appendEntries: peer="{Voter server-efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1 efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1}" error="append pipeline closed" +2025-06-15T03:16:35.722Z [INFO] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: aborting pipeline replication: peer="{Voter server-efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1 efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1}" +2025-06-15T03:16:35.722Z [INFO] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: aborting pipeline replication: peer="{Voter server-99a1b518-a908-d152-3235-b1916e97d9fb 99a1b518-a908-d152-3235-b1916e97d9fb}" +2025-06-15T03:16:35.722Z [INFO] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: aborting pipeline replication: peer="{Voter server-a09444a8-2bbe-4831-4cec-efa078fd9198 a09444a8-2bbe-4831-4cec-efa078fd9198}" +2025-06-15T03:16:35.729Z [ERROR] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: failed to heartbeat to: peer=99a1b518-a908-d152-3235-b1916e97d9fb backoff time=10ms error="failed to connect to peer: 99a1b518-a908-d152-3235-b1916e97d9fb" +2025-06-15T03:16:35.732Z [ERROR] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: failed to heartbeat to: peer=efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1 backoff time=10ms error="failed to connect to peer: efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1" +2025-06-15T03:16:35.732Z [ERROR] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: failed to appendEntries to: peer="{Voter server-efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1 efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1}" error="failed to connect to peer: efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1" +2025-06-15T03:16:35.732Z [ERROR] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: failed to appendEntries to: peer="{Voter server-99a1b518-a908-d152-3235-b1916e97d9fb 99a1b518-a908-d152-3235-b1916e97d9fb}" error="failed to connect to peer: 99a1b518-a908-d152-3235-b1916e97d9fb" +2025-06-15T03:16:35.748Z [ERROR] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: failed to heartbeat to: peer=99a1b518-a908-d152-3235-b1916e97d9fb backoff time=10ms error="failed to connect to peer: 99a1b518-a908-d152-3235-b1916e97d9fb" +2025-06-15T03:16:35.748Z [ERROR] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: failed to heartbeat to: peer=efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1 backoff time=10ms error="failed to connect to peer: efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1" +2025-06-15T03:16:35.752Z [ERROR] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: failed to appendEntries to: peer="{Voter server-efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1 efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1}" error="failed to connect to peer: efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1" +2025-06-15T03:16:35.752Z [ERROR] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: failed to appendEntries to: peer="{Voter server-99a1b518-a908-d152-3235-b1916e97d9fb 99a1b518-a908-d152-3235-b1916e97d9fb}" error="failed to connect to peer: 99a1b518-a908-d152-3235-b1916e97d9fb" +2025-06-15T03:16:35.764Z [ERROR] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: failed to heartbeat to: peer=99a1b518-a908-d152-3235-b1916e97d9fb backoff time=10ms error="failed to connect to peer: 99a1b518-a908-d152-3235-b1916e97d9fb" +2025-06-15T03:16:35.767Z [ERROR] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: failed to heartbeat to: peer=efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1 backoff time=10ms error="failed to connect to peer: efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1" +2025-06-15T03:16:35.772Z [ERROR] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: failed to appendEntries to: peer="{Voter server-efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1 efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1}" error="failed to connect to peer: efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1" +2025-06-15T03:16:35.772Z [WARN] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: failed to contact: server-id=server-99a1b518-a908-d152-3235-b1916e97d9fb time=50.101114ms +2025-06-15T03:16:35.772Z [WARN] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: failed to contact: server-id=server-efd7dc2f-4740-f8d9-1d7e-43a95eb9cff1 time=50.040704ms +2025-06-15T03:16:35.772Z [WARN] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: failed to contact quorum of nodes, stepping down +2025-06-15T03:16:35.772Z [INFO] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: entering follower state: follower="Node at e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f [Follower]" leader-address= leader-id= +2025-06-15T03:16:35.772Z [ERROR] server-e7a65ab0-c35d-b98d-3fd3-e41e1e565b5f: failed to appendEntries to: peer="{Voter server-99a1b518-a908-d152-3235-b1916e97d9fb 99a1b518-a908-d152-3235-b1916e97d9fb}" error="failed to connect to peer: 99a1b518-a908-d152-3235-b1916e97d9fb" +--- PASS: TestRaft_RemoveFollower_SplitCluster (0.43s) === RUN TestRaft_AddKnownPeer -2024-05-12T20:50:01.712Z [INFO] server-2442a139-d2ec-ced1-ddb3-b7b5dc75d95b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2442a139-d2ec-ced1-ddb3-b7b5dc75d95b Address:2442a139-d2ec-ced1-ddb3-b7b5dc75d95b} {Suffrage:Voter ID:server-76afcabe-0921-c8a9-1a6d-4669e0f4c693 Address:76afcabe-0921-c8a9-1a6d-4669e0f4c693} {Suffrage:Voter ID:server-237eae45-92f5-738f-ba00-0984c954bbf3 Address:237eae45-92f5-738f-ba00-0984c954bbf3}]" -2024-05-12T20:50:01.712Z [INFO] server-76afcabe-0921-c8a9-1a6d-4669e0f4c693: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2442a139-d2ec-ced1-ddb3-b7b5dc75d95b Address:2442a139-d2ec-ced1-ddb3-b7b5dc75d95b} {Suffrage:Voter ID:server-76afcabe-0921-c8a9-1a6d-4669e0f4c693 Address:76afcabe-0921-c8a9-1a6d-4669e0f4c693} {Suffrage:Voter ID:server-237eae45-92f5-738f-ba00-0984c954bbf3 Address:237eae45-92f5-738f-ba00-0984c954bbf3}]" -2024-05-12T20:50:01.712Z [INFO] server-237eae45-92f5-738f-ba00-0984c954bbf3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2442a139-d2ec-ced1-ddb3-b7b5dc75d95b Address:2442a139-d2ec-ced1-ddb3-b7b5dc75d95b} {Suffrage:Voter ID:server-76afcabe-0921-c8a9-1a6d-4669e0f4c693 Address:76afcabe-0921-c8a9-1a6d-4669e0f4c693} {Suffrage:Voter ID:server-237eae45-92f5-738f-ba00-0984c954bbf3 Address:237eae45-92f5-738f-ba00-0984c954bbf3}]" -2024-05-12T20:50:01.712Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:01.712Z [INFO] server-2442a139-d2ec-ced1-ddb3-b7b5dc75d95b: entering follower state: follower="Node at 2442a139-d2ec-ced1-ddb3-b7b5dc75d95b [Follower]" leader-address= leader-id= -2024-05-12T20:50:01.712Z [INFO] server-76afcabe-0921-c8a9-1a6d-4669e0f4c693: entering follower state: follower="Node at 76afcabe-0921-c8a9-1a6d-4669e0f4c693 [Follower]" leader-address= leader-id= -2024-05-12T20:50:01.713Z [INFO] server-237eae45-92f5-738f-ba00-0984c954bbf3: entering follower state: follower="Node at 237eae45-92f5-738f-ba00-0984c954bbf3 [Follower]" leader-address= leader-id= -2024-05-12T20:50:01.766Z [WARN] server-237eae45-92f5-738f-ba00-0984c954bbf3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:01.766Z [INFO] server-237eae45-92f5-738f-ba00-0984c954bbf3: entering candidate state: node="Node at 237eae45-92f5-738f-ba00-0984c954bbf3 [Candidate]" term=2 -2024-05-12T20:50:01.766Z [INFO] server-237eae45-92f5-738f-ba00-0984c954bbf3: election won: term=2 tally=2 -2024-05-12T20:50:01.766Z [INFO] server-237eae45-92f5-738f-ba00-0984c954bbf3: entering leader state: leader="Node at 237eae45-92f5-738f-ba00-0984c954bbf3 [Leader]" -2024-05-12T20:50:01.766Z [INFO] server-237eae45-92f5-738f-ba00-0984c954bbf3: added peer, starting replication: peer=server-2442a139-d2ec-ced1-ddb3-b7b5dc75d95b -2024-05-12T20:50:01.766Z [INFO] server-237eae45-92f5-738f-ba00-0984c954bbf3: added peer, starting replication: peer=server-76afcabe-0921-c8a9-1a6d-4669e0f4c693 -2024-05-12T20:50:01.767Z [INFO] server-237eae45-92f5-738f-ba00-0984c954bbf3: pipelining replication: peer="{Voter server-76afcabe-0921-c8a9-1a6d-4669e0f4c693 76afcabe-0921-c8a9-1a6d-4669e0f4c693}" -2024-05-12T20:50:01.767Z [INFO] server-237eae45-92f5-738f-ba00-0984c954bbf3: pipelining replication: peer="{Voter server-2442a139-d2ec-ced1-ddb3-b7b5dc75d95b 2442a139-d2ec-ced1-ddb3-b7b5dc75d95b}" -2024-05-12T20:50:01.874Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:01.767340946 +0000 UTC m=+13.048041261 (1 nodes), 54.429993ms from start of poll, 54.774079ms from cluster start. Timeout at 2024-05-12 20:50:01.874756485 +0000 UTC m=+13.155458107, 107.416846ms after stability -2024-05-12T20:50:01.874Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:01.982Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:01.874825702 +0000 UTC m=+13.155525986 (2 nodes), 1.848µs from start of poll, 162.258804ms from cluster start. Timeout at 2024-05-12 20:50:01.982770893 +0000 UTC m=+13.263473153, 107.947167ms after stability -2024-05-12T20:50:01.982Z [INFO] server-237eae45-92f5-738f-ba00-0984c954bbf3: updating configuration: command=AddVoter server-id=server-2442a139-d2ec-ced1-ddb3-b7b5dc75d95b server-addr=2442a139-d2ec-ced1-ddb3-b7b5dc75d95b servers="[{Suffrage:Voter ID:server-2442a139-d2ec-ced1-ddb3-b7b5dc75d95b Address:2442a139-d2ec-ced1-ddb3-b7b5dc75d95b} {Suffrage:Voter ID:server-76afcabe-0921-c8a9-1a6d-4669e0f4c693 Address:76afcabe-0921-c8a9-1a6d-4669e0f4c693} {Suffrage:Voter ID:server-237eae45-92f5-738f-ba00-0984c954bbf3 Address:237eae45-92f5-738f-ba00-0984c954bbf3}]" -2024-05-12T20:50:01.983Z [INFO] server-237eae45-92f5-738f-ba00-0984c954bbf3: aborting pipeline replication: peer="{Voter server-2442a139-d2ec-ced1-ddb3-b7b5dc75d95b 2442a139-d2ec-ced1-ddb3-b7b5dc75d95b}" -2024-05-12T20:50:01.983Z [INFO] server-237eae45-92f5-738f-ba00-0984c954bbf3: aborting pipeline replication: peer="{Voter server-76afcabe-0921-c8a9-1a6d-4669e0f4c693 76afcabe-0921-c8a9-1a6d-4669e0f4c693}" ---- PASS: TestRaft_AddKnownPeer (0.27s) +2025-06-15T03:16:35.774Z [INFO] server-9f1ade8d-e956-79ef-a8e5-e495e6daf6b3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9f1ade8d-e956-79ef-a8e5-e495e6daf6b3 Address:9f1ade8d-e956-79ef-a8e5-e495e6daf6b3} {Suffrage:Voter ID:server-b7c2cd19-ee04-87c8-a3c0-5ff958521bb4 Address:b7c2cd19-ee04-87c8-a3c0-5ff958521bb4} {Suffrage:Voter ID:server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05 Address:bcac20a2-8bcd-7145-f280-07d5bbd7fd05}]" +2025-06-15T03:16:35.774Z [INFO] server-9f1ade8d-e956-79ef-a8e5-e495e6daf6b3: entering follower state: follower="Node at 9f1ade8d-e956-79ef-a8e5-e495e6daf6b3 [Follower]" leader-address= leader-id= +2025-06-15T03:16:35.774Z [INFO] server-b7c2cd19-ee04-87c8-a3c0-5ff958521bb4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9f1ade8d-e956-79ef-a8e5-e495e6daf6b3 Address:9f1ade8d-e956-79ef-a8e5-e495e6daf6b3} {Suffrage:Voter ID:server-b7c2cd19-ee04-87c8-a3c0-5ff958521bb4 Address:b7c2cd19-ee04-87c8-a3c0-5ff958521bb4} {Suffrage:Voter ID:server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05 Address:bcac20a2-8bcd-7145-f280-07d5bbd7fd05}]" +2025-06-15T03:16:35.774Z [INFO] server-b7c2cd19-ee04-87c8-a3c0-5ff958521bb4: entering follower state: follower="Node at b7c2cd19-ee04-87c8-a3c0-5ff958521bb4 [Follower]" leader-address= leader-id= +2025-06-15T03:16:35.774Z [INFO] server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9f1ade8d-e956-79ef-a8e5-e495e6daf6b3 Address:9f1ade8d-e956-79ef-a8e5-e495e6daf6b3} {Suffrage:Voter ID:server-b7c2cd19-ee04-87c8-a3c0-5ff958521bb4 Address:b7c2cd19-ee04-87c8-a3c0-5ff958521bb4} {Suffrage:Voter ID:server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05 Address:bcac20a2-8bcd-7145-f280-07d5bbd7fd05}]" +2025-06-15T03:16:35.774Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:35.774Z [INFO] server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05: entering follower state: follower="Node at bcac20a2-8bcd-7145-f280-07d5bbd7fd05 [Follower]" leader-address= leader-id= +2025-06-15T03:16:35.862Z [WARN] server-9f1ade8d-e956-79ef-a8e5-e495e6daf6b3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:35.862Z [INFO] server-9f1ade8d-e956-79ef-a8e5-e495e6daf6b3: entering candidate state: node="Node at 9f1ade8d-e956-79ef-a8e5-e495e6daf6b3 [Candidate]" term=2 +2025-06-15T03:16:35.862Z [WARN] server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:35.862Z [INFO] server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05: entering candidate state: node="Node at bcac20a2-8bcd-7145-f280-07d5bbd7fd05 [Candidate]" term=2 +2025-06-15T03:16:35.862Z [INFO] server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05: election won: term=2 tally=2 +2025-06-15T03:16:35.862Z [INFO] server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05: entering leader state: leader="Node at bcac20a2-8bcd-7145-f280-07d5bbd7fd05 [Leader]" +2025-06-15T03:16:35.862Z [INFO] server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05: added peer, starting replication: peer=server-9f1ade8d-e956-79ef-a8e5-e495e6daf6b3 +2025-06-15T03:16:35.862Z [INFO] server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05: added peer, starting replication: peer=server-b7c2cd19-ee04-87c8-a3c0-5ff958521bb4 +2025-06-15T03:16:35.862Z [INFO] server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05: pipelining replication: peer="{Voter server-b7c2cd19-ee04-87c8-a3c0-5ff958521bb4 b7c2cd19-ee04-87c8-a3c0-5ff958521bb4}" +2025-06-15T03:16:35.862Z [WARN] server-b7c2cd19-ee04-87c8-a3c0-5ff958521bb4: rejecting vote request since we have a leader: from=9f1ade8d-e956-79ef-a8e5-e495e6daf6b3 leader=bcac20a2-8bcd-7145-f280-07d5bbd7fd05 leader-id=server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05 +2025-06-15T03:16:35.862Z [INFO] server-9f1ade8d-e956-79ef-a8e5-e495e6daf6b3: duplicate requestVote for same term: term=2 +2025-06-15T03:16:35.862Z [INFO] server-9f1ade8d-e956-79ef-a8e5-e495e6daf6b3: entering follower state: follower="Node at 9f1ade8d-e956-79ef-a8e5-e495e6daf6b3 [Follower]" leader-address=bcac20a2-8bcd-7145-f280-07d5bbd7fd05 leader-id=server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05 +2025-06-15T03:16:35.862Z [INFO] server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05: pipelining replication: peer="{Voter server-9f1ade8d-e956-79ef-a8e5-e495e6daf6b3 9f1ade8d-e956-79ef-a8e5-e495e6daf6b3}" +2025-06-15T03:16:35.862Z [WARN] server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05: rejecting vote request since we have a leader: from=9f1ade8d-e956-79ef-a8e5-e495e6daf6b3 leader=bcac20a2-8bcd-7145-f280-07d5bbd7fd05 leader-id=server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05 +2025-06-15T03:16:35.968Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:35.862895107 +0000 UTC m=+11.574573280 (1 nodes), 88.352196ms from start of poll, 88.851805ms from cluster start. Timeout at 2025-06-15 03:16:35.968914997 +0000 UTC m=+11.680593166, 106.019886ms after stability +2025-06-15T03:16:35.969Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:16:36.074Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:16:35.969020158 +0000 UTC m=+11.680698355 (2 nodes), 4.423µs from start of poll, 194.97688ms from cluster start. Timeout at 2025-06-15 03:16:36.074197499 +0000 UTC m=+11.785875711, 105.177356ms after stability +2025-06-15T03:16:36.074Z [INFO] server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05: updating configuration: command=AddVoter server-id=server-9f1ade8d-e956-79ef-a8e5-e495e6daf6b3 server-addr=9f1ade8d-e956-79ef-a8e5-e495e6daf6b3 servers="[{Suffrage:Voter ID:server-9f1ade8d-e956-79ef-a8e5-e495e6daf6b3 Address:9f1ade8d-e956-79ef-a8e5-e495e6daf6b3} {Suffrage:Voter ID:server-b7c2cd19-ee04-87c8-a3c0-5ff958521bb4 Address:b7c2cd19-ee04-87c8-a3c0-5ff958521bb4} {Suffrage:Voter ID:server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05 Address:bcac20a2-8bcd-7145-f280-07d5bbd7fd05}]" +2025-06-15T03:16:36.074Z [INFO] server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05: aborting pipeline replication: peer="{Voter server-b7c2cd19-ee04-87c8-a3c0-5ff958521bb4 b7c2cd19-ee04-87c8-a3c0-5ff958521bb4}" +2025-06-15T03:16:36.074Z [INFO] server-bcac20a2-8bcd-7145-f280-07d5bbd7fd05: aborting pipeline replication: peer="{Voter server-9f1ade8d-e956-79ef-a8e5-e495e6daf6b3 9f1ade8d-e956-79ef-a8e5-e495e6daf6b3}" +--- PASS: TestRaft_AddKnownPeer (0.30s) === RUN TestRaft_RemoveUnknownPeer -2024-05-12T20:50:01.984Z [INFO] server-eb469116-4bdf-e39e-0e14-28ff7348064a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eb469116-4bdf-e39e-0e14-28ff7348064a Address:eb469116-4bdf-e39e-0e14-28ff7348064a} {Suffrage:Voter ID:server-37c6aa6d-246b-469b-6d1e-b29ec2396fb1 Address:37c6aa6d-246b-469b-6d1e-b29ec2396fb1} {Suffrage:Voter ID:server-43c85437-2596-cd5f-111d-1b395b22c004 Address:43c85437-2596-cd5f-111d-1b395b22c004}]" -2024-05-12T20:50:01.984Z [INFO] server-37c6aa6d-246b-469b-6d1e-b29ec2396fb1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eb469116-4bdf-e39e-0e14-28ff7348064a Address:eb469116-4bdf-e39e-0e14-28ff7348064a} {Suffrage:Voter ID:server-37c6aa6d-246b-469b-6d1e-b29ec2396fb1 Address:37c6aa6d-246b-469b-6d1e-b29ec2396fb1} {Suffrage:Voter ID:server-43c85437-2596-cd5f-111d-1b395b22c004 Address:43c85437-2596-cd5f-111d-1b395b22c004}]" -2024-05-12T20:50:01.984Z [INFO] server-43c85437-2596-cd5f-111d-1b395b22c004: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eb469116-4bdf-e39e-0e14-28ff7348064a Address:eb469116-4bdf-e39e-0e14-28ff7348064a} {Suffrage:Voter ID:server-37c6aa6d-246b-469b-6d1e-b29ec2396fb1 Address:37c6aa6d-246b-469b-6d1e-b29ec2396fb1} {Suffrage:Voter ID:server-43c85437-2596-cd5f-111d-1b395b22c004 Address:43c85437-2596-cd5f-111d-1b395b22c004}]" -2024-05-12T20:50:01.984Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:01.984Z [INFO] server-eb469116-4bdf-e39e-0e14-28ff7348064a: entering follower state: follower="Node at eb469116-4bdf-e39e-0e14-28ff7348064a [Follower]" leader-address= leader-id= -2024-05-12T20:50:01.984Z [INFO] server-37c6aa6d-246b-469b-6d1e-b29ec2396fb1: entering follower state: follower="Node at 37c6aa6d-246b-469b-6d1e-b29ec2396fb1 [Follower]" leader-address= leader-id= -2024-05-12T20:50:01.985Z [INFO] server-43c85437-2596-cd5f-111d-1b395b22c004: entering follower state: follower="Node at 43c85437-2596-cd5f-111d-1b395b22c004 [Follower]" leader-address= leader-id= -2024-05-12T20:50:02.058Z [WARN] server-43c85437-2596-cd5f-111d-1b395b22c004: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:02.058Z [INFO] server-43c85437-2596-cd5f-111d-1b395b22c004: entering candidate state: node="Node at 43c85437-2596-cd5f-111d-1b395b22c004 [Candidate]" term=2 -2024-05-12T20:50:02.058Z [INFO] server-43c85437-2596-cd5f-111d-1b395b22c004: election won: term=2 tally=2 -2024-05-12T20:50:02.058Z [INFO] server-43c85437-2596-cd5f-111d-1b395b22c004: entering leader state: leader="Node at 43c85437-2596-cd5f-111d-1b395b22c004 [Leader]" -2024-05-12T20:50:02.058Z [INFO] server-43c85437-2596-cd5f-111d-1b395b22c004: added peer, starting replication: peer=server-eb469116-4bdf-e39e-0e14-28ff7348064a -2024-05-12T20:50:02.059Z [INFO] server-43c85437-2596-cd5f-111d-1b395b22c004: added peer, starting replication: peer=server-37c6aa6d-246b-469b-6d1e-b29ec2396fb1 -2024-05-12T20:50:02.059Z [INFO] server-43c85437-2596-cd5f-111d-1b395b22c004: pipelining replication: peer="{Voter server-37c6aa6d-246b-469b-6d1e-b29ec2396fb1 37c6aa6d-246b-469b-6d1e-b29ec2396fb1}" -2024-05-12T20:50:02.059Z [INFO] server-43c85437-2596-cd5f-111d-1b395b22c004: pipelining replication: peer="{Voter server-eb469116-4bdf-e39e-0e14-28ff7348064a eb469116-4bdf-e39e-0e14-28ff7348064a}" -2024-05-12T20:50:02.182Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:02.059411018 +0000 UTC m=+13.340111362 (1 nodes), 74.507616ms from start of poll, 74.812598ms from cluster start. Timeout at 2024-05-12 20:50:02.182809807 +0000 UTC m=+13.463510224, 123.398862ms after stability -2024-05-12T20:50:02.182Z [INFO] server-43c85437-2596-cd5f-111d-1b395b22c004: updating configuration: command=RemoveServer server-id=1fc41140-41b2-e79b-249e-bebf906f32d8 server-addr= servers="[{Suffrage:Voter ID:server-eb469116-4bdf-e39e-0e14-28ff7348064a Address:eb469116-4bdf-e39e-0e14-28ff7348064a} {Suffrage:Voter ID:server-37c6aa6d-246b-469b-6d1e-b29ec2396fb1 Address:37c6aa6d-246b-469b-6d1e-b29ec2396fb1} {Suffrage:Voter ID:server-43c85437-2596-cd5f-111d-1b395b22c004 Address:43c85437-2596-cd5f-111d-1b395b22c004}]" -2024-05-12T20:50:02.183Z [INFO] server-43c85437-2596-cd5f-111d-1b395b22c004: aborting pipeline replication: peer="{Voter server-37c6aa6d-246b-469b-6d1e-b29ec2396fb1 37c6aa6d-246b-469b-6d1e-b29ec2396fb1}" -2024-05-12T20:50:02.183Z [INFO] server-43c85437-2596-cd5f-111d-1b395b22c004: aborting pipeline replication: peer="{Voter server-eb469116-4bdf-e39e-0e14-28ff7348064a eb469116-4bdf-e39e-0e14-28ff7348064a}" ---- PASS: TestRaft_RemoveUnknownPeer (0.21s) +2025-06-15T03:16:36.076Z [INFO] server-9bcf9833-a417-9f00-b3bf-8404d01e040d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9bcf9833-a417-9f00-b3bf-8404d01e040d Address:9bcf9833-a417-9f00-b3bf-8404d01e040d} {Suffrage:Voter ID:server-857ff094-1b21-b984-7053-b8114b47284a Address:857ff094-1b21-b984-7053-b8114b47284a} {Suffrage:Voter ID:server-bf382df7-e928-c77b-f689-ad685747cee3 Address:bf382df7-e928-c77b-f689-ad685747cee3}]" +2025-06-15T03:16:36.076Z [INFO] server-857ff094-1b21-b984-7053-b8114b47284a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9bcf9833-a417-9f00-b3bf-8404d01e040d Address:9bcf9833-a417-9f00-b3bf-8404d01e040d} {Suffrage:Voter ID:server-857ff094-1b21-b984-7053-b8114b47284a Address:857ff094-1b21-b984-7053-b8114b47284a} {Suffrage:Voter ID:server-bf382df7-e928-c77b-f689-ad685747cee3 Address:bf382df7-e928-c77b-f689-ad685747cee3}]" +2025-06-15T03:16:36.082Z [INFO] server-bf382df7-e928-c77b-f689-ad685747cee3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9bcf9833-a417-9f00-b3bf-8404d01e040d Address:9bcf9833-a417-9f00-b3bf-8404d01e040d} {Suffrage:Voter ID:server-857ff094-1b21-b984-7053-b8114b47284a Address:857ff094-1b21-b984-7053-b8114b47284a} {Suffrage:Voter ID:server-bf382df7-e928-c77b-f689-ad685747cee3 Address:bf382df7-e928-c77b-f689-ad685747cee3}]" +2025-06-15T03:16:36.082Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:36.084Z [INFO] server-9bcf9833-a417-9f00-b3bf-8404d01e040d: entering follower state: follower="Node at 9bcf9833-a417-9f00-b3bf-8404d01e040d [Follower]" leader-address= leader-id= +2025-06-15T03:16:36.084Z [INFO] server-857ff094-1b21-b984-7053-b8114b47284a: entering follower state: follower="Node at 857ff094-1b21-b984-7053-b8114b47284a [Follower]" leader-address= leader-id= +2025-06-15T03:16:36.084Z [INFO] server-bf382df7-e928-c77b-f689-ad685747cee3: entering follower state: follower="Node at bf382df7-e928-c77b-f689-ad685747cee3 [Follower]" leader-address= leader-id= +2025-06-15T03:16:36.137Z [WARN] server-bf382df7-e928-c77b-f689-ad685747cee3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:36.137Z [INFO] server-bf382df7-e928-c77b-f689-ad685747cee3: entering candidate state: node="Node at bf382df7-e928-c77b-f689-ad685747cee3 [Candidate]" term=2 +2025-06-15T03:16:36.137Z [INFO] server-bf382df7-e928-c77b-f689-ad685747cee3: election won: term=2 tally=2 +2025-06-15T03:16:36.137Z [INFO] server-bf382df7-e928-c77b-f689-ad685747cee3: entering leader state: leader="Node at bf382df7-e928-c77b-f689-ad685747cee3 [Leader]" +2025-06-15T03:16:36.137Z [INFO] server-bf382df7-e928-c77b-f689-ad685747cee3: added peer, starting replication: peer=server-9bcf9833-a417-9f00-b3bf-8404d01e040d +2025-06-15T03:16:36.137Z [INFO] server-bf382df7-e928-c77b-f689-ad685747cee3: added peer, starting replication: peer=server-857ff094-1b21-b984-7053-b8114b47284a +2025-06-15T03:16:36.137Z [INFO] server-bf382df7-e928-c77b-f689-ad685747cee3: pipelining replication: peer="{Voter server-857ff094-1b21-b984-7053-b8114b47284a 857ff094-1b21-b984-7053-b8114b47284a}" +2025-06-15T03:16:36.137Z [INFO] server-bf382df7-e928-c77b-f689-ad685747cee3: pipelining replication: peer="{Voter server-9bcf9833-a417-9f00-b3bf-8404d01e040d 9bcf9833-a417-9f00-b3bf-8404d01e040d}" +2025-06-15T03:16:36.243Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:36.137924152 +0000 UTC m=+11.849602334 (1 nodes), 55.306161ms from start of poll, 61.763523ms from cluster start. Timeout at 2025-06-15 03:16:36.24343384 +0000 UTC m=+11.955112028, 105.509694ms after stability +2025-06-15T03:16:36.243Z [INFO] server-bf382df7-e928-c77b-f689-ad685747cee3: updating configuration: command=RemoveServer server-id=1bb94966-998e-0e0f-b7e0-25006775462e server-addr= servers="[{Suffrage:Voter ID:server-9bcf9833-a417-9f00-b3bf-8404d01e040d Address:9bcf9833-a417-9f00-b3bf-8404d01e040d} {Suffrage:Voter ID:server-857ff094-1b21-b984-7053-b8114b47284a Address:857ff094-1b21-b984-7053-b8114b47284a} {Suffrage:Voter ID:server-bf382df7-e928-c77b-f689-ad685747cee3 Address:bf382df7-e928-c77b-f689-ad685747cee3}]" +2025-06-15T03:16:36.243Z [INFO] server-bf382df7-e928-c77b-f689-ad685747cee3: aborting pipeline replication: peer="{Voter server-857ff094-1b21-b984-7053-b8114b47284a 857ff094-1b21-b984-7053-b8114b47284a}" +2025-06-15T03:16:36.243Z [INFO] server-bf382df7-e928-c77b-f689-ad685747cee3: aborting pipeline replication: peer="{Voter server-9bcf9833-a417-9f00-b3bf-8404d01e040d 9bcf9833-a417-9f00-b3bf-8404d01e040d}" +--- PASS: TestRaft_RemoveUnknownPeer (0.17s) === RUN TestRaft_SnapshotRestore -2024-05-12T20:50:02.191Z [INFO] server-8c4f88d0-ea79-a690-3930-34ed4faa4fdf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8c4f88d0-ea79-a690-3930-34ed4faa4fdf Address:8c4f88d0-ea79-a690-3930-34ed4faa4fdf}]" -2024-05-12T20:50:02.191Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:02.191Z [INFO] server-8c4f88d0-ea79-a690-3930-34ed4faa4fdf: entering follower state: follower="Node at 8c4f88d0-ea79-a690-3930-34ed4faa4fdf [Follower]" leader-address= leader-id= -2024-05-12T20:50:02.250Z [WARN] server-8c4f88d0-ea79-a690-3930-34ed4faa4fdf: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:02.250Z [INFO] server-8c4f88d0-ea79-a690-3930-34ed4faa4fdf: entering candidate state: node="Node at 8c4f88d0-ea79-a690-3930-34ed4faa4fdf [Candidate]" term=2 -2024-05-12T20:50:02.250Z [INFO] server-8c4f88d0-ea79-a690-3930-34ed4faa4fdf: election won: term=2 tally=1 -2024-05-12T20:50:02.250Z [INFO] server-8c4f88d0-ea79-a690-3930-34ed4faa4fdf: entering leader state: leader="Node at 8c4f88d0-ea79-a690-3930-34ed4faa4fdf [Leader]" -2024-05-12T20:50:02.358Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:02.250952243 +0000 UTC m=+13.531652568 (1 nodes), 59.229234ms from start of poll, 59.370961ms from cluster start. Timeout at 2024-05-12 20:50:02.358769841 +0000 UTC m=+13.639470168, 107.8176ms after stability -2024-05-12T20:50:02.362Z [INFO] server-8c4f88d0-ea79-a690-3930-34ed4faa4fdf: starting snapshot up to: index=102 -2024-05-12T20:50:02.362Z [INFO] creating new snapshot: path=/tmp/raft1009054680/snapshots/2-102-1715547002362.tmp -2024-05-12T20:50:02.363Z [INFO] server-8c4f88d0-ea79-a690-3930-34ed4faa4fdf: compacting logs: from=1 to=92 -2024-05-12T20:50:02.363Z [INFO] server-8c4f88d0-ea79-a690-3930-34ed4faa4fdf: snapshot complete up to: index=102 -2024-05-12T20:50:02.363Z [INFO] server-8c4f88d0-ea79-a690-3930-34ed4faa4fdf: starting restore from snapshot: id=2-102-1715547002362 last-index=102 last-term=2 size-in-bytes=693 -2024-05-12T20:50:02.363Z [INFO] server-8c4f88d0-ea79-a690-3930-34ed4faa4fdf: snapshot restore progress: id=2-102-1715547002362 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2024-05-12T20:50:02.363Z [INFO] server-8c4f88d0-ea79-a690-3930-34ed4faa4fdf: restored from snapshot: id=2-102-1715547002362 last-index=102 last-term=2 size-in-bytes=693 -2024-05-12T20:50:02.363Z [INFO] server-8c4f88d0-ea79-a690-3930-34ed4faa4fdf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8c4f88d0-ea79-a690-3930-34ed4faa4fdf Address:8c4f88d0-ea79-a690-3930-34ed4faa4fdf}]" ---- PASS: TestRaft_SnapshotRestore (0.17s) +2025-06-15T03:16:36.245Z [INFO] server-55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf Address:55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf}]" +2025-06-15T03:16:36.245Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:36.245Z [INFO] server-55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf: entering follower state: follower="Node at 55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf [Follower]" leader-address= leader-id= +2025-06-15T03:16:36.326Z [WARN] server-55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:36.326Z [INFO] server-55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf: entering candidate state: node="Node at 55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf [Candidate]" term=2 +2025-06-15T03:16:36.326Z [INFO] server-55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf: election won: term=2 tally=1 +2025-06-15T03:16:36.326Z [INFO] server-55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf: entering leader state: leader="Node at 55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf [Leader]" +2025-06-15T03:16:36.432Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:36.326328437 +0000 UTC m=+12.038006605 (1 nodes), 81.245969ms from start of poll, 81.41306ms from cluster start. Timeout at 2025-06-15 03:16:36.432409205 +0000 UTC m=+12.144087394, 106.080789ms after stability +2025-06-15T03:16:36.434Z [INFO] server-55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf: starting snapshot up to: index=102 +2025-06-15T03:16:36.434Z [INFO] creating new snapshot: path=/tmp/raft2937723696/snapshots/2-102-1749957396434.tmp +2025-06-15T03:16:36.435Z [INFO] server-55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf: compacting logs: from=1 to=92 +2025-06-15T03:16:36.435Z [INFO] server-55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf: snapshot complete up to: index=102 +2025-06-15T03:16:36.435Z [INFO] server-55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf: starting restore from snapshot: id=2-102-1749957396434 last-index=102 last-term=2 size-in-bytes=693 +2025-06-15T03:16:36.436Z [INFO] server-55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf: snapshot restore progress: id=2-102-1749957396434 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-06-15T03:16:36.436Z [INFO] server-55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf: restored from snapshot: id=2-102-1749957396434 last-index=102 last-term=2 size-in-bytes=693 +2025-06-15T03:16:36.436Z [INFO] server-55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf Address:55fffeeb-e141-ecc1-7c6c-6e0b303bbfdf}]" +--- PASS: TestRaft_SnapshotRestore (0.19s) === RUN TestRaft_RestoreSnapshotOnStartup_Monotonic -2024-05-12T20:50:02.364Z [INFO] server-60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6 Address:60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6}]" -2024-05-12T20:50:02.364Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:02.364Z [INFO] server-60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6: entering follower state: follower="Node at 60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6 [Follower]" leader-address= leader-id= -2024-05-12T20:50:02.446Z [WARN] server-60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:02.446Z [INFO] server-60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6: entering candidate state: node="Node at 60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6 [Candidate]" term=2 -2024-05-12T20:50:02.446Z [INFO] server-60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6: election won: term=2 tally=1 -2024-05-12T20:50:02.446Z [INFO] server-60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6: entering leader state: leader="Node at 60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6 [Leader]" -2024-05-12T20:50:02.554Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:02.446364256 +0000 UTC m=+13.727064677 (1 nodes), 81.563144ms from start of poll, 81.696287ms from cluster start. Timeout at 2024-05-12 20:50:02.554752713 +0000 UTC m=+13.835454246, 108.389569ms after stability -2024-05-12T20:50:02.567Z [INFO] server-60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6: starting snapshot up to: index=102 -2024-05-12T20:50:02.567Z [INFO] creating new snapshot: path=/tmp/raft3547244933/snapshots/2-102-1715547002567.tmp -2024-05-12T20:50:02.578Z [INFO] server-60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6: compacting logs: from=1 to=92 -2024-05-12T20:50:02.578Z [INFO] server-60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6: snapshot complete up to: index=102 -2024-05-12T20:50:02.579Z [INFO] server-60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6: starting restore from snapshot: id=2-102-1715547002567 last-index=102 last-term=2 size-in-bytes=693 -2024-05-12T20:50:02.579Z [INFO] server-60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6: snapshot restore progress: id=2-102-1715547002567 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2024-05-12T20:50:02.579Z [INFO] server-60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6: restored from snapshot: id=2-102-1715547002567 last-index=102 last-term=2 size-in-bytes=693 -2024-05-12T20:50:02.579Z [INFO] server-60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6 Address:60cb5bc0-23f7-87f0-5171-17e7e8ce2cb6}]" ---- PASS: TestRaft_RestoreSnapshotOnStartup_Monotonic (0.22s) +2025-06-15T03:16:36.437Z [INFO] server-290537a1-f0c8-ab53-58bb-a1ebceda62a4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-290537a1-f0c8-ab53-58bb-a1ebceda62a4 Address:290537a1-f0c8-ab53-58bb-a1ebceda62a4}]" +2025-06-15T03:16:36.437Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:36.437Z [INFO] server-290537a1-f0c8-ab53-58bb-a1ebceda62a4: entering follower state: follower="Node at 290537a1-f0c8-ab53-58bb-a1ebceda62a4 [Follower]" leader-address= leader-id= +2025-06-15T03:16:36.498Z [WARN] server-290537a1-f0c8-ab53-58bb-a1ebceda62a4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:36.498Z [INFO] server-290537a1-f0c8-ab53-58bb-a1ebceda62a4: entering candidate state: node="Node at 290537a1-f0c8-ab53-58bb-a1ebceda62a4 [Candidate]" term=2 +2025-06-15T03:16:36.498Z [INFO] server-290537a1-f0c8-ab53-58bb-a1ebceda62a4: election won: term=2 tally=1 +2025-06-15T03:16:36.498Z [INFO] server-290537a1-f0c8-ab53-58bb-a1ebceda62a4: entering leader state: leader="Node at 290537a1-f0c8-ab53-58bb-a1ebceda62a4 [Leader]" +2025-06-15T03:16:36.603Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:36.498517505 +0000 UTC m=+12.210195676 (1 nodes), 61.378775ms from start of poll, 61.550224ms from cluster start. Timeout at 2025-06-15 03:16:36.603662744 +0000 UTC m=+12.315340920, 105.145244ms after stability +2025-06-15T03:16:36.605Z [INFO] server-290537a1-f0c8-ab53-58bb-a1ebceda62a4: starting snapshot up to: index=102 +2025-06-15T03:16:36.605Z [INFO] creating new snapshot: path=/tmp/raft3619500273/snapshots/2-102-1749957396605.tmp +2025-06-15T03:16:36.606Z [INFO] server-290537a1-f0c8-ab53-58bb-a1ebceda62a4: compacting logs: from=1 to=92 +2025-06-15T03:16:36.606Z [INFO] server-290537a1-f0c8-ab53-58bb-a1ebceda62a4: snapshot complete up to: index=102 +2025-06-15T03:16:36.607Z [INFO] server-290537a1-f0c8-ab53-58bb-a1ebceda62a4: starting restore from snapshot: id=2-102-1749957396605 last-index=102 last-term=2 size-in-bytes=693 +2025-06-15T03:16:36.607Z [INFO] server-290537a1-f0c8-ab53-58bb-a1ebceda62a4: snapshot restore progress: id=2-102-1749957396605 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-06-15T03:16:36.607Z [INFO] server-290537a1-f0c8-ab53-58bb-a1ebceda62a4: restored from snapshot: id=2-102-1749957396605 last-index=102 last-term=2 size-in-bytes=693 +2025-06-15T03:16:36.607Z [INFO] server-290537a1-f0c8-ab53-58bb-a1ebceda62a4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-290537a1-f0c8-ab53-58bb-a1ebceda62a4 Address:290537a1-f0c8-ab53-58bb-a1ebceda62a4}]" +--- PASS: TestRaft_RestoreSnapshotOnStartup_Monotonic (0.17s) === RUN TestRaft_SnapshotRestore_Progress -2024-05-12T20:50:02.580Z [INFO] server-37771d2a-e4b9-7398-9438-bbb79edb2cf6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-37771d2a-e4b9-7398-9438-bbb79edb2cf6 Address:37771d2a-e4b9-7398-9438-bbb79edb2cf6}]" -2024-05-12T20:50:02.580Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:02.580Z [INFO] server-37771d2a-e4b9-7398-9438-bbb79edb2cf6: entering follower state: follower="Node at 37771d2a-e4b9-7398-9438-bbb79edb2cf6 [Follower]" leader-address= leader-id= -2024-05-12T20:50:02.650Z [WARN] server-37771d2a-e4b9-7398-9438-bbb79edb2cf6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:02.650Z [INFO] server-37771d2a-e4b9-7398-9438-bbb79edb2cf6: entering candidate state: node="Node at 37771d2a-e4b9-7398-9438-bbb79edb2cf6 [Candidate]" term=2 -2024-05-12T20:50:02.650Z [INFO] server-37771d2a-e4b9-7398-9438-bbb79edb2cf6: election won: term=2 tally=1 -2024-05-12T20:50:02.651Z [INFO] server-37771d2a-e4b9-7398-9438-bbb79edb2cf6: entering leader state: leader="Node at 37771d2a-e4b9-7398-9438-bbb79edb2cf6 [Leader]" -2024-05-12T20:50:02.758Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:02.651099328 +0000 UTC m=+13.931799674 (1 nodes), 70.313769ms from start of poll, 70.48373ms from cluster start. Timeout at 2024-05-12 20:50:02.7587598 +0000 UTC m=+14.039460703, 107.661029ms after stability -2024-05-12T20:50:02.762Z [INFO] server-37771d2a-e4b9-7398-9438-bbb79edb2cf6: starting snapshot up to: index=102 -2024-05-12T20:50:02.763Z [INFO] creating new snapshot: path=/tmp/raft2862533529/snapshots/2-102-1715547002763.tmp -2024-05-12T20:50:02.764Z [INFO] server-37771d2a-e4b9-7398-9438-bbb79edb2cf6: compacting logs: from=1 to=92 -2024-05-12T20:50:02.764Z [INFO] server-37771d2a-e4b9-7398-9438-bbb79edb2cf6: snapshot complete up to: index=102 +2025-06-15T03:16:36.608Z [INFO] server-070ded31-24b6-39d3-db6a-d4ff16345a48: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-070ded31-24b6-39d3-db6a-d4ff16345a48 Address:070ded31-24b6-39d3-db6a-d4ff16345a48}]" +2025-06-15T03:16:36.608Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:36.608Z [INFO] server-070ded31-24b6-39d3-db6a-d4ff16345a48: entering follower state: follower="Node at 070ded31-24b6-39d3-db6a-d4ff16345a48 [Follower]" leader-address= leader-id= +2025-06-15T03:16:36.680Z [WARN] server-070ded31-24b6-39d3-db6a-d4ff16345a48: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:36.680Z [INFO] server-070ded31-24b6-39d3-db6a-d4ff16345a48: entering candidate state: node="Node at 070ded31-24b6-39d3-db6a-d4ff16345a48 [Candidate]" term=2 +2025-06-15T03:16:36.680Z [INFO] server-070ded31-24b6-39d3-db6a-d4ff16345a48: election won: term=2 tally=1 +2025-06-15T03:16:36.680Z [INFO] server-070ded31-24b6-39d3-db6a-d4ff16345a48: entering leader state: leader="Node at 070ded31-24b6-39d3-db6a-d4ff16345a48 [Leader]" +2025-06-15T03:16:36.786Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:36.680279374 +0000 UTC m=+12.391957544 (1 nodes), 72.016462ms from start of poll, 72.190978ms from cluster start. Timeout at 2025-06-15 03:16:36.786246722 +0000 UTC m=+12.497924908, 105.967364ms after stability +2025-06-15T03:16:36.788Z [INFO] server-070ded31-24b6-39d3-db6a-d4ff16345a48: starting snapshot up to: index=102 +2025-06-15T03:16:36.788Z [INFO] creating new snapshot: path=/tmp/raft3743187595/snapshots/2-102-1749957396788.tmp +2025-06-15T03:16:36.789Z [INFO] server-070ded31-24b6-39d3-db6a-d4ff16345a48: compacting logs: from=1 to=92 +2025-06-15T03:16:36.789Z [INFO] server-070ded31-24b6-39d3-db6a-d4ff16345a48: snapshot complete up to: index=102 --- PASS: TestRaft_SnapshotRestore_Progress (0.19s) === RUN TestRaft_NoRestoreOnStart -2024-05-12T20:50:02.766Z [INFO] server-5731b5cb-08e3-a31f-0da7-5892bc512abc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5731b5cb-08e3-a31f-0da7-5892bc512abc Address:5731b5cb-08e3-a31f-0da7-5892bc512abc}]" -2024-05-12T20:50:02.766Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:02.766Z [INFO] server-5731b5cb-08e3-a31f-0da7-5892bc512abc: entering follower state: follower="Node at 5731b5cb-08e3-a31f-0da7-5892bc512abc [Follower]" leader-address= leader-id= -2024-05-12T20:50:02.834Z [WARN] server-5731b5cb-08e3-a31f-0da7-5892bc512abc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:02.834Z [INFO] server-5731b5cb-08e3-a31f-0da7-5892bc512abc: entering candidate state: node="Node at 5731b5cb-08e3-a31f-0da7-5892bc512abc [Candidate]" term=2 -2024-05-12T20:50:02.834Z [INFO] server-5731b5cb-08e3-a31f-0da7-5892bc512abc: election won: term=2 tally=1 -2024-05-12T20:50:02.834Z [INFO] server-5731b5cb-08e3-a31f-0da7-5892bc512abc: entering leader state: leader="Node at 5731b5cb-08e3-a31f-0da7-5892bc512abc [Leader]" -2024-05-12T20:50:02.942Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:02.835110319 +0000 UTC m=+14.115810680 (1 nodes), 69.043096ms from start of poll, 69.218416ms from cluster start. Timeout at 2024-05-12 20:50:02.942758768 +0000 UTC m=+14.223462447, 107.651767ms after stability -2024-05-12T20:50:02.946Z [INFO] server-5731b5cb-08e3-a31f-0da7-5892bc512abc: starting snapshot up to: index=102 -2024-05-12T20:50:02.946Z [INFO] creating new snapshot: path=/tmp/raft4074156082/snapshots/2-102-1715547002946.tmp -2024-05-12T20:50:02.948Z [INFO] server-5731b5cb-08e3-a31f-0da7-5892bc512abc: compacting logs: from=1 to=92 -2024-05-12T20:50:02.948Z [INFO] server-5731b5cb-08e3-a31f-0da7-5892bc512abc: snapshot complete up to: index=102 -2024-05-12T20:50:02.948Z [INFO] server-5731b5cb-08e3-a31f-0da7-5892bc512abc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5731b5cb-08e3-a31f-0da7-5892bc512abc Address:5731b5cb-08e3-a31f-0da7-5892bc512abc}]" ---- PASS: TestRaft_NoRestoreOnStart (0.18s) +2025-06-15T03:16:36.795Z [INFO] server-e0e77d70-b592-17f4-2d52-465772a65f92: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e0e77d70-b592-17f4-2d52-465772a65f92 Address:e0e77d70-b592-17f4-2d52-465772a65f92}]" +2025-06-15T03:16:36.795Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:36.795Z [INFO] server-e0e77d70-b592-17f4-2d52-465772a65f92: entering follower state: follower="Node at e0e77d70-b592-17f4-2d52-465772a65f92 [Follower]" leader-address= leader-id= +2025-06-15T03:16:36.870Z [WARN] server-e0e77d70-b592-17f4-2d52-465772a65f92: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:36.871Z [INFO] server-e0e77d70-b592-17f4-2d52-465772a65f92: entering candidate state: node="Node at e0e77d70-b592-17f4-2d52-465772a65f92 [Candidate]" term=2 +2025-06-15T03:16:36.871Z [INFO] server-e0e77d70-b592-17f4-2d52-465772a65f92: election won: term=2 tally=1 +2025-06-15T03:16:36.871Z [INFO] server-e0e77d70-b592-17f4-2d52-465772a65f92: entering leader state: leader="Node at e0e77d70-b592-17f4-2d52-465772a65f92 [Leader]" +2025-06-15T03:16:36.977Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:36.871900628 +0000 UTC m=+12.583578813 (1 nodes), 76.644138ms from start of poll, 76.821468ms from cluster start. Timeout at 2025-06-15 03:16:36.977242155 +0000 UTC m=+12.688920347, 105.341534ms after stability +2025-06-15T03:16:36.979Z [INFO] server-e0e77d70-b592-17f4-2d52-465772a65f92: starting snapshot up to: index=102 +2025-06-15T03:16:36.979Z [INFO] creating new snapshot: path=/tmp/raft4104910984/snapshots/2-102-1749957396979.tmp +2025-06-15T03:16:36.980Z [INFO] server-e0e77d70-b592-17f4-2d52-465772a65f92: compacting logs: from=1 to=92 +2025-06-15T03:16:36.980Z [INFO] server-e0e77d70-b592-17f4-2d52-465772a65f92: snapshot complete up to: index=102 +2025-06-15T03:16:36.980Z [INFO] server-e0e77d70-b592-17f4-2d52-465772a65f92: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e0e77d70-b592-17f4-2d52-465772a65f92 Address:e0e77d70-b592-17f4-2d52-465772a65f92}]" +--- PASS: TestRaft_NoRestoreOnStart (0.19s) === RUN TestRaft_SnapshotRestore_PeerChange raft_test.go:1266: Skipping unreliable test --- SKIP: TestRaft_SnapshotRestore_PeerChange (0.00s) === RUN TestRaft_AutoSnapshot -2024-05-12T20:50:02.948Z [INFO] server-7a67e679-b8ce-5a6f-bf89-f81d499c3a35: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a67e679-b8ce-5a6f-bf89-f81d499c3a35 Address:7a67e679-b8ce-5a6f-bf89-f81d499c3a35}]" -2024-05-12T20:50:02.949Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:02.949Z [INFO] server-5731b5cb-08e3-a31f-0da7-5892bc512abc: entering follower state: follower="Node at 5731b5cb-08e3-a31f-0da7-5892bc512abc [Follower]" leader-address= leader-id= -2024-05-12T20:50:02.949Z [INFO] server-7a67e679-b8ce-5a6f-bf89-f81d499c3a35: entering follower state: follower="Node at 7a67e679-b8ce-5a6f-bf89-f81d499c3a35 [Follower]" leader-address= leader-id= -2024-05-12T20:50:03.006Z [WARN] server-7a67e679-b8ce-5a6f-bf89-f81d499c3a35: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:03.006Z [INFO] server-7a67e679-b8ce-5a6f-bf89-f81d499c3a35: entering candidate state: node="Node at 7a67e679-b8ce-5a6f-bf89-f81d499c3a35 [Candidate]" term=2 -2024-05-12T20:50:03.007Z [INFO] server-7a67e679-b8ce-5a6f-bf89-f81d499c3a35: election won: term=2 tally=1 -2024-05-12T20:50:03.007Z [INFO] server-7a67e679-b8ce-5a6f-bf89-f81d499c3a35: entering leader state: leader="Node at 7a67e679-b8ce-5a6f-bf89-f81d499c3a35 [Leader]" -2024-05-12T20:50:03.014Z [WARN] server-5731b5cb-08e3-a31f-0da7-5892bc512abc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:03.014Z [INFO] server-5731b5cb-08e3-a31f-0da7-5892bc512abc: entering candidate state: node="Node at 5731b5cb-08e3-a31f-0da7-5892bc512abc [Candidate]" term=3 -2024-05-12T20:50:03.014Z [INFO] server-5731b5cb-08e3-a31f-0da7-5892bc512abc: election won: term=3 tally=1 -2024-05-12T20:50:03.014Z [INFO] server-5731b5cb-08e3-a31f-0da7-5892bc512abc: entering leader state: leader="Node at 5731b5cb-08e3-a31f-0da7-5892bc512abc [Leader]" -2024-05-12T20:50:03.114Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:03.007156651 +0000 UTC m=+14.287857062 (1 nodes), 58.139451ms from start of poll, 58.300207ms from cluster start. Timeout at 2024-05-12 20:50:03.114797418 +0000 UTC m=+14.395497847, 107.640785ms after stability -2024-05-12T20:50:03.119Z [INFO] server-7a67e679-b8ce-5a6f-bf89-f81d499c3a35: starting snapshot up to: index=72 -2024-05-12T20:50:03.119Z [INFO] creating new snapshot: path=/tmp/raft346967002/snapshots/2-72-1715547003119.tmp -2024-05-12T20:50:03.120Z [INFO] server-7a67e679-b8ce-5a6f-bf89-f81d499c3a35: compacting logs: from=1 to=72 -2024-05-12T20:50:03.120Z [INFO] server-7a67e679-b8ce-5a6f-bf89-f81d499c3a35: snapshot complete up to: index=72 +2025-06-15T03:16:36.980Z [INFO] server-122c4315-bdc9-ba41-c64c-fa93f316e7f1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-122c4315-bdc9-ba41-c64c-fa93f316e7f1 Address:122c4315-bdc9-ba41-c64c-fa93f316e7f1}]" +2025-06-15T03:16:36.980Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:36.980Z [INFO] server-e0e77d70-b592-17f4-2d52-465772a65f92: entering follower state: follower="Node at e0e77d70-b592-17f4-2d52-465772a65f92 [Follower]" leader-address= leader-id= +2025-06-15T03:16:36.980Z [INFO] server-122c4315-bdc9-ba41-c64c-fa93f316e7f1: entering follower state: follower="Node at 122c4315-bdc9-ba41-c64c-fa93f316e7f1 [Follower]" leader-address= leader-id= +2025-06-15T03:16:37.045Z [WARN] server-122c4315-bdc9-ba41-c64c-fa93f316e7f1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:37.045Z [INFO] server-122c4315-bdc9-ba41-c64c-fa93f316e7f1: entering candidate state: node="Node at 122c4315-bdc9-ba41-c64c-fa93f316e7f1 [Candidate]" term=2 +2025-06-15T03:16:37.045Z [INFO] server-122c4315-bdc9-ba41-c64c-fa93f316e7f1: election won: term=2 tally=1 +2025-06-15T03:16:37.045Z [INFO] server-122c4315-bdc9-ba41-c64c-fa93f316e7f1: entering leader state: leader="Node at 122c4315-bdc9-ba41-c64c-fa93f316e7f1 [Leader]" +2025-06-15T03:16:37.053Z [WARN] server-e0e77d70-b592-17f4-2d52-465772a65f92: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:37.053Z [INFO] server-e0e77d70-b592-17f4-2d52-465772a65f92: entering candidate state: node="Node at e0e77d70-b592-17f4-2d52-465772a65f92 [Candidate]" term=3 +2025-06-15T03:16:37.053Z [INFO] server-e0e77d70-b592-17f4-2d52-465772a65f92: election won: term=3 tally=1 +2025-06-15T03:16:37.053Z [INFO] server-e0e77d70-b592-17f4-2d52-465772a65f92: entering leader state: leader="Node at e0e77d70-b592-17f4-2d52-465772a65f92 [Leader]" +2025-06-15T03:16:37.151Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:37.045477893 +0000 UTC m=+12.757156071 (1 nodes), 64.672325ms from start of poll, 64.834283ms from cluster start. Timeout at 2025-06-15 03:16:37.151263958 +0000 UTC m=+12.862942152, 105.786081ms after stability +2025-06-15T03:16:37.158Z [INFO] server-122c4315-bdc9-ba41-c64c-fa93f316e7f1: starting snapshot up to: index=102 +2025-06-15T03:16:37.158Z [INFO] creating new snapshot: path=/tmp/raft3463299753/snapshots/2-102-1749957397158.tmp +2025-06-15T03:16:37.159Z [INFO] server-122c4315-bdc9-ba41-c64c-fa93f316e7f1: compacting logs: from=1 to=92 +2025-06-15T03:16:37.159Z [INFO] server-122c4315-bdc9-ba41-c64c-fa93f316e7f1: snapshot complete up to: index=102 --- PASS: TestRaft_AutoSnapshot (0.28s) === RUN TestRaft_UserSnapshot -2024-05-12T20:50:03.228Z [INFO] server-9af45c77-3d76-1995-cb82-8c7cb4e9b5e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9af45c77-3d76-1995-cb82-8c7cb4e9b5e9 Address:9af45c77-3d76-1995-cb82-8c7cb4e9b5e9}]" -2024-05-12T20:50:03.228Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:03.228Z [INFO] server-9af45c77-3d76-1995-cb82-8c7cb4e9b5e9: entering follower state: follower="Node at 9af45c77-3d76-1995-cb82-8c7cb4e9b5e9 [Follower]" leader-address= leader-id= -2024-05-12T20:50:03.302Z [WARN] server-9af45c77-3d76-1995-cb82-8c7cb4e9b5e9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:03.302Z [INFO] server-9af45c77-3d76-1995-cb82-8c7cb4e9b5e9: entering candidate state: node="Node at 9af45c77-3d76-1995-cb82-8c7cb4e9b5e9 [Candidate]" term=2 -2024-05-12T20:50:03.302Z [INFO] server-9af45c77-3d76-1995-cb82-8c7cb4e9b5e9: election won: term=2 tally=1 -2024-05-12T20:50:03.302Z [INFO] server-9af45c77-3d76-1995-cb82-8c7cb4e9b5e9: entering leader state: leader="Node at 9af45c77-3d76-1995-cb82-8c7cb4e9b5e9 [Leader]" -2024-05-12T20:50:03.410Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:03.303152519 +0000 UTC m=+14.583852920 (1 nodes), 75.087265ms from start of poll, 75.27062ms from cluster start. Timeout at 2024-05-12 20:50:03.410770391 +0000 UTC m=+14.691470773, 107.617853ms after stability -2024-05-12T20:50:03.410Z [ERROR] server-9af45c77-3d76-1995-cb82-8c7cb4e9b5e9: failed to take snapshot: error="nothing new to snapshot" -2024-05-12T20:50:03.411Z [INFO] server-9af45c77-3d76-1995-cb82-8c7cb4e9b5e9: starting snapshot up to: index=12 -2024-05-12T20:50:03.411Z [INFO] creating new snapshot: path=/tmp/raft3672730286/snapshots/2-12-1715547003411.tmp -2024-05-12T20:50:03.412Z [INFO] server-9af45c77-3d76-1995-cb82-8c7cb4e9b5e9: compacting logs: from=1 to=2 -2024-05-12T20:50:03.412Z [INFO] server-9af45c77-3d76-1995-cb82-8c7cb4e9b5e9: snapshot complete up to: index=12 ---- PASS: TestRaft_UserSnapshot (0.19s) +2025-06-15T03:16:37.262Z [INFO] server-8d44c976-42ba-aaa4-1b53-2122233795f2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8d44c976-42ba-aaa4-1b53-2122233795f2 Address:8d44c976-42ba-aaa4-1b53-2122233795f2}]" +2025-06-15T03:16:37.262Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:37.262Z [INFO] server-8d44c976-42ba-aaa4-1b53-2122233795f2: entering follower state: follower="Node at 8d44c976-42ba-aaa4-1b53-2122233795f2 [Follower]" leader-address= leader-id= +2025-06-15T03:16:37.366Z [WARN] server-8d44c976-42ba-aaa4-1b53-2122233795f2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:37.366Z [INFO] server-8d44c976-42ba-aaa4-1b53-2122233795f2: entering candidate state: node="Node at 8d44c976-42ba-aaa4-1b53-2122233795f2 [Candidate]" term=2 +2025-06-15T03:16:37.366Z [INFO] server-8d44c976-42ba-aaa4-1b53-2122233795f2: election won: term=2 tally=1 +2025-06-15T03:16:37.366Z [INFO] server-8d44c976-42ba-aaa4-1b53-2122233795f2: entering leader state: leader="Node at 8d44c976-42ba-aaa4-1b53-2122233795f2 [Leader]" +2025-06-15T03:16:37.471Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:37.366372279 +0000 UTC m=+13.078050461 (1 nodes), 103.66195ms from start of poll, 103.875073ms from cluster start. Timeout at 2025-06-15 03:16:37.471674473 +0000 UTC m=+13.183352654, 105.302193ms after stability +2025-06-15T03:16:37.471Z [ERROR] server-8d44c976-42ba-aaa4-1b53-2122233795f2: failed to take snapshot: error="nothing new to snapshot" +2025-06-15T03:16:37.472Z [INFO] server-8d44c976-42ba-aaa4-1b53-2122233795f2: starting snapshot up to: index=12 +2025-06-15T03:16:37.472Z [INFO] creating new snapshot: path=/tmp/raft2410151988/snapshots/2-12-1749957397472.tmp +2025-06-15T03:16:37.472Z [INFO] server-8d44c976-42ba-aaa4-1b53-2122233795f2: compacting logs: from=1 to=2 +2025-06-15T03:16:37.472Z [INFO] server-8d44c976-42ba-aaa4-1b53-2122233795f2: snapshot complete up to: index=12 +--- PASS: TestRaft_UserSnapshot (0.21s) === RUN TestRaft_UserRestore === RUN TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false -2024-05-12T20:50:03.413Z [INFO] server-c148ccf4-b148-92dc-2707-fb77fc6d2827: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c148ccf4-b148-92dc-2707-fb77fc6d2827 Address:c148ccf4-b148-92dc-2707-fb77fc6d2827} {Suffrage:Voter ID:server-93db09aa-51da-3b65-e149-8f4b2ff81bb1 Address:93db09aa-51da-3b65-e149-8f4b2ff81bb1} {Suffrage:Voter ID:server-bed68dbe-366d-9d01-a2b0-79b5216bdf92 Address:bed68dbe-366d-9d01-a2b0-79b5216bdf92}]" -2024-05-12T20:50:03.414Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c148ccf4-b148-92dc-2707-fb77fc6d2827 Address:c148ccf4-b148-92dc-2707-fb77fc6d2827} {Suffrage:Voter ID:server-93db09aa-51da-3b65-e149-8f4b2ff81bb1 Address:93db09aa-51da-3b65-e149-8f4b2ff81bb1} {Suffrage:Voter ID:server-bed68dbe-366d-9d01-a2b0-79b5216bdf92 Address:bed68dbe-366d-9d01-a2b0-79b5216bdf92}]" -2024-05-12T20:50:03.414Z [INFO] server-bed68dbe-366d-9d01-a2b0-79b5216bdf92: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c148ccf4-b148-92dc-2707-fb77fc6d2827 Address:c148ccf4-b148-92dc-2707-fb77fc6d2827} {Suffrage:Voter ID:server-93db09aa-51da-3b65-e149-8f4b2ff81bb1 Address:93db09aa-51da-3b65-e149-8f4b2ff81bb1} {Suffrage:Voter ID:server-bed68dbe-366d-9d01-a2b0-79b5216bdf92 Address:bed68dbe-366d-9d01-a2b0-79b5216bdf92}]" -2024-05-12T20:50:03.414Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:03.414Z [INFO] server-c148ccf4-b148-92dc-2707-fb77fc6d2827: entering follower state: follower="Node at c148ccf4-b148-92dc-2707-fb77fc6d2827 [Follower]" leader-address= leader-id= -2024-05-12T20:50:03.414Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: entering follower state: follower="Node at 93db09aa-51da-3b65-e149-8f4b2ff81bb1 [Follower]" leader-address= leader-id= -2024-05-12T20:50:03.414Z [INFO] server-bed68dbe-366d-9d01-a2b0-79b5216bdf92: entering follower state: follower="Node at bed68dbe-366d-9d01-a2b0-79b5216bdf92 [Follower]" leader-address= leader-id= -2024-05-12T20:50:04.154Z [WARN] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:04.154Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: entering candidate state: node="Node at 93db09aa-51da-3b65-e149-8f4b2ff81bb1 [Candidate]" term=2 -2024-05-12T20:50:04.155Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: election won: term=2 tally=2 -2024-05-12T20:50:04.155Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: entering leader state: leader="Node at 93db09aa-51da-3b65-e149-8f4b2ff81bb1 [Leader]" -2024-05-12T20:50:04.155Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: added peer, starting replication: peer=server-c148ccf4-b148-92dc-2707-fb77fc6d2827 -2024-05-12T20:50:04.155Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: added peer, starting replication: peer=server-bed68dbe-366d-9d01-a2b0-79b5216bdf92 -2024-05-12T20:50:04.155Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: pipelining replication: peer="{Voter server-bed68dbe-366d-9d01-a2b0-79b5216bdf92 bed68dbe-366d-9d01-a2b0-79b5216bdf92}" -2024-05-12T20:50:04.155Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: pipelining replication: peer="{Voter server-c148ccf4-b148-92dc-2707-fb77fc6d2827 c148ccf4-b148-92dc-2707-fb77fc6d2827}" -2024-05-12T20:50:05.162Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:04.155724207 +0000 UTC m=+15.436424607 (1 nodes), 741.505396ms from start of poll, 741.926358ms from cluster start. Timeout at 2024-05-12 20:50:05.162776611 +0000 UTC m=+16.443476506, 1.007051899s after stability -2024-05-12T20:50:05.164Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: starting snapshot up to: index=12 -2024-05-12T20:50:05.164Z [INFO] creating new snapshot: path=/tmp/raft4252098742/snapshots/2-12-1715547005164.tmp -2024-05-12T20:50:05.165Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: snapshot complete up to: index=12 -2024-05-12T20:50:05.167Z [INFO] creating new snapshot: path=/tmp/raft4252098742/snapshots/2-23-1715547005167.tmp -2024-05-12T20:50:05.168Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: copied to local snapshot: bytes=71 -2024-05-12T20:50:05.168Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: snapshot restore progress: id=2-23-1715547005167 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:05.168Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: restored user snapshot: index=23 -2024-05-12T20:50:05.168Z [ERROR] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: failed to get log: index=23 error="log not found" -2024-05-12T20:50:05.168Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: aborting pipeline replication: peer="{Voter server-bed68dbe-366d-9d01-a2b0-79b5216bdf92 bed68dbe-366d-9d01-a2b0-79b5216bdf92}" -2024-05-12T20:50:05.169Z [ERROR] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: failed to get log: index=23 error="log not found" -2024-05-12T20:50:05.169Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: aborting pipeline replication: peer="{Voter server-c148ccf4-b148-92dc-2707-fb77fc6d2827 c148ccf4-b148-92dc-2707-fb77fc6d2827}" -2024-05-12T20:50:05.178Z [ERROR] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: failed to get log: index=23 error="log not found" -2024-05-12T20:50:05.179Z [ERROR] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: failed to get log: index=23 error="log not found" -2024-05-12T20:50:05.180Z [INFO] creating new snapshot: path=/tmp/raft3381804302/snapshots/2-23-1715547005180.tmp -2024-05-12T20:50:05.180Z [INFO] server-c148ccf4-b148-92dc-2707-fb77fc6d2827: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:05.181Z [INFO] server-c148ccf4-b148-92dc-2707-fb77fc6d2827: copied to local snapshot: bytes=71 -2024-05-12T20:50:05.181Z [INFO] server-c148ccf4-b148-92dc-2707-fb77fc6d2827: snapshot restore progress: id=2-23-1715547005180 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:05.181Z [INFO] server-c148ccf4-b148-92dc-2707-fb77fc6d2827: Installed remote snapshot -2024-05-12T20:50:05.181Z [WARN] server-c148ccf4-b148-92dc-2707-fb77fc6d2827: skipping application of old log: index=22 -2024-05-12T20:50:05.182Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: pipelining replication: peer="{Voter server-c148ccf4-b148-92dc-2707-fb77fc6d2827 c148ccf4-b148-92dc-2707-fb77fc6d2827}" -2024-05-12T20:50:05.182Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:05.185Z [INFO] creating new snapshot: path=/tmp/raft3109179936/snapshots/2-23-1715547005185.tmp -2024-05-12T20:50:05.185Z [INFO] server-bed68dbe-366d-9d01-a2b0-79b5216bdf92: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:05.186Z [INFO] server-bed68dbe-366d-9d01-a2b0-79b5216bdf92: copied to local snapshot: bytes=71 -2024-05-12T20:50:05.186Z [INFO] server-bed68dbe-366d-9d01-a2b0-79b5216bdf92: snapshot restore progress: id=2-23-1715547005185 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:05.186Z [INFO] server-bed68dbe-366d-9d01-a2b0-79b5216bdf92: Installed remote snapshot -2024-05-12T20:50:05.187Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: pipelining replication: peer="{Voter server-bed68dbe-366d-9d01-a2b0-79b5216bdf92 bed68dbe-366d-9d01-a2b0-79b5216bdf92}" -2024-05-12T20:50:06.190Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:05.182114412 +0000 UTC m=+16.462814850 (2 nodes), 2.265µs from start of poll, 1.768316601s from cluster start. Timeout at 2024-05-12 20:50:06.190957722 +0000 UTC m=+17.471658195, 1.008843345s after stability -2024-05-12T20:50:06.206Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: aborting pipeline replication: peer="{Voter server-bed68dbe-366d-9d01-a2b0-79b5216bdf92 bed68dbe-366d-9d01-a2b0-79b5216bdf92}" -2024-05-12T20:50:06.207Z [INFO] server-93db09aa-51da-3b65-e149-8f4b2ff81bb1: aborting pipeline replication: peer="{Voter server-c148ccf4-b148-92dc-2707-fb77fc6d2827 c148ccf4-b148-92dc-2707-fb77fc6d2827}" +2025-06-15T03:16:37.474Z [INFO] server-51ad3a78-bb1b-9488-2c71-4a79f4938d7f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-51ad3a78-bb1b-9488-2c71-4a79f4938d7f Address:51ad3a78-bb1b-9488-2c71-4a79f4938d7f} {Suffrage:Voter ID:server-5f02237a-4279-31d6-c161-7a2fdb11fada Address:5f02237a-4279-31d6-c161-7a2fdb11fada} {Suffrage:Voter ID:server-a6aa7bce-e31e-238b-f88b-948719a2e4a2 Address:a6aa7bce-e31e-238b-f88b-948719a2e4a2}]" +2025-06-15T03:16:37.474Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-51ad3a78-bb1b-9488-2c71-4a79f4938d7f Address:51ad3a78-bb1b-9488-2c71-4a79f4938d7f} {Suffrage:Voter ID:server-5f02237a-4279-31d6-c161-7a2fdb11fada Address:5f02237a-4279-31d6-c161-7a2fdb11fada} {Suffrage:Voter ID:server-a6aa7bce-e31e-238b-f88b-948719a2e4a2 Address:a6aa7bce-e31e-238b-f88b-948719a2e4a2}]" +2025-06-15T03:16:37.474Z [INFO] server-a6aa7bce-e31e-238b-f88b-948719a2e4a2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-51ad3a78-bb1b-9488-2c71-4a79f4938d7f Address:51ad3a78-bb1b-9488-2c71-4a79f4938d7f} {Suffrage:Voter ID:server-5f02237a-4279-31d6-c161-7a2fdb11fada Address:5f02237a-4279-31d6-c161-7a2fdb11fada} {Suffrage:Voter ID:server-a6aa7bce-e31e-238b-f88b-948719a2e4a2 Address:a6aa7bce-e31e-238b-f88b-948719a2e4a2}]" +2025-06-15T03:16:37.474Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:37.474Z [INFO] server-51ad3a78-bb1b-9488-2c71-4a79f4938d7f: entering follower state: follower="Node at 51ad3a78-bb1b-9488-2c71-4a79f4938d7f [Follower]" leader-address= leader-id= +2025-06-15T03:16:37.474Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: entering follower state: follower="Node at 5f02237a-4279-31d6-c161-7a2fdb11fada [Follower]" leader-address= leader-id= +2025-06-15T03:16:37.474Z [INFO] server-a6aa7bce-e31e-238b-f88b-948719a2e4a2: entering follower state: follower="Node at a6aa7bce-e31e-238b-f88b-948719a2e4a2 [Follower]" leader-address= leader-id= +2025-06-15T03:16:38.082Z [WARN] server-5f02237a-4279-31d6-c161-7a2fdb11fada: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:38.083Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: entering candidate state: node="Node at 5f02237a-4279-31d6-c161-7a2fdb11fada [Candidate]" term=2 +2025-06-15T03:16:38.083Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: election won: term=2 tally=2 +2025-06-15T03:16:38.083Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: entering leader state: leader="Node at 5f02237a-4279-31d6-c161-7a2fdb11fada [Leader]" +2025-06-15T03:16:38.083Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: added peer, starting replication: peer=server-51ad3a78-bb1b-9488-2c71-4a79f4938d7f +2025-06-15T03:16:38.083Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: added peer, starting replication: peer=server-a6aa7bce-e31e-238b-f88b-948719a2e4a2 +2025-06-15T03:16:38.083Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: pipelining replication: peer="{Voter server-a6aa7bce-e31e-238b-f88b-948719a2e4a2 a6aa7bce-e31e-238b-f88b-948719a2e4a2}" +2025-06-15T03:16:38.083Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: pipelining replication: peer="{Voter server-51ad3a78-bb1b-9488-2c71-4a79f4938d7f 51ad3a78-bb1b-9488-2c71-4a79f4938d7f}" +2025-06-15T03:16:39.089Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:38.083571623 +0000 UTC m=+13.795249809 (1 nodes), 609.152656ms from start of poll, 609.611715ms from cluster start. Timeout at 2025-06-15 03:16:39.0894822 +0000 UTC m=+14.801160386, 1.005910577s after stability +2025-06-15T03:16:39.090Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: starting snapshot up to: index=12 +2025-06-15T03:16:39.090Z [INFO] creating new snapshot: path=/tmp/raft3097354575/snapshots/2-12-1749957399090.tmp +2025-06-15T03:16:39.123Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: snapshot complete up to: index=12 +2025-06-15T03:16:39.124Z [INFO] creating new snapshot: path=/tmp/raft3097354575/snapshots/2-23-1749957399124.tmp +2025-06-15T03:16:39.125Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: copied to local snapshot: bytes=71 +2025-06-15T03:16:39.125Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: snapshot restore progress: id=2-23-1749957399124 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:39.125Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: restored user snapshot: index=23 +2025-06-15T03:16:39.125Z [ERROR] server-5f02237a-4279-31d6-c161-7a2fdb11fada: failed to get log: index=23 error="log not found" +2025-06-15T03:16:39.125Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: aborting pipeline replication: peer="{Voter server-a6aa7bce-e31e-238b-f88b-948719a2e4a2 a6aa7bce-e31e-238b-f88b-948719a2e4a2}" +2025-06-15T03:16:39.125Z [ERROR] server-5f02237a-4279-31d6-c161-7a2fdb11fada: failed to get log: index=23 error="log not found" +2025-06-15T03:16:39.125Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: aborting pipeline replication: peer="{Voter server-51ad3a78-bb1b-9488-2c71-4a79f4938d7f 51ad3a78-bb1b-9488-2c71-4a79f4938d7f}" +2025-06-15T03:16:39.131Z [ERROR] server-5f02237a-4279-31d6-c161-7a2fdb11fada: failed to get log: index=23 error="log not found" +2025-06-15T03:16:39.132Z [INFO] creating new snapshot: path=/tmp/raft1191549473/snapshots/2-23-1749957399132.tmp +2025-06-15T03:16:39.132Z [INFO] server-a6aa7bce-e31e-238b-f88b-948719a2e4a2: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:39.133Z [ERROR] server-5f02237a-4279-31d6-c161-7a2fdb11fada: failed to get log: index=23 error="log not found" +2025-06-15T03:16:39.133Z [INFO] creating new snapshot: path=/tmp/raft4154069581/snapshots/2-23-1749957399133.tmp +2025-06-15T03:16:39.134Z [INFO] server-51ad3a78-bb1b-9488-2c71-4a79f4938d7f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:39.341Z [INFO] server-51ad3a78-bb1b-9488-2c71-4a79f4938d7f: copied to local snapshot: bytes=71 +2025-06-15T03:16:39.342Z [INFO] server-51ad3a78-bb1b-9488-2c71-4a79f4938d7f: snapshot restore progress: id=2-23-1749957399133 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:39.342Z [INFO] server-51ad3a78-bb1b-9488-2c71-4a79f4938d7f: Installed remote snapshot +2025-06-15T03:16:39.342Z [WARN] server-51ad3a78-bb1b-9488-2c71-4a79f4938d7f: skipping application of old log: index=22 +2025-06-15T03:16:39.342Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: pipelining replication: peer="{Voter server-51ad3a78-bb1b-9488-2c71-4a79f4938d7f 51ad3a78-bb1b-9488-2c71-4a79f4938d7f}" +2025-06-15T03:16:39.342Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:16:39.343Z [INFO] server-a6aa7bce-e31e-238b-f88b-948719a2e4a2: copied to local snapshot: bytes=71 +2025-06-15T03:16:39.343Z [INFO] server-a6aa7bce-e31e-238b-f88b-948719a2e4a2: snapshot restore progress: id=2-23-1749957399132 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:39.343Z [INFO] server-a6aa7bce-e31e-238b-f88b-948719a2e4a2: Installed remote snapshot +2025-06-15T03:16:39.343Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: pipelining replication: peer="{Voter server-a6aa7bce-e31e-238b-f88b-948719a2e4a2 a6aa7bce-e31e-238b-f88b-948719a2e4a2}" +2025-06-15T03:16:40.348Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:16:39.342384407 +0000 UTC m=+15.054062573 (2 nodes), 987ns from start of poll, 1.868424479s from cluster start. Timeout at 2025-06-15 03:16:40.348124022 +0000 UTC m=+16.059802187, 1.005739614s after stability +2025-06-15T03:16:40.360Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: aborting pipeline replication: peer="{Voter server-a6aa7bce-e31e-238b-f88b-948719a2e4a2 a6aa7bce-e31e-238b-f88b-948719a2e4a2}" +2025-06-15T03:16:40.360Z [INFO] server-5f02237a-4279-31d6-c161-7a2fdb11fada: aborting pipeline replication: peer="{Voter server-51ad3a78-bb1b-9488-2c71-4a79f4938d7f 51ad3a78-bb1b-9488-2c71-4a79f4938d7f}" === RUN TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false -2024-05-12T20:50:06.209Z [INFO] server-35fb5885-8aaa-4cb1-efde-b13572a702df: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35fb5885-8aaa-4cb1-efde-b13572a702df Address:35fb5885-8aaa-4cb1-efde-b13572a702df} {Suffrage:Voter ID:server-725618f4-604f-527f-8942-7125bc399859 Address:725618f4-604f-527f-8942-7125bc399859} {Suffrage:Voter ID:server-b07c6d80-f773-9503-33f7-f17ae4f368c2 Address:b07c6d80-f773-9503-33f7-f17ae4f368c2}]" -2024-05-12T20:50:06.209Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35fb5885-8aaa-4cb1-efde-b13572a702df Address:35fb5885-8aaa-4cb1-efde-b13572a702df} {Suffrage:Voter ID:server-725618f4-604f-527f-8942-7125bc399859 Address:725618f4-604f-527f-8942-7125bc399859} {Suffrage:Voter ID:server-b07c6d80-f773-9503-33f7-f17ae4f368c2 Address:b07c6d80-f773-9503-33f7-f17ae4f368c2}]" -2024-05-12T20:50:06.209Z [INFO] server-b07c6d80-f773-9503-33f7-f17ae4f368c2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35fb5885-8aaa-4cb1-efde-b13572a702df Address:35fb5885-8aaa-4cb1-efde-b13572a702df} {Suffrage:Voter ID:server-725618f4-604f-527f-8942-7125bc399859 Address:725618f4-604f-527f-8942-7125bc399859} {Suffrage:Voter ID:server-b07c6d80-f773-9503-33f7-f17ae4f368c2 Address:b07c6d80-f773-9503-33f7-f17ae4f368c2}]" -2024-05-12T20:50:06.209Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:06.209Z [INFO] server-35fb5885-8aaa-4cb1-efde-b13572a702df: entering follower state: follower="Node at 35fb5885-8aaa-4cb1-efde-b13572a702df [Follower]" leader-address= leader-id= -2024-05-12T20:50:06.209Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: entering follower state: follower="Node at 725618f4-604f-527f-8942-7125bc399859 [Follower]" leader-address= leader-id= -2024-05-12T20:50:06.209Z [INFO] server-b07c6d80-f773-9503-33f7-f17ae4f368c2: entering follower state: follower="Node at b07c6d80-f773-9503-33f7-f17ae4f368c2 [Follower]" leader-address= leader-id= -2024-05-12T20:50:06.830Z [WARN] server-725618f4-604f-527f-8942-7125bc399859: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:06.830Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: entering candidate state: node="Node at 725618f4-604f-527f-8942-7125bc399859 [Candidate]" term=2 -2024-05-12T20:50:06.831Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: election won: term=2 tally=2 -2024-05-12T20:50:06.831Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: entering leader state: leader="Node at 725618f4-604f-527f-8942-7125bc399859 [Leader]" -2024-05-12T20:50:06.831Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: added peer, starting replication: peer=server-35fb5885-8aaa-4cb1-efde-b13572a702df -2024-05-12T20:50:06.831Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: added peer, starting replication: peer=server-b07c6d80-f773-9503-33f7-f17ae4f368c2 -2024-05-12T20:50:06.831Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: pipelining replication: peer="{Voter server-b07c6d80-f773-9503-33f7-f17ae4f368c2 b07c6d80-f773-9503-33f7-f17ae4f368c2}" -2024-05-12T20:50:06.831Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: pipelining replication: peer="{Voter server-35fb5885-8aaa-4cb1-efde-b13572a702df 35fb5885-8aaa-4cb1-efde-b13572a702df}" -2024-05-12T20:50:07.838Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:06.831571785 +0000 UTC m=+18.112272126 (1 nodes), 622.031313ms from start of poll, 622.451487ms from cluster start. Timeout at 2024-05-12 20:50:07.83883438 +0000 UTC m=+19.119534784, 1.007262658s after stability -2024-05-12T20:50:07.840Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: starting snapshot up to: index=12 -2024-05-12T20:50:07.840Z [INFO] creating new snapshot: path=/tmp/raft354294236/snapshots/2-12-1715547007840.tmp -2024-05-12T20:50:07.841Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: snapshot complete up to: index=12 -2024-05-12T20:50:07.842Z [INFO] creating new snapshot: path=/tmp/raft354294236/snapshots/2-23-1715547007842.tmp -2024-05-12T20:50:07.843Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: copied to local snapshot: bytes=71 -2024-05-12T20:50:07.843Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: snapshot restore progress: id=2-23-1715547007842 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:07.844Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: removing all old logs from log store -2024-05-12T20:50:07.844Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: compacting logs: from=1 to=22 -2024-05-12T20:50:07.844Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: restored user snapshot: index=23 -2024-05-12T20:50:07.844Z [ERROR] server-725618f4-604f-527f-8942-7125bc399859: failed to get log: index=22 error="log not found" -2024-05-12T20:50:07.844Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: aborting pipeline replication: peer="{Voter server-b07c6d80-f773-9503-33f7-f17ae4f368c2 b07c6d80-f773-9503-33f7-f17ae4f368c2}" -2024-05-12T20:50:07.844Z [ERROR] server-725618f4-604f-527f-8942-7125bc399859: failed to get log: index=22 error="log not found" -2024-05-12T20:50:07.844Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: aborting pipeline replication: peer="{Voter server-35fb5885-8aaa-4cb1-efde-b13572a702df 35fb5885-8aaa-4cb1-efde-b13572a702df}" -2024-05-12T20:50:07.850Z [ERROR] server-725618f4-604f-527f-8942-7125bc399859: failed to get log: index=22 error="log not found" -2024-05-12T20:50:07.851Z [ERROR] server-725618f4-604f-527f-8942-7125bc399859: failed to get log: index=22 error="log not found" -2024-05-12T20:50:07.851Z [INFO] creating new snapshot: path=/tmp/raft3917098134/snapshots/2-23-1715547007851.tmp -2024-05-12T20:50:07.852Z [INFO] server-b07c6d80-f773-9503-33f7-f17ae4f368c2: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:07.852Z [INFO] server-b07c6d80-f773-9503-33f7-f17ae4f368c2: copied to local snapshot: bytes=71 -2024-05-12T20:50:07.852Z [INFO] server-b07c6d80-f773-9503-33f7-f17ae4f368c2: snapshot restore progress: id=2-23-1715547007851 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:07.853Z [INFO] server-b07c6d80-f773-9503-33f7-f17ae4f368c2: removing all old logs from log store -2024-05-12T20:50:07.853Z [INFO] server-b07c6d80-f773-9503-33f7-f17ae4f368c2: compacting logs: from=1 to=22 -2024-05-12T20:50:07.853Z [INFO] server-b07c6d80-f773-9503-33f7-f17ae4f368c2: Installed remote snapshot -2024-05-12T20:50:07.853Z [WARN] server-b07c6d80-f773-9503-33f7-f17ae4f368c2: skipping application of old log: index=22 -2024-05-12T20:50:07.853Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: pipelining replication: peer="{Voter server-b07c6d80-f773-9503-33f7-f17ae4f368c2 b07c6d80-f773-9503-33f7-f17ae4f368c2}" -2024-05-12T20:50:07.853Z [INFO] creating new snapshot: path=/tmp/raft1468564089/snapshots/2-23-1715547007853.tmp -2024-05-12T20:50:07.853Z [INFO] server-35fb5885-8aaa-4cb1-efde-b13572a702df: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:07.853Z [INFO] server-35fb5885-8aaa-4cb1-efde-b13572a702df: copied to local snapshot: bytes=71 -2024-05-12T20:50:07.853Z [INFO] server-35fb5885-8aaa-4cb1-efde-b13572a702df: snapshot restore progress: id=2-23-1715547007853 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:07.853Z [INFO] server-35fb5885-8aaa-4cb1-efde-b13572a702df: removing all old logs from log store -2024-05-12T20:50:07.853Z [INFO] server-35fb5885-8aaa-4cb1-efde-b13572a702df: compacting logs: from=1 to=22 -2024-05-12T20:50:07.854Z [INFO] server-35fb5885-8aaa-4cb1-efde-b13572a702df: Installed remote snapshot -2024-05-12T20:50:07.854Z [WARN] server-35fb5885-8aaa-4cb1-efde-b13572a702df: skipping application of old log: index=22 -2024-05-12T20:50:07.854Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: pipelining replication: peer="{Voter server-35fb5885-8aaa-4cb1-efde-b13572a702df 35fb5885-8aaa-4cb1-efde-b13572a702df}" -2024-05-12T20:50:07.854Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:08.870Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:07.854210134 +0000 UTC m=+19.134910426 (2 nodes), 1.712µs from start of poll, 1.645089787s from cluster start. Timeout at 2024-05-12 20:50:08.870820764 +0000 UTC m=+20.151521473, 1.016611047s after stability -2024-05-12T20:50:08.886Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: aborting pipeline replication: peer="{Voter server-b07c6d80-f773-9503-33f7-f17ae4f368c2 b07c6d80-f773-9503-33f7-f17ae4f368c2}" -2024-05-12T20:50:08.887Z [INFO] server-725618f4-604f-527f-8942-7125bc399859: aborting pipeline replication: peer="{Voter server-35fb5885-8aaa-4cb1-efde-b13572a702df 35fb5885-8aaa-4cb1-efde-b13572a702df}" +2025-06-15T03:16:40.361Z [INFO] server-0d0830a1-f7f1-c064-e42f-a9324792151f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0d0830a1-f7f1-c064-e42f-a9324792151f Address:0d0830a1-f7f1-c064-e42f-a9324792151f} {Suffrage:Voter ID:server-f5ac19ae-b843-9a8a-4e98-895f290cb26f Address:f5ac19ae-b843-9a8a-4e98-895f290cb26f} {Suffrage:Voter ID:server-3017454c-67ce-6d60-16d0-d884f2c4e5aa Address:3017454c-67ce-6d60-16d0-d884f2c4e5aa}]" +2025-06-15T03:16:40.361Z [INFO] server-f5ac19ae-b843-9a8a-4e98-895f290cb26f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0d0830a1-f7f1-c064-e42f-a9324792151f Address:0d0830a1-f7f1-c064-e42f-a9324792151f} {Suffrage:Voter ID:server-f5ac19ae-b843-9a8a-4e98-895f290cb26f Address:f5ac19ae-b843-9a8a-4e98-895f290cb26f} {Suffrage:Voter ID:server-3017454c-67ce-6d60-16d0-d884f2c4e5aa Address:3017454c-67ce-6d60-16d0-d884f2c4e5aa}]" +2025-06-15T03:16:40.361Z [INFO] server-0d0830a1-f7f1-c064-e42f-a9324792151f: entering follower state: follower="Node at 0d0830a1-f7f1-c064-e42f-a9324792151f [Follower]" leader-address= leader-id= +2025-06-15T03:16:40.361Z [INFO] server-f5ac19ae-b843-9a8a-4e98-895f290cb26f: entering follower state: follower="Node at f5ac19ae-b843-9a8a-4e98-895f290cb26f [Follower]" leader-address= leader-id= +2025-06-15T03:16:40.361Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0d0830a1-f7f1-c064-e42f-a9324792151f Address:0d0830a1-f7f1-c064-e42f-a9324792151f} {Suffrage:Voter ID:server-f5ac19ae-b843-9a8a-4e98-895f290cb26f Address:f5ac19ae-b843-9a8a-4e98-895f290cb26f} {Suffrage:Voter ID:server-3017454c-67ce-6d60-16d0-d884f2c4e5aa Address:3017454c-67ce-6d60-16d0-d884f2c4e5aa}]" +2025-06-15T03:16:40.361Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:40.361Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: entering follower state: follower="Node at 3017454c-67ce-6d60-16d0-d884f2c4e5aa [Follower]" leader-address= leader-id= +2025-06-15T03:16:41.273Z [WARN] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:41.273Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: entering candidate state: node="Node at 3017454c-67ce-6d60-16d0-d884f2c4e5aa [Candidate]" term=2 +2025-06-15T03:16:41.273Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: election won: term=2 tally=2 +2025-06-15T03:16:41.273Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: entering leader state: leader="Node at 3017454c-67ce-6d60-16d0-d884f2c4e5aa [Leader]" +2025-06-15T03:16:41.273Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: added peer, starting replication: peer=server-0d0830a1-f7f1-c064-e42f-a9324792151f +2025-06-15T03:16:41.274Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: added peer, starting replication: peer=server-f5ac19ae-b843-9a8a-4e98-895f290cb26f +2025-06-15T03:16:41.274Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: pipelining replication: peer="{Voter server-f5ac19ae-b843-9a8a-4e98-895f290cb26f f5ac19ae-b843-9a8a-4e98-895f290cb26f}" +2025-06-15T03:16:41.274Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: pipelining replication: peer="{Voter server-0d0830a1-f7f1-c064-e42f-a9324792151f 0d0830a1-f7f1-c064-e42f-a9324792151f}" +2025-06-15T03:16:42.279Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:41.273966924 +0000 UTC m=+16.985645149 (1 nodes), 912.123169ms from start of poll, 912.50731ms from cluster start. Timeout at 2025-06-15 03:16:42.279569894 +0000 UTC m=+17.991248055, 1.005602906s after stability +2025-06-15T03:16:42.280Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: starting snapshot up to: index=12 +2025-06-15T03:16:42.280Z [INFO] creating new snapshot: path=/tmp/raft2632179266/snapshots/2-12-1749957402280.tmp +2025-06-15T03:16:42.280Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: snapshot complete up to: index=12 +2025-06-15T03:16:42.281Z [INFO] creating new snapshot: path=/tmp/raft2632179266/snapshots/2-23-1749957402281.tmp +2025-06-15T03:16:42.281Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: copied to local snapshot: bytes=71 +2025-06-15T03:16:42.281Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: snapshot restore progress: id=2-23-1749957402281 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:42.282Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: removing all old logs from log store +2025-06-15T03:16:42.282Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: compacting logs: from=1 to=22 +2025-06-15T03:16:42.282Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: restored user snapshot: index=23 +2025-06-15T03:16:42.282Z [ERROR] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: failed to get log: index=22 error="log not found" +2025-06-15T03:16:42.282Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: aborting pipeline replication: peer="{Voter server-0d0830a1-f7f1-c064-e42f-a9324792151f 0d0830a1-f7f1-c064-e42f-a9324792151f}" +2025-06-15T03:16:42.282Z [ERROR] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: failed to get log: index=22 error="log not found" +2025-06-15T03:16:42.282Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: aborting pipeline replication: peer="{Voter server-f5ac19ae-b843-9a8a-4e98-895f290cb26f f5ac19ae-b843-9a8a-4e98-895f290cb26f}" +2025-06-15T03:16:42.291Z [ERROR] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: failed to get log: index=22 error="log not found" +2025-06-15T03:16:42.291Z [ERROR] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: failed to get log: index=22 error="log not found" +2025-06-15T03:16:42.291Z [INFO] creating new snapshot: path=/tmp/raft3431713383/snapshots/2-23-1749957402291.tmp +2025-06-15T03:16:42.291Z [INFO] creating new snapshot: path=/tmp/raft161196733/snapshots/2-23-1749957402291.tmp +2025-06-15T03:16:42.291Z [INFO] server-0d0830a1-f7f1-c064-e42f-a9324792151f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:42.291Z [INFO] server-f5ac19ae-b843-9a8a-4e98-895f290cb26f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:42.292Z [INFO] server-0d0830a1-f7f1-c064-e42f-a9324792151f: copied to local snapshot: bytes=71 +2025-06-15T03:16:42.292Z [INFO] server-f5ac19ae-b843-9a8a-4e98-895f290cb26f: copied to local snapshot: bytes=71 +2025-06-15T03:16:42.292Z [INFO] server-0d0830a1-f7f1-c064-e42f-a9324792151f: snapshot restore progress: id=2-23-1749957402291 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:42.292Z [INFO] server-0d0830a1-f7f1-c064-e42f-a9324792151f: removing all old logs from log store +2025-06-15T03:16:42.292Z [INFO] server-0d0830a1-f7f1-c064-e42f-a9324792151f: compacting logs: from=1 to=22 +2025-06-15T03:16:42.292Z [INFO] server-0d0830a1-f7f1-c064-e42f-a9324792151f: Installed remote snapshot +2025-06-15T03:16:42.292Z [WARN] server-0d0830a1-f7f1-c064-e42f-a9324792151f: skipping application of old log: index=22 +2025-06-15T03:16:42.292Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: pipelining replication: peer="{Voter server-0d0830a1-f7f1-c064-e42f-a9324792151f 0d0830a1-f7f1-c064-e42f-a9324792151f}" +2025-06-15T03:16:42.292Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:16:42.292Z [INFO] server-f5ac19ae-b843-9a8a-4e98-895f290cb26f: snapshot restore progress: id=2-23-1749957402291 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:42.292Z [INFO] server-f5ac19ae-b843-9a8a-4e98-895f290cb26f: removing all old logs from log store +2025-06-15T03:16:42.292Z [INFO] server-f5ac19ae-b843-9a8a-4e98-895f290cb26f: compacting logs: from=1 to=22 +2025-06-15T03:16:42.292Z [INFO] server-f5ac19ae-b843-9a8a-4e98-895f290cb26f: Installed remote snapshot +2025-06-15T03:16:42.292Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: pipelining replication: peer="{Voter server-f5ac19ae-b843-9a8a-4e98-895f290cb26f f5ac19ae-b843-9a8a-4e98-895f290cb26f}" +2025-06-15T03:16:43.298Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:16:42.292324437 +0000 UTC m=+18.004002598 (2 nodes), 659ns from start of poll, 1.930864759s from cluster start. Timeout at 2025-06-15 03:16:43.298370278 +0000 UTC m=+19.010048443, 1.006045845s after stability +2025-06-15T03:16:43.310Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: aborting pipeline replication: peer="{Voter server-f5ac19ae-b843-9a8a-4e98-895f290cb26f f5ac19ae-b843-9a8a-4e98-895f290cb26f}" +2025-06-15T03:16:43.310Z [INFO] server-3017454c-67ce-6d60-16d0-d884f2c4e5aa: aborting pipeline replication: peer="{Voter server-0d0830a1-f7f1-c064-e42f-a9324792151f 0d0830a1-f7f1-c064-e42f-a9324792151f}" === RUN TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true -2024-05-12T20:50:08.889Z [INFO] server-497802ee-c2da-e343-86d9-4dfdc227e329: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-497802ee-c2da-e343-86d9-4dfdc227e329 Address:497802ee-c2da-e343-86d9-4dfdc227e329} {Suffrage:Voter ID:server-68a4e40b-0c23-15c7-a7e5-69cf73021dfa Address:68a4e40b-0c23-15c7-a7e5-69cf73021dfa} {Suffrage:Voter ID:server-a13a57f9-c6be-6a24-5169-f1e56cf988ba Address:a13a57f9-c6be-6a24-5169-f1e56cf988ba}]" -2024-05-12T20:50:08.889Z [INFO] server-68a4e40b-0c23-15c7-a7e5-69cf73021dfa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-497802ee-c2da-e343-86d9-4dfdc227e329 Address:497802ee-c2da-e343-86d9-4dfdc227e329} {Suffrage:Voter ID:server-68a4e40b-0c23-15c7-a7e5-69cf73021dfa Address:68a4e40b-0c23-15c7-a7e5-69cf73021dfa} {Suffrage:Voter ID:server-a13a57f9-c6be-6a24-5169-f1e56cf988ba Address:a13a57f9-c6be-6a24-5169-f1e56cf988ba}]" -2024-05-12T20:50:08.889Z [INFO] server-a13a57f9-c6be-6a24-5169-f1e56cf988ba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-497802ee-c2da-e343-86d9-4dfdc227e329 Address:497802ee-c2da-e343-86d9-4dfdc227e329} {Suffrage:Voter ID:server-68a4e40b-0c23-15c7-a7e5-69cf73021dfa Address:68a4e40b-0c23-15c7-a7e5-69cf73021dfa} {Suffrage:Voter ID:server-a13a57f9-c6be-6a24-5169-f1e56cf988ba Address:a13a57f9-c6be-6a24-5169-f1e56cf988ba}]" -2024-05-12T20:50:08.889Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:08.889Z [INFO] server-497802ee-c2da-e343-86d9-4dfdc227e329: entering follower state: follower="Node at 497802ee-c2da-e343-86d9-4dfdc227e329 [Follower]" leader-address= leader-id= -2024-05-12T20:50:08.889Z [INFO] server-68a4e40b-0c23-15c7-a7e5-69cf73021dfa: entering follower state: follower="Node at 68a4e40b-0c23-15c7-a7e5-69cf73021dfa [Follower]" leader-address= leader-id= -2024-05-12T20:50:08.889Z [INFO] server-a13a57f9-c6be-6a24-5169-f1e56cf988ba: entering follower state: follower="Node at a13a57f9-c6be-6a24-5169-f1e56cf988ba [Follower]" leader-address= leader-id= -2024-05-12T20:50:09.394Z [WARN] server-497802ee-c2da-e343-86d9-4dfdc227e329: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:09.394Z [INFO] server-497802ee-c2da-e343-86d9-4dfdc227e329: entering candidate state: node="Node at 497802ee-c2da-e343-86d9-4dfdc227e329 [Candidate]" term=2 -2024-05-12T20:50:09.395Z [INFO] server-497802ee-c2da-e343-86d9-4dfdc227e329: election won: term=2 tally=2 -2024-05-12T20:50:09.395Z [INFO] server-497802ee-c2da-e343-86d9-4dfdc227e329: entering leader state: leader="Node at 497802ee-c2da-e343-86d9-4dfdc227e329 [Leader]" -2024-05-12T20:50:09.395Z [INFO] server-497802ee-c2da-e343-86d9-4dfdc227e329: added peer, starting replication: peer=server-68a4e40b-0c23-15c7-a7e5-69cf73021dfa -2024-05-12T20:50:09.395Z [INFO] server-497802ee-c2da-e343-86d9-4dfdc227e329: added peer, starting replication: peer=server-a13a57f9-c6be-6a24-5169-f1e56cf988ba -2024-05-12T20:50:09.395Z [INFO] server-497802ee-c2da-e343-86d9-4dfdc227e329: pipelining replication: peer="{Voter server-a13a57f9-c6be-6a24-5169-f1e56cf988ba a13a57f9-c6be-6a24-5169-f1e56cf988ba}" -2024-05-12T20:50:09.395Z [INFO] server-497802ee-c2da-e343-86d9-4dfdc227e329: pipelining replication: peer="{Voter server-68a4e40b-0c23-15c7-a7e5-69cf73021dfa 68a4e40b-0c23-15c7-a7e5-69cf73021dfa}" -2024-05-12T20:50:10.402Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:09.395672515 +0000 UTC m=+20.676372855 (1 nodes), 505.964661ms from start of poll, 506.384619ms from cluster start. Timeout at 2024-05-12 20:50:10.402767679 +0000 UTC m=+21.683468151, 1.007095296s after stability -2024-05-12T20:50:10.404Z [INFO] server-497802ee-c2da-e343-86d9-4dfdc227e329: starting snapshot up to: index=12 -2024-05-12T20:50:10.404Z [INFO] creating new snapshot: path=/tmp/raft3853258240/snapshots/2-12-1715547010404.tmp -2024-05-12T20:50:10.415Z [INFO] server-497802ee-c2da-e343-86d9-4dfdc227e329: snapshot complete up to: index=12 -2024-05-12T20:50:10.418Z [INFO] server-7d85b732-94ae-def0-449a-f1a91bf378bb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d85b732-94ae-def0-449a-f1a91bf378bb Address:7d85b732-94ae-def0-449a-f1a91bf378bb} {Suffrage:Voter ID:server-7faae4e8-f063-2864-bd42-b7e365bfc5b3 Address:7faae4e8-f063-2864-bd42-b7e365bfc5b3} {Suffrage:Voter ID:server-a591c468-61ea-b73b-1190-ec990fb3a5f8 Address:a591c468-61ea-b73b-1190-ec990fb3a5f8}]" -2024-05-12T20:50:10.418Z [INFO] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d85b732-94ae-def0-449a-f1a91bf378bb Address:7d85b732-94ae-def0-449a-f1a91bf378bb} {Suffrage:Voter ID:server-7faae4e8-f063-2864-bd42-b7e365bfc5b3 Address:7faae4e8-f063-2864-bd42-b7e365bfc5b3} {Suffrage:Voter ID:server-a591c468-61ea-b73b-1190-ec990fb3a5f8 Address:a591c468-61ea-b73b-1190-ec990fb3a5f8}]" -2024-05-12T20:50:10.418Z [INFO] server-a591c468-61ea-b73b-1190-ec990fb3a5f8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d85b732-94ae-def0-449a-f1a91bf378bb Address:7d85b732-94ae-def0-449a-f1a91bf378bb} {Suffrage:Voter ID:server-7faae4e8-f063-2864-bd42-b7e365bfc5b3 Address:7faae4e8-f063-2864-bd42-b7e365bfc5b3} {Suffrage:Voter ID:server-a591c468-61ea-b73b-1190-ec990fb3a5f8 Address:a591c468-61ea-b73b-1190-ec990fb3a5f8}]" -2024-05-12T20:50:10.418Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:10.418Z [INFO] server-7d85b732-94ae-def0-449a-f1a91bf378bb: entering follower state: follower="Node at 7d85b732-94ae-def0-449a-f1a91bf378bb [Follower]" leader-address= leader-id= -2024-05-12T20:50:10.419Z [INFO] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: entering follower state: follower="Node at 7faae4e8-f063-2864-bd42-b7e365bfc5b3 [Follower]" leader-address= leader-id= -2024-05-12T20:50:10.419Z [INFO] server-a591c468-61ea-b73b-1190-ec990fb3a5f8: entering follower state: follower="Node at a591c468-61ea-b73b-1190-ec990fb3a5f8 [Follower]" leader-address= leader-id= -2024-05-12T20:50:10.938Z [WARN] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:10.938Z [INFO] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: entering candidate state: node="Node at 7faae4e8-f063-2864-bd42-b7e365bfc5b3 [Candidate]" term=2 -2024-05-12T20:50:10.938Z [INFO] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: election won: term=2 tally=2 -2024-05-12T20:50:10.939Z [INFO] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: entering leader state: leader="Node at 7faae4e8-f063-2864-bd42-b7e365bfc5b3 [Leader]" -2024-05-12T20:50:10.939Z [INFO] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: added peer, starting replication: peer=server-7d85b732-94ae-def0-449a-f1a91bf378bb -2024-05-12T20:50:10.939Z [INFO] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: added peer, starting replication: peer=server-a591c468-61ea-b73b-1190-ec990fb3a5f8 -2024-05-12T20:50:10.939Z [INFO] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: pipelining replication: peer="{Voter server-a591c468-61ea-b73b-1190-ec990fb3a5f8 a591c468-61ea-b73b-1190-ec990fb3a5f8}" -2024-05-12T20:50:10.939Z [INFO] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: pipelining replication: peer="{Voter server-7d85b732-94ae-def0-449a-f1a91bf378bb 7d85b732-94ae-def0-449a-f1a91bf378bb}" -2024-05-12T20:50:11.948Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:10.939519158 +0000 UTC m=+22.220219582 (1 nodes), 520.962542ms from start of poll, 521.353521ms from cluster start. Timeout at 2024-05-12 20:50:11.948867619 +0000 UTC m=+23.229567974, 1.009348392s after stability -2024-05-12T20:50:11.949Z [INFO] creating new snapshot: path=/tmp/raft443438392/snapshots/2-13-1715547011949.tmp -2024-05-12T20:50:11.950Z [INFO] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: copied to local snapshot: bytes=71 -2024-05-12T20:50:11.950Z [INFO] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: snapshot restore progress: id=2-13-1715547011949 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:11.950Z [INFO] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: restored user snapshot: index=13 -2024-05-12T20:50:11.950Z [ERROR] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: failed to get log: index=3 error="log not found" -2024-05-12T20:50:11.950Z [ERROR] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: failed to get log: index=3 error="log not found" -2024-05-12T20:50:11.950Z [INFO] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: aborting pipeline replication: peer="{Voter server-7d85b732-94ae-def0-449a-f1a91bf378bb 7d85b732-94ae-def0-449a-f1a91bf378bb}" -2024-05-12T20:50:11.950Z [INFO] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: aborting pipeline replication: peer="{Voter server-a591c468-61ea-b73b-1190-ec990fb3a5f8 a591c468-61ea-b73b-1190-ec990fb3a5f8}" -2024-05-12T20:50:11.959Z [ERROR] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: failed to get log: index=3 error="log not found" -2024-05-12T20:50:11.961Z [ERROR] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: failed to get log: index=3 error="log not found" -2024-05-12T20:50:11.962Z [INFO] creating new snapshot: path=/tmp/raft1516369690/snapshots/2-13-1715547011962.tmp -2024-05-12T20:50:11.962Z [INFO] server-a591c468-61ea-b73b-1190-ec990fb3a5f8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:11.963Z [INFO] server-a591c468-61ea-b73b-1190-ec990fb3a5f8: copied to local snapshot: bytes=71 -2024-05-12T20:50:11.963Z [INFO] server-a591c468-61ea-b73b-1190-ec990fb3a5f8: snapshot restore progress: id=2-13-1715547011962 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:11.963Z [INFO] server-a591c468-61ea-b73b-1190-ec990fb3a5f8: Installed remote snapshot -2024-05-12T20:50:11.963Z [INFO] creating new snapshot: path=/tmp/raft4073059443/snapshots/2-13-1715547011963.tmp -2024-05-12T20:50:11.967Z [INFO] server-7d85b732-94ae-def0-449a-f1a91bf378bb: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:11.968Z [INFO] server-7d85b732-94ae-def0-449a-f1a91bf378bb: copied to local snapshot: bytes=71 -2024-05-12T20:50:11.968Z [INFO] server-7d85b732-94ae-def0-449a-f1a91bf378bb: snapshot restore progress: id=2-13-1715547011963 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:11.968Z [INFO] server-7d85b732-94ae-def0-449a-f1a91bf378bb: Installed remote snapshot -2024-05-12T20:50:11.968Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:11.968Z [INFO] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: pipelining replication: peer="{Voter server-a591c468-61ea-b73b-1190-ec990fb3a5f8 a591c468-61ea-b73b-1190-ec990fb3a5f8}" -2024-05-12T20:50:11.968Z [WARN] server-7d85b732-94ae-def0-449a-f1a91bf378bb: skipping application of old log: index=13 -2024-05-12T20:50:11.968Z [INFO] server-7faae4e8-f063-2864-bd42-b7e365bfc5b3: pipelining replication: peer="{Voter server-7d85b732-94ae-def0-449a-f1a91bf378bb 7d85b732-94ae-def0-449a-f1a91bf378bb}" -2024-05-12T20:50:12.975Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:11.968611417 +0000 UTC m=+23.249311731 (2 nodes), 1.815µs from start of poll, 1.55044567s from cluster start. Timeout at 2024-05-12 20:50:12.975022746 +0000 UTC m=+24.255723380, 1.006411649s after stability -2024-05-12T20:50:12.991Z [INFO] server-497802ee-c2da-e343-86d9-4dfdc227e329: aborting pipeline replication: peer="{Voter server-a13a57f9-c6be-6a24-5169-f1e56cf988ba a13a57f9-c6be-6a24-5169-f1e56cf988ba}" -2024-05-12T20:50:12.991Z [INFO] server-497802ee-c2da-e343-86d9-4dfdc227e329: aborting pipeline replication: peer="{Voter server-68a4e40b-0c23-15c7-a7e5-69cf73021dfa 68a4e40b-0c23-15c7-a7e5-69cf73021dfa}" +2025-06-15T03:16:43.311Z [INFO] server-e877c8e0-bb74-1fce-147c-f7c3f8502329: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e877c8e0-bb74-1fce-147c-f7c3f8502329 Address:e877c8e0-bb74-1fce-147c-f7c3f8502329} {Suffrage:Voter ID:server-bcc8f0ff-9e90-bd9b-80f2-dce58ddaec8c Address:bcc8f0ff-9e90-bd9b-80f2-dce58ddaec8c} {Suffrage:Voter ID:server-43886b2b-e5a2-1bbc-f51f-8c37f877382e Address:43886b2b-e5a2-1bbc-f51f-8c37f877382e}]" +2025-06-15T03:16:43.311Z [INFO] server-e877c8e0-bb74-1fce-147c-f7c3f8502329: entering follower state: follower="Node at e877c8e0-bb74-1fce-147c-f7c3f8502329 [Follower]" leader-address= leader-id= +2025-06-15T03:16:43.311Z [INFO] server-bcc8f0ff-9e90-bd9b-80f2-dce58ddaec8c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e877c8e0-bb74-1fce-147c-f7c3f8502329 Address:e877c8e0-bb74-1fce-147c-f7c3f8502329} {Suffrage:Voter ID:server-bcc8f0ff-9e90-bd9b-80f2-dce58ddaec8c Address:bcc8f0ff-9e90-bd9b-80f2-dce58ddaec8c} {Suffrage:Voter ID:server-43886b2b-e5a2-1bbc-f51f-8c37f877382e Address:43886b2b-e5a2-1bbc-f51f-8c37f877382e}]" +2025-06-15T03:16:43.311Z [INFO] server-43886b2b-e5a2-1bbc-f51f-8c37f877382e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e877c8e0-bb74-1fce-147c-f7c3f8502329 Address:e877c8e0-bb74-1fce-147c-f7c3f8502329} {Suffrage:Voter ID:server-bcc8f0ff-9e90-bd9b-80f2-dce58ddaec8c Address:bcc8f0ff-9e90-bd9b-80f2-dce58ddaec8c} {Suffrage:Voter ID:server-43886b2b-e5a2-1bbc-f51f-8c37f877382e Address:43886b2b-e5a2-1bbc-f51f-8c37f877382e}]" +2025-06-15T03:16:43.311Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:43.311Z [INFO] server-bcc8f0ff-9e90-bd9b-80f2-dce58ddaec8c: entering follower state: follower="Node at bcc8f0ff-9e90-bd9b-80f2-dce58ddaec8c [Follower]" leader-address= leader-id= +2025-06-15T03:16:43.311Z [INFO] server-43886b2b-e5a2-1bbc-f51f-8c37f877382e: entering follower state: follower="Node at 43886b2b-e5a2-1bbc-f51f-8c37f877382e [Follower]" leader-address= leader-id= +2025-06-15T03:16:43.872Z [WARN] server-e877c8e0-bb74-1fce-147c-f7c3f8502329: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:43.872Z [INFO] server-e877c8e0-bb74-1fce-147c-f7c3f8502329: entering candidate state: node="Node at e877c8e0-bb74-1fce-147c-f7c3f8502329 [Candidate]" term=2 +2025-06-15T03:16:43.873Z [INFO] server-e877c8e0-bb74-1fce-147c-f7c3f8502329: election won: term=2 tally=2 +2025-06-15T03:16:43.873Z [INFO] server-e877c8e0-bb74-1fce-147c-f7c3f8502329: entering leader state: leader="Node at e877c8e0-bb74-1fce-147c-f7c3f8502329 [Leader]" +2025-06-15T03:16:43.873Z [INFO] server-e877c8e0-bb74-1fce-147c-f7c3f8502329: added peer, starting replication: peer=server-bcc8f0ff-9e90-bd9b-80f2-dce58ddaec8c +2025-06-15T03:16:43.873Z [INFO] server-e877c8e0-bb74-1fce-147c-f7c3f8502329: added peer, starting replication: peer=server-43886b2b-e5a2-1bbc-f51f-8c37f877382e +2025-06-15T03:16:43.873Z [INFO] server-e877c8e0-bb74-1fce-147c-f7c3f8502329: pipelining replication: peer="{Voter server-43886b2b-e5a2-1bbc-f51f-8c37f877382e 43886b2b-e5a2-1bbc-f51f-8c37f877382e}" +2025-06-15T03:16:43.873Z [INFO] server-e877c8e0-bb74-1fce-147c-f7c3f8502329: pipelining replication: peer="{Voter server-bcc8f0ff-9e90-bd9b-80f2-dce58ddaec8c bcc8f0ff-9e90-bd9b-80f2-dce58ddaec8c}" +2025-06-15T03:16:44.878Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:43.873101731 +0000 UTC m=+19.584779892 (1 nodes), 561.183815ms from start of poll, 561.532609ms from cluster start. Timeout at 2025-06-15 03:16:44.878483928 +0000 UTC m=+20.590162094, 1.005382202s after stability +2025-06-15T03:16:44.880Z [INFO] server-e877c8e0-bb74-1fce-147c-f7c3f8502329: starting snapshot up to: index=12 +2025-06-15T03:16:44.880Z [INFO] creating new snapshot: path=/tmp/raft2954699886/snapshots/2-12-1749957404880.tmp +2025-06-15T03:16:44.880Z [INFO] server-e877c8e0-bb74-1fce-147c-f7c3f8502329: snapshot complete up to: index=12 +2025-06-15T03:16:44.882Z [INFO] server-00c2e34b-24b4-2ccb-0e37-f74fd2f26778: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-00c2e34b-24b4-2ccb-0e37-f74fd2f26778 Address:00c2e34b-24b4-2ccb-0e37-f74fd2f26778} {Suffrage:Voter ID:server-e2c2b4d3-0b81-a619-ca64-7122ba743cd8 Address:e2c2b4d3-0b81-a619-ca64-7122ba743cd8} {Suffrage:Voter ID:server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d Address:8f83507a-177c-2e4c-e55b-97c6e86d5f4d}]" +2025-06-15T03:16:44.882Z [INFO] server-00c2e34b-24b4-2ccb-0e37-f74fd2f26778: entering follower state: follower="Node at 00c2e34b-24b4-2ccb-0e37-f74fd2f26778 [Follower]" leader-address= leader-id= +2025-06-15T03:16:44.882Z [INFO] server-e2c2b4d3-0b81-a619-ca64-7122ba743cd8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-00c2e34b-24b4-2ccb-0e37-f74fd2f26778 Address:00c2e34b-24b4-2ccb-0e37-f74fd2f26778} {Suffrage:Voter ID:server-e2c2b4d3-0b81-a619-ca64-7122ba743cd8 Address:e2c2b4d3-0b81-a619-ca64-7122ba743cd8} {Suffrage:Voter ID:server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d Address:8f83507a-177c-2e4c-e55b-97c6e86d5f4d}]" +2025-06-15T03:16:44.882Z [INFO] server-e2c2b4d3-0b81-a619-ca64-7122ba743cd8: entering follower state: follower="Node at e2c2b4d3-0b81-a619-ca64-7122ba743cd8 [Follower]" leader-address= leader-id= +2025-06-15T03:16:44.882Z [INFO] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-00c2e34b-24b4-2ccb-0e37-f74fd2f26778 Address:00c2e34b-24b4-2ccb-0e37-f74fd2f26778} {Suffrage:Voter ID:server-e2c2b4d3-0b81-a619-ca64-7122ba743cd8 Address:e2c2b4d3-0b81-a619-ca64-7122ba743cd8} {Suffrage:Voter ID:server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d Address:8f83507a-177c-2e4c-e55b-97c6e86d5f4d}]" +2025-06-15T03:16:44.882Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:44.882Z [INFO] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: entering follower state: follower="Node at 8f83507a-177c-2e4c-e55b-97c6e86d5f4d [Follower]" leader-address= leader-id= +2025-06-15T03:16:45.692Z [WARN] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:45.693Z [INFO] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: entering candidate state: node="Node at 8f83507a-177c-2e4c-e55b-97c6e86d5f4d [Candidate]" term=2 +2025-06-15T03:16:45.693Z [INFO] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: election won: term=2 tally=2 +2025-06-15T03:16:45.693Z [INFO] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: entering leader state: leader="Node at 8f83507a-177c-2e4c-e55b-97c6e86d5f4d [Leader]" +2025-06-15T03:16:45.693Z [INFO] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: added peer, starting replication: peer=server-00c2e34b-24b4-2ccb-0e37-f74fd2f26778 +2025-06-15T03:16:45.693Z [INFO] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: added peer, starting replication: peer=server-e2c2b4d3-0b81-a619-ca64-7122ba743cd8 +2025-06-15T03:16:45.693Z [INFO] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: pipelining replication: peer="{Voter server-e2c2b4d3-0b81-a619-ca64-7122ba743cd8 e2c2b4d3-0b81-a619-ca64-7122ba743cd8}" +2025-06-15T03:16:45.693Z [INFO] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: pipelining replication: peer="{Voter server-00c2e34b-24b4-2ccb-0e37-f74fd2f26778 00c2e34b-24b4-2ccb-0e37-f74fd2f26778}" +2025-06-15T03:16:46.699Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:45.693226724 +0000 UTC m=+21.404904889 (1 nodes), 810.600606ms from start of poll, 810.92577ms from cluster start. Timeout at 2025-06-15 03:16:46.699085178 +0000 UTC m=+22.410763359, 1.00585847s after stability +2025-06-15T03:16:46.699Z [INFO] creating new snapshot: path=/tmp/raft4180196168/snapshots/2-13-1749957406699.tmp +2025-06-15T03:16:46.700Z [INFO] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: copied to local snapshot: bytes=71 +2025-06-15T03:16:46.700Z [INFO] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: snapshot restore progress: id=2-13-1749957406699 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:46.700Z [INFO] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: restored user snapshot: index=13 +2025-06-15T03:16:46.700Z [ERROR] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: failed to get log: index=3 error="log not found" +2025-06-15T03:16:46.700Z [INFO] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: aborting pipeline replication: peer="{Voter server-e2c2b4d3-0b81-a619-ca64-7122ba743cd8 e2c2b4d3-0b81-a619-ca64-7122ba743cd8}" +2025-06-15T03:16:46.700Z [ERROR] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: failed to get log: index=3 error="log not found" +2025-06-15T03:16:46.700Z [INFO] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: aborting pipeline replication: peer="{Voter server-00c2e34b-24b4-2ccb-0e37-f74fd2f26778 00c2e34b-24b4-2ccb-0e37-f74fd2f26778}" +2025-06-15T03:16:46.706Z [ERROR] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: failed to get log: index=3 error="log not found" +2025-06-15T03:16:46.707Z [INFO] creating new snapshot: path=/tmp/raft3093949507/snapshots/2-13-1749957406707.tmp +2025-06-15T03:16:46.707Z [INFO] server-00c2e34b-24b4-2ccb-0e37-f74fd2f26778: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:46.707Z [INFO] server-00c2e34b-24b4-2ccb-0e37-f74fd2f26778: copied to local snapshot: bytes=71 +2025-06-15T03:16:46.707Z [INFO] server-00c2e34b-24b4-2ccb-0e37-f74fd2f26778: snapshot restore progress: id=2-13-1749957406707 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:46.707Z [INFO] server-00c2e34b-24b4-2ccb-0e37-f74fd2f26778: Installed remote snapshot +2025-06-15T03:16:46.707Z [INFO] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: pipelining replication: peer="{Voter server-00c2e34b-24b4-2ccb-0e37-f74fd2f26778 00c2e34b-24b4-2ccb-0e37-f74fd2f26778}" +2025-06-15T03:16:46.707Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:16:46.710Z [ERROR] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: failed to get log: index=3 error="log not found" +2025-06-15T03:16:46.710Z [INFO] creating new snapshot: path=/tmp/raft335457245/snapshots/2-13-1749957406710.tmp +2025-06-15T03:16:46.710Z [INFO] server-e2c2b4d3-0b81-a619-ca64-7122ba743cd8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:46.710Z [INFO] server-e2c2b4d3-0b81-a619-ca64-7122ba743cd8: copied to local snapshot: bytes=71 +2025-06-15T03:16:46.711Z [INFO] server-e2c2b4d3-0b81-a619-ca64-7122ba743cd8: snapshot restore progress: id=2-13-1749957406710 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:46.711Z [INFO] server-e2c2b4d3-0b81-a619-ca64-7122ba743cd8: Installed remote snapshot +2025-06-15T03:16:46.711Z [INFO] server-8f83507a-177c-2e4c-e55b-97c6e86d5f4d: pipelining replication: peer="{Voter server-e2c2b4d3-0b81-a619-ca64-7122ba743cd8 e2c2b4d3-0b81-a619-ca64-7122ba743cd8}" +2025-06-15T03:16:47.713Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:16:46.707839098 +0000 UTC m=+22.419517264 (2 nodes), 1.234µs from start of poll, 1.825538145s from cluster start. Timeout at 2025-06-15 03:16:47.713085662 +0000 UTC m=+23.424763823, 1.005246559s after stability +2025-06-15T03:16:47.724Z [INFO] server-e877c8e0-bb74-1fce-147c-f7c3f8502329: aborting pipeline replication: peer="{Voter server-43886b2b-e5a2-1bbc-f51f-8c37f877382e 43886b2b-e5a2-1bbc-f51f-8c37f877382e}" +2025-06-15T03:16:47.724Z [INFO] server-e877c8e0-bb74-1fce-147c-f7c3f8502329: aborting pipeline replication: peer="{Voter server-bcc8f0ff-9e90-bd9b-80f2-dce58ddaec8c bcc8f0ff-9e90-bd9b-80f2-dce58ddaec8c}" === RUN TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true -2024-05-12T20:50:13.000Z [INFO] server-ab5c0224-df02-ec95-dd21-0f14896c224e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ab5c0224-df02-ec95-dd21-0f14896c224e Address:ab5c0224-df02-ec95-dd21-0f14896c224e} {Suffrage:Voter ID:server-b0f3b599-ecdd-b838-5916-b4089ca3d69b Address:b0f3b599-ecdd-b838-5916-b4089ca3d69b} {Suffrage:Voter ID:server-dbca5fc5-50f1-110d-b124-64477c7f4a7b Address:dbca5fc5-50f1-110d-b124-64477c7f4a7b}]" -2024-05-12T20:50:13.000Z [INFO] server-b0f3b599-ecdd-b838-5916-b4089ca3d69b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ab5c0224-df02-ec95-dd21-0f14896c224e Address:ab5c0224-df02-ec95-dd21-0f14896c224e} {Suffrage:Voter ID:server-b0f3b599-ecdd-b838-5916-b4089ca3d69b Address:b0f3b599-ecdd-b838-5916-b4089ca3d69b} {Suffrage:Voter ID:server-dbca5fc5-50f1-110d-b124-64477c7f4a7b Address:dbca5fc5-50f1-110d-b124-64477c7f4a7b}]" -2024-05-12T20:50:13.001Z [INFO] server-dbca5fc5-50f1-110d-b124-64477c7f4a7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ab5c0224-df02-ec95-dd21-0f14896c224e Address:ab5c0224-df02-ec95-dd21-0f14896c224e} {Suffrage:Voter ID:server-b0f3b599-ecdd-b838-5916-b4089ca3d69b Address:b0f3b599-ecdd-b838-5916-b4089ca3d69b} {Suffrage:Voter ID:server-dbca5fc5-50f1-110d-b124-64477c7f4a7b Address:dbca5fc5-50f1-110d-b124-64477c7f4a7b}]" -2024-05-12T20:50:13.001Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:13.001Z [INFO] server-ab5c0224-df02-ec95-dd21-0f14896c224e: entering follower state: follower="Node at ab5c0224-df02-ec95-dd21-0f14896c224e [Follower]" leader-address= leader-id= -2024-05-12T20:50:13.001Z [INFO] server-b0f3b599-ecdd-b838-5916-b4089ca3d69b: entering follower state: follower="Node at b0f3b599-ecdd-b838-5916-b4089ca3d69b [Follower]" leader-address= leader-id= -2024-05-12T20:50:13.001Z [INFO] server-dbca5fc5-50f1-110d-b124-64477c7f4a7b: entering follower state: follower="Node at dbca5fc5-50f1-110d-b124-64477c7f4a7b [Follower]" leader-address= leader-id= -2024-05-12T20:50:13.657Z [WARN] server-ab5c0224-df02-ec95-dd21-0f14896c224e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:13.657Z [INFO] server-ab5c0224-df02-ec95-dd21-0f14896c224e: entering candidate state: node="Node at ab5c0224-df02-ec95-dd21-0f14896c224e [Candidate]" term=2 -2024-05-12T20:50:13.657Z [INFO] server-ab5c0224-df02-ec95-dd21-0f14896c224e: election won: term=2 tally=2 -2024-05-12T20:50:13.657Z [INFO] server-ab5c0224-df02-ec95-dd21-0f14896c224e: entering leader state: leader="Node at ab5c0224-df02-ec95-dd21-0f14896c224e [Leader]" -2024-05-12T20:50:13.657Z [INFO] server-ab5c0224-df02-ec95-dd21-0f14896c224e: added peer, starting replication: peer=server-b0f3b599-ecdd-b838-5916-b4089ca3d69b -2024-05-12T20:50:13.657Z [INFO] server-ab5c0224-df02-ec95-dd21-0f14896c224e: added peer, starting replication: peer=server-dbca5fc5-50f1-110d-b124-64477c7f4a7b -2024-05-12T20:50:13.658Z [INFO] server-ab5c0224-df02-ec95-dd21-0f14896c224e: pipelining replication: peer="{Voter server-b0f3b599-ecdd-b838-5916-b4089ca3d69b b0f3b599-ecdd-b838-5916-b4089ca3d69b}" -2024-05-12T20:50:13.658Z [INFO] server-ab5c0224-df02-ec95-dd21-0f14896c224e: pipelining replication: peer="{Voter server-dbca5fc5-50f1-110d-b124-64477c7f4a7b dbca5fc5-50f1-110d-b124-64477c7f4a7b}" -2024-05-12T20:50:14.666Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:13.658264527 +0000 UTC m=+24.938964874 (1 nodes), 657.156306ms from start of poll, 657.600663ms from cluster start. Timeout at 2024-05-12 20:50:14.66688209 +0000 UTC m=+25.947581985, 1.008617111s after stability -2024-05-12T20:50:14.668Z [INFO] server-ab5c0224-df02-ec95-dd21-0f14896c224e: starting snapshot up to: index=12 -2024-05-12T20:50:14.668Z [INFO] creating new snapshot: path=/tmp/raft1441810253/snapshots/2-12-1715547014668.tmp -2024-05-12T20:50:14.669Z [INFO] server-ab5c0224-df02-ec95-dd21-0f14896c224e: snapshot complete up to: index=12 -2024-05-12T20:50:14.672Z [INFO] server-87358d57-312c-baae-2a01-225538426026: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-87358d57-312c-baae-2a01-225538426026 Address:87358d57-312c-baae-2a01-225538426026} {Suffrage:Voter ID:server-27458f7f-762c-8b83-38e3-fd8b2ecebba8 Address:27458f7f-762c-8b83-38e3-fd8b2ecebba8} {Suffrage:Voter ID:server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be Address:742aa142-21eb-bcb1-8ea7-62d0fcacb0be}]" -2024-05-12T20:50:14.673Z [INFO] server-27458f7f-762c-8b83-38e3-fd8b2ecebba8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-87358d57-312c-baae-2a01-225538426026 Address:87358d57-312c-baae-2a01-225538426026} {Suffrage:Voter ID:server-27458f7f-762c-8b83-38e3-fd8b2ecebba8 Address:27458f7f-762c-8b83-38e3-fd8b2ecebba8} {Suffrage:Voter ID:server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be Address:742aa142-21eb-bcb1-8ea7-62d0fcacb0be}]" -2024-05-12T20:50:14.673Z [INFO] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-87358d57-312c-baae-2a01-225538426026 Address:87358d57-312c-baae-2a01-225538426026} {Suffrage:Voter ID:server-27458f7f-762c-8b83-38e3-fd8b2ecebba8 Address:27458f7f-762c-8b83-38e3-fd8b2ecebba8} {Suffrage:Voter ID:server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be Address:742aa142-21eb-bcb1-8ea7-62d0fcacb0be}]" -2024-05-12T20:50:14.673Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:14.673Z [INFO] server-87358d57-312c-baae-2a01-225538426026: entering follower state: follower="Node at 87358d57-312c-baae-2a01-225538426026 [Follower]" leader-address= leader-id= -2024-05-12T20:50:14.673Z [INFO] server-27458f7f-762c-8b83-38e3-fd8b2ecebba8: entering follower state: follower="Node at 27458f7f-762c-8b83-38e3-fd8b2ecebba8 [Follower]" leader-address= leader-id= -2024-05-12T20:50:14.673Z [INFO] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: entering follower state: follower="Node at 742aa142-21eb-bcb1-8ea7-62d0fcacb0be [Follower]" leader-address= leader-id= -2024-05-12T20:50:15.262Z [WARN] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:15.263Z [INFO] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: entering candidate state: node="Node at 742aa142-21eb-bcb1-8ea7-62d0fcacb0be [Candidate]" term=2 -2024-05-12T20:50:15.263Z [INFO] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: election won: term=2 tally=2 -2024-05-12T20:50:15.263Z [INFO] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: entering leader state: leader="Node at 742aa142-21eb-bcb1-8ea7-62d0fcacb0be [Leader]" -2024-05-12T20:50:15.263Z [INFO] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: added peer, starting replication: peer=server-87358d57-312c-baae-2a01-225538426026 -2024-05-12T20:50:15.263Z [INFO] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: added peer, starting replication: peer=server-27458f7f-762c-8b83-38e3-fd8b2ecebba8 -2024-05-12T20:50:15.263Z [INFO] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: pipelining replication: peer="{Voter server-27458f7f-762c-8b83-38e3-fd8b2ecebba8 27458f7f-762c-8b83-38e3-fd8b2ecebba8}" -2024-05-12T20:50:15.263Z [INFO] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: pipelining replication: peer="{Voter server-87358d57-312c-baae-2a01-225538426026 87358d57-312c-baae-2a01-225538426026}" -2024-05-12T20:50:16.271Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:15.263832844 +0000 UTC m=+26.544533259 (1 nodes), 590.56519ms from start of poll, 591.117319ms from cluster start. Timeout at 2024-05-12 20:50:16.271014521 +0000 UTC m=+27.551715001, 1.007181742s after stability -2024-05-12T20:50:16.271Z [INFO] creating new snapshot: path=/tmp/raft2064455345/snapshots/2-13-1715547016271.tmp -2024-05-12T20:50:16.272Z [INFO] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: copied to local snapshot: bytes=71 -2024-05-12T20:50:16.273Z [INFO] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: snapshot restore progress: id=2-13-1715547016271 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:16.273Z [INFO] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: removing all old logs from log store -2024-05-12T20:50:16.273Z [INFO] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: compacting logs: from=1 to=2 -2024-05-12T20:50:16.273Z [INFO] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: restored user snapshot: index=13 -2024-05-12T20:50:16.273Z [ERROR] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: failed to get log: index=2 error="log not found" -2024-05-12T20:50:16.273Z [INFO] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: aborting pipeline replication: peer="{Voter server-27458f7f-762c-8b83-38e3-fd8b2ecebba8 27458f7f-762c-8b83-38e3-fd8b2ecebba8}" -2024-05-12T20:50:16.273Z [ERROR] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: failed to get log: index=2 error="log not found" -2024-05-12T20:50:16.273Z [INFO] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: aborting pipeline replication: peer="{Voter server-87358d57-312c-baae-2a01-225538426026 87358d57-312c-baae-2a01-225538426026}" -2024-05-12T20:50:16.282Z [ERROR] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: failed to get log: index=2 error="log not found" -2024-05-12T20:50:16.283Z [ERROR] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: failed to get log: index=2 error="log not found" -2024-05-12T20:50:16.287Z [INFO] creating new snapshot: path=/tmp/raft2653823683/snapshots/2-13-1715547016287.tmp -2024-05-12T20:50:16.287Z [INFO] server-87358d57-312c-baae-2a01-225538426026: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:16.288Z [INFO] server-87358d57-312c-baae-2a01-225538426026: copied to local snapshot: bytes=71 -2024-05-12T20:50:16.288Z [INFO] server-87358d57-312c-baae-2a01-225538426026: snapshot restore progress: id=2-13-1715547016287 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:16.288Z [INFO] server-87358d57-312c-baae-2a01-225538426026: removing all old logs from log store -2024-05-12T20:50:16.288Z [INFO] server-87358d57-312c-baae-2a01-225538426026: compacting logs: from=1 to=2 -2024-05-12T20:50:16.288Z [INFO] server-87358d57-312c-baae-2a01-225538426026: Installed remote snapshot -2024-05-12T20:50:16.288Z [INFO] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: pipelining replication: peer="{Voter server-87358d57-312c-baae-2a01-225538426026 87358d57-312c-baae-2a01-225538426026}" -2024-05-12T20:50:16.288Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:16.289Z [INFO] creating new snapshot: path=/tmp/raft2275094324/snapshots/2-13-1715547016289.tmp -2024-05-12T20:50:16.289Z [INFO] server-27458f7f-762c-8b83-38e3-fd8b2ecebba8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:16.289Z [INFO] server-27458f7f-762c-8b83-38e3-fd8b2ecebba8: copied to local snapshot: bytes=71 -2024-05-12T20:50:16.290Z [INFO] server-27458f7f-762c-8b83-38e3-fd8b2ecebba8: snapshot restore progress: id=2-13-1715547016289 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:16.290Z [INFO] server-27458f7f-762c-8b83-38e3-fd8b2ecebba8: removing all old logs from log store -2024-05-12T20:50:16.290Z [INFO] server-27458f7f-762c-8b83-38e3-fd8b2ecebba8: compacting logs: from=1 to=2 -2024-05-12T20:50:16.290Z [INFO] server-27458f7f-762c-8b83-38e3-fd8b2ecebba8: Installed remote snapshot -2024-05-12T20:50:16.290Z [INFO] server-742aa142-21eb-bcb1-8ea7-62d0fcacb0be: pipelining replication: peer="{Voter server-27458f7f-762c-8b83-38e3-fd8b2ecebba8 27458f7f-762c-8b83-38e3-fd8b2ecebba8}" -2024-05-12T20:50:17.298Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:16.288844714 +0000 UTC m=+27.569545106 (2 nodes), 2.409µs from start of poll, 1.616129166s from cluster start. Timeout at 2024-05-12 20:50:17.298778323 +0000 UTC m=+28.579478218, 1.009933112s after stability -2024-05-12T20:50:17.335Z [INFO] server-ab5c0224-df02-ec95-dd21-0f14896c224e: aborting pipeline replication: peer="{Voter server-dbca5fc5-50f1-110d-b124-64477c7f4a7b dbca5fc5-50f1-110d-b124-64477c7f4a7b}" -2024-05-12T20:50:17.335Z [INFO] server-ab5c0224-df02-ec95-dd21-0f14896c224e: aborting pipeline replication: peer="{Voter server-b0f3b599-ecdd-b838-5916-b4089ca3d69b b0f3b599-ecdd-b838-5916-b4089ca3d69b}" +2025-06-15T03:16:47.725Z [INFO] server-1eb2bf99-5771-6d96-a925-545c775d3805: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1eb2bf99-5771-6d96-a925-545c775d3805 Address:1eb2bf99-5771-6d96-a925-545c775d3805} {Suffrage:Voter ID:server-18d5ec34-2c12-cd28-7b8d-6545db762b6d Address:18d5ec34-2c12-cd28-7b8d-6545db762b6d} {Suffrage:Voter ID:server-a70ed36c-077b-afd9-712b-c981cb14be7a Address:a70ed36c-077b-afd9-712b-c981cb14be7a}]" +2025-06-15T03:16:47.725Z [INFO] server-1eb2bf99-5771-6d96-a925-545c775d3805: entering follower state: follower="Node at 1eb2bf99-5771-6d96-a925-545c775d3805 [Follower]" leader-address= leader-id= +2025-06-15T03:16:47.725Z [INFO] server-18d5ec34-2c12-cd28-7b8d-6545db762b6d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1eb2bf99-5771-6d96-a925-545c775d3805 Address:1eb2bf99-5771-6d96-a925-545c775d3805} {Suffrage:Voter ID:server-18d5ec34-2c12-cd28-7b8d-6545db762b6d Address:18d5ec34-2c12-cd28-7b8d-6545db762b6d} {Suffrage:Voter ID:server-a70ed36c-077b-afd9-712b-c981cb14be7a Address:a70ed36c-077b-afd9-712b-c981cb14be7a}]" +2025-06-15T03:16:47.725Z [INFO] server-18d5ec34-2c12-cd28-7b8d-6545db762b6d: entering follower state: follower="Node at 18d5ec34-2c12-cd28-7b8d-6545db762b6d [Follower]" leader-address= leader-id= +2025-06-15T03:16:47.725Z [INFO] server-a70ed36c-077b-afd9-712b-c981cb14be7a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1eb2bf99-5771-6d96-a925-545c775d3805 Address:1eb2bf99-5771-6d96-a925-545c775d3805} {Suffrage:Voter ID:server-18d5ec34-2c12-cd28-7b8d-6545db762b6d Address:18d5ec34-2c12-cd28-7b8d-6545db762b6d} {Suffrage:Voter ID:server-a70ed36c-077b-afd9-712b-c981cb14be7a Address:a70ed36c-077b-afd9-712b-c981cb14be7a}]" +2025-06-15T03:16:47.725Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:47.725Z [INFO] server-a70ed36c-077b-afd9-712b-c981cb14be7a: entering follower state: follower="Node at a70ed36c-077b-afd9-712b-c981cb14be7a [Follower]" leader-address= leader-id= +2025-06-15T03:16:48.258Z [WARN] server-1eb2bf99-5771-6d96-a925-545c775d3805: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:48.258Z [INFO] server-1eb2bf99-5771-6d96-a925-545c775d3805: entering candidate state: node="Node at 1eb2bf99-5771-6d96-a925-545c775d3805 [Candidate]" term=2 +2025-06-15T03:16:48.258Z [INFO] server-1eb2bf99-5771-6d96-a925-545c775d3805: election won: term=2 tally=2 +2025-06-15T03:16:48.258Z [INFO] server-1eb2bf99-5771-6d96-a925-545c775d3805: entering leader state: leader="Node at 1eb2bf99-5771-6d96-a925-545c775d3805 [Leader]" +2025-06-15T03:16:48.258Z [INFO] server-1eb2bf99-5771-6d96-a925-545c775d3805: added peer, starting replication: peer=server-18d5ec34-2c12-cd28-7b8d-6545db762b6d +2025-06-15T03:16:48.258Z [INFO] server-1eb2bf99-5771-6d96-a925-545c775d3805: added peer, starting replication: peer=server-a70ed36c-077b-afd9-712b-c981cb14be7a +2025-06-15T03:16:48.258Z [WARN] server-18d5ec34-2c12-cd28-7b8d-6545db762b6d: rejecting vote request since our last term is greater: candidate=1eb2bf99-5771-6d96-a925-545c775d3805 last-term=2 last-candidate-term=1 +2025-06-15T03:16:48.258Z [INFO] server-1eb2bf99-5771-6d96-a925-545c775d3805: pipelining replication: peer="{Voter server-a70ed36c-077b-afd9-712b-c981cb14be7a a70ed36c-077b-afd9-712b-c981cb14be7a}" +2025-06-15T03:16:48.258Z [INFO] server-1eb2bf99-5771-6d96-a925-545c775d3805: pipelining replication: peer="{Voter server-18d5ec34-2c12-cd28-7b8d-6545db762b6d 18d5ec34-2c12-cd28-7b8d-6545db762b6d}" +2025-06-15T03:16:49.264Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:48.258620354 +0000 UTC m=+23.970298518 (1 nodes), 533.238939ms from start of poll, 533.54307ms from cluster start. Timeout at 2025-06-15 03:16:49.264678021 +0000 UTC m=+24.976356187, 1.006057669s after stability +2025-06-15T03:16:49.265Z [INFO] server-1eb2bf99-5771-6d96-a925-545c775d3805: starting snapshot up to: index=12 +2025-06-15T03:16:49.265Z [INFO] creating new snapshot: path=/tmp/raft949779907/snapshots/2-12-1749957409265.tmp +2025-06-15T03:16:49.265Z [INFO] server-1eb2bf99-5771-6d96-a925-545c775d3805: snapshot complete up to: index=12 +2025-06-15T03:16:49.266Z [INFO] server-969fac36-b2b4-e892-a0c5-21a6371f1331: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-969fac36-b2b4-e892-a0c5-21a6371f1331 Address:969fac36-b2b4-e892-a0c5-21a6371f1331} {Suffrage:Voter ID:server-3fa90b8d-36f1-d271-e086-0b48a19124af Address:3fa90b8d-36f1-d271-e086-0b48a19124af} {Suffrage:Voter ID:server-7738d9e7-c66a-d7ac-0df3-6b97cb43057d Address:7738d9e7-c66a-d7ac-0df3-6b97cb43057d}]" +2025-06-15T03:16:49.266Z [INFO] server-969fac36-b2b4-e892-a0c5-21a6371f1331: entering follower state: follower="Node at 969fac36-b2b4-e892-a0c5-21a6371f1331 [Follower]" leader-address= leader-id= +2025-06-15T03:16:49.267Z [INFO] server-3fa90b8d-36f1-d271-e086-0b48a19124af: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-969fac36-b2b4-e892-a0c5-21a6371f1331 Address:969fac36-b2b4-e892-a0c5-21a6371f1331} {Suffrage:Voter ID:server-3fa90b8d-36f1-d271-e086-0b48a19124af Address:3fa90b8d-36f1-d271-e086-0b48a19124af} {Suffrage:Voter ID:server-7738d9e7-c66a-d7ac-0df3-6b97cb43057d Address:7738d9e7-c66a-d7ac-0df3-6b97cb43057d}]" +2025-06-15T03:16:49.267Z [INFO] server-3fa90b8d-36f1-d271-e086-0b48a19124af: entering follower state: follower="Node at 3fa90b8d-36f1-d271-e086-0b48a19124af [Follower]" leader-address= leader-id= +2025-06-15T03:16:49.267Z [INFO] server-7738d9e7-c66a-d7ac-0df3-6b97cb43057d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-969fac36-b2b4-e892-a0c5-21a6371f1331 Address:969fac36-b2b4-e892-a0c5-21a6371f1331} {Suffrage:Voter ID:server-3fa90b8d-36f1-d271-e086-0b48a19124af Address:3fa90b8d-36f1-d271-e086-0b48a19124af} {Suffrage:Voter ID:server-7738d9e7-c66a-d7ac-0df3-6b97cb43057d Address:7738d9e7-c66a-d7ac-0df3-6b97cb43057d}]" +2025-06-15T03:16:49.267Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:49.267Z [INFO] server-7738d9e7-c66a-d7ac-0df3-6b97cb43057d: entering follower state: follower="Node at 7738d9e7-c66a-d7ac-0df3-6b97cb43057d [Follower]" leader-address= leader-id= +2025-06-15T03:16:49.800Z [WARN] server-3fa90b8d-36f1-d271-e086-0b48a19124af: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:49.800Z [INFO] server-3fa90b8d-36f1-d271-e086-0b48a19124af: entering candidate state: node="Node at 3fa90b8d-36f1-d271-e086-0b48a19124af [Candidate]" term=2 +2025-06-15T03:16:49.800Z [INFO] server-3fa90b8d-36f1-d271-e086-0b48a19124af: election won: term=2 tally=2 +2025-06-15T03:16:49.800Z [INFO] server-3fa90b8d-36f1-d271-e086-0b48a19124af: entering leader state: leader="Node at 3fa90b8d-36f1-d271-e086-0b48a19124af [Leader]" +2025-06-15T03:16:49.800Z [INFO] server-3fa90b8d-36f1-d271-e086-0b48a19124af: added peer, starting replication: peer=server-969fac36-b2b4-e892-a0c5-21a6371f1331 +2025-06-15T03:16:49.800Z [INFO] server-3fa90b8d-36f1-d271-e086-0b48a19124af: added peer, starting replication: peer=server-7738d9e7-c66a-d7ac-0df3-6b97cb43057d +2025-06-15T03:16:49.800Z [INFO] server-3fa90b8d-36f1-d271-e086-0b48a19124af: pipelining replication: peer="{Voter server-969fac36-b2b4-e892-a0c5-21a6371f1331 969fac36-b2b4-e892-a0c5-21a6371f1331}" +2025-06-15T03:16:49.800Z [INFO] server-3fa90b8d-36f1-d271-e086-0b48a19124af: pipelining replication: peer="{Voter server-7738d9e7-c66a-d7ac-0df3-6b97cb43057d 7738d9e7-c66a-d7ac-0df3-6b97cb43057d}" +2025-06-15T03:16:50.805Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:49.800203217 +0000 UTC m=+25.511881394 (1 nodes), 533.04844ms from start of poll, 533.460175ms from cluster start. Timeout at 2025-06-15 03:16:50.805472639 +0000 UTC m=+26.517150838, 1.005269444s after stability +2025-06-15T03:16:50.805Z [INFO] creating new snapshot: path=/tmp/raft1421422180/snapshots/2-13-1749957410805.tmp +2025-06-15T03:16:50.909Z [INFO] server-3fa90b8d-36f1-d271-e086-0b48a19124af: copied to local snapshot: bytes=71 +2025-06-15T03:16:50.909Z [INFO] server-3fa90b8d-36f1-d271-e086-0b48a19124af: snapshot restore progress: id=2-13-1749957410805 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:50.909Z [INFO] server-3fa90b8d-36f1-d271-e086-0b48a19124af: removing all old logs from log store +2025-06-15T03:16:50.909Z [INFO] server-3fa90b8d-36f1-d271-e086-0b48a19124af: compacting logs: from=1 to=2 +2025-06-15T03:16:50.909Z [INFO] server-3fa90b8d-36f1-d271-e086-0b48a19124af: restored user snapshot: index=13 +2025-06-15T03:16:50.909Z [ERROR] server-3fa90b8d-36f1-d271-e086-0b48a19124af: failed to get log: index=2 error="log not found" +2025-06-15T03:16:50.909Z [INFO] server-3fa90b8d-36f1-d271-e086-0b48a19124af: aborting pipeline replication: peer="{Voter server-7738d9e7-c66a-d7ac-0df3-6b97cb43057d 7738d9e7-c66a-d7ac-0df3-6b97cb43057d}" +2025-06-15T03:16:50.909Z [ERROR] server-3fa90b8d-36f1-d271-e086-0b48a19124af: failed to get log: index=2 error="log not found" +2025-06-15T03:16:50.909Z [INFO] server-3fa90b8d-36f1-d271-e086-0b48a19124af: aborting pipeline replication: peer="{Voter server-969fac36-b2b4-e892-a0c5-21a6371f1331 969fac36-b2b4-e892-a0c5-21a6371f1331}" +2025-06-15T03:16:50.916Z [ERROR] server-3fa90b8d-36f1-d271-e086-0b48a19124af: failed to get log: index=2 error="log not found" +2025-06-15T03:16:50.917Z [INFO] creating new snapshot: path=/tmp/raft4007007121/snapshots/2-13-1749957410917.tmp +2025-06-15T03:16:50.917Z [INFO] server-7738d9e7-c66a-d7ac-0df3-6b97cb43057d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:50.919Z [ERROR] server-3fa90b8d-36f1-d271-e086-0b48a19124af: failed to get log: index=2 error="log not found" +2025-06-15T03:16:50.920Z [INFO] creating new snapshot: path=/tmp/raft3543393193/snapshots/2-13-1749957410920.tmp +2025-06-15T03:16:50.920Z [INFO] server-969fac36-b2b4-e892-a0c5-21a6371f1331: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:51.055Z [INFO] server-7738d9e7-c66a-d7ac-0df3-6b97cb43057d: copied to local snapshot: bytes=71 +2025-06-15T03:16:51.055Z [INFO] server-969fac36-b2b4-e892-a0c5-21a6371f1331: copied to local snapshot: bytes=71 +2025-06-15T03:16:51.055Z [INFO] server-969fac36-b2b4-e892-a0c5-21a6371f1331: snapshot restore progress: id=2-13-1749957410920 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:51.055Z [INFO] server-7738d9e7-c66a-d7ac-0df3-6b97cb43057d: snapshot restore progress: id=2-13-1749957410917 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:51.055Z [INFO] server-969fac36-b2b4-e892-a0c5-21a6371f1331: removing all old logs from log store +2025-06-15T03:16:51.055Z [INFO] server-969fac36-b2b4-e892-a0c5-21a6371f1331: compacting logs: from=1 to=2 +2025-06-15T03:16:51.055Z [INFO] server-7738d9e7-c66a-d7ac-0df3-6b97cb43057d: removing all old logs from log store +2025-06-15T03:16:51.055Z [INFO] server-969fac36-b2b4-e892-a0c5-21a6371f1331: Installed remote snapshot +2025-06-15T03:16:51.055Z [INFO] server-7738d9e7-c66a-d7ac-0df3-6b97cb43057d: compacting logs: from=1 to=2 +2025-06-15T03:16:51.055Z [INFO] server-7738d9e7-c66a-d7ac-0df3-6b97cb43057d: Installed remote snapshot +2025-06-15T03:16:51.056Z [INFO] server-3fa90b8d-36f1-d271-e086-0b48a19124af: pipelining replication: peer="{Voter server-969fac36-b2b4-e892-a0c5-21a6371f1331 969fac36-b2b4-e892-a0c5-21a6371f1331}" +2025-06-15T03:16:51.056Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:16:51.056Z [INFO] server-3fa90b8d-36f1-d271-e086-0b48a19124af: pipelining replication: peer="{Voter server-7738d9e7-c66a-d7ac-0df3-6b97cb43057d 7738d9e7-c66a-d7ac-0df3-6b97cb43057d}" +2025-06-15T03:16:52.061Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:16:51.056124767 +0000 UTC m=+26.767802948 (2 nodes), 1.094µs from start of poll, 1.789381729s from cluster start. Timeout at 2025-06-15 03:16:52.061703294 +0000 UTC m=+27.773381455, 1.005578507s after stability +2025-06-15T03:16:52.073Z [INFO] server-1eb2bf99-5771-6d96-a925-545c775d3805: aborting pipeline replication: peer="{Voter server-a70ed36c-077b-afd9-712b-c981cb14be7a a70ed36c-077b-afd9-712b-c981cb14be7a}" +2025-06-15T03:16:52.073Z [INFO] server-1eb2bf99-5771-6d96-a925-545c775d3805: aborting pipeline replication: peer="{Voter server-18d5ec34-2c12-cd28-7b8d-6545db762b6d 18d5ec34-2c12-cd28-7b8d-6545db762b6d}" === RUN TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false -2024-05-12T20:50:17.337Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-571ecbe7-37d6-8287-de93-712c050c6f8c Address:571ecbe7-37d6-8287-de93-712c050c6f8c} {Suffrage:Voter ID:server-9370cdb0-e1b1-4032-af80-0d141d448f8a Address:9370cdb0-e1b1-4032-af80-0d141d448f8a} {Suffrage:Voter ID:server-f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe Address:f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe}]" -2024-05-12T20:50:17.337Z [INFO] server-9370cdb0-e1b1-4032-af80-0d141d448f8a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-571ecbe7-37d6-8287-de93-712c050c6f8c Address:571ecbe7-37d6-8287-de93-712c050c6f8c} {Suffrage:Voter ID:server-9370cdb0-e1b1-4032-af80-0d141d448f8a Address:9370cdb0-e1b1-4032-af80-0d141d448f8a} {Suffrage:Voter ID:server-f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe Address:f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe}]" -2024-05-12T20:50:17.337Z [INFO] server-f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-571ecbe7-37d6-8287-de93-712c050c6f8c Address:571ecbe7-37d6-8287-de93-712c050c6f8c} {Suffrage:Voter ID:server-9370cdb0-e1b1-4032-af80-0d141d448f8a Address:9370cdb0-e1b1-4032-af80-0d141d448f8a} {Suffrage:Voter ID:server-f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe Address:f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe}]" -2024-05-12T20:50:17.337Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:17.337Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: entering follower state: follower="Node at 571ecbe7-37d6-8287-de93-712c050c6f8c [Follower]" leader-address= leader-id= -2024-05-12T20:50:17.337Z [INFO] server-9370cdb0-e1b1-4032-af80-0d141d448f8a: entering follower state: follower="Node at 9370cdb0-e1b1-4032-af80-0d141d448f8a [Follower]" leader-address= leader-id= -2024-05-12T20:50:17.337Z [INFO] server-f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe: entering follower state: follower="Node at f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe [Follower]" leader-address= leader-id= -2024-05-12T20:50:18.082Z [WARN] server-571ecbe7-37d6-8287-de93-712c050c6f8c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:18.082Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: entering candidate state: node="Node at 571ecbe7-37d6-8287-de93-712c050c6f8c [Candidate]" term=2 -2024-05-12T20:50:18.083Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: election won: term=2 tally=2 -2024-05-12T20:50:18.083Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: entering leader state: leader="Node at 571ecbe7-37d6-8287-de93-712c050c6f8c [Leader]" -2024-05-12T20:50:18.083Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: added peer, starting replication: peer=server-9370cdb0-e1b1-4032-af80-0d141d448f8a -2024-05-12T20:50:18.083Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: added peer, starting replication: peer=server-f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe -2024-05-12T20:50:18.083Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: pipelining replication: peer="{Voter server-f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe}" -2024-05-12T20:50:18.083Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: pipelining replication: peer="{Voter server-9370cdb0-e1b1-4032-af80-0d141d448f8a 9370cdb0-e1b1-4032-af80-0d141d448f8a}" -2024-05-12T20:50:19.090Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:18.083665231 +0000 UTC m=+29.364365670 (1 nodes), 746.093621ms from start of poll, 746.481091ms from cluster start. Timeout at 2024-05-12 20:50:19.090793549 +0000 UTC m=+30.371493873, 1.007128203s after stability -2024-05-12T20:50:19.092Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: starting snapshot up to: index=12 -2024-05-12T20:50:19.092Z [INFO] creating new snapshot: path=/tmp/raft1093364572/snapshots/2-12-1715547019092.tmp -2024-05-12T20:50:19.093Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: snapshot complete up to: index=12 -2024-05-12T20:50:19.094Z [INFO] creating new snapshot: path=/tmp/raft1093364572/snapshots/2-23-1715547019094.tmp -2024-05-12T20:50:19.095Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: copied to local snapshot: bytes=71 -2024-05-12T20:50:19.096Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: snapshot restore progress: id=2-23-1715547019094 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:19.096Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: restored user snapshot: index=23 -2024-05-12T20:50:19.096Z [ERROR] server-571ecbe7-37d6-8287-de93-712c050c6f8c: failed to get log: index=23 error="log not found" -2024-05-12T20:50:19.096Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: aborting pipeline replication: peer="{Voter server-f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe}" -2024-05-12T20:50:19.096Z [ERROR] server-571ecbe7-37d6-8287-de93-712c050c6f8c: failed to get log: index=23 error="log not found" -2024-05-12T20:50:19.096Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: aborting pipeline replication: peer="{Voter server-9370cdb0-e1b1-4032-af80-0d141d448f8a 9370cdb0-e1b1-4032-af80-0d141d448f8a}" -2024-05-12T20:50:19.103Z [ERROR] server-571ecbe7-37d6-8287-de93-712c050c6f8c: failed to get log: index=23 error="log not found" -2024-05-12T20:50:19.104Z [INFO] creating new snapshot: path=/tmp/raft1129056136/snapshots/2-23-1715547019104.tmp -2024-05-12T20:50:19.104Z [INFO] server-f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:19.105Z [INFO] server-f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe: copied to local snapshot: bytes=71 -2024-05-12T20:50:19.105Z [INFO] server-f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe: snapshot restore progress: id=2-23-1715547019104 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:19.105Z [INFO] server-f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe: Installed remote snapshot -2024-05-12T20:50:19.105Z [WARN] server-f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe: skipping application of old log: index=22 -2024-05-12T20:50:19.105Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: pipelining replication: peer="{Voter server-f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe}" -2024-05-12T20:50:19.105Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:19.107Z [ERROR] server-571ecbe7-37d6-8287-de93-712c050c6f8c: failed to get log: index=23 error="log not found" -2024-05-12T20:50:19.108Z [INFO] creating new snapshot: path=/tmp/raft2001769339/snapshots/2-23-1715547019108.tmp -2024-05-12T20:50:19.108Z [INFO] server-9370cdb0-e1b1-4032-af80-0d141d448f8a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:19.108Z [INFO] server-9370cdb0-e1b1-4032-af80-0d141d448f8a: copied to local snapshot: bytes=71 -2024-05-12T20:50:19.109Z [INFO] server-9370cdb0-e1b1-4032-af80-0d141d448f8a: snapshot restore progress: id=2-23-1715547019108 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:19.109Z [INFO] server-9370cdb0-e1b1-4032-af80-0d141d448f8a: Installed remote snapshot -2024-05-12T20:50:19.109Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: pipelining replication: peer="{Voter server-9370cdb0-e1b1-4032-af80-0d141d448f8a 9370cdb0-e1b1-4032-af80-0d141d448f8a}" -2024-05-12T20:50:20.110Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:19.105540572 +0000 UTC m=+30.386240840 (2 nodes), 1.55µs from start of poll, 1.768356261s from cluster start. Timeout at 2024-05-12 20:50:20.110766899 +0000 UTC m=+31.391467299, 1.005226459s after stability -2024-05-12T20:50:20.134Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: aborting pipeline replication: peer="{Voter server-f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe f1f572f4-36de-9a21-ec7a-df1b1b1e6bbe}" -2024-05-12T20:50:20.135Z [INFO] server-571ecbe7-37d6-8287-de93-712c050c6f8c: aborting pipeline replication: peer="{Voter server-9370cdb0-e1b1-4032-af80-0d141d448f8a 9370cdb0-e1b1-4032-af80-0d141d448f8a}" +2025-06-15T03:16:52.075Z [INFO] server-4ee0d77a-880f-1ebe-99c3-f2daca487acc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ee0d77a-880f-1ebe-99c3-f2daca487acc Address:4ee0d77a-880f-1ebe-99c3-f2daca487acc} {Suffrage:Voter ID:server-bec1def8-c90c-8086-e6e0-fb1e38e76dfc Address:bec1def8-c90c-8086-e6e0-fb1e38e76dfc} {Suffrage:Voter ID:server-d8815f2b-a396-183d-b5c6-5cbfe0152116 Address:d8815f2b-a396-183d-b5c6-5cbfe0152116}]" +2025-06-15T03:16:52.075Z [INFO] server-4ee0d77a-880f-1ebe-99c3-f2daca487acc: entering follower state: follower="Node at 4ee0d77a-880f-1ebe-99c3-f2daca487acc [Follower]" leader-address= leader-id= +2025-06-15T03:16:52.075Z [INFO] server-bec1def8-c90c-8086-e6e0-fb1e38e76dfc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ee0d77a-880f-1ebe-99c3-f2daca487acc Address:4ee0d77a-880f-1ebe-99c3-f2daca487acc} {Suffrage:Voter ID:server-bec1def8-c90c-8086-e6e0-fb1e38e76dfc Address:bec1def8-c90c-8086-e6e0-fb1e38e76dfc} {Suffrage:Voter ID:server-d8815f2b-a396-183d-b5c6-5cbfe0152116 Address:d8815f2b-a396-183d-b5c6-5cbfe0152116}]" +2025-06-15T03:16:52.075Z [INFO] server-bec1def8-c90c-8086-e6e0-fb1e38e76dfc: entering follower state: follower="Node at bec1def8-c90c-8086-e6e0-fb1e38e76dfc [Follower]" leader-address= leader-id= +2025-06-15T03:16:52.075Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ee0d77a-880f-1ebe-99c3-f2daca487acc Address:4ee0d77a-880f-1ebe-99c3-f2daca487acc} {Suffrage:Voter ID:server-bec1def8-c90c-8086-e6e0-fb1e38e76dfc Address:bec1def8-c90c-8086-e6e0-fb1e38e76dfc} {Suffrage:Voter ID:server-d8815f2b-a396-183d-b5c6-5cbfe0152116 Address:d8815f2b-a396-183d-b5c6-5cbfe0152116}]" +2025-06-15T03:16:52.075Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:52.075Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: entering follower state: follower="Node at d8815f2b-a396-183d-b5c6-5cbfe0152116 [Follower]" leader-address= leader-id= +2025-06-15T03:16:52.661Z [WARN] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:52.661Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: entering candidate state: node="Node at d8815f2b-a396-183d-b5c6-5cbfe0152116 [Candidate]" term=2 +2025-06-15T03:16:52.661Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: election won: term=2 tally=2 +2025-06-15T03:16:52.661Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: entering leader state: leader="Node at d8815f2b-a396-183d-b5c6-5cbfe0152116 [Leader]" +2025-06-15T03:16:52.661Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: added peer, starting replication: peer=server-4ee0d77a-880f-1ebe-99c3-f2daca487acc +2025-06-15T03:16:52.661Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: added peer, starting replication: peer=server-bec1def8-c90c-8086-e6e0-fb1e38e76dfc +2025-06-15T03:16:52.662Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: pipelining replication: peer="{Voter server-bec1def8-c90c-8086-e6e0-fb1e38e76dfc bec1def8-c90c-8086-e6e0-fb1e38e76dfc}" +2025-06-15T03:16:52.662Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: pipelining replication: peer="{Voter server-4ee0d77a-880f-1ebe-99c3-f2daca487acc 4ee0d77a-880f-1ebe-99c3-f2daca487acc}" +2025-06-15T03:16:53.667Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:52.662561495 +0000 UTC m=+28.374239676 (1 nodes), 587.279031ms from start of poll, 587.650958ms from cluster start. Timeout at 2025-06-15 03:16:53.667700232 +0000 UTC m=+29.379378397, 1.005138721s after stability +2025-06-15T03:16:53.668Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: starting snapshot up to: index=12 +2025-06-15T03:16:53.668Z [INFO] creating new snapshot: path=/tmp/raft1691894363/snapshots/2-12-1749957413668.tmp +2025-06-15T03:16:53.804Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: snapshot complete up to: index=12 +2025-06-15T03:16:53.805Z [INFO] creating new snapshot: path=/tmp/raft1691894363/snapshots/2-23-1749957413805.tmp +2025-06-15T03:16:53.805Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: copied to local snapshot: bytes=71 +2025-06-15T03:16:53.805Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: snapshot restore progress: id=2-23-1749957413805 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:53.805Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: restored user snapshot: index=23 +2025-06-15T03:16:53.805Z [ERROR] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: failed to get log: index=23 error="log not found" +2025-06-15T03:16:53.805Z [ERROR] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: failed to get log: index=23 error="log not found" +2025-06-15T03:16:53.805Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: aborting pipeline replication: peer="{Voter server-4ee0d77a-880f-1ebe-99c3-f2daca487acc 4ee0d77a-880f-1ebe-99c3-f2daca487acc}" +2025-06-15T03:16:53.805Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: aborting pipeline replication: peer="{Voter server-bec1def8-c90c-8086-e6e0-fb1e38e76dfc bec1def8-c90c-8086-e6e0-fb1e38e76dfc}" +2025-06-15T03:16:53.811Z [ERROR] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: failed to get log: index=23 error="log not found" +2025-06-15T03:16:53.812Z [INFO] creating new snapshot: path=/tmp/raft879491927/snapshots/2-23-1749957413812.tmp +2025-06-15T03:16:53.812Z [INFO] server-bec1def8-c90c-8086-e6e0-fb1e38e76dfc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:53.814Z [ERROR] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: failed to get log: index=23 error="log not found" +2025-06-15T03:16:53.814Z [INFO] creating new snapshot: path=/tmp/raft3643002752/snapshots/2-23-1749957413814.tmp +2025-06-15T03:16:53.814Z [INFO] server-4ee0d77a-880f-1ebe-99c3-f2daca487acc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:53.981Z [INFO] server-4ee0d77a-880f-1ebe-99c3-f2daca487acc: copied to local snapshot: bytes=71 +2025-06-15T03:16:53.981Z [INFO] server-bec1def8-c90c-8086-e6e0-fb1e38e76dfc: copied to local snapshot: bytes=71 +2025-06-15T03:16:53.981Z [INFO] server-bec1def8-c90c-8086-e6e0-fb1e38e76dfc: snapshot restore progress: id=2-23-1749957413812 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:53.981Z [INFO] server-bec1def8-c90c-8086-e6e0-fb1e38e76dfc: Installed remote snapshot +2025-06-15T03:16:53.981Z [INFO] server-4ee0d77a-880f-1ebe-99c3-f2daca487acc: snapshot restore progress: id=2-23-1749957413814 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:53.981Z [INFO] server-4ee0d77a-880f-1ebe-99c3-f2daca487acc: Installed remote snapshot +2025-06-15T03:16:53.981Z [WARN] server-bec1def8-c90c-8086-e6e0-fb1e38e76dfc: skipping application of old log: index=22 +2025-06-15T03:16:53.981Z [WARN] server-4ee0d77a-880f-1ebe-99c3-f2daca487acc: skipping application of old log: index=22 +2025-06-15T03:16:53.981Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: pipelining replication: peer="{Voter server-bec1def8-c90c-8086-e6e0-fb1e38e76dfc bec1def8-c90c-8086-e6e0-fb1e38e76dfc}" +2025-06-15T03:16:53.981Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: pipelining replication: peer="{Voter server-4ee0d77a-880f-1ebe-99c3-f2daca487acc 4ee0d77a-880f-1ebe-99c3-f2daca487acc}" +2025-06-15T03:16:53.981Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:16:54.986Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:16:53.981692404 +0000 UTC m=+29.693370580 (2 nodes), 887ns from start of poll, 1.906781862s from cluster start. Timeout at 2025-06-15 03:16:54.986753726 +0000 UTC m=+30.698431925, 1.005061345s after stability +2025-06-15T03:16:54.998Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: aborting pipeline replication: peer="{Voter server-bec1def8-c90c-8086-e6e0-fb1e38e76dfc bec1def8-c90c-8086-e6e0-fb1e38e76dfc}" +2025-06-15T03:16:54.998Z [INFO] server-d8815f2b-a396-183d-b5c6-5cbfe0152116: aborting pipeline replication: peer="{Voter server-4ee0d77a-880f-1ebe-99c3-f2daca487acc 4ee0d77a-880f-1ebe-99c3-f2daca487acc}" === RUN TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false -2024-05-12T20:50:20.137Z [INFO] server-aceb86ce-86e5-10e5-9c86-4d3a737a7258: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aceb86ce-86e5-10e5-9c86-4d3a737a7258 Address:aceb86ce-86e5-10e5-9c86-4d3a737a7258} {Suffrage:Voter ID:server-0039c2cd-5f0f-6356-8eeb-68b7baee7a4d Address:0039c2cd-5f0f-6356-8eeb-68b7baee7a4d} {Suffrage:Voter ID:server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92 Address:5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92}]" -2024-05-12T20:50:20.137Z [INFO] server-0039c2cd-5f0f-6356-8eeb-68b7baee7a4d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aceb86ce-86e5-10e5-9c86-4d3a737a7258 Address:aceb86ce-86e5-10e5-9c86-4d3a737a7258} {Suffrage:Voter ID:server-0039c2cd-5f0f-6356-8eeb-68b7baee7a4d Address:0039c2cd-5f0f-6356-8eeb-68b7baee7a4d} {Suffrage:Voter ID:server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92 Address:5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92}]" -2024-05-12T20:50:20.137Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aceb86ce-86e5-10e5-9c86-4d3a737a7258 Address:aceb86ce-86e5-10e5-9c86-4d3a737a7258} {Suffrage:Voter ID:server-0039c2cd-5f0f-6356-8eeb-68b7baee7a4d Address:0039c2cd-5f0f-6356-8eeb-68b7baee7a4d} {Suffrage:Voter ID:server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92 Address:5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92}]" -2024-05-12T20:50:20.137Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:20.137Z [INFO] server-aceb86ce-86e5-10e5-9c86-4d3a737a7258: entering follower state: follower="Node at aceb86ce-86e5-10e5-9c86-4d3a737a7258 [Follower]" leader-address= leader-id= -2024-05-12T20:50:20.137Z [INFO] server-0039c2cd-5f0f-6356-8eeb-68b7baee7a4d: entering follower state: follower="Node at 0039c2cd-5f0f-6356-8eeb-68b7baee7a4d [Follower]" leader-address= leader-id= -2024-05-12T20:50:20.137Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: entering follower state: follower="Node at 5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92 [Follower]" leader-address= leader-id= -2024-05-12T20:50:20.929Z [WARN] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:20.929Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: entering candidate state: node="Node at 5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92 [Candidate]" term=2 -2024-05-12T20:50:20.929Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: election won: term=2 tally=2 -2024-05-12T20:50:20.929Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: entering leader state: leader="Node at 5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92 [Leader]" -2024-05-12T20:50:20.929Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: added peer, starting replication: peer=server-aceb86ce-86e5-10e5-9c86-4d3a737a7258 -2024-05-12T20:50:20.929Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: added peer, starting replication: peer=server-0039c2cd-5f0f-6356-8eeb-68b7baee7a4d -2024-05-12T20:50:20.929Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: pipelining replication: peer="{Voter server-0039c2cd-5f0f-6356-8eeb-68b7baee7a4d 0039c2cd-5f0f-6356-8eeb-68b7baee7a4d}" -2024-05-12T20:50:20.929Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: pipelining replication: peer="{Voter server-aceb86ce-86e5-10e5-9c86-4d3a737a7258 aceb86ce-86e5-10e5-9c86-4d3a737a7258}" -2024-05-12T20:50:21.938Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:20.930102758 +0000 UTC m=+32.210803086 (1 nodes), 792.615073ms from start of poll, 792.981025ms from cluster start. Timeout at 2024-05-12 20:50:21.938769845 +0000 UTC m=+33.219469740, 1.008666654s after stability -2024-05-12T20:50:21.940Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: starting snapshot up to: index=12 -2024-05-12T20:50:21.940Z [INFO] creating new snapshot: path=/tmp/raft4184761089/snapshots/2-12-1715547021940.tmp -2024-05-12T20:50:21.947Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: snapshot complete up to: index=12 -2024-05-12T20:50:21.969Z [INFO] creating new snapshot: path=/tmp/raft4184761089/snapshots/2-23-1715547021969.tmp -2024-05-12T20:50:21.970Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: copied to local snapshot: bytes=71 -2024-05-12T20:50:21.975Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: snapshot restore progress: id=2-23-1715547021969 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:21.975Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: removing all old logs from log store -2024-05-12T20:50:21.975Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: compacting logs: from=1 to=22 -2024-05-12T20:50:21.975Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: restored user snapshot: index=23 -2024-05-12T20:50:21.975Z [ERROR] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: failed to get log: index=22 error="log not found" -2024-05-12T20:50:21.975Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: aborting pipeline replication: peer="{Voter server-0039c2cd-5f0f-6356-8eeb-68b7baee7a4d 0039c2cd-5f0f-6356-8eeb-68b7baee7a4d}" -2024-05-12T20:50:21.975Z [ERROR] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: failed to get log: index=22 error="log not found" -2024-05-12T20:50:21.975Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: aborting pipeline replication: peer="{Voter server-aceb86ce-86e5-10e5-9c86-4d3a737a7258 aceb86ce-86e5-10e5-9c86-4d3a737a7258}" -2024-05-12T20:50:21.987Z [ERROR] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: failed to get log: index=22 error="log not found" -2024-05-12T20:50:21.987Z [ERROR] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: failed to get log: index=22 error="log not found" -2024-05-12T20:50:21.987Z [INFO] creating new snapshot: path=/tmp/raft2257228955/snapshots/2-23-1715547021987.tmp -2024-05-12T20:50:21.987Z [INFO] creating new snapshot: path=/tmp/raft1930655595/snapshots/2-23-1715547021987.tmp -2024-05-12T20:50:21.988Z [INFO] server-aceb86ce-86e5-10e5-9c86-4d3a737a7258: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:21.988Z [INFO] server-0039c2cd-5f0f-6356-8eeb-68b7baee7a4d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:21.988Z [INFO] server-0039c2cd-5f0f-6356-8eeb-68b7baee7a4d: copied to local snapshot: bytes=71 -2024-05-12T20:50:21.988Z [INFO] server-0039c2cd-5f0f-6356-8eeb-68b7baee7a4d: snapshot restore progress: id=2-23-1715547021987 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:21.988Z [INFO] server-0039c2cd-5f0f-6356-8eeb-68b7baee7a4d: removing all old logs from log store -2024-05-12T20:50:21.988Z [INFO] server-0039c2cd-5f0f-6356-8eeb-68b7baee7a4d: compacting logs: from=1 to=22 -2024-05-12T20:50:21.988Z [INFO] server-0039c2cd-5f0f-6356-8eeb-68b7baee7a4d: Installed remote snapshot -2024-05-12T20:50:21.988Z [INFO] server-aceb86ce-86e5-10e5-9c86-4d3a737a7258: copied to local snapshot: bytes=71 -2024-05-12T20:50:21.988Z [WARN] server-0039c2cd-5f0f-6356-8eeb-68b7baee7a4d: skipping application of old log: index=22 -2024-05-12T20:50:21.989Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: pipelining replication: peer="{Voter server-0039c2cd-5f0f-6356-8eeb-68b7baee7a4d 0039c2cd-5f0f-6356-8eeb-68b7baee7a4d}" -2024-05-12T20:50:21.989Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:21.989Z [INFO] server-aceb86ce-86e5-10e5-9c86-4d3a737a7258: snapshot restore progress: id=2-23-1715547021987 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:21.989Z [INFO] server-aceb86ce-86e5-10e5-9c86-4d3a737a7258: removing all old logs from log store -2024-05-12T20:50:21.989Z [INFO] server-aceb86ce-86e5-10e5-9c86-4d3a737a7258: compacting logs: from=1 to=22 -2024-05-12T20:50:21.989Z [INFO] server-aceb86ce-86e5-10e5-9c86-4d3a737a7258: Installed remote snapshot -2024-05-12T20:50:21.989Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: pipelining replication: peer="{Voter server-aceb86ce-86e5-10e5-9c86-4d3a737a7258 aceb86ce-86e5-10e5-9c86-4d3a737a7258}" -2024-05-12T20:50:22.994Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:21.989127536 +0000 UTC m=+33.269827981 (2 nodes), 1.814µs from start of poll, 1.85200592s from cluster start. Timeout at 2024-05-12 20:50:22.994816681 +0000 UTC m=+34.275517164, 1.005689183s after stability -2024-05-12T20:50:23.015Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: aborting pipeline replication: peer="{Voter server-0039c2cd-5f0f-6356-8eeb-68b7baee7a4d 0039c2cd-5f0f-6356-8eeb-68b7baee7a4d}" -2024-05-12T20:50:23.015Z [INFO] server-5ebf2aaa-1cef-e4e1-2aa9-4f3326b7dc92: aborting pipeline replication: peer="{Voter server-aceb86ce-86e5-10e5-9c86-4d3a737a7258 aceb86ce-86e5-10e5-9c86-4d3a737a7258}" +2025-06-15T03:16:55.000Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b Address:46b69dc8-e76f-2f53-ae8a-a8a73a77a96b} {Suffrage:Voter ID:server-ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd Address:ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd} {Suffrage:Voter ID:server-1e12d441-ba6b-dda8-e071-6042201730f0 Address:1e12d441-ba6b-dda8-e071-6042201730f0}]" +2025-06-15T03:16:55.000Z [INFO] server-ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b Address:46b69dc8-e76f-2f53-ae8a-a8a73a77a96b} {Suffrage:Voter ID:server-ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd Address:ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd} {Suffrage:Voter ID:server-1e12d441-ba6b-dda8-e071-6042201730f0 Address:1e12d441-ba6b-dda8-e071-6042201730f0}]" +2025-06-15T03:16:55.000Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: entering follower state: follower="Node at 46b69dc8-e76f-2f53-ae8a-a8a73a77a96b [Follower]" leader-address= leader-id= +2025-06-15T03:16:55.000Z [INFO] server-ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd: entering follower state: follower="Node at ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd [Follower]" leader-address= leader-id= +2025-06-15T03:16:55.000Z [INFO] server-1e12d441-ba6b-dda8-e071-6042201730f0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b Address:46b69dc8-e76f-2f53-ae8a-a8a73a77a96b} {Suffrage:Voter ID:server-ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd Address:ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd} {Suffrage:Voter ID:server-1e12d441-ba6b-dda8-e071-6042201730f0 Address:1e12d441-ba6b-dda8-e071-6042201730f0}]" +2025-06-15T03:16:55.000Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:55.000Z [INFO] server-1e12d441-ba6b-dda8-e071-6042201730f0: entering follower state: follower="Node at 1e12d441-ba6b-dda8-e071-6042201730f0 [Follower]" leader-address= leader-id= +2025-06-15T03:16:55.868Z [WARN] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:55.869Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: entering candidate state: node="Node at 46b69dc8-e76f-2f53-ae8a-a8a73a77a96b [Candidate]" term=2 +2025-06-15T03:16:55.869Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: election won: term=2 tally=2 +2025-06-15T03:16:55.869Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: entering leader state: leader="Node at 46b69dc8-e76f-2f53-ae8a-a8a73a77a96b [Leader]" +2025-06-15T03:16:55.869Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: added peer, starting replication: peer=server-ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd +2025-06-15T03:16:55.869Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: added peer, starting replication: peer=server-1e12d441-ba6b-dda8-e071-6042201730f0 +2025-06-15T03:16:55.869Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: pipelining replication: peer="{Voter server-1e12d441-ba6b-dda8-e071-6042201730f0 1e12d441-ba6b-dda8-e071-6042201730f0}" +2025-06-15T03:16:55.869Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: pipelining replication: peer="{Voter server-ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd}" +2025-06-15T03:16:56.875Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:55.869218999 +0000 UTC m=+31.580897170 (1 nodes), 868.548354ms from start of poll, 868.935447ms from cluster start. Timeout at 2025-06-15 03:16:56.875241176 +0000 UTC m=+32.586919338, 1.006022168s after stability +2025-06-15T03:16:56.875Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: starting snapshot up to: index=12 +2025-06-15T03:16:56.875Z [INFO] creating new snapshot: path=/tmp/raft3259987016/snapshots/2-12-1749957416875.tmp +2025-06-15T03:16:56.915Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: snapshot complete up to: index=12 +2025-06-15T03:16:56.915Z [INFO] creating new snapshot: path=/tmp/raft3259987016/snapshots/2-23-1749957416915.tmp +2025-06-15T03:16:56.916Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: copied to local snapshot: bytes=71 +2025-06-15T03:16:56.916Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: snapshot restore progress: id=2-23-1749957416915 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:56.916Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: removing all old logs from log store +2025-06-15T03:16:56.916Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: compacting logs: from=1 to=22 +2025-06-15T03:16:56.916Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: restored user snapshot: index=23 +2025-06-15T03:16:56.916Z [ERROR] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: failed to get log: index=22 error="log not found" +2025-06-15T03:16:56.916Z [ERROR] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: failed to get log: index=22 error="log not found" +2025-06-15T03:16:56.916Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: aborting pipeline replication: peer="{Voter server-ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd}" +2025-06-15T03:16:56.916Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: aborting pipeline replication: peer="{Voter server-1e12d441-ba6b-dda8-e071-6042201730f0 1e12d441-ba6b-dda8-e071-6042201730f0}" +2025-06-15T03:16:56.922Z [ERROR] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: failed to get log: index=22 error="log not found" +2025-06-15T03:16:56.922Z [INFO] creating new snapshot: path=/tmp/raft2127481473/snapshots/2-23-1749957416922.tmp +2025-06-15T03:16:56.922Z [INFO] server-1e12d441-ba6b-dda8-e071-6042201730f0: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:56.922Z [INFO] server-1e12d441-ba6b-dda8-e071-6042201730f0: copied to local snapshot: bytes=71 +2025-06-15T03:16:56.923Z [INFO] server-1e12d441-ba6b-dda8-e071-6042201730f0: snapshot restore progress: id=2-23-1749957416922 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:56.923Z [INFO] server-1e12d441-ba6b-dda8-e071-6042201730f0: removing all old logs from log store +2025-06-15T03:16:56.923Z [INFO] server-1e12d441-ba6b-dda8-e071-6042201730f0: compacting logs: from=1 to=22 +2025-06-15T03:16:56.923Z [INFO] server-1e12d441-ba6b-dda8-e071-6042201730f0: Installed remote snapshot +2025-06-15T03:16:56.923Z [WARN] server-1e12d441-ba6b-dda8-e071-6042201730f0: skipping application of old log: index=22 +2025-06-15T03:16:56.923Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: pipelining replication: peer="{Voter server-1e12d441-ba6b-dda8-e071-6042201730f0 1e12d441-ba6b-dda8-e071-6042201730f0}" +2025-06-15T03:16:56.923Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:16:56.926Z [ERROR] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: failed to get log: index=22 error="log not found" +2025-06-15T03:16:56.926Z [INFO] creating new snapshot: path=/tmp/raft3305971665/snapshots/2-23-1749957416926.tmp +2025-06-15T03:16:56.927Z [INFO] server-ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:56.927Z [INFO] server-ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd: copied to local snapshot: bytes=71 +2025-06-15T03:16:56.927Z [INFO] server-ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd: snapshot restore progress: id=2-23-1749957416926 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:16:56.927Z [INFO] server-ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd: removing all old logs from log store +2025-06-15T03:16:56.927Z [INFO] server-ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd: compacting logs: from=1 to=22 +2025-06-15T03:16:56.927Z [INFO] server-ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd: Installed remote snapshot +2025-06-15T03:16:56.927Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: pipelining replication: peer="{Voter server-ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd}" +2025-06-15T03:16:57.928Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:16:56.923210263 +0000 UTC m=+32.634888429 (2 nodes), 548ns from start of poll, 1.922926706s from cluster start. Timeout at 2025-06-15 03:16:57.928756982 +0000 UTC m=+33.640435147, 1.005546718s after stability +2025-06-15T03:16:57.940Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: aborting pipeline replication: peer="{Voter server-1e12d441-ba6b-dda8-e071-6042201730f0 1e12d441-ba6b-dda8-e071-6042201730f0}" +2025-06-15T03:16:57.940Z [INFO] server-46b69dc8-e76f-2f53-ae8a-a8a73a77a96b: aborting pipeline replication: peer="{Voter server-ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd ad3a7bf2-5fd8-9e1f-39e4-39f35f3a49fd}" === RUN TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true -2024-05-12T20:50:23.018Z [INFO] server-0dbd5ac7-f6f4-6181-78dd-b825637e93c0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0dbd5ac7-f6f4-6181-78dd-b825637e93c0 Address:0dbd5ac7-f6f4-6181-78dd-b825637e93c0} {Suffrage:Voter ID:server-088fbba9-6f18-a0bf-43ad-18098f3eeb53 Address:088fbba9-6f18-a0bf-43ad-18098f3eeb53} {Suffrage:Voter ID:server-d981699b-d972-8a2e-6e2d-ab815720803a Address:d981699b-d972-8a2e-6e2d-ab815720803a}]" -2024-05-12T20:50:23.018Z [INFO] server-088fbba9-6f18-a0bf-43ad-18098f3eeb53: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0dbd5ac7-f6f4-6181-78dd-b825637e93c0 Address:0dbd5ac7-f6f4-6181-78dd-b825637e93c0} {Suffrage:Voter ID:server-088fbba9-6f18-a0bf-43ad-18098f3eeb53 Address:088fbba9-6f18-a0bf-43ad-18098f3eeb53} {Suffrage:Voter ID:server-d981699b-d972-8a2e-6e2d-ab815720803a Address:d981699b-d972-8a2e-6e2d-ab815720803a}]" -2024-05-12T20:50:23.018Z [INFO] server-d981699b-d972-8a2e-6e2d-ab815720803a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0dbd5ac7-f6f4-6181-78dd-b825637e93c0 Address:0dbd5ac7-f6f4-6181-78dd-b825637e93c0} {Suffrage:Voter ID:server-088fbba9-6f18-a0bf-43ad-18098f3eeb53 Address:088fbba9-6f18-a0bf-43ad-18098f3eeb53} {Suffrage:Voter ID:server-d981699b-d972-8a2e-6e2d-ab815720803a Address:d981699b-d972-8a2e-6e2d-ab815720803a}]" -2024-05-12T20:50:23.018Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:23.018Z [INFO] server-0dbd5ac7-f6f4-6181-78dd-b825637e93c0: entering follower state: follower="Node at 0dbd5ac7-f6f4-6181-78dd-b825637e93c0 [Follower]" leader-address= leader-id= -2024-05-12T20:50:23.018Z [INFO] server-d981699b-d972-8a2e-6e2d-ab815720803a: entering follower state: follower="Node at d981699b-d972-8a2e-6e2d-ab815720803a [Follower]" leader-address= leader-id= -2024-05-12T20:50:23.018Z [INFO] server-088fbba9-6f18-a0bf-43ad-18098f3eeb53: entering follower state: follower="Node at 088fbba9-6f18-a0bf-43ad-18098f3eeb53 [Follower]" leader-address= leader-id= -2024-05-12T20:50:23.522Z [WARN] server-0dbd5ac7-f6f4-6181-78dd-b825637e93c0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:23.522Z [INFO] server-0dbd5ac7-f6f4-6181-78dd-b825637e93c0: entering candidate state: node="Node at 0dbd5ac7-f6f4-6181-78dd-b825637e93c0 [Candidate]" term=2 -2024-05-12T20:50:23.523Z [INFO] server-0dbd5ac7-f6f4-6181-78dd-b825637e93c0: election won: term=2 tally=2 -2024-05-12T20:50:23.523Z [INFO] server-0dbd5ac7-f6f4-6181-78dd-b825637e93c0: entering leader state: leader="Node at 0dbd5ac7-f6f4-6181-78dd-b825637e93c0 [Leader]" -2024-05-12T20:50:23.523Z [INFO] server-0dbd5ac7-f6f4-6181-78dd-b825637e93c0: added peer, starting replication: peer=server-088fbba9-6f18-a0bf-43ad-18098f3eeb53 -2024-05-12T20:50:23.523Z [INFO] server-0dbd5ac7-f6f4-6181-78dd-b825637e93c0: added peer, starting replication: peer=server-d981699b-d972-8a2e-6e2d-ab815720803a -2024-05-12T20:50:23.523Z [INFO] server-0dbd5ac7-f6f4-6181-78dd-b825637e93c0: pipelining replication: peer="{Voter server-d981699b-d972-8a2e-6e2d-ab815720803a d981699b-d972-8a2e-6e2d-ab815720803a}" -2024-05-12T20:50:23.523Z [INFO] server-0dbd5ac7-f6f4-6181-78dd-b825637e93c0: pipelining replication: peer="{Voter server-088fbba9-6f18-a0bf-43ad-18098f3eeb53 088fbba9-6f18-a0bf-43ad-18098f3eeb53}" -2024-05-12T20:50:24.539Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:23.523234117 +0000 UTC m=+34.803934516 (1 nodes), 504.689316ms from start of poll, 505.134655ms from cluster start. Timeout at 2024-05-12 20:50:24.530759368 +0000 UTC m=+35.811459263, 1.007524747s after stability -2024-05-12T20:50:24.540Z [INFO] server-0dbd5ac7-f6f4-6181-78dd-b825637e93c0: starting snapshot up to: index=12 -2024-05-12T20:50:24.540Z [INFO] creating new snapshot: path=/tmp/raft2075982004/snapshots/2-12-1715547024540.tmp -2024-05-12T20:50:24.541Z [INFO] server-0dbd5ac7-f6f4-6181-78dd-b825637e93c0: snapshot complete up to: index=12 -2024-05-12T20:50:24.543Z [INFO] server-2dffa43d-6266-bce9-bf01-bd2151b25218: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2dffa43d-6266-bce9-bf01-bd2151b25218 Address:2dffa43d-6266-bce9-bf01-bd2151b25218} {Suffrage:Voter ID:server-eeb50158-8454-b073-44ba-ef0dd60e45bc Address:eeb50158-8454-b073-44ba-ef0dd60e45bc} {Suffrage:Voter ID:server-f4416791-f9f1-d9fc-ac32-16972dddc00f Address:f4416791-f9f1-d9fc-ac32-16972dddc00f}]" -2024-05-12T20:50:24.543Z [INFO] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2dffa43d-6266-bce9-bf01-bd2151b25218 Address:2dffa43d-6266-bce9-bf01-bd2151b25218} {Suffrage:Voter ID:server-eeb50158-8454-b073-44ba-ef0dd60e45bc Address:eeb50158-8454-b073-44ba-ef0dd60e45bc} {Suffrage:Voter ID:server-f4416791-f9f1-d9fc-ac32-16972dddc00f Address:f4416791-f9f1-d9fc-ac32-16972dddc00f}]" -2024-05-12T20:50:24.543Z [INFO] server-f4416791-f9f1-d9fc-ac32-16972dddc00f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2dffa43d-6266-bce9-bf01-bd2151b25218 Address:2dffa43d-6266-bce9-bf01-bd2151b25218} {Suffrage:Voter ID:server-eeb50158-8454-b073-44ba-ef0dd60e45bc Address:eeb50158-8454-b073-44ba-ef0dd60e45bc} {Suffrage:Voter ID:server-f4416791-f9f1-d9fc-ac32-16972dddc00f Address:f4416791-f9f1-d9fc-ac32-16972dddc00f}]" -2024-05-12T20:50:24.543Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:24.543Z [INFO] server-2dffa43d-6266-bce9-bf01-bd2151b25218: entering follower state: follower="Node at 2dffa43d-6266-bce9-bf01-bd2151b25218 [Follower]" leader-address= leader-id= -2024-05-12T20:50:24.543Z [INFO] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: entering follower state: follower="Node at eeb50158-8454-b073-44ba-ef0dd60e45bc [Follower]" leader-address= leader-id= -2024-05-12T20:50:24.543Z [INFO] server-f4416791-f9f1-d9fc-ac32-16972dddc00f: entering follower state: follower="Node at f4416791-f9f1-d9fc-ac32-16972dddc00f [Follower]" leader-address= leader-id= -2024-05-12T20:50:25.194Z [WARN] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:25.194Z [INFO] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: entering candidate state: node="Node at eeb50158-8454-b073-44ba-ef0dd60e45bc [Candidate]" term=2 -2024-05-12T20:50:25.195Z [INFO] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: election won: term=2 tally=2 -2024-05-12T20:50:25.195Z [INFO] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: entering leader state: leader="Node at eeb50158-8454-b073-44ba-ef0dd60e45bc [Leader]" -2024-05-12T20:50:25.195Z [INFO] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: added peer, starting replication: peer=server-2dffa43d-6266-bce9-bf01-bd2151b25218 -2024-05-12T20:50:25.195Z [INFO] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: added peer, starting replication: peer=server-f4416791-f9f1-d9fc-ac32-16972dddc00f -2024-05-12T20:50:25.195Z [INFO] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: pipelining replication: peer="{Voter server-f4416791-f9f1-d9fc-ac32-16972dddc00f f4416791-f9f1-d9fc-ac32-16972dddc00f}" -2024-05-12T20:50:25.195Z [INFO] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: pipelining replication: peer="{Voter server-2dffa43d-6266-bce9-bf01-bd2151b25218 2dffa43d-6266-bce9-bf01-bd2151b25218}" -2024-05-12T20:50:26.203Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:25.195180069 +0000 UTC m=+36.475880851 (1 nodes), 651.368664ms from start of poll, 651.735641ms from cluster start. Timeout at 2024-05-12 20:50:26.202973713 +0000 UTC m=+37.483674048, 1.007793197s after stability -2024-05-12T20:50:26.203Z [INFO] creating new snapshot: path=/tmp/raft3223538527/snapshots/2-14-1715547026203.tmp -2024-05-12T20:50:26.204Z [INFO] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: copied to local snapshot: bytes=71 -2024-05-12T20:50:26.205Z [INFO] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: snapshot restore progress: id=2-14-1715547026203 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:26.205Z [INFO] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: restored user snapshot: index=14 -2024-05-12T20:50:26.205Z [ERROR] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: failed to get log: index=3 error="log not found" -2024-05-12T20:50:26.205Z [INFO] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: aborting pipeline replication: peer="{Voter server-f4416791-f9f1-d9fc-ac32-16972dddc00f f4416791-f9f1-d9fc-ac32-16972dddc00f}" -2024-05-12T20:50:26.205Z [ERROR] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: failed to get log: index=3 error="log not found" -2024-05-12T20:50:26.205Z [INFO] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: aborting pipeline replication: peer="{Voter server-2dffa43d-6266-bce9-bf01-bd2151b25218 2dffa43d-6266-bce9-bf01-bd2151b25218}" -2024-05-12T20:50:26.215Z [ERROR] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: failed to get log: index=3 error="log not found" -2024-05-12T20:50:26.215Z [ERROR] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: failed to get log: index=3 error="log not found" -2024-05-12T20:50:26.215Z [INFO] creating new snapshot: path=/tmp/raft2007533637/snapshots/2-14-1715547026215.tmp -2024-05-12T20:50:26.215Z [INFO] creating new snapshot: path=/tmp/raft1877303258/snapshots/2-14-1715547026215.tmp -2024-05-12T20:50:26.215Z [INFO] server-2dffa43d-6266-bce9-bf01-bd2151b25218: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:26.215Z [INFO] server-f4416791-f9f1-d9fc-ac32-16972dddc00f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:26.216Z [INFO] server-f4416791-f9f1-d9fc-ac32-16972dddc00f: copied to local snapshot: bytes=71 -2024-05-12T20:50:26.216Z [INFO] server-f4416791-f9f1-d9fc-ac32-16972dddc00f: snapshot restore progress: id=2-14-1715547026215 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:26.216Z [INFO] server-2dffa43d-6266-bce9-bf01-bd2151b25218: copied to local snapshot: bytes=71 -2024-05-12T20:50:26.216Z [INFO] server-f4416791-f9f1-d9fc-ac32-16972dddc00f: Installed remote snapshot -2024-05-12T20:50:26.217Z [INFO] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: pipelining replication: peer="{Voter server-f4416791-f9f1-d9fc-ac32-16972dddc00f f4416791-f9f1-d9fc-ac32-16972dddc00f}" -2024-05-12T20:50:26.217Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:26.217Z [INFO] server-2dffa43d-6266-bce9-bf01-bd2151b25218: snapshot restore progress: id=2-14-1715547026215 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:26.217Z [INFO] server-2dffa43d-6266-bce9-bf01-bd2151b25218: Installed remote snapshot -2024-05-12T20:50:26.217Z [INFO] server-eeb50158-8454-b073-44ba-ef0dd60e45bc: pipelining replication: peer="{Voter server-2dffa43d-6266-bce9-bf01-bd2151b25218 2dffa43d-6266-bce9-bf01-bd2151b25218}" -2024-05-12T20:50:27.226Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:26.217137269 +0000 UTC m=+37.497837630 (2 nodes), 1.782µs from start of poll, 1.67369242s from cluster start. Timeout at 2024-05-12 20:50:27.226798914 +0000 UTC m=+38.507499235, 1.009661605s after stability -2024-05-12T20:50:27.241Z [INFO] server-0dbd5ac7-f6f4-6181-78dd-b825637e93c0: aborting pipeline replication: peer="{Voter server-d981699b-d972-8a2e-6e2d-ab815720803a d981699b-d972-8a2e-6e2d-ab815720803a}" -2024-05-12T20:50:27.241Z [INFO] server-0dbd5ac7-f6f4-6181-78dd-b825637e93c0: aborting pipeline replication: peer="{Voter server-088fbba9-6f18-a0bf-43ad-18098f3eeb53 088fbba9-6f18-a0bf-43ad-18098f3eeb53}" +2025-06-15T03:16:57.941Z [INFO] server-825d4d20-c813-030a-8b44-92bed2390708: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-825d4d20-c813-030a-8b44-92bed2390708 Address:825d4d20-c813-030a-8b44-92bed2390708} {Suffrage:Voter ID:server-bd7adc8e-3dfe-1f9a-72b0-fca87704e1f3 Address:bd7adc8e-3dfe-1f9a-72b0-fca87704e1f3} {Suffrage:Voter ID:server-0e4fddcb-b994-5727-3dd3-5ab876ba19fc Address:0e4fddcb-b994-5727-3dd3-5ab876ba19fc}]" +2025-06-15T03:16:57.941Z [INFO] server-825d4d20-c813-030a-8b44-92bed2390708: entering follower state: follower="Node at 825d4d20-c813-030a-8b44-92bed2390708 [Follower]" leader-address= leader-id= +2025-06-15T03:16:57.941Z [INFO] server-bd7adc8e-3dfe-1f9a-72b0-fca87704e1f3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-825d4d20-c813-030a-8b44-92bed2390708 Address:825d4d20-c813-030a-8b44-92bed2390708} {Suffrage:Voter ID:server-bd7adc8e-3dfe-1f9a-72b0-fca87704e1f3 Address:bd7adc8e-3dfe-1f9a-72b0-fca87704e1f3} {Suffrage:Voter ID:server-0e4fddcb-b994-5727-3dd3-5ab876ba19fc Address:0e4fddcb-b994-5727-3dd3-5ab876ba19fc}]" +2025-06-15T03:16:57.941Z [INFO] server-bd7adc8e-3dfe-1f9a-72b0-fca87704e1f3: entering follower state: follower="Node at bd7adc8e-3dfe-1f9a-72b0-fca87704e1f3 [Follower]" leader-address= leader-id= +2025-06-15T03:16:57.941Z [INFO] server-0e4fddcb-b994-5727-3dd3-5ab876ba19fc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-825d4d20-c813-030a-8b44-92bed2390708 Address:825d4d20-c813-030a-8b44-92bed2390708} {Suffrage:Voter ID:server-bd7adc8e-3dfe-1f9a-72b0-fca87704e1f3 Address:bd7adc8e-3dfe-1f9a-72b0-fca87704e1f3} {Suffrage:Voter ID:server-0e4fddcb-b994-5727-3dd3-5ab876ba19fc Address:0e4fddcb-b994-5727-3dd3-5ab876ba19fc}]" +2025-06-15T03:16:57.941Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:57.941Z [INFO] server-0e4fddcb-b994-5727-3dd3-5ab876ba19fc: entering follower state: follower="Node at 0e4fddcb-b994-5727-3dd3-5ab876ba19fc [Follower]" leader-address= leader-id= +2025-06-15T03:16:58.479Z [WARN] server-825d4d20-c813-030a-8b44-92bed2390708: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:16:58.479Z [INFO] server-825d4d20-c813-030a-8b44-92bed2390708: entering candidate state: node="Node at 825d4d20-c813-030a-8b44-92bed2390708 [Candidate]" term=2 +2025-06-15T03:16:58.479Z [INFO] server-825d4d20-c813-030a-8b44-92bed2390708: election won: term=2 tally=2 +2025-06-15T03:16:58.479Z [INFO] server-825d4d20-c813-030a-8b44-92bed2390708: entering leader state: leader="Node at 825d4d20-c813-030a-8b44-92bed2390708 [Leader]" +2025-06-15T03:16:58.479Z [INFO] server-825d4d20-c813-030a-8b44-92bed2390708: added peer, starting replication: peer=server-bd7adc8e-3dfe-1f9a-72b0-fca87704e1f3 +2025-06-15T03:16:58.479Z [INFO] server-825d4d20-c813-030a-8b44-92bed2390708: added peer, starting replication: peer=server-0e4fddcb-b994-5727-3dd3-5ab876ba19fc +2025-06-15T03:16:58.479Z [INFO] server-825d4d20-c813-030a-8b44-92bed2390708: pipelining replication: peer="{Voter server-0e4fddcb-b994-5727-3dd3-5ab876ba19fc 0e4fddcb-b994-5727-3dd3-5ab876ba19fc}" +2025-06-15T03:16:58.479Z [INFO] server-825d4d20-c813-030a-8b44-92bed2390708: pipelining replication: peer="{Voter server-bd7adc8e-3dfe-1f9a-72b0-fca87704e1f3 bd7adc8e-3dfe-1f9a-72b0-fca87704e1f3}" +2025-06-15T03:16:59.485Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:16:58.479895622 +0000 UTC m=+34.191573822 (1 nodes), 538.342784ms from start of poll, 538.617718ms from cluster start. Timeout at 2025-06-15 03:16:59.48567876 +0000 UTC m=+35.197356927, 1.005783105s after stability +2025-06-15T03:16:59.486Z [INFO] server-825d4d20-c813-030a-8b44-92bed2390708: starting snapshot up to: index=12 +2025-06-15T03:16:59.486Z [INFO] creating new snapshot: path=/tmp/raft1782766031/snapshots/2-12-1749957419486.tmp +2025-06-15T03:16:59.486Z [INFO] server-825d4d20-c813-030a-8b44-92bed2390708: snapshot complete up to: index=12 +2025-06-15T03:16:59.487Z [INFO] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8e302a7f-06b9-ddfa-502b-bea237b1391b Address:8e302a7f-06b9-ddfa-502b-bea237b1391b} {Suffrage:Voter ID:server-0529f177-c6b4-21f9-cf67-616c2818affc Address:0529f177-c6b4-21f9-cf67-616c2818affc} {Suffrage:Voter ID:server-5c311877-45e2-917b-1962-8755227497e7 Address:5c311877-45e2-917b-1962-8755227497e7}]" +2025-06-15T03:16:59.487Z [INFO] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: entering follower state: follower="Node at 8e302a7f-06b9-ddfa-502b-bea237b1391b [Follower]" leader-address= leader-id= +2025-06-15T03:16:59.487Z [INFO] server-0529f177-c6b4-21f9-cf67-616c2818affc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8e302a7f-06b9-ddfa-502b-bea237b1391b Address:8e302a7f-06b9-ddfa-502b-bea237b1391b} {Suffrage:Voter ID:server-0529f177-c6b4-21f9-cf67-616c2818affc Address:0529f177-c6b4-21f9-cf67-616c2818affc} {Suffrage:Voter ID:server-5c311877-45e2-917b-1962-8755227497e7 Address:5c311877-45e2-917b-1962-8755227497e7}]" +2025-06-15T03:16:59.487Z [INFO] server-0529f177-c6b4-21f9-cf67-616c2818affc: entering follower state: follower="Node at 0529f177-c6b4-21f9-cf67-616c2818affc [Follower]" leader-address= leader-id= +2025-06-15T03:16:59.487Z [INFO] server-5c311877-45e2-917b-1962-8755227497e7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8e302a7f-06b9-ddfa-502b-bea237b1391b Address:8e302a7f-06b9-ddfa-502b-bea237b1391b} {Suffrage:Voter ID:server-0529f177-c6b4-21f9-cf67-616c2818affc Address:0529f177-c6b4-21f9-cf67-616c2818affc} {Suffrage:Voter ID:server-5c311877-45e2-917b-1962-8755227497e7 Address:5c311877-45e2-917b-1962-8755227497e7}]" +2025-06-15T03:16:59.487Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:16:59.487Z [INFO] server-5c311877-45e2-917b-1962-8755227497e7: entering follower state: follower="Node at 5c311877-45e2-917b-1962-8755227497e7 [Follower]" leader-address= leader-id= +2025-06-15T03:17:00.111Z [WARN] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:00.111Z [INFO] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: entering candidate state: node="Node at 8e302a7f-06b9-ddfa-502b-bea237b1391b [Candidate]" term=2 +2025-06-15T03:17:00.111Z [INFO] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: election won: term=2 tally=2 +2025-06-15T03:17:00.111Z [INFO] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: entering leader state: leader="Node at 8e302a7f-06b9-ddfa-502b-bea237b1391b [Leader]" +2025-06-15T03:17:00.111Z [INFO] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: added peer, starting replication: peer=server-0529f177-c6b4-21f9-cf67-616c2818affc +2025-06-15T03:17:00.111Z [INFO] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: added peer, starting replication: peer=server-5c311877-45e2-917b-1962-8755227497e7 +2025-06-15T03:17:00.111Z [INFO] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: pipelining replication: peer="{Voter server-5c311877-45e2-917b-1962-8755227497e7 5c311877-45e2-917b-1962-8755227497e7}" +2025-06-15T03:17:00.111Z [INFO] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: pipelining replication: peer="{Voter server-0529f177-c6b4-21f9-cf67-616c2818affc 0529f177-c6b4-21f9-cf67-616c2818affc}" +2025-06-15T03:17:01.116Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:00.111635052 +0000 UTC m=+35.823313234 (1 nodes), 623.868943ms from start of poll, 624.252468ms from cluster start. Timeout at 2025-06-15 03:17:01.1168817 +0000 UTC m=+36.828559865, 1.005246631s after stability +2025-06-15T03:17:01.117Z [INFO] creating new snapshot: path=/tmp/raft645859301/snapshots/2-14-1749957421117.tmp +2025-06-15T03:17:01.118Z [INFO] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: copied to local snapshot: bytes=71 +2025-06-15T03:17:01.118Z [INFO] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: snapshot restore progress: id=2-14-1749957421117 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:01.118Z [INFO] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: restored user snapshot: index=14 +2025-06-15T03:17:01.118Z [ERROR] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: failed to get log: index=3 error="log not found" +2025-06-15T03:17:01.118Z [INFO] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: aborting pipeline replication: peer="{Voter server-5c311877-45e2-917b-1962-8755227497e7 5c311877-45e2-917b-1962-8755227497e7}" +2025-06-15T03:17:01.118Z [ERROR] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: failed to get log: index=3 error="log not found" +2025-06-15T03:17:01.118Z [INFO] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: aborting pipeline replication: peer="{Voter server-0529f177-c6b4-21f9-cf67-616c2818affc 0529f177-c6b4-21f9-cf67-616c2818affc}" +2025-06-15T03:17:01.124Z [ERROR] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: failed to get log: index=3 error="log not found" +2025-06-15T03:17:01.124Z [INFO] creating new snapshot: path=/tmp/raft2439034523/snapshots/2-14-1749957421124.tmp +2025-06-15T03:17:01.124Z [INFO] server-5c311877-45e2-917b-1962-8755227497e7: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:01.125Z [INFO] server-5c311877-45e2-917b-1962-8755227497e7: copied to local snapshot: bytes=71 +2025-06-15T03:17:01.125Z [INFO] server-5c311877-45e2-917b-1962-8755227497e7: snapshot restore progress: id=2-14-1749957421124 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:01.125Z [INFO] server-5c311877-45e2-917b-1962-8755227497e7: Installed remote snapshot +2025-06-15T03:17:01.125Z [INFO] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: pipelining replication: peer="{Voter server-5c311877-45e2-917b-1962-8755227497e7 5c311877-45e2-917b-1962-8755227497e7}" +2025-06-15T03:17:01.125Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:17:01.127Z [ERROR] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: failed to get log: index=3 error="log not found" +2025-06-15T03:17:01.128Z [INFO] creating new snapshot: path=/tmp/raft644690575/snapshots/2-14-1749957421128.tmp +2025-06-15T03:17:01.128Z [INFO] server-0529f177-c6b4-21f9-cf67-616c2818affc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:01.128Z [INFO] server-0529f177-c6b4-21f9-cf67-616c2818affc: copied to local snapshot: bytes=71 +2025-06-15T03:17:01.128Z [INFO] server-0529f177-c6b4-21f9-cf67-616c2818affc: snapshot restore progress: id=2-14-1749957421128 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:01.128Z [INFO] server-0529f177-c6b4-21f9-cf67-616c2818affc: Installed remote snapshot +2025-06-15T03:17:01.128Z [INFO] server-8e302a7f-06b9-ddfa-502b-bea237b1391b: pipelining replication: peer="{Voter server-0529f177-c6b4-21f9-cf67-616c2818affc 0529f177-c6b4-21f9-cf67-616c2818affc}" +2025-06-15T03:17:02.131Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:17:01.125421609 +0000 UTC m=+36.837099773 (2 nodes), 732ns from start of poll, 1.638039007s from cluster start. Timeout at 2025-06-15 03:17:02.131316305 +0000 UTC m=+37.842994471, 1.005894698s after stability +2025-06-15T03:17:02.144Z [INFO] server-825d4d20-c813-030a-8b44-92bed2390708: aborting pipeline replication: peer="{Voter server-0e4fddcb-b994-5727-3dd3-5ab876ba19fc 0e4fddcb-b994-5727-3dd3-5ab876ba19fc}" +2025-06-15T03:17:02.144Z [INFO] server-825d4d20-c813-030a-8b44-92bed2390708: aborting pipeline replication: peer="{Voter server-bd7adc8e-3dfe-1f9a-72b0-fca87704e1f3 bd7adc8e-3dfe-1f9a-72b0-fca87704e1f3}" === RUN TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true -2024-05-12T20:50:27.244Z [INFO] server-53a9fff0-fcbf-1d3e-815b-367c0747460d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53a9fff0-fcbf-1d3e-815b-367c0747460d Address:53a9fff0-fcbf-1d3e-815b-367c0747460d} {Suffrage:Voter ID:server-f9365861-37be-9285-e721-55f7a7d17e9d Address:f9365861-37be-9285-e721-55f7a7d17e9d} {Suffrage:Voter ID:server-803e27f4-ca2d-28de-cf1f-7090c5b9bed1 Address:803e27f4-ca2d-28de-cf1f-7090c5b9bed1}]" -2024-05-12T20:50:27.244Z [INFO] server-f9365861-37be-9285-e721-55f7a7d17e9d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53a9fff0-fcbf-1d3e-815b-367c0747460d Address:53a9fff0-fcbf-1d3e-815b-367c0747460d} {Suffrage:Voter ID:server-f9365861-37be-9285-e721-55f7a7d17e9d Address:f9365861-37be-9285-e721-55f7a7d17e9d} {Suffrage:Voter ID:server-803e27f4-ca2d-28de-cf1f-7090c5b9bed1 Address:803e27f4-ca2d-28de-cf1f-7090c5b9bed1}]" -2024-05-12T20:50:27.244Z [INFO] server-803e27f4-ca2d-28de-cf1f-7090c5b9bed1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53a9fff0-fcbf-1d3e-815b-367c0747460d Address:53a9fff0-fcbf-1d3e-815b-367c0747460d} {Suffrage:Voter ID:server-f9365861-37be-9285-e721-55f7a7d17e9d Address:f9365861-37be-9285-e721-55f7a7d17e9d} {Suffrage:Voter ID:server-803e27f4-ca2d-28de-cf1f-7090c5b9bed1 Address:803e27f4-ca2d-28de-cf1f-7090c5b9bed1}]" -2024-05-12T20:50:27.244Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:27.244Z [INFO] server-53a9fff0-fcbf-1d3e-815b-367c0747460d: entering follower state: follower="Node at 53a9fff0-fcbf-1d3e-815b-367c0747460d [Follower]" leader-address= leader-id= -2024-05-12T20:50:27.244Z [INFO] server-f9365861-37be-9285-e721-55f7a7d17e9d: entering follower state: follower="Node at f9365861-37be-9285-e721-55f7a7d17e9d [Follower]" leader-address= leader-id= -2024-05-12T20:50:27.244Z [INFO] server-803e27f4-ca2d-28de-cf1f-7090c5b9bed1: entering follower state: follower="Node at 803e27f4-ca2d-28de-cf1f-7090c5b9bed1 [Follower]" leader-address= leader-id= -2024-05-12T20:50:27.867Z [WARN] server-f9365861-37be-9285-e721-55f7a7d17e9d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:27.867Z [INFO] server-f9365861-37be-9285-e721-55f7a7d17e9d: entering candidate state: node="Node at f9365861-37be-9285-e721-55f7a7d17e9d [Candidate]" term=2 -2024-05-12T20:50:27.867Z [INFO] server-f9365861-37be-9285-e721-55f7a7d17e9d: election won: term=2 tally=2 -2024-05-12T20:50:27.867Z [INFO] server-f9365861-37be-9285-e721-55f7a7d17e9d: entering leader state: leader="Node at f9365861-37be-9285-e721-55f7a7d17e9d [Leader]" -2024-05-12T20:50:27.867Z [INFO] server-f9365861-37be-9285-e721-55f7a7d17e9d: added peer, starting replication: peer=server-53a9fff0-fcbf-1d3e-815b-367c0747460d -2024-05-12T20:50:27.867Z [INFO] server-f9365861-37be-9285-e721-55f7a7d17e9d: added peer, starting replication: peer=server-803e27f4-ca2d-28de-cf1f-7090c5b9bed1 -2024-05-12T20:50:27.867Z [INFO] server-f9365861-37be-9285-e721-55f7a7d17e9d: pipelining replication: peer="{Voter server-803e27f4-ca2d-28de-cf1f-7090c5b9bed1 803e27f4-ca2d-28de-cf1f-7090c5b9bed1}" -2024-05-12T20:50:27.867Z [INFO] server-f9365861-37be-9285-e721-55f7a7d17e9d: pipelining replication: peer="{Voter server-53a9fff0-fcbf-1d3e-815b-367c0747460d 53a9fff0-fcbf-1d3e-815b-367c0747460d}" -2024-05-12T20:50:28.878Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:27.867680572 +0000 UTC m=+39.148380877 (1 nodes), 623.349051ms from start of poll, 623.76049ms from cluster start. Timeout at 2024-05-12 20:50:28.878380675 +0000 UTC m=+40.159081139, 1.010700262s after stability -2024-05-12T20:50:28.880Z [INFO] server-f9365861-37be-9285-e721-55f7a7d17e9d: starting snapshot up to: index=12 -2024-05-12T20:50:28.880Z [INFO] creating new snapshot: path=/tmp/raft395433448/snapshots/2-12-1715547028880.tmp -2024-05-12T20:50:28.880Z [INFO] server-f9365861-37be-9285-e721-55f7a7d17e9d: snapshot complete up to: index=12 -2024-05-12T20:50:28.884Z [INFO] server-1b92cdd6-1f56-1807-2901-4b206b1fd6a3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b92cdd6-1f56-1807-2901-4b206b1fd6a3 Address:1b92cdd6-1f56-1807-2901-4b206b1fd6a3} {Suffrage:Voter ID:server-f1117adc-4ee8-0a02-4a2c-76c31b67fd0f Address:f1117adc-4ee8-0a02-4a2c-76c31b67fd0f} {Suffrage:Voter ID:server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec Address:fef24803-c69c-da45-7ded-bc8b5fd3b1ec}]" -2024-05-12T20:50:28.884Z [INFO] server-f1117adc-4ee8-0a02-4a2c-76c31b67fd0f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b92cdd6-1f56-1807-2901-4b206b1fd6a3 Address:1b92cdd6-1f56-1807-2901-4b206b1fd6a3} {Suffrage:Voter ID:server-f1117adc-4ee8-0a02-4a2c-76c31b67fd0f Address:f1117adc-4ee8-0a02-4a2c-76c31b67fd0f} {Suffrage:Voter ID:server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec Address:fef24803-c69c-da45-7ded-bc8b5fd3b1ec}]" -2024-05-12T20:50:28.884Z [INFO] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b92cdd6-1f56-1807-2901-4b206b1fd6a3 Address:1b92cdd6-1f56-1807-2901-4b206b1fd6a3} {Suffrage:Voter ID:server-f1117adc-4ee8-0a02-4a2c-76c31b67fd0f Address:f1117adc-4ee8-0a02-4a2c-76c31b67fd0f} {Suffrage:Voter ID:server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec Address:fef24803-c69c-da45-7ded-bc8b5fd3b1ec}]" -2024-05-12T20:50:28.884Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:28.884Z [INFO] server-1b92cdd6-1f56-1807-2901-4b206b1fd6a3: entering follower state: follower="Node at 1b92cdd6-1f56-1807-2901-4b206b1fd6a3 [Follower]" leader-address= leader-id= -2024-05-12T20:50:28.884Z [INFO] server-f1117adc-4ee8-0a02-4a2c-76c31b67fd0f: entering follower state: follower="Node at f1117adc-4ee8-0a02-4a2c-76c31b67fd0f [Follower]" leader-address= leader-id= -2024-05-12T20:50:28.884Z [INFO] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: entering follower state: follower="Node at fef24803-c69c-da45-7ded-bc8b5fd3b1ec [Follower]" leader-address= leader-id= -2024-05-12T20:50:29.466Z [WARN] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:29.466Z [INFO] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: entering candidate state: node="Node at fef24803-c69c-da45-7ded-bc8b5fd3b1ec [Candidate]" term=2 -2024-05-12T20:50:29.467Z [INFO] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: election won: term=2 tally=2 -2024-05-12T20:50:29.467Z [INFO] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: entering leader state: leader="Node at fef24803-c69c-da45-7ded-bc8b5fd3b1ec [Leader]" -2024-05-12T20:50:29.467Z [INFO] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: added peer, starting replication: peer=server-1b92cdd6-1f56-1807-2901-4b206b1fd6a3 -2024-05-12T20:50:29.467Z [INFO] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: added peer, starting replication: peer=server-f1117adc-4ee8-0a02-4a2c-76c31b67fd0f -2024-05-12T20:50:29.467Z [INFO] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: pipelining replication: peer="{Voter server-f1117adc-4ee8-0a02-4a2c-76c31b67fd0f f1117adc-4ee8-0a02-4a2c-76c31b67fd0f}" -2024-05-12T20:50:29.467Z [WARN] server-1b92cdd6-1f56-1807-2901-4b206b1fd6a3: rejecting vote request since our last term is greater: candidate=fef24803-c69c-da45-7ded-bc8b5fd3b1ec last-term=2 last-candidate-term=1 -2024-05-12T20:50:29.467Z [INFO] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: pipelining replication: peer="{Voter server-1b92cdd6-1f56-1807-2901-4b206b1fd6a3 1b92cdd6-1f56-1807-2901-4b206b1fd6a3}" -2024-05-12T20:50:30.475Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:29.467409391 +0000 UTC m=+40.748109817 (1 nodes), 583.022013ms from start of poll, 583.493246ms from cluster start. Timeout at 2024-05-12 20:50:30.475674904 +0000 UTC m=+41.756375227, 1.00826541s after stability -2024-05-12T20:50:30.476Z [INFO] creating new snapshot: path=/tmp/raft3382854521/snapshots/2-14-1715547030476.tmp -2024-05-12T20:50:30.477Z [INFO] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: copied to local snapshot: bytes=71 -2024-05-12T20:50:30.477Z [INFO] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: snapshot restore progress: id=2-14-1715547030476 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:30.477Z [INFO] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: removing all old logs from log store -2024-05-12T20:50:30.477Z [INFO] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: compacting logs: from=1 to=2 -2024-05-12T20:50:30.477Z [INFO] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: restored user snapshot: index=14 -2024-05-12T20:50:30.477Z [ERROR] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: failed to get log: index=2 error="log not found" -2024-05-12T20:50:30.477Z [INFO] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: aborting pipeline replication: peer="{Voter server-f1117adc-4ee8-0a02-4a2c-76c31b67fd0f f1117adc-4ee8-0a02-4a2c-76c31b67fd0f}" -2024-05-12T20:50:30.478Z [ERROR] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: failed to get log: index=2 error="log not found" -2024-05-12T20:50:30.478Z [INFO] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: aborting pipeline replication: peer="{Voter server-1b92cdd6-1f56-1807-2901-4b206b1fd6a3 1b92cdd6-1f56-1807-2901-4b206b1fd6a3}" -2024-05-12T20:50:30.487Z [ERROR] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: failed to get log: index=2 error="log not found" -2024-05-12T20:50:30.487Z [INFO] creating new snapshot: path=/tmp/raft4255985661/snapshots/2-14-1715547030487.tmp -2024-05-12T20:50:30.488Z [INFO] server-1b92cdd6-1f56-1807-2901-4b206b1fd6a3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:30.490Z [INFO] server-1b92cdd6-1f56-1807-2901-4b206b1fd6a3: copied to local snapshot: bytes=71 -2024-05-12T20:50:30.495Z [ERROR] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: failed to get log: index=2 error="log not found" -2024-05-12T20:50:30.506Z [INFO] creating new snapshot: path=/tmp/raft2845346855/snapshots/2-14-1715547030506.tmp -2024-05-12T20:50:30.507Z [INFO] server-f1117adc-4ee8-0a02-4a2c-76c31b67fd0f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:30.507Z [INFO] server-f1117adc-4ee8-0a02-4a2c-76c31b67fd0f: copied to local snapshot: bytes=71 -2024-05-12T20:50:30.508Z [INFO] server-f1117adc-4ee8-0a02-4a2c-76c31b67fd0f: snapshot restore progress: id=2-14-1715547030506 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:30.508Z [INFO] server-f1117adc-4ee8-0a02-4a2c-76c31b67fd0f: removing all old logs from log store -2024-05-12T20:50:30.508Z [INFO] server-f1117adc-4ee8-0a02-4a2c-76c31b67fd0f: compacting logs: from=1 to=2 -2024-05-12T20:50:30.508Z [INFO] server-f1117adc-4ee8-0a02-4a2c-76c31b67fd0f: Installed remote snapshot -2024-05-12T20:50:30.508Z [INFO] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: pipelining replication: peer="{Voter server-f1117adc-4ee8-0a02-4a2c-76c31b67fd0f f1117adc-4ee8-0a02-4a2c-76c31b67fd0f}" -2024-05-12T20:50:30.508Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:30.508Z [INFO] server-1b92cdd6-1f56-1807-2901-4b206b1fd6a3: snapshot restore progress: id=2-14-1715547030487 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:30.508Z [INFO] server-1b92cdd6-1f56-1807-2901-4b206b1fd6a3: removing all old logs from log store -2024-05-12T20:50:30.508Z [INFO] server-1b92cdd6-1f56-1807-2901-4b206b1fd6a3: compacting logs: from=1 to=2 -2024-05-12T20:50:30.508Z [INFO] server-1b92cdd6-1f56-1807-2901-4b206b1fd6a3: Installed remote snapshot -2024-05-12T20:50:30.509Z [INFO] server-fef24803-c69c-da45-7ded-bc8b5fd3b1ec: pipelining replication: peer="{Voter server-1b92cdd6-1f56-1807-2901-4b206b1fd6a3 1b92cdd6-1f56-1807-2901-4b206b1fd6a3}" -2024-05-12T20:50:31.515Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:30.508537378 +0000 UTC m=+41.789237777 (2 nodes), 2.001µs from start of poll, 1.624621206s from cluster start. Timeout at 2024-05-12 20:50:31.515297856 +0000 UTC m=+42.795998322, 1.006760545s after stability -2024-05-12T20:50:31.529Z [INFO] server-f9365861-37be-9285-e721-55f7a7d17e9d: aborting pipeline replication: peer="{Voter server-803e27f4-ca2d-28de-cf1f-7090c5b9bed1 803e27f4-ca2d-28de-cf1f-7090c5b9bed1}" -2024-05-12T20:50:31.530Z [INFO] server-f9365861-37be-9285-e721-55f7a7d17e9d: aborting pipeline replication: peer="{Voter server-53a9fff0-fcbf-1d3e-815b-367c0747460d 53a9fff0-fcbf-1d3e-815b-367c0747460d}" +2025-06-15T03:17:02.145Z [INFO] server-cc538058-a8d6-1eea-71ff-5fab76914291: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cc538058-a8d6-1eea-71ff-5fab76914291 Address:cc538058-a8d6-1eea-71ff-5fab76914291} {Suffrage:Voter ID:server-7243c961-468a-ab56-a4e1-3ff83ee08836 Address:7243c961-468a-ab56-a4e1-3ff83ee08836} {Suffrage:Voter ID:server-47efea5c-541f-ad8f-a542-0f2b57b7695c Address:47efea5c-541f-ad8f-a542-0f2b57b7695c}]" +2025-06-15T03:17:02.145Z [INFO] server-7243c961-468a-ab56-a4e1-3ff83ee08836: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cc538058-a8d6-1eea-71ff-5fab76914291 Address:cc538058-a8d6-1eea-71ff-5fab76914291} {Suffrage:Voter ID:server-7243c961-468a-ab56-a4e1-3ff83ee08836 Address:7243c961-468a-ab56-a4e1-3ff83ee08836} {Suffrage:Voter ID:server-47efea5c-541f-ad8f-a542-0f2b57b7695c Address:47efea5c-541f-ad8f-a542-0f2b57b7695c}]" +2025-06-15T03:17:02.145Z [INFO] server-cc538058-a8d6-1eea-71ff-5fab76914291: entering follower state: follower="Node at cc538058-a8d6-1eea-71ff-5fab76914291 [Follower]" leader-address= leader-id= +2025-06-15T03:17:02.145Z [INFO] server-47efea5c-541f-ad8f-a542-0f2b57b7695c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cc538058-a8d6-1eea-71ff-5fab76914291 Address:cc538058-a8d6-1eea-71ff-5fab76914291} {Suffrage:Voter ID:server-7243c961-468a-ab56-a4e1-3ff83ee08836 Address:7243c961-468a-ab56-a4e1-3ff83ee08836} {Suffrage:Voter ID:server-47efea5c-541f-ad8f-a542-0f2b57b7695c Address:47efea5c-541f-ad8f-a542-0f2b57b7695c}]" +2025-06-15T03:17:02.145Z [INFO] server-7243c961-468a-ab56-a4e1-3ff83ee08836: entering follower state: follower="Node at 7243c961-468a-ab56-a4e1-3ff83ee08836 [Follower]" leader-address= leader-id= +2025-06-15T03:17:02.145Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:02.145Z [INFO] server-47efea5c-541f-ad8f-a542-0f2b57b7695c: entering follower state: follower="Node at 47efea5c-541f-ad8f-a542-0f2b57b7695c [Follower]" leader-address= leader-id= +2025-06-15T03:17:02.658Z [WARN] server-7243c961-468a-ab56-a4e1-3ff83ee08836: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:02.658Z [INFO] server-7243c961-468a-ab56-a4e1-3ff83ee08836: entering candidate state: node="Node at 7243c961-468a-ab56-a4e1-3ff83ee08836 [Candidate]" term=2 +2025-06-15T03:17:02.658Z [INFO] server-7243c961-468a-ab56-a4e1-3ff83ee08836: election won: term=2 tally=2 +2025-06-15T03:17:02.658Z [INFO] server-7243c961-468a-ab56-a4e1-3ff83ee08836: entering leader state: leader="Node at 7243c961-468a-ab56-a4e1-3ff83ee08836 [Leader]" +2025-06-15T03:17:02.659Z [INFO] server-7243c961-468a-ab56-a4e1-3ff83ee08836: added peer, starting replication: peer=server-cc538058-a8d6-1eea-71ff-5fab76914291 +2025-06-15T03:17:02.659Z [INFO] server-7243c961-468a-ab56-a4e1-3ff83ee08836: added peer, starting replication: peer=server-47efea5c-541f-ad8f-a542-0f2b57b7695c +2025-06-15T03:17:02.659Z [INFO] server-7243c961-468a-ab56-a4e1-3ff83ee08836: pipelining replication: peer="{Voter server-cc538058-a8d6-1eea-71ff-5fab76914291 cc538058-a8d6-1eea-71ff-5fab76914291}" +2025-06-15T03:17:02.659Z [INFO] server-7243c961-468a-ab56-a4e1-3ff83ee08836: pipelining replication: peer="{Voter server-47efea5c-541f-ad8f-a542-0f2b57b7695c 47efea5c-541f-ad8f-a542-0f2b57b7695c}" +2025-06-15T03:17:03.664Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:02.659127905 +0000 UTC m=+38.370806086 (1 nodes), 513.621126ms from start of poll, 513.94287ms from cluster start. Timeout at 2025-06-15 03:17:03.664932652 +0000 UTC m=+39.376610817, 1.005804731s after stability +2025-06-15T03:17:03.665Z [INFO] server-7243c961-468a-ab56-a4e1-3ff83ee08836: starting snapshot up to: index=12 +2025-06-15T03:17:03.665Z [INFO] creating new snapshot: path=/tmp/raft439592351/snapshots/2-12-1749957423665.tmp +2025-06-15T03:17:03.665Z [INFO] server-7243c961-468a-ab56-a4e1-3ff83ee08836: snapshot complete up to: index=12 +2025-06-15T03:17:03.667Z [INFO] server-5ba58fae-e75a-458d-c20f-7236a56f7d49: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5ba58fae-e75a-458d-c20f-7236a56f7d49 Address:5ba58fae-e75a-458d-c20f-7236a56f7d49} {Suffrage:Voter ID:server-fda1265c-3235-5252-fd0b-883522a21aab Address:fda1265c-3235-5252-fd0b-883522a21aab} {Suffrage:Voter ID:server-8b1a9662-2537-6b1f-49ab-d8d1472363c6 Address:8b1a9662-2537-6b1f-49ab-d8d1472363c6}]" +2025-06-15T03:17:03.667Z [INFO] server-5ba58fae-e75a-458d-c20f-7236a56f7d49: entering follower state: follower="Node at 5ba58fae-e75a-458d-c20f-7236a56f7d49 [Follower]" leader-address= leader-id= +2025-06-15T03:17:03.667Z [INFO] server-fda1265c-3235-5252-fd0b-883522a21aab: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5ba58fae-e75a-458d-c20f-7236a56f7d49 Address:5ba58fae-e75a-458d-c20f-7236a56f7d49} {Suffrage:Voter ID:server-fda1265c-3235-5252-fd0b-883522a21aab Address:fda1265c-3235-5252-fd0b-883522a21aab} {Suffrage:Voter ID:server-8b1a9662-2537-6b1f-49ab-d8d1472363c6 Address:8b1a9662-2537-6b1f-49ab-d8d1472363c6}]" +2025-06-15T03:17:03.667Z [INFO] server-fda1265c-3235-5252-fd0b-883522a21aab: entering follower state: follower="Node at fda1265c-3235-5252-fd0b-883522a21aab [Follower]" leader-address= leader-id= +2025-06-15T03:17:03.667Z [INFO] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5ba58fae-e75a-458d-c20f-7236a56f7d49 Address:5ba58fae-e75a-458d-c20f-7236a56f7d49} {Suffrage:Voter ID:server-fda1265c-3235-5252-fd0b-883522a21aab Address:fda1265c-3235-5252-fd0b-883522a21aab} {Suffrage:Voter ID:server-8b1a9662-2537-6b1f-49ab-d8d1472363c6 Address:8b1a9662-2537-6b1f-49ab-d8d1472363c6}]" +2025-06-15T03:17:03.667Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:03.667Z [INFO] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: entering follower state: follower="Node at 8b1a9662-2537-6b1f-49ab-d8d1472363c6 [Follower]" leader-address= leader-id= +2025-06-15T03:17:04.341Z [WARN] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:04.341Z [INFO] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: entering candidate state: node="Node at 8b1a9662-2537-6b1f-49ab-d8d1472363c6 [Candidate]" term=2 +2025-06-15T03:17:04.341Z [INFO] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: election won: term=2 tally=2 +2025-06-15T03:17:04.341Z [INFO] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: entering leader state: leader="Node at 8b1a9662-2537-6b1f-49ab-d8d1472363c6 [Leader]" +2025-06-15T03:17:04.341Z [INFO] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: added peer, starting replication: peer=server-5ba58fae-e75a-458d-c20f-7236a56f7d49 +2025-06-15T03:17:04.341Z [INFO] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: added peer, starting replication: peer=server-fda1265c-3235-5252-fd0b-883522a21aab +2025-06-15T03:17:04.341Z [INFO] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: pipelining replication: peer="{Voter server-fda1265c-3235-5252-fd0b-883522a21aab fda1265c-3235-5252-fd0b-883522a21aab}" +2025-06-15T03:17:04.341Z [INFO] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: pipelining replication: peer="{Voter server-5ba58fae-e75a-458d-c20f-7236a56f7d49 5ba58fae-e75a-458d-c20f-7236a56f7d49}" +2025-06-15T03:17:05.347Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:04.34161158 +0000 UTC m=+40.053289752 (1 nodes), 673.621236ms from start of poll, 673.993734ms from cluster start. Timeout at 2025-06-15 03:17:05.347475426 +0000 UTC m=+41.059153655, 1.005863903s after stability +2025-06-15T03:17:05.347Z [INFO] creating new snapshot: path=/tmp/raft1036268257/snapshots/2-14-1749957425347.tmp +2025-06-15T03:17:05.348Z [INFO] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: copied to local snapshot: bytes=71 +2025-06-15T03:17:05.348Z [INFO] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: snapshot restore progress: id=2-14-1749957425347 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:05.348Z [INFO] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: removing all old logs from log store +2025-06-15T03:17:05.348Z [INFO] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: compacting logs: from=1 to=2 +2025-06-15T03:17:05.348Z [INFO] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: restored user snapshot: index=14 +2025-06-15T03:17:05.348Z [ERROR] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: failed to get log: index=2 error="log not found" +2025-06-15T03:17:05.348Z [INFO] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: aborting pipeline replication: peer="{Voter server-fda1265c-3235-5252-fd0b-883522a21aab fda1265c-3235-5252-fd0b-883522a21aab}" +2025-06-15T03:17:05.348Z [ERROR] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: failed to get log: index=2 error="log not found" +2025-06-15T03:17:05.348Z [INFO] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: aborting pipeline replication: peer="{Voter server-5ba58fae-e75a-458d-c20f-7236a56f7d49 5ba58fae-e75a-458d-c20f-7236a56f7d49}" +2025-06-15T03:17:05.354Z [ERROR] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: failed to get log: index=2 error="log not found" +2025-06-15T03:17:05.354Z [INFO] creating new snapshot: path=/tmp/raft3435238669/snapshots/2-14-1749957425354.tmp +2025-06-15T03:17:05.354Z [INFO] server-5ba58fae-e75a-458d-c20f-7236a56f7d49: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:05.355Z [INFO] server-5ba58fae-e75a-458d-c20f-7236a56f7d49: copied to local snapshot: bytes=71 +2025-06-15T03:17:05.355Z [INFO] server-5ba58fae-e75a-458d-c20f-7236a56f7d49: snapshot restore progress: id=2-14-1749957425354 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:05.355Z [INFO] server-5ba58fae-e75a-458d-c20f-7236a56f7d49: removing all old logs from log store +2025-06-15T03:17:05.356Z [INFO] server-5ba58fae-e75a-458d-c20f-7236a56f7d49: compacting logs: from=1 to=2 +2025-06-15T03:17:05.356Z [INFO] server-5ba58fae-e75a-458d-c20f-7236a56f7d49: Installed remote snapshot +2025-06-15T03:17:05.356Z [WARN] server-5ba58fae-e75a-458d-c20f-7236a56f7d49: skipping application of old log: index=14 +2025-06-15T03:17:05.356Z [INFO] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: pipelining replication: peer="{Voter server-5ba58fae-e75a-458d-c20f-7236a56f7d49 5ba58fae-e75a-458d-c20f-7236a56f7d49}" +2025-06-15T03:17:05.356Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:17:05.357Z [ERROR] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: failed to get log: index=2 error="log not found" +2025-06-15T03:17:05.357Z [INFO] creating new snapshot: path=/tmp/raft3077035758/snapshots/2-14-1749957425357.tmp +2025-06-15T03:17:05.357Z [INFO] server-fda1265c-3235-5252-fd0b-883522a21aab: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:05.358Z [INFO] server-fda1265c-3235-5252-fd0b-883522a21aab: copied to local snapshot: bytes=71 +2025-06-15T03:17:05.358Z [INFO] server-fda1265c-3235-5252-fd0b-883522a21aab: snapshot restore progress: id=2-14-1749957425357 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:05.358Z [INFO] server-fda1265c-3235-5252-fd0b-883522a21aab: removing all old logs from log store +2025-06-15T03:17:05.358Z [INFO] server-fda1265c-3235-5252-fd0b-883522a21aab: compacting logs: from=1 to=2 +2025-06-15T03:17:05.358Z [INFO] server-fda1265c-3235-5252-fd0b-883522a21aab: Installed remote snapshot +2025-06-15T03:17:05.358Z [INFO] server-8b1a9662-2537-6b1f-49ab-d8d1472363c6: pipelining replication: peer="{Voter server-fda1265c-3235-5252-fd0b-883522a21aab fda1265c-3235-5252-fd0b-883522a21aab}" +2025-06-15T03:17:06.361Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:17:05.356591394 +0000 UTC m=+41.068269568 (2 nodes), 904ns from start of poll, 1.68897355s from cluster start. Timeout at 2025-06-15 03:17:06.361784014 +0000 UTC m=+42.073462179, 1.005192611s after stability +2025-06-15T03:17:06.373Z [INFO] server-7243c961-468a-ab56-a4e1-3ff83ee08836: aborting pipeline replication: peer="{Voter server-47efea5c-541f-ad8f-a542-0f2b57b7695c 47efea5c-541f-ad8f-a542-0f2b57b7695c}" +2025-06-15T03:17:06.373Z [INFO] server-7243c961-468a-ab56-a4e1-3ff83ee08836: aborting pipeline replication: peer="{Voter server-cc538058-a8d6-1eea-71ff-5fab76914291 cc538058-a8d6-1eea-71ff-5fab76914291}" === RUN TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false -2024-05-12T20:50:31.532Z [INFO] server-35316ca0-ac75-1f99-50ff-8efd4dc5b575: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35316ca0-ac75-1f99-50ff-8efd4dc5b575 Address:35316ca0-ac75-1f99-50ff-8efd4dc5b575} {Suffrage:Voter ID:server-52587903-40ab-ab66-88af-91df59c14e14 Address:52587903-40ab-ab66-88af-91df59c14e14} {Suffrage:Voter ID:server-5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43 Address:5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43}]" -2024-05-12T20:50:31.532Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35316ca0-ac75-1f99-50ff-8efd4dc5b575 Address:35316ca0-ac75-1f99-50ff-8efd4dc5b575} {Suffrage:Voter ID:server-52587903-40ab-ab66-88af-91df59c14e14 Address:52587903-40ab-ab66-88af-91df59c14e14} {Suffrage:Voter ID:server-5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43 Address:5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43}]" -2024-05-12T20:50:31.532Z [INFO] server-5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35316ca0-ac75-1f99-50ff-8efd4dc5b575 Address:35316ca0-ac75-1f99-50ff-8efd4dc5b575} {Suffrage:Voter ID:server-52587903-40ab-ab66-88af-91df59c14e14 Address:52587903-40ab-ab66-88af-91df59c14e14} {Suffrage:Voter ID:server-5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43 Address:5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43}]" -2024-05-12T20:50:31.532Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:31.532Z [INFO] server-35316ca0-ac75-1f99-50ff-8efd4dc5b575: entering follower state: follower="Node at 35316ca0-ac75-1f99-50ff-8efd4dc5b575 [Follower]" leader-address= leader-id= -2024-05-12T20:50:31.533Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: entering follower state: follower="Node at 52587903-40ab-ab66-88af-91df59c14e14 [Follower]" leader-address= leader-id= -2024-05-12T20:50:31.533Z [INFO] server-5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43: entering follower state: follower="Node at 5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43 [Follower]" leader-address= leader-id= -2024-05-12T20:50:32.147Z [WARN] server-35316ca0-ac75-1f99-50ff-8efd4dc5b575: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:32.147Z [INFO] server-35316ca0-ac75-1f99-50ff-8efd4dc5b575: entering candidate state: node="Node at 35316ca0-ac75-1f99-50ff-8efd4dc5b575 [Candidate]" term=2 -2024-05-12T20:50:32.147Z [WARN] server-52587903-40ab-ab66-88af-91df59c14e14: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:32.147Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: entering candidate state: node="Node at 52587903-40ab-ab66-88af-91df59c14e14 [Candidate]" term=2 -2024-05-12T20:50:32.147Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: duplicate requestVote for same term: term=2 -2024-05-12T20:50:32.147Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: election won: term=2 tally=2 -2024-05-12T20:50:32.147Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: entering leader state: leader="Node at 52587903-40ab-ab66-88af-91df59c14e14 [Leader]" -2024-05-12T20:50:32.147Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: added peer, starting replication: peer=server-35316ca0-ac75-1f99-50ff-8efd4dc5b575 -2024-05-12T20:50:32.147Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: added peer, starting replication: peer=server-5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43 -2024-05-12T20:50:32.147Z [INFO] server-5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43: duplicate requestVote for same term: term=2 -2024-05-12T20:50:32.147Z [INFO] server-35316ca0-ac75-1f99-50ff-8efd4dc5b575: duplicate requestVote for same term: term=2 -2024-05-12T20:50:32.147Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: pipelining replication: peer="{Voter server-5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43 5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43}" -2024-05-12T20:50:32.147Z [INFO] server-35316ca0-ac75-1f99-50ff-8efd4dc5b575: entering follower state: follower="Node at 35316ca0-ac75-1f99-50ff-8efd4dc5b575 [Follower]" leader-address=52587903-40ab-ab66-88af-91df59c14e14 leader-id=server-52587903-40ab-ab66-88af-91df59c14e14 -2024-05-12T20:50:32.147Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: pipelining replication: peer="{Voter server-35316ca0-ac75-1f99-50ff-8efd4dc5b575 35316ca0-ac75-1f99-50ff-8efd4dc5b575}" -2024-05-12T20:50:33.159Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:32.147851094 +0000 UTC m=+43.428551571 (1 nodes), 614.968403ms from start of poll, 615.436893ms from cluster start. Timeout at 2024-05-12 20:50:33.159162711 +0000 UTC m=+44.439862606, 1.011311035s after stability -2024-05-12T20:50:33.160Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: starting snapshot up to: index=12 -2024-05-12T20:50:33.160Z [INFO] creating new snapshot: path=/tmp/raft2407257659/snapshots/2-12-1715547033160.tmp -2024-05-12T20:50:33.161Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: snapshot complete up to: index=12 -2024-05-12T20:50:33.195Z [INFO] creating new snapshot: path=/tmp/raft2407257659/snapshots/2-23-1715547033195.tmp -2024-05-12T20:50:33.196Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: copied to local snapshot: bytes=71 -2024-05-12T20:50:33.197Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: snapshot restore progress: id=2-23-1715547033195 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:33.197Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: restored user snapshot: index=23 -2024-05-12T20:50:33.197Z [ERROR] server-52587903-40ab-ab66-88af-91df59c14e14: failed to get log: index=23 error="log not found" -2024-05-12T20:50:33.197Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: aborting pipeline replication: peer="{Voter server-5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43 5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43}" -2024-05-12T20:50:33.197Z [ERROR] server-52587903-40ab-ab66-88af-91df59c14e14: failed to get log: index=23 error="log not found" -2024-05-12T20:50:33.197Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: aborting pipeline replication: peer="{Voter server-35316ca0-ac75-1f99-50ff-8efd4dc5b575 35316ca0-ac75-1f99-50ff-8efd4dc5b575}" -2024-05-12T20:50:33.206Z [ERROR] server-52587903-40ab-ab66-88af-91df59c14e14: failed to get log: index=23 error="log not found" -2024-05-12T20:50:33.207Z [INFO] creating new snapshot: path=/tmp/raft2511251786/snapshots/2-23-1715547033207.tmp -2024-05-12T20:50:33.207Z [INFO] server-35316ca0-ac75-1f99-50ff-8efd4dc5b575: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:33.208Z [INFO] server-35316ca0-ac75-1f99-50ff-8efd4dc5b575: copied to local snapshot: bytes=71 -2024-05-12T20:50:33.209Z [INFO] server-35316ca0-ac75-1f99-50ff-8efd4dc5b575: snapshot restore progress: id=2-23-1715547033207 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:33.209Z [INFO] server-35316ca0-ac75-1f99-50ff-8efd4dc5b575: Installed remote snapshot -2024-05-12T20:50:33.209Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: pipelining replication: peer="{Voter server-35316ca0-ac75-1f99-50ff-8efd4dc5b575 35316ca0-ac75-1f99-50ff-8efd4dc5b575}" -2024-05-12T20:50:33.209Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:33.209Z [ERROR] server-52587903-40ab-ab66-88af-91df59c14e14: failed to get log: index=23 error="log not found" -2024-05-12T20:50:33.210Z [INFO] creating new snapshot: path=/tmp/raft362695539/snapshots/2-23-1715547033210.tmp -2024-05-12T20:50:33.210Z [INFO] server-5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:33.211Z [INFO] server-5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43: copied to local snapshot: bytes=71 -2024-05-12T20:50:33.211Z [INFO] server-5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43: snapshot restore progress: id=2-23-1715547033210 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:33.211Z [INFO] server-5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43: Installed remote snapshot -2024-05-12T20:50:33.211Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: pipelining replication: peer="{Voter server-5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43 5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43}" -2024-05-12T20:50:34.219Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:33.20969878 +0000 UTC m=+44.490399098 (2 nodes), 1.88µs from start of poll, 1.67728442s from cluster start. Timeout at 2024-05-12 20:50:34.218990582 +0000 UTC m=+45.499690898, 1.0092918s after stability -2024-05-12T20:50:34.243Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: aborting pipeline replication: peer="{Voter server-5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43 5cf9a97d-1cac-3d56-b7ae-924f2b1e9f43}" -2024-05-12T20:50:34.243Z [INFO] server-52587903-40ab-ab66-88af-91df59c14e14: aborting pipeline replication: peer="{Voter server-35316ca0-ac75-1f99-50ff-8efd4dc5b575 35316ca0-ac75-1f99-50ff-8efd4dc5b575}" +2025-06-15T03:17:06.375Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-387d2597-8dc4-eb63-8220-c4a6c9d593d8 Address:387d2597-8dc4-eb63-8220-c4a6c9d593d8} {Suffrage:Voter ID:server-cc863b5a-b1f4-4aa7-4030-3b3b567512ae Address:cc863b5a-b1f4-4aa7-4030-3b3b567512ae} {Suffrage:Voter ID:server-e335bb9b-35f3-b15f-bbf9-2d382ae20d3a Address:e335bb9b-35f3-b15f-bbf9-2d382ae20d3a}]" +2025-06-15T03:17:06.375Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: entering follower state: follower="Node at 387d2597-8dc4-eb63-8220-c4a6c9d593d8 [Follower]" leader-address= leader-id= +2025-06-15T03:17:06.375Z [INFO] server-cc863b5a-b1f4-4aa7-4030-3b3b567512ae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-387d2597-8dc4-eb63-8220-c4a6c9d593d8 Address:387d2597-8dc4-eb63-8220-c4a6c9d593d8} {Suffrage:Voter ID:server-cc863b5a-b1f4-4aa7-4030-3b3b567512ae Address:cc863b5a-b1f4-4aa7-4030-3b3b567512ae} {Suffrage:Voter ID:server-e335bb9b-35f3-b15f-bbf9-2d382ae20d3a Address:e335bb9b-35f3-b15f-bbf9-2d382ae20d3a}]" +2025-06-15T03:17:06.375Z [INFO] server-cc863b5a-b1f4-4aa7-4030-3b3b567512ae: entering follower state: follower="Node at cc863b5a-b1f4-4aa7-4030-3b3b567512ae [Follower]" leader-address= leader-id= +2025-06-15T03:17:06.375Z [INFO] server-e335bb9b-35f3-b15f-bbf9-2d382ae20d3a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-387d2597-8dc4-eb63-8220-c4a6c9d593d8 Address:387d2597-8dc4-eb63-8220-c4a6c9d593d8} {Suffrage:Voter ID:server-cc863b5a-b1f4-4aa7-4030-3b3b567512ae Address:cc863b5a-b1f4-4aa7-4030-3b3b567512ae} {Suffrage:Voter ID:server-e335bb9b-35f3-b15f-bbf9-2d382ae20d3a Address:e335bb9b-35f3-b15f-bbf9-2d382ae20d3a}]" +2025-06-15T03:17:06.375Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:06.375Z [INFO] server-e335bb9b-35f3-b15f-bbf9-2d382ae20d3a: entering follower state: follower="Node at e335bb9b-35f3-b15f-bbf9-2d382ae20d3a [Follower]" leader-address= leader-id= +2025-06-15T03:17:06.931Z [WARN] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:06.931Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: entering candidate state: node="Node at 387d2597-8dc4-eb63-8220-c4a6c9d593d8 [Candidate]" term=2 +2025-06-15T03:17:06.931Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: election won: term=2 tally=2 +2025-06-15T03:17:06.931Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: entering leader state: leader="Node at 387d2597-8dc4-eb63-8220-c4a6c9d593d8 [Leader]" +2025-06-15T03:17:06.931Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: added peer, starting replication: peer=server-cc863b5a-b1f4-4aa7-4030-3b3b567512ae +2025-06-15T03:17:06.931Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: added peer, starting replication: peer=server-e335bb9b-35f3-b15f-bbf9-2d382ae20d3a +2025-06-15T03:17:06.931Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: pipelining replication: peer="{Voter server-e335bb9b-35f3-b15f-bbf9-2d382ae20d3a e335bb9b-35f3-b15f-bbf9-2d382ae20d3a}" +2025-06-15T03:17:06.931Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: pipelining replication: peer="{Voter server-cc863b5a-b1f4-4aa7-4030-3b3b567512ae cc863b5a-b1f4-4aa7-4030-3b3b567512ae}" +2025-06-15T03:17:07.937Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:06.931756988 +0000 UTC m=+42.643435153 (1 nodes), 556.067614ms from start of poll, 556.348661ms from cluster start. Timeout at 2025-06-15 03:17:07.937535018 +0000 UTC m=+43.649213185, 1.005778032s after stability +2025-06-15T03:17:07.938Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: starting snapshot up to: index=12 +2025-06-15T03:17:07.938Z [INFO] creating new snapshot: path=/tmp/raft3143568345/snapshots/2-12-1749957427938.tmp +2025-06-15T03:17:07.939Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: snapshot complete up to: index=12 +2025-06-15T03:17:07.940Z [INFO] creating new snapshot: path=/tmp/raft3143568345/snapshots/2-23-1749957427940.tmp +2025-06-15T03:17:07.940Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: copied to local snapshot: bytes=71 +2025-06-15T03:17:07.940Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: snapshot restore progress: id=2-23-1749957427940 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:07.940Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: restored user snapshot: index=23 +2025-06-15T03:17:07.940Z [ERROR] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: failed to get log: index=23 error="log not found" +2025-06-15T03:17:07.940Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: aborting pipeline replication: peer="{Voter server-e335bb9b-35f3-b15f-bbf9-2d382ae20d3a e335bb9b-35f3-b15f-bbf9-2d382ae20d3a}" +2025-06-15T03:17:07.940Z [ERROR] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: failed to get log: index=23 error="log not found" +2025-06-15T03:17:07.940Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: aborting pipeline replication: peer="{Voter server-cc863b5a-b1f4-4aa7-4030-3b3b567512ae cc863b5a-b1f4-4aa7-4030-3b3b567512ae}" +2025-06-15T03:17:07.947Z [ERROR] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: failed to get log: index=23 error="log not found" +2025-06-15T03:17:07.947Z [INFO] creating new snapshot: path=/tmp/raft602803712/snapshots/2-23-1749957427947.tmp +2025-06-15T03:17:07.947Z [INFO] server-cc863b5a-b1f4-4aa7-4030-3b3b567512ae: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:07.948Z [INFO] server-cc863b5a-b1f4-4aa7-4030-3b3b567512ae: copied to local snapshot: bytes=71 +2025-06-15T03:17:07.948Z [INFO] server-cc863b5a-b1f4-4aa7-4030-3b3b567512ae: snapshot restore progress: id=2-23-1749957427947 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:07.948Z [INFO] server-cc863b5a-b1f4-4aa7-4030-3b3b567512ae: Installed remote snapshot +2025-06-15T03:17:07.948Z [WARN] server-cc863b5a-b1f4-4aa7-4030-3b3b567512ae: skipping application of old log: index=22 +2025-06-15T03:17:07.948Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: pipelining replication: peer="{Voter server-cc863b5a-b1f4-4aa7-4030-3b3b567512ae cc863b5a-b1f4-4aa7-4030-3b3b567512ae}" +2025-06-15T03:17:07.948Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:17:07.949Z [ERROR] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: failed to get log: index=23 error="log not found" +2025-06-15T03:17:07.950Z [INFO] creating new snapshot: path=/tmp/raft1617853931/snapshots/2-23-1749957427950.tmp +2025-06-15T03:17:07.950Z [INFO] server-e335bb9b-35f3-b15f-bbf9-2d382ae20d3a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:07.950Z [INFO] server-e335bb9b-35f3-b15f-bbf9-2d382ae20d3a: copied to local snapshot: bytes=71 +2025-06-15T03:17:07.950Z [INFO] server-e335bb9b-35f3-b15f-bbf9-2d382ae20d3a: snapshot restore progress: id=2-23-1749957427950 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:07.950Z [INFO] server-e335bb9b-35f3-b15f-bbf9-2d382ae20d3a: Installed remote snapshot +2025-06-15T03:17:07.950Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: pipelining replication: peer="{Voter server-e335bb9b-35f3-b15f-bbf9-2d382ae20d3a e335bb9b-35f3-b15f-bbf9-2d382ae20d3a}" +2025-06-15T03:17:08.953Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:17:07.948512038 +0000 UTC m=+43.660190201 (2 nodes), 574ns from start of poll, 1.573103709s from cluster start. Timeout at 2025-06-15 03:17:08.953831164 +0000 UTC m=+44.665509330, 1.005319129s after stability +2025-06-15T03:17:08.965Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: aborting pipeline replication: peer="{Voter server-e335bb9b-35f3-b15f-bbf9-2d382ae20d3a e335bb9b-35f3-b15f-bbf9-2d382ae20d3a}" +2025-06-15T03:17:08.965Z [INFO] server-387d2597-8dc4-eb63-8220-c4a6c9d593d8: aborting pipeline replication: peer="{Voter server-cc863b5a-b1f4-4aa7-4030-3b3b567512ae cc863b5a-b1f4-4aa7-4030-3b3b567512ae}" === RUN TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false -2024-05-12T20:50:34.245Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-804cd157-9513-d9a5-3859-7e0cac79c4ce Address:804cd157-9513-d9a5-3859-7e0cac79c4ce} {Suffrage:Voter ID:server-1bfc6e64-789b-0280-b213-6698cdbe2e5e Address:1bfc6e64-789b-0280-b213-6698cdbe2e5e} {Suffrage:Voter ID:server-60229e8e-177a-30d6-9843-73121d6f67d8 Address:60229e8e-177a-30d6-9843-73121d6f67d8}]" -2024-05-12T20:50:34.245Z [INFO] server-1bfc6e64-789b-0280-b213-6698cdbe2e5e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-804cd157-9513-d9a5-3859-7e0cac79c4ce Address:804cd157-9513-d9a5-3859-7e0cac79c4ce} {Suffrage:Voter ID:server-1bfc6e64-789b-0280-b213-6698cdbe2e5e Address:1bfc6e64-789b-0280-b213-6698cdbe2e5e} {Suffrage:Voter ID:server-60229e8e-177a-30d6-9843-73121d6f67d8 Address:60229e8e-177a-30d6-9843-73121d6f67d8}]" -2024-05-12T20:50:34.245Z [INFO] server-60229e8e-177a-30d6-9843-73121d6f67d8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-804cd157-9513-d9a5-3859-7e0cac79c4ce Address:804cd157-9513-d9a5-3859-7e0cac79c4ce} {Suffrage:Voter ID:server-1bfc6e64-789b-0280-b213-6698cdbe2e5e Address:1bfc6e64-789b-0280-b213-6698cdbe2e5e} {Suffrage:Voter ID:server-60229e8e-177a-30d6-9843-73121d6f67d8 Address:60229e8e-177a-30d6-9843-73121d6f67d8}]" -2024-05-12T20:50:34.246Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:34.246Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: entering follower state: follower="Node at 804cd157-9513-d9a5-3859-7e0cac79c4ce [Follower]" leader-address= leader-id= -2024-05-12T20:50:34.246Z [INFO] server-1bfc6e64-789b-0280-b213-6698cdbe2e5e: entering follower state: follower="Node at 1bfc6e64-789b-0280-b213-6698cdbe2e5e [Follower]" leader-address= leader-id= -2024-05-12T20:50:34.246Z [INFO] server-60229e8e-177a-30d6-9843-73121d6f67d8: entering follower state: follower="Node at 60229e8e-177a-30d6-9843-73121d6f67d8 [Follower]" leader-address= leader-id= -2024-05-12T20:50:34.782Z [WARN] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:34.782Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: entering candidate state: node="Node at 804cd157-9513-d9a5-3859-7e0cac79c4ce [Candidate]" term=2 -2024-05-12T20:50:34.783Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: election won: term=2 tally=2 -2024-05-12T20:50:34.783Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: entering leader state: leader="Node at 804cd157-9513-d9a5-3859-7e0cac79c4ce [Leader]" -2024-05-12T20:50:34.783Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: added peer, starting replication: peer=server-1bfc6e64-789b-0280-b213-6698cdbe2e5e -2024-05-12T20:50:34.783Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: added peer, starting replication: peer=server-60229e8e-177a-30d6-9843-73121d6f67d8 -2024-05-12T20:50:34.783Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: pipelining replication: peer="{Voter server-60229e8e-177a-30d6-9843-73121d6f67d8 60229e8e-177a-30d6-9843-73121d6f67d8}" -2024-05-12T20:50:34.783Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: pipelining replication: peer="{Voter server-1bfc6e64-789b-0280-b213-6698cdbe2e5e 1bfc6e64-789b-0280-b213-6698cdbe2e5e}" -2024-05-12T20:50:35.790Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:34.783257185 +0000 UTC m=+46.063957904 (1 nodes), 537.227017ms from start of poll, 537.598089ms from cluster start. Timeout at 2024-05-12 20:50:35.790828297 +0000 UTC m=+47.071528676, 1.007570772s after stability -2024-05-12T20:50:35.792Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: starting snapshot up to: index=12 -2024-05-12T20:50:35.792Z [INFO] creating new snapshot: path=/tmp/raft901888032/snapshots/2-12-1715547035792.tmp -2024-05-12T20:50:35.793Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: snapshot complete up to: index=12 -2024-05-12T20:50:35.795Z [INFO] creating new snapshot: path=/tmp/raft901888032/snapshots/2-23-1715547035795.tmp -2024-05-12T20:50:35.796Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: copied to local snapshot: bytes=71 -2024-05-12T20:50:35.796Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: snapshot restore progress: id=2-23-1715547035795 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:35.799Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: removing all old logs from log store -2024-05-12T20:50:35.799Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: compacting logs: from=1 to=22 -2024-05-12T20:50:35.799Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: restored user snapshot: index=23 -2024-05-12T20:50:35.799Z [ERROR] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: failed to get log: index=22 error="log not found" -2024-05-12T20:50:35.799Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: aborting pipeline replication: peer="{Voter server-60229e8e-177a-30d6-9843-73121d6f67d8 60229e8e-177a-30d6-9843-73121d6f67d8}" -2024-05-12T20:50:35.799Z [ERROR] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: failed to get log: index=22 error="log not found" -2024-05-12T20:50:35.799Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: aborting pipeline replication: peer="{Voter server-1bfc6e64-789b-0280-b213-6698cdbe2e5e 1bfc6e64-789b-0280-b213-6698cdbe2e5e}" -2024-05-12T20:50:35.811Z [ERROR] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: failed to get log: index=22 error="log not found" -2024-05-12T20:50:35.811Z [INFO] creating new snapshot: path=/tmp/raft3267577122/snapshots/2-23-1715547035811.tmp -2024-05-12T20:50:35.812Z [INFO] server-1bfc6e64-789b-0280-b213-6698cdbe2e5e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:35.812Z [INFO] server-1bfc6e64-789b-0280-b213-6698cdbe2e5e: copied to local snapshot: bytes=71 -2024-05-12T20:50:35.811Z [ERROR] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: failed to get log: index=22 error="log not found" -2024-05-12T20:50:35.813Z [INFO] creating new snapshot: path=/tmp/raft1824648321/snapshots/2-23-1715547035813.tmp -2024-05-12T20:50:35.813Z [INFO] server-60229e8e-177a-30d6-9843-73121d6f67d8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:35.813Z [INFO] server-1bfc6e64-789b-0280-b213-6698cdbe2e5e: snapshot restore progress: id=2-23-1715547035811 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:35.813Z [INFO] server-1bfc6e64-789b-0280-b213-6698cdbe2e5e: removing all old logs from log store -2024-05-12T20:50:35.813Z [INFO] server-1bfc6e64-789b-0280-b213-6698cdbe2e5e: compacting logs: from=1 to=22 -2024-05-12T20:50:35.813Z [INFO] server-1bfc6e64-789b-0280-b213-6698cdbe2e5e: Installed remote snapshot -2024-05-12T20:50:35.813Z [WARN] server-1bfc6e64-789b-0280-b213-6698cdbe2e5e: skipping application of old log: index=22 -2024-05-12T20:50:35.813Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: pipelining replication: peer="{Voter server-1bfc6e64-789b-0280-b213-6698cdbe2e5e 1bfc6e64-789b-0280-b213-6698cdbe2e5e}" -2024-05-12T20:50:35.813Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:35.813Z [INFO] server-60229e8e-177a-30d6-9843-73121d6f67d8: copied to local snapshot: bytes=71 -2024-05-12T20:50:35.814Z [INFO] server-60229e8e-177a-30d6-9843-73121d6f67d8: snapshot restore progress: id=2-23-1715547035813 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:35.814Z [INFO] server-60229e8e-177a-30d6-9843-73121d6f67d8: removing all old logs from log store -2024-05-12T20:50:35.814Z [INFO] server-60229e8e-177a-30d6-9843-73121d6f67d8: compacting logs: from=1 to=22 -2024-05-12T20:50:35.814Z [INFO] server-60229e8e-177a-30d6-9843-73121d6f67d8: Installed remote snapshot -2024-05-12T20:50:35.814Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: pipelining replication: peer="{Voter server-60229e8e-177a-30d6-9843-73121d6f67d8 60229e8e-177a-30d6-9843-73121d6f67d8}" -2024-05-12T20:50:36.820Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:35.813960362 +0000 UTC m=+47.094660731 (2 nodes), 2.109µs from start of poll, 1.568300916s from cluster start. Timeout at 2024-05-12 20:50:36.82022825 +0000 UTC m=+48.100928558, 1.006267827s after stability -2024-05-12T20:50:36.844Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: aborting pipeline replication: peer="{Voter server-60229e8e-177a-30d6-9843-73121d6f67d8 60229e8e-177a-30d6-9843-73121d6f67d8}" -2024-05-12T20:50:36.844Z [INFO] server-804cd157-9513-d9a5-3859-7e0cac79c4ce: aborting pipeline replication: peer="{Voter server-1bfc6e64-789b-0280-b213-6698cdbe2e5e 1bfc6e64-789b-0280-b213-6698cdbe2e5e}" +2025-06-15T03:17:08.967Z [INFO] server-507e248c-2c4e-79ae-c616-39b7cedc6e5c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-507e248c-2c4e-79ae-c616-39b7cedc6e5c Address:507e248c-2c4e-79ae-c616-39b7cedc6e5c} {Suffrage:Voter ID:server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63 Address:d4d508ae-77fa-9f10-fc3e-3efb23ea9b63} {Suffrage:Voter ID:server-86582199-dab4-8b54-768f-f110b47d31ce Address:86582199-dab4-8b54-768f-f110b47d31ce}]" +2025-06-15T03:17:08.967Z [INFO] server-507e248c-2c4e-79ae-c616-39b7cedc6e5c: entering follower state: follower="Node at 507e248c-2c4e-79ae-c616-39b7cedc6e5c [Follower]" leader-address= leader-id= +2025-06-15T03:17:08.967Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-507e248c-2c4e-79ae-c616-39b7cedc6e5c Address:507e248c-2c4e-79ae-c616-39b7cedc6e5c} {Suffrage:Voter ID:server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63 Address:d4d508ae-77fa-9f10-fc3e-3efb23ea9b63} {Suffrage:Voter ID:server-86582199-dab4-8b54-768f-f110b47d31ce Address:86582199-dab4-8b54-768f-f110b47d31ce}]" +2025-06-15T03:17:08.967Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: entering follower state: follower="Node at d4d508ae-77fa-9f10-fc3e-3efb23ea9b63 [Follower]" leader-address= leader-id= +2025-06-15T03:17:08.967Z [INFO] server-86582199-dab4-8b54-768f-f110b47d31ce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-507e248c-2c4e-79ae-c616-39b7cedc6e5c Address:507e248c-2c4e-79ae-c616-39b7cedc6e5c} {Suffrage:Voter ID:server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63 Address:d4d508ae-77fa-9f10-fc3e-3efb23ea9b63} {Suffrage:Voter ID:server-86582199-dab4-8b54-768f-f110b47d31ce Address:86582199-dab4-8b54-768f-f110b47d31ce}]" +2025-06-15T03:17:08.967Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:08.967Z [INFO] server-86582199-dab4-8b54-768f-f110b47d31ce: entering follower state: follower="Node at 86582199-dab4-8b54-768f-f110b47d31ce [Follower]" leader-address= leader-id= +2025-06-15T03:17:09.577Z [WARN] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:09.577Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: entering candidate state: node="Node at d4d508ae-77fa-9f10-fc3e-3efb23ea9b63 [Candidate]" term=2 +2025-06-15T03:17:09.577Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: election won: term=2 tally=2 +2025-06-15T03:17:09.577Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: entering leader state: leader="Node at d4d508ae-77fa-9f10-fc3e-3efb23ea9b63 [Leader]" +2025-06-15T03:17:09.577Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: added peer, starting replication: peer=server-507e248c-2c4e-79ae-c616-39b7cedc6e5c +2025-06-15T03:17:09.577Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: added peer, starting replication: peer=server-86582199-dab4-8b54-768f-f110b47d31ce +2025-06-15T03:17:09.577Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: pipelining replication: peer="{Voter server-507e248c-2c4e-79ae-c616-39b7cedc6e5c 507e248c-2c4e-79ae-c616-39b7cedc6e5c}" +2025-06-15T03:17:09.577Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: pipelining replication: peer="{Voter server-86582199-dab4-8b54-768f-f110b47d31ce 86582199-dab4-8b54-768f-f110b47d31ce}" +2025-06-15T03:17:10.583Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:09.577628725 +0000 UTC m=+45.289306890 (1 nodes), 610.337935ms from start of poll, 610.71746ms from cluster start. Timeout at 2025-06-15 03:17:10.58345374 +0000 UTC m=+46.295131942, 1.005825052s after stability +2025-06-15T03:17:10.584Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: starting snapshot up to: index=12 +2025-06-15T03:17:10.584Z [INFO] creating new snapshot: path=/tmp/raft1402843168/snapshots/2-12-1749957430584.tmp +2025-06-15T03:17:10.585Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: snapshot complete up to: index=12 +2025-06-15T03:17:10.585Z [INFO] creating new snapshot: path=/tmp/raft1402843168/snapshots/2-23-1749957430585.tmp +2025-06-15T03:17:10.586Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: copied to local snapshot: bytes=71 +2025-06-15T03:17:10.586Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: snapshot restore progress: id=2-23-1749957430585 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:10.586Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: removing all old logs from log store +2025-06-15T03:17:10.586Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: compacting logs: from=1 to=22 +2025-06-15T03:17:10.586Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: restored user snapshot: index=23 +2025-06-15T03:17:10.586Z [ERROR] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: failed to get log: index=22 error="log not found" +2025-06-15T03:17:10.586Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: aborting pipeline replication: peer="{Voter server-86582199-dab4-8b54-768f-f110b47d31ce 86582199-dab4-8b54-768f-f110b47d31ce}" +2025-06-15T03:17:10.586Z [ERROR] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: failed to get log: index=22 error="log not found" +2025-06-15T03:17:10.586Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: aborting pipeline replication: peer="{Voter server-507e248c-2c4e-79ae-c616-39b7cedc6e5c 507e248c-2c4e-79ae-c616-39b7cedc6e5c}" +2025-06-15T03:17:10.592Z [ERROR] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: failed to get log: index=22 error="log not found" +2025-06-15T03:17:10.592Z [ERROR] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: failed to get log: index=22 error="log not found" +2025-06-15T03:17:10.592Z [INFO] creating new snapshot: path=/tmp/raft2119442370/snapshots/2-23-1749957430592.tmp +2025-06-15T03:17:10.592Z [INFO] creating new snapshot: path=/tmp/raft2145780714/snapshots/2-23-1749957430592.tmp +2025-06-15T03:17:10.593Z [INFO] server-507e248c-2c4e-79ae-c616-39b7cedc6e5c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:10.593Z [INFO] server-86582199-dab4-8b54-768f-f110b47d31ce: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:10.593Z [INFO] server-507e248c-2c4e-79ae-c616-39b7cedc6e5c: copied to local snapshot: bytes=71 +2025-06-15T03:17:10.593Z [INFO] server-86582199-dab4-8b54-768f-f110b47d31ce: copied to local snapshot: bytes=71 +2025-06-15T03:17:10.593Z [INFO] server-507e248c-2c4e-79ae-c616-39b7cedc6e5c: snapshot restore progress: id=2-23-1749957430592 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:10.593Z [INFO] server-86582199-dab4-8b54-768f-f110b47d31ce: snapshot restore progress: id=2-23-1749957430592 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:10.593Z [INFO] server-86582199-dab4-8b54-768f-f110b47d31ce: removing all old logs from log store +2025-06-15T03:17:10.593Z [INFO] server-86582199-dab4-8b54-768f-f110b47d31ce: compacting logs: from=1 to=22 +2025-06-15T03:17:10.593Z [INFO] server-86582199-dab4-8b54-768f-f110b47d31ce: Installed remote snapshot +2025-06-15T03:17:10.593Z [INFO] server-507e248c-2c4e-79ae-c616-39b7cedc6e5c: removing all old logs from log store +2025-06-15T03:17:10.594Z [INFO] server-507e248c-2c4e-79ae-c616-39b7cedc6e5c: compacting logs: from=1 to=22 +2025-06-15T03:17:10.594Z [WARN] server-86582199-dab4-8b54-768f-f110b47d31ce: skipping application of old log: index=22 +2025-06-15T03:17:10.594Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: pipelining replication: peer="{Voter server-86582199-dab4-8b54-768f-f110b47d31ce 86582199-dab4-8b54-768f-f110b47d31ce}" +2025-06-15T03:17:10.594Z [INFO] server-507e248c-2c4e-79ae-c616-39b7cedc6e5c: Installed remote snapshot +2025-06-15T03:17:10.594Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:17:10.594Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: pipelining replication: peer="{Voter server-507e248c-2c4e-79ae-c616-39b7cedc6e5c 507e248c-2c4e-79ae-c616-39b7cedc6e5c}" +2025-06-15T03:17:11.599Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:17:10.594205892 +0000 UTC m=+46.305884082 (2 nodes), 895ns from start of poll, 1.627294652s from cluster start. Timeout at 2025-06-15 03:17:11.599529699 +0000 UTC m=+47.311207898, 1.005323816s after stability +2025-06-15T03:17:11.611Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: aborting pipeline replication: peer="{Voter server-86582199-dab4-8b54-768f-f110b47d31ce 86582199-dab4-8b54-768f-f110b47d31ce}" +2025-06-15T03:17:11.611Z [INFO] server-d4d508ae-77fa-9f10-fc3e-3efb23ea9b63: aborting pipeline replication: peer="{Voter server-507e248c-2c4e-79ae-c616-39b7cedc6e5c 507e248c-2c4e-79ae-c616-39b7cedc6e5c}" === RUN TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true -2024-05-12T20:50:36.846Z [INFO] server-27f36be9-6652-6561-fa29-77cc8d016ddf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27f36be9-6652-6561-fa29-77cc8d016ddf Address:27f36be9-6652-6561-fa29-77cc8d016ddf} {Suffrage:Voter ID:server-5964381d-a63d-fb7b-a595-5ca386f8603d Address:5964381d-a63d-fb7b-a595-5ca386f8603d} {Suffrage:Voter ID:server-ea81a483-9ce2-b72d-4714-020c3c513b9e Address:ea81a483-9ce2-b72d-4714-020c3c513b9e}]" -2024-05-12T20:50:36.846Z [INFO] server-5964381d-a63d-fb7b-a595-5ca386f8603d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27f36be9-6652-6561-fa29-77cc8d016ddf Address:27f36be9-6652-6561-fa29-77cc8d016ddf} {Suffrage:Voter ID:server-5964381d-a63d-fb7b-a595-5ca386f8603d Address:5964381d-a63d-fb7b-a595-5ca386f8603d} {Suffrage:Voter ID:server-ea81a483-9ce2-b72d-4714-020c3c513b9e Address:ea81a483-9ce2-b72d-4714-020c3c513b9e}]" -2024-05-12T20:50:36.846Z [INFO] server-27f36be9-6652-6561-fa29-77cc8d016ddf: entering follower state: follower="Node at 27f36be9-6652-6561-fa29-77cc8d016ddf [Follower]" leader-address= leader-id= -2024-05-12T20:50:36.847Z [INFO] server-5964381d-a63d-fb7b-a595-5ca386f8603d: entering follower state: follower="Node at 5964381d-a63d-fb7b-a595-5ca386f8603d [Follower]" leader-address= leader-id= -2024-05-12T20:50:36.847Z [INFO] server-ea81a483-9ce2-b72d-4714-020c3c513b9e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27f36be9-6652-6561-fa29-77cc8d016ddf Address:27f36be9-6652-6561-fa29-77cc8d016ddf} {Suffrage:Voter ID:server-5964381d-a63d-fb7b-a595-5ca386f8603d Address:5964381d-a63d-fb7b-a595-5ca386f8603d} {Suffrage:Voter ID:server-ea81a483-9ce2-b72d-4714-020c3c513b9e Address:ea81a483-9ce2-b72d-4714-020c3c513b9e}]" -2024-05-12T20:50:36.847Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:36.847Z [INFO] server-ea81a483-9ce2-b72d-4714-020c3c513b9e: entering follower state: follower="Node at ea81a483-9ce2-b72d-4714-020c3c513b9e [Follower]" leader-address= leader-id= -2024-05-12T20:50:37.358Z [WARN] server-ea81a483-9ce2-b72d-4714-020c3c513b9e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:37.359Z [INFO] server-ea81a483-9ce2-b72d-4714-020c3c513b9e: entering candidate state: node="Node at ea81a483-9ce2-b72d-4714-020c3c513b9e [Candidate]" term=2 -2024-05-12T20:50:37.359Z [INFO] server-ea81a483-9ce2-b72d-4714-020c3c513b9e: election won: term=2 tally=2 -2024-05-12T20:50:37.359Z [INFO] server-ea81a483-9ce2-b72d-4714-020c3c513b9e: entering leader state: leader="Node at ea81a483-9ce2-b72d-4714-020c3c513b9e [Leader]" -2024-05-12T20:50:37.359Z [INFO] server-ea81a483-9ce2-b72d-4714-020c3c513b9e: added peer, starting replication: peer=server-27f36be9-6652-6561-fa29-77cc8d016ddf -2024-05-12T20:50:37.359Z [INFO] server-ea81a483-9ce2-b72d-4714-020c3c513b9e: added peer, starting replication: peer=server-5964381d-a63d-fb7b-a595-5ca386f8603d -2024-05-12T20:50:37.359Z [INFO] server-ea81a483-9ce2-b72d-4714-020c3c513b9e: pipelining replication: peer="{Voter server-5964381d-a63d-fb7b-a595-5ca386f8603d 5964381d-a63d-fb7b-a595-5ca386f8603d}" -2024-05-12T20:50:37.360Z [INFO] server-ea81a483-9ce2-b72d-4714-020c3c513b9e: pipelining replication: peer="{Voter server-27f36be9-6652-6561-fa29-77cc8d016ddf 27f36be9-6652-6561-fa29-77cc8d016ddf}" -2024-05-12T20:50:38.391Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:37.360317344 +0000 UTC m=+48.641017705 (1 nodes), 513.197221ms from start of poll, 513.778665ms from cluster start. Timeout at 2024-05-12 20:50:38.370786422 +0000 UTC m=+49.651486317, 1.010468612s after stability -2024-05-12T20:50:38.392Z [INFO] server-ea81a483-9ce2-b72d-4714-020c3c513b9e: starting snapshot up to: index=12 -2024-05-12T20:50:38.392Z [INFO] creating new snapshot: path=/tmp/raft1623425310/snapshots/2-12-1715547038392.tmp -2024-05-12T20:50:38.394Z [INFO] server-ea81a483-9ce2-b72d-4714-020c3c513b9e: snapshot complete up to: index=12 -2024-05-12T20:50:38.396Z [INFO] server-47acbbb8-711c-2a70-9161-56ffcdd41784: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-47acbbb8-711c-2a70-9161-56ffcdd41784 Address:47acbbb8-711c-2a70-9161-56ffcdd41784} {Suffrage:Voter ID:server-af57c149-4313-5ef1-9036-cd3898224bfc Address:af57c149-4313-5ef1-9036-cd3898224bfc} {Suffrage:Voter ID:server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7 Address:4010e27b-a8e4-9c6c-2f89-697afd8d2dc7}]" -2024-05-12T20:50:38.396Z [INFO] server-af57c149-4313-5ef1-9036-cd3898224bfc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-47acbbb8-711c-2a70-9161-56ffcdd41784 Address:47acbbb8-711c-2a70-9161-56ffcdd41784} {Suffrage:Voter ID:server-af57c149-4313-5ef1-9036-cd3898224bfc Address:af57c149-4313-5ef1-9036-cd3898224bfc} {Suffrage:Voter ID:server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7 Address:4010e27b-a8e4-9c6c-2f89-697afd8d2dc7}]" -2024-05-12T20:50:38.396Z [INFO] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-47acbbb8-711c-2a70-9161-56ffcdd41784 Address:47acbbb8-711c-2a70-9161-56ffcdd41784} {Suffrage:Voter ID:server-af57c149-4313-5ef1-9036-cd3898224bfc Address:af57c149-4313-5ef1-9036-cd3898224bfc} {Suffrage:Voter ID:server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7 Address:4010e27b-a8e4-9c6c-2f89-697afd8d2dc7}]" -2024-05-12T20:50:38.396Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:38.397Z [INFO] server-47acbbb8-711c-2a70-9161-56ffcdd41784: entering follower state: follower="Node at 47acbbb8-711c-2a70-9161-56ffcdd41784 [Follower]" leader-address= leader-id= -2024-05-12T20:50:38.397Z [INFO] server-af57c149-4313-5ef1-9036-cd3898224bfc: entering follower state: follower="Node at af57c149-4313-5ef1-9036-cd3898224bfc [Follower]" leader-address= leader-id= -2024-05-12T20:50:38.397Z [INFO] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: entering follower state: follower="Node at 4010e27b-a8e4-9c6c-2f89-697afd8d2dc7 [Follower]" leader-address= leader-id= -2024-05-12T20:50:38.959Z [WARN] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:38.959Z [INFO] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: entering candidate state: node="Node at 4010e27b-a8e4-9c6c-2f89-697afd8d2dc7 [Candidate]" term=2 -2024-05-12T20:50:38.959Z [INFO] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: election won: term=2 tally=2 -2024-05-12T20:50:38.959Z [INFO] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: entering leader state: leader="Node at 4010e27b-a8e4-9c6c-2f89-697afd8d2dc7 [Leader]" -2024-05-12T20:50:38.959Z [INFO] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: added peer, starting replication: peer=server-47acbbb8-711c-2a70-9161-56ffcdd41784 -2024-05-12T20:50:38.959Z [INFO] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: added peer, starting replication: peer=server-af57c149-4313-5ef1-9036-cd3898224bfc -2024-05-12T20:50:38.959Z [INFO] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: pipelining replication: peer="{Voter server-af57c149-4313-5ef1-9036-cd3898224bfc af57c149-4313-5ef1-9036-cd3898224bfc}" -2024-05-12T20:50:38.960Z [INFO] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: pipelining replication: peer="{Voter server-47acbbb8-711c-2a70-9161-56ffcdd41784 47acbbb8-711c-2a70-9161-56ffcdd41784}" -2024-05-12T20:50:39.967Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:38.960110205 +0000 UTC m=+50.240810578 (1 nodes), 563.112366ms from start of poll, 563.517078ms from cluster start. Timeout at 2024-05-12 20:50:39.967149812 +0000 UTC m=+51.247850543, 1.007039965s after stability -2024-05-12T20:50:39.967Z [INFO] creating new snapshot: path=/tmp/raft4011944720/snapshots/2-15-1715547039967.tmp -2024-05-12T20:50:39.968Z [INFO] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: copied to local snapshot: bytes=71 -2024-05-12T20:50:39.969Z [INFO] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: snapshot restore progress: id=2-15-1715547039967 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:39.969Z [INFO] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: restored user snapshot: index=15 -2024-05-12T20:50:39.969Z [ERROR] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: failed to get log: index=3 error="log not found" -2024-05-12T20:50:39.969Z [INFO] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: aborting pipeline replication: peer="{Voter server-af57c149-4313-5ef1-9036-cd3898224bfc af57c149-4313-5ef1-9036-cd3898224bfc}" -2024-05-12T20:50:39.969Z [ERROR] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: failed to get log: index=3 error="log not found" -2024-05-12T20:50:39.969Z [INFO] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: aborting pipeline replication: peer="{Voter server-47acbbb8-711c-2a70-9161-56ffcdd41784 47acbbb8-711c-2a70-9161-56ffcdd41784}" -2024-05-12T20:50:39.978Z [ERROR] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: failed to get log: index=3 error="log not found" -2024-05-12T20:50:39.979Z [INFO] creating new snapshot: path=/tmp/raft2723210011/snapshots/2-15-1715547039979.tmp -2024-05-12T20:50:39.979Z [INFO] server-47acbbb8-711c-2a70-9161-56ffcdd41784: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:39.980Z [INFO] server-47acbbb8-711c-2a70-9161-56ffcdd41784: copied to local snapshot: bytes=71 -2024-05-12T20:50:39.980Z [INFO] server-47acbbb8-711c-2a70-9161-56ffcdd41784: snapshot restore progress: id=2-15-1715547039979 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:39.980Z [INFO] server-47acbbb8-711c-2a70-9161-56ffcdd41784: Installed remote snapshot -2024-05-12T20:50:39.981Z [INFO] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: pipelining replication: peer="{Voter server-47acbbb8-711c-2a70-9161-56ffcdd41784 47acbbb8-711c-2a70-9161-56ffcdd41784}" -2024-05-12T20:50:39.981Z [ERROR] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: failed to get log: index=3 error="log not found" -2024-05-12T20:50:39.981Z [INFO] creating new snapshot: path=/tmp/raft577478951/snapshots/2-15-1715547039981.tmp -2024-05-12T20:50:39.982Z [INFO] server-af57c149-4313-5ef1-9036-cd3898224bfc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:39.982Z [INFO] server-af57c149-4313-5ef1-9036-cd3898224bfc: copied to local snapshot: bytes=71 -2024-05-12T20:50:39.983Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:39.999Z [INFO] server-af57c149-4313-5ef1-9036-cd3898224bfc: snapshot restore progress: id=2-15-1715547039981 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:39.999Z [INFO] server-af57c149-4313-5ef1-9036-cd3898224bfc: Installed remote snapshot -2024-05-12T20:50:39.999Z [INFO] server-4010e27b-a8e4-9c6c-2f89-697afd8d2dc7: pipelining replication: peer="{Voter server-af57c149-4313-5ef1-9036-cd3898224bfc af57c149-4313-5ef1-9036-cd3898224bfc}" -2024-05-12T20:50:41.003Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:39.983469068 +0000 UTC m=+51.264169377 (2 nodes), 2.843µs from start of poll, 1.586875877s from cluster start. Timeout at 2024-05-12 20:50:41.002995448 +0000 UTC m=+52.283695343, 1.019525966s after stability -2024-05-12T20:50:41.019Z [INFO] server-ea81a483-9ce2-b72d-4714-020c3c513b9e: aborting pipeline replication: peer="{Voter server-5964381d-a63d-fb7b-a595-5ca386f8603d 5964381d-a63d-fb7b-a595-5ca386f8603d}" -2024-05-12T20:50:41.019Z [INFO] server-ea81a483-9ce2-b72d-4714-020c3c513b9e: aborting pipeline replication: peer="{Voter server-27f36be9-6652-6561-fa29-77cc8d016ddf 27f36be9-6652-6561-fa29-77cc8d016ddf}" +2025-06-15T03:17:11.620Z [INFO] server-4cc8a21b-8ed2-64e2-f39f-0c5bb8ab48db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4cc8a21b-8ed2-64e2-f39f-0c5bb8ab48db Address:4cc8a21b-8ed2-64e2-f39f-0c5bb8ab48db} {Suffrage:Voter ID:server-732678b9-5d78-6277-ead9-583e1a5b6906 Address:732678b9-5d78-6277-ead9-583e1a5b6906} {Suffrage:Voter ID:server-73deb0fe-f680-2157-d7c0-27cbc1546cf8 Address:73deb0fe-f680-2157-d7c0-27cbc1546cf8}]" +2025-06-15T03:17:11.620Z [INFO] server-732678b9-5d78-6277-ead9-583e1a5b6906: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4cc8a21b-8ed2-64e2-f39f-0c5bb8ab48db Address:4cc8a21b-8ed2-64e2-f39f-0c5bb8ab48db} {Suffrage:Voter ID:server-732678b9-5d78-6277-ead9-583e1a5b6906 Address:732678b9-5d78-6277-ead9-583e1a5b6906} {Suffrage:Voter ID:server-73deb0fe-f680-2157-d7c0-27cbc1546cf8 Address:73deb0fe-f680-2157-d7c0-27cbc1546cf8}]" +2025-06-15T03:17:11.620Z [INFO] server-4cc8a21b-8ed2-64e2-f39f-0c5bb8ab48db: entering follower state: follower="Node at 4cc8a21b-8ed2-64e2-f39f-0c5bb8ab48db [Follower]" leader-address= leader-id= +2025-06-15T03:17:11.620Z [INFO] server-732678b9-5d78-6277-ead9-583e1a5b6906: entering follower state: follower="Node at 732678b9-5d78-6277-ead9-583e1a5b6906 [Follower]" leader-address= leader-id= +2025-06-15T03:17:11.620Z [INFO] server-73deb0fe-f680-2157-d7c0-27cbc1546cf8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4cc8a21b-8ed2-64e2-f39f-0c5bb8ab48db Address:4cc8a21b-8ed2-64e2-f39f-0c5bb8ab48db} {Suffrage:Voter ID:server-732678b9-5d78-6277-ead9-583e1a5b6906 Address:732678b9-5d78-6277-ead9-583e1a5b6906} {Suffrage:Voter ID:server-73deb0fe-f680-2157-d7c0-27cbc1546cf8 Address:73deb0fe-f680-2157-d7c0-27cbc1546cf8}]" +2025-06-15T03:17:11.620Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:11.620Z [INFO] server-73deb0fe-f680-2157-d7c0-27cbc1546cf8: entering follower state: follower="Node at 73deb0fe-f680-2157-d7c0-27cbc1546cf8 [Follower]" leader-address= leader-id= +2025-06-15T03:17:12.134Z [WARN] server-73deb0fe-f680-2157-d7c0-27cbc1546cf8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:12.134Z [INFO] server-73deb0fe-f680-2157-d7c0-27cbc1546cf8: entering candidate state: node="Node at 73deb0fe-f680-2157-d7c0-27cbc1546cf8 [Candidate]" term=2 +2025-06-15T03:17:12.134Z [INFO] server-73deb0fe-f680-2157-d7c0-27cbc1546cf8: election won: term=2 tally=2 +2025-06-15T03:17:12.134Z [INFO] server-73deb0fe-f680-2157-d7c0-27cbc1546cf8: entering leader state: leader="Node at 73deb0fe-f680-2157-d7c0-27cbc1546cf8 [Leader]" +2025-06-15T03:17:12.134Z [INFO] server-73deb0fe-f680-2157-d7c0-27cbc1546cf8: added peer, starting replication: peer=server-4cc8a21b-8ed2-64e2-f39f-0c5bb8ab48db +2025-06-15T03:17:12.134Z [INFO] server-73deb0fe-f680-2157-d7c0-27cbc1546cf8: added peer, starting replication: peer=server-732678b9-5d78-6277-ead9-583e1a5b6906 +2025-06-15T03:17:12.134Z [INFO] server-73deb0fe-f680-2157-d7c0-27cbc1546cf8: pipelining replication: peer="{Voter server-732678b9-5d78-6277-ead9-583e1a5b6906 732678b9-5d78-6277-ead9-583e1a5b6906}" +2025-06-15T03:17:12.134Z [INFO] server-73deb0fe-f680-2157-d7c0-27cbc1546cf8: pipelining replication: peer="{Voter server-4cc8a21b-8ed2-64e2-f39f-0c5bb8ab48db 4cc8a21b-8ed2-64e2-f39f-0c5bb8ab48db}" +2025-06-15T03:17:13.139Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:12.134898825 +0000 UTC m=+47.846576999 (1 nodes), 514.323698ms from start of poll, 514.62759ms from cluster start. Timeout at 2025-06-15 03:17:13.139951372 +0000 UTC m=+48.851629537, 1.005052538s after stability +2025-06-15T03:17:13.140Z [INFO] server-73deb0fe-f680-2157-d7c0-27cbc1546cf8: starting snapshot up to: index=12 +2025-06-15T03:17:13.140Z [INFO] creating new snapshot: path=/tmp/raft3325347766/snapshots/2-12-1749957433140.tmp +2025-06-15T03:17:13.141Z [INFO] server-73deb0fe-f680-2157-d7c0-27cbc1546cf8: snapshot complete up to: index=12 +2025-06-15T03:17:13.141Z [INFO] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e Address:c9b3e649-ddff-2cbc-ec6b-b753cadd263e} {Suffrage:Voter ID:server-f1157af3-9ce3-6134-e46b-d6d249760017 Address:f1157af3-9ce3-6134-e46b-d6d249760017} {Suffrage:Voter ID:server-cff828e2-26bc-6e46-0477-4463d81d1bd8 Address:cff828e2-26bc-6e46-0477-4463d81d1bd8}]" +2025-06-15T03:17:13.142Z [INFO] server-f1157af3-9ce3-6134-e46b-d6d249760017: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e Address:c9b3e649-ddff-2cbc-ec6b-b753cadd263e} {Suffrage:Voter ID:server-f1157af3-9ce3-6134-e46b-d6d249760017 Address:f1157af3-9ce3-6134-e46b-d6d249760017} {Suffrage:Voter ID:server-cff828e2-26bc-6e46-0477-4463d81d1bd8 Address:cff828e2-26bc-6e46-0477-4463d81d1bd8}]" +2025-06-15T03:17:13.142Z [INFO] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: entering follower state: follower="Node at c9b3e649-ddff-2cbc-ec6b-b753cadd263e [Follower]" leader-address= leader-id= +2025-06-15T03:17:13.142Z [INFO] server-f1157af3-9ce3-6134-e46b-d6d249760017: entering follower state: follower="Node at f1157af3-9ce3-6134-e46b-d6d249760017 [Follower]" leader-address= leader-id= +2025-06-15T03:17:13.142Z [INFO] server-cff828e2-26bc-6e46-0477-4463d81d1bd8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e Address:c9b3e649-ddff-2cbc-ec6b-b753cadd263e} {Suffrage:Voter ID:server-f1157af3-9ce3-6134-e46b-d6d249760017 Address:f1157af3-9ce3-6134-e46b-d6d249760017} {Suffrage:Voter ID:server-cff828e2-26bc-6e46-0477-4463d81d1bd8 Address:cff828e2-26bc-6e46-0477-4463d81d1bd8}]" +2025-06-15T03:17:13.142Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:13.142Z [INFO] server-cff828e2-26bc-6e46-0477-4463d81d1bd8: entering follower state: follower="Node at cff828e2-26bc-6e46-0477-4463d81d1bd8 [Follower]" leader-address= leader-id= +2025-06-15T03:17:13.756Z [WARN] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:13.756Z [INFO] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: entering candidate state: node="Node at c9b3e649-ddff-2cbc-ec6b-b753cadd263e [Candidate]" term=2 +2025-06-15T03:17:13.756Z [INFO] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: election won: term=2 tally=2 +2025-06-15T03:17:13.756Z [INFO] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: entering leader state: leader="Node at c9b3e649-ddff-2cbc-ec6b-b753cadd263e [Leader]" +2025-06-15T03:17:13.756Z [INFO] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: added peer, starting replication: peer=server-f1157af3-9ce3-6134-e46b-d6d249760017 +2025-06-15T03:17:13.756Z [INFO] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: added peer, starting replication: peer=server-cff828e2-26bc-6e46-0477-4463d81d1bd8 +2025-06-15T03:17:13.756Z [INFO] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: pipelining replication: peer="{Voter server-cff828e2-26bc-6e46-0477-4463d81d1bd8 cff828e2-26bc-6e46-0477-4463d81d1bd8}" +2025-06-15T03:17:13.756Z [INFO] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: pipelining replication: peer="{Voter server-f1157af3-9ce3-6134-e46b-d6d249760017 f1157af3-9ce3-6134-e46b-d6d249760017}" +2025-06-15T03:17:14.762Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:13.756583883 +0000 UTC m=+49.468262060 (1 nodes), 614.311768ms from start of poll, 614.676053ms from cluster start. Timeout at 2025-06-15 03:17:14.762160152 +0000 UTC m=+50.473838361, 1.005576301s after stability +2025-06-15T03:17:14.762Z [INFO] creating new snapshot: path=/tmp/raft1264475825/snapshots/2-15-1749957434762.tmp +2025-06-15T03:17:14.762Z [INFO] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: copied to local snapshot: bytes=71 +2025-06-15T03:17:14.763Z [INFO] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: snapshot restore progress: id=2-15-1749957434762 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:14.763Z [INFO] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: restored user snapshot: index=15 +2025-06-15T03:17:14.763Z [ERROR] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: failed to get log: index=3 error="log not found" +2025-06-15T03:17:14.763Z [INFO] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: aborting pipeline replication: peer="{Voter server-cff828e2-26bc-6e46-0477-4463d81d1bd8 cff828e2-26bc-6e46-0477-4463d81d1bd8}" +2025-06-15T03:17:14.763Z [ERROR] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: failed to get log: index=3 error="log not found" +2025-06-15T03:17:14.763Z [INFO] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: aborting pipeline replication: peer="{Voter server-f1157af3-9ce3-6134-e46b-d6d249760017 f1157af3-9ce3-6134-e46b-d6d249760017}" +2025-06-15T03:17:14.772Z [ERROR] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: failed to get log: index=3 error="log not found" +2025-06-15T03:17:14.772Z [ERROR] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: failed to get log: index=3 error="log not found" +2025-06-15T03:17:14.773Z [INFO] creating new snapshot: path=/tmp/raft4197713093/snapshots/2-15-1749957434773.tmp +2025-06-15T03:17:14.773Z [INFO] server-cff828e2-26bc-6e46-0477-4463d81d1bd8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:14.773Z [INFO] creating new snapshot: path=/tmp/raft1019522179/snapshots/2-15-1749957434773.tmp +2025-06-15T03:17:14.773Z [INFO] server-cff828e2-26bc-6e46-0477-4463d81d1bd8: copied to local snapshot: bytes=71 +2025-06-15T03:17:14.773Z [INFO] server-cff828e2-26bc-6e46-0477-4463d81d1bd8: snapshot restore progress: id=2-15-1749957434773 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:14.773Z [INFO] server-cff828e2-26bc-6e46-0477-4463d81d1bd8: Installed remote snapshot +2025-06-15T03:17:14.773Z [INFO] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: pipelining replication: peer="{Voter server-cff828e2-26bc-6e46-0477-4463d81d1bd8 cff828e2-26bc-6e46-0477-4463d81d1bd8}" +2025-06-15T03:17:14.773Z [INFO] server-f1157af3-9ce3-6134-e46b-d6d249760017: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:14.773Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:17:14.774Z [INFO] server-f1157af3-9ce3-6134-e46b-d6d249760017: copied to local snapshot: bytes=71 +2025-06-15T03:17:14.774Z [INFO] server-f1157af3-9ce3-6134-e46b-d6d249760017: snapshot restore progress: id=2-15-1749957434773 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:14.774Z [INFO] server-f1157af3-9ce3-6134-e46b-d6d249760017: Installed remote snapshot +2025-06-15T03:17:14.774Z [INFO] server-c9b3e649-ddff-2cbc-ec6b-b753cadd263e: pipelining replication: peer="{Voter server-f1157af3-9ce3-6134-e46b-d6d249760017 f1157af3-9ce3-6134-e46b-d6d249760017}" +2025-06-15T03:17:15.779Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:17:14.773790752 +0000 UTC m=+50.485468914 (2 nodes), 492ns from start of poll, 1.631882907s from cluster start. Timeout at 2025-06-15 03:17:15.779577123 +0000 UTC m=+51.491255325, 1.005786411s after stability +2025-06-15T03:17:15.791Z [INFO] server-73deb0fe-f680-2157-d7c0-27cbc1546cf8: aborting pipeline replication: peer="{Voter server-732678b9-5d78-6277-ead9-583e1a5b6906 732678b9-5d78-6277-ead9-583e1a5b6906}" +2025-06-15T03:17:15.791Z [INFO] server-73deb0fe-f680-2157-d7c0-27cbc1546cf8: aborting pipeline replication: peer="{Voter server-4cc8a21b-8ed2-64e2-f39f-0c5bb8ab48db 4cc8a21b-8ed2-64e2-f39f-0c5bb8ab48db}" === RUN TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true -2024-05-12T20:50:41.021Z [INFO] server-5124933c-3940-e51a-0433-7e1672ff3c93: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5124933c-3940-e51a-0433-7e1672ff3c93 Address:5124933c-3940-e51a-0433-7e1672ff3c93} {Suffrage:Voter ID:server-2d0a11f7-6f35-20a3-b625-25043eccdedf Address:2d0a11f7-6f35-20a3-b625-25043eccdedf} {Suffrage:Voter ID:server-99cafbb5-fe75-ef9f-69e6-c6c0cc7dc18f Address:99cafbb5-fe75-ef9f-69e6-c6c0cc7dc18f}]" -2024-05-12T20:50:41.021Z [INFO] server-2d0a11f7-6f35-20a3-b625-25043eccdedf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5124933c-3940-e51a-0433-7e1672ff3c93 Address:5124933c-3940-e51a-0433-7e1672ff3c93} {Suffrage:Voter ID:server-2d0a11f7-6f35-20a3-b625-25043eccdedf Address:2d0a11f7-6f35-20a3-b625-25043eccdedf} {Suffrage:Voter ID:server-99cafbb5-fe75-ef9f-69e6-c6c0cc7dc18f Address:99cafbb5-fe75-ef9f-69e6-c6c0cc7dc18f}]" -2024-05-12T20:50:41.022Z [INFO] server-99cafbb5-fe75-ef9f-69e6-c6c0cc7dc18f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5124933c-3940-e51a-0433-7e1672ff3c93 Address:5124933c-3940-e51a-0433-7e1672ff3c93} {Suffrage:Voter ID:server-2d0a11f7-6f35-20a3-b625-25043eccdedf Address:2d0a11f7-6f35-20a3-b625-25043eccdedf} {Suffrage:Voter ID:server-99cafbb5-fe75-ef9f-69e6-c6c0cc7dc18f Address:99cafbb5-fe75-ef9f-69e6-c6c0cc7dc18f}]" -2024-05-12T20:50:41.022Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:41.022Z [INFO] server-5124933c-3940-e51a-0433-7e1672ff3c93: entering follower state: follower="Node at 5124933c-3940-e51a-0433-7e1672ff3c93 [Follower]" leader-address= leader-id= -2024-05-12T20:50:41.022Z [INFO] server-2d0a11f7-6f35-20a3-b625-25043eccdedf: entering follower state: follower="Node at 2d0a11f7-6f35-20a3-b625-25043eccdedf [Follower]" leader-address= leader-id= -2024-05-12T20:50:41.022Z [INFO] server-99cafbb5-fe75-ef9f-69e6-c6c0cc7dc18f: entering follower state: follower="Node at 99cafbb5-fe75-ef9f-69e6-c6c0cc7dc18f [Follower]" leader-address= leader-id= -2024-05-12T20:50:41.619Z [WARN] server-2d0a11f7-6f35-20a3-b625-25043eccdedf: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:41.619Z [INFO] server-2d0a11f7-6f35-20a3-b625-25043eccdedf: entering candidate state: node="Node at 2d0a11f7-6f35-20a3-b625-25043eccdedf [Candidate]" term=2 -2024-05-12T20:50:41.619Z [INFO] server-2d0a11f7-6f35-20a3-b625-25043eccdedf: election won: term=2 tally=2 -2024-05-12T20:50:41.619Z [INFO] server-2d0a11f7-6f35-20a3-b625-25043eccdedf: entering leader state: leader="Node at 2d0a11f7-6f35-20a3-b625-25043eccdedf [Leader]" -2024-05-12T20:50:41.619Z [INFO] server-2d0a11f7-6f35-20a3-b625-25043eccdedf: added peer, starting replication: peer=server-5124933c-3940-e51a-0433-7e1672ff3c93 -2024-05-12T20:50:41.619Z [INFO] server-2d0a11f7-6f35-20a3-b625-25043eccdedf: added peer, starting replication: peer=server-99cafbb5-fe75-ef9f-69e6-c6c0cc7dc18f -2024-05-12T20:50:41.619Z [INFO] server-2d0a11f7-6f35-20a3-b625-25043eccdedf: pipelining replication: peer="{Voter server-99cafbb5-fe75-ef9f-69e6-c6c0cc7dc18f 99cafbb5-fe75-ef9f-69e6-c6c0cc7dc18f}" -2024-05-12T20:50:41.619Z [INFO] server-2d0a11f7-6f35-20a3-b625-25043eccdedf: pipelining replication: peer="{Voter server-5124933c-3940-e51a-0433-7e1672ff3c93 5124933c-3940-e51a-0433-7e1672ff3c93}" -2024-05-12T20:50:42.626Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:41.6200692 +0000 UTC m=+52.900769685 (1 nodes), 597.932772ms from start of poll, 598.525679ms from cluster start. Timeout at 2024-05-12 20:50:42.626835119 +0000 UTC m=+53.907535526, 1.006765841s after stability -2024-05-12T20:50:42.628Z [INFO] server-2d0a11f7-6f35-20a3-b625-25043eccdedf: starting snapshot up to: index=12 -2024-05-12T20:50:42.628Z [INFO] creating new snapshot: path=/tmp/raft2745222846/snapshots/2-12-1715547042628.tmp -2024-05-12T20:50:42.629Z [INFO] server-2d0a11f7-6f35-20a3-b625-25043eccdedf: snapshot complete up to: index=12 -2024-05-12T20:50:42.632Z [INFO] server-8fd55d55-6330-3505-a75f-6480bd31c298: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8fd55d55-6330-3505-a75f-6480bd31c298 Address:8fd55d55-6330-3505-a75f-6480bd31c298} {Suffrage:Voter ID:server-9126ac4f-e093-0757-b530-b351cab689d8 Address:9126ac4f-e093-0757-b530-b351cab689d8} {Suffrage:Voter ID:server-87365a08-9c32-db26-2205-c549f4d60bd7 Address:87365a08-9c32-db26-2205-c549f4d60bd7}]" -2024-05-12T20:50:42.632Z [INFO] server-9126ac4f-e093-0757-b530-b351cab689d8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8fd55d55-6330-3505-a75f-6480bd31c298 Address:8fd55d55-6330-3505-a75f-6480bd31c298} {Suffrage:Voter ID:server-9126ac4f-e093-0757-b530-b351cab689d8 Address:9126ac4f-e093-0757-b530-b351cab689d8} {Suffrage:Voter ID:server-87365a08-9c32-db26-2205-c549f4d60bd7 Address:87365a08-9c32-db26-2205-c549f4d60bd7}]" -2024-05-12T20:50:42.633Z [INFO] server-87365a08-9c32-db26-2205-c549f4d60bd7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8fd55d55-6330-3505-a75f-6480bd31c298 Address:8fd55d55-6330-3505-a75f-6480bd31c298} {Suffrage:Voter ID:server-9126ac4f-e093-0757-b530-b351cab689d8 Address:9126ac4f-e093-0757-b530-b351cab689d8} {Suffrage:Voter ID:server-87365a08-9c32-db26-2205-c549f4d60bd7 Address:87365a08-9c32-db26-2205-c549f4d60bd7}]" -2024-05-12T20:50:42.633Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:42.633Z [INFO] server-8fd55d55-6330-3505-a75f-6480bd31c298: entering follower state: follower="Node at 8fd55d55-6330-3505-a75f-6480bd31c298 [Follower]" leader-address= leader-id= -2024-05-12T20:50:42.633Z [INFO] server-9126ac4f-e093-0757-b530-b351cab689d8: entering follower state: follower="Node at 9126ac4f-e093-0757-b530-b351cab689d8 [Follower]" leader-address= leader-id= -2024-05-12T20:50:42.633Z [INFO] server-87365a08-9c32-db26-2205-c549f4d60bd7: entering follower state: follower="Node at 87365a08-9c32-db26-2205-c549f4d60bd7 [Follower]" leader-address= leader-id= -2024-05-12T20:50:43.438Z [WARN] server-87365a08-9c32-db26-2205-c549f4d60bd7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:43.438Z [INFO] server-87365a08-9c32-db26-2205-c549f4d60bd7: entering candidate state: node="Node at 87365a08-9c32-db26-2205-c549f4d60bd7 [Candidate]" term=2 -2024-05-12T20:50:43.439Z [INFO] server-87365a08-9c32-db26-2205-c549f4d60bd7: election won: term=2 tally=2 -2024-05-12T20:50:43.439Z [INFO] server-87365a08-9c32-db26-2205-c549f4d60bd7: entering leader state: leader="Node at 87365a08-9c32-db26-2205-c549f4d60bd7 [Leader]" -2024-05-12T20:50:43.439Z [INFO] server-87365a08-9c32-db26-2205-c549f4d60bd7: added peer, starting replication: peer=server-8fd55d55-6330-3505-a75f-6480bd31c298 -2024-05-12T20:50:43.439Z [INFO] server-87365a08-9c32-db26-2205-c549f4d60bd7: added peer, starting replication: peer=server-9126ac4f-e093-0757-b530-b351cab689d8 -2024-05-12T20:50:43.439Z [INFO] server-87365a08-9c32-db26-2205-c549f4d60bd7: pipelining replication: peer="{Voter server-8fd55d55-6330-3505-a75f-6480bd31c298 8fd55d55-6330-3505-a75f-6480bd31c298}" -2024-05-12T20:50:43.439Z [INFO] server-87365a08-9c32-db26-2205-c549f4d60bd7: pipelining replication: peer="{Voter server-9126ac4f-e093-0757-b530-b351cab689d8 9126ac4f-e093-0757-b530-b351cab689d8}" -2024-05-12T20:50:44.446Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:43.439215993 +0000 UTC m=+54.719915888 (1 nodes), 806.10809ms from start of poll, 806.511316ms from cluster start. Timeout at 2024-05-12 20:50:44.44691887 +0000 UTC m=+55.727619336, 1.007703448s after stability -2024-05-12T20:50:44.447Z [INFO] creating new snapshot: path=/tmp/raft15746642/snapshots/2-15-1715547044447.tmp -2024-05-12T20:50:44.448Z [INFO] server-87365a08-9c32-db26-2205-c549f4d60bd7: copied to local snapshot: bytes=71 -2024-05-12T20:50:44.449Z [INFO] server-87365a08-9c32-db26-2205-c549f4d60bd7: snapshot restore progress: id=2-15-1715547044447 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:44.449Z [INFO] server-87365a08-9c32-db26-2205-c549f4d60bd7: removing all old logs from log store -2024-05-12T20:50:44.449Z [INFO] server-87365a08-9c32-db26-2205-c549f4d60bd7: compacting logs: from=1 to=2 -2024-05-12T20:50:44.449Z [INFO] server-87365a08-9c32-db26-2205-c549f4d60bd7: restored user snapshot: index=15 -2024-05-12T20:50:44.449Z [ERROR] server-87365a08-9c32-db26-2205-c549f4d60bd7: failed to get log: index=2 error="log not found" -2024-05-12T20:50:44.449Z [INFO] server-87365a08-9c32-db26-2205-c549f4d60bd7: aborting pipeline replication: peer="{Voter server-9126ac4f-e093-0757-b530-b351cab689d8 9126ac4f-e093-0757-b530-b351cab689d8}" -2024-05-12T20:50:44.449Z [ERROR] server-87365a08-9c32-db26-2205-c549f4d60bd7: failed to get log: index=2 error="log not found" -2024-05-12T20:50:44.449Z [INFO] server-87365a08-9c32-db26-2205-c549f4d60bd7: aborting pipeline replication: peer="{Voter server-8fd55d55-6330-3505-a75f-6480bd31c298 8fd55d55-6330-3505-a75f-6480bd31c298}" -2024-05-12T20:50:44.454Z [ERROR] server-87365a08-9c32-db26-2205-c549f4d60bd7: failed to get log: index=2 error="log not found" -2024-05-12T20:50:44.455Z [ERROR] server-87365a08-9c32-db26-2205-c549f4d60bd7: failed to get log: index=2 error="log not found" -2024-05-12T20:50:44.455Z [INFO] creating new snapshot: path=/tmp/raft2273351729/snapshots/2-15-1715547044455.tmp -2024-05-12T20:50:44.455Z [INFO] creating new snapshot: path=/tmp/raft3403710562/snapshots/2-15-1715547044455.tmp -2024-05-12T20:50:44.456Z [INFO] server-9126ac4f-e093-0757-b530-b351cab689d8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:44.456Z [INFO] server-8fd55d55-6330-3505-a75f-6480bd31c298: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:44.456Z [INFO] server-8fd55d55-6330-3505-a75f-6480bd31c298: copied to local snapshot: bytes=71 -2024-05-12T20:50:44.457Z [INFO] server-8fd55d55-6330-3505-a75f-6480bd31c298: snapshot restore progress: id=2-15-1715547044455 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:44.457Z [INFO] server-8fd55d55-6330-3505-a75f-6480bd31c298: removing all old logs from log store -2024-05-12T20:50:44.457Z [INFO] server-8fd55d55-6330-3505-a75f-6480bd31c298: compacting logs: from=1 to=2 -2024-05-12T20:50:44.457Z [INFO] server-8fd55d55-6330-3505-a75f-6480bd31c298: Installed remote snapshot -2024-05-12T20:50:44.457Z [INFO] server-9126ac4f-e093-0757-b530-b351cab689d8: copied to local snapshot: bytes=71 -2024-05-12T20:50:44.457Z [INFO] server-87365a08-9c32-db26-2205-c549f4d60bd7: pipelining replication: peer="{Voter server-8fd55d55-6330-3505-a75f-6480bd31c298 8fd55d55-6330-3505-a75f-6480bd31c298}" -2024-05-12T20:50:44.457Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:44.457Z [INFO] server-9126ac4f-e093-0757-b530-b351cab689d8: snapshot restore progress: id=2-15-1715547044455 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:44.457Z [INFO] server-9126ac4f-e093-0757-b530-b351cab689d8: removing all old logs from log store -2024-05-12T20:50:44.457Z [INFO] server-9126ac4f-e093-0757-b530-b351cab689d8: compacting logs: from=1 to=2 -2024-05-12T20:50:44.457Z [INFO] server-9126ac4f-e093-0757-b530-b351cab689d8: Installed remote snapshot -2024-05-12T20:50:44.458Z [INFO] server-87365a08-9c32-db26-2205-c549f4d60bd7: pipelining replication: peer="{Voter server-9126ac4f-e093-0757-b530-b351cab689d8 9126ac4f-e093-0757-b530-b351cab689d8}" -2024-05-12T20:50:45.467Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:44.457630881 +0000 UTC m=+55.738331255 (2 nodes), 2.457µs from start of poll, 1.824926683s from cluster start. Timeout at 2024-05-12 20:50:45.466977474 +0000 UTC m=+56.747677819, 1.009346564s after stability -2024-05-12T20:50:45.487Z [INFO] server-2d0a11f7-6f35-20a3-b625-25043eccdedf: aborting pipeline replication: peer="{Voter server-99cafbb5-fe75-ef9f-69e6-c6c0cc7dc18f 99cafbb5-fe75-ef9f-69e6-c6c0cc7dc18f}" -2024-05-12T20:50:45.487Z [INFO] server-2d0a11f7-6f35-20a3-b625-25043eccdedf: aborting pipeline replication: peer="{Voter server-5124933c-3940-e51a-0433-7e1672ff3c93 5124933c-3940-e51a-0433-7e1672ff3c93}" +2025-06-15T03:17:15.792Z [INFO] server-46144aee-113e-8a30-d94b-3aa9e3037811: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-46144aee-113e-8a30-d94b-3aa9e3037811 Address:46144aee-113e-8a30-d94b-3aa9e3037811} {Suffrage:Voter ID:server-09d8a710-4d6b-6ab9-b66c-f36d15953574 Address:09d8a710-4d6b-6ab9-b66c-f36d15953574} {Suffrage:Voter ID:server-00dc7775-2a8f-cc80-3b92-0bc3d40f435f Address:00dc7775-2a8f-cc80-3b92-0bc3d40f435f}]" +2025-06-15T03:17:15.792Z [INFO] server-46144aee-113e-8a30-d94b-3aa9e3037811: entering follower state: follower="Node at 46144aee-113e-8a30-d94b-3aa9e3037811 [Follower]" leader-address= leader-id= +2025-06-15T03:17:15.792Z [INFO] server-09d8a710-4d6b-6ab9-b66c-f36d15953574: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-46144aee-113e-8a30-d94b-3aa9e3037811 Address:46144aee-113e-8a30-d94b-3aa9e3037811} {Suffrage:Voter ID:server-09d8a710-4d6b-6ab9-b66c-f36d15953574 Address:09d8a710-4d6b-6ab9-b66c-f36d15953574} {Suffrage:Voter ID:server-00dc7775-2a8f-cc80-3b92-0bc3d40f435f Address:00dc7775-2a8f-cc80-3b92-0bc3d40f435f}]" +2025-06-15T03:17:15.792Z [INFO] server-09d8a710-4d6b-6ab9-b66c-f36d15953574: entering follower state: follower="Node at 09d8a710-4d6b-6ab9-b66c-f36d15953574 [Follower]" leader-address= leader-id= +2025-06-15T03:17:15.792Z [INFO] server-00dc7775-2a8f-cc80-3b92-0bc3d40f435f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-46144aee-113e-8a30-d94b-3aa9e3037811 Address:46144aee-113e-8a30-d94b-3aa9e3037811} {Suffrage:Voter ID:server-09d8a710-4d6b-6ab9-b66c-f36d15953574 Address:09d8a710-4d6b-6ab9-b66c-f36d15953574} {Suffrage:Voter ID:server-00dc7775-2a8f-cc80-3b92-0bc3d40f435f Address:00dc7775-2a8f-cc80-3b92-0bc3d40f435f}]" +2025-06-15T03:17:15.792Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:15.792Z [INFO] server-00dc7775-2a8f-cc80-3b92-0bc3d40f435f: entering follower state: follower="Node at 00dc7775-2a8f-cc80-3b92-0bc3d40f435f [Follower]" leader-address= leader-id= +2025-06-15T03:17:16.349Z [WARN] server-09d8a710-4d6b-6ab9-b66c-f36d15953574: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:16.349Z [INFO] server-09d8a710-4d6b-6ab9-b66c-f36d15953574: entering candidate state: node="Node at 09d8a710-4d6b-6ab9-b66c-f36d15953574 [Candidate]" term=2 +2025-06-15T03:17:16.349Z [INFO] server-09d8a710-4d6b-6ab9-b66c-f36d15953574: election won: term=2 tally=2 +2025-06-15T03:17:16.349Z [INFO] server-09d8a710-4d6b-6ab9-b66c-f36d15953574: entering leader state: leader="Node at 09d8a710-4d6b-6ab9-b66c-f36d15953574 [Leader]" +2025-06-15T03:17:16.349Z [INFO] server-09d8a710-4d6b-6ab9-b66c-f36d15953574: added peer, starting replication: peer=server-46144aee-113e-8a30-d94b-3aa9e3037811 +2025-06-15T03:17:16.349Z [INFO] server-09d8a710-4d6b-6ab9-b66c-f36d15953574: added peer, starting replication: peer=server-00dc7775-2a8f-cc80-3b92-0bc3d40f435f +2025-06-15T03:17:16.350Z [INFO] server-09d8a710-4d6b-6ab9-b66c-f36d15953574: pipelining replication: peer="{Voter server-46144aee-113e-8a30-d94b-3aa9e3037811 46144aee-113e-8a30-d94b-3aa9e3037811}" +2025-06-15T03:17:16.350Z [INFO] server-09d8a710-4d6b-6ab9-b66c-f36d15953574: pipelining replication: peer="{Voter server-00dc7775-2a8f-cc80-3b92-0bc3d40f435f 00dc7775-2a8f-cc80-3b92-0bc3d40f435f}" +2025-06-15T03:17:17.355Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:16.349991652 +0000 UTC m=+52.061669817 (1 nodes), 557.140883ms from start of poll, 557.411038ms from cluster start. Timeout at 2025-06-15 03:17:17.355474362 +0000 UTC m=+53.067152561, 1.005482744s after stability +2025-06-15T03:17:17.355Z [INFO] server-09d8a710-4d6b-6ab9-b66c-f36d15953574: starting snapshot up to: index=12 +2025-06-15T03:17:17.355Z [INFO] creating new snapshot: path=/tmp/raft3168215811/snapshots/2-12-1749957437355.tmp +2025-06-15T03:17:17.356Z [INFO] server-09d8a710-4d6b-6ab9-b66c-f36d15953574: snapshot complete up to: index=12 +2025-06-15T03:17:17.357Z [INFO] server-fb11e60e-1261-8487-6adf-d632f03e38b6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fb11e60e-1261-8487-6adf-d632f03e38b6 Address:fb11e60e-1261-8487-6adf-d632f03e38b6} {Suffrage:Voter ID:server-fe17d037-f4b7-ef8b-94a1-a77b997965cf Address:fe17d037-f4b7-ef8b-94a1-a77b997965cf} {Suffrage:Voter ID:server-3eb0d601-b921-0dc2-ad66-2ea69b33c790 Address:3eb0d601-b921-0dc2-ad66-2ea69b33c790}]" +2025-06-15T03:17:17.357Z [INFO] server-fb11e60e-1261-8487-6adf-d632f03e38b6: entering follower state: follower="Node at fb11e60e-1261-8487-6adf-d632f03e38b6 [Follower]" leader-address= leader-id= +2025-06-15T03:17:17.357Z [INFO] server-fe17d037-f4b7-ef8b-94a1-a77b997965cf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fb11e60e-1261-8487-6adf-d632f03e38b6 Address:fb11e60e-1261-8487-6adf-d632f03e38b6} {Suffrage:Voter ID:server-fe17d037-f4b7-ef8b-94a1-a77b997965cf Address:fe17d037-f4b7-ef8b-94a1-a77b997965cf} {Suffrage:Voter ID:server-3eb0d601-b921-0dc2-ad66-2ea69b33c790 Address:3eb0d601-b921-0dc2-ad66-2ea69b33c790}]" +2025-06-15T03:17:17.357Z [INFO] server-fe17d037-f4b7-ef8b-94a1-a77b997965cf: entering follower state: follower="Node at fe17d037-f4b7-ef8b-94a1-a77b997965cf [Follower]" leader-address= leader-id= +2025-06-15T03:17:17.357Z [INFO] server-3eb0d601-b921-0dc2-ad66-2ea69b33c790: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fb11e60e-1261-8487-6adf-d632f03e38b6 Address:fb11e60e-1261-8487-6adf-d632f03e38b6} {Suffrage:Voter ID:server-fe17d037-f4b7-ef8b-94a1-a77b997965cf Address:fe17d037-f4b7-ef8b-94a1-a77b997965cf} {Suffrage:Voter ID:server-3eb0d601-b921-0dc2-ad66-2ea69b33c790 Address:3eb0d601-b921-0dc2-ad66-2ea69b33c790}]" +2025-06-15T03:17:17.357Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:17.357Z [INFO] server-3eb0d601-b921-0dc2-ad66-2ea69b33c790: entering follower state: follower="Node at 3eb0d601-b921-0dc2-ad66-2ea69b33c790 [Follower]" leader-address= leader-id= +2025-06-15T03:17:18.038Z [WARN] server-fb11e60e-1261-8487-6adf-d632f03e38b6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:18.038Z [INFO] server-fb11e60e-1261-8487-6adf-d632f03e38b6: entering candidate state: node="Node at fb11e60e-1261-8487-6adf-d632f03e38b6 [Candidate]" term=2 +2025-06-15T03:17:18.038Z [INFO] server-fb11e60e-1261-8487-6adf-d632f03e38b6: election won: term=2 tally=2 +2025-06-15T03:17:18.038Z [INFO] server-fb11e60e-1261-8487-6adf-d632f03e38b6: entering leader state: leader="Node at fb11e60e-1261-8487-6adf-d632f03e38b6 [Leader]" +2025-06-15T03:17:18.038Z [INFO] server-fb11e60e-1261-8487-6adf-d632f03e38b6: added peer, starting replication: peer=server-fe17d037-f4b7-ef8b-94a1-a77b997965cf +2025-06-15T03:17:18.038Z [INFO] server-fb11e60e-1261-8487-6adf-d632f03e38b6: added peer, starting replication: peer=server-3eb0d601-b921-0dc2-ad66-2ea69b33c790 +2025-06-15T03:17:18.039Z [INFO] server-fb11e60e-1261-8487-6adf-d632f03e38b6: pipelining replication: peer="{Voter server-fe17d037-f4b7-ef8b-94a1-a77b997965cf fe17d037-f4b7-ef8b-94a1-a77b997965cf}" +2025-06-15T03:17:18.039Z [INFO] server-fb11e60e-1261-8487-6adf-d632f03e38b6: pipelining replication: peer="{Voter server-3eb0d601-b921-0dc2-ad66-2ea69b33c790 3eb0d601-b921-0dc2-ad66-2ea69b33c790}" +2025-06-15T03:17:19.044Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:18.039054734 +0000 UTC m=+53.750732901 (1 nodes), 681.703754ms from start of poll, 681.953084ms from cluster start. Timeout at 2025-06-15 03:17:19.044410899 +0000 UTC m=+54.756089098, 1.005356197s after stability +2025-06-15T03:17:19.044Z [INFO] creating new snapshot: path=/tmp/raft3616613904/snapshots/2-15-1749957439044.tmp +2025-06-15T03:17:19.045Z [INFO] server-fb11e60e-1261-8487-6adf-d632f03e38b6: copied to local snapshot: bytes=71 +2025-06-15T03:17:19.045Z [INFO] server-fb11e60e-1261-8487-6adf-d632f03e38b6: snapshot restore progress: id=2-15-1749957439044 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:19.045Z [INFO] server-fb11e60e-1261-8487-6adf-d632f03e38b6: removing all old logs from log store +2025-06-15T03:17:19.045Z [INFO] server-fb11e60e-1261-8487-6adf-d632f03e38b6: compacting logs: from=1 to=2 +2025-06-15T03:17:19.045Z [INFO] server-fb11e60e-1261-8487-6adf-d632f03e38b6: restored user snapshot: index=15 +2025-06-15T03:17:19.045Z [ERROR] server-fb11e60e-1261-8487-6adf-d632f03e38b6: failed to get log: index=2 error="log not found" +2025-06-15T03:17:19.045Z [INFO] server-fb11e60e-1261-8487-6adf-d632f03e38b6: aborting pipeline replication: peer="{Voter server-fe17d037-f4b7-ef8b-94a1-a77b997965cf fe17d037-f4b7-ef8b-94a1-a77b997965cf}" +2025-06-15T03:17:19.045Z [ERROR] server-fb11e60e-1261-8487-6adf-d632f03e38b6: failed to get log: index=2 error="log not found" +2025-06-15T03:17:19.045Z [INFO] server-fb11e60e-1261-8487-6adf-d632f03e38b6: aborting pipeline replication: peer="{Voter server-3eb0d601-b921-0dc2-ad66-2ea69b33c790 3eb0d601-b921-0dc2-ad66-2ea69b33c790}" +2025-06-15T03:17:19.051Z [ERROR] server-fb11e60e-1261-8487-6adf-d632f03e38b6: failed to get log: index=2 error="log not found" +2025-06-15T03:17:19.051Z [INFO] creating new snapshot: path=/tmp/raft406189975/snapshots/2-15-1749957439051.tmp +2025-06-15T03:17:19.051Z [ERROR] server-fb11e60e-1261-8487-6adf-d632f03e38b6: failed to get log: index=2 error="log not found" +2025-06-15T03:17:19.052Z [INFO] server-3eb0d601-b921-0dc2-ad66-2ea69b33c790: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:19.052Z [INFO] creating new snapshot: path=/tmp/raft445379024/snapshots/2-15-1749957439052.tmp +2025-06-15T03:17:19.052Z [INFO] server-3eb0d601-b921-0dc2-ad66-2ea69b33c790: copied to local snapshot: bytes=71 +2025-06-15T03:17:19.052Z [INFO] server-3eb0d601-b921-0dc2-ad66-2ea69b33c790: snapshot restore progress: id=2-15-1749957439051 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:19.052Z [INFO] server-3eb0d601-b921-0dc2-ad66-2ea69b33c790: removing all old logs from log store +2025-06-15T03:17:19.052Z [INFO] server-3eb0d601-b921-0dc2-ad66-2ea69b33c790: compacting logs: from=1 to=2 +2025-06-15T03:17:19.052Z [INFO] server-3eb0d601-b921-0dc2-ad66-2ea69b33c790: Installed remote snapshot +2025-06-15T03:17:19.052Z [INFO] server-fb11e60e-1261-8487-6adf-d632f03e38b6: pipelining replication: peer="{Voter server-3eb0d601-b921-0dc2-ad66-2ea69b33c790 3eb0d601-b921-0dc2-ad66-2ea69b33c790}" +2025-06-15T03:17:19.052Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:17:19.052Z [INFO] server-fe17d037-f4b7-ef8b-94a1-a77b997965cf: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:19.052Z [INFO] server-fe17d037-f4b7-ef8b-94a1-a77b997965cf: copied to local snapshot: bytes=71 +2025-06-15T03:17:19.052Z [INFO] server-fe17d037-f4b7-ef8b-94a1-a77b997965cf: snapshot restore progress: id=2-15-1749957439052 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:19.052Z [INFO] server-fe17d037-f4b7-ef8b-94a1-a77b997965cf: removing all old logs from log store +2025-06-15T03:17:19.052Z [INFO] server-fe17d037-f4b7-ef8b-94a1-a77b997965cf: compacting logs: from=1 to=2 +2025-06-15T03:17:19.052Z [INFO] server-fe17d037-f4b7-ef8b-94a1-a77b997965cf: Installed remote snapshot +2025-06-15T03:17:19.053Z [INFO] server-fb11e60e-1261-8487-6adf-d632f03e38b6: pipelining replication: peer="{Voter server-fe17d037-f4b7-ef8b-94a1-a77b997965cf fe17d037-f4b7-ef8b-94a1-a77b997965cf}" +2025-06-15T03:17:20.057Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:17:19.052577995 +0000 UTC m=+54.764256160 (2 nodes), 607ns from start of poll, 1.695476343s from cluster start. Timeout at 2025-06-15 03:17:20.057904558 +0000 UTC m=+55.769582757, 1.005326597s after stability +2025-06-15T03:17:20.070Z [INFO] server-09d8a710-4d6b-6ab9-b66c-f36d15953574: aborting pipeline replication: peer="{Voter server-00dc7775-2a8f-cc80-3b92-0bc3d40f435f 00dc7775-2a8f-cc80-3b92-0bc3d40f435f}" +2025-06-15T03:17:20.070Z [INFO] server-09d8a710-4d6b-6ab9-b66c-f36d15953574: aborting pipeline replication: peer="{Voter server-46144aee-113e-8a30-d94b-3aa9e3037811 46144aee-113e-8a30-d94b-3aa9e3037811}" === RUN TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false -2024-05-12T20:50:45.489Z [INFO] server-289a76b4-3ba1-b059-0911-8a066690e34f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-289a76b4-3ba1-b059-0911-8a066690e34f Address:289a76b4-3ba1-b059-0911-8a066690e34f} {Suffrage:Voter ID:server-6a11da2d-c315-93c2-f616-c06e2146e09c Address:6a11da2d-c315-93c2-f616-c06e2146e09c} {Suffrage:Voter ID:server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e Address:1fa81adc-4098-33de-dfbe-29f8c09b3e8e}]" -2024-05-12T20:50:45.490Z [INFO] server-6a11da2d-c315-93c2-f616-c06e2146e09c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-289a76b4-3ba1-b059-0911-8a066690e34f Address:289a76b4-3ba1-b059-0911-8a066690e34f} {Suffrage:Voter ID:server-6a11da2d-c315-93c2-f616-c06e2146e09c Address:6a11da2d-c315-93c2-f616-c06e2146e09c} {Suffrage:Voter ID:server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e Address:1fa81adc-4098-33de-dfbe-29f8c09b3e8e}]" -2024-05-12T20:50:45.490Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-289a76b4-3ba1-b059-0911-8a066690e34f Address:289a76b4-3ba1-b059-0911-8a066690e34f} {Suffrage:Voter ID:server-6a11da2d-c315-93c2-f616-c06e2146e09c Address:6a11da2d-c315-93c2-f616-c06e2146e09c} {Suffrage:Voter ID:server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e Address:1fa81adc-4098-33de-dfbe-29f8c09b3e8e}]" -2024-05-12T20:50:45.490Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:45.490Z [INFO] server-289a76b4-3ba1-b059-0911-8a066690e34f: entering follower state: follower="Node at 289a76b4-3ba1-b059-0911-8a066690e34f [Follower]" leader-address= leader-id= -2024-05-12T20:50:45.490Z [INFO] server-6a11da2d-c315-93c2-f616-c06e2146e09c: entering follower state: follower="Node at 6a11da2d-c315-93c2-f616-c06e2146e09c [Follower]" leader-address= leader-id= -2024-05-12T20:50:45.490Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: entering follower state: follower="Node at 1fa81adc-4098-33de-dfbe-29f8c09b3e8e [Follower]" leader-address= leader-id= -2024-05-12T20:50:45.995Z [WARN] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:45.995Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: entering candidate state: node="Node at 1fa81adc-4098-33de-dfbe-29f8c09b3e8e [Candidate]" term=2 -2024-05-12T20:50:45.995Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: election won: term=2 tally=2 -2024-05-12T20:50:45.995Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: entering leader state: leader="Node at 1fa81adc-4098-33de-dfbe-29f8c09b3e8e [Leader]" -2024-05-12T20:50:45.995Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: added peer, starting replication: peer=server-289a76b4-3ba1-b059-0911-8a066690e34f -2024-05-12T20:50:45.995Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: added peer, starting replication: peer=server-6a11da2d-c315-93c2-f616-c06e2146e09c -2024-05-12T20:50:45.995Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: pipelining replication: peer="{Voter server-6a11da2d-c315-93c2-f616-c06e2146e09c 6a11da2d-c315-93c2-f616-c06e2146e09c}" -2024-05-12T20:50:45.996Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: pipelining replication: peer="{Voter server-289a76b4-3ba1-b059-0911-8a066690e34f 289a76b4-3ba1-b059-0911-8a066690e34f}" -2024-05-12T20:50:47.016Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:45.996296234 +0000 UTC m=+57.276996643 (1 nodes), 506.017239ms from start of poll, 506.520523ms from cluster start. Timeout at 2024-05-12 20:50:47.002817242 +0000 UTC m=+58.283517137, 1.006520494s after stability -2024-05-12T20:50:47.017Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: starting snapshot up to: index=12 -2024-05-12T20:50:47.017Z [INFO] creating new snapshot: path=/tmp/raft3733328066/snapshots/2-12-1715547047017.tmp -2024-05-12T20:50:47.018Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: snapshot complete up to: index=12 -2024-05-12T20:50:47.019Z [INFO] creating new snapshot: path=/tmp/raft3733328066/snapshots/2-113-1715547047019.tmp -2024-05-12T20:50:47.020Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: copied to local snapshot: bytes=71 -2024-05-12T20:50:47.021Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: snapshot restore progress: id=2-113-1715547047019 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:47.021Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: restored user snapshot: index=113 -2024-05-12T20:50:47.021Z [ERROR] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: failed to get log: index=23 error="log not found" -2024-05-12T20:50:47.021Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: aborting pipeline replication: peer="{Voter server-6a11da2d-c315-93c2-f616-c06e2146e09c 6a11da2d-c315-93c2-f616-c06e2146e09c}" -2024-05-12T20:50:47.021Z [ERROR] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: failed to get log: index=23 error="log not found" -2024-05-12T20:50:47.021Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: aborting pipeline replication: peer="{Voter server-289a76b4-3ba1-b059-0911-8a066690e34f 289a76b4-3ba1-b059-0911-8a066690e34f}" -2024-05-12T20:50:47.030Z [ERROR] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: failed to get log: index=23 error="log not found" -2024-05-12T20:50:47.031Z [INFO] creating new snapshot: path=/tmp/raft1941444323/snapshots/2-113-1715547047031.tmp -2024-05-12T20:50:47.030Z [ERROR] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: failed to get log: index=23 error="log not found" -2024-05-12T20:50:47.031Z [INFO] server-289a76b4-3ba1-b059-0911-8a066690e34f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:47.032Z [INFO] creating new snapshot: path=/tmp/raft799422700/snapshots/2-113-1715547047032.tmp -2024-05-12T20:50:47.032Z [INFO] server-6a11da2d-c315-93c2-f616-c06e2146e09c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:47.032Z [INFO] server-289a76b4-3ba1-b059-0911-8a066690e34f: copied to local snapshot: bytes=71 -2024-05-12T20:50:47.033Z [INFO] server-289a76b4-3ba1-b059-0911-8a066690e34f: snapshot restore progress: id=2-113-1715547047031 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:47.033Z [INFO] server-289a76b4-3ba1-b059-0911-8a066690e34f: Installed remote snapshot -2024-05-12T20:50:47.033Z [WARN] server-289a76b4-3ba1-b059-0911-8a066690e34f: skipping application of old log: index=22 -2024-05-12T20:50:47.033Z [INFO] server-6a11da2d-c315-93c2-f616-c06e2146e09c: copied to local snapshot: bytes=71 -2024-05-12T20:50:47.033Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: pipelining replication: peer="{Voter server-289a76b4-3ba1-b059-0911-8a066690e34f 289a76b4-3ba1-b059-0911-8a066690e34f}" -2024-05-12T20:50:47.033Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:47.033Z [INFO] server-6a11da2d-c315-93c2-f616-c06e2146e09c: snapshot restore progress: id=2-113-1715547047032 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:47.033Z [INFO] server-6a11da2d-c315-93c2-f616-c06e2146e09c: Installed remote snapshot -2024-05-12T20:50:47.033Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: pipelining replication: peer="{Voter server-6a11da2d-c315-93c2-f616-c06e2146e09c 6a11da2d-c315-93c2-f616-c06e2146e09c}" -2024-05-12T20:50:48.038Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:47.033439396 +0000 UTC m=+58.314139777 (2 nodes), 2.367µs from start of poll, 1.543663657s from cluster start. Timeout at 2024-05-12 20:50:48.038817802 +0000 UTC m=+59.319518335, 1.005378558s after stability -2024-05-12T20:50:48.091Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: aborting pipeline replication: peer="{Voter server-6a11da2d-c315-93c2-f616-c06e2146e09c 6a11da2d-c315-93c2-f616-c06e2146e09c}" -2024-05-12T20:50:48.091Z [INFO] server-1fa81adc-4098-33de-dfbe-29f8c09b3e8e: aborting pipeline replication: peer="{Voter server-289a76b4-3ba1-b059-0911-8a066690e34f 289a76b4-3ba1-b059-0911-8a066690e34f}" +2025-06-15T03:17:20.071Z [INFO] server-99ddd352-44ac-ebbf-ad51-b88cf1b1e718: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99ddd352-44ac-ebbf-ad51-b88cf1b1e718 Address:99ddd352-44ac-ebbf-ad51-b88cf1b1e718} {Suffrage:Voter ID:server-a9f41536-69ab-c84b-748b-84297ce046cb Address:a9f41536-69ab-c84b-748b-84297ce046cb} {Suffrage:Voter ID:server-9a909e3e-8e7a-ef7f-24e6-1d6865064d36 Address:9a909e3e-8e7a-ef7f-24e6-1d6865064d36}]" +2025-06-15T03:17:20.071Z [INFO] server-99ddd352-44ac-ebbf-ad51-b88cf1b1e718: entering follower state: follower="Node at 99ddd352-44ac-ebbf-ad51-b88cf1b1e718 [Follower]" leader-address= leader-id= +2025-06-15T03:17:20.071Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99ddd352-44ac-ebbf-ad51-b88cf1b1e718 Address:99ddd352-44ac-ebbf-ad51-b88cf1b1e718} {Suffrage:Voter ID:server-a9f41536-69ab-c84b-748b-84297ce046cb Address:a9f41536-69ab-c84b-748b-84297ce046cb} {Suffrage:Voter ID:server-9a909e3e-8e7a-ef7f-24e6-1d6865064d36 Address:9a909e3e-8e7a-ef7f-24e6-1d6865064d36}]" +2025-06-15T03:17:20.071Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: entering follower state: follower="Node at a9f41536-69ab-c84b-748b-84297ce046cb [Follower]" leader-address= leader-id= +2025-06-15T03:17:20.071Z [INFO] server-9a909e3e-8e7a-ef7f-24e6-1d6865064d36: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99ddd352-44ac-ebbf-ad51-b88cf1b1e718 Address:99ddd352-44ac-ebbf-ad51-b88cf1b1e718} {Suffrage:Voter ID:server-a9f41536-69ab-c84b-748b-84297ce046cb Address:a9f41536-69ab-c84b-748b-84297ce046cb} {Suffrage:Voter ID:server-9a909e3e-8e7a-ef7f-24e6-1d6865064d36 Address:9a909e3e-8e7a-ef7f-24e6-1d6865064d36}]" +2025-06-15T03:17:20.071Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:20.071Z [INFO] server-9a909e3e-8e7a-ef7f-24e6-1d6865064d36: entering follower state: follower="Node at 9a909e3e-8e7a-ef7f-24e6-1d6865064d36 [Follower]" leader-address= leader-id= +2025-06-15T03:17:20.818Z [WARN] server-a9f41536-69ab-c84b-748b-84297ce046cb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:20.818Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: entering candidate state: node="Node at a9f41536-69ab-c84b-748b-84297ce046cb [Candidate]" term=2 +2025-06-15T03:17:20.818Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: election won: term=2 tally=2 +2025-06-15T03:17:20.818Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: entering leader state: leader="Node at a9f41536-69ab-c84b-748b-84297ce046cb [Leader]" +2025-06-15T03:17:20.818Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: added peer, starting replication: peer=server-99ddd352-44ac-ebbf-ad51-b88cf1b1e718 +2025-06-15T03:17:20.818Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: added peer, starting replication: peer=server-9a909e3e-8e7a-ef7f-24e6-1d6865064d36 +2025-06-15T03:17:20.818Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: pipelining replication: peer="{Voter server-9a909e3e-8e7a-ef7f-24e6-1d6865064d36 9a909e3e-8e7a-ef7f-24e6-1d6865064d36}" +2025-06-15T03:17:20.818Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: pipelining replication: peer="{Voter server-99ddd352-44ac-ebbf-ad51-b88cf1b1e718 99ddd352-44ac-ebbf-ad51-b88cf1b1e718}" +2025-06-15T03:17:21.823Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:20.818175651 +0000 UTC m=+56.529853812 (1 nodes), 746.606962ms from start of poll, 746.963118ms from cluster start. Timeout at 2025-06-15 03:17:21.823208338 +0000 UTC m=+57.534886571, 1.005032759s after stability +2025-06-15T03:17:21.823Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: starting snapshot up to: index=12 +2025-06-15T03:17:21.823Z [INFO] creating new snapshot: path=/tmp/raft3324789371/snapshots/2-12-1749957441823.tmp +2025-06-15T03:17:21.823Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: snapshot complete up to: index=12 +2025-06-15T03:17:21.824Z [INFO] creating new snapshot: path=/tmp/raft3324789371/snapshots/2-113-1749957441824.tmp +2025-06-15T03:17:21.824Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: copied to local snapshot: bytes=71 +2025-06-15T03:17:21.824Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: snapshot restore progress: id=2-113-1749957441824 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:21.825Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: restored user snapshot: index=113 +2025-06-15T03:17:21.825Z [ERROR] server-a9f41536-69ab-c84b-748b-84297ce046cb: failed to get log: index=23 error="log not found" +2025-06-15T03:17:21.825Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: aborting pipeline replication: peer="{Voter server-9a909e3e-8e7a-ef7f-24e6-1d6865064d36 9a909e3e-8e7a-ef7f-24e6-1d6865064d36}" +2025-06-15T03:17:21.825Z [ERROR] server-a9f41536-69ab-c84b-748b-84297ce046cb: failed to get log: index=23 error="log not found" +2025-06-15T03:17:21.825Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: aborting pipeline replication: peer="{Voter server-99ddd352-44ac-ebbf-ad51-b88cf1b1e718 99ddd352-44ac-ebbf-ad51-b88cf1b1e718}" +2025-06-15T03:17:21.832Z [ERROR] server-a9f41536-69ab-c84b-748b-84297ce046cb: failed to get log: index=23 error="log not found" +2025-06-15T03:17:21.832Z [ERROR] server-a9f41536-69ab-c84b-748b-84297ce046cb: failed to get log: index=23 error="log not found" +2025-06-15T03:17:21.832Z [INFO] creating new snapshot: path=/tmp/raft743796100/snapshots/2-113-1749957441832.tmp +2025-06-15T03:17:21.832Z [INFO] creating new snapshot: path=/tmp/raft444764954/snapshots/2-113-1749957441832.tmp +2025-06-15T03:17:21.833Z [INFO] server-99ddd352-44ac-ebbf-ad51-b88cf1b1e718: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:21.833Z [INFO] server-9a909e3e-8e7a-ef7f-24e6-1d6865064d36: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:21.833Z [INFO] server-9a909e3e-8e7a-ef7f-24e6-1d6865064d36: copied to local snapshot: bytes=71 +2025-06-15T03:17:21.833Z [INFO] server-9a909e3e-8e7a-ef7f-24e6-1d6865064d36: snapshot restore progress: id=2-113-1749957441832 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:21.833Z [INFO] server-9a909e3e-8e7a-ef7f-24e6-1d6865064d36: Installed remote snapshot +2025-06-15T03:17:21.833Z [WARN] server-9a909e3e-8e7a-ef7f-24e6-1d6865064d36: skipping application of old log: index=22 +2025-06-15T03:17:21.833Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: pipelining replication: peer="{Voter server-9a909e3e-8e7a-ef7f-24e6-1d6865064d36 9a909e3e-8e7a-ef7f-24e6-1d6865064d36}" +2025-06-15T03:17:21.834Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:17:21.834Z [INFO] server-99ddd352-44ac-ebbf-ad51-b88cf1b1e718: copied to local snapshot: bytes=71 +2025-06-15T03:17:21.834Z [INFO] server-99ddd352-44ac-ebbf-ad51-b88cf1b1e718: snapshot restore progress: id=2-113-1749957441832 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:21.834Z [INFO] server-99ddd352-44ac-ebbf-ad51-b88cf1b1e718: Installed remote snapshot +2025-06-15T03:17:21.834Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: pipelining replication: peer="{Voter server-99ddd352-44ac-ebbf-ad51-b88cf1b1e718 99ddd352-44ac-ebbf-ad51-b88cf1b1e718}" +2025-06-15T03:17:22.839Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:17:21.834095905 +0000 UTC m=+57.545774071 (2 nodes), 2.831µs from start of poll, 1.762883377s from cluster start. Timeout at 2025-06-15 03:17:22.839615338 +0000 UTC m=+58.551293538, 1.005519467s after stability +2025-06-15T03:17:22.850Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: aborting pipeline replication: peer="{Voter server-9a909e3e-8e7a-ef7f-24e6-1d6865064d36 9a909e3e-8e7a-ef7f-24e6-1d6865064d36}" +2025-06-15T03:17:22.850Z [INFO] server-a9f41536-69ab-c84b-748b-84297ce046cb: aborting pipeline replication: peer="{Voter server-99ddd352-44ac-ebbf-ad51-b88cf1b1e718 99ddd352-44ac-ebbf-ad51-b88cf1b1e718}" === RUN TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false -2024-05-12T20:50:48.093Z [INFO] server-63f11900-63df-5c68-7ae7-1f214e875616: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-63f11900-63df-5c68-7ae7-1f214e875616 Address:63f11900-63df-5c68-7ae7-1f214e875616} {Suffrage:Voter ID:server-1413702d-5612-c0b5-069b-0f8f4eb0732d Address:1413702d-5612-c0b5-069b-0f8f4eb0732d} {Suffrage:Voter ID:server-30a503ba-4cae-cdc0-443d-d41c1800608c Address:30a503ba-4cae-cdc0-443d-d41c1800608c}]" -2024-05-12T20:50:48.094Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-63f11900-63df-5c68-7ae7-1f214e875616 Address:63f11900-63df-5c68-7ae7-1f214e875616} {Suffrage:Voter ID:server-1413702d-5612-c0b5-069b-0f8f4eb0732d Address:1413702d-5612-c0b5-069b-0f8f4eb0732d} {Suffrage:Voter ID:server-30a503ba-4cae-cdc0-443d-d41c1800608c Address:30a503ba-4cae-cdc0-443d-d41c1800608c}]" -2024-05-12T20:50:48.094Z [INFO] server-30a503ba-4cae-cdc0-443d-d41c1800608c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-63f11900-63df-5c68-7ae7-1f214e875616 Address:63f11900-63df-5c68-7ae7-1f214e875616} {Suffrage:Voter ID:server-1413702d-5612-c0b5-069b-0f8f4eb0732d Address:1413702d-5612-c0b5-069b-0f8f4eb0732d} {Suffrage:Voter ID:server-30a503ba-4cae-cdc0-443d-d41c1800608c Address:30a503ba-4cae-cdc0-443d-d41c1800608c}]" -2024-05-12T20:50:48.094Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:48.094Z [INFO] server-63f11900-63df-5c68-7ae7-1f214e875616: entering follower state: follower="Node at 63f11900-63df-5c68-7ae7-1f214e875616 [Follower]" leader-address= leader-id= -2024-05-12T20:50:48.094Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: entering follower state: follower="Node at 1413702d-5612-c0b5-069b-0f8f4eb0732d [Follower]" leader-address= leader-id= -2024-05-12T20:50:48.094Z [INFO] server-30a503ba-4cae-cdc0-443d-d41c1800608c: entering follower state: follower="Node at 30a503ba-4cae-cdc0-443d-d41c1800608c [Follower]" leader-address= leader-id= -2024-05-12T20:50:48.878Z [WARN] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:48.878Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: entering candidate state: node="Node at 1413702d-5612-c0b5-069b-0f8f4eb0732d [Candidate]" term=2 -2024-05-12T20:50:48.879Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: election won: term=2 tally=2 -2024-05-12T20:50:48.879Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: entering leader state: leader="Node at 1413702d-5612-c0b5-069b-0f8f4eb0732d [Leader]" -2024-05-12T20:50:48.879Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: added peer, starting replication: peer=server-63f11900-63df-5c68-7ae7-1f214e875616 -2024-05-12T20:50:48.879Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: added peer, starting replication: peer=server-30a503ba-4cae-cdc0-443d-d41c1800608c -2024-05-12T20:50:48.879Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: pipelining replication: peer="{Voter server-30a503ba-4cae-cdc0-443d-d41c1800608c 30a503ba-4cae-cdc0-443d-d41c1800608c}" -2024-05-12T20:50:48.879Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: pipelining replication: peer="{Voter server-63f11900-63df-5c68-7ae7-1f214e875616 63f11900-63df-5c68-7ae7-1f214e875616}" -2024-05-12T20:50:49.887Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:48.879551539 +0000 UTC m=+60.160251902 (1 nodes), 785.307295ms from start of poll, 785.717045ms from cluster start. Timeout at 2024-05-12 20:50:49.886822002 +0000 UTC m=+61.167521897, 1.007269995s after stability -2024-05-12T20:50:49.889Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: starting snapshot up to: index=12 -2024-05-12T20:50:49.889Z [INFO] creating new snapshot: path=/tmp/raft2436697979/snapshots/2-12-1715547049889.tmp -2024-05-12T20:50:49.895Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: snapshot complete up to: index=12 -2024-05-12T20:50:49.897Z [INFO] creating new snapshot: path=/tmp/raft2436697979/snapshots/2-113-1715547049897.tmp -2024-05-12T20:50:49.907Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: copied to local snapshot: bytes=71 -2024-05-12T20:50:49.907Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: snapshot restore progress: id=2-113-1715547049897 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:49.907Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: removing all old logs from log store -2024-05-12T20:50:49.907Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: compacting logs: from=1 to=22 -2024-05-12T20:50:49.907Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: restored user snapshot: index=113 -2024-05-12T20:50:49.908Z [ERROR] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: failed to get log: index=22 error="log not found" -2024-05-12T20:50:49.908Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: aborting pipeline replication: peer="{Voter server-30a503ba-4cae-cdc0-443d-d41c1800608c 30a503ba-4cae-cdc0-443d-d41c1800608c}" -2024-05-12T20:50:49.908Z [ERROR] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: failed to get log: index=22 error="log not found" -2024-05-12T20:50:49.908Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: aborting pipeline replication: peer="{Voter server-63f11900-63df-5c68-7ae7-1f214e875616 63f11900-63df-5c68-7ae7-1f214e875616}" -2024-05-12T20:50:49.914Z [ERROR] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: failed to get log: index=22 error="log not found" -2024-05-12T20:50:49.915Z [ERROR] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: failed to get log: index=22 error="log not found" -2024-05-12T20:50:49.915Z [INFO] creating new snapshot: path=/tmp/raft1868463187/snapshots/2-113-1715547049915.tmp -2024-05-12T20:50:49.916Z [INFO] creating new snapshot: path=/tmp/raft3148642117/snapshots/2-113-1715547049916.tmp -2024-05-12T20:50:49.916Z [INFO] server-63f11900-63df-5c68-7ae7-1f214e875616: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:49.916Z [INFO] server-30a503ba-4cae-cdc0-443d-d41c1800608c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:49.917Z [INFO] server-30a503ba-4cae-cdc0-443d-d41c1800608c: copied to local snapshot: bytes=71 -2024-05-12T20:50:49.917Z [INFO] server-30a503ba-4cae-cdc0-443d-d41c1800608c: snapshot restore progress: id=2-113-1715547049916 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:49.917Z [INFO] server-30a503ba-4cae-cdc0-443d-d41c1800608c: removing all old logs from log store -2024-05-12T20:50:49.917Z [INFO] server-30a503ba-4cae-cdc0-443d-d41c1800608c: compacting logs: from=1 to=22 -2024-05-12T20:50:49.917Z [INFO] server-30a503ba-4cae-cdc0-443d-d41c1800608c: Installed remote snapshot -2024-05-12T20:50:49.917Z [INFO] server-63f11900-63df-5c68-7ae7-1f214e875616: copied to local snapshot: bytes=71 -2024-05-12T20:50:49.917Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: pipelining replication: peer="{Voter server-30a503ba-4cae-cdc0-443d-d41c1800608c 30a503ba-4cae-cdc0-443d-d41c1800608c}" -2024-05-12T20:50:49.917Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:49.917Z [INFO] server-63f11900-63df-5c68-7ae7-1f214e875616: snapshot restore progress: id=2-113-1715547049915 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:49.917Z [INFO] server-63f11900-63df-5c68-7ae7-1f214e875616: removing all old logs from log store -2024-05-12T20:50:49.917Z [INFO] server-63f11900-63df-5c68-7ae7-1f214e875616: compacting logs: from=1 to=22 -2024-05-12T20:50:49.917Z [INFO] server-63f11900-63df-5c68-7ae7-1f214e875616: Installed remote snapshot -2024-05-12T20:50:49.918Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: pipelining replication: peer="{Voter server-63f11900-63df-5c68-7ae7-1f214e875616 63f11900-63df-5c68-7ae7-1f214e875616}" -2024-05-12T20:50:50.923Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:49.917775149 +0000 UTC m=+61.198475527 (2 nodes), 2.004µs from start of poll, 1.82394067s from cluster start. Timeout at 2024-05-12 20:50:50.923292208 +0000 UTC m=+62.203992602, 1.005517075s after stability -2024-05-12T20:50:50.938Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: aborting pipeline replication: peer="{Voter server-30a503ba-4cae-cdc0-443d-d41c1800608c 30a503ba-4cae-cdc0-443d-d41c1800608c}" -2024-05-12T20:50:50.939Z [INFO] server-1413702d-5612-c0b5-069b-0f8f4eb0732d: aborting pipeline replication: peer="{Voter server-63f11900-63df-5c68-7ae7-1f214e875616 63f11900-63df-5c68-7ae7-1f214e875616}" +2025-06-15T03:17:22.854Z [INFO] server-bcd7dedf-c157-ce42-080c-e563393d5075: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bcd7dedf-c157-ce42-080c-e563393d5075 Address:bcd7dedf-c157-ce42-080c-e563393d5075} {Suffrage:Voter ID:server-eced0263-3661-c669-c95f-7c59156efe6f Address:eced0263-3661-c669-c95f-7c59156efe6f} {Suffrage:Voter ID:server-e2954937-79a4-2484-8f04-47f93b86ac98 Address:e2954937-79a4-2484-8f04-47f93b86ac98}]" +2025-06-15T03:17:22.854Z [INFO] server-bcd7dedf-c157-ce42-080c-e563393d5075: entering follower state: follower="Node at bcd7dedf-c157-ce42-080c-e563393d5075 [Follower]" leader-address= leader-id= +2025-06-15T03:17:22.854Z [INFO] server-eced0263-3661-c669-c95f-7c59156efe6f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bcd7dedf-c157-ce42-080c-e563393d5075 Address:bcd7dedf-c157-ce42-080c-e563393d5075} {Suffrage:Voter ID:server-eced0263-3661-c669-c95f-7c59156efe6f Address:eced0263-3661-c669-c95f-7c59156efe6f} {Suffrage:Voter ID:server-e2954937-79a4-2484-8f04-47f93b86ac98 Address:e2954937-79a4-2484-8f04-47f93b86ac98}]" +2025-06-15T03:17:22.854Z [INFO] server-eced0263-3661-c669-c95f-7c59156efe6f: entering follower state: follower="Node at eced0263-3661-c669-c95f-7c59156efe6f [Follower]" leader-address= leader-id= +2025-06-15T03:17:22.854Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bcd7dedf-c157-ce42-080c-e563393d5075 Address:bcd7dedf-c157-ce42-080c-e563393d5075} {Suffrage:Voter ID:server-eced0263-3661-c669-c95f-7c59156efe6f Address:eced0263-3661-c669-c95f-7c59156efe6f} {Suffrage:Voter ID:server-e2954937-79a4-2484-8f04-47f93b86ac98 Address:e2954937-79a4-2484-8f04-47f93b86ac98}]" +2025-06-15T03:17:22.854Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:22.854Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: entering follower state: follower="Node at e2954937-79a4-2484-8f04-47f93b86ac98 [Follower]" leader-address= leader-id= +2025-06-15T03:17:23.478Z [WARN] server-e2954937-79a4-2484-8f04-47f93b86ac98: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:23.478Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: entering candidate state: node="Node at e2954937-79a4-2484-8f04-47f93b86ac98 [Candidate]" term=2 +2025-06-15T03:17:23.478Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: election won: term=2 tally=2 +2025-06-15T03:17:23.478Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: entering leader state: leader="Node at e2954937-79a4-2484-8f04-47f93b86ac98 [Leader]" +2025-06-15T03:17:23.478Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: added peer, starting replication: peer=server-bcd7dedf-c157-ce42-080c-e563393d5075 +2025-06-15T03:17:23.478Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: added peer, starting replication: peer=server-eced0263-3661-c669-c95f-7c59156efe6f +2025-06-15T03:17:23.478Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: pipelining replication: peer="{Voter server-eced0263-3661-c669-c95f-7c59156efe6f eced0263-3661-c669-c95f-7c59156efe6f}" +2025-06-15T03:17:23.479Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: pipelining replication: peer="{Voter server-bcd7dedf-c157-ce42-080c-e563393d5075 bcd7dedf-c157-ce42-080c-e563393d5075}" +2025-06-15T03:17:24.485Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:23.478956613 +0000 UTC m=+59.190634816 (1 nodes), 624.074435ms from start of poll, 624.351648ms from cluster start. Timeout at 2025-06-15 03:17:24.484995425 +0000 UTC m=+60.196673624, 1.006038808s after stability +2025-06-15T03:17:24.486Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: starting snapshot up to: index=12 +2025-06-15T03:17:24.486Z [INFO] creating new snapshot: path=/tmp/raft3854065537/snapshots/2-12-1749957444486.tmp +2025-06-15T03:17:24.487Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: snapshot complete up to: index=12 +2025-06-15T03:17:24.487Z [INFO] creating new snapshot: path=/tmp/raft3854065537/snapshots/2-113-1749957444487.tmp +2025-06-15T03:17:24.488Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: copied to local snapshot: bytes=71 +2025-06-15T03:17:24.488Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: snapshot restore progress: id=2-113-1749957444487 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:24.488Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: removing all old logs from log store +2025-06-15T03:17:24.488Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: compacting logs: from=1 to=22 +2025-06-15T03:17:24.488Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: restored user snapshot: index=113 +2025-06-15T03:17:24.488Z [ERROR] server-e2954937-79a4-2484-8f04-47f93b86ac98: failed to get log: index=22 error="log not found" +2025-06-15T03:17:24.488Z [ERROR] server-e2954937-79a4-2484-8f04-47f93b86ac98: failed to get log: index=22 error="log not found" +2025-06-15T03:17:24.488Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: aborting pipeline replication: peer="{Voter server-bcd7dedf-c157-ce42-080c-e563393d5075 bcd7dedf-c157-ce42-080c-e563393d5075}" +2025-06-15T03:17:24.488Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: aborting pipeline replication: peer="{Voter server-eced0263-3661-c669-c95f-7c59156efe6f eced0263-3661-c669-c95f-7c59156efe6f}" +2025-06-15T03:17:24.494Z [ERROR] server-e2954937-79a4-2484-8f04-47f93b86ac98: failed to get log: index=22 error="log not found" +2025-06-15T03:17:24.494Z [ERROR] server-e2954937-79a4-2484-8f04-47f93b86ac98: failed to get log: index=22 error="log not found" +2025-06-15T03:17:24.494Z [INFO] creating new snapshot: path=/tmp/raft949808467/snapshots/2-113-1749957444494.tmp +2025-06-15T03:17:24.494Z [INFO] creating new snapshot: path=/tmp/raft2499611321/snapshots/2-113-1749957444494.tmp +2025-06-15T03:17:24.494Z [INFO] server-bcd7dedf-c157-ce42-080c-e563393d5075: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:24.494Z [INFO] server-eced0263-3661-c669-c95f-7c59156efe6f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:24.494Z [INFO] server-bcd7dedf-c157-ce42-080c-e563393d5075: copied to local snapshot: bytes=71 +2025-06-15T03:17:24.494Z [INFO] server-bcd7dedf-c157-ce42-080c-e563393d5075: snapshot restore progress: id=2-113-1749957444494 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:24.494Z [INFO] server-bcd7dedf-c157-ce42-080c-e563393d5075: removing all old logs from log store +2025-06-15T03:17:24.495Z [INFO] server-bcd7dedf-c157-ce42-080c-e563393d5075: compacting logs: from=1 to=22 +2025-06-15T03:17:24.495Z [INFO] server-bcd7dedf-c157-ce42-080c-e563393d5075: Installed remote snapshot +2025-06-15T03:17:24.495Z [WARN] server-bcd7dedf-c157-ce42-080c-e563393d5075: skipping application of old log: index=22 +2025-06-15T03:17:24.495Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: pipelining replication: peer="{Voter server-bcd7dedf-c157-ce42-080c-e563393d5075 bcd7dedf-c157-ce42-080c-e563393d5075}" +2025-06-15T03:17:24.495Z [INFO] server-eced0263-3661-c669-c95f-7c59156efe6f: copied to local snapshot: bytes=71 +2025-06-15T03:17:24.495Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:17:24.495Z [INFO] server-eced0263-3661-c669-c95f-7c59156efe6f: snapshot restore progress: id=2-113-1749957444494 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:24.495Z [INFO] server-eced0263-3661-c669-c95f-7c59156efe6f: removing all old logs from log store +2025-06-15T03:17:24.495Z [INFO] server-eced0263-3661-c669-c95f-7c59156efe6f: compacting logs: from=1 to=22 +2025-06-15T03:17:24.495Z [INFO] server-eced0263-3661-c669-c95f-7c59156efe6f: Installed remote snapshot +2025-06-15T03:17:24.495Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: pipelining replication: peer="{Voter server-eced0263-3661-c669-c95f-7c59156efe6f eced0263-3661-c669-c95f-7c59156efe6f}" +2025-06-15T03:17:25.500Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:17:24.495114719 +0000 UTC m=+60.206792884 (2 nodes), 619ns from start of poll, 1.640509716s from cluster start. Timeout at 2025-06-15 03:17:25.500141058 +0000 UTC m=+61.211819220, 1.005026336s after stability +2025-06-15T03:17:25.511Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: aborting pipeline replication: peer="{Voter server-eced0263-3661-c669-c95f-7c59156efe6f eced0263-3661-c669-c95f-7c59156efe6f}" +2025-06-15T03:17:25.511Z [INFO] server-e2954937-79a4-2484-8f04-47f93b86ac98: aborting pipeline replication: peer="{Voter server-bcd7dedf-c157-ce42-080c-e563393d5075 bcd7dedf-c157-ce42-080c-e563393d5075}" === RUN TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true -2024-05-12T20:50:50.942Z [INFO] server-6f498b42-ac6e-fced-e004-29acaf4af0d4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f498b42-ac6e-fced-e004-29acaf4af0d4 Address:6f498b42-ac6e-fced-e004-29acaf4af0d4} {Suffrage:Voter ID:server-bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5 Address:bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5} {Suffrage:Voter ID:server-010401ec-f040-7351-8388-50938dc0db5b Address:010401ec-f040-7351-8388-50938dc0db5b}]" -2024-05-12T20:50:50.942Z [INFO] server-bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f498b42-ac6e-fced-e004-29acaf4af0d4 Address:6f498b42-ac6e-fced-e004-29acaf4af0d4} {Suffrage:Voter ID:server-bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5 Address:bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5} {Suffrage:Voter ID:server-010401ec-f040-7351-8388-50938dc0db5b Address:010401ec-f040-7351-8388-50938dc0db5b}]" -2024-05-12T20:50:50.942Z [INFO] server-010401ec-f040-7351-8388-50938dc0db5b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f498b42-ac6e-fced-e004-29acaf4af0d4 Address:6f498b42-ac6e-fced-e004-29acaf4af0d4} {Suffrage:Voter ID:server-bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5 Address:bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5} {Suffrage:Voter ID:server-010401ec-f040-7351-8388-50938dc0db5b Address:010401ec-f040-7351-8388-50938dc0db5b}]" -2024-05-12T20:50:50.942Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:50.942Z [INFO] server-6f498b42-ac6e-fced-e004-29acaf4af0d4: entering follower state: follower="Node at 6f498b42-ac6e-fced-e004-29acaf4af0d4 [Follower]" leader-address= leader-id= -2024-05-12T20:50:50.942Z [INFO] server-bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5: entering follower state: follower="Node at bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5 [Follower]" leader-address= leader-id= -2024-05-12T20:50:50.942Z [INFO] server-010401ec-f040-7351-8388-50938dc0db5b: entering follower state: follower="Node at 010401ec-f040-7351-8388-50938dc0db5b [Follower]" leader-address= leader-id= -2024-05-12T20:50:51.450Z [WARN] server-6f498b42-ac6e-fced-e004-29acaf4af0d4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:51.450Z [INFO] server-6f498b42-ac6e-fced-e004-29acaf4af0d4: entering candidate state: node="Node at 6f498b42-ac6e-fced-e004-29acaf4af0d4 [Candidate]" term=2 -2024-05-12T20:50:51.451Z [INFO] server-6f498b42-ac6e-fced-e004-29acaf4af0d4: election won: term=2 tally=2 -2024-05-12T20:50:51.451Z [INFO] server-6f498b42-ac6e-fced-e004-29acaf4af0d4: entering leader state: leader="Node at 6f498b42-ac6e-fced-e004-29acaf4af0d4 [Leader]" -2024-05-12T20:50:51.451Z [INFO] server-6f498b42-ac6e-fced-e004-29acaf4af0d4: added peer, starting replication: peer=server-bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5 -2024-05-12T20:50:51.451Z [WARN] server-bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:51.451Z [INFO] server-6f498b42-ac6e-fced-e004-29acaf4af0d4: added peer, starting replication: peer=server-010401ec-f040-7351-8388-50938dc0db5b -2024-05-12T20:50:51.451Z [INFO] server-bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5: entering candidate state: node="Node at bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5 [Candidate]" term=2 -2024-05-12T20:50:51.451Z [INFO] server-010401ec-f040-7351-8388-50938dc0db5b: duplicate requestVote for same term: term=2 -2024-05-12T20:50:51.451Z [INFO] server-bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5: duplicate requestVote for same term: term=2 -2024-05-12T20:50:51.451Z [INFO] server-bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5: entering follower state: follower="Node at bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5 [Follower]" leader-address=6f498b42-ac6e-fced-e004-29acaf4af0d4 leader-id=server-6f498b42-ac6e-fced-e004-29acaf4af0d4 -2024-05-12T20:50:51.451Z [INFO] server-6f498b42-ac6e-fced-e004-29acaf4af0d4: pipelining replication: peer="{Voter server-010401ec-f040-7351-8388-50938dc0db5b 010401ec-f040-7351-8388-50938dc0db5b}" -2024-05-12T20:50:51.451Z [INFO] server-6f498b42-ac6e-fced-e004-29acaf4af0d4: pipelining replication: peer="{Voter server-bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5 bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5}" -2024-05-12T20:50:51.451Z [WARN] server-6f498b42-ac6e-fced-e004-29acaf4af0d4: rejecting vote request since we have a leader: from=bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5 leader=6f498b42-ac6e-fced-e004-29acaf4af0d4 leader-id=server-6f498b42-ac6e-fced-e004-29acaf4af0d4 -2024-05-12T20:50:52.459Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:51.451484209 +0000 UTC m=+62.732184657 (1 nodes), 508.88077ms from start of poll, 509.311286ms from cluster start. Timeout at 2024-05-12 20:50:52.458833953 +0000 UTC m=+63.739534691, 1.007350034s after stability -2024-05-12T20:50:52.460Z [INFO] server-6f498b42-ac6e-fced-e004-29acaf4af0d4: starting snapshot up to: index=12 -2024-05-12T20:50:52.460Z [INFO] creating new snapshot: path=/tmp/raft1485374270/snapshots/2-12-1715547052460.tmp -2024-05-12T20:50:52.461Z [INFO] server-6f498b42-ac6e-fced-e004-29acaf4af0d4: snapshot complete up to: index=12 -2024-05-12T20:50:52.464Z [INFO] server-274f8bbf-d65f-a7e7-e483-555898f7c48f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-274f8bbf-d65f-a7e7-e483-555898f7c48f Address:274f8bbf-d65f-a7e7-e483-555898f7c48f} {Suffrage:Voter ID:server-6a4977b6-3a03-b51c-7997-ab243ffc48e0 Address:6a4977b6-3a03-b51c-7997-ab243ffc48e0} {Suffrage:Voter ID:server-9605cde0-50a1-91da-d1ae-93a6bb5b208c Address:9605cde0-50a1-91da-d1ae-93a6bb5b208c}]" -2024-05-12T20:50:52.464Z [INFO] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-274f8bbf-d65f-a7e7-e483-555898f7c48f Address:274f8bbf-d65f-a7e7-e483-555898f7c48f} {Suffrage:Voter ID:server-6a4977b6-3a03-b51c-7997-ab243ffc48e0 Address:6a4977b6-3a03-b51c-7997-ab243ffc48e0} {Suffrage:Voter ID:server-9605cde0-50a1-91da-d1ae-93a6bb5b208c Address:9605cde0-50a1-91da-d1ae-93a6bb5b208c}]" -2024-05-12T20:50:52.464Z [INFO] server-9605cde0-50a1-91da-d1ae-93a6bb5b208c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-274f8bbf-d65f-a7e7-e483-555898f7c48f Address:274f8bbf-d65f-a7e7-e483-555898f7c48f} {Suffrage:Voter ID:server-6a4977b6-3a03-b51c-7997-ab243ffc48e0 Address:6a4977b6-3a03-b51c-7997-ab243ffc48e0} {Suffrage:Voter ID:server-9605cde0-50a1-91da-d1ae-93a6bb5b208c Address:9605cde0-50a1-91da-d1ae-93a6bb5b208c}]" -2024-05-12T20:50:52.464Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:52.464Z [INFO] server-274f8bbf-d65f-a7e7-e483-555898f7c48f: entering follower state: follower="Node at 274f8bbf-d65f-a7e7-e483-555898f7c48f [Follower]" leader-address= leader-id= -2024-05-12T20:50:52.464Z [INFO] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: entering follower state: follower="Node at 6a4977b6-3a03-b51c-7997-ab243ffc48e0 [Follower]" leader-address= leader-id= -2024-05-12T20:50:52.465Z [INFO] server-9605cde0-50a1-91da-d1ae-93a6bb5b208c: entering follower state: follower="Node at 9605cde0-50a1-91da-d1ae-93a6bb5b208c [Follower]" leader-address= leader-id= -2024-05-12T20:50:53.111Z [WARN] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:53.111Z [INFO] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: entering candidate state: node="Node at 6a4977b6-3a03-b51c-7997-ab243ffc48e0 [Candidate]" term=2 -2024-05-12T20:50:53.111Z [INFO] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: election won: term=2 tally=2 -2024-05-12T20:50:53.111Z [INFO] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: entering leader state: leader="Node at 6a4977b6-3a03-b51c-7997-ab243ffc48e0 [Leader]" -2024-05-12T20:50:53.111Z [INFO] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: added peer, starting replication: peer=server-274f8bbf-d65f-a7e7-e483-555898f7c48f -2024-05-12T20:50:53.111Z [INFO] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: added peer, starting replication: peer=server-9605cde0-50a1-91da-d1ae-93a6bb5b208c -2024-05-12T20:50:53.112Z [INFO] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: pipelining replication: peer="{Voter server-9605cde0-50a1-91da-d1ae-93a6bb5b208c 9605cde0-50a1-91da-d1ae-93a6bb5b208c}" -2024-05-12T20:50:53.112Z [INFO] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: pipelining replication: peer="{Voter server-274f8bbf-d65f-a7e7-e483-555898f7c48f 274f8bbf-d65f-a7e7-e483-555898f7c48f}" -2024-05-12T20:50:54.118Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:53.112472909 +0000 UTC m=+64.393173302 (1 nodes), 647.653689ms from start of poll, 648.132718ms from cluster start. Timeout at 2024-05-12 20:50:54.118766123 +0000 UTC m=+65.399466925, 1.006293623s after stability -2024-05-12T20:50:54.119Z [INFO] creating new snapshot: path=/tmp/raft3313787136/snapshots/2-113-1715547054119.tmp -2024-05-12T20:50:54.120Z [INFO] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: copied to local snapshot: bytes=71 -2024-05-12T20:50:54.120Z [INFO] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: snapshot restore progress: id=2-113-1715547054119 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:54.120Z [INFO] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: restored user snapshot: index=113 -2024-05-12T20:50:54.120Z [ERROR] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: failed to get log: index=3 error="log not found" -2024-05-12T20:50:54.120Z [INFO] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: aborting pipeline replication: peer="{Voter server-274f8bbf-d65f-a7e7-e483-555898f7c48f 274f8bbf-d65f-a7e7-e483-555898f7c48f}" -2024-05-12T20:50:54.120Z [ERROR] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: failed to get log: index=3 error="log not found" -2024-05-12T20:50:54.120Z [INFO] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: aborting pipeline replication: peer="{Voter server-9605cde0-50a1-91da-d1ae-93a6bb5b208c 9605cde0-50a1-91da-d1ae-93a6bb5b208c}" -2024-05-12T20:50:54.127Z [ERROR] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: failed to get log: index=3 error="log not found" -2024-05-12T20:50:54.127Z [INFO] creating new snapshot: path=/tmp/raft2736531942/snapshots/2-113-1715547054127.tmp -2024-05-12T20:50:54.128Z [INFO] server-9605cde0-50a1-91da-d1ae-93a6bb5b208c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:54.129Z [INFO] server-9605cde0-50a1-91da-d1ae-93a6bb5b208c: copied to local snapshot: bytes=71 -2024-05-12T20:50:54.130Z [INFO] server-9605cde0-50a1-91da-d1ae-93a6bb5b208c: snapshot restore progress: id=2-113-1715547054127 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:54.130Z [INFO] server-9605cde0-50a1-91da-d1ae-93a6bb5b208c: Installed remote snapshot -2024-05-12T20:50:54.130Z [INFO] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: pipelining replication: peer="{Voter server-9605cde0-50a1-91da-d1ae-93a6bb5b208c 9605cde0-50a1-91da-d1ae-93a6bb5b208c}" -2024-05-12T20:50:54.130Z [ERROR] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: failed to get log: index=3 error="log not found" -2024-05-12T20:50:54.130Z [INFO] creating new snapshot: path=/tmp/raft3183957687/snapshots/2-113-1715547054130.tmp -2024-05-12T20:50:54.130Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:54.130Z [INFO] server-274f8bbf-d65f-a7e7-e483-555898f7c48f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:54.131Z [INFO] server-274f8bbf-d65f-a7e7-e483-555898f7c48f: copied to local snapshot: bytes=71 -2024-05-12T20:50:54.132Z [INFO] server-274f8bbf-d65f-a7e7-e483-555898f7c48f: snapshot restore progress: id=2-113-1715547054130 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:54.132Z [INFO] server-274f8bbf-d65f-a7e7-e483-555898f7c48f: Installed remote snapshot -2024-05-12T20:50:54.132Z [INFO] server-6a4977b6-3a03-b51c-7997-ab243ffc48e0: pipelining replication: peer="{Voter server-274f8bbf-d65f-a7e7-e483-555898f7c48f 274f8bbf-d65f-a7e7-e483-555898f7c48f}" -2024-05-12T20:50:55.139Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:54.131037889 +0000 UTC m=+65.411739047 (2 nodes), 2.706µs from start of poll, 1.666698463s from cluster start. Timeout at 2024-05-12 20:50:55.139277615 +0000 UTC m=+66.419978111, 1.008239064s after stability -2024-05-12T20:50:55.155Z [INFO] server-6f498b42-ac6e-fced-e004-29acaf4af0d4: aborting pipeline replication: peer="{Voter server-010401ec-f040-7351-8388-50938dc0db5b 010401ec-f040-7351-8388-50938dc0db5b}" -2024-05-12T20:50:55.155Z [INFO] server-6f498b42-ac6e-fced-e004-29acaf4af0d4: aborting pipeline replication: peer="{Voter server-bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5 bd70ce4d-04db-e21a-8ae9-142f7ae2a3f5}" +2025-06-15T03:17:25.513Z [INFO] server-5b66c78b-6a5d-22d9-370b-a5820b7b957c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5b66c78b-6a5d-22d9-370b-a5820b7b957c Address:5b66c78b-6a5d-22d9-370b-a5820b7b957c} {Suffrage:Voter ID:server-7672771c-592b-a458-70bf-c4f4db64dd76 Address:7672771c-592b-a458-70bf-c4f4db64dd76} {Suffrage:Voter ID:server-ffa40a56-8cad-ec9b-a626-50975d1771d7 Address:ffa40a56-8cad-ec9b-a626-50975d1771d7}]" +2025-06-15T03:17:25.513Z [INFO] server-5b66c78b-6a5d-22d9-370b-a5820b7b957c: entering follower state: follower="Node at 5b66c78b-6a5d-22d9-370b-a5820b7b957c [Follower]" leader-address= leader-id= +2025-06-15T03:17:25.513Z [INFO] server-7672771c-592b-a458-70bf-c4f4db64dd76: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5b66c78b-6a5d-22d9-370b-a5820b7b957c Address:5b66c78b-6a5d-22d9-370b-a5820b7b957c} {Suffrage:Voter ID:server-7672771c-592b-a458-70bf-c4f4db64dd76 Address:7672771c-592b-a458-70bf-c4f4db64dd76} {Suffrage:Voter ID:server-ffa40a56-8cad-ec9b-a626-50975d1771d7 Address:ffa40a56-8cad-ec9b-a626-50975d1771d7}]" +2025-06-15T03:17:25.513Z [INFO] server-7672771c-592b-a458-70bf-c4f4db64dd76: entering follower state: follower="Node at 7672771c-592b-a458-70bf-c4f4db64dd76 [Follower]" leader-address= leader-id= +2025-06-15T03:17:25.513Z [INFO] server-ffa40a56-8cad-ec9b-a626-50975d1771d7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5b66c78b-6a5d-22d9-370b-a5820b7b957c Address:5b66c78b-6a5d-22d9-370b-a5820b7b957c} {Suffrage:Voter ID:server-7672771c-592b-a458-70bf-c4f4db64dd76 Address:7672771c-592b-a458-70bf-c4f4db64dd76} {Suffrage:Voter ID:server-ffa40a56-8cad-ec9b-a626-50975d1771d7 Address:ffa40a56-8cad-ec9b-a626-50975d1771d7}]" +2025-06-15T03:17:25.513Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:25.513Z [INFO] server-ffa40a56-8cad-ec9b-a626-50975d1771d7: entering follower state: follower="Node at ffa40a56-8cad-ec9b-a626-50975d1771d7 [Follower]" leader-address= leader-id= +2025-06-15T03:17:26.114Z [WARN] server-5b66c78b-6a5d-22d9-370b-a5820b7b957c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:26.114Z [INFO] server-5b66c78b-6a5d-22d9-370b-a5820b7b957c: entering candidate state: node="Node at 5b66c78b-6a5d-22d9-370b-a5820b7b957c [Candidate]" term=2 +2025-06-15T03:17:26.115Z [INFO] server-5b66c78b-6a5d-22d9-370b-a5820b7b957c: election won: term=2 tally=2 +2025-06-15T03:17:26.115Z [INFO] server-5b66c78b-6a5d-22d9-370b-a5820b7b957c: entering leader state: leader="Node at 5b66c78b-6a5d-22d9-370b-a5820b7b957c [Leader]" +2025-06-15T03:17:26.115Z [INFO] server-5b66c78b-6a5d-22d9-370b-a5820b7b957c: added peer, starting replication: peer=server-7672771c-592b-a458-70bf-c4f4db64dd76 +2025-06-15T03:17:26.115Z [INFO] server-5b66c78b-6a5d-22d9-370b-a5820b7b957c: added peer, starting replication: peer=server-ffa40a56-8cad-ec9b-a626-50975d1771d7 +2025-06-15T03:17:26.115Z [INFO] server-5b66c78b-6a5d-22d9-370b-a5820b7b957c: pipelining replication: peer="{Voter server-7672771c-592b-a458-70bf-c4f4db64dd76 7672771c-592b-a458-70bf-c4f4db64dd76}" +2025-06-15T03:17:26.115Z [INFO] server-5b66c78b-6a5d-22d9-370b-a5820b7b957c: pipelining replication: peer="{Voter server-ffa40a56-8cad-ec9b-a626-50975d1771d7 ffa40a56-8cad-ec9b-a626-50975d1771d7}" +2025-06-15T03:17:27.120Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:26.115071694 +0000 UTC m=+61.826749859 (1 nodes), 601.699241ms from start of poll, 602.014865ms from cluster start. Timeout at 2025-06-15 03:17:27.120143294 +0000 UTC m=+62.831821493, 1.005071634s after stability +2025-06-15T03:17:27.121Z [INFO] server-5b66c78b-6a5d-22d9-370b-a5820b7b957c: starting snapshot up to: index=12 +2025-06-15T03:17:27.121Z [INFO] creating new snapshot: path=/tmp/raft990987318/snapshots/2-12-1749957447121.tmp +2025-06-15T03:17:27.121Z [INFO] server-5b66c78b-6a5d-22d9-370b-a5820b7b957c: snapshot complete up to: index=12 +2025-06-15T03:17:27.122Z [INFO] server-051efa30-55b7-93b2-049f-50f3edf817e3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-051efa30-55b7-93b2-049f-50f3edf817e3 Address:051efa30-55b7-93b2-049f-50f3edf817e3} {Suffrage:Voter ID:server-a9d663e1-24c0-2d0b-b47b-38e215a92cac Address:a9d663e1-24c0-2d0b-b47b-38e215a92cac} {Suffrage:Voter ID:server-97984c73-ce74-dcea-e363-eecb1d50349a Address:97984c73-ce74-dcea-e363-eecb1d50349a}]" +2025-06-15T03:17:27.122Z [INFO] server-051efa30-55b7-93b2-049f-50f3edf817e3: entering follower state: follower="Node at 051efa30-55b7-93b2-049f-50f3edf817e3 [Follower]" leader-address= leader-id= +2025-06-15T03:17:27.122Z [INFO] server-a9d663e1-24c0-2d0b-b47b-38e215a92cac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-051efa30-55b7-93b2-049f-50f3edf817e3 Address:051efa30-55b7-93b2-049f-50f3edf817e3} {Suffrage:Voter ID:server-a9d663e1-24c0-2d0b-b47b-38e215a92cac Address:a9d663e1-24c0-2d0b-b47b-38e215a92cac} {Suffrage:Voter ID:server-97984c73-ce74-dcea-e363-eecb1d50349a Address:97984c73-ce74-dcea-e363-eecb1d50349a}]" +2025-06-15T03:17:27.122Z [INFO] server-a9d663e1-24c0-2d0b-b47b-38e215a92cac: entering follower state: follower="Node at a9d663e1-24c0-2d0b-b47b-38e215a92cac [Follower]" leader-address= leader-id= +2025-06-15T03:17:27.122Z [INFO] server-97984c73-ce74-dcea-e363-eecb1d50349a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-051efa30-55b7-93b2-049f-50f3edf817e3 Address:051efa30-55b7-93b2-049f-50f3edf817e3} {Suffrage:Voter ID:server-a9d663e1-24c0-2d0b-b47b-38e215a92cac Address:a9d663e1-24c0-2d0b-b47b-38e215a92cac} {Suffrage:Voter ID:server-97984c73-ce74-dcea-e363-eecb1d50349a Address:97984c73-ce74-dcea-e363-eecb1d50349a}]" +2025-06-15T03:17:27.122Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:27.123Z [INFO] server-97984c73-ce74-dcea-e363-eecb1d50349a: entering follower state: follower="Node at 97984c73-ce74-dcea-e363-eecb1d50349a [Follower]" leader-address= leader-id= +2025-06-15T03:17:27.634Z [WARN] server-97984c73-ce74-dcea-e363-eecb1d50349a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:27.634Z [INFO] server-97984c73-ce74-dcea-e363-eecb1d50349a: entering candidate state: node="Node at 97984c73-ce74-dcea-e363-eecb1d50349a [Candidate]" term=2 +2025-06-15T03:17:27.634Z [INFO] server-97984c73-ce74-dcea-e363-eecb1d50349a: election won: term=2 tally=2 +2025-06-15T03:17:27.634Z [INFO] server-97984c73-ce74-dcea-e363-eecb1d50349a: entering leader state: leader="Node at 97984c73-ce74-dcea-e363-eecb1d50349a [Leader]" +2025-06-15T03:17:27.634Z [INFO] server-97984c73-ce74-dcea-e363-eecb1d50349a: added peer, starting replication: peer=server-051efa30-55b7-93b2-049f-50f3edf817e3 +2025-06-15T03:17:27.634Z [INFO] server-97984c73-ce74-dcea-e363-eecb1d50349a: added peer, starting replication: peer=server-a9d663e1-24c0-2d0b-b47b-38e215a92cac +2025-06-15T03:17:27.635Z [INFO] server-97984c73-ce74-dcea-e363-eecb1d50349a: pipelining replication: peer="{Voter server-a9d663e1-24c0-2d0b-b47b-38e215a92cac a9d663e1-24c0-2d0b-b47b-38e215a92cac}" +2025-06-15T03:17:27.635Z [INFO] server-97984c73-ce74-dcea-e363-eecb1d50349a: pipelining replication: peer="{Voter server-051efa30-55b7-93b2-049f-50f3edf817e3 051efa30-55b7-93b2-049f-50f3edf817e3}" +2025-06-15T03:17:28.640Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:27.635175032 +0000 UTC m=+63.346853195 (1 nodes), 512.176358ms from start of poll, 512.5486ms from cluster start. Timeout at 2025-06-15 03:17:28.640228755 +0000 UTC m=+64.351906954, 1.005053759s after stability +2025-06-15T03:17:28.640Z [INFO] creating new snapshot: path=/tmp/raft3221011997/snapshots/2-113-1749957448640.tmp +2025-06-15T03:17:28.640Z [INFO] server-97984c73-ce74-dcea-e363-eecb1d50349a: copied to local snapshot: bytes=71 +2025-06-15T03:17:28.640Z [INFO] server-97984c73-ce74-dcea-e363-eecb1d50349a: snapshot restore progress: id=2-113-1749957448640 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:28.641Z [INFO] server-97984c73-ce74-dcea-e363-eecb1d50349a: restored user snapshot: index=113 +2025-06-15T03:17:28.641Z [ERROR] server-97984c73-ce74-dcea-e363-eecb1d50349a: failed to get log: index=3 error="log not found" +2025-06-15T03:17:28.641Z [INFO] server-97984c73-ce74-dcea-e363-eecb1d50349a: aborting pipeline replication: peer="{Voter server-a9d663e1-24c0-2d0b-b47b-38e215a92cac a9d663e1-24c0-2d0b-b47b-38e215a92cac}" +2025-06-15T03:17:28.641Z [ERROR] server-97984c73-ce74-dcea-e363-eecb1d50349a: failed to get log: index=3 error="log not found" +2025-06-15T03:17:28.641Z [INFO] server-97984c73-ce74-dcea-e363-eecb1d50349a: aborting pipeline replication: peer="{Voter server-051efa30-55b7-93b2-049f-50f3edf817e3 051efa30-55b7-93b2-049f-50f3edf817e3}" +2025-06-15T03:17:28.648Z [ERROR] server-97984c73-ce74-dcea-e363-eecb1d50349a: failed to get log: index=3 error="log not found" +2025-06-15T03:17:28.648Z [INFO] creating new snapshot: path=/tmp/raft1590721449/snapshots/2-113-1749957448648.tmp +2025-06-15T03:17:28.648Z [ERROR] server-97984c73-ce74-dcea-e363-eecb1d50349a: failed to get log: index=3 error="log not found" +2025-06-15T03:17:28.649Z [INFO] server-a9d663e1-24c0-2d0b-b47b-38e215a92cac: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:28.649Z [INFO] creating new snapshot: path=/tmp/raft1715150208/snapshots/2-113-1749957448649.tmp +2025-06-15T03:17:28.649Z [INFO] server-a9d663e1-24c0-2d0b-b47b-38e215a92cac: copied to local snapshot: bytes=71 +2025-06-15T03:17:28.649Z [INFO] server-051efa30-55b7-93b2-049f-50f3edf817e3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:28.649Z [INFO] server-a9d663e1-24c0-2d0b-b47b-38e215a92cac: snapshot restore progress: id=2-113-1749957448648 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:28.649Z [INFO] server-a9d663e1-24c0-2d0b-b47b-38e215a92cac: Installed remote snapshot +2025-06-15T03:17:28.649Z [INFO] server-97984c73-ce74-dcea-e363-eecb1d50349a: pipelining replication: peer="{Voter server-a9d663e1-24c0-2d0b-b47b-38e215a92cac a9d663e1-24c0-2d0b-b47b-38e215a92cac}" +2025-06-15T03:17:28.649Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:17:28.649Z [INFO] server-051efa30-55b7-93b2-049f-50f3edf817e3: copied to local snapshot: bytes=71 +2025-06-15T03:17:28.649Z [INFO] server-051efa30-55b7-93b2-049f-50f3edf817e3: snapshot restore progress: id=2-113-1749957448649 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:28.649Z [INFO] server-051efa30-55b7-93b2-049f-50f3edf817e3: Installed remote snapshot +2025-06-15T03:17:28.649Z [INFO] server-97984c73-ce74-dcea-e363-eecb1d50349a: pipelining replication: peer="{Voter server-051efa30-55b7-93b2-049f-50f3edf817e3 051efa30-55b7-93b2-049f-50f3edf817e3}" +2025-06-15T03:17:29.654Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:17:28.649626948 +0000 UTC m=+64.361305120 (2 nodes), 745ns from start of poll, 1.527000525s from cluster start. Timeout at 2025-06-15 03:17:29.654664664 +0000 UTC m=+65.366342864, 1.005037744s after stability +2025-06-15T03:17:29.666Z [INFO] server-5b66c78b-6a5d-22d9-370b-a5820b7b957c: aborting pipeline replication: peer="{Voter server-ffa40a56-8cad-ec9b-a626-50975d1771d7 ffa40a56-8cad-ec9b-a626-50975d1771d7}" +2025-06-15T03:17:29.666Z [INFO] server-5b66c78b-6a5d-22d9-370b-a5820b7b957c: aborting pipeline replication: peer="{Voter server-7672771c-592b-a458-70bf-c4f4db64dd76 7672771c-592b-a458-70bf-c4f4db64dd76}" === RUN TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true -2024-05-12T20:50:55.157Z [INFO] server-0361ec8a-bd78-8b73-5104-81753432ea2b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0361ec8a-bd78-8b73-5104-81753432ea2b Address:0361ec8a-bd78-8b73-5104-81753432ea2b} {Suffrage:Voter ID:server-f05f0835-dd65-d347-e01e-a500d5fbd8bc Address:f05f0835-dd65-d347-e01e-a500d5fbd8bc} {Suffrage:Voter ID:server-23e9ccbc-ab82-62f6-60cd-2bf5a9359c76 Address:23e9ccbc-ab82-62f6-60cd-2bf5a9359c76}]" -2024-05-12T20:50:55.157Z [INFO] server-f05f0835-dd65-d347-e01e-a500d5fbd8bc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0361ec8a-bd78-8b73-5104-81753432ea2b Address:0361ec8a-bd78-8b73-5104-81753432ea2b} {Suffrage:Voter ID:server-f05f0835-dd65-d347-e01e-a500d5fbd8bc Address:f05f0835-dd65-d347-e01e-a500d5fbd8bc} {Suffrage:Voter ID:server-23e9ccbc-ab82-62f6-60cd-2bf5a9359c76 Address:23e9ccbc-ab82-62f6-60cd-2bf5a9359c76}]" -2024-05-12T20:50:55.157Z [INFO] server-23e9ccbc-ab82-62f6-60cd-2bf5a9359c76: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0361ec8a-bd78-8b73-5104-81753432ea2b Address:0361ec8a-bd78-8b73-5104-81753432ea2b} {Suffrage:Voter ID:server-f05f0835-dd65-d347-e01e-a500d5fbd8bc Address:f05f0835-dd65-d347-e01e-a500d5fbd8bc} {Suffrage:Voter ID:server-23e9ccbc-ab82-62f6-60cd-2bf5a9359c76 Address:23e9ccbc-ab82-62f6-60cd-2bf5a9359c76}]" -2024-05-12T20:50:55.157Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:55.157Z [INFO] server-0361ec8a-bd78-8b73-5104-81753432ea2b: entering follower state: follower="Node at 0361ec8a-bd78-8b73-5104-81753432ea2b [Follower]" leader-address= leader-id= -2024-05-12T20:50:55.157Z [INFO] server-f05f0835-dd65-d347-e01e-a500d5fbd8bc: entering follower state: follower="Node at f05f0835-dd65-d347-e01e-a500d5fbd8bc [Follower]" leader-address= leader-id= -2024-05-12T20:50:55.157Z [INFO] server-23e9ccbc-ab82-62f6-60cd-2bf5a9359c76: entering follower state: follower="Node at 23e9ccbc-ab82-62f6-60cd-2bf5a9359c76 [Follower]" leader-address= leader-id= -2024-05-12T20:50:55.727Z [WARN] server-23e9ccbc-ab82-62f6-60cd-2bf5a9359c76: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:55.727Z [INFO] server-23e9ccbc-ab82-62f6-60cd-2bf5a9359c76: entering candidate state: node="Node at 23e9ccbc-ab82-62f6-60cd-2bf5a9359c76 [Candidate]" term=2 -2024-05-12T20:50:55.727Z [INFO] server-23e9ccbc-ab82-62f6-60cd-2bf5a9359c76: election won: term=2 tally=2 -2024-05-12T20:50:55.727Z [INFO] server-23e9ccbc-ab82-62f6-60cd-2bf5a9359c76: entering leader state: leader="Node at 23e9ccbc-ab82-62f6-60cd-2bf5a9359c76 [Leader]" -2024-05-12T20:50:55.727Z [INFO] server-23e9ccbc-ab82-62f6-60cd-2bf5a9359c76: added peer, starting replication: peer=server-0361ec8a-bd78-8b73-5104-81753432ea2b -2024-05-12T20:50:55.727Z [INFO] server-23e9ccbc-ab82-62f6-60cd-2bf5a9359c76: added peer, starting replication: peer=server-f05f0835-dd65-d347-e01e-a500d5fbd8bc -2024-05-12T20:50:55.727Z [INFO] server-23e9ccbc-ab82-62f6-60cd-2bf5a9359c76: pipelining replication: peer="{Voter server-f05f0835-dd65-d347-e01e-a500d5fbd8bc f05f0835-dd65-d347-e01e-a500d5fbd8bc}" -2024-05-12T20:50:55.727Z [INFO] server-23e9ccbc-ab82-62f6-60cd-2bf5a9359c76: pipelining replication: peer="{Voter server-0361ec8a-bd78-8b73-5104-81753432ea2b 0361ec8a-bd78-8b73-5104-81753432ea2b}" -2024-05-12T20:50:56.734Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:55.728043369 +0000 UTC m=+67.008743695 (1 nodes), 570.513915ms from start of poll, 570.915598ms from cluster start. Timeout at 2024-05-12 20:50:56.734752769 +0000 UTC m=+68.015456228, 1.006712533s after stability -2024-05-12T20:50:56.736Z [INFO] server-23e9ccbc-ab82-62f6-60cd-2bf5a9359c76: starting snapshot up to: index=12 -2024-05-12T20:50:56.736Z [INFO] creating new snapshot: path=/tmp/raft186734292/snapshots/2-12-1715547056736.tmp -2024-05-12T20:50:56.737Z [INFO] server-23e9ccbc-ab82-62f6-60cd-2bf5a9359c76: snapshot complete up to: index=12 -2024-05-12T20:50:56.740Z [INFO] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5 Address:7dc93746-ced7-09e1-15d1-6b3f16aae3f5} {Suffrage:Voter ID:server-25f10caa-3a82-5ead-7e58-1816f5632b03 Address:25f10caa-3a82-5ead-7e58-1816f5632b03} {Suffrage:Voter ID:server-148bec15-4525-d6d9-b7be-b02cacab8409 Address:148bec15-4525-d6d9-b7be-b02cacab8409}]" -2024-05-12T20:50:56.740Z [INFO] server-25f10caa-3a82-5ead-7e58-1816f5632b03: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5 Address:7dc93746-ced7-09e1-15d1-6b3f16aae3f5} {Suffrage:Voter ID:server-25f10caa-3a82-5ead-7e58-1816f5632b03 Address:25f10caa-3a82-5ead-7e58-1816f5632b03} {Suffrage:Voter ID:server-148bec15-4525-d6d9-b7be-b02cacab8409 Address:148bec15-4525-d6d9-b7be-b02cacab8409}]" -2024-05-12T20:50:56.740Z [INFO] server-148bec15-4525-d6d9-b7be-b02cacab8409: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5 Address:7dc93746-ced7-09e1-15d1-6b3f16aae3f5} {Suffrage:Voter ID:server-25f10caa-3a82-5ead-7e58-1816f5632b03 Address:25f10caa-3a82-5ead-7e58-1816f5632b03} {Suffrage:Voter ID:server-148bec15-4525-d6d9-b7be-b02cacab8409 Address:148bec15-4525-d6d9-b7be-b02cacab8409}]" -2024-05-12T20:50:56.740Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:56.740Z [INFO] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: entering follower state: follower="Node at 7dc93746-ced7-09e1-15d1-6b3f16aae3f5 [Follower]" leader-address= leader-id= -2024-05-12T20:50:56.740Z [INFO] server-25f10caa-3a82-5ead-7e58-1816f5632b03: entering follower state: follower="Node at 25f10caa-3a82-5ead-7e58-1816f5632b03 [Follower]" leader-address= leader-id= -2024-05-12T20:50:56.740Z [INFO] server-148bec15-4525-d6d9-b7be-b02cacab8409: entering follower state: follower="Node at 148bec15-4525-d6d9-b7be-b02cacab8409 [Follower]" leader-address= leader-id= -2024-05-12T20:50:57.426Z [WARN] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:57.426Z [INFO] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: entering candidate state: node="Node at 7dc93746-ced7-09e1-15d1-6b3f16aae3f5 [Candidate]" term=2 -2024-05-12T20:50:57.427Z [INFO] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: election won: term=2 tally=2 -2024-05-12T20:50:57.427Z [INFO] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: entering leader state: leader="Node at 7dc93746-ced7-09e1-15d1-6b3f16aae3f5 [Leader]" -2024-05-12T20:50:57.427Z [INFO] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: added peer, starting replication: peer=server-25f10caa-3a82-5ead-7e58-1816f5632b03 -2024-05-12T20:50:57.427Z [INFO] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: added peer, starting replication: peer=server-148bec15-4525-d6d9-b7be-b02cacab8409 -2024-05-12T20:50:57.427Z [INFO] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: pipelining replication: peer="{Voter server-25f10caa-3a82-5ead-7e58-1816f5632b03 25f10caa-3a82-5ead-7e58-1816f5632b03}" -2024-05-12T20:50:57.427Z [INFO] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: pipelining replication: peer="{Voter server-148bec15-4525-d6d9-b7be-b02cacab8409 148bec15-4525-d6d9-b7be-b02cacab8409}" -2024-05-12T20:50:58.434Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:57.427428288 +0000 UTC m=+68.708128674 (1 nodes), 686.931667ms from start of poll, 687.412874ms from cluster start. Timeout at 2024-05-12 20:50:58.434786176 +0000 UTC m=+69.715486706, 1.007358032s after stability -2024-05-12T20:50:58.435Z [INFO] creating new snapshot: path=/tmp/raft314831891/snapshots/2-113-1715547058435.tmp -2024-05-12T20:50:58.436Z [INFO] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: copied to local snapshot: bytes=71 -2024-05-12T20:50:58.436Z [INFO] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: snapshot restore progress: id=2-113-1715547058435 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:58.436Z [INFO] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: removing all old logs from log store -2024-05-12T20:50:58.436Z [INFO] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: compacting logs: from=1 to=2 -2024-05-12T20:50:58.436Z [INFO] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: restored user snapshot: index=113 -2024-05-12T20:50:58.437Z [ERROR] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: failed to get log: index=2 error="log not found" -2024-05-12T20:50:58.437Z [INFO] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: aborting pipeline replication: peer="{Voter server-148bec15-4525-d6d9-b7be-b02cacab8409 148bec15-4525-d6d9-b7be-b02cacab8409}" -2024-05-12T20:50:58.437Z [ERROR] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: failed to get log: index=2 error="log not found" -2024-05-12T20:50:58.437Z [INFO] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: aborting pipeline replication: peer="{Voter server-25f10caa-3a82-5ead-7e58-1816f5632b03 25f10caa-3a82-5ead-7e58-1816f5632b03}" -2024-05-12T20:50:58.443Z [ERROR] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: failed to get log: index=2 error="log not found" -2024-05-12T20:50:58.443Z [INFO] creating new snapshot: path=/tmp/raft3178573115/snapshots/2-113-1715547058443.tmp -2024-05-12T20:50:58.443Z [INFO] server-148bec15-4525-d6d9-b7be-b02cacab8409: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:58.444Z [INFO] server-148bec15-4525-d6d9-b7be-b02cacab8409: copied to local snapshot: bytes=71 -2024-05-12T20:50:58.444Z [INFO] server-148bec15-4525-d6d9-b7be-b02cacab8409: snapshot restore progress: id=2-113-1715547058443 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:58.444Z [INFO] server-148bec15-4525-d6d9-b7be-b02cacab8409: removing all old logs from log store -2024-05-12T20:50:58.444Z [INFO] server-148bec15-4525-d6d9-b7be-b02cacab8409: compacting logs: from=1 to=2 -2024-05-12T20:50:58.444Z [INFO] server-148bec15-4525-d6d9-b7be-b02cacab8409: Installed remote snapshot -2024-05-12T20:50:58.444Z [ERROR] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: failed to get log: index=2 error="log not found" -2024-05-12T20:50:58.445Z [INFO] creating new snapshot: path=/tmp/raft4244441847/snapshots/2-113-1715547058445.tmp -2024-05-12T20:50:58.445Z [INFO] server-25f10caa-3a82-5ead-7e58-1816f5632b03: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:58.445Z [INFO] server-25f10caa-3a82-5ead-7e58-1816f5632b03: copied to local snapshot: bytes=71 -2024-05-12T20:50:58.446Z [INFO] server-25f10caa-3a82-5ead-7e58-1816f5632b03: snapshot restore progress: id=2-113-1715547058445 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:50:58.446Z [INFO] server-25f10caa-3a82-5ead-7e58-1816f5632b03: removing all old logs from log store -2024-05-12T20:50:58.446Z [INFO] server-25f10caa-3a82-5ead-7e58-1816f5632b03: compacting logs: from=1 to=2 -2024-05-12T20:50:58.446Z [INFO] server-25f10caa-3a82-5ead-7e58-1816f5632b03: Installed remote snapshot -2024-05-12T20:50:58.446Z [INFO] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: pipelining replication: peer="{Voter server-25f10caa-3a82-5ead-7e58-1816f5632b03 25f10caa-3a82-5ead-7e58-1816f5632b03}" -2024-05-12T20:50:58.446Z [INFO] server-7dc93746-ced7-09e1-15d1-6b3f16aae3f5: pipelining replication: peer="{Voter server-148bec15-4525-d6d9-b7be-b02cacab8409 148bec15-4525-d6d9-b7be-b02cacab8409}" -2024-05-12T20:50:58.446Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:50:59.463Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:50:58.446558066 +0000 UTC m=+69.727258439 (2 nodes), 1.741µs from start of poll, 1.706542639s from cluster start. Timeout at 2024-05-12 20:50:59.463801227 +0000 UTC m=+70.744501584, 1.017243145s after stability -2024-05-12T20:50:59.487Z [INFO] server-23e9ccbc-ab82-62f6-60cd-2bf5a9359c76: aborting pipeline replication: peer="{Voter server-f05f0835-dd65-d347-e01e-a500d5fbd8bc f05f0835-dd65-d347-e01e-a500d5fbd8bc}" -2024-05-12T20:50:59.487Z [INFO] server-23e9ccbc-ab82-62f6-60cd-2bf5a9359c76: aborting pipeline replication: peer="{Voter server-0361ec8a-bd78-8b73-5104-81753432ea2b 0361ec8a-bd78-8b73-5104-81753432ea2b}" +2025-06-15T03:17:29.667Z [INFO] server-dcfe4088-c351-1103-e1f7-08fb55de91e8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dcfe4088-c351-1103-e1f7-08fb55de91e8 Address:dcfe4088-c351-1103-e1f7-08fb55de91e8} {Suffrage:Voter ID:server-d6c3f5b6-ea2b-02a2-b4ea-5dc5e1e9b577 Address:d6c3f5b6-ea2b-02a2-b4ea-5dc5e1e9b577} {Suffrage:Voter ID:server-d42fdfaa-c611-0ac0-e63e-5ecac17c10cc Address:d42fdfaa-c611-0ac0-e63e-5ecac17c10cc}]" +2025-06-15T03:17:29.667Z [INFO] server-d6c3f5b6-ea2b-02a2-b4ea-5dc5e1e9b577: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dcfe4088-c351-1103-e1f7-08fb55de91e8 Address:dcfe4088-c351-1103-e1f7-08fb55de91e8} {Suffrage:Voter ID:server-d6c3f5b6-ea2b-02a2-b4ea-5dc5e1e9b577 Address:d6c3f5b6-ea2b-02a2-b4ea-5dc5e1e9b577} {Suffrage:Voter ID:server-d42fdfaa-c611-0ac0-e63e-5ecac17c10cc Address:d42fdfaa-c611-0ac0-e63e-5ecac17c10cc}]" +2025-06-15T03:17:29.667Z [INFO] server-d42fdfaa-c611-0ac0-e63e-5ecac17c10cc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dcfe4088-c351-1103-e1f7-08fb55de91e8 Address:dcfe4088-c351-1103-e1f7-08fb55de91e8} {Suffrage:Voter ID:server-d6c3f5b6-ea2b-02a2-b4ea-5dc5e1e9b577 Address:d6c3f5b6-ea2b-02a2-b4ea-5dc5e1e9b577} {Suffrage:Voter ID:server-d42fdfaa-c611-0ac0-e63e-5ecac17c10cc Address:d42fdfaa-c611-0ac0-e63e-5ecac17c10cc}]" +2025-06-15T03:17:29.667Z [INFO] server-d6c3f5b6-ea2b-02a2-b4ea-5dc5e1e9b577: entering follower state: follower="Node at d6c3f5b6-ea2b-02a2-b4ea-5dc5e1e9b577 [Follower]" leader-address= leader-id= +2025-06-15T03:17:29.668Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:29.667Z [INFO] server-dcfe4088-c351-1103-e1f7-08fb55de91e8: entering follower state: follower="Node at dcfe4088-c351-1103-e1f7-08fb55de91e8 [Follower]" leader-address= leader-id= +2025-06-15T03:17:29.668Z [INFO] server-d42fdfaa-c611-0ac0-e63e-5ecac17c10cc: entering follower state: follower="Node at d42fdfaa-c611-0ac0-e63e-5ecac17c10cc [Follower]" leader-address= leader-id= +2025-06-15T03:17:30.195Z [WARN] server-d42fdfaa-c611-0ac0-e63e-5ecac17c10cc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:30.195Z [INFO] server-d42fdfaa-c611-0ac0-e63e-5ecac17c10cc: entering candidate state: node="Node at d42fdfaa-c611-0ac0-e63e-5ecac17c10cc [Candidate]" term=2 +2025-06-15T03:17:30.195Z [INFO] server-d42fdfaa-c611-0ac0-e63e-5ecac17c10cc: election won: term=2 tally=2 +2025-06-15T03:17:30.195Z [INFO] server-d42fdfaa-c611-0ac0-e63e-5ecac17c10cc: entering leader state: leader="Node at d42fdfaa-c611-0ac0-e63e-5ecac17c10cc [Leader]" +2025-06-15T03:17:30.195Z [INFO] server-d42fdfaa-c611-0ac0-e63e-5ecac17c10cc: added peer, starting replication: peer=server-dcfe4088-c351-1103-e1f7-08fb55de91e8 +2025-06-15T03:17:30.195Z [INFO] server-d42fdfaa-c611-0ac0-e63e-5ecac17c10cc: added peer, starting replication: peer=server-d6c3f5b6-ea2b-02a2-b4ea-5dc5e1e9b577 +2025-06-15T03:17:30.195Z [INFO] server-d42fdfaa-c611-0ac0-e63e-5ecac17c10cc: pipelining replication: peer="{Voter server-dcfe4088-c351-1103-e1f7-08fb55de91e8 dcfe4088-c351-1103-e1f7-08fb55de91e8}" +2025-06-15T03:17:30.195Z [INFO] server-d42fdfaa-c611-0ac0-e63e-5ecac17c10cc: pipelining replication: peer="{Voter server-d6c3f5b6-ea2b-02a2-b4ea-5dc5e1e9b577 d6c3f5b6-ea2b-02a2-b4ea-5dc5e1e9b577}" +2025-06-15T03:17:31.201Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:30.19545576 +0000 UTC m=+65.907133925 (1 nodes), 527.445312ms from start of poll, 527.726575ms from cluster start. Timeout at 2025-06-15 03:17:31.201115774 +0000 UTC m=+66.912793973, 1.005660048s after stability +2025-06-15T03:17:31.201Z [INFO] server-d42fdfaa-c611-0ac0-e63e-5ecac17c10cc: starting snapshot up to: index=12 +2025-06-15T03:17:31.201Z [INFO] creating new snapshot: path=/tmp/raft907802943/snapshots/2-12-1749957451201.tmp +2025-06-15T03:17:31.202Z [INFO] server-d42fdfaa-c611-0ac0-e63e-5ecac17c10cc: snapshot complete up to: index=12 +2025-06-15T03:17:31.202Z [INFO] server-03d59252-b130-6ae0-5198-7b709d2ca95d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03d59252-b130-6ae0-5198-7b709d2ca95d Address:03d59252-b130-6ae0-5198-7b709d2ca95d} {Suffrage:Voter ID:server-1f24d0f5-70c8-f43e-100e-8738f667c657 Address:1f24d0f5-70c8-f43e-100e-8738f667c657} {Suffrage:Voter ID:server-a78734ae-f77d-d8ac-fe84-8b43c31eb58b Address:a78734ae-f77d-d8ac-fe84-8b43c31eb58b}]" +2025-06-15T03:17:31.203Z [INFO] server-1f24d0f5-70c8-f43e-100e-8738f667c657: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03d59252-b130-6ae0-5198-7b709d2ca95d Address:03d59252-b130-6ae0-5198-7b709d2ca95d} {Suffrage:Voter ID:server-1f24d0f5-70c8-f43e-100e-8738f667c657 Address:1f24d0f5-70c8-f43e-100e-8738f667c657} {Suffrage:Voter ID:server-a78734ae-f77d-d8ac-fe84-8b43c31eb58b Address:a78734ae-f77d-d8ac-fe84-8b43c31eb58b}]" +2025-06-15T03:17:31.203Z [INFO] server-03d59252-b130-6ae0-5198-7b709d2ca95d: entering follower state: follower="Node at 03d59252-b130-6ae0-5198-7b709d2ca95d [Follower]" leader-address= leader-id= +2025-06-15T03:17:31.203Z [INFO] server-1f24d0f5-70c8-f43e-100e-8738f667c657: entering follower state: follower="Node at 1f24d0f5-70c8-f43e-100e-8738f667c657 [Follower]" leader-address= leader-id= +2025-06-15T03:17:31.203Z [INFO] server-a78734ae-f77d-d8ac-fe84-8b43c31eb58b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03d59252-b130-6ae0-5198-7b709d2ca95d Address:03d59252-b130-6ae0-5198-7b709d2ca95d} {Suffrage:Voter ID:server-1f24d0f5-70c8-f43e-100e-8738f667c657 Address:1f24d0f5-70c8-f43e-100e-8738f667c657} {Suffrage:Voter ID:server-a78734ae-f77d-d8ac-fe84-8b43c31eb58b Address:a78734ae-f77d-d8ac-fe84-8b43c31eb58b}]" +2025-06-15T03:17:31.203Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:31.203Z [INFO] server-a78734ae-f77d-d8ac-fe84-8b43c31eb58b: entering follower state: follower="Node at a78734ae-f77d-d8ac-fe84-8b43c31eb58b [Follower]" leader-address= leader-id= +2025-06-15T03:17:31.884Z [WARN] server-03d59252-b130-6ae0-5198-7b709d2ca95d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:31.884Z [INFO] server-03d59252-b130-6ae0-5198-7b709d2ca95d: entering candidate state: node="Node at 03d59252-b130-6ae0-5198-7b709d2ca95d [Candidate]" term=2 +2025-06-15T03:17:31.884Z [INFO] server-03d59252-b130-6ae0-5198-7b709d2ca95d: election won: term=2 tally=2 +2025-06-15T03:17:31.884Z [INFO] server-03d59252-b130-6ae0-5198-7b709d2ca95d: entering leader state: leader="Node at 03d59252-b130-6ae0-5198-7b709d2ca95d [Leader]" +2025-06-15T03:17:31.884Z [INFO] server-03d59252-b130-6ae0-5198-7b709d2ca95d: added peer, starting replication: peer=server-1f24d0f5-70c8-f43e-100e-8738f667c657 +2025-06-15T03:17:31.884Z [INFO] server-03d59252-b130-6ae0-5198-7b709d2ca95d: added peer, starting replication: peer=server-a78734ae-f77d-d8ac-fe84-8b43c31eb58b +2025-06-15T03:17:31.884Z [INFO] server-03d59252-b130-6ae0-5198-7b709d2ca95d: pipelining replication: peer="{Voter server-1f24d0f5-70c8-f43e-100e-8738f667c657 1f24d0f5-70c8-f43e-100e-8738f667c657}" +2025-06-15T03:17:31.884Z [INFO] server-03d59252-b130-6ae0-5198-7b709d2ca95d: pipelining replication: peer="{Voter server-a78734ae-f77d-d8ac-fe84-8b43c31eb58b a78734ae-f77d-d8ac-fe84-8b43c31eb58b}" +2025-06-15T03:17:32.889Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:31.884435176 +0000 UTC m=+67.596113342 (1 nodes), 681.342613ms from start of poll, 681.608038ms from cluster start. Timeout at 2025-06-15 03:17:32.889687495 +0000 UTC m=+68.601365694, 1.005252352s after stability +2025-06-15T03:17:32.890Z [INFO] creating new snapshot: path=/tmp/raft2468963816/snapshots/2-113-1749957452890.tmp +2025-06-15T03:17:32.890Z [INFO] server-03d59252-b130-6ae0-5198-7b709d2ca95d: copied to local snapshot: bytes=71 +2025-06-15T03:17:32.890Z [INFO] server-03d59252-b130-6ae0-5198-7b709d2ca95d: snapshot restore progress: id=2-113-1749957452890 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:32.890Z [INFO] server-03d59252-b130-6ae0-5198-7b709d2ca95d: removing all old logs from log store +2025-06-15T03:17:32.890Z [INFO] server-03d59252-b130-6ae0-5198-7b709d2ca95d: compacting logs: from=1 to=2 +2025-06-15T03:17:32.890Z [INFO] server-03d59252-b130-6ae0-5198-7b709d2ca95d: restored user snapshot: index=113 +2025-06-15T03:17:32.890Z [ERROR] server-03d59252-b130-6ae0-5198-7b709d2ca95d: failed to get log: index=2 error="log not found" +2025-06-15T03:17:32.890Z [ERROR] server-03d59252-b130-6ae0-5198-7b709d2ca95d: failed to get log: index=2 error="log not found" +2025-06-15T03:17:32.890Z [INFO] server-03d59252-b130-6ae0-5198-7b709d2ca95d: aborting pipeline replication: peer="{Voter server-a78734ae-f77d-d8ac-fe84-8b43c31eb58b a78734ae-f77d-d8ac-fe84-8b43c31eb58b}" +2025-06-15T03:17:32.890Z [INFO] server-03d59252-b130-6ae0-5198-7b709d2ca95d: aborting pipeline replication: peer="{Voter server-1f24d0f5-70c8-f43e-100e-8738f667c657 1f24d0f5-70c8-f43e-100e-8738f667c657}" +2025-06-15T03:17:32.896Z [ERROR] server-03d59252-b130-6ae0-5198-7b709d2ca95d: failed to get log: index=2 error="log not found" +2025-06-15T03:17:32.896Z [ERROR] server-03d59252-b130-6ae0-5198-7b709d2ca95d: failed to get log: index=2 error="log not found" +2025-06-15T03:17:32.896Z [INFO] creating new snapshot: path=/tmp/raft697924630/snapshots/2-113-1749957452896.tmp +2025-06-15T03:17:32.896Z [INFO] creating new snapshot: path=/tmp/raft485434134/snapshots/2-113-1749957452896.tmp +2025-06-15T03:17:32.897Z [INFO] server-a78734ae-f77d-d8ac-fe84-8b43c31eb58b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:32.897Z [INFO] server-1f24d0f5-70c8-f43e-100e-8738f667c657: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:32.897Z [INFO] server-a78734ae-f77d-d8ac-fe84-8b43c31eb58b: copied to local snapshot: bytes=71 +2025-06-15T03:17:32.897Z [INFO] server-1f24d0f5-70c8-f43e-100e-8738f667c657: copied to local snapshot: bytes=71 +2025-06-15T03:17:32.897Z [INFO] server-a78734ae-f77d-d8ac-fe84-8b43c31eb58b: snapshot restore progress: id=2-113-1749957452896 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:32.897Z [INFO] server-a78734ae-f77d-d8ac-fe84-8b43c31eb58b: removing all old logs from log store +2025-06-15T03:17:32.897Z [INFO] server-a78734ae-f77d-d8ac-fe84-8b43c31eb58b: compacting logs: from=1 to=2 +2025-06-15T03:17:32.897Z [INFO] server-a78734ae-f77d-d8ac-fe84-8b43c31eb58b: Installed remote snapshot +2025-06-15T03:17:32.897Z [INFO] server-03d59252-b130-6ae0-5198-7b709d2ca95d: pipelining replication: peer="{Voter server-a78734ae-f77d-d8ac-fe84-8b43c31eb58b a78734ae-f77d-d8ac-fe84-8b43c31eb58b}" +2025-06-15T03:17:32.897Z [INFO] server-1f24d0f5-70c8-f43e-100e-8738f667c657: snapshot restore progress: id=2-113-1749957452896 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:32.897Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:17:32.897Z [INFO] server-1f24d0f5-70c8-f43e-100e-8738f667c657: removing all old logs from log store +2025-06-15T03:17:32.897Z [INFO] server-1f24d0f5-70c8-f43e-100e-8738f667c657: compacting logs: from=1 to=2 +2025-06-15T03:17:32.897Z [INFO] server-1f24d0f5-70c8-f43e-100e-8738f667c657: Installed remote snapshot +2025-06-15T03:17:32.898Z [INFO] server-03d59252-b130-6ae0-5198-7b709d2ca95d: pipelining replication: peer="{Voter server-1f24d0f5-70c8-f43e-100e-8738f667c657 1f24d0f5-70c8-f43e-100e-8738f667c657}" +2025-06-15T03:17:33.903Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:17:32.897952058 +0000 UTC m=+68.609630223 (2 nodes), 506ns from start of poll, 1.695124919s from cluster start. Timeout at 2025-06-15 03:17:33.903840761 +0000 UTC m=+69.615518964, 1.005888741s after stability +2025-06-15T03:17:33.920Z [INFO] server-d42fdfaa-c611-0ac0-e63e-5ecac17c10cc: aborting pipeline replication: peer="{Voter server-d6c3f5b6-ea2b-02a2-b4ea-5dc5e1e9b577 d6c3f5b6-ea2b-02a2-b4ea-5dc5e1e9b577}" +2025-06-15T03:17:33.920Z [INFO] server-d42fdfaa-c611-0ac0-e63e-5ecac17c10cc: aborting pipeline replication: peer="{Voter server-dcfe4088-c351-1103-e1f7-08fb55de91e8 dcfe4088-c351-1103-e1f7-08fb55de91e8}" === RUN TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false -2024-05-12T20:50:59.490Z [INFO] server-0195eadb-dd26-0920-5e02-b8113ec83c1a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0195eadb-dd26-0920-5e02-b8113ec83c1a Address:0195eadb-dd26-0920-5e02-b8113ec83c1a} {Suffrage:Voter ID:server-d9093aae-11e0-4e0b-9cab-704259a0c56c Address:d9093aae-11e0-4e0b-9cab-704259a0c56c} {Suffrage:Voter ID:server-03e67525-7d24-7798-583a-6b94c23002e5 Address:03e67525-7d24-7798-583a-6b94c23002e5}]" -2024-05-12T20:50:59.490Z [INFO] server-d9093aae-11e0-4e0b-9cab-704259a0c56c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0195eadb-dd26-0920-5e02-b8113ec83c1a Address:0195eadb-dd26-0920-5e02-b8113ec83c1a} {Suffrage:Voter ID:server-d9093aae-11e0-4e0b-9cab-704259a0c56c Address:d9093aae-11e0-4e0b-9cab-704259a0c56c} {Suffrage:Voter ID:server-03e67525-7d24-7798-583a-6b94c23002e5 Address:03e67525-7d24-7798-583a-6b94c23002e5}]" -2024-05-12T20:50:59.491Z [INFO] server-0195eadb-dd26-0920-5e02-b8113ec83c1a: entering follower state: follower="Node at 0195eadb-dd26-0920-5e02-b8113ec83c1a [Follower]" leader-address= leader-id= -2024-05-12T20:50:59.491Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0195eadb-dd26-0920-5e02-b8113ec83c1a Address:0195eadb-dd26-0920-5e02-b8113ec83c1a} {Suffrage:Voter ID:server-d9093aae-11e0-4e0b-9cab-704259a0c56c Address:d9093aae-11e0-4e0b-9cab-704259a0c56c} {Suffrage:Voter ID:server-03e67525-7d24-7798-583a-6b94c23002e5 Address:03e67525-7d24-7798-583a-6b94c23002e5}]" -2024-05-12T20:50:59.491Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:50:59.491Z [INFO] server-d9093aae-11e0-4e0b-9cab-704259a0c56c: entering follower state: follower="Node at d9093aae-11e0-4e0b-9cab-704259a0c56c [Follower]" leader-address= leader-id= -2024-05-12T20:50:59.491Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: entering follower state: follower="Node at 03e67525-7d24-7798-583a-6b94c23002e5 [Follower]" leader-address= leader-id= -2024-05-12T20:50:59.994Z [WARN] server-03e67525-7d24-7798-583a-6b94c23002e5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:50:59.994Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: entering candidate state: node="Node at 03e67525-7d24-7798-583a-6b94c23002e5 [Candidate]" term=2 -2024-05-12T20:50:59.995Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: election won: term=2 tally=2 -2024-05-12T20:50:59.995Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: entering leader state: leader="Node at 03e67525-7d24-7798-583a-6b94c23002e5 [Leader]" -2024-05-12T20:50:59.995Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: added peer, starting replication: peer=server-0195eadb-dd26-0920-5e02-b8113ec83c1a -2024-05-12T20:50:59.995Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: added peer, starting replication: peer=server-d9093aae-11e0-4e0b-9cab-704259a0c56c -2024-05-12T20:50:59.995Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: pipelining replication: peer="{Voter server-d9093aae-11e0-4e0b-9cab-704259a0c56c d9093aae-11e0-4e0b-9cab-704259a0c56c}" -2024-05-12T20:50:59.995Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: pipelining replication: peer="{Voter server-0195eadb-dd26-0920-5e02-b8113ec83c1a 0195eadb-dd26-0920-5e02-b8113ec83c1a}" -2024-05-12T20:51:01.002Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:50:59.995360323 +0000 UTC m=+71.276060711 (1 nodes), 503.880024ms from start of poll, 505.041626ms from cluster start. Timeout at 2024-05-12 20:51:01.002925949 +0000 UTC m=+72.283625844, 1.007565133s after stability -2024-05-12T20:51:01.004Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: starting snapshot up to: index=12 -2024-05-12T20:51:01.004Z [INFO] creating new snapshot: path=/tmp/raft462638532/snapshots/2-12-1715547061004.tmp -2024-05-12T20:51:01.005Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: snapshot complete up to: index=12 -2024-05-12T20:51:01.012Z [INFO] creating new snapshot: path=/tmp/raft462638532/snapshots/2-1013-1715547061012.tmp -2024-05-12T20:51:01.013Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: copied to local snapshot: bytes=71 -2024-05-12T20:51:01.014Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: snapshot restore progress: id=2-1013-1715547061012 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:01.014Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: restored user snapshot: index=1013 -2024-05-12T20:51:01.014Z [ERROR] server-03e67525-7d24-7798-583a-6b94c23002e5: failed to get log: index=23 error="log not found" -2024-05-12T20:51:01.014Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: aborting pipeline replication: peer="{Voter server-d9093aae-11e0-4e0b-9cab-704259a0c56c d9093aae-11e0-4e0b-9cab-704259a0c56c}" -2024-05-12T20:51:01.014Z [ERROR] server-03e67525-7d24-7798-583a-6b94c23002e5: failed to get log: index=23 error="log not found" -2024-05-12T20:51:01.014Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: aborting pipeline replication: peer="{Voter server-0195eadb-dd26-0920-5e02-b8113ec83c1a 0195eadb-dd26-0920-5e02-b8113ec83c1a}" -2024-05-12T20:51:01.023Z [ERROR] server-03e67525-7d24-7798-583a-6b94c23002e5: failed to get log: index=23 error="log not found" -2024-05-12T20:51:01.023Z [ERROR] server-03e67525-7d24-7798-583a-6b94c23002e5: failed to get log: index=23 error="log not found" -2024-05-12T20:51:01.023Z [INFO] creating new snapshot: path=/tmp/raft227740278/snapshots/2-1013-1715547061023.tmp -2024-05-12T20:51:01.024Z [INFO] creating new snapshot: path=/tmp/raft1793715070/snapshots/2-1013-1715547061024.tmp -2024-05-12T20:51:01.024Z [INFO] server-0195eadb-dd26-0920-5e02-b8113ec83c1a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:01.024Z [INFO] server-d9093aae-11e0-4e0b-9cab-704259a0c56c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:01.024Z [INFO] server-d9093aae-11e0-4e0b-9cab-704259a0c56c: copied to local snapshot: bytes=71 -2024-05-12T20:51:01.025Z [INFO] server-0195eadb-dd26-0920-5e02-b8113ec83c1a: copied to local snapshot: bytes=71 -2024-05-12T20:51:01.025Z [INFO] server-d9093aae-11e0-4e0b-9cab-704259a0c56c: snapshot restore progress: id=2-1013-1715547061024 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:01.025Z [INFO] server-d9093aae-11e0-4e0b-9cab-704259a0c56c: Installed remote snapshot -2024-05-12T20:51:01.025Z [WARN] server-d9093aae-11e0-4e0b-9cab-704259a0c56c: skipping application of old log: index=22 -2024-05-12T20:51:01.025Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: pipelining replication: peer="{Voter server-d9093aae-11e0-4e0b-9cab-704259a0c56c d9093aae-11e0-4e0b-9cab-704259a0c56c}" -2024-05-12T20:51:01.025Z [INFO] server-0195eadb-dd26-0920-5e02-b8113ec83c1a: snapshot restore progress: id=2-1013-1715547061023 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:01.025Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:01.025Z [INFO] server-0195eadb-dd26-0920-5e02-b8113ec83c1a: Installed remote snapshot -2024-05-12T20:51:01.025Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: pipelining replication: peer="{Voter server-0195eadb-dd26-0920-5e02-b8113ec83c1a 0195eadb-dd26-0920-5e02-b8113ec83c1a}" -2024-05-12T20:51:02.035Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:01.025587504 +0000 UTC m=+72.306287984 (2 nodes), 2.161µs from start of poll, 1.535268899s from cluster start. Timeout at 2024-05-12 20:51:02.034976039 +0000 UTC m=+73.315676460, 1.009388476s after stability -2024-05-12T20:51:02.051Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: aborting pipeline replication: peer="{Voter server-d9093aae-11e0-4e0b-9cab-704259a0c56c d9093aae-11e0-4e0b-9cab-704259a0c56c}" -2024-05-12T20:51:02.051Z [INFO] server-03e67525-7d24-7798-583a-6b94c23002e5: aborting pipeline replication: peer="{Voter server-0195eadb-dd26-0920-5e02-b8113ec83c1a 0195eadb-dd26-0920-5e02-b8113ec83c1a}" +2025-06-15T03:17:33.922Z [INFO] server-152336ba-1842-88cd-5607-e6b3b8a8953c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-152336ba-1842-88cd-5607-e6b3b8a8953c Address:152336ba-1842-88cd-5607-e6b3b8a8953c} {Suffrage:Voter ID:server-844d167d-48c2-676a-045b-b139e776e339 Address:844d167d-48c2-676a-045b-b139e776e339} {Suffrage:Voter ID:server-67628780-9a56-21bd-5ef7-c3d43c49b5e0 Address:67628780-9a56-21bd-5ef7-c3d43c49b5e0}]" +2025-06-15T03:17:33.923Z [INFO] server-152336ba-1842-88cd-5607-e6b3b8a8953c: entering follower state: follower="Node at 152336ba-1842-88cd-5607-e6b3b8a8953c [Follower]" leader-address= leader-id= +2025-06-15T03:17:33.923Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-152336ba-1842-88cd-5607-e6b3b8a8953c Address:152336ba-1842-88cd-5607-e6b3b8a8953c} {Suffrage:Voter ID:server-844d167d-48c2-676a-045b-b139e776e339 Address:844d167d-48c2-676a-045b-b139e776e339} {Suffrage:Voter ID:server-67628780-9a56-21bd-5ef7-c3d43c49b5e0 Address:67628780-9a56-21bd-5ef7-c3d43c49b5e0}]" +2025-06-15T03:17:33.923Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: entering follower state: follower="Node at 844d167d-48c2-676a-045b-b139e776e339 [Follower]" leader-address= leader-id= +2025-06-15T03:17:33.923Z [INFO] server-67628780-9a56-21bd-5ef7-c3d43c49b5e0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-152336ba-1842-88cd-5607-e6b3b8a8953c Address:152336ba-1842-88cd-5607-e6b3b8a8953c} {Suffrage:Voter ID:server-844d167d-48c2-676a-045b-b139e776e339 Address:844d167d-48c2-676a-045b-b139e776e339} {Suffrage:Voter ID:server-67628780-9a56-21bd-5ef7-c3d43c49b5e0 Address:67628780-9a56-21bd-5ef7-c3d43c49b5e0}]" +2025-06-15T03:17:33.923Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:33.923Z [INFO] server-67628780-9a56-21bd-5ef7-c3d43c49b5e0: entering follower state: follower="Node at 67628780-9a56-21bd-5ef7-c3d43c49b5e0 [Follower]" leader-address= leader-id= +2025-06-15T03:17:34.588Z [WARN] server-844d167d-48c2-676a-045b-b139e776e339: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:34.588Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: entering candidate state: node="Node at 844d167d-48c2-676a-045b-b139e776e339 [Candidate]" term=2 +2025-06-15T03:17:34.588Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: election won: term=2 tally=2 +2025-06-15T03:17:34.588Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: entering leader state: leader="Node at 844d167d-48c2-676a-045b-b139e776e339 [Leader]" +2025-06-15T03:17:34.588Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: added peer, starting replication: peer=server-152336ba-1842-88cd-5607-e6b3b8a8953c +2025-06-15T03:17:34.588Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: added peer, starting replication: peer=server-67628780-9a56-21bd-5ef7-c3d43c49b5e0 +2025-06-15T03:17:34.588Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: pipelining replication: peer="{Voter server-152336ba-1842-88cd-5607-e6b3b8a8953c 152336ba-1842-88cd-5607-e6b3b8a8953c}" +2025-06-15T03:17:34.588Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: pipelining replication: peer="{Voter server-67628780-9a56-21bd-5ef7-c3d43c49b5e0 67628780-9a56-21bd-5ef7-c3d43c49b5e0}" +2025-06-15T03:17:35.593Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:34.58853916 +0000 UTC m=+70.300217359 (1 nodes), 665.387861ms from start of poll, 665.632569ms from cluster start. Timeout at 2025-06-15 03:17:35.593611495 +0000 UTC m=+71.305289672, 1.005072313s after stability +2025-06-15T03:17:35.594Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: starting snapshot up to: index=12 +2025-06-15T03:17:35.594Z [INFO] creating new snapshot: path=/tmp/raft912256638/snapshots/2-12-1749957455594.tmp +2025-06-15T03:17:35.595Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: snapshot complete up to: index=12 +2025-06-15T03:17:35.595Z [INFO] creating new snapshot: path=/tmp/raft912256638/snapshots/2-1013-1749957455595.tmp +2025-06-15T03:17:35.596Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: copied to local snapshot: bytes=71 +2025-06-15T03:17:35.596Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: snapshot restore progress: id=2-1013-1749957455595 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:35.596Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: restored user snapshot: index=1013 +2025-06-15T03:17:35.596Z [ERROR] server-844d167d-48c2-676a-045b-b139e776e339: failed to get log: index=23 error="log not found" +2025-06-15T03:17:35.596Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: aborting pipeline replication: peer="{Voter server-67628780-9a56-21bd-5ef7-c3d43c49b5e0 67628780-9a56-21bd-5ef7-c3d43c49b5e0}" +2025-06-15T03:17:35.596Z [ERROR] server-844d167d-48c2-676a-045b-b139e776e339: failed to get log: index=23 error="log not found" +2025-06-15T03:17:35.596Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: aborting pipeline replication: peer="{Voter server-152336ba-1842-88cd-5607-e6b3b8a8953c 152336ba-1842-88cd-5607-e6b3b8a8953c}" +2025-06-15T03:17:35.605Z [ERROR] server-844d167d-48c2-676a-045b-b139e776e339: failed to get log: index=23 error="log not found" +2025-06-15T03:17:35.605Z [ERROR] server-844d167d-48c2-676a-045b-b139e776e339: failed to get log: index=23 error="log not found" +2025-06-15T03:17:35.605Z [INFO] creating new snapshot: path=/tmp/raft334034710/snapshots/2-1013-1749957455605.tmp +2025-06-15T03:17:35.605Z [INFO] creating new snapshot: path=/tmp/raft2401067313/snapshots/2-1013-1749957455605.tmp +2025-06-15T03:17:35.605Z [INFO] server-67628780-9a56-21bd-5ef7-c3d43c49b5e0: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:35.606Z [INFO] server-152336ba-1842-88cd-5607-e6b3b8a8953c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:35.606Z [INFO] server-67628780-9a56-21bd-5ef7-c3d43c49b5e0: copied to local snapshot: bytes=71 +2025-06-15T03:17:35.606Z [INFO] server-152336ba-1842-88cd-5607-e6b3b8a8953c: copied to local snapshot: bytes=71 +2025-06-15T03:17:35.606Z [INFO] server-67628780-9a56-21bd-5ef7-c3d43c49b5e0: snapshot restore progress: id=2-1013-1749957455605 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:35.606Z [INFO] server-67628780-9a56-21bd-5ef7-c3d43c49b5e0: Installed remote snapshot +2025-06-15T03:17:35.606Z [INFO] server-152336ba-1842-88cd-5607-e6b3b8a8953c: snapshot restore progress: id=2-1013-1749957455605 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:35.606Z [WARN] server-67628780-9a56-21bd-5ef7-c3d43c49b5e0: skipping application of old log: index=22 +2025-06-15T03:17:35.606Z [INFO] server-152336ba-1842-88cd-5607-e6b3b8a8953c: Installed remote snapshot +2025-06-15T03:17:35.606Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: pipelining replication: peer="{Voter server-67628780-9a56-21bd-5ef7-c3d43c49b5e0 67628780-9a56-21bd-5ef7-c3d43c49b5e0}" +2025-06-15T03:17:35.606Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:17:35.606Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: pipelining replication: peer="{Voter server-152336ba-1842-88cd-5607-e6b3b8a8953c 152336ba-1842-88cd-5607-e6b3b8a8953c}" +2025-06-15T03:17:36.611Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:17:35.606593374 +0000 UTC m=+71.318271535 (2 nodes), 486ns from start of poll, 1.683686745s from cluster start. Timeout at 2025-06-15 03:17:36.611733278 +0000 UTC m=+72.323411445, 1.00513991s after stability +2025-06-15T03:17:36.622Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: aborting pipeline replication: peer="{Voter server-67628780-9a56-21bd-5ef7-c3d43c49b5e0 67628780-9a56-21bd-5ef7-c3d43c49b5e0}" +2025-06-15T03:17:36.622Z [INFO] server-844d167d-48c2-676a-045b-b139e776e339: aborting pipeline replication: peer="{Voter server-152336ba-1842-88cd-5607-e6b3b8a8953c 152336ba-1842-88cd-5607-e6b3b8a8953c}" === RUN TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false -2024-05-12T20:51:02.054Z [INFO] server-e531e9ff-7c12-bf36-ae7e-7aea551ad781: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e531e9ff-7c12-bf36-ae7e-7aea551ad781 Address:e531e9ff-7c12-bf36-ae7e-7aea551ad781} {Suffrage:Voter ID:server-d8c33984-7a5f-53e9-1c2a-bafd6ababa8e Address:d8c33984-7a5f-53e9-1c2a-bafd6ababa8e} {Suffrage:Voter ID:server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5 Address:87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5}]" -2024-05-12T20:51:02.054Z [INFO] server-d8c33984-7a5f-53e9-1c2a-bafd6ababa8e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e531e9ff-7c12-bf36-ae7e-7aea551ad781 Address:e531e9ff-7c12-bf36-ae7e-7aea551ad781} {Suffrage:Voter ID:server-d8c33984-7a5f-53e9-1c2a-bafd6ababa8e Address:d8c33984-7a5f-53e9-1c2a-bafd6ababa8e} {Suffrage:Voter ID:server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5 Address:87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5}]" -2024-05-12T20:51:02.054Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e531e9ff-7c12-bf36-ae7e-7aea551ad781 Address:e531e9ff-7c12-bf36-ae7e-7aea551ad781} {Suffrage:Voter ID:server-d8c33984-7a5f-53e9-1c2a-bafd6ababa8e Address:d8c33984-7a5f-53e9-1c2a-bafd6ababa8e} {Suffrage:Voter ID:server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5 Address:87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5}]" -2024-05-12T20:51:02.054Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:02.054Z [INFO] server-e531e9ff-7c12-bf36-ae7e-7aea551ad781: entering follower state: follower="Node at e531e9ff-7c12-bf36-ae7e-7aea551ad781 [Follower]" leader-address= leader-id= -2024-05-12T20:51:02.054Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: entering follower state: follower="Node at 87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5 [Follower]" leader-address= leader-id= -2024-05-12T20:51:02.054Z [INFO] server-d8c33984-7a5f-53e9-1c2a-bafd6ababa8e: entering follower state: follower="Node at d8c33984-7a5f-53e9-1c2a-bafd6ababa8e [Follower]" leader-address= leader-id= -2024-05-12T20:51:02.759Z [WARN] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:02.759Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: entering candidate state: node="Node at 87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5 [Candidate]" term=2 -2024-05-12T20:51:02.759Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: election won: term=2 tally=2 -2024-05-12T20:51:02.759Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: entering leader state: leader="Node at 87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5 [Leader]" -2024-05-12T20:51:02.759Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: added peer, starting replication: peer=server-e531e9ff-7c12-bf36-ae7e-7aea551ad781 -2024-05-12T20:51:02.759Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: added peer, starting replication: peer=server-d8c33984-7a5f-53e9-1c2a-bafd6ababa8e -2024-05-12T20:51:02.759Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: pipelining replication: peer="{Voter server-d8c33984-7a5f-53e9-1c2a-bafd6ababa8e d8c33984-7a5f-53e9-1c2a-bafd6ababa8e}" -2024-05-12T20:51:02.759Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: pipelining replication: peer="{Voter server-e531e9ff-7c12-bf36-ae7e-7aea551ad781 e531e9ff-7c12-bf36-ae7e-7aea551ad781}" -2024-05-12T20:51:03.770Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:02.759824492 +0000 UTC m=+74.040524824 (1 nodes), 704.992703ms from start of poll, 705.605829ms from cluster start. Timeout at 2024-05-12 20:51:03.770796674 +0000 UTC m=+75.051497239, 1.010972415s after stability -2024-05-12T20:51:03.772Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: starting snapshot up to: index=12 -2024-05-12T20:51:03.772Z [INFO] creating new snapshot: path=/tmp/raft250582763/snapshots/2-12-1715547063772.tmp -2024-05-12T20:51:03.773Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: snapshot complete up to: index=12 -2024-05-12T20:51:03.775Z [INFO] creating new snapshot: path=/tmp/raft250582763/snapshots/2-1013-1715547063775.tmp -2024-05-12T20:51:03.776Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: copied to local snapshot: bytes=71 -2024-05-12T20:51:03.776Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: snapshot restore progress: id=2-1013-1715547063775 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:03.776Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: removing all old logs from log store -2024-05-12T20:51:03.776Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: compacting logs: from=1 to=22 -2024-05-12T20:51:03.776Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: restored user snapshot: index=1013 -2024-05-12T20:51:03.776Z [ERROR] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: failed to get log: index=22 error="log not found" -2024-05-12T20:51:03.776Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: aborting pipeline replication: peer="{Voter server-e531e9ff-7c12-bf36-ae7e-7aea551ad781 e531e9ff-7c12-bf36-ae7e-7aea551ad781}" -2024-05-12T20:51:03.776Z [ERROR] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: failed to get log: index=22 error="log not found" -2024-05-12T20:51:03.776Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: aborting pipeline replication: peer="{Voter server-d8c33984-7a5f-53e9-1c2a-bafd6ababa8e d8c33984-7a5f-53e9-1c2a-bafd6ababa8e}" -2024-05-12T20:51:03.790Z [ERROR] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: failed to get log: index=22 error="log not found" -2024-05-12T20:51:03.791Z [ERROR] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: failed to get log: index=22 error="log not found" -2024-05-12T20:51:03.791Z [INFO] creating new snapshot: path=/tmp/raft4142640199/snapshots/2-1013-1715547063791.tmp -2024-05-12T20:51:03.791Z [INFO] creating new snapshot: path=/tmp/raft3208142809/snapshots/2-1013-1715547063791.tmp -2024-05-12T20:51:03.792Z [INFO] server-e531e9ff-7c12-bf36-ae7e-7aea551ad781: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:03.792Z [INFO] server-d8c33984-7a5f-53e9-1c2a-bafd6ababa8e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:03.792Z [INFO] server-d8c33984-7a5f-53e9-1c2a-bafd6ababa8e: copied to local snapshot: bytes=71 -2024-05-12T20:51:03.792Z [INFO] server-e531e9ff-7c12-bf36-ae7e-7aea551ad781: copied to local snapshot: bytes=71 -2024-05-12T20:51:03.792Z [INFO] server-d8c33984-7a5f-53e9-1c2a-bafd6ababa8e: snapshot restore progress: id=2-1013-1715547063791 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:03.793Z [INFO] server-d8c33984-7a5f-53e9-1c2a-bafd6ababa8e: removing all old logs from log store -2024-05-12T20:51:03.793Z [INFO] server-d8c33984-7a5f-53e9-1c2a-bafd6ababa8e: compacting logs: from=1 to=22 -2024-05-12T20:51:03.793Z [INFO] server-d8c33984-7a5f-53e9-1c2a-bafd6ababa8e: Installed remote snapshot -2024-05-12T20:51:03.793Z [INFO] server-e531e9ff-7c12-bf36-ae7e-7aea551ad781: snapshot restore progress: id=2-1013-1715547063791 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:03.793Z [INFO] server-e531e9ff-7c12-bf36-ae7e-7aea551ad781: removing all old logs from log store -2024-05-12T20:51:03.793Z [INFO] server-e531e9ff-7c12-bf36-ae7e-7aea551ad781: compacting logs: from=1 to=22 -2024-05-12T20:51:03.793Z [INFO] server-e531e9ff-7c12-bf36-ae7e-7aea551ad781: Installed remote snapshot -2024-05-12T20:51:03.793Z [WARN] server-d8c33984-7a5f-53e9-1c2a-bafd6ababa8e: skipping application of old log: index=22 -2024-05-12T20:51:03.793Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: pipelining replication: peer="{Voter server-d8c33984-7a5f-53e9-1c2a-bafd6ababa8e d8c33984-7a5f-53e9-1c2a-bafd6ababa8e}" -2024-05-12T20:51:03.793Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:03.793Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: pipelining replication: peer="{Voter server-e531e9ff-7c12-bf36-ae7e-7aea551ad781 e531e9ff-7c12-bf36-ae7e-7aea551ad781}" -2024-05-12T20:51:04.798Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:03.793511363 +0000 UTC m=+75.074212214 (2 nodes), 2.595µs from start of poll, 1.739293219s from cluster start. Timeout at 2024-05-12 20:51:04.798776639 +0000 UTC m=+76.079477172, 1.005264958s after stability -2024-05-12T20:51:04.815Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: aborting pipeline replication: peer="{Voter server-e531e9ff-7c12-bf36-ae7e-7aea551ad781 e531e9ff-7c12-bf36-ae7e-7aea551ad781}" -2024-05-12T20:51:04.816Z [INFO] server-87335c1d-74ae-03b9-6bd6-d5e7c6c9abf5: aborting pipeline replication: peer="{Voter server-d8c33984-7a5f-53e9-1c2a-bafd6ababa8e d8c33984-7a5f-53e9-1c2a-bafd6ababa8e}" +2025-06-15T03:17:36.623Z [INFO] server-fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc Address:fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc} {Suffrage:Voter ID:server-56d7d729-d642-0e53-4f33-f3e28010a72d Address:56d7d729-d642-0e53-4f33-f3e28010a72d} {Suffrage:Voter ID:server-78adaf6b-1442-f18a-1157-dc40aea19f74 Address:78adaf6b-1442-f18a-1157-dc40aea19f74}]" +2025-06-15T03:17:36.623Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc Address:fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc} {Suffrage:Voter ID:server-56d7d729-d642-0e53-4f33-f3e28010a72d Address:56d7d729-d642-0e53-4f33-f3e28010a72d} {Suffrage:Voter ID:server-78adaf6b-1442-f18a-1157-dc40aea19f74 Address:78adaf6b-1442-f18a-1157-dc40aea19f74}]" +2025-06-15T03:17:36.623Z [INFO] server-fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc: entering follower state: follower="Node at fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc [Follower]" leader-address= leader-id= +2025-06-15T03:17:36.623Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: entering follower state: follower="Node at 56d7d729-d642-0e53-4f33-f3e28010a72d [Follower]" leader-address= leader-id= +2025-06-15T03:17:36.623Z [INFO] server-78adaf6b-1442-f18a-1157-dc40aea19f74: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc Address:fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc} {Suffrage:Voter ID:server-56d7d729-d642-0e53-4f33-f3e28010a72d Address:56d7d729-d642-0e53-4f33-f3e28010a72d} {Suffrage:Voter ID:server-78adaf6b-1442-f18a-1157-dc40aea19f74 Address:78adaf6b-1442-f18a-1157-dc40aea19f74}]" +2025-06-15T03:17:36.624Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:36.624Z [INFO] server-78adaf6b-1442-f18a-1157-dc40aea19f74: entering follower state: follower="Node at 78adaf6b-1442-f18a-1157-dc40aea19f74 [Follower]" leader-address= leader-id= +2025-06-15T03:17:37.278Z [WARN] server-56d7d729-d642-0e53-4f33-f3e28010a72d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:37.278Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: entering candidate state: node="Node at 56d7d729-d642-0e53-4f33-f3e28010a72d [Candidate]" term=2 +2025-06-15T03:17:37.278Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: election won: term=2 tally=2 +2025-06-15T03:17:37.278Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: entering leader state: leader="Node at 56d7d729-d642-0e53-4f33-f3e28010a72d [Leader]" +2025-06-15T03:17:37.278Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: added peer, starting replication: peer=server-fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc +2025-06-15T03:17:37.278Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: added peer, starting replication: peer=server-78adaf6b-1442-f18a-1157-dc40aea19f74 +2025-06-15T03:17:37.278Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: pipelining replication: peer="{Voter server-78adaf6b-1442-f18a-1157-dc40aea19f74 78adaf6b-1442-f18a-1157-dc40aea19f74}" +2025-06-15T03:17:37.278Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: pipelining replication: peer="{Voter server-fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc}" +2025-06-15T03:17:38.284Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:37.278461422 +0000 UTC m=+72.990139604 (1 nodes), 654.430169ms from start of poll, 654.684812ms from cluster start. Timeout at 2025-06-15 03:17:38.284309712 +0000 UTC m=+73.995987911, 1.005848307s after stability +2025-06-15T03:17:38.284Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: starting snapshot up to: index=12 +2025-06-15T03:17:38.284Z [INFO] creating new snapshot: path=/tmp/raft2582717619/snapshots/2-12-1749957458284.tmp +2025-06-15T03:17:38.285Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: snapshot complete up to: index=12 +2025-06-15T03:17:38.285Z [INFO] creating new snapshot: path=/tmp/raft2582717619/snapshots/2-1013-1749957458285.tmp +2025-06-15T03:17:38.285Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: copied to local snapshot: bytes=71 +2025-06-15T03:17:38.285Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: snapshot restore progress: id=2-1013-1749957458285 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:38.285Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: removing all old logs from log store +2025-06-15T03:17:38.285Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: compacting logs: from=1 to=22 +2025-06-15T03:17:38.285Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: restored user snapshot: index=1013 +2025-06-15T03:17:38.285Z [ERROR] server-56d7d729-d642-0e53-4f33-f3e28010a72d: failed to get log: index=22 error="log not found" +2025-06-15T03:17:38.285Z [ERROR] server-56d7d729-d642-0e53-4f33-f3e28010a72d: failed to get log: index=22 error="log not found" +2025-06-15T03:17:38.285Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: aborting pipeline replication: peer="{Voter server-fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc}" +2025-06-15T03:17:38.285Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: aborting pipeline replication: peer="{Voter server-78adaf6b-1442-f18a-1157-dc40aea19f74 78adaf6b-1442-f18a-1157-dc40aea19f74}" +2025-06-15T03:17:38.294Z [ERROR] server-56d7d729-d642-0e53-4f33-f3e28010a72d: failed to get log: index=22 error="log not found" +2025-06-15T03:17:38.294Z [ERROR] server-56d7d729-d642-0e53-4f33-f3e28010a72d: failed to get log: index=22 error="log not found" +2025-06-15T03:17:38.295Z [INFO] creating new snapshot: path=/tmp/raft2517882292/snapshots/2-1013-1749957458295.tmp +2025-06-15T03:17:38.295Z [INFO] creating new snapshot: path=/tmp/raft3906746827/snapshots/2-1013-1749957458295.tmp +2025-06-15T03:17:38.295Z [INFO] server-78adaf6b-1442-f18a-1157-dc40aea19f74: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:38.295Z [INFO] server-fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:38.295Z [INFO] server-fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc: copied to local snapshot: bytes=71 +2025-06-15T03:17:38.295Z [INFO] server-78adaf6b-1442-f18a-1157-dc40aea19f74: copied to local snapshot: bytes=71 +2025-06-15T03:17:38.295Z [INFO] server-fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc: snapshot restore progress: id=2-1013-1749957458295 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:38.295Z [INFO] server-fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc: removing all old logs from log store +2025-06-15T03:17:38.296Z [INFO] server-fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc: compacting logs: from=1 to=22 +2025-06-15T03:17:38.296Z [INFO] server-fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc: Installed remote snapshot +2025-06-15T03:17:38.296Z [WARN] server-fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc: skipping application of old log: index=22 +2025-06-15T03:17:38.296Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: pipelining replication: peer="{Voter server-fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc}" +2025-06-15T03:17:38.296Z [INFO] server-78adaf6b-1442-f18a-1157-dc40aea19f74: snapshot restore progress: id=2-1013-1749957458295 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:38.296Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:17:38.296Z [INFO] server-78adaf6b-1442-f18a-1157-dc40aea19f74: removing all old logs from log store +2025-06-15T03:17:38.296Z [INFO] server-78adaf6b-1442-f18a-1157-dc40aea19f74: compacting logs: from=1 to=22 +2025-06-15T03:17:38.296Z [INFO] server-78adaf6b-1442-f18a-1157-dc40aea19f74: Installed remote snapshot +2025-06-15T03:17:38.296Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: pipelining replication: peer="{Voter server-78adaf6b-1442-f18a-1157-dc40aea19f74 78adaf6b-1442-f18a-1157-dc40aea19f74}" +2025-06-15T03:17:39.302Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:17:38.296291258 +0000 UTC m=+74.007969435 (2 nodes), 45.448µs from start of poll, 1.672514643s from cluster start. Timeout at 2025-06-15 03:17:39.30231544 +0000 UTC m=+75.013993639, 1.006024204s after stability +2025-06-15T03:17:39.313Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: aborting pipeline replication: peer="{Voter server-78adaf6b-1442-f18a-1157-dc40aea19f74 78adaf6b-1442-f18a-1157-dc40aea19f74}" +2025-06-15T03:17:39.313Z [INFO] server-56d7d729-d642-0e53-4f33-f3e28010a72d: aborting pipeline replication: peer="{Voter server-fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc fe2b08a9-9b9e-4c2b-b8bd-b9d1157836bc}" === RUN TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true -2024-05-12T20:51:04.819Z [INFO] server-7e6652ef-4cd1-201f-b2ec-6337db33fe31: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7e6652ef-4cd1-201f-b2ec-6337db33fe31 Address:7e6652ef-4cd1-201f-b2ec-6337db33fe31} {Suffrage:Voter ID:server-f94d2520-9e60-be2f-dc3d-69517974e0a7 Address:f94d2520-9e60-be2f-dc3d-69517974e0a7} {Suffrage:Voter ID:server-9b9afbd2-e498-11b8-a210-4143bd8932e0 Address:9b9afbd2-e498-11b8-a210-4143bd8932e0}]" -2024-05-12T20:51:04.819Z [INFO] server-f94d2520-9e60-be2f-dc3d-69517974e0a7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7e6652ef-4cd1-201f-b2ec-6337db33fe31 Address:7e6652ef-4cd1-201f-b2ec-6337db33fe31} {Suffrage:Voter ID:server-f94d2520-9e60-be2f-dc3d-69517974e0a7 Address:f94d2520-9e60-be2f-dc3d-69517974e0a7} {Suffrage:Voter ID:server-9b9afbd2-e498-11b8-a210-4143bd8932e0 Address:9b9afbd2-e498-11b8-a210-4143bd8932e0}]" -2024-05-12T20:51:04.819Z [INFO] server-9b9afbd2-e498-11b8-a210-4143bd8932e0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7e6652ef-4cd1-201f-b2ec-6337db33fe31 Address:7e6652ef-4cd1-201f-b2ec-6337db33fe31} {Suffrage:Voter ID:server-f94d2520-9e60-be2f-dc3d-69517974e0a7 Address:f94d2520-9e60-be2f-dc3d-69517974e0a7} {Suffrage:Voter ID:server-9b9afbd2-e498-11b8-a210-4143bd8932e0 Address:9b9afbd2-e498-11b8-a210-4143bd8932e0}]" -2024-05-12T20:51:04.819Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:04.819Z [INFO] server-7e6652ef-4cd1-201f-b2ec-6337db33fe31: entering follower state: follower="Node at 7e6652ef-4cd1-201f-b2ec-6337db33fe31 [Follower]" leader-address= leader-id= -2024-05-12T20:51:04.819Z [INFO] server-f94d2520-9e60-be2f-dc3d-69517974e0a7: entering follower state: follower="Node at f94d2520-9e60-be2f-dc3d-69517974e0a7 [Follower]" leader-address= leader-id= -2024-05-12T20:51:04.819Z [INFO] server-9b9afbd2-e498-11b8-a210-4143bd8932e0: entering follower state: follower="Node at 9b9afbd2-e498-11b8-a210-4143bd8932e0 [Follower]" leader-address= leader-id= -2024-05-12T20:51:05.425Z [WARN] server-7e6652ef-4cd1-201f-b2ec-6337db33fe31: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:05.425Z [INFO] server-7e6652ef-4cd1-201f-b2ec-6337db33fe31: entering candidate state: node="Node at 7e6652ef-4cd1-201f-b2ec-6337db33fe31 [Candidate]" term=2 -2024-05-12T20:51:05.425Z [INFO] server-7e6652ef-4cd1-201f-b2ec-6337db33fe31: election won: term=2 tally=2 -2024-05-12T20:51:05.425Z [INFO] server-7e6652ef-4cd1-201f-b2ec-6337db33fe31: entering leader state: leader="Node at 7e6652ef-4cd1-201f-b2ec-6337db33fe31 [Leader]" -2024-05-12T20:51:05.425Z [INFO] server-7e6652ef-4cd1-201f-b2ec-6337db33fe31: added peer, starting replication: peer=server-f94d2520-9e60-be2f-dc3d-69517974e0a7 -2024-05-12T20:51:05.425Z [INFO] server-7e6652ef-4cd1-201f-b2ec-6337db33fe31: added peer, starting replication: peer=server-9b9afbd2-e498-11b8-a210-4143bd8932e0 -2024-05-12T20:51:05.425Z [INFO] server-7e6652ef-4cd1-201f-b2ec-6337db33fe31: pipelining replication: peer="{Voter server-9b9afbd2-e498-11b8-a210-4143bd8932e0 9b9afbd2-e498-11b8-a210-4143bd8932e0}" -2024-05-12T20:51:05.440Z [INFO] server-7e6652ef-4cd1-201f-b2ec-6337db33fe31: pipelining replication: peer="{Voter server-f94d2520-9e60-be2f-dc3d-69517974e0a7 f94d2520-9e60-be2f-dc3d-69517974e0a7}" -2024-05-12T20:51:06.446Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:05.440354706 +0000 UTC m=+76.721055042 (1 nodes), 620.774147ms from start of poll, 621.249402ms from cluster start. Timeout at 2024-05-12 20:51:06.446815711 +0000 UTC m=+77.727516731, 1.006461689s after stability -2024-05-12T20:51:06.461Z [INFO] server-7e6652ef-4cd1-201f-b2ec-6337db33fe31: starting snapshot up to: index=12 -2024-05-12T20:51:06.461Z [INFO] creating new snapshot: path=/tmp/raft2368515616/snapshots/2-12-1715547066461.tmp -2024-05-12T20:51:06.464Z [INFO] server-7e6652ef-4cd1-201f-b2ec-6337db33fe31: snapshot complete up to: index=12 -2024-05-12T20:51:06.485Z [INFO] server-c4ce91df-743e-363e-0e74-339afc7e7fd3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c4ce91df-743e-363e-0e74-339afc7e7fd3 Address:c4ce91df-743e-363e-0e74-339afc7e7fd3} {Suffrage:Voter ID:server-1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7 Address:1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7} {Suffrage:Voter ID:server-4ed81d36-4792-6207-2c98-25b4554b4b3e Address:4ed81d36-4792-6207-2c98-25b4554b4b3e}]" -2024-05-12T20:51:06.485Z [INFO] server-1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c4ce91df-743e-363e-0e74-339afc7e7fd3 Address:c4ce91df-743e-363e-0e74-339afc7e7fd3} {Suffrage:Voter ID:server-1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7 Address:1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7} {Suffrage:Voter ID:server-4ed81d36-4792-6207-2c98-25b4554b4b3e Address:4ed81d36-4792-6207-2c98-25b4554b4b3e}]" -2024-05-12T20:51:06.485Z [INFO] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c4ce91df-743e-363e-0e74-339afc7e7fd3 Address:c4ce91df-743e-363e-0e74-339afc7e7fd3} {Suffrage:Voter ID:server-1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7 Address:1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7} {Suffrage:Voter ID:server-4ed81d36-4792-6207-2c98-25b4554b4b3e Address:4ed81d36-4792-6207-2c98-25b4554b4b3e}]" -2024-05-12T20:51:06.485Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:06.485Z [INFO] server-c4ce91df-743e-363e-0e74-339afc7e7fd3: entering follower state: follower="Node at c4ce91df-743e-363e-0e74-339afc7e7fd3 [Follower]" leader-address= leader-id= -2024-05-12T20:51:06.485Z [INFO] server-1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7: entering follower state: follower="Node at 1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7 [Follower]" leader-address= leader-id= -2024-05-12T20:51:06.485Z [INFO] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: entering follower state: follower="Node at 4ed81d36-4792-6207-2c98-25b4554b4b3e [Follower]" leader-address= leader-id= -2024-05-12T20:51:07.094Z [WARN] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:07.094Z [INFO] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: entering candidate state: node="Node at 4ed81d36-4792-6207-2c98-25b4554b4b3e [Candidate]" term=2 -2024-05-12T20:51:07.094Z [INFO] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: election won: term=2 tally=2 -2024-05-12T20:51:07.094Z [INFO] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: entering leader state: leader="Node at 4ed81d36-4792-6207-2c98-25b4554b4b3e [Leader]" -2024-05-12T20:51:07.094Z [INFO] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: added peer, starting replication: peer=server-c4ce91df-743e-363e-0e74-339afc7e7fd3 -2024-05-12T20:51:07.094Z [INFO] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: added peer, starting replication: peer=server-1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7 -2024-05-12T20:51:07.094Z [INFO] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: pipelining replication: peer="{Voter server-1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7 1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7}" -2024-05-12T20:51:07.094Z [INFO] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: pipelining replication: peer="{Voter server-c4ce91df-743e-363e-0e74-339afc7e7fd3 c4ce91df-743e-363e-0e74-339afc7e7fd3}" -2024-05-12T20:51:07.094Z [WARN] server-c4ce91df-743e-363e-0e74-339afc7e7fd3: rejecting vote request since our last term is greater: candidate=4ed81d36-4792-6207-2c98-25b4554b4b3e last-term=2 last-candidate-term=1 -2024-05-12T20:51:08.103Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:07.094951592 +0000 UTC m=+78.375651982 (1 nodes), 609.116759ms from start of poll, 609.578151ms from cluster start. Timeout at 2024-05-12 20:51:08.103417294 +0000 UTC m=+79.384117189, 1.008465207s after stability -2024-05-12T20:51:08.103Z [INFO] creating new snapshot: path=/tmp/raft3319255234/snapshots/2-1013-1715547068103.tmp -2024-05-12T20:51:08.105Z [INFO] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: copied to local snapshot: bytes=71 -2024-05-12T20:51:08.105Z [INFO] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: snapshot restore progress: id=2-1013-1715547068103 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:08.105Z [INFO] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: restored user snapshot: index=1013 -2024-05-12T20:51:08.105Z [ERROR] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: failed to get log: index=3 error="log not found" -2024-05-12T20:51:08.105Z [INFO] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: aborting pipeline replication: peer="{Voter server-c4ce91df-743e-363e-0e74-339afc7e7fd3 c4ce91df-743e-363e-0e74-339afc7e7fd3}" -2024-05-12T20:51:08.105Z [ERROR] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: failed to get log: index=3 error="log not found" -2024-05-12T20:51:08.106Z [INFO] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: aborting pipeline replication: peer="{Voter server-1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7 1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7}" -2024-05-12T20:51:08.114Z [ERROR] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: failed to get log: index=3 error="log not found" -2024-05-12T20:51:08.115Z [ERROR] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: failed to get log: index=3 error="log not found" -2024-05-12T20:51:08.115Z [INFO] creating new snapshot: path=/tmp/raft79232416/snapshots/2-1013-1715547068115.tmp -2024-05-12T20:51:08.116Z [INFO] server-c4ce91df-743e-363e-0e74-339afc7e7fd3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:08.115Z [INFO] creating new snapshot: path=/tmp/raft1972874573/snapshots/2-1013-1715547068115.tmp -2024-05-12T20:51:08.116Z [INFO] server-1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:08.116Z [INFO] server-c4ce91df-743e-363e-0e74-339afc7e7fd3: copied to local snapshot: bytes=71 -2024-05-12T20:51:08.117Z [INFO] server-c4ce91df-743e-363e-0e74-339afc7e7fd3: snapshot restore progress: id=2-1013-1715547068115 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:08.117Z [INFO] server-1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7: copied to local snapshot: bytes=71 -2024-05-12T20:51:08.117Z [INFO] server-c4ce91df-743e-363e-0e74-339afc7e7fd3: Installed remote snapshot -2024-05-12T20:51:08.117Z [INFO] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: pipelining replication: peer="{Voter server-c4ce91df-743e-363e-0e74-339afc7e7fd3 c4ce91df-743e-363e-0e74-339afc7e7fd3}" -2024-05-12T20:51:08.117Z [INFO] server-1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7: snapshot restore progress: id=2-1013-1715547068115 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:08.117Z [INFO] server-1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7: Installed remote snapshot -2024-05-12T20:51:08.117Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:08.117Z [INFO] server-4ed81d36-4792-6207-2c98-25b4554b4b3e: pipelining replication: peer="{Voter server-1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7 1b60b3cc-68ae-6217-40f7-3ee25ce5e6b7}" -2024-05-12T20:51:09.122Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:08.117545658 +0000 UTC m=+79.398245998 (2 nodes), 1.688µs from start of poll, 1.632172167s from cluster start. Timeout at 2024-05-12 20:51:09.122922655 +0000 UTC m=+80.403623086, 1.005377088s after stability -2024-05-12T20:51:09.138Z [INFO] server-7e6652ef-4cd1-201f-b2ec-6337db33fe31: aborting pipeline replication: peer="{Voter server-9b9afbd2-e498-11b8-a210-4143bd8932e0 9b9afbd2-e498-11b8-a210-4143bd8932e0}" -2024-05-12T20:51:09.138Z [INFO] server-7e6652ef-4cd1-201f-b2ec-6337db33fe31: aborting pipeline replication: peer="{Voter server-f94d2520-9e60-be2f-dc3d-69517974e0a7 f94d2520-9e60-be2f-dc3d-69517974e0a7}" +2025-06-15T03:17:39.315Z [INFO] server-bee166fc-565e-398b-8ab7-c164a94d7599: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bee166fc-565e-398b-8ab7-c164a94d7599 Address:bee166fc-565e-398b-8ab7-c164a94d7599} {Suffrage:Voter ID:server-d51a1f23-36f2-6ee8-ac97-8c55a0e31166 Address:d51a1f23-36f2-6ee8-ac97-8c55a0e31166} {Suffrage:Voter ID:server-62db3e58-ce2b-9b04-dfc4-186eb619261c Address:62db3e58-ce2b-9b04-dfc4-186eb619261c}]" +2025-06-15T03:17:39.315Z [INFO] server-d51a1f23-36f2-6ee8-ac97-8c55a0e31166: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bee166fc-565e-398b-8ab7-c164a94d7599 Address:bee166fc-565e-398b-8ab7-c164a94d7599} {Suffrage:Voter ID:server-d51a1f23-36f2-6ee8-ac97-8c55a0e31166 Address:d51a1f23-36f2-6ee8-ac97-8c55a0e31166} {Suffrage:Voter ID:server-62db3e58-ce2b-9b04-dfc4-186eb619261c Address:62db3e58-ce2b-9b04-dfc4-186eb619261c}]" +2025-06-15T03:17:39.315Z [INFO] server-bee166fc-565e-398b-8ab7-c164a94d7599: entering follower state: follower="Node at bee166fc-565e-398b-8ab7-c164a94d7599 [Follower]" leader-address= leader-id= +2025-06-15T03:17:39.315Z [INFO] server-d51a1f23-36f2-6ee8-ac97-8c55a0e31166: entering follower state: follower="Node at d51a1f23-36f2-6ee8-ac97-8c55a0e31166 [Follower]" leader-address= leader-id= +2025-06-15T03:17:39.315Z [INFO] server-62db3e58-ce2b-9b04-dfc4-186eb619261c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bee166fc-565e-398b-8ab7-c164a94d7599 Address:bee166fc-565e-398b-8ab7-c164a94d7599} {Suffrage:Voter ID:server-d51a1f23-36f2-6ee8-ac97-8c55a0e31166 Address:d51a1f23-36f2-6ee8-ac97-8c55a0e31166} {Suffrage:Voter ID:server-62db3e58-ce2b-9b04-dfc4-186eb619261c Address:62db3e58-ce2b-9b04-dfc4-186eb619261c}]" +2025-06-15T03:17:39.315Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:39.315Z [INFO] server-62db3e58-ce2b-9b04-dfc4-186eb619261c: entering follower state: follower="Node at 62db3e58-ce2b-9b04-dfc4-186eb619261c [Follower]" leader-address= leader-id= +2025-06-15T03:17:40.067Z [WARN] server-62db3e58-ce2b-9b04-dfc4-186eb619261c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:40.067Z [INFO] server-62db3e58-ce2b-9b04-dfc4-186eb619261c: entering candidate state: node="Node at 62db3e58-ce2b-9b04-dfc4-186eb619261c [Candidate]" term=2 +2025-06-15T03:17:40.067Z [INFO] server-62db3e58-ce2b-9b04-dfc4-186eb619261c: election won: term=2 tally=2 +2025-06-15T03:17:40.067Z [INFO] server-62db3e58-ce2b-9b04-dfc4-186eb619261c: entering leader state: leader="Node at 62db3e58-ce2b-9b04-dfc4-186eb619261c [Leader]" +2025-06-15T03:17:40.067Z [INFO] server-62db3e58-ce2b-9b04-dfc4-186eb619261c: added peer, starting replication: peer=server-bee166fc-565e-398b-8ab7-c164a94d7599 +2025-06-15T03:17:40.067Z [INFO] server-62db3e58-ce2b-9b04-dfc4-186eb619261c: added peer, starting replication: peer=server-d51a1f23-36f2-6ee8-ac97-8c55a0e31166 +2025-06-15T03:17:40.067Z [INFO] server-62db3e58-ce2b-9b04-dfc4-186eb619261c: pipelining replication: peer="{Voter server-bee166fc-565e-398b-8ab7-c164a94d7599 bee166fc-565e-398b-8ab7-c164a94d7599}" +2025-06-15T03:17:40.067Z [INFO] server-62db3e58-ce2b-9b04-dfc4-186eb619261c: pipelining replication: peer="{Voter server-d51a1f23-36f2-6ee8-ac97-8c55a0e31166 d51a1f23-36f2-6ee8-ac97-8c55a0e31166}" +2025-06-15T03:17:41.073Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:40.067613453 +0000 UTC m=+75.779291689 (1 nodes), 752.338675ms from start of poll, 752.662631ms from cluster start. Timeout at 2025-06-15 03:17:41.073209265 +0000 UTC m=+76.784887430, 1.005595741s after stability +2025-06-15T03:17:41.073Z [INFO] server-62db3e58-ce2b-9b04-dfc4-186eb619261c: starting snapshot up to: index=12 +2025-06-15T03:17:41.073Z [INFO] creating new snapshot: path=/tmp/raft528072511/snapshots/2-12-1749957461073.tmp +2025-06-15T03:17:41.073Z [INFO] server-62db3e58-ce2b-9b04-dfc4-186eb619261c: snapshot complete up to: index=12 +2025-06-15T03:17:41.074Z [INFO] server-b34028d9-b144-4f9a-58f3-fc1918d2107e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b34028d9-b144-4f9a-58f3-fc1918d2107e Address:b34028d9-b144-4f9a-58f3-fc1918d2107e} {Suffrage:Voter ID:server-4830320c-5c0a-89d2-5020-99996ea78179 Address:4830320c-5c0a-89d2-5020-99996ea78179} {Suffrage:Voter ID:server-36621827-8d86-4687-6cb5-522ae6e66611 Address:36621827-8d86-4687-6cb5-522ae6e66611}]" +2025-06-15T03:17:41.074Z [INFO] server-b34028d9-b144-4f9a-58f3-fc1918d2107e: entering follower state: follower="Node at b34028d9-b144-4f9a-58f3-fc1918d2107e [Follower]" leader-address= leader-id= +2025-06-15T03:17:41.074Z [INFO] server-4830320c-5c0a-89d2-5020-99996ea78179: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b34028d9-b144-4f9a-58f3-fc1918d2107e Address:b34028d9-b144-4f9a-58f3-fc1918d2107e} {Suffrage:Voter ID:server-4830320c-5c0a-89d2-5020-99996ea78179 Address:4830320c-5c0a-89d2-5020-99996ea78179} {Suffrage:Voter ID:server-36621827-8d86-4687-6cb5-522ae6e66611 Address:36621827-8d86-4687-6cb5-522ae6e66611}]" +2025-06-15T03:17:41.075Z [INFO] server-4830320c-5c0a-89d2-5020-99996ea78179: entering follower state: follower="Node at 4830320c-5c0a-89d2-5020-99996ea78179 [Follower]" leader-address= leader-id= +2025-06-15T03:17:41.075Z [INFO] server-36621827-8d86-4687-6cb5-522ae6e66611: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b34028d9-b144-4f9a-58f3-fc1918d2107e Address:b34028d9-b144-4f9a-58f3-fc1918d2107e} {Suffrage:Voter ID:server-4830320c-5c0a-89d2-5020-99996ea78179 Address:4830320c-5c0a-89d2-5020-99996ea78179} {Suffrage:Voter ID:server-36621827-8d86-4687-6cb5-522ae6e66611 Address:36621827-8d86-4687-6cb5-522ae6e66611}]" +2025-06-15T03:17:41.075Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:41.075Z [INFO] server-36621827-8d86-4687-6cb5-522ae6e66611: entering follower state: follower="Node at 36621827-8d86-4687-6cb5-522ae6e66611 [Follower]" leader-address= leader-id= +2025-06-15T03:17:41.747Z [WARN] server-4830320c-5c0a-89d2-5020-99996ea78179: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:41.747Z [INFO] server-4830320c-5c0a-89d2-5020-99996ea78179: entering candidate state: node="Node at 4830320c-5c0a-89d2-5020-99996ea78179 [Candidate]" term=2 +2025-06-15T03:17:41.747Z [INFO] server-4830320c-5c0a-89d2-5020-99996ea78179: election won: term=2 tally=2 +2025-06-15T03:17:41.747Z [INFO] server-4830320c-5c0a-89d2-5020-99996ea78179: entering leader state: leader="Node at 4830320c-5c0a-89d2-5020-99996ea78179 [Leader]" +2025-06-15T03:17:41.747Z [INFO] server-4830320c-5c0a-89d2-5020-99996ea78179: added peer, starting replication: peer=server-b34028d9-b144-4f9a-58f3-fc1918d2107e +2025-06-15T03:17:41.747Z [INFO] server-4830320c-5c0a-89d2-5020-99996ea78179: added peer, starting replication: peer=server-36621827-8d86-4687-6cb5-522ae6e66611 +2025-06-15T03:17:41.748Z [INFO] server-4830320c-5c0a-89d2-5020-99996ea78179: pipelining replication: peer="{Voter server-36621827-8d86-4687-6cb5-522ae6e66611 36621827-8d86-4687-6cb5-522ae6e66611}" +2025-06-15T03:17:41.748Z [INFO] server-4830320c-5c0a-89d2-5020-99996ea78179: pipelining replication: peer="{Voter server-b34028d9-b144-4f9a-58f3-fc1918d2107e b34028d9-b144-4f9a-58f3-fc1918d2107e}" +2025-06-15T03:17:42.754Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:41.748069955 +0000 UTC m=+77.459748121 (1 nodes), 672.985276ms from start of poll, 673.272248ms from cluster start. Timeout at 2025-06-15 03:17:42.753973381 +0000 UTC m=+78.465651583, 1.005903462s after stability +2025-06-15T03:17:42.754Z [INFO] creating new snapshot: path=/tmp/raft3891269386/snapshots/2-1013-1749957462754.tmp +2025-06-15T03:17:42.755Z [INFO] server-4830320c-5c0a-89d2-5020-99996ea78179: copied to local snapshot: bytes=71 +2025-06-15T03:17:42.756Z [INFO] server-4830320c-5c0a-89d2-5020-99996ea78179: snapshot restore progress: id=2-1013-1749957462754 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:42.756Z [INFO] server-4830320c-5c0a-89d2-5020-99996ea78179: restored user snapshot: index=1013 +2025-06-15T03:17:42.756Z [ERROR] server-4830320c-5c0a-89d2-5020-99996ea78179: failed to get log: index=3 error="log not found" +2025-06-15T03:17:42.756Z [ERROR] server-4830320c-5c0a-89d2-5020-99996ea78179: failed to get log: index=3 error="log not found" +2025-06-15T03:17:42.756Z [INFO] server-4830320c-5c0a-89d2-5020-99996ea78179: aborting pipeline replication: peer="{Voter server-b34028d9-b144-4f9a-58f3-fc1918d2107e b34028d9-b144-4f9a-58f3-fc1918d2107e}" +2025-06-15T03:17:42.756Z [INFO] server-4830320c-5c0a-89d2-5020-99996ea78179: aborting pipeline replication: peer="{Voter server-36621827-8d86-4687-6cb5-522ae6e66611 36621827-8d86-4687-6cb5-522ae6e66611}" +2025-06-15T03:17:42.762Z [ERROR] server-4830320c-5c0a-89d2-5020-99996ea78179: failed to get log: index=3 error="log not found" +2025-06-15T03:17:42.762Z [ERROR] server-4830320c-5c0a-89d2-5020-99996ea78179: failed to get log: index=3 error="log not found" +2025-06-15T03:17:42.762Z [INFO] creating new snapshot: path=/tmp/raft1557313535/snapshots/2-1013-1749957462762.tmp +2025-06-15T03:17:42.762Z [INFO] creating new snapshot: path=/tmp/raft3309220673/snapshots/2-1013-1749957462762.tmp +2025-06-15T03:17:42.762Z [INFO] server-b34028d9-b144-4f9a-58f3-fc1918d2107e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:42.762Z [INFO] server-36621827-8d86-4687-6cb5-522ae6e66611: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:42.763Z [INFO] server-b34028d9-b144-4f9a-58f3-fc1918d2107e: copied to local snapshot: bytes=71 +2025-06-15T03:17:42.763Z [INFO] server-36621827-8d86-4687-6cb5-522ae6e66611: copied to local snapshot: bytes=71 +2025-06-15T03:17:42.763Z [INFO] server-b34028d9-b144-4f9a-58f3-fc1918d2107e: snapshot restore progress: id=2-1013-1749957462762 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:42.763Z [INFO] server-b34028d9-b144-4f9a-58f3-fc1918d2107e: Installed remote snapshot +2025-06-15T03:17:42.763Z [INFO] server-4830320c-5c0a-89d2-5020-99996ea78179: pipelining replication: peer="{Voter server-b34028d9-b144-4f9a-58f3-fc1918d2107e b34028d9-b144-4f9a-58f3-fc1918d2107e}" +2025-06-15T03:17:42.763Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:17:42.763Z [INFO] server-36621827-8d86-4687-6cb5-522ae6e66611: snapshot restore progress: id=2-1013-1749957462762 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:42.763Z [INFO] server-36621827-8d86-4687-6cb5-522ae6e66611: Installed remote snapshot +2025-06-15T03:17:42.763Z [INFO] server-4830320c-5c0a-89d2-5020-99996ea78179: pipelining replication: peer="{Voter server-36621827-8d86-4687-6cb5-522ae6e66611 36621827-8d86-4687-6cb5-522ae6e66611}" +2025-06-15T03:17:43.768Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:17:42.763302933 +0000 UTC m=+78.474981098 (2 nodes), 541ns from start of poll, 1.688505225s from cluster start. Timeout at 2025-06-15 03:17:43.768335399 +0000 UTC m=+79.480013598, 1.0050325s after stability +2025-06-15T03:17:43.779Z [INFO] server-62db3e58-ce2b-9b04-dfc4-186eb619261c: aborting pipeline replication: peer="{Voter server-bee166fc-565e-398b-8ab7-c164a94d7599 bee166fc-565e-398b-8ab7-c164a94d7599}" +2025-06-15T03:17:43.779Z [INFO] server-62db3e58-ce2b-9b04-dfc4-186eb619261c: aborting pipeline replication: peer="{Voter server-d51a1f23-36f2-6ee8-ac97-8c55a0e31166 d51a1f23-36f2-6ee8-ac97-8c55a0e31166}" === RUN TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true -2024-05-12T20:51:09.147Z [INFO] server-429bbff3-71be-7524-07c3-26a2c0f4d21a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-429bbff3-71be-7524-07c3-26a2c0f4d21a Address:429bbff3-71be-7524-07c3-26a2c0f4d21a} {Suffrage:Voter ID:server-089bd1fb-cbfe-a5c7-2419-b575a98477b2 Address:089bd1fb-cbfe-a5c7-2419-b575a98477b2} {Suffrage:Voter ID:server-29905062-da5b-1a0c-721c-93c3f860bff4 Address:29905062-da5b-1a0c-721c-93c3f860bff4}]" -2024-05-12T20:51:09.148Z [INFO] server-089bd1fb-cbfe-a5c7-2419-b575a98477b2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-429bbff3-71be-7524-07c3-26a2c0f4d21a Address:429bbff3-71be-7524-07c3-26a2c0f4d21a} {Suffrage:Voter ID:server-089bd1fb-cbfe-a5c7-2419-b575a98477b2 Address:089bd1fb-cbfe-a5c7-2419-b575a98477b2} {Suffrage:Voter ID:server-29905062-da5b-1a0c-721c-93c3f860bff4 Address:29905062-da5b-1a0c-721c-93c3f860bff4}]" -2024-05-12T20:51:09.148Z [INFO] server-29905062-da5b-1a0c-721c-93c3f860bff4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-429bbff3-71be-7524-07c3-26a2c0f4d21a Address:429bbff3-71be-7524-07c3-26a2c0f4d21a} {Suffrage:Voter ID:server-089bd1fb-cbfe-a5c7-2419-b575a98477b2 Address:089bd1fb-cbfe-a5c7-2419-b575a98477b2} {Suffrage:Voter ID:server-29905062-da5b-1a0c-721c-93c3f860bff4 Address:29905062-da5b-1a0c-721c-93c3f860bff4}]" -2024-05-12T20:51:09.148Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:09.152Z [INFO] server-29905062-da5b-1a0c-721c-93c3f860bff4: entering follower state: follower="Node at 29905062-da5b-1a0c-721c-93c3f860bff4 [Follower]" leader-address= leader-id= -2024-05-12T20:51:09.153Z [INFO] server-089bd1fb-cbfe-a5c7-2419-b575a98477b2: entering follower state: follower="Node at 089bd1fb-cbfe-a5c7-2419-b575a98477b2 [Follower]" leader-address= leader-id= -2024-05-12T20:51:09.171Z [INFO] server-429bbff3-71be-7524-07c3-26a2c0f4d21a: entering follower state: follower="Node at 429bbff3-71be-7524-07c3-26a2c0f4d21a [Follower]" leader-address= leader-id= -2024-05-12T20:51:09.956Z [WARN] server-29905062-da5b-1a0c-721c-93c3f860bff4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:09.956Z [INFO] server-29905062-da5b-1a0c-721c-93c3f860bff4: entering candidate state: node="Node at 29905062-da5b-1a0c-721c-93c3f860bff4 [Candidate]" term=2 -2024-05-12T20:51:09.956Z [INFO] server-29905062-da5b-1a0c-721c-93c3f860bff4: election won: term=2 tally=2 -2024-05-12T20:51:09.956Z [INFO] server-29905062-da5b-1a0c-721c-93c3f860bff4: entering leader state: leader="Node at 29905062-da5b-1a0c-721c-93c3f860bff4 [Leader]" -2024-05-12T20:51:09.956Z [INFO] server-29905062-da5b-1a0c-721c-93c3f860bff4: added peer, starting replication: peer=server-429bbff3-71be-7524-07c3-26a2c0f4d21a -2024-05-12T20:51:09.956Z [INFO] server-29905062-da5b-1a0c-721c-93c3f860bff4: added peer, starting replication: peer=server-089bd1fb-cbfe-a5c7-2419-b575a98477b2 -2024-05-12T20:51:09.957Z [INFO] server-29905062-da5b-1a0c-721c-93c3f860bff4: pipelining replication: peer="{Voter server-089bd1fb-cbfe-a5c7-2419-b575a98477b2 089bd1fb-cbfe-a5c7-2419-b575a98477b2}" -2024-05-12T20:51:09.957Z [INFO] server-29905062-da5b-1a0c-721c-93c3f860bff4: pipelining replication: peer="{Voter server-429bbff3-71be-7524-07c3-26a2c0f4d21a 429bbff3-71be-7524-07c3-26a2c0f4d21a}" -2024-05-12T20:51:10.962Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:09.957722073 +0000 UTC m=+81.238422443 (1 nodes), 809.259599ms from start of poll, 816.943171ms from cluster start. Timeout at 2024-05-12 20:51:10.96282608 +0000 UTC m=+82.243526665, 1.005104222s after stability -2024-05-12T20:51:10.964Z [INFO] server-29905062-da5b-1a0c-721c-93c3f860bff4: starting snapshot up to: index=12 -2024-05-12T20:51:10.964Z [INFO] creating new snapshot: path=/tmp/raft995996098/snapshots/2-12-1715547070964.tmp -2024-05-12T20:51:10.965Z [INFO] server-29905062-da5b-1a0c-721c-93c3f860bff4: snapshot complete up to: index=12 -2024-05-12T20:51:10.971Z [INFO] server-3934eb68-24e2-e222-77a3-621884577611: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3934eb68-24e2-e222-77a3-621884577611 Address:3934eb68-24e2-e222-77a3-621884577611} {Suffrage:Voter ID:server-426b45da-82b9-2c98-5909-f8ac67011417 Address:426b45da-82b9-2c98-5909-f8ac67011417} {Suffrage:Voter ID:server-c14929fa-95f6-2110-f686-e0ded37b567a Address:c14929fa-95f6-2110-f686-e0ded37b567a}]" -2024-05-12T20:51:10.971Z [INFO] server-426b45da-82b9-2c98-5909-f8ac67011417: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3934eb68-24e2-e222-77a3-621884577611 Address:3934eb68-24e2-e222-77a3-621884577611} {Suffrage:Voter ID:server-426b45da-82b9-2c98-5909-f8ac67011417 Address:426b45da-82b9-2c98-5909-f8ac67011417} {Suffrage:Voter ID:server-c14929fa-95f6-2110-f686-e0ded37b567a Address:c14929fa-95f6-2110-f686-e0ded37b567a}]" -2024-05-12T20:51:10.971Z [INFO] server-3934eb68-24e2-e222-77a3-621884577611: entering follower state: follower="Node at 3934eb68-24e2-e222-77a3-621884577611 [Follower]" leader-address= leader-id= -2024-05-12T20:51:10.972Z [INFO] server-c14929fa-95f6-2110-f686-e0ded37b567a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3934eb68-24e2-e222-77a3-621884577611 Address:3934eb68-24e2-e222-77a3-621884577611} {Suffrage:Voter ID:server-426b45da-82b9-2c98-5909-f8ac67011417 Address:426b45da-82b9-2c98-5909-f8ac67011417} {Suffrage:Voter ID:server-c14929fa-95f6-2110-f686-e0ded37b567a Address:c14929fa-95f6-2110-f686-e0ded37b567a}]" -2024-05-12T20:51:10.972Z [INFO] server-426b45da-82b9-2c98-5909-f8ac67011417: entering follower state: follower="Node at 426b45da-82b9-2c98-5909-f8ac67011417 [Follower]" leader-address= leader-id= -2024-05-12T20:51:10.972Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:10.972Z [INFO] server-c14929fa-95f6-2110-f686-e0ded37b567a: entering follower state: follower="Node at c14929fa-95f6-2110-f686-e0ded37b567a [Follower]" leader-address= leader-id= -2024-05-12T20:51:11.591Z [WARN] server-c14929fa-95f6-2110-f686-e0ded37b567a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:11.591Z [INFO] server-c14929fa-95f6-2110-f686-e0ded37b567a: entering candidate state: node="Node at c14929fa-95f6-2110-f686-e0ded37b567a [Candidate]" term=2 -2024-05-12T20:51:11.591Z [INFO] server-c14929fa-95f6-2110-f686-e0ded37b567a: election won: term=2 tally=2 -2024-05-12T20:51:11.591Z [INFO] server-c14929fa-95f6-2110-f686-e0ded37b567a: entering leader state: leader="Node at c14929fa-95f6-2110-f686-e0ded37b567a [Leader]" -2024-05-12T20:51:11.591Z [INFO] server-c14929fa-95f6-2110-f686-e0ded37b567a: added peer, starting replication: peer=server-3934eb68-24e2-e222-77a3-621884577611 -2024-05-12T20:51:11.591Z [INFO] server-c14929fa-95f6-2110-f686-e0ded37b567a: added peer, starting replication: peer=server-426b45da-82b9-2c98-5909-f8ac67011417 -2024-05-12T20:51:11.591Z [INFO] server-c14929fa-95f6-2110-f686-e0ded37b567a: pipelining replication: peer="{Voter server-426b45da-82b9-2c98-5909-f8ac67011417 426b45da-82b9-2c98-5909-f8ac67011417}" -2024-05-12T20:51:11.592Z [INFO] server-c14929fa-95f6-2110-f686-e0ded37b567a: pipelining replication: peer="{Voter server-3934eb68-24e2-e222-77a3-621884577611 3934eb68-24e2-e222-77a3-621884577611}" -2024-05-12T20:51:12.597Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:11.592135878 +0000 UTC m=+82.872836430 (1 nodes), 619.96064ms from start of poll, 620.471623ms from cluster start. Timeout at 2024-05-12 20:51:12.597142655 +0000 UTC m=+83.877843016, 1.005006586s after stability -2024-05-12T20:51:12.597Z [INFO] creating new snapshot: path=/tmp/raft42277436/snapshots/2-1013-1715547072597.tmp -2024-05-12T20:51:12.598Z [INFO] server-c14929fa-95f6-2110-f686-e0ded37b567a: copied to local snapshot: bytes=71 -2024-05-12T20:51:12.599Z [INFO] server-c14929fa-95f6-2110-f686-e0ded37b567a: snapshot restore progress: id=2-1013-1715547072597 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:12.599Z [INFO] server-c14929fa-95f6-2110-f686-e0ded37b567a: removing all old logs from log store -2024-05-12T20:51:12.599Z [INFO] server-c14929fa-95f6-2110-f686-e0ded37b567a: compacting logs: from=1 to=2 -2024-05-12T20:51:12.599Z [INFO] server-c14929fa-95f6-2110-f686-e0ded37b567a: restored user snapshot: index=1013 -2024-05-12T20:51:12.599Z [ERROR] server-c14929fa-95f6-2110-f686-e0ded37b567a: failed to get log: index=2 error="log not found" -2024-05-12T20:51:12.599Z [INFO] server-c14929fa-95f6-2110-f686-e0ded37b567a: aborting pipeline replication: peer="{Voter server-426b45da-82b9-2c98-5909-f8ac67011417 426b45da-82b9-2c98-5909-f8ac67011417}" -2024-05-12T20:51:12.599Z [ERROR] server-c14929fa-95f6-2110-f686-e0ded37b567a: failed to get log: index=2 error="log not found" -2024-05-12T20:51:12.599Z [INFO] server-c14929fa-95f6-2110-f686-e0ded37b567a: aborting pipeline replication: peer="{Voter server-3934eb68-24e2-e222-77a3-621884577611 3934eb68-24e2-e222-77a3-621884577611}" -2024-05-12T20:51:12.610Z [ERROR] server-c14929fa-95f6-2110-f686-e0ded37b567a: failed to get log: index=2 error="log not found" -2024-05-12T20:51:12.611Z [ERROR] server-c14929fa-95f6-2110-f686-e0ded37b567a: failed to get log: index=2 error="log not found" -2024-05-12T20:51:12.611Z [INFO] creating new snapshot: path=/tmp/raft568230865/snapshots/2-1013-1715547072611.tmp -2024-05-12T20:51:12.611Z [INFO] creating new snapshot: path=/tmp/raft3319604613/snapshots/2-1013-1715547072611.tmp -2024-05-12T20:51:12.611Z [INFO] server-3934eb68-24e2-e222-77a3-621884577611: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:12.612Z [INFO] server-426b45da-82b9-2c98-5909-f8ac67011417: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:12.612Z [INFO] server-3934eb68-24e2-e222-77a3-621884577611: copied to local snapshot: bytes=71 -2024-05-12T20:51:12.612Z [INFO] server-3934eb68-24e2-e222-77a3-621884577611: snapshot restore progress: id=2-1013-1715547072611 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:12.612Z [INFO] server-3934eb68-24e2-e222-77a3-621884577611: removing all old logs from log store -2024-05-12T20:51:12.612Z [INFO] server-3934eb68-24e2-e222-77a3-621884577611: compacting logs: from=1 to=2 -2024-05-12T20:51:12.612Z [INFO] server-3934eb68-24e2-e222-77a3-621884577611: Installed remote snapshot -2024-05-12T20:51:12.612Z [INFO] server-c14929fa-95f6-2110-f686-e0ded37b567a: pipelining replication: peer="{Voter server-3934eb68-24e2-e222-77a3-621884577611 3934eb68-24e2-e222-77a3-621884577611}" -2024-05-12T20:51:12.612Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:12.613Z [INFO] server-426b45da-82b9-2c98-5909-f8ac67011417: copied to local snapshot: bytes=71 -2024-05-12T20:51:12.613Z [INFO] server-426b45da-82b9-2c98-5909-f8ac67011417: snapshot restore progress: id=2-1013-1715547072611 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:12.613Z [INFO] server-426b45da-82b9-2c98-5909-f8ac67011417: removing all old logs from log store -2024-05-12T20:51:12.613Z [INFO] server-426b45da-82b9-2c98-5909-f8ac67011417: compacting logs: from=1 to=2 -2024-05-12T20:51:12.613Z [INFO] server-426b45da-82b9-2c98-5909-f8ac67011417: Installed remote snapshot -2024-05-12T20:51:12.613Z [INFO] server-c14929fa-95f6-2110-f686-e0ded37b567a: pipelining replication: peer="{Voter server-426b45da-82b9-2c98-5909-f8ac67011417 426b45da-82b9-2c98-5909-f8ac67011417}" -2024-05-12T20:51:13.618Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:12.613007131 +0000 UTC m=+83.893707508 (2 nodes), 1.826µs from start of poll, 1.641342701s from cluster start. Timeout at 2024-05-12 20:51:13.618810524 +0000 UTC m=+84.899510419, 1.005802911s after stability -2024-05-12T20:51:13.635Z [INFO] server-29905062-da5b-1a0c-721c-93c3f860bff4: aborting pipeline replication: peer="{Voter server-089bd1fb-cbfe-a5c7-2419-b575a98477b2 089bd1fb-cbfe-a5c7-2419-b575a98477b2}" -2024-05-12T20:51:13.635Z [INFO] server-29905062-da5b-1a0c-721c-93c3f860bff4: aborting pipeline replication: peer="{Voter server-429bbff3-71be-7524-07c3-26a2c0f4d21a 429bbff3-71be-7524-07c3-26a2c0f4d21a}" +2025-06-15T03:17:43.780Z [INFO] server-b664a6a8-86c6-b1ed-ae4a-11509aeb1abd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b664a6a8-86c6-b1ed-ae4a-11509aeb1abd Address:b664a6a8-86c6-b1ed-ae4a-11509aeb1abd} {Suffrage:Voter ID:server-fdc552b9-ce78-ed6c-0eac-3ca20306628a Address:fdc552b9-ce78-ed6c-0eac-3ca20306628a} {Suffrage:Voter ID:server-3c19ec49-ce06-9065-d4f9-89a4e39cf9fb Address:3c19ec49-ce06-9065-d4f9-89a4e39cf9fb}]" +2025-06-15T03:17:43.780Z [INFO] server-fdc552b9-ce78-ed6c-0eac-3ca20306628a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b664a6a8-86c6-b1ed-ae4a-11509aeb1abd Address:b664a6a8-86c6-b1ed-ae4a-11509aeb1abd} {Suffrage:Voter ID:server-fdc552b9-ce78-ed6c-0eac-3ca20306628a Address:fdc552b9-ce78-ed6c-0eac-3ca20306628a} {Suffrage:Voter ID:server-3c19ec49-ce06-9065-d4f9-89a4e39cf9fb Address:3c19ec49-ce06-9065-d4f9-89a4e39cf9fb}]" +2025-06-15T03:17:43.780Z [INFO] server-b664a6a8-86c6-b1ed-ae4a-11509aeb1abd: entering follower state: follower="Node at b664a6a8-86c6-b1ed-ae4a-11509aeb1abd [Follower]" leader-address= leader-id= +2025-06-15T03:17:43.780Z [INFO] server-fdc552b9-ce78-ed6c-0eac-3ca20306628a: entering follower state: follower="Node at fdc552b9-ce78-ed6c-0eac-3ca20306628a [Follower]" leader-address= leader-id= +2025-06-15T03:17:43.780Z [INFO] server-3c19ec49-ce06-9065-d4f9-89a4e39cf9fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b664a6a8-86c6-b1ed-ae4a-11509aeb1abd Address:b664a6a8-86c6-b1ed-ae4a-11509aeb1abd} {Suffrage:Voter ID:server-fdc552b9-ce78-ed6c-0eac-3ca20306628a Address:fdc552b9-ce78-ed6c-0eac-3ca20306628a} {Suffrage:Voter ID:server-3c19ec49-ce06-9065-d4f9-89a4e39cf9fb Address:3c19ec49-ce06-9065-d4f9-89a4e39cf9fb}]" +2025-06-15T03:17:43.781Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:43.781Z [INFO] server-3c19ec49-ce06-9065-d4f9-89a4e39cf9fb: entering follower state: follower="Node at 3c19ec49-ce06-9065-d4f9-89a4e39cf9fb [Follower]" leader-address= leader-id= +2025-06-15T03:17:44.384Z [WARN] server-3c19ec49-ce06-9065-d4f9-89a4e39cf9fb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:44.384Z [INFO] server-3c19ec49-ce06-9065-d4f9-89a4e39cf9fb: entering candidate state: node="Node at 3c19ec49-ce06-9065-d4f9-89a4e39cf9fb [Candidate]" term=2 +2025-06-15T03:17:44.385Z [INFO] server-3c19ec49-ce06-9065-d4f9-89a4e39cf9fb: election won: term=2 tally=2 +2025-06-15T03:17:44.385Z [INFO] server-3c19ec49-ce06-9065-d4f9-89a4e39cf9fb: entering leader state: leader="Node at 3c19ec49-ce06-9065-d4f9-89a4e39cf9fb [Leader]" +2025-06-15T03:17:44.385Z [INFO] server-3c19ec49-ce06-9065-d4f9-89a4e39cf9fb: added peer, starting replication: peer=server-b664a6a8-86c6-b1ed-ae4a-11509aeb1abd +2025-06-15T03:17:44.385Z [INFO] server-3c19ec49-ce06-9065-d4f9-89a4e39cf9fb: added peer, starting replication: peer=server-fdc552b9-ce78-ed6c-0eac-3ca20306628a +2025-06-15T03:17:44.385Z [INFO] server-3c19ec49-ce06-9065-d4f9-89a4e39cf9fb: pipelining replication: peer="{Voter server-fdc552b9-ce78-ed6c-0eac-3ca20306628a fdc552b9-ce78-ed6c-0eac-3ca20306628a}" +2025-06-15T03:17:44.385Z [INFO] server-3c19ec49-ce06-9065-d4f9-89a4e39cf9fb: pipelining replication: peer="{Voter server-b664a6a8-86c6-b1ed-ae4a-11509aeb1abd b664a6a8-86c6-b1ed-ae4a-11509aeb1abd}" +2025-06-15T03:17:45.390Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:44.385220747 +0000 UTC m=+80.096898914 (1 nodes), 604.213407ms from start of poll, 604.537078ms from cluster start. Timeout at 2025-06-15 03:17:45.390297 +0000 UTC m=+81.101975161, 1.005076247s after stability +2025-06-15T03:17:45.390Z [INFO] server-3c19ec49-ce06-9065-d4f9-89a4e39cf9fb: starting snapshot up to: index=12 +2025-06-15T03:17:45.390Z [INFO] creating new snapshot: path=/tmp/raft3726464951/snapshots/2-12-1749957465390.tmp +2025-06-15T03:17:45.390Z [INFO] server-3c19ec49-ce06-9065-d4f9-89a4e39cf9fb: snapshot complete up to: index=12 +2025-06-15T03:17:45.391Z [INFO] server-7ccc08ca-545e-ed19-815e-a4c72d8ca242: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7ccc08ca-545e-ed19-815e-a4c72d8ca242 Address:7ccc08ca-545e-ed19-815e-a4c72d8ca242} {Suffrage:Voter ID:server-47a26cca-230a-b40e-fbf4-b78eb50a2cf9 Address:47a26cca-230a-b40e-fbf4-b78eb50a2cf9} {Suffrage:Voter ID:server-8879e19b-3122-6dcb-5e52-2b624cd80b70 Address:8879e19b-3122-6dcb-5e52-2b624cd80b70}]" +2025-06-15T03:17:45.391Z [INFO] server-7ccc08ca-545e-ed19-815e-a4c72d8ca242: entering follower state: follower="Node at 7ccc08ca-545e-ed19-815e-a4c72d8ca242 [Follower]" leader-address= leader-id= +2025-06-15T03:17:45.391Z [INFO] server-47a26cca-230a-b40e-fbf4-b78eb50a2cf9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7ccc08ca-545e-ed19-815e-a4c72d8ca242 Address:7ccc08ca-545e-ed19-815e-a4c72d8ca242} {Suffrage:Voter ID:server-47a26cca-230a-b40e-fbf4-b78eb50a2cf9 Address:47a26cca-230a-b40e-fbf4-b78eb50a2cf9} {Suffrage:Voter ID:server-8879e19b-3122-6dcb-5e52-2b624cd80b70 Address:8879e19b-3122-6dcb-5e52-2b624cd80b70}]" +2025-06-15T03:17:45.391Z [INFO] server-47a26cca-230a-b40e-fbf4-b78eb50a2cf9: entering follower state: follower="Node at 47a26cca-230a-b40e-fbf4-b78eb50a2cf9 [Follower]" leader-address= leader-id= +2025-06-15T03:17:45.391Z [INFO] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7ccc08ca-545e-ed19-815e-a4c72d8ca242 Address:7ccc08ca-545e-ed19-815e-a4c72d8ca242} {Suffrage:Voter ID:server-47a26cca-230a-b40e-fbf4-b78eb50a2cf9 Address:47a26cca-230a-b40e-fbf4-b78eb50a2cf9} {Suffrage:Voter ID:server-8879e19b-3122-6dcb-5e52-2b624cd80b70 Address:8879e19b-3122-6dcb-5e52-2b624cd80b70}]" +2025-06-15T03:17:45.391Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:45.391Z [INFO] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: entering follower state: follower="Node at 8879e19b-3122-6dcb-5e52-2b624cd80b70 [Follower]" leader-address= leader-id= +2025-06-15T03:17:46.230Z [WARN] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:46.230Z [INFO] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: entering candidate state: node="Node at 8879e19b-3122-6dcb-5e52-2b624cd80b70 [Candidate]" term=2 +2025-06-15T03:17:46.231Z [INFO] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: election won: term=2 tally=2 +2025-06-15T03:17:46.231Z [INFO] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: entering leader state: leader="Node at 8879e19b-3122-6dcb-5e52-2b624cd80b70 [Leader]" +2025-06-15T03:17:46.231Z [INFO] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: added peer, starting replication: peer=server-7ccc08ca-545e-ed19-815e-a4c72d8ca242 +2025-06-15T03:17:46.231Z [INFO] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: added peer, starting replication: peer=server-47a26cca-230a-b40e-fbf4-b78eb50a2cf9 +2025-06-15T03:17:46.231Z [INFO] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: pipelining replication: peer="{Voter server-7ccc08ca-545e-ed19-815e-a4c72d8ca242 7ccc08ca-545e-ed19-815e-a4c72d8ca242}" +2025-06-15T03:17:46.231Z [INFO] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: pipelining replication: peer="{Voter server-47a26cca-230a-b40e-fbf4-b78eb50a2cf9 47a26cca-230a-b40e-fbf4-b78eb50a2cf9}" +2025-06-15T03:17:47.237Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:46.231086453 +0000 UTC m=+81.942764653 (1 nodes), 839.240574ms from start of poll, 839.483801ms from cluster start. Timeout at 2025-06-15 03:17:47.236994082 +0000 UTC m=+82.948672281, 1.005907628s after stability +2025-06-15T03:17:47.237Z [INFO] creating new snapshot: path=/tmp/raft590302289/snapshots/2-1013-1749957467237.tmp +2025-06-15T03:17:47.237Z [INFO] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: copied to local snapshot: bytes=71 +2025-06-15T03:17:47.238Z [INFO] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: snapshot restore progress: id=2-1013-1749957467237 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:47.238Z [INFO] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: removing all old logs from log store +2025-06-15T03:17:47.238Z [INFO] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: compacting logs: from=1 to=2 +2025-06-15T03:17:47.238Z [INFO] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: restored user snapshot: index=1013 +2025-06-15T03:17:47.238Z [ERROR] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: failed to get log: index=2 error="log not found" +2025-06-15T03:17:47.238Z [INFO] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: aborting pipeline replication: peer="{Voter server-47a26cca-230a-b40e-fbf4-b78eb50a2cf9 47a26cca-230a-b40e-fbf4-b78eb50a2cf9}" +2025-06-15T03:17:47.238Z [ERROR] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: failed to get log: index=2 error="log not found" +2025-06-15T03:17:47.238Z [INFO] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: aborting pipeline replication: peer="{Voter server-7ccc08ca-545e-ed19-815e-a4c72d8ca242 7ccc08ca-545e-ed19-815e-a4c72d8ca242}" +2025-06-15T03:17:47.246Z [ERROR] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: failed to get log: index=2 error="log not found" +2025-06-15T03:17:47.246Z [INFO] creating new snapshot: path=/tmp/raft620264741/snapshots/2-1013-1749957467246.tmp +2025-06-15T03:17:47.246Z [INFO] server-7ccc08ca-545e-ed19-815e-a4c72d8ca242: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:47.247Z [INFO] server-7ccc08ca-545e-ed19-815e-a4c72d8ca242: copied to local snapshot: bytes=71 +2025-06-15T03:17:47.247Z [ERROR] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: failed to get log: index=2 error="log not found" +2025-06-15T03:17:47.247Z [INFO] server-7ccc08ca-545e-ed19-815e-a4c72d8ca242: snapshot restore progress: id=2-1013-1749957467246 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:47.247Z [INFO] server-7ccc08ca-545e-ed19-815e-a4c72d8ca242: removing all old logs from log store +2025-06-15T03:17:47.247Z [INFO] server-7ccc08ca-545e-ed19-815e-a4c72d8ca242: compacting logs: from=1 to=2 +2025-06-15T03:17:47.247Z [INFO] server-7ccc08ca-545e-ed19-815e-a4c72d8ca242: Installed remote snapshot +2025-06-15T03:17:47.247Z [INFO] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: pipelining replication: peer="{Voter server-7ccc08ca-545e-ed19-815e-a4c72d8ca242 7ccc08ca-545e-ed19-815e-a4c72d8ca242}" +2025-06-15T03:17:47.247Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:17:47.247Z [INFO] creating new snapshot: path=/tmp/raft3712778191/snapshots/2-1013-1749957467247.tmp +2025-06-15T03:17:47.247Z [INFO] server-47a26cca-230a-b40e-fbf4-b78eb50a2cf9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:47.247Z [INFO] server-47a26cca-230a-b40e-fbf4-b78eb50a2cf9: copied to local snapshot: bytes=71 +2025-06-15T03:17:47.248Z [INFO] server-47a26cca-230a-b40e-fbf4-b78eb50a2cf9: snapshot restore progress: id=2-1013-1749957467247 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:47.248Z [INFO] server-47a26cca-230a-b40e-fbf4-b78eb50a2cf9: removing all old logs from log store +2025-06-15T03:17:47.248Z [INFO] server-47a26cca-230a-b40e-fbf4-b78eb50a2cf9: compacting logs: from=1 to=2 +2025-06-15T03:17:47.248Z [INFO] server-47a26cca-230a-b40e-fbf4-b78eb50a2cf9: Installed remote snapshot +2025-06-15T03:17:47.248Z [INFO] server-8879e19b-3122-6dcb-5e52-2b624cd80b70: pipelining replication: peer="{Voter server-47a26cca-230a-b40e-fbf4-b78eb50a2cf9 47a26cca-230a-b40e-fbf4-b78eb50a2cf9}" +2025-06-15T03:17:48.253Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:17:47.247545912 +0000 UTC m=+82.959224079 (2 nodes), 468ns from start of poll, 1.855943227s from cluster start. Timeout at 2025-06-15 03:17:48.253177473 +0000 UTC m=+83.964855639, 1.00563156s after stability +2025-06-15T03:17:48.269Z [INFO] server-3c19ec49-ce06-9065-d4f9-89a4e39cf9fb: aborting pipeline replication: peer="{Voter server-fdc552b9-ce78-ed6c-0eac-3ca20306628a fdc552b9-ce78-ed6c-0eac-3ca20306628a}" +2025-06-15T03:17:48.269Z [INFO] server-3c19ec49-ce06-9065-d4f9-89a4e39cf9fb: aborting pipeline replication: peer="{Voter server-b664a6a8-86c6-b1ed-ae4a-11509aeb1abd b664a6a8-86c6-b1ed-ae4a-11509aeb1abd}" === RUN TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false -2024-05-12T20:51:13.638Z [INFO] server-38cb0d97-a16d-794d-3edd-8003db632b97: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-38cb0d97-a16d-794d-3edd-8003db632b97 Address:38cb0d97-a16d-794d-3edd-8003db632b97} {Suffrage:Voter ID:server-4b224d47-c229-845d-84c3-a33528b473a3 Address:4b224d47-c229-845d-84c3-a33528b473a3} {Suffrage:Voter ID:server-196a3892-0e4c-1e84-8159-00b2f3b4b1b8 Address:196a3892-0e4c-1e84-8159-00b2f3b4b1b8}]" -2024-05-12T20:51:13.638Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-38cb0d97-a16d-794d-3edd-8003db632b97 Address:38cb0d97-a16d-794d-3edd-8003db632b97} {Suffrage:Voter ID:server-4b224d47-c229-845d-84c3-a33528b473a3 Address:4b224d47-c229-845d-84c3-a33528b473a3} {Suffrage:Voter ID:server-196a3892-0e4c-1e84-8159-00b2f3b4b1b8 Address:196a3892-0e4c-1e84-8159-00b2f3b4b1b8}]" -2024-05-12T20:51:13.638Z [INFO] server-196a3892-0e4c-1e84-8159-00b2f3b4b1b8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-38cb0d97-a16d-794d-3edd-8003db632b97 Address:38cb0d97-a16d-794d-3edd-8003db632b97} {Suffrage:Voter ID:server-4b224d47-c229-845d-84c3-a33528b473a3 Address:4b224d47-c229-845d-84c3-a33528b473a3} {Suffrage:Voter ID:server-196a3892-0e4c-1e84-8159-00b2f3b4b1b8 Address:196a3892-0e4c-1e84-8159-00b2f3b4b1b8}]" -2024-05-12T20:51:13.638Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:13.639Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: entering follower state: follower="Node at 4b224d47-c229-845d-84c3-a33528b473a3 [Follower]" leader-address= leader-id= -2024-05-12T20:51:13.639Z [INFO] server-196a3892-0e4c-1e84-8159-00b2f3b4b1b8: entering follower state: follower="Node at 196a3892-0e4c-1e84-8159-00b2f3b4b1b8 [Follower]" leader-address= leader-id= -2024-05-12T20:51:13.639Z [INFO] server-38cb0d97-a16d-794d-3edd-8003db632b97: entering follower state: follower="Node at 38cb0d97-a16d-794d-3edd-8003db632b97 [Follower]" leader-address= leader-id= -2024-05-12T20:51:14.539Z [WARN] server-4b224d47-c229-845d-84c3-a33528b473a3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:14.539Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: entering candidate state: node="Node at 4b224d47-c229-845d-84c3-a33528b473a3 [Candidate]" term=2 -2024-05-12T20:51:14.540Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: election won: term=2 tally=2 -2024-05-12T20:51:14.540Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: entering leader state: leader="Node at 4b224d47-c229-845d-84c3-a33528b473a3 [Leader]" -2024-05-12T20:51:14.540Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: added peer, starting replication: peer=server-38cb0d97-a16d-794d-3edd-8003db632b97 -2024-05-12T20:51:14.540Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: added peer, starting replication: peer=server-196a3892-0e4c-1e84-8159-00b2f3b4b1b8 -2024-05-12T20:51:14.540Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: pipelining replication: peer="{Voter server-196a3892-0e4c-1e84-8159-00b2f3b4b1b8 196a3892-0e4c-1e84-8159-00b2f3b4b1b8}" -2024-05-12T20:51:14.541Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: pipelining replication: peer="{Voter server-38cb0d97-a16d-794d-3edd-8003db632b97 38cb0d97-a16d-794d-3edd-8003db632b97}" -2024-05-12T20:51:15.550Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:14.541389831 +0000 UTC m=+85.822090306 (1 nodes), 902.731813ms from start of poll, 903.269326ms from cluster start. Timeout at 2024-05-12 20:51:15.550807826 +0000 UTC m=+86.831507721, 1.009417415s after stability -2024-05-12T20:51:15.552Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: starting snapshot up to: index=12 -2024-05-12T20:51:15.552Z [INFO] creating new snapshot: path=/tmp/raft60081281/snapshots/2-12-1715547075552.tmp -2024-05-12T20:51:15.554Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: snapshot complete up to: index=12 -2024-05-12T20:51:15.556Z [INFO] creating new snapshot: path=/tmp/raft60081281/snapshots/2-10013-1715547075556.tmp -2024-05-12T20:51:15.556Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: copied to local snapshot: bytes=71 -2024-05-12T20:51:15.557Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: snapshot restore progress: id=2-10013-1715547075556 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:15.557Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: restored user snapshot: index=10013 -2024-05-12T20:51:15.557Z [ERROR] server-4b224d47-c229-845d-84c3-a33528b473a3: failed to get log: index=23 error="log not found" -2024-05-12T20:51:15.557Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: aborting pipeline replication: peer="{Voter server-196a3892-0e4c-1e84-8159-00b2f3b4b1b8 196a3892-0e4c-1e84-8159-00b2f3b4b1b8}" -2024-05-12T20:51:15.557Z [ERROR] server-4b224d47-c229-845d-84c3-a33528b473a3: failed to get log: index=23 error="log not found" -2024-05-12T20:51:15.557Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: aborting pipeline replication: peer="{Voter server-38cb0d97-a16d-794d-3edd-8003db632b97 38cb0d97-a16d-794d-3edd-8003db632b97}" -2024-05-12T20:51:15.557Z [ERROR] server-4b224d47-c229-845d-84c3-a33528b473a3: failed to get log: index=23 error="log not found" -2024-05-12T20:51:15.557Z [INFO] creating new snapshot: path=/tmp/raft2054292928/snapshots/2-10013-1715547075557.tmp -2024-05-12T20:51:15.558Z [INFO] server-38cb0d97-a16d-794d-3edd-8003db632b97: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:15.558Z [INFO] server-38cb0d97-a16d-794d-3edd-8003db632b97: copied to local snapshot: bytes=71 -2024-05-12T20:51:15.558Z [INFO] server-38cb0d97-a16d-794d-3edd-8003db632b97: snapshot restore progress: id=2-10013-1715547075557 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:15.558Z [INFO] server-38cb0d97-a16d-794d-3edd-8003db632b97: Installed remote snapshot -2024-05-12T20:51:15.575Z [ERROR] server-4b224d47-c229-845d-84c3-a33528b473a3: failed to get log: index=23 error="log not found" -2024-05-12T20:51:15.581Z [WARN] server-38cb0d97-a16d-794d-3edd-8003db632b97: skipping application of old log: index=22 -2024-05-12T20:51:15.582Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: pipelining replication: peer="{Voter server-38cb0d97-a16d-794d-3edd-8003db632b97 38cb0d97-a16d-794d-3edd-8003db632b97}" -2024-05-12T20:51:15.582Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:15.582Z [INFO] creating new snapshot: path=/tmp/raft3990241770/snapshots/2-10013-1715547075582.tmp -2024-05-12T20:51:15.611Z [INFO] server-196a3892-0e4c-1e84-8159-00b2f3b4b1b8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:15.612Z [INFO] server-196a3892-0e4c-1e84-8159-00b2f3b4b1b8: copied to local snapshot: bytes=71 -2024-05-12T20:51:15.612Z [INFO] server-196a3892-0e4c-1e84-8159-00b2f3b4b1b8: snapshot restore progress: id=2-10013-1715547075582 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:15.612Z [INFO] server-196a3892-0e4c-1e84-8159-00b2f3b4b1b8: Installed remote snapshot -2024-05-12T20:51:15.613Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: pipelining replication: peer="{Voter server-196a3892-0e4c-1e84-8159-00b2f3b4b1b8 196a3892-0e4c-1e84-8159-00b2f3b4b1b8}" -2024-05-12T20:51:16.590Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:15.582308109 +0000 UTC m=+86.863008487 (2 nodes), 1.994µs from start of poll, 1.944187507s from cluster start. Timeout at 2024-05-12 20:51:16.590794157 +0000 UTC m=+87.871494742, 1.008486255s after stability -2024-05-12T20:51:16.611Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: aborting pipeline replication: peer="{Voter server-196a3892-0e4c-1e84-8159-00b2f3b4b1b8 196a3892-0e4c-1e84-8159-00b2f3b4b1b8}" -2024-05-12T20:51:16.611Z [INFO] server-4b224d47-c229-845d-84c3-a33528b473a3: aborting pipeline replication: peer="{Voter server-38cb0d97-a16d-794d-3edd-8003db632b97 38cb0d97-a16d-794d-3edd-8003db632b97}" +2025-06-15T03:17:48.270Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b7719e25-5d73-6693-2c6b-904e0c702a1b Address:b7719e25-5d73-6693-2c6b-904e0c702a1b} {Suffrage:Voter ID:server-87bb5214-aa7f-b39e-0011-b57193e7f60b Address:87bb5214-aa7f-b39e-0011-b57193e7f60b} {Suffrage:Voter ID:server-6a27d8b1-de13-dcdc-7e4d-2205ae685064 Address:6a27d8b1-de13-dcdc-7e4d-2205ae685064}]" +2025-06-15T03:17:48.270Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: entering follower state: follower="Node at b7719e25-5d73-6693-2c6b-904e0c702a1b [Follower]" leader-address= leader-id= +2025-06-15T03:17:48.271Z [INFO] server-87bb5214-aa7f-b39e-0011-b57193e7f60b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b7719e25-5d73-6693-2c6b-904e0c702a1b Address:b7719e25-5d73-6693-2c6b-904e0c702a1b} {Suffrage:Voter ID:server-87bb5214-aa7f-b39e-0011-b57193e7f60b Address:87bb5214-aa7f-b39e-0011-b57193e7f60b} {Suffrage:Voter ID:server-6a27d8b1-de13-dcdc-7e4d-2205ae685064 Address:6a27d8b1-de13-dcdc-7e4d-2205ae685064}]" +2025-06-15T03:17:48.271Z [INFO] server-87bb5214-aa7f-b39e-0011-b57193e7f60b: entering follower state: follower="Node at 87bb5214-aa7f-b39e-0011-b57193e7f60b [Follower]" leader-address= leader-id= +2025-06-15T03:17:48.271Z [INFO] server-6a27d8b1-de13-dcdc-7e4d-2205ae685064: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b7719e25-5d73-6693-2c6b-904e0c702a1b Address:b7719e25-5d73-6693-2c6b-904e0c702a1b} {Suffrage:Voter ID:server-87bb5214-aa7f-b39e-0011-b57193e7f60b Address:87bb5214-aa7f-b39e-0011-b57193e7f60b} {Suffrage:Voter ID:server-6a27d8b1-de13-dcdc-7e4d-2205ae685064 Address:6a27d8b1-de13-dcdc-7e4d-2205ae685064}]" +2025-06-15T03:17:48.271Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:48.271Z [INFO] server-6a27d8b1-de13-dcdc-7e4d-2205ae685064: entering follower state: follower="Node at 6a27d8b1-de13-dcdc-7e4d-2205ae685064 [Follower]" leader-address= leader-id= +2025-06-15T03:17:49.043Z [WARN] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:49.043Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: entering candidate state: node="Node at b7719e25-5d73-6693-2c6b-904e0c702a1b [Candidate]" term=2 +2025-06-15T03:17:49.043Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: election won: term=2 tally=2 +2025-06-15T03:17:49.043Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: entering leader state: leader="Node at b7719e25-5d73-6693-2c6b-904e0c702a1b [Leader]" +2025-06-15T03:17:49.043Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: added peer, starting replication: peer=server-87bb5214-aa7f-b39e-0011-b57193e7f60b +2025-06-15T03:17:49.043Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: added peer, starting replication: peer=server-6a27d8b1-de13-dcdc-7e4d-2205ae685064 +2025-06-15T03:17:49.043Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: pipelining replication: peer="{Voter server-6a27d8b1-de13-dcdc-7e4d-2205ae685064 6a27d8b1-de13-dcdc-7e4d-2205ae685064}" +2025-06-15T03:17:49.043Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: pipelining replication: peer="{Voter server-87bb5214-aa7f-b39e-0011-b57193e7f60b 87bb5214-aa7f-b39e-0011-b57193e7f60b}" +2025-06-15T03:17:50.048Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:49.043302662 +0000 UTC m=+84.754980827 (1 nodes), 771.968969ms from start of poll, 772.660171ms from cluster start. Timeout at 2025-06-15 03:17:50.048320917 +0000 UTC m=+85.759999082, 1.005018255s after stability +2025-06-15T03:17:50.048Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: starting snapshot up to: index=12 +2025-06-15T03:17:50.048Z [INFO] creating new snapshot: path=/tmp/raft2224330382/snapshots/2-12-1749957470048.tmp +2025-06-15T03:17:50.049Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: snapshot complete up to: index=12 +2025-06-15T03:17:50.049Z [INFO] creating new snapshot: path=/tmp/raft2224330382/snapshots/2-10013-1749957470049.tmp +2025-06-15T03:17:50.050Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: copied to local snapshot: bytes=71 +2025-06-15T03:17:50.050Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: snapshot restore progress: id=2-10013-1749957470049 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:50.050Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: restored user snapshot: index=10013 +2025-06-15T03:17:50.050Z [ERROR] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: failed to get log: index=23 error="log not found" +2025-06-15T03:17:50.050Z [ERROR] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: failed to get log: index=23 error="log not found" +2025-06-15T03:17:50.050Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: aborting pipeline replication: peer="{Voter server-87bb5214-aa7f-b39e-0011-b57193e7f60b 87bb5214-aa7f-b39e-0011-b57193e7f60b}" +2025-06-15T03:17:50.050Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: aborting pipeline replication: peer="{Voter server-6a27d8b1-de13-dcdc-7e4d-2205ae685064 6a27d8b1-de13-dcdc-7e4d-2205ae685064}" +2025-06-15T03:17:50.056Z [ERROR] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: failed to get log: index=23 error="log not found" +2025-06-15T03:17:50.056Z [ERROR] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: failed to get log: index=23 error="log not found" +2025-06-15T03:17:50.056Z [INFO] creating new snapshot: path=/tmp/raft106401313/snapshots/2-10013-1749957470056.tmp +2025-06-15T03:17:50.056Z [INFO] creating new snapshot: path=/tmp/raft2396983308/snapshots/2-10013-1749957470056.tmp +2025-06-15T03:17:50.057Z [INFO] server-87bb5214-aa7f-b39e-0011-b57193e7f60b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:50.057Z [INFO] server-6a27d8b1-de13-dcdc-7e4d-2205ae685064: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:50.057Z [INFO] server-6a27d8b1-de13-dcdc-7e4d-2205ae685064: copied to local snapshot: bytes=71 +2025-06-15T03:17:50.057Z [INFO] server-87bb5214-aa7f-b39e-0011-b57193e7f60b: copied to local snapshot: bytes=71 +2025-06-15T03:17:50.057Z [INFO] server-6a27d8b1-de13-dcdc-7e4d-2205ae685064: snapshot restore progress: id=2-10013-1749957470056 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:50.057Z [INFO] server-6a27d8b1-de13-dcdc-7e4d-2205ae685064: Installed remote snapshot +2025-06-15T03:17:50.057Z [INFO] server-87bb5214-aa7f-b39e-0011-b57193e7f60b: snapshot restore progress: id=2-10013-1749957470056 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:50.057Z [WARN] server-6a27d8b1-de13-dcdc-7e4d-2205ae685064: skipping application of old log: index=22 +2025-06-15T03:17:50.057Z [INFO] server-87bb5214-aa7f-b39e-0011-b57193e7f60b: Installed remote snapshot +2025-06-15T03:17:50.057Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: pipelining replication: peer="{Voter server-6a27d8b1-de13-dcdc-7e4d-2205ae685064 6a27d8b1-de13-dcdc-7e4d-2205ae685064}" +2025-06-15T03:17:50.057Z [WARN] server-87bb5214-aa7f-b39e-0011-b57193e7f60b: skipping application of old log: index=10013 +2025-06-15T03:17:50.057Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:17:50.057Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: pipelining replication: peer="{Voter server-87bb5214-aa7f-b39e-0011-b57193e7f60b 87bb5214-aa7f-b39e-0011-b57193e7f60b}" +2025-06-15T03:17:51.063Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:17:50.057548709 +0000 UTC m=+85.769226874 (2 nodes), 560ns from start of poll, 1.786906218s from cluster start. Timeout at 2025-06-15 03:17:51.063284662 +0000 UTC m=+86.774962827, 1.005735953s after stability +2025-06-15T03:17:51.075Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: aborting pipeline replication: peer="{Voter server-6a27d8b1-de13-dcdc-7e4d-2205ae685064 6a27d8b1-de13-dcdc-7e4d-2205ae685064}" +2025-06-15T03:17:51.075Z [INFO] server-b7719e25-5d73-6693-2c6b-904e0c702a1b: aborting pipeline replication: peer="{Voter server-87bb5214-aa7f-b39e-0011-b57193e7f60b 87bb5214-aa7f-b39e-0011-b57193e7f60b}" === RUN TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false -2024-05-12T20:51:16.614Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed Address:8a1e4392-2c2a-1e2e-2780-b2b70e2754ed} {Suffrage:Voter ID:server-b4135339-4f62-343f-4fcf-98d3da350750 Address:b4135339-4f62-343f-4fcf-98d3da350750} {Suffrage:Voter ID:server-7d0200e5-00e0-be0d-d3b5-09f924554179 Address:7d0200e5-00e0-be0d-d3b5-09f924554179}]" -2024-05-12T20:51:16.614Z [INFO] server-b4135339-4f62-343f-4fcf-98d3da350750: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed Address:8a1e4392-2c2a-1e2e-2780-b2b70e2754ed} {Suffrage:Voter ID:server-b4135339-4f62-343f-4fcf-98d3da350750 Address:b4135339-4f62-343f-4fcf-98d3da350750} {Suffrage:Voter ID:server-7d0200e5-00e0-be0d-d3b5-09f924554179 Address:7d0200e5-00e0-be0d-d3b5-09f924554179}]" -2024-05-12T20:51:16.615Z [INFO] server-7d0200e5-00e0-be0d-d3b5-09f924554179: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed Address:8a1e4392-2c2a-1e2e-2780-b2b70e2754ed} {Suffrage:Voter ID:server-b4135339-4f62-343f-4fcf-98d3da350750 Address:b4135339-4f62-343f-4fcf-98d3da350750} {Suffrage:Voter ID:server-7d0200e5-00e0-be0d-d3b5-09f924554179 Address:7d0200e5-00e0-be0d-d3b5-09f924554179}]" -2024-05-12T20:51:16.615Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:16.615Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: entering follower state: follower="Node at 8a1e4392-2c2a-1e2e-2780-b2b70e2754ed [Follower]" leader-address= leader-id= -2024-05-12T20:51:16.615Z [INFO] server-7d0200e5-00e0-be0d-d3b5-09f924554179: entering follower state: follower="Node at 7d0200e5-00e0-be0d-d3b5-09f924554179 [Follower]" leader-address= leader-id= -2024-05-12T20:51:16.615Z [INFO] server-b4135339-4f62-343f-4fcf-98d3da350750: entering follower state: follower="Node at b4135339-4f62-343f-4fcf-98d3da350750 [Follower]" leader-address= leader-id= -2024-05-12T20:51:17.314Z [WARN] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:17.314Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: entering candidate state: node="Node at 8a1e4392-2c2a-1e2e-2780-b2b70e2754ed [Candidate]" term=2 -2024-05-12T20:51:17.315Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: election won: term=2 tally=2 -2024-05-12T20:51:17.315Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: entering leader state: leader="Node at 8a1e4392-2c2a-1e2e-2780-b2b70e2754ed [Leader]" -2024-05-12T20:51:17.315Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: added peer, starting replication: peer=server-b4135339-4f62-343f-4fcf-98d3da350750 -2024-05-12T20:51:17.315Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: added peer, starting replication: peer=server-7d0200e5-00e0-be0d-d3b5-09f924554179 -2024-05-12T20:51:17.315Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: pipelining replication: peer="{Voter server-7d0200e5-00e0-be0d-d3b5-09f924554179 7d0200e5-00e0-be0d-d3b5-09f924554179}" -2024-05-12T20:51:17.315Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: pipelining replication: peer="{Voter server-b4135339-4f62-343f-4fcf-98d3da350750 b4135339-4f62-343f-4fcf-98d3da350750}" -2024-05-12T20:51:18.326Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:17.315404399 +0000 UTC m=+88.596104808 (1 nodes), 700.2441ms from start of poll, 701.021766ms from cluster start. Timeout at 2024-05-12 20:51:18.326784165 +0000 UTC m=+89.607485884, 1.011381076s after stability -2024-05-12T20:51:18.328Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: starting snapshot up to: index=12 -2024-05-12T20:51:18.328Z [INFO] creating new snapshot: path=/tmp/raft1021890663/snapshots/2-12-1715547078328.tmp -2024-05-12T20:51:18.329Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: snapshot complete up to: index=12 -2024-05-12T20:51:18.331Z [INFO] creating new snapshot: path=/tmp/raft1021890663/snapshots/2-10013-1715547078331.tmp -2024-05-12T20:51:18.332Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: copied to local snapshot: bytes=71 -2024-05-12T20:51:18.338Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: snapshot restore progress: id=2-10013-1715547078331 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:18.339Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: removing all old logs from log store -2024-05-12T20:51:18.339Z [ERROR] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: failed to get log: index=23 error="log not found" -2024-05-12T20:51:18.339Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: aborting pipeline replication: peer="{Voter server-7d0200e5-00e0-be0d-d3b5-09f924554179 7d0200e5-00e0-be0d-d3b5-09f924554179}" -2024-05-12T20:51:18.339Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: compacting logs: from=1 to=22 -2024-05-12T20:51:18.339Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: restored user snapshot: index=10013 -2024-05-12T20:51:18.339Z [ERROR] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: failed to get log: index=22 error="log not found" -2024-05-12T20:51:18.339Z [ERROR] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: failed to get log: index=22 error="log not found" -2024-05-12T20:51:18.339Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: aborting pipeline replication: peer="{Voter server-b4135339-4f62-343f-4fcf-98d3da350750 b4135339-4f62-343f-4fcf-98d3da350750}" -2024-05-12T20:51:18.339Z [INFO] creating new snapshot: path=/tmp/raft161144009/snapshots/2-10013-1715547078339.tmp -2024-05-12T20:51:18.339Z [INFO] server-7d0200e5-00e0-be0d-d3b5-09f924554179: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:18.340Z [INFO] server-7d0200e5-00e0-be0d-d3b5-09f924554179: copied to local snapshot: bytes=71 -2024-05-12T20:51:18.340Z [INFO] server-7d0200e5-00e0-be0d-d3b5-09f924554179: snapshot restore progress: id=2-10013-1715547078339 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:18.340Z [INFO] server-7d0200e5-00e0-be0d-d3b5-09f924554179: removing all old logs from log store -2024-05-12T20:51:18.340Z [INFO] server-7d0200e5-00e0-be0d-d3b5-09f924554179: compacting logs: from=1 to=22 -2024-05-12T20:51:18.340Z [INFO] server-7d0200e5-00e0-be0d-d3b5-09f924554179: Installed remote snapshot -2024-05-12T20:51:18.340Z [WARN] server-7d0200e5-00e0-be0d-d3b5-09f924554179: skipping application of old log: index=22 -2024-05-12T20:51:18.341Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: pipelining replication: peer="{Voter server-7d0200e5-00e0-be0d-d3b5-09f924554179 7d0200e5-00e0-be0d-d3b5-09f924554179}" -2024-05-12T20:51:18.341Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:18.346Z [ERROR] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: failed to get log: index=22 error="log not found" -2024-05-12T20:51:18.348Z [INFO] creating new snapshot: path=/tmp/raft4068220900/snapshots/2-10013-1715547078348.tmp -2024-05-12T20:51:18.348Z [INFO] server-b4135339-4f62-343f-4fcf-98d3da350750: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:18.349Z [INFO] server-b4135339-4f62-343f-4fcf-98d3da350750: copied to local snapshot: bytes=71 -2024-05-12T20:51:18.349Z [INFO] server-b4135339-4f62-343f-4fcf-98d3da350750: snapshot restore progress: id=2-10013-1715547078348 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:18.349Z [INFO] server-b4135339-4f62-343f-4fcf-98d3da350750: removing all old logs from log store -2024-05-12T20:51:18.349Z [INFO] server-b4135339-4f62-343f-4fcf-98d3da350750: compacting logs: from=1 to=22 -2024-05-12T20:51:18.349Z [INFO] server-b4135339-4f62-343f-4fcf-98d3da350750: Installed remote snapshot -2024-05-12T20:51:18.349Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: pipelining replication: peer="{Voter server-b4135339-4f62-343f-4fcf-98d3da350750 b4135339-4f62-343f-4fcf-98d3da350750}" -2024-05-12T20:51:19.347Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:18.341104587 +0000 UTC m=+89.621804982 (2 nodes), 1.842µs from start of poll, 1.72672194s from cluster start. Timeout at 2024-05-12 20:51:19.346997278 +0000 UTC m=+90.627697708, 1.005892726s after stability -2024-05-12T20:51:19.371Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: aborting pipeline replication: peer="{Voter server-7d0200e5-00e0-be0d-d3b5-09f924554179 7d0200e5-00e0-be0d-d3b5-09f924554179}" -2024-05-12T20:51:19.371Z [INFO] server-8a1e4392-2c2a-1e2e-2780-b2b70e2754ed: aborting pipeline replication: peer="{Voter server-b4135339-4f62-343f-4fcf-98d3da350750 b4135339-4f62-343f-4fcf-98d3da350750}" +2025-06-15T03:17:51.076Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55e3eaa6-2113-29ea-856c-e82e2f79780d Address:55e3eaa6-2113-29ea-856c-e82e2f79780d} {Suffrage:Voter ID:server-8789dc75-b4d8-71dc-fc2d-374466b5bafe Address:8789dc75-b4d8-71dc-fc2d-374466b5bafe} {Suffrage:Voter ID:server-6959b9af-9baf-17d1-dc7e-7f1d2c5123ea Address:6959b9af-9baf-17d1-dc7e-7f1d2c5123ea}]" +2025-06-15T03:17:51.077Z [INFO] server-8789dc75-b4d8-71dc-fc2d-374466b5bafe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55e3eaa6-2113-29ea-856c-e82e2f79780d Address:55e3eaa6-2113-29ea-856c-e82e2f79780d} {Suffrage:Voter ID:server-8789dc75-b4d8-71dc-fc2d-374466b5bafe Address:8789dc75-b4d8-71dc-fc2d-374466b5bafe} {Suffrage:Voter ID:server-6959b9af-9baf-17d1-dc7e-7f1d2c5123ea Address:6959b9af-9baf-17d1-dc7e-7f1d2c5123ea}]" +2025-06-15T03:17:51.077Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: entering follower state: follower="Node at 55e3eaa6-2113-29ea-856c-e82e2f79780d [Follower]" leader-address= leader-id= +2025-06-15T03:17:51.077Z [INFO] server-8789dc75-b4d8-71dc-fc2d-374466b5bafe: entering follower state: follower="Node at 8789dc75-b4d8-71dc-fc2d-374466b5bafe [Follower]" leader-address= leader-id= +2025-06-15T03:17:51.077Z [INFO] server-6959b9af-9baf-17d1-dc7e-7f1d2c5123ea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55e3eaa6-2113-29ea-856c-e82e2f79780d Address:55e3eaa6-2113-29ea-856c-e82e2f79780d} {Suffrage:Voter ID:server-8789dc75-b4d8-71dc-fc2d-374466b5bafe Address:8789dc75-b4d8-71dc-fc2d-374466b5bafe} {Suffrage:Voter ID:server-6959b9af-9baf-17d1-dc7e-7f1d2c5123ea Address:6959b9af-9baf-17d1-dc7e-7f1d2c5123ea}]" +2025-06-15T03:17:51.077Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:51.077Z [INFO] server-6959b9af-9baf-17d1-dc7e-7f1d2c5123ea: entering follower state: follower="Node at 6959b9af-9baf-17d1-dc7e-7f1d2c5123ea [Follower]" leader-address= leader-id= +2025-06-15T03:17:51.604Z [WARN] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:51.604Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: entering candidate state: node="Node at 55e3eaa6-2113-29ea-856c-e82e2f79780d [Candidate]" term=2 +2025-06-15T03:17:51.605Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: election won: term=2 tally=2 +2025-06-15T03:17:51.605Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: entering leader state: leader="Node at 55e3eaa6-2113-29ea-856c-e82e2f79780d [Leader]" +2025-06-15T03:17:51.605Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: added peer, starting replication: peer=server-8789dc75-b4d8-71dc-fc2d-374466b5bafe +2025-06-15T03:17:51.605Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: added peer, starting replication: peer=server-6959b9af-9baf-17d1-dc7e-7f1d2c5123ea +2025-06-15T03:17:51.605Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: pipelining replication: peer="{Voter server-6959b9af-9baf-17d1-dc7e-7f1d2c5123ea 6959b9af-9baf-17d1-dc7e-7f1d2c5123ea}" +2025-06-15T03:17:51.605Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: pipelining replication: peer="{Voter server-8789dc75-b4d8-71dc-fc2d-374466b5bafe 8789dc75-b4d8-71dc-fc2d-374466b5bafe}" +2025-06-15T03:17:52.611Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:51.605151068 +0000 UTC m=+87.316829267 (1 nodes), 527.950675ms from start of poll, 528.314801ms from cluster start. Timeout at 2025-06-15 03:17:52.611184648 +0000 UTC m=+88.322862814, 1.006033547s after stability +2025-06-15T03:17:52.611Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: starting snapshot up to: index=12 +2025-06-15T03:17:52.611Z [INFO] creating new snapshot: path=/tmp/raft4264958844/snapshots/2-12-1749957472611.tmp +2025-06-15T03:17:52.611Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: snapshot complete up to: index=12 +2025-06-15T03:17:52.612Z [INFO] creating new snapshot: path=/tmp/raft4264958844/snapshots/2-10013-1749957472612.tmp +2025-06-15T03:17:52.612Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: copied to local snapshot: bytes=71 +2025-06-15T03:17:52.613Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: snapshot restore progress: id=2-10013-1749957472612 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:52.613Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: removing all old logs from log store +2025-06-15T03:17:52.613Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: compacting logs: from=1 to=22 +2025-06-15T03:17:52.613Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: restored user snapshot: index=10013 +2025-06-15T03:17:52.613Z [ERROR] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: failed to get log: index=22 error="log not found" +2025-06-15T03:17:52.613Z [ERROR] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: failed to get log: index=22 error="log not found" +2025-06-15T03:17:52.613Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: aborting pipeline replication: peer="{Voter server-6959b9af-9baf-17d1-dc7e-7f1d2c5123ea 6959b9af-9baf-17d1-dc7e-7f1d2c5123ea}" +2025-06-15T03:17:52.613Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: aborting pipeline replication: peer="{Voter server-8789dc75-b4d8-71dc-fc2d-374466b5bafe 8789dc75-b4d8-71dc-fc2d-374466b5bafe}" +2025-06-15T03:17:52.621Z [ERROR] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: failed to get log: index=22 error="log not found" +2025-06-15T03:17:52.621Z [ERROR] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: failed to get log: index=22 error="log not found" +2025-06-15T03:17:52.622Z [INFO] creating new snapshot: path=/tmp/raft3579644406/snapshots/2-10013-1749957472622.tmp +2025-06-15T03:17:52.622Z [INFO] creating new snapshot: path=/tmp/raft3917613092/snapshots/2-10013-1749957472622.tmp +2025-06-15T03:17:52.622Z [INFO] server-6959b9af-9baf-17d1-dc7e-7f1d2c5123ea: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:52.622Z [INFO] server-8789dc75-b4d8-71dc-fc2d-374466b5bafe: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:52.622Z [INFO] server-6959b9af-9baf-17d1-dc7e-7f1d2c5123ea: copied to local snapshot: bytes=71 +2025-06-15T03:17:52.622Z [INFO] server-8789dc75-b4d8-71dc-fc2d-374466b5bafe: copied to local snapshot: bytes=71 +2025-06-15T03:17:52.622Z [INFO] server-6959b9af-9baf-17d1-dc7e-7f1d2c5123ea: snapshot restore progress: id=2-10013-1749957472622 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:52.622Z [INFO] server-6959b9af-9baf-17d1-dc7e-7f1d2c5123ea: removing all old logs from log store +2025-06-15T03:17:52.622Z [INFO] server-6959b9af-9baf-17d1-dc7e-7f1d2c5123ea: compacting logs: from=1 to=22 +2025-06-15T03:17:52.622Z [INFO] server-6959b9af-9baf-17d1-dc7e-7f1d2c5123ea: Installed remote snapshot +2025-06-15T03:17:52.622Z [WARN] server-6959b9af-9baf-17d1-dc7e-7f1d2c5123ea: skipping application of old log: index=22 +2025-06-15T03:17:52.622Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: pipelining replication: peer="{Voter server-6959b9af-9baf-17d1-dc7e-7f1d2c5123ea 6959b9af-9baf-17d1-dc7e-7f1d2c5123ea}" +2025-06-15T03:17:52.622Z [INFO] server-8789dc75-b4d8-71dc-fc2d-374466b5bafe: snapshot restore progress: id=2-10013-1749957472622 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:52.622Z [INFO] server-8789dc75-b4d8-71dc-fc2d-374466b5bafe: removing all old logs from log store +2025-06-15T03:17:52.622Z [INFO] server-8789dc75-b4d8-71dc-fc2d-374466b5bafe: compacting logs: from=1 to=22 +2025-06-15T03:17:52.622Z [INFO] server-8789dc75-b4d8-71dc-fc2d-374466b5bafe: Installed remote snapshot +2025-06-15T03:17:52.622Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:17:52.622Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: pipelining replication: peer="{Voter server-8789dc75-b4d8-71dc-fc2d-374466b5bafe 8789dc75-b4d8-71dc-fc2d-374466b5bafe}" +2025-06-15T03:17:53.627Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:17:52.622843921 +0000 UTC m=+88.334522091 (2 nodes), 2.3µs from start of poll, 1.546007625s from cluster start. Timeout at 2025-06-15 03:17:53.627877662 +0000 UTC m=+89.339555861, 1.00503377s after stability +2025-06-15T03:17:53.639Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: aborting pipeline replication: peer="{Voter server-6959b9af-9baf-17d1-dc7e-7f1d2c5123ea 6959b9af-9baf-17d1-dc7e-7f1d2c5123ea}" +2025-06-15T03:17:53.639Z [INFO] server-55e3eaa6-2113-29ea-856c-e82e2f79780d: aborting pipeline replication: peer="{Voter server-8789dc75-b4d8-71dc-fc2d-374466b5bafe 8789dc75-b4d8-71dc-fc2d-374466b5bafe}" === RUN TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true -2024-05-12T20:51:19.373Z [INFO] server-9713aa45-70c3-d529-31a1-b4228e928df5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9713aa45-70c3-d529-31a1-b4228e928df5 Address:9713aa45-70c3-d529-31a1-b4228e928df5} {Suffrage:Voter ID:server-5cf4b816-4239-13bf-7346-920863c048d3 Address:5cf4b816-4239-13bf-7346-920863c048d3} {Suffrage:Voter ID:server-f656e597-e5bb-7101-501c-1bfe3af37792 Address:f656e597-e5bb-7101-501c-1bfe3af37792}]" -2024-05-12T20:51:19.374Z [INFO] server-5cf4b816-4239-13bf-7346-920863c048d3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9713aa45-70c3-d529-31a1-b4228e928df5 Address:9713aa45-70c3-d529-31a1-b4228e928df5} {Suffrage:Voter ID:server-5cf4b816-4239-13bf-7346-920863c048d3 Address:5cf4b816-4239-13bf-7346-920863c048d3} {Suffrage:Voter ID:server-f656e597-e5bb-7101-501c-1bfe3af37792 Address:f656e597-e5bb-7101-501c-1bfe3af37792}]" -2024-05-12T20:51:19.374Z [INFO] server-f656e597-e5bb-7101-501c-1bfe3af37792: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9713aa45-70c3-d529-31a1-b4228e928df5 Address:9713aa45-70c3-d529-31a1-b4228e928df5} {Suffrage:Voter ID:server-5cf4b816-4239-13bf-7346-920863c048d3 Address:5cf4b816-4239-13bf-7346-920863c048d3} {Suffrage:Voter ID:server-f656e597-e5bb-7101-501c-1bfe3af37792 Address:f656e597-e5bb-7101-501c-1bfe3af37792}]" -2024-05-12T20:51:19.374Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:19.374Z [INFO] server-9713aa45-70c3-d529-31a1-b4228e928df5: entering follower state: follower="Node at 9713aa45-70c3-d529-31a1-b4228e928df5 [Follower]" leader-address= leader-id= -2024-05-12T20:51:19.374Z [INFO] server-5cf4b816-4239-13bf-7346-920863c048d3: entering follower state: follower="Node at 5cf4b816-4239-13bf-7346-920863c048d3 [Follower]" leader-address= leader-id= -2024-05-12T20:51:19.374Z [INFO] server-f656e597-e5bb-7101-501c-1bfe3af37792: entering follower state: follower="Node at f656e597-e5bb-7101-501c-1bfe3af37792 [Follower]" leader-address= leader-id= -2024-05-12T20:51:19.887Z [WARN] server-9713aa45-70c3-d529-31a1-b4228e928df5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:19.887Z [INFO] server-9713aa45-70c3-d529-31a1-b4228e928df5: entering candidate state: node="Node at 9713aa45-70c3-d529-31a1-b4228e928df5 [Candidate]" term=2 -2024-05-12T20:51:19.887Z [INFO] server-9713aa45-70c3-d529-31a1-b4228e928df5: election won: term=2 tally=2 -2024-05-12T20:51:19.887Z [INFO] server-9713aa45-70c3-d529-31a1-b4228e928df5: entering leader state: leader="Node at 9713aa45-70c3-d529-31a1-b4228e928df5 [Leader]" -2024-05-12T20:51:19.887Z [INFO] server-9713aa45-70c3-d529-31a1-b4228e928df5: added peer, starting replication: peer=server-5cf4b816-4239-13bf-7346-920863c048d3 -2024-05-12T20:51:19.887Z [INFO] server-9713aa45-70c3-d529-31a1-b4228e928df5: added peer, starting replication: peer=server-f656e597-e5bb-7101-501c-1bfe3af37792 -2024-05-12T20:51:19.887Z [INFO] server-9713aa45-70c3-d529-31a1-b4228e928df5: pipelining replication: peer="{Voter server-f656e597-e5bb-7101-501c-1bfe3af37792 f656e597-e5bb-7101-501c-1bfe3af37792}" -2024-05-12T20:51:19.887Z [INFO] server-9713aa45-70c3-d529-31a1-b4228e928df5: pipelining replication: peer="{Voter server-5cf4b816-4239-13bf-7346-920863c048d3 5cf4b816-4239-13bf-7346-920863c048d3}" -2024-05-12T20:51:20.894Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:19.887700354 +0000 UTC m=+91.168400699 (1 nodes), 513.546558ms from start of poll, 513.974509ms from cluster start. Timeout at 2024-05-12 20:51:20.894779402 +0000 UTC m=+92.175479297, 1.007078598s after stability -2024-05-12T20:51:20.896Z [INFO] server-9713aa45-70c3-d529-31a1-b4228e928df5: starting snapshot up to: index=12 -2024-05-12T20:51:20.896Z [INFO] creating new snapshot: path=/tmp/raft3102396140/snapshots/2-12-1715547080896.tmp -2024-05-12T20:51:20.897Z [INFO] server-9713aa45-70c3-d529-31a1-b4228e928df5: snapshot complete up to: index=12 -2024-05-12T20:51:20.900Z [INFO] server-80c148f3-3bac-49a3-7141-fda82d532594: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-80c148f3-3bac-49a3-7141-fda82d532594 Address:80c148f3-3bac-49a3-7141-fda82d532594} {Suffrage:Voter ID:server-02c6fd16-7f08-41bd-667b-741c6501ff28 Address:02c6fd16-7f08-41bd-667b-741c6501ff28} {Suffrage:Voter ID:server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e Address:4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e}]" -2024-05-12T20:51:20.900Z [INFO] server-02c6fd16-7f08-41bd-667b-741c6501ff28: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-80c148f3-3bac-49a3-7141-fda82d532594 Address:80c148f3-3bac-49a3-7141-fda82d532594} {Suffrage:Voter ID:server-02c6fd16-7f08-41bd-667b-741c6501ff28 Address:02c6fd16-7f08-41bd-667b-741c6501ff28} {Suffrage:Voter ID:server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e Address:4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e}]" -2024-05-12T20:51:20.900Z [INFO] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-80c148f3-3bac-49a3-7141-fda82d532594 Address:80c148f3-3bac-49a3-7141-fda82d532594} {Suffrage:Voter ID:server-02c6fd16-7f08-41bd-667b-741c6501ff28 Address:02c6fd16-7f08-41bd-667b-741c6501ff28} {Suffrage:Voter ID:server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e Address:4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e}]" -2024-05-12T20:51:20.900Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:20.900Z [INFO] server-80c148f3-3bac-49a3-7141-fda82d532594: entering follower state: follower="Node at 80c148f3-3bac-49a3-7141-fda82d532594 [Follower]" leader-address= leader-id= -2024-05-12T20:51:20.900Z [INFO] server-02c6fd16-7f08-41bd-667b-741c6501ff28: entering follower state: follower="Node at 02c6fd16-7f08-41bd-667b-741c6501ff28 [Follower]" leader-address= leader-id= -2024-05-12T20:51:20.900Z [INFO] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: entering follower state: follower="Node at 4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e [Follower]" leader-address= leader-id= -2024-05-12T20:51:21.434Z [WARN] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:21.434Z [INFO] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: entering candidate state: node="Node at 4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e [Candidate]" term=2 -2024-05-12T20:51:21.435Z [INFO] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: election won: term=2 tally=2 -2024-05-12T20:51:21.435Z [INFO] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: entering leader state: leader="Node at 4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e [Leader]" -2024-05-12T20:51:21.435Z [INFO] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: added peer, starting replication: peer=server-80c148f3-3bac-49a3-7141-fda82d532594 -2024-05-12T20:51:21.435Z [INFO] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: added peer, starting replication: peer=server-02c6fd16-7f08-41bd-667b-741c6501ff28 -2024-05-12T20:51:21.435Z [INFO] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: pipelining replication: peer="{Voter server-02c6fd16-7f08-41bd-667b-741c6501ff28 02c6fd16-7f08-41bd-667b-741c6501ff28}" -2024-05-12T20:51:21.435Z [INFO] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: pipelining replication: peer="{Voter server-80c148f3-3bac-49a3-7141-fda82d532594 80c148f3-3bac-49a3-7141-fda82d532594}" -2024-05-12T20:51:22.451Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:21.435710964 +0000 UTC m=+92.716411344 (1 nodes), 535.102604ms from start of poll, 535.508814ms from cluster start. Timeout at 2024-05-12 20:51:22.451037605 +0000 UTC m=+93.731739675, 1.015328331s after stability -2024-05-12T20:51:22.451Z [INFO] creating new snapshot: path=/tmp/raft109764960/snapshots/2-10013-1715547082451.tmp -2024-05-12T20:51:22.452Z [INFO] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: copied to local snapshot: bytes=71 -2024-05-12T20:51:22.453Z [INFO] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: snapshot restore progress: id=2-10013-1715547082451 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:22.453Z [INFO] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: restored user snapshot: index=10013 -2024-05-12T20:51:22.453Z [ERROR] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: failed to get log: index=3 error="log not found" -2024-05-12T20:51:22.453Z [INFO] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: aborting pipeline replication: peer="{Voter server-02c6fd16-7f08-41bd-667b-741c6501ff28 02c6fd16-7f08-41bd-667b-741c6501ff28}" -2024-05-12T20:51:22.454Z [ERROR] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: failed to get log: index=3 error="log not found" -2024-05-12T20:51:22.454Z [INFO] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: aborting pipeline replication: peer="{Voter server-80c148f3-3bac-49a3-7141-fda82d532594 80c148f3-3bac-49a3-7141-fda82d532594}" -2024-05-12T20:51:22.462Z [ERROR] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: failed to get log: index=3 error="log not found" -2024-05-12T20:51:22.463Z [ERROR] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: failed to get log: index=3 error="log not found" -2024-05-12T20:51:22.463Z [INFO] creating new snapshot: path=/tmp/raft1447051167/snapshots/2-10013-1715547082463.tmp -2024-05-12T20:51:22.463Z [INFO] creating new snapshot: path=/tmp/raft177241028/snapshots/2-10013-1715547082463.tmp -2024-05-12T20:51:22.463Z [INFO] server-02c6fd16-7f08-41bd-667b-741c6501ff28: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:22.464Z [INFO] server-80c148f3-3bac-49a3-7141-fda82d532594: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:22.464Z [INFO] server-02c6fd16-7f08-41bd-667b-741c6501ff28: copied to local snapshot: bytes=71 -2024-05-12T20:51:22.464Z [INFO] server-02c6fd16-7f08-41bd-667b-741c6501ff28: snapshot restore progress: id=2-10013-1715547082463 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:22.464Z [INFO] server-02c6fd16-7f08-41bd-667b-741c6501ff28: Installed remote snapshot -2024-05-12T20:51:22.465Z [INFO] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: pipelining replication: peer="{Voter server-02c6fd16-7f08-41bd-667b-741c6501ff28 02c6fd16-7f08-41bd-667b-741c6501ff28}" -2024-05-12T20:51:22.465Z [INFO] server-80c148f3-3bac-49a3-7141-fda82d532594: copied to local snapshot: bytes=71 -2024-05-12T20:51:22.465Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:22.466Z [INFO] server-80c148f3-3bac-49a3-7141-fda82d532594: snapshot restore progress: id=2-10013-1715547082463 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:22.466Z [INFO] server-80c148f3-3bac-49a3-7141-fda82d532594: Installed remote snapshot -2024-05-12T20:51:22.466Z [INFO] server-4ee3a5fc-f75d-dea3-6a60-a1e8d28d224e: pipelining replication: peer="{Voter server-80c148f3-3bac-49a3-7141-fda82d532594 80c148f3-3bac-49a3-7141-fda82d532594}" -2024-05-12T20:51:23.474Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:22.465517367 +0000 UTC m=+93.746217676 (2 nodes), 1.808µs from start of poll, 1.565315146s from cluster start. Timeout at 2024-05-12 20:51:23.474812957 +0000 UTC m=+94.755512852, 1.009295176s after stability -2024-05-12T20:51:23.491Z [INFO] server-9713aa45-70c3-d529-31a1-b4228e928df5: aborting pipeline replication: peer="{Voter server-f656e597-e5bb-7101-501c-1bfe3af37792 f656e597-e5bb-7101-501c-1bfe3af37792}" -2024-05-12T20:51:23.491Z [INFO] server-9713aa45-70c3-d529-31a1-b4228e928df5: aborting pipeline replication: peer="{Voter server-5cf4b816-4239-13bf-7346-920863c048d3 5cf4b816-4239-13bf-7346-920863c048d3}" +2025-06-15T03:17:53.641Z [INFO] server-8c892185-c254-e9f8-562f-3ef8e6040155: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8c892185-c254-e9f8-562f-3ef8e6040155 Address:8c892185-c254-e9f8-562f-3ef8e6040155} {Suffrage:Voter ID:server-cb839c33-dfad-d819-12bf-f23e31a0c844 Address:cb839c33-dfad-d819-12bf-f23e31a0c844} {Suffrage:Voter ID:server-c97aa1e1-ccb3-6358-528f-c9b69a542d3d Address:c97aa1e1-ccb3-6358-528f-c9b69a542d3d}]" +2025-06-15T03:17:53.641Z [INFO] server-8c892185-c254-e9f8-562f-3ef8e6040155: entering follower state: follower="Node at 8c892185-c254-e9f8-562f-3ef8e6040155 [Follower]" leader-address= leader-id= +2025-06-15T03:17:53.641Z [INFO] server-cb839c33-dfad-d819-12bf-f23e31a0c844: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8c892185-c254-e9f8-562f-3ef8e6040155 Address:8c892185-c254-e9f8-562f-3ef8e6040155} {Suffrage:Voter ID:server-cb839c33-dfad-d819-12bf-f23e31a0c844 Address:cb839c33-dfad-d819-12bf-f23e31a0c844} {Suffrage:Voter ID:server-c97aa1e1-ccb3-6358-528f-c9b69a542d3d Address:c97aa1e1-ccb3-6358-528f-c9b69a542d3d}]" +2025-06-15T03:17:53.641Z [INFO] server-cb839c33-dfad-d819-12bf-f23e31a0c844: entering follower state: follower="Node at cb839c33-dfad-d819-12bf-f23e31a0c844 [Follower]" leader-address= leader-id= +2025-06-15T03:17:53.641Z [INFO] server-c97aa1e1-ccb3-6358-528f-c9b69a542d3d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8c892185-c254-e9f8-562f-3ef8e6040155 Address:8c892185-c254-e9f8-562f-3ef8e6040155} {Suffrage:Voter ID:server-cb839c33-dfad-d819-12bf-f23e31a0c844 Address:cb839c33-dfad-d819-12bf-f23e31a0c844} {Suffrage:Voter ID:server-c97aa1e1-ccb3-6358-528f-c9b69a542d3d Address:c97aa1e1-ccb3-6358-528f-c9b69a542d3d}]" +2025-06-15T03:17:53.641Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:53.641Z [INFO] server-c97aa1e1-ccb3-6358-528f-c9b69a542d3d: entering follower state: follower="Node at c97aa1e1-ccb3-6358-528f-c9b69a542d3d [Follower]" leader-address= leader-id= +2025-06-15T03:17:54.260Z [WARN] server-8c892185-c254-e9f8-562f-3ef8e6040155: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:54.260Z [INFO] server-8c892185-c254-e9f8-562f-3ef8e6040155: entering candidate state: node="Node at 8c892185-c254-e9f8-562f-3ef8e6040155 [Candidate]" term=2 +2025-06-15T03:17:54.260Z [INFO] server-8c892185-c254-e9f8-562f-3ef8e6040155: election won: term=2 tally=2 +2025-06-15T03:17:54.260Z [INFO] server-8c892185-c254-e9f8-562f-3ef8e6040155: entering leader state: leader="Node at 8c892185-c254-e9f8-562f-3ef8e6040155 [Leader]" +2025-06-15T03:17:54.260Z [INFO] server-8c892185-c254-e9f8-562f-3ef8e6040155: added peer, starting replication: peer=server-cb839c33-dfad-d819-12bf-f23e31a0c844 +2025-06-15T03:17:54.260Z [INFO] server-8c892185-c254-e9f8-562f-3ef8e6040155: added peer, starting replication: peer=server-c97aa1e1-ccb3-6358-528f-c9b69a542d3d +2025-06-15T03:17:54.260Z [INFO] server-8c892185-c254-e9f8-562f-3ef8e6040155: pipelining replication: peer="{Voter server-cb839c33-dfad-d819-12bf-f23e31a0c844 cb839c33-dfad-d819-12bf-f23e31a0c844}" +2025-06-15T03:17:54.260Z [INFO] server-8c892185-c254-e9f8-562f-3ef8e6040155: pipelining replication: peer="{Voter server-c97aa1e1-ccb3-6358-528f-c9b69a542d3d c97aa1e1-ccb3-6358-528f-c9b69a542d3d}" +2025-06-15T03:17:55.266Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:54.26081754 +0000 UTC m=+89.972495739 (1 nodes), 619.469887ms from start of poll, 619.856601ms from cluster start. Timeout at 2025-06-15 03:17:55.266236746 +0000 UTC m=+90.977914911, 1.005419172s after stability +2025-06-15T03:17:55.266Z [INFO] server-8c892185-c254-e9f8-562f-3ef8e6040155: starting snapshot up to: index=12 +2025-06-15T03:17:55.266Z [INFO] creating new snapshot: path=/tmp/raft985660527/snapshots/2-12-1749957475266.tmp +2025-06-15T03:17:55.266Z [INFO] server-8c892185-c254-e9f8-562f-3ef8e6040155: snapshot complete up to: index=12 +2025-06-15T03:17:55.268Z [INFO] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6a4a1543-70fc-f36a-b09d-46ba885e9e42 Address:6a4a1543-70fc-f36a-b09d-46ba885e9e42} {Suffrage:Voter ID:server-ef6dff12-9a40-7973-4740-6e1431eb2fb0 Address:ef6dff12-9a40-7973-4740-6e1431eb2fb0} {Suffrage:Voter ID:server-cee81054-cb77-3c3c-f3a2-a7ecfe924a6b Address:cee81054-cb77-3c3c-f3a2-a7ecfe924a6b}]" +2025-06-15T03:17:55.268Z [INFO] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: entering follower state: follower="Node at 6a4a1543-70fc-f36a-b09d-46ba885e9e42 [Follower]" leader-address= leader-id= +2025-06-15T03:17:55.268Z [INFO] server-ef6dff12-9a40-7973-4740-6e1431eb2fb0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6a4a1543-70fc-f36a-b09d-46ba885e9e42 Address:6a4a1543-70fc-f36a-b09d-46ba885e9e42} {Suffrage:Voter ID:server-ef6dff12-9a40-7973-4740-6e1431eb2fb0 Address:ef6dff12-9a40-7973-4740-6e1431eb2fb0} {Suffrage:Voter ID:server-cee81054-cb77-3c3c-f3a2-a7ecfe924a6b Address:cee81054-cb77-3c3c-f3a2-a7ecfe924a6b}]" +2025-06-15T03:17:55.268Z [INFO] server-ef6dff12-9a40-7973-4740-6e1431eb2fb0: entering follower state: follower="Node at ef6dff12-9a40-7973-4740-6e1431eb2fb0 [Follower]" leader-address= leader-id= +2025-06-15T03:17:55.268Z [INFO] server-cee81054-cb77-3c3c-f3a2-a7ecfe924a6b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6a4a1543-70fc-f36a-b09d-46ba885e9e42 Address:6a4a1543-70fc-f36a-b09d-46ba885e9e42} {Suffrage:Voter ID:server-ef6dff12-9a40-7973-4740-6e1431eb2fb0 Address:ef6dff12-9a40-7973-4740-6e1431eb2fb0} {Suffrage:Voter ID:server-cee81054-cb77-3c3c-f3a2-a7ecfe924a6b Address:cee81054-cb77-3c3c-f3a2-a7ecfe924a6b}]" +2025-06-15T03:17:55.268Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:55.268Z [INFO] server-cee81054-cb77-3c3c-f3a2-a7ecfe924a6b: entering follower state: follower="Node at cee81054-cb77-3c3c-f3a2-a7ecfe924a6b [Follower]" leader-address= leader-id= +2025-06-15T03:17:55.846Z [WARN] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:55.846Z [INFO] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: entering candidate state: node="Node at 6a4a1543-70fc-f36a-b09d-46ba885e9e42 [Candidate]" term=2 +2025-06-15T03:17:55.846Z [INFO] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: election won: term=2 tally=2 +2025-06-15T03:17:55.846Z [INFO] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: entering leader state: leader="Node at 6a4a1543-70fc-f36a-b09d-46ba885e9e42 [Leader]" +2025-06-15T03:17:55.846Z [INFO] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: added peer, starting replication: peer=server-ef6dff12-9a40-7973-4740-6e1431eb2fb0 +2025-06-15T03:17:55.846Z [INFO] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: added peer, starting replication: peer=server-cee81054-cb77-3c3c-f3a2-a7ecfe924a6b +2025-06-15T03:17:55.847Z [INFO] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: pipelining replication: peer="{Voter server-ef6dff12-9a40-7973-4740-6e1431eb2fb0 ef6dff12-9a40-7973-4740-6e1431eb2fb0}" +2025-06-15T03:17:55.847Z [INFO] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: pipelining replication: peer="{Voter server-cee81054-cb77-3c3c-f3a2-a7ecfe924a6b cee81054-cb77-3c3c-f3a2-a7ecfe924a6b}" +2025-06-15T03:17:56.852Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:55.846997911 +0000 UTC m=+91.558676133 (1 nodes), 578.029892ms from start of poll, 578.789682ms from cluster start. Timeout at 2025-06-15 03:17:56.852835351 +0000 UTC m=+92.564513550, 1.005837417s after stability +2025-06-15T03:17:56.853Z [INFO] creating new snapshot: path=/tmp/raft2685962769/snapshots/2-10013-1749957476853.tmp +2025-06-15T03:17:56.853Z [INFO] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: copied to local snapshot: bytes=71 +2025-06-15T03:17:56.853Z [INFO] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: snapshot restore progress: id=2-10013-1749957476853 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:56.853Z [INFO] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: restored user snapshot: index=10013 +2025-06-15T03:17:56.853Z [ERROR] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: failed to get log: index=3 error="log not found" +2025-06-15T03:17:56.853Z [INFO] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: aborting pipeline replication: peer="{Voter server-cee81054-cb77-3c3c-f3a2-a7ecfe924a6b cee81054-cb77-3c3c-f3a2-a7ecfe924a6b}" +2025-06-15T03:17:56.853Z [ERROR] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: failed to get log: index=3 error="log not found" +2025-06-15T03:17:56.853Z [INFO] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: aborting pipeline replication: peer="{Voter server-ef6dff12-9a40-7973-4740-6e1431eb2fb0 ef6dff12-9a40-7973-4740-6e1431eb2fb0}" +2025-06-15T03:17:56.862Z [ERROR] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: failed to get log: index=3 error="log not found" +2025-06-15T03:17:56.862Z [ERROR] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: failed to get log: index=3 error="log not found" +2025-06-15T03:17:56.862Z [INFO] creating new snapshot: path=/tmp/raft3396453050/snapshots/2-10013-1749957476862.tmp +2025-06-15T03:17:56.862Z [INFO] creating new snapshot: path=/tmp/raft4234597098/snapshots/2-10013-1749957476862.tmp +2025-06-15T03:17:56.862Z [INFO] server-cee81054-cb77-3c3c-f3a2-a7ecfe924a6b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:56.862Z [INFO] server-ef6dff12-9a40-7973-4740-6e1431eb2fb0: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:56.863Z [INFO] server-cee81054-cb77-3c3c-f3a2-a7ecfe924a6b: copied to local snapshot: bytes=71 +2025-06-15T03:17:56.863Z [INFO] server-ef6dff12-9a40-7973-4740-6e1431eb2fb0: copied to local snapshot: bytes=71 +2025-06-15T03:17:56.863Z [INFO] server-cee81054-cb77-3c3c-f3a2-a7ecfe924a6b: snapshot restore progress: id=2-10013-1749957476862 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:56.863Z [INFO] server-cee81054-cb77-3c3c-f3a2-a7ecfe924a6b: Installed remote snapshot +2025-06-15T03:17:56.863Z [INFO] server-ef6dff12-9a40-7973-4740-6e1431eb2fb0: snapshot restore progress: id=2-10013-1749957476862 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:17:56.863Z [INFO] server-ef6dff12-9a40-7973-4740-6e1431eb2fb0: Installed remote snapshot +2025-06-15T03:17:56.863Z [INFO] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: pipelining replication: peer="{Voter server-cee81054-cb77-3c3c-f3a2-a7ecfe924a6b cee81054-cb77-3c3c-f3a2-a7ecfe924a6b}" +2025-06-15T03:17:56.863Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:17:56.863Z [INFO] server-6a4a1543-70fc-f36a-b09d-46ba885e9e42: pipelining replication: peer="{Voter server-ef6dff12-9a40-7973-4740-6e1431eb2fb0 ef6dff12-9a40-7973-4740-6e1431eb2fb0}" +2025-06-15T03:17:57.868Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:17:56.863345911 +0000 UTC m=+92.575024076 (2 nodes), 391ns from start of poll, 1.595137625s from cluster start. Timeout at 2025-06-15 03:17:57.8689327 +0000 UTC m=+93.580610865, 1.005586789s after stability +2025-06-15T03:17:57.879Z [INFO] server-8c892185-c254-e9f8-562f-3ef8e6040155: aborting pipeline replication: peer="{Voter server-c97aa1e1-ccb3-6358-528f-c9b69a542d3d c97aa1e1-ccb3-6358-528f-c9b69a542d3d}" +2025-06-15T03:17:57.879Z [INFO] server-8c892185-c254-e9f8-562f-3ef8e6040155: aborting pipeline replication: peer="{Voter server-cb839c33-dfad-d819-12bf-f23e31a0c844 cb839c33-dfad-d819-12bf-f23e31a0c844}" === RUN TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true -2024-05-12T20:51:23.493Z [INFO] server-b098ba09-e7c5-6734-7eff-8e62b8684424: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b098ba09-e7c5-6734-7eff-8e62b8684424 Address:b098ba09-e7c5-6734-7eff-8e62b8684424} {Suffrage:Voter ID:server-a5e17834-9e79-9c84-28f2-84f73ccd230a Address:a5e17834-9e79-9c84-28f2-84f73ccd230a} {Suffrage:Voter ID:server-f07a6d45-c80d-ab0e-c90b-280a40984b7a Address:f07a6d45-c80d-ab0e-c90b-280a40984b7a}]" -2024-05-12T20:51:23.494Z [INFO] server-a5e17834-9e79-9c84-28f2-84f73ccd230a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b098ba09-e7c5-6734-7eff-8e62b8684424 Address:b098ba09-e7c5-6734-7eff-8e62b8684424} {Suffrage:Voter ID:server-a5e17834-9e79-9c84-28f2-84f73ccd230a Address:a5e17834-9e79-9c84-28f2-84f73ccd230a} {Suffrage:Voter ID:server-f07a6d45-c80d-ab0e-c90b-280a40984b7a Address:f07a6d45-c80d-ab0e-c90b-280a40984b7a}]" -2024-05-12T20:51:23.494Z [INFO] server-f07a6d45-c80d-ab0e-c90b-280a40984b7a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b098ba09-e7c5-6734-7eff-8e62b8684424 Address:b098ba09-e7c5-6734-7eff-8e62b8684424} {Suffrage:Voter ID:server-a5e17834-9e79-9c84-28f2-84f73ccd230a Address:a5e17834-9e79-9c84-28f2-84f73ccd230a} {Suffrage:Voter ID:server-f07a6d45-c80d-ab0e-c90b-280a40984b7a Address:f07a6d45-c80d-ab0e-c90b-280a40984b7a}]" -2024-05-12T20:51:23.494Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:23.494Z [INFO] server-b098ba09-e7c5-6734-7eff-8e62b8684424: entering follower state: follower="Node at b098ba09-e7c5-6734-7eff-8e62b8684424 [Follower]" leader-address= leader-id= -2024-05-12T20:51:23.494Z [INFO] server-a5e17834-9e79-9c84-28f2-84f73ccd230a: entering follower state: follower="Node at a5e17834-9e79-9c84-28f2-84f73ccd230a [Follower]" leader-address= leader-id= -2024-05-12T20:51:23.494Z [INFO] server-f07a6d45-c80d-ab0e-c90b-280a40984b7a: entering follower state: follower="Node at f07a6d45-c80d-ab0e-c90b-280a40984b7a [Follower]" leader-address= leader-id= -2024-05-12T20:51:24.142Z [WARN] server-b098ba09-e7c5-6734-7eff-8e62b8684424: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:24.142Z [INFO] server-b098ba09-e7c5-6734-7eff-8e62b8684424: entering candidate state: node="Node at b098ba09-e7c5-6734-7eff-8e62b8684424 [Candidate]" term=2 -2024-05-12T20:51:24.143Z [INFO] server-b098ba09-e7c5-6734-7eff-8e62b8684424: election won: term=2 tally=2 -2024-05-12T20:51:24.143Z [INFO] server-b098ba09-e7c5-6734-7eff-8e62b8684424: entering leader state: leader="Node at b098ba09-e7c5-6734-7eff-8e62b8684424 [Leader]" -2024-05-12T20:51:24.143Z [INFO] server-b098ba09-e7c5-6734-7eff-8e62b8684424: added peer, starting replication: peer=server-a5e17834-9e79-9c84-28f2-84f73ccd230a -2024-05-12T20:51:24.143Z [INFO] server-b098ba09-e7c5-6734-7eff-8e62b8684424: added peer, starting replication: peer=server-f07a6d45-c80d-ab0e-c90b-280a40984b7a -2024-05-12T20:51:24.143Z [WARN] server-f07a6d45-c80d-ab0e-c90b-280a40984b7a: rejecting vote request since our last term is greater: candidate=b098ba09-e7c5-6734-7eff-8e62b8684424 last-term=2 last-candidate-term=1 -2024-05-12T20:51:24.143Z [INFO] server-b098ba09-e7c5-6734-7eff-8e62b8684424: pipelining replication: peer="{Voter server-f07a6d45-c80d-ab0e-c90b-280a40984b7a f07a6d45-c80d-ab0e-c90b-280a40984b7a}" -2024-05-12T20:51:24.143Z [INFO] server-b098ba09-e7c5-6734-7eff-8e62b8684424: pipelining replication: peer="{Voter server-a5e17834-9e79-9c84-28f2-84f73ccd230a a5e17834-9e79-9c84-28f2-84f73ccd230a}" -2024-05-12T20:51:25.150Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:24.143511832 +0000 UTC m=+95.424211727 (1 nodes), 649.304527ms from start of poll, 649.870897ms from cluster start. Timeout at 2024-05-12 20:51:25.150776405 +0000 UTC m=+96.431476300, 1.007264573s after stability -2024-05-12T20:51:25.152Z [INFO] server-b098ba09-e7c5-6734-7eff-8e62b8684424: starting snapshot up to: index=12 -2024-05-12T20:51:25.152Z [INFO] creating new snapshot: path=/tmp/raft419517906/snapshots/2-12-1715547085152.tmp -2024-05-12T20:51:25.153Z [INFO] server-b098ba09-e7c5-6734-7eff-8e62b8684424: snapshot complete up to: index=12 -2024-05-12T20:51:25.156Z [INFO] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9 Address:7729e4b8-edea-f147-c7ea-d093ef7c0bc9} {Suffrage:Voter ID:server-f8b7b079-e04a-dc96-5823-a3d6e24ae823 Address:f8b7b079-e04a-dc96-5823-a3d6e24ae823} {Suffrage:Voter ID:server-85ecc494-40c3-d99f-3200-a175b5c4053b Address:85ecc494-40c3-d99f-3200-a175b5c4053b}]" -2024-05-12T20:51:25.156Z [INFO] server-f8b7b079-e04a-dc96-5823-a3d6e24ae823: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9 Address:7729e4b8-edea-f147-c7ea-d093ef7c0bc9} {Suffrage:Voter ID:server-f8b7b079-e04a-dc96-5823-a3d6e24ae823 Address:f8b7b079-e04a-dc96-5823-a3d6e24ae823} {Suffrage:Voter ID:server-85ecc494-40c3-d99f-3200-a175b5c4053b Address:85ecc494-40c3-d99f-3200-a175b5c4053b}]" -2024-05-12T20:51:25.157Z [INFO] server-85ecc494-40c3-d99f-3200-a175b5c4053b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9 Address:7729e4b8-edea-f147-c7ea-d093ef7c0bc9} {Suffrage:Voter ID:server-f8b7b079-e04a-dc96-5823-a3d6e24ae823 Address:f8b7b079-e04a-dc96-5823-a3d6e24ae823} {Suffrage:Voter ID:server-85ecc494-40c3-d99f-3200-a175b5c4053b Address:85ecc494-40c3-d99f-3200-a175b5c4053b}]" -2024-05-12T20:51:25.157Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:25.157Z [INFO] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: entering follower state: follower="Node at 7729e4b8-edea-f147-c7ea-d093ef7c0bc9 [Follower]" leader-address= leader-id= -2024-05-12T20:51:25.157Z [INFO] server-f8b7b079-e04a-dc96-5823-a3d6e24ae823: entering follower state: follower="Node at f8b7b079-e04a-dc96-5823-a3d6e24ae823 [Follower]" leader-address= leader-id= -2024-05-12T20:51:25.157Z [INFO] server-85ecc494-40c3-d99f-3200-a175b5c4053b: entering follower state: follower="Node at 85ecc494-40c3-d99f-3200-a175b5c4053b [Follower]" leader-address= leader-id= -2024-05-12T20:51:25.790Z [WARN] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:25.790Z [INFO] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: entering candidate state: node="Node at 7729e4b8-edea-f147-c7ea-d093ef7c0bc9 [Candidate]" term=2 -2024-05-12T20:51:25.790Z [INFO] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: election won: term=2 tally=2 -2024-05-12T20:51:25.790Z [INFO] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: entering leader state: leader="Node at 7729e4b8-edea-f147-c7ea-d093ef7c0bc9 [Leader]" -2024-05-12T20:51:25.790Z [INFO] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: added peer, starting replication: peer=server-f8b7b079-e04a-dc96-5823-a3d6e24ae823 -2024-05-12T20:51:25.791Z [INFO] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: added peer, starting replication: peer=server-85ecc494-40c3-d99f-3200-a175b5c4053b -2024-05-12T20:51:25.791Z [INFO] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: pipelining replication: peer="{Voter server-85ecc494-40c3-d99f-3200-a175b5c4053b 85ecc494-40c3-d99f-3200-a175b5c4053b}" -2024-05-12T20:51:25.791Z [INFO] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: pipelining replication: peer="{Voter server-f8b7b079-e04a-dc96-5823-a3d6e24ae823 f8b7b079-e04a-dc96-5823-a3d6e24ae823}" -2024-05-12T20:51:26.798Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:25.791268607 +0000 UTC m=+97.071968960 (1 nodes), 634.207014ms from start of poll, 634.72521ms from cluster start. Timeout at 2024-05-12 20:51:26.798799905 +0000 UTC m=+98.079499800, 1.00753084s after stability -2024-05-12T20:51:26.799Z [INFO] creating new snapshot: path=/tmp/raft3024238162/snapshots/2-10013-1715547086799.tmp -2024-05-12T20:51:26.800Z [INFO] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: copied to local snapshot: bytes=71 -2024-05-12T20:51:26.800Z [INFO] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: snapshot restore progress: id=2-10013-1715547086799 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:26.800Z [INFO] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: removing all old logs from log store -2024-05-12T20:51:26.800Z [INFO] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: compacting logs: from=1 to=2 -2024-05-12T20:51:26.800Z [INFO] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: restored user snapshot: index=10013 -2024-05-12T20:51:26.800Z [ERROR] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: failed to get log: index=2 error="log not found" -2024-05-12T20:51:26.800Z [INFO] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: aborting pipeline replication: peer="{Voter server-85ecc494-40c3-d99f-3200-a175b5c4053b 85ecc494-40c3-d99f-3200-a175b5c4053b}" -2024-05-12T20:51:26.801Z [ERROR] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: failed to get log: index=2 error="log not found" -2024-05-12T20:51:26.801Z [INFO] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: aborting pipeline replication: peer="{Voter server-f8b7b079-e04a-dc96-5823-a3d6e24ae823 f8b7b079-e04a-dc96-5823-a3d6e24ae823}" -2024-05-12T20:51:26.808Z [ERROR] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: failed to get log: index=2 error="log not found" -2024-05-12T20:51:26.808Z [INFO] creating new snapshot: path=/tmp/raft2720790554/snapshots/2-10013-1715547086808.tmp -2024-05-12T20:51:26.808Z [INFO] server-85ecc494-40c3-d99f-3200-a175b5c4053b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:26.808Z [INFO] server-85ecc494-40c3-d99f-3200-a175b5c4053b: copied to local snapshot: bytes=71 -2024-05-12T20:51:26.809Z [INFO] server-85ecc494-40c3-d99f-3200-a175b5c4053b: snapshot restore progress: id=2-10013-1715547086808 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:26.809Z [INFO] server-85ecc494-40c3-d99f-3200-a175b5c4053b: removing all old logs from log store -2024-05-12T20:51:26.809Z [INFO] server-85ecc494-40c3-d99f-3200-a175b5c4053b: compacting logs: from=1 to=2 -2024-05-12T20:51:26.809Z [INFO] server-85ecc494-40c3-d99f-3200-a175b5c4053b: Installed remote snapshot -2024-05-12T20:51:26.809Z [INFO] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: pipelining replication: peer="{Voter server-85ecc494-40c3-d99f-3200-a175b5c4053b 85ecc494-40c3-d99f-3200-a175b5c4053b}" -2024-05-12T20:51:26.809Z [ERROR] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: failed to get log: index=2 error="log not found" -2024-05-12T20:51:26.809Z [INFO] creating new snapshot: path=/tmp/raft1321825537/snapshots/2-10013-1715547086809.tmp -2024-05-12T20:51:26.810Z [INFO] server-f8b7b079-e04a-dc96-5823-a3d6e24ae823: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:26.810Z [INFO] server-f8b7b079-e04a-dc96-5823-a3d6e24ae823: copied to local snapshot: bytes=71 -2024-05-12T20:51:26.810Z [INFO] server-f8b7b079-e04a-dc96-5823-a3d6e24ae823: snapshot restore progress: id=2-10013-1715547086809 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2024-05-12T20:51:26.810Z [INFO] server-f8b7b079-e04a-dc96-5823-a3d6e24ae823: removing all old logs from log store -2024-05-12T20:51:26.810Z [INFO] server-f8b7b079-e04a-dc96-5823-a3d6e24ae823: compacting logs: from=1 to=2 -2024-05-12T20:51:26.810Z [INFO] server-f8b7b079-e04a-dc96-5823-a3d6e24ae823: Installed remote snapshot -2024-05-12T20:51:26.811Z [INFO] server-7729e4b8-edea-f147-c7ea-d093ef7c0bc9: pipelining replication: peer="{Voter server-f8b7b079-e04a-dc96-5823-a3d6e24ae823 f8b7b079-e04a-dc96-5823-a3d6e24ae823}" -2024-05-12T20:51:26.811Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:27.818Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:26.811090492 +0000 UTC m=+98.091791496 (2 nodes), 2.524µs from start of poll, 1.654547746s from cluster start. Timeout at 2024-05-12 20:51:27.818773596 +0000 UTC m=+99.099474142, 1.007682646s after stability -2024-05-12T20:51:27.835Z [INFO] server-b098ba09-e7c5-6734-7eff-8e62b8684424: aborting pipeline replication: peer="{Voter server-f07a6d45-c80d-ab0e-c90b-280a40984b7a f07a6d45-c80d-ab0e-c90b-280a40984b7a}" -2024-05-12T20:51:27.835Z [INFO] server-b098ba09-e7c5-6734-7eff-8e62b8684424: aborting pipeline replication: peer="{Voter server-a5e17834-9e79-9c84-28f2-84f73ccd230a a5e17834-9e79-9c84-28f2-84f73ccd230a}" ---- PASS: TestRaft_UserRestore (84.42s) - --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false (2.80s) - --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false (2.68s) - --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true (4.11s) - --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true (4.34s) - --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false (2.80s) - --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false (2.88s) - --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true (4.23s) - --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true (4.29s) - --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false (2.71s) - --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false (2.60s) +2025-06-15T03:17:57.880Z [INFO] server-80886c59-d9e1-fd78-7ac7-418ec7eac2ba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-80886c59-d9e1-fd78-7ac7-418ec7eac2ba Address:80886c59-d9e1-fd78-7ac7-418ec7eac2ba} {Suffrage:Voter ID:server-0e928166-a81b-1d1b-bba9-46e15459990f Address:0e928166-a81b-1d1b-bba9-46e15459990f} {Suffrage:Voter ID:server-b574133c-28c3-668b-8b3f-dc15a191862a Address:b574133c-28c3-668b-8b3f-dc15a191862a}]" +2025-06-15T03:17:57.880Z [INFO] server-80886c59-d9e1-fd78-7ac7-418ec7eac2ba: entering follower state: follower="Node at 80886c59-d9e1-fd78-7ac7-418ec7eac2ba [Follower]" leader-address= leader-id= +2025-06-15T03:17:57.880Z [INFO] server-0e928166-a81b-1d1b-bba9-46e15459990f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-80886c59-d9e1-fd78-7ac7-418ec7eac2ba Address:80886c59-d9e1-fd78-7ac7-418ec7eac2ba} {Suffrage:Voter ID:server-0e928166-a81b-1d1b-bba9-46e15459990f Address:0e928166-a81b-1d1b-bba9-46e15459990f} {Suffrage:Voter ID:server-b574133c-28c3-668b-8b3f-dc15a191862a Address:b574133c-28c3-668b-8b3f-dc15a191862a}]" +2025-06-15T03:17:57.880Z [INFO] server-0e928166-a81b-1d1b-bba9-46e15459990f: entering follower state: follower="Node at 0e928166-a81b-1d1b-bba9-46e15459990f [Follower]" leader-address= leader-id= +2025-06-15T03:17:57.880Z [INFO] server-b574133c-28c3-668b-8b3f-dc15a191862a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-80886c59-d9e1-fd78-7ac7-418ec7eac2ba Address:80886c59-d9e1-fd78-7ac7-418ec7eac2ba} {Suffrage:Voter ID:server-0e928166-a81b-1d1b-bba9-46e15459990f Address:0e928166-a81b-1d1b-bba9-46e15459990f} {Suffrage:Voter ID:server-b574133c-28c3-668b-8b3f-dc15a191862a Address:b574133c-28c3-668b-8b3f-dc15a191862a}]" +2025-06-15T03:17:57.880Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:57.880Z [INFO] server-b574133c-28c3-668b-8b3f-dc15a191862a: entering follower state: follower="Node at b574133c-28c3-668b-8b3f-dc15a191862a [Follower]" leader-address= leader-id= +2025-06-15T03:17:58.543Z [WARN] server-b574133c-28c3-668b-8b3f-dc15a191862a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:17:58.544Z [INFO] server-b574133c-28c3-668b-8b3f-dc15a191862a: entering candidate state: node="Node at b574133c-28c3-668b-8b3f-dc15a191862a [Candidate]" term=2 +2025-06-15T03:17:58.544Z [INFO] server-b574133c-28c3-668b-8b3f-dc15a191862a: election won: term=2 tally=2 +2025-06-15T03:17:58.544Z [INFO] server-b574133c-28c3-668b-8b3f-dc15a191862a: entering leader state: leader="Node at b574133c-28c3-668b-8b3f-dc15a191862a [Leader]" +2025-06-15T03:17:58.544Z [INFO] server-b574133c-28c3-668b-8b3f-dc15a191862a: added peer, starting replication: peer=server-80886c59-d9e1-fd78-7ac7-418ec7eac2ba +2025-06-15T03:17:58.544Z [INFO] server-b574133c-28c3-668b-8b3f-dc15a191862a: added peer, starting replication: peer=server-0e928166-a81b-1d1b-bba9-46e15459990f +2025-06-15T03:17:58.544Z [INFO] server-b574133c-28c3-668b-8b3f-dc15a191862a: pipelining replication: peer="{Voter server-80886c59-d9e1-fd78-7ac7-418ec7eac2ba 80886c59-d9e1-fd78-7ac7-418ec7eac2ba}" +2025-06-15T03:17:58.544Z [INFO] server-b574133c-28c3-668b-8b3f-dc15a191862a: pipelining replication: peer="{Voter server-0e928166-a81b-1d1b-bba9-46e15459990f 0e928166-a81b-1d1b-bba9-46e15459990f}" +2025-06-15T03:17:59.549Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:17:58.544123447 +0000 UTC m=+94.255801617 (1 nodes), 663.238179ms from start of poll, 663.500916ms from cluster start. Timeout at 2025-06-15 03:17:59.549810282 +0000 UTC m=+95.261488447, 1.00568683s after stability +2025-06-15T03:17:59.550Z [INFO] server-b574133c-28c3-668b-8b3f-dc15a191862a: starting snapshot up to: index=12 +2025-06-15T03:17:59.550Z [INFO] creating new snapshot: path=/tmp/raft278503279/snapshots/2-12-1749957479550.tmp +2025-06-15T03:17:59.552Z [INFO] server-b574133c-28c3-668b-8b3f-dc15a191862a: snapshot complete up to: index=12 +2025-06-15T03:17:59.553Z [INFO] server-a0fe6889-8194-c141-5efa-e9e09f2a8185: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0fe6889-8194-c141-5efa-e9e09f2a8185 Address:a0fe6889-8194-c141-5efa-e9e09f2a8185} {Suffrage:Voter ID:server-6dabeffb-20f9-0b46-a2e6-637446fdef26 Address:6dabeffb-20f9-0b46-a2e6-637446fdef26} {Suffrage:Voter ID:server-f5d80c67-08b1-8535-99b5-249e91a024ae Address:f5d80c67-08b1-8535-99b5-249e91a024ae}]" +2025-06-15T03:17:59.554Z [INFO] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0fe6889-8194-c141-5efa-e9e09f2a8185 Address:a0fe6889-8194-c141-5efa-e9e09f2a8185} {Suffrage:Voter ID:server-6dabeffb-20f9-0b46-a2e6-637446fdef26 Address:6dabeffb-20f9-0b46-a2e6-637446fdef26} {Suffrage:Voter ID:server-f5d80c67-08b1-8535-99b5-249e91a024ae Address:f5d80c67-08b1-8535-99b5-249e91a024ae}]" +2025-06-15T03:17:59.554Z [INFO] server-a0fe6889-8194-c141-5efa-e9e09f2a8185: entering follower state: follower="Node at a0fe6889-8194-c141-5efa-e9e09f2a8185 [Follower]" leader-address= leader-id= +2025-06-15T03:17:59.554Z [INFO] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: entering follower state: follower="Node at 6dabeffb-20f9-0b46-a2e6-637446fdef26 [Follower]" leader-address= leader-id= +2025-06-15T03:17:59.554Z [INFO] server-f5d80c67-08b1-8535-99b5-249e91a024ae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0fe6889-8194-c141-5efa-e9e09f2a8185 Address:a0fe6889-8194-c141-5efa-e9e09f2a8185} {Suffrage:Voter ID:server-6dabeffb-20f9-0b46-a2e6-637446fdef26 Address:6dabeffb-20f9-0b46-a2e6-637446fdef26} {Suffrage:Voter ID:server-f5d80c67-08b1-8535-99b5-249e91a024ae Address:f5d80c67-08b1-8535-99b5-249e91a024ae}]" +2025-06-15T03:17:59.554Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:17:59.554Z [INFO] server-f5d80c67-08b1-8535-99b5-249e91a024ae: entering follower state: follower="Node at f5d80c67-08b1-8535-99b5-249e91a024ae [Follower]" leader-address= leader-id= +2025-06-15T03:18:00.177Z [WARN] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:00.177Z [INFO] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: entering candidate state: node="Node at 6dabeffb-20f9-0b46-a2e6-637446fdef26 [Candidate]" term=2 +2025-06-15T03:18:00.177Z [INFO] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: election won: term=2 tally=2 +2025-06-15T03:18:00.177Z [INFO] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: entering leader state: leader="Node at 6dabeffb-20f9-0b46-a2e6-637446fdef26 [Leader]" +2025-06-15T03:18:00.177Z [INFO] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: added peer, starting replication: peer=server-a0fe6889-8194-c141-5efa-e9e09f2a8185 +2025-06-15T03:18:00.177Z [INFO] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: added peer, starting replication: peer=server-f5d80c67-08b1-8535-99b5-249e91a024ae +2025-06-15T03:18:00.177Z [INFO] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: pipelining replication: peer="{Voter server-a0fe6889-8194-c141-5efa-e9e09f2a8185 a0fe6889-8194-c141-5efa-e9e09f2a8185}" +2025-06-15T03:18:00.177Z [INFO] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: pipelining replication: peer="{Voter server-f5d80c67-08b1-8535-99b5-249e91a024ae f5d80c67-08b1-8535-99b5-249e91a024ae}" +2025-06-15T03:18:01.183Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:00.17727027 +0000 UTC m=+95.888948473 (1 nodes), 622.922127ms from start of poll, 623.856207ms from cluster start. Timeout at 2025-06-15 03:18:01.183300265 +0000 UTC m=+96.894978498, 1.006030025s after stability +2025-06-15T03:18:01.183Z [INFO] creating new snapshot: path=/tmp/raft1268676630/snapshots/2-10013-1749957481183.tmp +2025-06-15T03:18:01.183Z [INFO] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: copied to local snapshot: bytes=71 +2025-06-15T03:18:01.184Z [INFO] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: snapshot restore progress: id=2-10013-1749957481183 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:18:01.184Z [INFO] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: removing all old logs from log store +2025-06-15T03:18:01.184Z [INFO] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: compacting logs: from=1 to=2 +2025-06-15T03:18:01.184Z [INFO] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: restored user snapshot: index=10013 +2025-06-15T03:18:01.184Z [ERROR] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: failed to get log: index=2 error="log not found" +2025-06-15T03:18:01.184Z [INFO] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: aborting pipeline replication: peer="{Voter server-f5d80c67-08b1-8535-99b5-249e91a024ae f5d80c67-08b1-8535-99b5-249e91a024ae}" +2025-06-15T03:18:01.184Z [ERROR] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: failed to get log: index=2 error="log not found" +2025-06-15T03:18:01.184Z [INFO] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: aborting pipeline replication: peer="{Voter server-a0fe6889-8194-c141-5efa-e9e09f2a8185 a0fe6889-8194-c141-5efa-e9e09f2a8185}" +2025-06-15T03:18:01.189Z [ERROR] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: failed to get log: index=2 error="log not found" +2025-06-15T03:18:01.189Z [INFO] creating new snapshot: path=/tmp/raft1216542698/snapshots/2-10013-1749957481189.tmp +2025-06-15T03:18:01.189Z [INFO] server-f5d80c67-08b1-8535-99b5-249e91a024ae: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:18:01.190Z [INFO] server-f5d80c67-08b1-8535-99b5-249e91a024ae: copied to local snapshot: bytes=71 +2025-06-15T03:18:01.190Z [INFO] server-f5d80c67-08b1-8535-99b5-249e91a024ae: snapshot restore progress: id=2-10013-1749957481189 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:18:01.190Z [INFO] server-f5d80c67-08b1-8535-99b5-249e91a024ae: removing all old logs from log store +2025-06-15T03:18:01.190Z [INFO] server-f5d80c67-08b1-8535-99b5-249e91a024ae: compacting logs: from=1 to=2 +2025-06-15T03:18:01.190Z [INFO] server-f5d80c67-08b1-8535-99b5-249e91a024ae: Installed remote snapshot +2025-06-15T03:18:01.190Z [INFO] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: pipelining replication: peer="{Voter server-f5d80c67-08b1-8535-99b5-249e91a024ae f5d80c67-08b1-8535-99b5-249e91a024ae}" +2025-06-15T03:18:01.190Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:01.194Z [ERROR] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: failed to get log: index=2 error="log not found" +2025-06-15T03:18:01.194Z [INFO] creating new snapshot: path=/tmp/raft4279914336/snapshots/2-10013-1749957481194.tmp +2025-06-15T03:18:01.194Z [INFO] server-a0fe6889-8194-c141-5efa-e9e09f2a8185: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-06-15T03:18:01.194Z [INFO] server-a0fe6889-8194-c141-5efa-e9e09f2a8185: copied to local snapshot: bytes=71 +2025-06-15T03:18:01.194Z [INFO] server-a0fe6889-8194-c141-5efa-e9e09f2a8185: snapshot restore progress: id=2-10013-1749957481194 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-06-15T03:18:01.194Z [INFO] server-a0fe6889-8194-c141-5efa-e9e09f2a8185: removing all old logs from log store +2025-06-15T03:18:01.194Z [INFO] server-a0fe6889-8194-c141-5efa-e9e09f2a8185: compacting logs: from=1 to=2 +2025-06-15T03:18:01.194Z [INFO] server-a0fe6889-8194-c141-5efa-e9e09f2a8185: Installed remote snapshot +2025-06-15T03:18:01.195Z [INFO] server-6dabeffb-20f9-0b46-a2e6-637446fdef26: pipelining replication: peer="{Voter server-a0fe6889-8194-c141-5efa-e9e09f2a8185 a0fe6889-8194-c141-5efa-e9e09f2a8185}" +2025-06-15T03:18:02.196Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:01.190405538 +0000 UTC m=+96.902083703 (2 nodes), 504ns from start of poll, 1.636991437s from cluster start. Timeout at 2025-06-15 03:18:02.196354325 +0000 UTC m=+97.908032491, 1.005948788s after stability +2025-06-15T03:18:02.208Z [INFO] server-b574133c-28c3-668b-8b3f-dc15a191862a: aborting pipeline replication: peer="{Voter server-0e928166-a81b-1d1b-bba9-46e15459990f 0e928166-a81b-1d1b-bba9-46e15459990f}" +2025-06-15T03:18:02.208Z [INFO] server-b574133c-28c3-668b-8b3f-dc15a191862a: aborting pipeline replication: peer="{Voter server-80886c59-d9e1-fd78-7ac7-418ec7eac2ba 80886c59-d9e1-fd78-7ac7-418ec7eac2ba}" +--- PASS: TestRaft_UserRestore (84.74s) + --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false (2.89s) + --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false (2.95s) + --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true (4.41s) + --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true (4.35s) + --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false (2.93s) + --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false (2.94s) + --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true (4.20s) + --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true (4.23s) + --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false (2.59s) + --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false (2.65s) --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true (4.17s) - --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true (4.47s) - --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false (2.60s) - --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false (2.85s) - --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true (4.21s) - --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true (4.33s) - --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false (2.56s) - --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false (2.76s) - --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true (4.32s) - --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true (4.50s) - --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false (2.98s) - --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false (2.76s) - --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true (4.12s) - --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true (4.34s) + --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true (4.28s) + --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false (2.78s) + --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false (2.66s) + --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true (4.15s) + --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true (4.26s) + --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false (2.70s) + --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false (2.69s) + --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true (4.47s) + --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true (4.49s) + --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false (2.81s) + --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false (2.56s) + --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true (4.24s) + --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true (4.33s) === RUN TestRaft_SendSnapshotFollower -2024-05-12T20:51:27.837Z [INFO] server-789a8717-2fdb-c83f-4edb-084f480d8fb5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-789a8717-2fdb-c83f-4edb-084f480d8fb5 Address:789a8717-2fdb-c83f-4edb-084f480d8fb5} {Suffrage:Voter ID:server-63c3fc8d-1885-591f-4c38-5afd36beccb9 Address:63c3fc8d-1885-591f-4c38-5afd36beccb9} {Suffrage:Voter ID:server-cca70723-6e40-9454-0661-fc1f1126e7fb Address:cca70723-6e40-9454-0661-fc1f1126e7fb}]" -2024-05-12T20:51:27.837Z [INFO] server-63c3fc8d-1885-591f-4c38-5afd36beccb9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-789a8717-2fdb-c83f-4edb-084f480d8fb5 Address:789a8717-2fdb-c83f-4edb-084f480d8fb5} {Suffrage:Voter ID:server-63c3fc8d-1885-591f-4c38-5afd36beccb9 Address:63c3fc8d-1885-591f-4c38-5afd36beccb9} {Suffrage:Voter ID:server-cca70723-6e40-9454-0661-fc1f1126e7fb Address:cca70723-6e40-9454-0661-fc1f1126e7fb}]" -2024-05-12T20:51:27.837Z [INFO] server-cca70723-6e40-9454-0661-fc1f1126e7fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-789a8717-2fdb-c83f-4edb-084f480d8fb5 Address:789a8717-2fdb-c83f-4edb-084f480d8fb5} {Suffrage:Voter ID:server-63c3fc8d-1885-591f-4c38-5afd36beccb9 Address:63c3fc8d-1885-591f-4c38-5afd36beccb9} {Suffrage:Voter ID:server-cca70723-6e40-9454-0661-fc1f1126e7fb Address:cca70723-6e40-9454-0661-fc1f1126e7fb}]" -2024-05-12T20:51:27.837Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:27.837Z [INFO] server-789a8717-2fdb-c83f-4edb-084f480d8fb5: entering follower state: follower="Node at 789a8717-2fdb-c83f-4edb-084f480d8fb5 [Follower]" leader-address= leader-id= -2024-05-12T20:51:27.837Z [INFO] server-63c3fc8d-1885-591f-4c38-5afd36beccb9: entering follower state: follower="Node at 63c3fc8d-1885-591f-4c38-5afd36beccb9 [Follower]" leader-address= leader-id= -2024-05-12T20:51:27.837Z [INFO] server-cca70723-6e40-9454-0661-fc1f1126e7fb: entering follower state: follower="Node at cca70723-6e40-9454-0661-fc1f1126e7fb [Follower]" leader-address= leader-id= -2024-05-12T20:51:27.894Z [WARN] server-cca70723-6e40-9454-0661-fc1f1126e7fb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:27.894Z [INFO] server-cca70723-6e40-9454-0661-fc1f1126e7fb: entering candidate state: node="Node at cca70723-6e40-9454-0661-fc1f1126e7fb [Candidate]" term=2 -2024-05-12T20:51:27.895Z [INFO] server-cca70723-6e40-9454-0661-fc1f1126e7fb: election won: term=2 tally=2 -2024-05-12T20:51:27.895Z [INFO] server-cca70723-6e40-9454-0661-fc1f1126e7fb: entering leader state: leader="Node at cca70723-6e40-9454-0661-fc1f1126e7fb [Leader]" -2024-05-12T20:51:27.895Z [INFO] server-cca70723-6e40-9454-0661-fc1f1126e7fb: added peer, starting replication: peer=server-789a8717-2fdb-c83f-4edb-084f480d8fb5 -2024-05-12T20:51:27.895Z [INFO] server-cca70723-6e40-9454-0661-fc1f1126e7fb: added peer, starting replication: peer=server-63c3fc8d-1885-591f-4c38-5afd36beccb9 -2024-05-12T20:51:27.895Z [INFO] server-cca70723-6e40-9454-0661-fc1f1126e7fb: pipelining replication: peer="{Voter server-63c3fc8d-1885-591f-4c38-5afd36beccb9 63c3fc8d-1885-591f-4c38-5afd36beccb9}" -2024-05-12T20:51:27.895Z [WARN] server-789a8717-2fdb-c83f-4edb-084f480d8fb5: rejecting vote request since our last term is greater: candidate=cca70723-6e40-9454-0661-fc1f1126e7fb last-term=2 last-candidate-term=1 -2024-05-12T20:51:27.895Z [INFO] server-cca70723-6e40-9454-0661-fc1f1126e7fb: pipelining replication: peer="{Voter server-789a8717-2fdb-c83f-4edb-084f480d8fb5 789a8717-2fdb-c83f-4edb-084f480d8fb5}" -2024-05-12T20:51:28.007Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:27.895581213 +0000 UTC m=+99.176281776 (2 nodes), 57.823714ms from start of poll, 58.21051ms from cluster start. Timeout at 2024-05-12 20:51:28.007580636 +0000 UTC m=+99.288281144, 111.999368ms after stability -2024-05-12T20:51:28.007Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:28.119Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:28.007712264 +0000 UTC m=+99.288412614 (1 nodes), 3.81µs from start of poll, 170.341348ms from cluster start. Timeout at 2024-05-12 20:51:28.119378843 +0000 UTC m=+99.400079561, 111.666947ms after stability -2024-05-12T20:51:28.119Z [ERROR] server-cca70723-6e40-9454-0661-fc1f1126e7fb: failed to pipeline appendEntries: peer="{Voter server-789a8717-2fdb-c83f-4edb-084f480d8fb5 789a8717-2fdb-c83f-4edb-084f480d8fb5}" error="append pipeline closed" -2024-05-12T20:51:28.119Z [INFO] server-cca70723-6e40-9454-0661-fc1f1126e7fb: aborting pipeline replication: peer="{Voter server-789a8717-2fdb-c83f-4edb-084f480d8fb5 789a8717-2fdb-c83f-4edb-084f480d8fb5}" -2024-05-12T20:51:28.119Z [ERROR] server-cca70723-6e40-9454-0661-fc1f1126e7fb: failed to appendEntries to: peer="{Voter server-789a8717-2fdb-c83f-4edb-084f480d8fb5 789a8717-2fdb-c83f-4edb-084f480d8fb5}" error="failed to connect to peer: 789a8717-2fdb-c83f-4edb-084f480d8fb5" -2024-05-12T20:51:28.123Z [ERROR] server-cca70723-6e40-9454-0661-fc1f1126e7fb: failed to heartbeat to: peer=789a8717-2fdb-c83f-4edb-084f480d8fb5 backoff time=10ms error="failed to connect to peer: 789a8717-2fdb-c83f-4edb-084f480d8fb5" +2025-06-15T03:18:02.209Z [INFO] server-4028b91f-7b62-e212-190d-1cd1a491a667: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4028b91f-7b62-e212-190d-1cd1a491a667 Address:4028b91f-7b62-e212-190d-1cd1a491a667} {Suffrage:Voter ID:server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a Address:742e1ae8-9b5a-6736-a6e1-ff07013c7d0a} {Suffrage:Voter ID:server-b3803b51-81b8-4140-e0af-2a644323abf6 Address:b3803b51-81b8-4140-e0af-2a644323abf6}]" +2025-06-15T03:18:02.209Z [INFO] server-4028b91f-7b62-e212-190d-1cd1a491a667: entering follower state: follower="Node at 4028b91f-7b62-e212-190d-1cd1a491a667 [Follower]" leader-address= leader-id= +2025-06-15T03:18:02.209Z [INFO] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4028b91f-7b62-e212-190d-1cd1a491a667 Address:4028b91f-7b62-e212-190d-1cd1a491a667} {Suffrage:Voter ID:server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a Address:742e1ae8-9b5a-6736-a6e1-ff07013c7d0a} {Suffrage:Voter ID:server-b3803b51-81b8-4140-e0af-2a644323abf6 Address:b3803b51-81b8-4140-e0af-2a644323abf6}]" +2025-06-15T03:18:02.209Z [INFO] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: entering follower state: follower="Node at 742e1ae8-9b5a-6736-a6e1-ff07013c7d0a [Follower]" leader-address= leader-id= +2025-06-15T03:18:02.209Z [INFO] server-b3803b51-81b8-4140-e0af-2a644323abf6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4028b91f-7b62-e212-190d-1cd1a491a667 Address:4028b91f-7b62-e212-190d-1cd1a491a667} {Suffrage:Voter ID:server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a Address:742e1ae8-9b5a-6736-a6e1-ff07013c7d0a} {Suffrage:Voter ID:server-b3803b51-81b8-4140-e0af-2a644323abf6 Address:b3803b51-81b8-4140-e0af-2a644323abf6}]" +2025-06-15T03:18:02.209Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:02.209Z [INFO] server-b3803b51-81b8-4140-e0af-2a644323abf6: entering follower state: follower="Node at b3803b51-81b8-4140-e0af-2a644323abf6 [Follower]" leader-address= leader-id= +2025-06-15T03:18:02.283Z [WARN] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:02.283Z [INFO] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: entering candidate state: node="Node at 742e1ae8-9b5a-6736-a6e1-ff07013c7d0a [Candidate]" term=2 +2025-06-15T03:18:02.283Z [INFO] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: election won: term=2 tally=2 +2025-06-15T03:18:02.283Z [INFO] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: entering leader state: leader="Node at 742e1ae8-9b5a-6736-a6e1-ff07013c7d0a [Leader]" +2025-06-15T03:18:02.283Z [INFO] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: added peer, starting replication: peer=server-4028b91f-7b62-e212-190d-1cd1a491a667 +2025-06-15T03:18:02.283Z [INFO] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: added peer, starting replication: peer=server-b3803b51-81b8-4140-e0af-2a644323abf6 +2025-06-15T03:18:02.283Z [INFO] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: pipelining replication: peer="{Voter server-4028b91f-7b62-e212-190d-1cd1a491a667 4028b91f-7b62-e212-190d-1cd1a491a667}" +2025-06-15T03:18:02.283Z [INFO] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: pipelining replication: peer="{Voter server-b3803b51-81b8-4140-e0af-2a644323abf6 b3803b51-81b8-4140-e0af-2a644323abf6}" +2025-06-15T03:18:02.389Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:02.283912611 +0000 UTC m=+97.995590772 (2 nodes), 74.201311ms from start of poll, 74.483996ms from cluster start. Timeout at 2025-06-15 03:18:02.389638489 +0000 UTC m=+98.101316688, 105.725916ms after stability +2025-06-15T03:18:02.389Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:02.494Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:02.38972556 +0000 UTC m=+98.101403760 (1 nodes), 1.567µs from start of poll, 180.296984ms from cluster start. Timeout at 2025-06-15 03:18:02.494864311 +0000 UTC m=+98.206542511, 105.138751ms after stability +2025-06-15T03:18:02.495Z [ERROR] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: failed to pipeline appendEntries: peer="{Voter server-4028b91f-7b62-e212-190d-1cd1a491a667 4028b91f-7b62-e212-190d-1cd1a491a667}" error="append pipeline closed" +2025-06-15T03:18:02.495Z [INFO] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: aborting pipeline replication: peer="{Voter server-4028b91f-7b62-e212-190d-1cd1a491a667 4028b91f-7b62-e212-190d-1cd1a491a667}" +2025-06-15T03:18:02.495Z [ERROR] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: failed to appendEntries to: peer="{Voter server-4028b91f-7b62-e212-190d-1cd1a491a667 4028b91f-7b62-e212-190d-1cd1a491a667}" error="failed to connect to peer: 4028b91f-7b62-e212-190d-1cd1a491a667" raft_test.go:1627: [INFO] Finished apply without behind follower -2024-05-12T20:51:28.126Z [ERROR] server-789a8717-2fdb-c83f-4edb-084f480d8fb5: failed to take snapshot: error="nothing new to snapshot" -2024-05-12T20:51:28.126Z [INFO] server-63c3fc8d-1885-591f-4c38-5afd36beccb9: starting snapshot up to: index=98 -2024-05-12T20:51:28.126Z [INFO] creating new snapshot: path=/tmp/raft102637006/snapshots/2-98-1715547088126.tmp -2024-05-12T20:51:28.128Z [INFO] server-63c3fc8d-1885-591f-4c38-5afd36beccb9: compacting logs: from=1 to=92 -2024-05-12T20:51:28.128Z [INFO] server-63c3fc8d-1885-591f-4c38-5afd36beccb9: snapshot complete up to: index=98 -2024-05-12T20:51:28.128Z [INFO] server-cca70723-6e40-9454-0661-fc1f1126e7fb: starting snapshot up to: index=102 -2024-05-12T20:51:28.128Z [INFO] creating new snapshot: path=/tmp/raft3202578273/snapshots/2-102-1715547088128.tmp -2024-05-12T20:51:28.129Z [INFO] server-cca70723-6e40-9454-0661-fc1f1126e7fb: compacting logs: from=1 to=92 -2024-05-12T20:51:28.129Z [INFO] server-cca70723-6e40-9454-0661-fc1f1126e7fb: snapshot complete up to: index=102 -2024-05-12T20:51:28.135Z [ERROR] server-cca70723-6e40-9454-0661-fc1f1126e7fb: failed to get log: index=2 error="log not found" -2024-05-12T20:51:28.135Z [INFO] creating new snapshot: path=/tmp/raft140845045/snapshots/2-102-1715547088135.tmp -2024-05-12T20:51:28.135Z [INFO] server-789a8717-2fdb-c83f-4edb-084f480d8fb5: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% -2024-05-12T20:51:28.136Z [INFO] server-789a8717-2fdb-c83f-4edb-084f480d8fb5: copied to local snapshot: bytes=693 -2024-05-12T20:51:28.136Z [INFO] server-789a8717-2fdb-c83f-4edb-084f480d8fb5: snapshot restore progress: id=2-102-1715547088135 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2024-05-12T20:51:28.136Z [INFO] server-789a8717-2fdb-c83f-4edb-084f480d8fb5: Installed remote snapshot -2024-05-12T20:51:28.136Z [WARN] server-789a8717-2fdb-c83f-4edb-084f480d8fb5: skipping application of old log: index=102 -2024-05-12T20:51:28.136Z [INFO] server-cca70723-6e40-9454-0661-fc1f1126e7fb: pipelining replication: peer="{Voter server-789a8717-2fdb-c83f-4edb-084f480d8fb5 789a8717-2fdb-c83f-4edb-084f480d8fb5}" -2024-05-12T20:51:28.151Z [INFO] server-cca70723-6e40-9454-0661-fc1f1126e7fb: aborting pipeline replication: peer="{Voter server-63c3fc8d-1885-591f-4c38-5afd36beccb9 63c3fc8d-1885-591f-4c38-5afd36beccb9}" -2024-05-12T20:51:28.151Z [INFO] server-cca70723-6e40-9454-0661-fc1f1126e7fb: aborting pipeline replication: peer="{Voter server-789a8717-2fdb-c83f-4edb-084f480d8fb5 789a8717-2fdb-c83f-4edb-084f480d8fb5}" ---- PASS: TestRaft_SendSnapshotFollower (0.32s) +2025-06-15T03:18:02.496Z [ERROR] server-4028b91f-7b62-e212-190d-1cd1a491a667: failed to take snapshot: error="nothing new to snapshot" +2025-06-15T03:18:02.496Z [INFO] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: starting snapshot up to: index=102 +2025-06-15T03:18:02.496Z [INFO] creating new snapshot: path=/tmp/raft3679793988/snapshots/2-102-1749957482496.tmp +2025-06-15T03:18:02.497Z [INFO] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: compacting logs: from=1 to=92 +2025-06-15T03:18:02.497Z [INFO] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: snapshot complete up to: index=102 +2025-06-15T03:18:02.497Z [INFO] server-b3803b51-81b8-4140-e0af-2a644323abf6: starting snapshot up to: index=98 +2025-06-15T03:18:02.497Z [INFO] creating new snapshot: path=/tmp/raft4046262661/snapshots/2-98-1749957482497.tmp +2025-06-15T03:18:02.497Z [INFO] server-b3803b51-81b8-4140-e0af-2a644323abf6: compacting logs: from=1 to=92 +2025-06-15T03:18:02.497Z [INFO] server-b3803b51-81b8-4140-e0af-2a644323abf6: snapshot complete up to: index=98 +2025-06-15T03:18:02.505Z [ERROR] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: failed to get log: index=2 error="log not found" +2025-06-15T03:18:02.505Z [INFO] creating new snapshot: path=/tmp/raft3102278280/snapshots/2-102-1749957482505.tmp +2025-06-15T03:18:02.506Z [INFO] server-4028b91f-7b62-e212-190d-1cd1a491a667: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% +2025-06-15T03:18:02.506Z [INFO] server-4028b91f-7b62-e212-190d-1cd1a491a667: copied to local snapshot: bytes=693 +2025-06-15T03:18:02.506Z [INFO] server-4028b91f-7b62-e212-190d-1cd1a491a667: snapshot restore progress: id=2-102-1749957482505 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-06-15T03:18:02.506Z [INFO] server-4028b91f-7b62-e212-190d-1cd1a491a667: Installed remote snapshot +2025-06-15T03:18:02.506Z [WARN] server-4028b91f-7b62-e212-190d-1cd1a491a667: skipping application of old log: index=102 +2025-06-15T03:18:02.506Z [INFO] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: pipelining replication: peer="{Voter server-4028b91f-7b62-e212-190d-1cd1a491a667 4028b91f-7b62-e212-190d-1cd1a491a667}" +2025-06-15T03:18:02.508Z [INFO] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: aborting pipeline replication: peer="{Voter server-b3803b51-81b8-4140-e0af-2a644323abf6 b3803b51-81b8-4140-e0af-2a644323abf6}" +2025-06-15T03:18:02.508Z [INFO] server-742e1ae8-9b5a-6736-a6e1-ff07013c7d0a: aborting pipeline replication: peer="{Voter server-4028b91f-7b62-e212-190d-1cd1a491a667 4028b91f-7b62-e212-190d-1cd1a491a667}" +--- PASS: TestRaft_SendSnapshotFollower (0.30s) === RUN TestRaft_SendSnapshotAndLogsFollower -2024-05-12T20:51:28.153Z [INFO] server-80959482-d8ba-fbfa-fe27-b41d48e9a710: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-80959482-d8ba-fbfa-fe27-b41d48e9a710 Address:80959482-d8ba-fbfa-fe27-b41d48e9a710} {Suffrage:Voter ID:server-fbeb472f-aad0-a759-9ef8-e049e8f7b2d4 Address:fbeb472f-aad0-a759-9ef8-e049e8f7b2d4} {Suffrage:Voter ID:server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5 Address:7e796dd1-742e-9b64-9ed4-0591fb99c5a5}]" -2024-05-12T20:51:28.154Z [INFO] server-fbeb472f-aad0-a759-9ef8-e049e8f7b2d4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-80959482-d8ba-fbfa-fe27-b41d48e9a710 Address:80959482-d8ba-fbfa-fe27-b41d48e9a710} {Suffrage:Voter ID:server-fbeb472f-aad0-a759-9ef8-e049e8f7b2d4 Address:fbeb472f-aad0-a759-9ef8-e049e8f7b2d4} {Suffrage:Voter ID:server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5 Address:7e796dd1-742e-9b64-9ed4-0591fb99c5a5}]" -2024-05-12T20:51:28.154Z [INFO] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-80959482-d8ba-fbfa-fe27-b41d48e9a710 Address:80959482-d8ba-fbfa-fe27-b41d48e9a710} {Suffrage:Voter ID:server-fbeb472f-aad0-a759-9ef8-e049e8f7b2d4 Address:fbeb472f-aad0-a759-9ef8-e049e8f7b2d4} {Suffrage:Voter ID:server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5 Address:7e796dd1-742e-9b64-9ed4-0591fb99c5a5}]" -2024-05-12T20:51:28.154Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:28.154Z [INFO] server-80959482-d8ba-fbfa-fe27-b41d48e9a710: entering follower state: follower="Node at 80959482-d8ba-fbfa-fe27-b41d48e9a710 [Follower]" leader-address= leader-id= -2024-05-12T20:51:28.154Z [INFO] server-fbeb472f-aad0-a759-9ef8-e049e8f7b2d4: entering follower state: follower="Node at fbeb472f-aad0-a759-9ef8-e049e8f7b2d4 [Follower]" leader-address= leader-id= -2024-05-12T20:51:28.154Z [INFO] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: entering follower state: follower="Node at 7e796dd1-742e-9b64-9ed4-0591fb99c5a5 [Follower]" leader-address= leader-id= -2024-05-12T20:51:28.230Z [WARN] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:28.230Z [INFO] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: entering candidate state: node="Node at 7e796dd1-742e-9b64-9ed4-0591fb99c5a5 [Candidate]" term=2 -2024-05-12T20:51:28.231Z [INFO] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: election won: term=2 tally=2 -2024-05-12T20:51:28.231Z [INFO] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: entering leader state: leader="Node at 7e796dd1-742e-9b64-9ed4-0591fb99c5a5 [Leader]" -2024-05-12T20:51:28.231Z [INFO] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: added peer, starting replication: peer=server-80959482-d8ba-fbfa-fe27-b41d48e9a710 -2024-05-12T20:51:28.231Z [INFO] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: added peer, starting replication: peer=server-fbeb472f-aad0-a759-9ef8-e049e8f7b2d4 -2024-05-12T20:51:28.231Z [INFO] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: pipelining replication: peer="{Voter server-fbeb472f-aad0-a759-9ef8-e049e8f7b2d4 fbeb472f-aad0-a759-9ef8-e049e8f7b2d4}" -2024-05-12T20:51:28.231Z [INFO] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: pipelining replication: peer="{Voter server-80959482-d8ba-fbfa-fe27-b41d48e9a710 80959482-d8ba-fbfa-fe27-b41d48e9a710}" -2024-05-12T20:51:28.339Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:28.231505755 +0000 UTC m=+99.512206105 (2 nodes), 77.137618ms from start of poll, 77.73121ms from cluster start. Timeout at 2024-05-12 20:51:28.339133705 +0000 UTC m=+99.619833600, 107.627495ms after stability -2024-05-12T20:51:28.339Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:28.447Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:28.33926083 +0000 UTC m=+99.619961285 (1 nodes), 1.004µs from start of poll, 185.48639ms from cluster start. Timeout at 2024-05-12 20:51:28.447552593 +0000 UTC m=+99.728253419, 108.292134ms after stability -2024-05-12T20:51:28.447Z [ERROR] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: failed to pipeline appendEntries: peer="{Voter server-80959482-d8ba-fbfa-fe27-b41d48e9a710 80959482-d8ba-fbfa-fe27-b41d48e9a710}" error="append pipeline closed" -2024-05-12T20:51:28.447Z [INFO] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: aborting pipeline replication: peer="{Voter server-80959482-d8ba-fbfa-fe27-b41d48e9a710 80959482-d8ba-fbfa-fe27-b41d48e9a710}" -2024-05-12T20:51:28.447Z [ERROR] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: failed to appendEntries to: peer="{Voter server-80959482-d8ba-fbfa-fe27-b41d48e9a710 80959482-d8ba-fbfa-fe27-b41d48e9a710}" error="failed to connect to peer: 80959482-d8ba-fbfa-fe27-b41d48e9a710" -2024-05-12T20:51:28.473Z [ERROR] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: failed to appendEntries to: peer="{Voter server-80959482-d8ba-fbfa-fe27-b41d48e9a710 80959482-d8ba-fbfa-fe27-b41d48e9a710}" error="failed to connect to peer: 80959482-d8ba-fbfa-fe27-b41d48e9a710" -2024-05-12T20:51:28.473Z [ERROR] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: failed to heartbeat to: peer=80959482-d8ba-fbfa-fe27-b41d48e9a710 backoff time=10ms error="failed to connect to peer: 80959482-d8ba-fbfa-fe27-b41d48e9a710" +2025-06-15T03:18:02.510Z [INFO] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1a8523bd-5c23-6e30-b326-3fae8c6174c9 Address:1a8523bd-5c23-6e30-b326-3fae8c6174c9} {Suffrage:Voter ID:server-53446988-5c3a-fc94-695b-4a2b09d01f91 Address:53446988-5c3a-fc94-695b-4a2b09d01f91} {Suffrage:Voter ID:server-492aab20-0fd8-7dfe-6638-88ca695e60bb Address:492aab20-0fd8-7dfe-6638-88ca695e60bb}]" +2025-06-15T03:18:02.510Z [INFO] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: entering follower state: follower="Node at 1a8523bd-5c23-6e30-b326-3fae8c6174c9 [Follower]" leader-address= leader-id= +2025-06-15T03:18:02.510Z [INFO] server-53446988-5c3a-fc94-695b-4a2b09d01f91: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1a8523bd-5c23-6e30-b326-3fae8c6174c9 Address:1a8523bd-5c23-6e30-b326-3fae8c6174c9} {Suffrage:Voter ID:server-53446988-5c3a-fc94-695b-4a2b09d01f91 Address:53446988-5c3a-fc94-695b-4a2b09d01f91} {Suffrage:Voter ID:server-492aab20-0fd8-7dfe-6638-88ca695e60bb Address:492aab20-0fd8-7dfe-6638-88ca695e60bb}]" +2025-06-15T03:18:02.511Z [INFO] server-53446988-5c3a-fc94-695b-4a2b09d01f91: entering follower state: follower="Node at 53446988-5c3a-fc94-695b-4a2b09d01f91 [Follower]" leader-address= leader-id= +2025-06-15T03:18:02.511Z [INFO] server-492aab20-0fd8-7dfe-6638-88ca695e60bb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1a8523bd-5c23-6e30-b326-3fae8c6174c9 Address:1a8523bd-5c23-6e30-b326-3fae8c6174c9} {Suffrage:Voter ID:server-53446988-5c3a-fc94-695b-4a2b09d01f91 Address:53446988-5c3a-fc94-695b-4a2b09d01f91} {Suffrage:Voter ID:server-492aab20-0fd8-7dfe-6638-88ca695e60bb Address:492aab20-0fd8-7dfe-6638-88ca695e60bb}]" +2025-06-15T03:18:02.511Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:02.511Z [INFO] server-492aab20-0fd8-7dfe-6638-88ca695e60bb: entering follower state: follower="Node at 492aab20-0fd8-7dfe-6638-88ca695e60bb [Follower]" leader-address= leader-id= +2025-06-15T03:18:02.571Z [WARN] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:02.571Z [INFO] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: entering candidate state: node="Node at 1a8523bd-5c23-6e30-b326-3fae8c6174c9 [Candidate]" term=2 +2025-06-15T03:18:02.571Z [INFO] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: election won: term=2 tally=2 +2025-06-15T03:18:02.572Z [INFO] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: entering leader state: leader="Node at 1a8523bd-5c23-6e30-b326-3fae8c6174c9 [Leader]" +2025-06-15T03:18:02.572Z [INFO] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: added peer, starting replication: peer=server-53446988-5c3a-fc94-695b-4a2b09d01f91 +2025-06-15T03:18:02.572Z [INFO] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: added peer, starting replication: peer=server-492aab20-0fd8-7dfe-6638-88ca695e60bb +2025-06-15T03:18:02.572Z [INFO] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: pipelining replication: peer="{Voter server-492aab20-0fd8-7dfe-6638-88ca695e60bb 492aab20-0fd8-7dfe-6638-88ca695e60bb}" +2025-06-15T03:18:02.572Z [INFO] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: pipelining replication: peer="{Voter server-53446988-5c3a-fc94-695b-4a2b09d01f91 53446988-5c3a-fc94-695b-4a2b09d01f91}" +2025-06-15T03:18:02.678Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:02.572088891 +0000 UTC m=+98.283767052 (2 nodes), 60.951338ms from start of poll, 61.388033ms from cluster start. Timeout at 2025-06-15 03:18:02.677980566 +0000 UTC m=+98.389658769, 105.891717ms after stability +2025-06-15T03:18:02.678Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:02.783Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:02.678092648 +0000 UTC m=+98.389770848 (1 nodes), 1.039µs from start of poll, 167.391829ms from cluster start. Timeout at 2025-06-15 03:18:02.78396203 +0000 UTC m=+98.495640197, 105.869349ms after stability +2025-06-15T03:18:02.784Z [ERROR] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: failed to pipeline appendEntries: peer="{Voter server-53446988-5c3a-fc94-695b-4a2b09d01f91 53446988-5c3a-fc94-695b-4a2b09d01f91}" error="append pipeline closed" +2025-06-15T03:18:02.784Z [INFO] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: aborting pipeline replication: peer="{Voter server-53446988-5c3a-fc94-695b-4a2b09d01f91 53446988-5c3a-fc94-695b-4a2b09d01f91}" +2025-06-15T03:18:02.784Z [ERROR] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: failed to appendEntries to: peer="{Voter server-53446988-5c3a-fc94-695b-4a2b09d01f91 53446988-5c3a-fc94-695b-4a2b09d01f91}" error="failed to connect to peer: 53446988-5c3a-fc94-695b-4a2b09d01f91" +2025-06-15T03:18:02.784Z [ERROR] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: failed to heartbeat to: peer=53446988-5c3a-fc94-695b-4a2b09d01f91 backoff time=10ms error="failed to connect to peer: 53446988-5c3a-fc94-695b-4a2b09d01f91" raft_test.go:1669: [INFO] Finished apply without behind follower -2024-05-12T20:51:28.498Z [ERROR] server-80959482-d8ba-fbfa-fe27-b41d48e9a710: failed to take snapshot: error="nothing new to snapshot" -2024-05-12T20:51:28.498Z [INFO] server-fbeb472f-aad0-a759-9ef8-e049e8f7b2d4: starting snapshot up to: index=99 -2024-05-12T20:51:28.498Z [INFO] creating new snapshot: path=/tmp/raft3822326941/snapshots/2-99-1715547088498.tmp -2024-05-12T20:51:28.498Z [WARN] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: failed to contact: server-id=server-80959482-d8ba-fbfa-fe27-b41d48e9a710 time=51.066552ms -2024-05-12T20:51:28.498Z [ERROR] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: failed to appendEntries to: peer="{Voter server-80959482-d8ba-fbfa-fe27-b41d48e9a710 80959482-d8ba-fbfa-fe27-b41d48e9a710}" error="failed to connect to peer: 80959482-d8ba-fbfa-fe27-b41d48e9a710" -2024-05-12T20:51:28.500Z [INFO] server-fbeb472f-aad0-a759-9ef8-e049e8f7b2d4: compacting logs: from=1 to=92 -2024-05-12T20:51:28.500Z [INFO] server-fbeb472f-aad0-a759-9ef8-e049e8f7b2d4: snapshot complete up to: index=99 -2024-05-12T20:51:28.500Z [INFO] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: starting snapshot up to: index=102 -2024-05-12T20:51:28.500Z [INFO] creating new snapshot: path=/tmp/raft2068528133/snapshots/2-102-1715547088500.tmp -2024-05-12T20:51:28.501Z [INFO] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: compacting logs: from=1 to=92 -2024-05-12T20:51:28.501Z [INFO] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: snapshot complete up to: index=102 +2025-06-15T03:18:02.785Z [INFO] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: starting snapshot up to: index=102 +2025-06-15T03:18:02.785Z [INFO] creating new snapshot: path=/tmp/raft1711002543/snapshots/2-102-1749957482785.tmp +2025-06-15T03:18:02.786Z [INFO] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: compacting logs: from=1 to=92 +2025-06-15T03:18:02.786Z [INFO] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: snapshot complete up to: index=102 +2025-06-15T03:18:02.786Z [ERROR] server-53446988-5c3a-fc94-695b-4a2b09d01f91: failed to take snapshot: error="nothing new to snapshot" +2025-06-15T03:18:02.786Z [INFO] server-492aab20-0fd8-7dfe-6638-88ca695e60bb: starting snapshot up to: index=93 +2025-06-15T03:18:02.786Z [INFO] creating new snapshot: path=/tmp/raft2728527767/snapshots/2-93-1749957482786.tmp +2025-06-15T03:18:02.786Z [INFO] server-492aab20-0fd8-7dfe-6638-88ca695e60bb: compacting logs: from=1 to=92 +2025-06-15T03:18:02.786Z [INFO] server-492aab20-0fd8-7dfe-6638-88ca695e60bb: snapshot complete up to: index=93 raft_test.go:1690: [INFO] Finished apply without behind follower -2024-05-12T20:51:28.519Z [ERROR] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: failed to get log: index=2 error="log not found" -2024-05-12T20:51:28.519Z [INFO] creating new snapshot: path=/tmp/raft890097370/snapshots/2-102-1715547088519.tmp -2024-05-12T20:51:28.519Z [INFO] server-80959482-d8ba-fbfa-fe27-b41d48e9a710: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% -2024-05-12T20:51:28.520Z [INFO] server-80959482-d8ba-fbfa-fe27-b41d48e9a710: copied to local snapshot: bytes=693 -2024-05-12T20:51:28.520Z [INFO] server-80959482-d8ba-fbfa-fe27-b41d48e9a710: snapshot restore progress: id=2-102-1715547088519 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2024-05-12T20:51:28.520Z [INFO] server-80959482-d8ba-fbfa-fe27-b41d48e9a710: Installed remote snapshot -2024-05-12T20:51:28.521Z [INFO] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: pipelining replication: peer="{Voter server-80959482-d8ba-fbfa-fe27-b41d48e9a710 80959482-d8ba-fbfa-fe27-b41d48e9a710}" -2024-05-12T20:51:28.523Z [INFO] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: aborting pipeline replication: peer="{Voter server-fbeb472f-aad0-a759-9ef8-e049e8f7b2d4 fbeb472f-aad0-a759-9ef8-e049e8f7b2d4}" -2024-05-12T20:51:28.523Z [INFO] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: aborting pipeline replication: peer="{Voter server-80959482-d8ba-fbfa-fe27-b41d48e9a710 80959482-d8ba-fbfa-fe27-b41d48e9a710}" -2024-05-12T20:51:29.023Z [ERROR] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: failed to heartbeat to: peer=80959482-d8ba-fbfa-fe27-b41d48e9a710 backoff time=10ms error="command timed out" -2024-05-12T20:51:29.023Z [ERROR] server-7e796dd1-742e-9b64-9ed4-0591fb99c5a5: failed to heartbeat to: peer=fbeb472f-aad0-a759-9ef8-e049e8f7b2d4 backoff time=10ms error="command timed out" ---- PASS: TestRaft_SendSnapshotAndLogsFollower (0.87s) +2025-06-15T03:18:02.795Z [ERROR] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: failed to get log: index=2 error="log not found" +2025-06-15T03:18:02.795Z [INFO] creating new snapshot: path=/tmp/raft4246536088/snapshots/2-102-1749957482795.tmp +2025-06-15T03:18:02.795Z [INFO] server-53446988-5c3a-fc94-695b-4a2b09d01f91: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% +2025-06-15T03:18:02.795Z [INFO] server-53446988-5c3a-fc94-695b-4a2b09d01f91: copied to local snapshot: bytes=693 +2025-06-15T03:18:02.795Z [INFO] server-53446988-5c3a-fc94-695b-4a2b09d01f91: snapshot restore progress: id=2-102-1749957482795 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-06-15T03:18:02.795Z [INFO] server-53446988-5c3a-fc94-695b-4a2b09d01f91: Installed remote snapshot +2025-06-15T03:18:02.795Z [INFO] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: pipelining replication: peer="{Voter server-53446988-5c3a-fc94-695b-4a2b09d01f91 53446988-5c3a-fc94-695b-4a2b09d01f91}" +2025-06-15T03:18:02.799Z [INFO] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: aborting pipeline replication: peer="{Voter server-492aab20-0fd8-7dfe-6638-88ca695e60bb 492aab20-0fd8-7dfe-6638-88ca695e60bb}" +2025-06-15T03:18:02.799Z [INFO] server-1a8523bd-5c23-6e30-b326-3fae8c6174c9: aborting pipeline replication: peer="{Voter server-53446988-5c3a-fc94-695b-4a2b09d01f91 53446988-5c3a-fc94-695b-4a2b09d01f91}" +--- PASS: TestRaft_SendSnapshotAndLogsFollower (0.29s) === RUN TestRaft_ReJoinFollower -2024-05-12T20:51:29.025Z [INFO] server-aaa82b28-908a-a7a1-044f-01291b265f10: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aaa82b28-908a-a7a1-044f-01291b265f10 Address:aaa82b28-908a-a7a1-044f-01291b265f10} {Suffrage:Voter ID:server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7 Address:539f9829-7bf7-cdb5-02a5-13ca222d0bb7} {Suffrage:Voter ID:server-55c8da24-873a-e10c-0706-38499af0be5b Address:55c8da24-873a-e10c-0706-38499af0be5b}]" -2024-05-12T20:51:29.025Z [INFO] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aaa82b28-908a-a7a1-044f-01291b265f10 Address:aaa82b28-908a-a7a1-044f-01291b265f10} {Suffrage:Voter ID:server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7 Address:539f9829-7bf7-cdb5-02a5-13ca222d0bb7} {Suffrage:Voter ID:server-55c8da24-873a-e10c-0706-38499af0be5b Address:55c8da24-873a-e10c-0706-38499af0be5b}]" -2024-05-12T20:51:29.026Z [INFO] server-55c8da24-873a-e10c-0706-38499af0be5b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aaa82b28-908a-a7a1-044f-01291b265f10 Address:aaa82b28-908a-a7a1-044f-01291b265f10} {Suffrage:Voter ID:server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7 Address:539f9829-7bf7-cdb5-02a5-13ca222d0bb7} {Suffrage:Voter ID:server-55c8da24-873a-e10c-0706-38499af0be5b Address:55c8da24-873a-e10c-0706-38499af0be5b}]" -2024-05-12T20:51:29.026Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:29.026Z [INFO] server-aaa82b28-908a-a7a1-044f-01291b265f10: entering follower state: follower="Node at aaa82b28-908a-a7a1-044f-01291b265f10 [Follower]" leader-address= leader-id= -2024-05-12T20:51:29.026Z [INFO] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: entering follower state: follower="Node at 539f9829-7bf7-cdb5-02a5-13ca222d0bb7 [Follower]" leader-address= leader-id= -2024-05-12T20:51:29.026Z [INFO] server-55c8da24-873a-e10c-0706-38499af0be5b: entering follower state: follower="Node at 55c8da24-873a-e10c-0706-38499af0be5b [Follower]" leader-address= leader-id= -2024-05-12T20:51:29.087Z [WARN] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:29.087Z [INFO] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: entering candidate state: node="Node at 539f9829-7bf7-cdb5-02a5-13ca222d0bb7 [Candidate]" term=2 -2024-05-12T20:51:29.087Z [INFO] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: election won: term=2 tally=2 -2024-05-12T20:51:29.087Z [INFO] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: entering leader state: leader="Node at 539f9829-7bf7-cdb5-02a5-13ca222d0bb7 [Leader]" -2024-05-12T20:51:29.087Z [INFO] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: added peer, starting replication: peer=server-aaa82b28-908a-a7a1-044f-01291b265f10 -2024-05-12T20:51:29.087Z [INFO] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: added peer, starting replication: peer=server-55c8da24-873a-e10c-0706-38499af0be5b -2024-05-12T20:51:29.087Z [INFO] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: pipelining replication: peer="{Voter server-55c8da24-873a-e10c-0706-38499af0be5b 55c8da24-873a-e10c-0706-38499af0be5b}" -2024-05-12T20:51:29.088Z [INFO] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: pipelining replication: peer="{Voter server-aaa82b28-908a-a7a1-044f-01291b265f10 aaa82b28-908a-a7a1-044f-01291b265f10}" -2024-05-12T20:51:29.194Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:29.088021718 +0000 UTC m=+100.368722060 (1 nodes), 61.867124ms from start of poll, 62.351251ms from cluster start. Timeout at 2024-05-12 20:51:29.194781912 +0000 UTC m=+100.475482464, 106.760404ms after stability -2024-05-12T20:51:29.203Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:29.311Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:29.203091472 +0000 UTC m=+100.483791873 (2 nodes), 3.995µs from start of poll, 177.421064ms from cluster start. Timeout at 2024-05-12 20:51:29.311156341 +0000 UTC m=+100.591856236, 108.064363ms after stability -2024-05-12T20:51:29.311Z [INFO] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: updating configuration: command=RemoveServer server-id=server-aaa82b28-908a-a7a1-044f-01291b265f10 server-addr= servers="[{Suffrage:Voter ID:server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7 Address:539f9829-7bf7-cdb5-02a5-13ca222d0bb7} {Suffrage:Voter ID:server-55c8da24-873a-e10c-0706-38499af0be5b Address:55c8da24-873a-e10c-0706-38499af0be5b}]" -2024-05-12T20:51:29.311Z [INFO] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: removed peer, stopping replication: peer=server-aaa82b28-908a-a7a1-044f-01291b265f10 last-index=3 -2024-05-12T20:51:29.311Z [INFO] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: aborting pipeline replication: peer="{Voter server-aaa82b28-908a-a7a1-044f-01291b265f10 aaa82b28-908a-a7a1-044f-01291b265f10}" -2024-05-12T20:51:29.374Z [WARN] server-aaa82b28-908a-a7a1-044f-01291b265f10: not part of stable configuration, aborting election -2024-05-12T20:51:29.418Z [INFO] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: updating configuration: command=AddVoter server-id=server-aaa82b28-908a-a7a1-044f-01291b265f10 server-addr=aaa82b28-908a-a7a1-044f-01291b265f10 servers="[{Suffrage:Voter ID:server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7 Address:539f9829-7bf7-cdb5-02a5-13ca222d0bb7} {Suffrage:Voter ID:server-55c8da24-873a-e10c-0706-38499af0be5b Address:55c8da24-873a-e10c-0706-38499af0be5b} {Suffrage:Voter ID:server-aaa82b28-908a-a7a1-044f-01291b265f10 Address:aaa82b28-908a-a7a1-044f-01291b265f10}]" -2024-05-12T20:51:29.419Z [INFO] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: added peer, starting replication: peer=server-aaa82b28-908a-a7a1-044f-01291b265f10 -2024-05-12T20:51:29.419Z [WARN] server-aaa82b28-908a-a7a1-044f-01291b265f10: failed to get previous log: previous-index=4 last-index=3 error="log not found" -2024-05-12T20:51:29.419Z [WARN] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: appendEntries rejected, sending older logs: peer="{Voter server-aaa82b28-908a-a7a1-044f-01291b265f10 aaa82b28-908a-a7a1-044f-01291b265f10}" next=4 -2024-05-12T20:51:29.419Z [INFO] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: pipelining replication: peer="{Voter server-aaa82b28-908a-a7a1-044f-01291b265f10 aaa82b28-908a-a7a1-044f-01291b265f10}" -2024-05-12T20:51:29.419Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:29.527Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:29.419526282 +0000 UTC m=+100.700226589 (1 nodes), 1.933µs from start of poll, 393.85578ms from cluster start. Timeout at 2024-05-12 20:51:29.526862964 +0000 UTC m=+100.807563530, 107.336941ms after stability -2024-05-12T20:51:29.527Z [INFO] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: aborting pipeline replication: peer="{Voter server-55c8da24-873a-e10c-0706-38499af0be5b 55c8da24-873a-e10c-0706-38499af0be5b}" -2024-05-12T20:51:29.527Z [INFO] server-539f9829-7bf7-cdb5-02a5-13ca222d0bb7: aborting pipeline replication: peer="{Voter server-aaa82b28-908a-a7a1-044f-01291b265f10 aaa82b28-908a-a7a1-044f-01291b265f10}" +2025-06-15T03:18:02.800Z [INFO] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-54245e80-28ac-547d-d999-b4bd49fcc4a1 Address:54245e80-28ac-547d-d999-b4bd49fcc4a1} {Suffrage:Voter ID:server-2b48e971-b669-3658-eb36-2b317b42cda1 Address:2b48e971-b669-3658-eb36-2b317b42cda1} {Suffrage:Voter ID:server-ab63c4e6-9f85-67c7-3430-1bc02533d869 Address:ab63c4e6-9f85-67c7-3430-1bc02533d869}]" +2025-06-15T03:18:02.800Z [INFO] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: entering follower state: follower="Node at 54245e80-28ac-547d-d999-b4bd49fcc4a1 [Follower]" leader-address= leader-id= +2025-06-15T03:18:02.800Z [INFO] server-2b48e971-b669-3658-eb36-2b317b42cda1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-54245e80-28ac-547d-d999-b4bd49fcc4a1 Address:54245e80-28ac-547d-d999-b4bd49fcc4a1} {Suffrage:Voter ID:server-2b48e971-b669-3658-eb36-2b317b42cda1 Address:2b48e971-b669-3658-eb36-2b317b42cda1} {Suffrage:Voter ID:server-ab63c4e6-9f85-67c7-3430-1bc02533d869 Address:ab63c4e6-9f85-67c7-3430-1bc02533d869}]" +2025-06-15T03:18:02.800Z [INFO] server-2b48e971-b669-3658-eb36-2b317b42cda1: entering follower state: follower="Node at 2b48e971-b669-3658-eb36-2b317b42cda1 [Follower]" leader-address= leader-id= +2025-06-15T03:18:02.800Z [INFO] server-ab63c4e6-9f85-67c7-3430-1bc02533d869: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-54245e80-28ac-547d-d999-b4bd49fcc4a1 Address:54245e80-28ac-547d-d999-b4bd49fcc4a1} {Suffrage:Voter ID:server-2b48e971-b669-3658-eb36-2b317b42cda1 Address:2b48e971-b669-3658-eb36-2b317b42cda1} {Suffrage:Voter ID:server-ab63c4e6-9f85-67c7-3430-1bc02533d869 Address:ab63c4e6-9f85-67c7-3430-1bc02533d869}]" +2025-06-15T03:18:02.800Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:02.800Z [INFO] server-ab63c4e6-9f85-67c7-3430-1bc02533d869: entering follower state: follower="Node at ab63c4e6-9f85-67c7-3430-1bc02533d869 [Follower]" leader-address= leader-id= +2025-06-15T03:18:02.866Z [WARN] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:02.866Z [INFO] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: entering candidate state: node="Node at 54245e80-28ac-547d-d999-b4bd49fcc4a1 [Candidate]" term=2 +2025-06-15T03:18:02.866Z [INFO] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: election won: term=2 tally=2 +2025-06-15T03:18:02.866Z [INFO] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: entering leader state: leader="Node at 54245e80-28ac-547d-d999-b4bd49fcc4a1 [Leader]" +2025-06-15T03:18:02.866Z [INFO] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: added peer, starting replication: peer=server-2b48e971-b669-3658-eb36-2b317b42cda1 +2025-06-15T03:18:02.866Z [INFO] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: added peer, starting replication: peer=server-ab63c4e6-9f85-67c7-3430-1bc02533d869 +2025-06-15T03:18:02.866Z [INFO] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: pipelining replication: peer="{Voter server-ab63c4e6-9f85-67c7-3430-1bc02533d869 ab63c4e6-9f85-67c7-3430-1bc02533d869}" +2025-06-15T03:18:02.866Z [INFO] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: pipelining replication: peer="{Voter server-2b48e971-b669-3658-eb36-2b317b42cda1 2b48e971-b669-3658-eb36-2b317b42cda1}" +2025-06-15T03:18:02.972Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:02.866794853 +0000 UTC m=+98.578473052 (1 nodes), 65.976357ms from start of poll, 66.2424ms from cluster start. Timeout at 2025-06-15 03:18:02.972199381 +0000 UTC m=+98.683877580, 105.404528ms after stability +2025-06-15T03:18:02.977Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:03.082Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:02.977569034 +0000 UTC m=+98.689247233 (2 nodes), 1.283µs from start of poll, 177.016581ms from cluster start. Timeout at 2025-06-15 03:18:03.082791184 +0000 UTC m=+98.794469383, 105.22215ms after stability +2025-06-15T03:18:03.082Z [INFO] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: updating configuration: command=RemoveServer server-id=server-2b48e971-b669-3658-eb36-2b317b42cda1 server-addr= servers="[{Suffrage:Voter ID:server-54245e80-28ac-547d-d999-b4bd49fcc4a1 Address:54245e80-28ac-547d-d999-b4bd49fcc4a1} {Suffrage:Voter ID:server-ab63c4e6-9f85-67c7-3430-1bc02533d869 Address:ab63c4e6-9f85-67c7-3430-1bc02533d869}]" +2025-06-15T03:18:03.083Z [INFO] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: removed peer, stopping replication: peer=server-2b48e971-b669-3658-eb36-2b317b42cda1 last-index=3 +2025-06-15T03:18:03.083Z [INFO] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: aborting pipeline replication: peer="{Voter server-2b48e971-b669-3658-eb36-2b317b42cda1 2b48e971-b669-3658-eb36-2b317b42cda1}" +2025-06-15T03:18:03.134Z [WARN] server-2b48e971-b669-3658-eb36-2b317b42cda1: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-06-15T03:18:03.189Z [INFO] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: updating configuration: command=AddVoter server-id=server-2b48e971-b669-3658-eb36-2b317b42cda1 server-addr=2b48e971-b669-3658-eb36-2b317b42cda1 servers="[{Suffrage:Voter ID:server-54245e80-28ac-547d-d999-b4bd49fcc4a1 Address:54245e80-28ac-547d-d999-b4bd49fcc4a1} {Suffrage:Voter ID:server-ab63c4e6-9f85-67c7-3430-1bc02533d869 Address:ab63c4e6-9f85-67c7-3430-1bc02533d869} {Suffrage:Voter ID:server-2b48e971-b669-3658-eb36-2b317b42cda1 Address:2b48e971-b669-3658-eb36-2b317b42cda1}]" +2025-06-15T03:18:03.189Z [INFO] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: added peer, starting replication: peer=server-2b48e971-b669-3658-eb36-2b317b42cda1 +2025-06-15T03:18:03.189Z [WARN] server-2b48e971-b669-3658-eb36-2b317b42cda1: failed to get previous log: previous-index=4 last-index=3 error="log not found" +2025-06-15T03:18:03.189Z [WARN] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: appendEntries rejected, sending older logs: peer="{Voter server-2b48e971-b669-3658-eb36-2b317b42cda1 2b48e971-b669-3658-eb36-2b317b42cda1}" next=4 +2025-06-15T03:18:03.189Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:03.189Z [INFO] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: pipelining replication: peer="{Voter server-2b48e971-b669-3658-eb36-2b317b42cda1 2b48e971-b669-3658-eb36-2b317b42cda1}" +2025-06-15T03:18:03.294Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:03.189350677 +0000 UTC m=+98.901028842 (1 nodes), 477ns from start of poll, 388.79819ms from cluster start. Timeout at 2025-06-15 03:18:03.294585564 +0000 UTC m=+99.006263763, 105.234921ms after stability +2025-06-15T03:18:03.294Z [INFO] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: aborting pipeline replication: peer="{Voter server-ab63c4e6-9f85-67c7-3430-1bc02533d869 ab63c4e6-9f85-67c7-3430-1bc02533d869}" +2025-06-15T03:18:03.294Z [INFO] server-54245e80-28ac-547d-d999-b4bd49fcc4a1: aborting pipeline replication: peer="{Voter server-2b48e971-b669-3658-eb36-2b317b42cda1 2b48e971-b669-3658-eb36-2b317b42cda1}" --- PASS: TestRaft_ReJoinFollower (0.50s) === RUN TestRaft_LeaderLeaseExpire -2024-05-12T20:51:29.529Z [INFO] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105 Address:d01ec6e4-ee6f-cd50-b9bc-e14b676f4105} {Suffrage:Voter ID:server-a4491af3-07ec-38a4-b367-b541efd9bfb7 Address:a4491af3-07ec-38a4-b367-b541efd9bfb7}]" -2024-05-12T20:51:29.529Z [INFO] server-a4491af3-07ec-38a4-b367-b541efd9bfb7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105 Address:d01ec6e4-ee6f-cd50-b9bc-e14b676f4105} {Suffrage:Voter ID:server-a4491af3-07ec-38a4-b367-b541efd9bfb7 Address:a4491af3-07ec-38a4-b367-b541efd9bfb7}]" -2024-05-12T20:51:29.529Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:29.529Z [INFO] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: entering follower state: follower="Node at d01ec6e4-ee6f-cd50-b9bc-e14b676f4105 [Follower]" leader-address= leader-id= -2024-05-12T20:51:29.529Z [INFO] server-a4491af3-07ec-38a4-b367-b541efd9bfb7: entering follower state: follower="Node at a4491af3-07ec-38a4-b367-b541efd9bfb7 [Follower]" leader-address= leader-id= -2024-05-12T20:51:29.614Z [WARN] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:29.614Z [INFO] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: entering candidate state: node="Node at d01ec6e4-ee6f-cd50-b9bc-e14b676f4105 [Candidate]" term=2 -2024-05-12T20:51:29.615Z [INFO] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: election won: term=2 tally=2 -2024-05-12T20:51:29.615Z [INFO] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: entering leader state: leader="Node at d01ec6e4-ee6f-cd50-b9bc-e14b676f4105 [Leader]" -2024-05-12T20:51:29.615Z [INFO] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: added peer, starting replication: peer=server-a4491af3-07ec-38a4-b367-b541efd9bfb7 -2024-05-12T20:51:29.615Z [INFO] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: pipelining replication: peer="{Voter server-a4491af3-07ec-38a4-b367-b541efd9bfb7 a4491af3-07ec-38a4-b367-b541efd9bfb7}" -2024-05-12T20:51:29.721Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:29.615237089 +0000 UTC m=+100.895937424 (1 nodes), 85.568248ms from start of poll, 85.85246ms from cluster start. Timeout at 2024-05-12 20:51:29.721363592 +0000 UTC m=+101.002063989, 106.126565ms after stability -2024-05-12T20:51:29.727Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:29.834Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:29.727077218 +0000 UTC m=+101.007777113 (1 nodes), 0s from start of poll, 197.692149ms from cluster start. Timeout at 2024-05-12 20:51:29.834792502 +0000 UTC m=+101.115492397, 107.715284ms after stability - raft_test.go:1799: [INFO] Disconnecting Node at a4491af3-07ec-38a4-b367-b541efd9bfb7 [Follower] -2024-05-12T20:51:29.835Z [ERROR] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: failed to pipeline appendEntries: peer="{Voter server-a4491af3-07ec-38a4-b367-b541efd9bfb7 a4491af3-07ec-38a4-b367-b541efd9bfb7}" error="append pipeline closed" -2024-05-12T20:51:29.835Z [INFO] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: aborting pipeline replication: peer="{Voter server-a4491af3-07ec-38a4-b367-b541efd9bfb7 a4491af3-07ec-38a4-b367-b541efd9bfb7}" -2024-05-12T20:51:29.837Z [ERROR] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: failed to heartbeat to: peer=a4491af3-07ec-38a4-b367-b541efd9bfb7 backoff time=10ms error="failed to connect to peer: a4491af3-07ec-38a4-b367-b541efd9bfb7" -2024-05-12T20:51:29.845Z [ERROR] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: failed to appendEntries to: peer="{Voter server-a4491af3-07ec-38a4-b367-b541efd9bfb7 a4491af3-07ec-38a4-b367-b541efd9bfb7}" error="failed to connect to peer: a4491af3-07ec-38a4-b367-b541efd9bfb7" -2024-05-12T20:51:29.854Z [ERROR] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: failed to heartbeat to: peer=a4491af3-07ec-38a4-b367-b541efd9bfb7 backoff time=10ms error="failed to connect to peer: a4491af3-07ec-38a4-b367-b541efd9bfb7" -2024-05-12T20:51:29.861Z [ERROR] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: failed to appendEntries to: peer="{Voter server-a4491af3-07ec-38a4-b367-b541efd9bfb7 a4491af3-07ec-38a4-b367-b541efd9bfb7}" error="failed to connect to peer: a4491af3-07ec-38a4-b367-b541efd9bfb7" -2024-05-12T20:51:29.875Z [ERROR] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: failed to heartbeat to: peer=a4491af3-07ec-38a4-b367-b541efd9bfb7 backoff time=10ms error="failed to connect to peer: a4491af3-07ec-38a4-b367-b541efd9bfb7" -2024-05-12T20:51:29.879Z [WARN] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: failed to contact: server-id=server-a4491af3-07ec-38a4-b367-b541efd9bfb7 time=50.031792ms -2024-05-12T20:51:29.879Z [WARN] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: failed to contact quorum of nodes, stepping down -2024-05-12T20:51:29.879Z [INFO] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: entering follower state: follower="Node at d01ec6e4-ee6f-cd50-b9bc-e14b676f4105 [Follower]" leader-address= leader-id= -2024-05-12T20:51:29.879Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:29.883Z [ERROR] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: failed to appendEntries to: peer="{Voter server-a4491af3-07ec-38a4-b367-b541efd9bfb7 a4491af3-07ec-38a4-b367-b541efd9bfb7}" error="failed to connect to peer: a4491af3-07ec-38a4-b367-b541efd9bfb7" -2024-05-12T20:51:29.927Z [WARN] server-a4491af3-07ec-38a4-b367-b541efd9bfb7: heartbeat timeout reached, starting election: last-leader-addr=d01ec6e4-ee6f-cd50-b9bc-e14b676f4105 last-leader-id=server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105 -2024-05-12T20:51:29.927Z [INFO] server-a4491af3-07ec-38a4-b367-b541efd9bfb7: entering candidate state: node="Node at a4491af3-07ec-38a4-b367-b541efd9bfb7 [Candidate]" term=3 -2024-05-12T20:51:29.927Z [ERROR] server-a4491af3-07ec-38a4-b367-b541efd9bfb7: failed to make requestVote RPC: target="{Voter server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105 d01ec6e4-ee6f-cd50-b9bc-e14b676f4105}" error="failed to connect to peer: d01ec6e4-ee6f-cd50-b9bc-e14b676f4105" term=3 -2024-05-12T20:51:29.933Z [WARN] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:29.933Z [INFO] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: entering candidate state: node="Node at d01ec6e4-ee6f-cd50-b9bc-e14b676f4105 [Candidate]" term=3 -2024-05-12T20:51:29.933Z [ERROR] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: failed to make requestVote RPC: target="{Voter server-a4491af3-07ec-38a4-b367-b541efd9bfb7 a4491af3-07ec-38a4-b367-b541efd9bfb7}" error="failed to connect to peer: a4491af3-07ec-38a4-b367-b541efd9bfb7" term=3 -2024-05-12T20:51:29.991Z [WARN] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: Election timeout reached, restarting election -2024-05-12T20:51:29.991Z [INFO] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: entering candidate state: node="Node at d01ec6e4-ee6f-cd50-b9bc-e14b676f4105 [Candidate]" term=4 -2024-05-12T20:51:29.991Z [ERROR] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: failed to make requestVote RPC: target="{Voter server-a4491af3-07ec-38a4-b367-b541efd9bfb7 a4491af3-07ec-38a4-b367-b541efd9bfb7}" error="failed to connect to peer: a4491af3-07ec-38a4-b367-b541efd9bfb7" term=4 -2024-05-12T20:51:30.015Z [WARN] server-a4491af3-07ec-38a4-b367-b541efd9bfb7: Election timeout reached, restarting election -2024-05-12T20:51:30.015Z [INFO] server-a4491af3-07ec-38a4-b367-b541efd9bfb7: entering candidate state: node="Node at a4491af3-07ec-38a4-b367-b541efd9bfb7 [Candidate]" term=4 -2024-05-12T20:51:30.015Z [ERROR] server-a4491af3-07ec-38a4-b367-b541efd9bfb7: failed to make requestVote RPC: target="{Voter server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105 d01ec6e4-ee6f-cd50-b9bc-e14b676f4105}" error="failed to connect to peer: d01ec6e4-ee6f-cd50-b9bc-e14b676f4105" term=4 -2024-05-12T20:51:30.038Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:29.933812278 +0000 UTC m=+101.214512620 (0 nodes), 54.543132ms from start of poll, 404.427656ms from cluster start. Timeout at 2024-05-12 20:51:30.038818112 +0000 UTC m=+101.319518007, 105.005387ms after stability -2024-05-12T20:51:30.079Z [WARN] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: Election timeout reached, restarting election -2024-05-12T20:51:30.079Z [INFO] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: entering candidate state: node="Node at d01ec6e4-ee6f-cd50-b9bc-e14b676f4105 [Candidate]" term=5 -2024-05-12T20:51:30.079Z [ERROR] server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105: failed to make requestVote RPC: target="{Voter server-a4491af3-07ec-38a4-b367-b541efd9bfb7 a4491af3-07ec-38a4-b367-b541efd9bfb7}" error="failed to connect to peer: a4491af3-07ec-38a4-b367-b541efd9bfb7" term=5 -2024-05-12T20:51:30.098Z [WARN] server-a4491af3-07ec-38a4-b367-b541efd9bfb7: Election timeout reached, restarting election -2024-05-12T20:51:30.098Z [INFO] server-a4491af3-07ec-38a4-b367-b541efd9bfb7: entering candidate state: node="Node at a4491af3-07ec-38a4-b367-b541efd9bfb7 [Candidate]" term=5 -2024-05-12T20:51:30.098Z [ERROR] server-a4491af3-07ec-38a4-b367-b541efd9bfb7: failed to make requestVote RPC: target="{Voter server-d01ec6e4-ee6f-cd50-b9bc-e14b676f4105 d01ec6e4-ee6f-cd50-b9bc-e14b676f4105}" error="failed to connect to peer: d01ec6e4-ee6f-cd50-b9bc-e14b676f4105" term=5 ---- PASS: TestRaft_LeaderLeaseExpire (0.62s) +2025-06-15T03:18:03.295Z [INFO] server-f32b41be-316c-b62d-0871-ff2655b5172a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f32b41be-316c-b62d-0871-ff2655b5172a Address:f32b41be-316c-b62d-0871-ff2655b5172a} {Suffrage:Voter ID:server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 Address:4f14a926-0cc5-8ca6-c56b-74dfb6a5d774}]" +2025-06-15T03:18:03.295Z [INFO] server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f32b41be-316c-b62d-0871-ff2655b5172a Address:f32b41be-316c-b62d-0871-ff2655b5172a} {Suffrage:Voter ID:server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 Address:4f14a926-0cc5-8ca6-c56b-74dfb6a5d774}]" +2025-06-15T03:18:03.295Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:03.295Z [INFO] server-f32b41be-316c-b62d-0871-ff2655b5172a: entering follower state: follower="Node at f32b41be-316c-b62d-0871-ff2655b5172a [Follower]" leader-address= leader-id= +2025-06-15T03:18:03.295Z [INFO] server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774: entering follower state: follower="Node at 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 [Follower]" leader-address= leader-id= +2025-06-15T03:18:03.354Z [WARN] server-f32b41be-316c-b62d-0871-ff2655b5172a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:03.354Z [INFO] server-f32b41be-316c-b62d-0871-ff2655b5172a: entering candidate state: node="Node at f32b41be-316c-b62d-0871-ff2655b5172a [Candidate]" term=2 +2025-06-15T03:18:03.354Z [INFO] server-f32b41be-316c-b62d-0871-ff2655b5172a: election won: term=2 tally=2 +2025-06-15T03:18:03.354Z [INFO] server-f32b41be-316c-b62d-0871-ff2655b5172a: entering leader state: leader="Node at f32b41be-316c-b62d-0871-ff2655b5172a [Leader]" +2025-06-15T03:18:03.354Z [INFO] server-f32b41be-316c-b62d-0871-ff2655b5172a: added peer, starting replication: peer=server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 +2025-06-15T03:18:03.354Z [INFO] server-f32b41be-316c-b62d-0871-ff2655b5172a: pipelining replication: peer="{Voter server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774}" +2025-06-15T03:18:03.459Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:03.354777778 +0000 UTC m=+99.066455944 (1 nodes), 59.078534ms from start of poll, 59.27743ms from cluster start. Timeout at 2025-06-15 03:18:03.459781392 +0000 UTC m=+99.171459591, 105.003647ms after stability +2025-06-15T03:18:03.464Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:03.570Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:03.464894923 +0000 UTC m=+99.176573129 (1 nodes), 2.125µs from start of poll, 169.394615ms from cluster start. Timeout at 2025-06-15 03:18:03.570642554 +0000 UTC m=+99.282320753, 105.747624ms after stability + raft_test.go:1799: [INFO] Disconnecting Node at 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 [Follower] +2025-06-15T03:18:03.575Z [ERROR] server-f32b41be-316c-b62d-0871-ff2655b5172a: failed to pipeline appendEntries: peer="{Voter server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774}" error="append pipeline closed" +2025-06-15T03:18:03.575Z [INFO] server-f32b41be-316c-b62d-0871-ff2655b5172a: aborting pipeline replication: peer="{Voter server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774}" +2025-06-15T03:18:03.580Z [ERROR] server-f32b41be-316c-b62d-0871-ff2655b5172a: failed to heartbeat to: peer=4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 backoff time=10ms error="failed to connect to peer: 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774" +2025-06-15T03:18:03.582Z [ERROR] server-f32b41be-316c-b62d-0871-ff2655b5172a: failed to appendEntries to: peer="{Voter server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774}" error="failed to connect to peer: 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774" +2025-06-15T03:18:03.599Z [ERROR] server-f32b41be-316c-b62d-0871-ff2655b5172a: failed to heartbeat to: peer=4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 backoff time=10ms error="failed to connect to peer: 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774" +2025-06-15T03:18:03.601Z [ERROR] server-f32b41be-316c-b62d-0871-ff2655b5172a: failed to appendEntries to: peer="{Voter server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774}" error="failed to connect to peer: 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774" +2025-06-15T03:18:03.618Z [ERROR] server-f32b41be-316c-b62d-0871-ff2655b5172a: failed to heartbeat to: peer=4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 backoff time=10ms error="failed to connect to peer: 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774" +2025-06-15T03:18:03.620Z [WARN] server-f32b41be-316c-b62d-0871-ff2655b5172a: failed to contact: server-id=server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 time=50.11108ms +2025-06-15T03:18:03.620Z [WARN] server-f32b41be-316c-b62d-0871-ff2655b5172a: failed to contact quorum of nodes, stepping down +2025-06-15T03:18:03.620Z [INFO] server-f32b41be-316c-b62d-0871-ff2655b5172a: entering follower state: follower="Node at f32b41be-316c-b62d-0871-ff2655b5172a [Follower]" leader-address= leader-id= +2025-06-15T03:18:03.620Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:03.621Z [ERROR] server-f32b41be-316c-b62d-0871-ff2655b5172a: failed to appendEntries to: peer="{Voter server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774}" error="failed to connect to peer: 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774" +2025-06-15T03:18:03.646Z [WARN] server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774: heartbeat timeout reached, starting election: last-leader-addr=f32b41be-316c-b62d-0871-ff2655b5172a last-leader-id=server-f32b41be-316c-b62d-0871-ff2655b5172a +2025-06-15T03:18:03.646Z [INFO] server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774: entering candidate state: node="Node at 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 [Candidate]" term=3 +2025-06-15T03:18:03.646Z [ERROR] server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774: failed to make requestVote RPC: target="{Voter server-f32b41be-316c-b62d-0871-ff2655b5172a f32b41be-316c-b62d-0871-ff2655b5172a}" error="failed to connect to peer: f32b41be-316c-b62d-0871-ff2655b5172a" term=3 +2025-06-15T03:18:03.681Z [WARN] server-f32b41be-316c-b62d-0871-ff2655b5172a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:03.681Z [INFO] server-f32b41be-316c-b62d-0871-ff2655b5172a: entering candidate state: node="Node at f32b41be-316c-b62d-0871-ff2655b5172a [Candidate]" term=3 +2025-06-15T03:18:03.681Z [ERROR] server-f32b41be-316c-b62d-0871-ff2655b5172a: failed to make requestVote RPC: target="{Voter server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774}" error="failed to connect to peer: 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774" term=3 +2025-06-15T03:18:03.742Z [WARN] server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774: Election timeout reached, restarting election +2025-06-15T03:18:03.742Z [INFO] server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774: entering candidate state: node="Node at 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 [Candidate]" term=4 +2025-06-15T03:18:03.742Z [ERROR] server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774: failed to make requestVote RPC: target="{Voter server-f32b41be-316c-b62d-0871-ff2655b5172a f32b41be-316c-b62d-0871-ff2655b5172a}" error="failed to connect to peer: f32b41be-316c-b62d-0871-ff2655b5172a" term=4 +2025-06-15T03:18:03.757Z [WARN] server-f32b41be-316c-b62d-0871-ff2655b5172a: Election timeout reached, restarting election +2025-06-15T03:18:03.757Z [INFO] server-f32b41be-316c-b62d-0871-ff2655b5172a: entering candidate state: node="Node at f32b41be-316c-b62d-0871-ff2655b5172a [Candidate]" term=4 +2025-06-15T03:18:03.757Z [ERROR] server-f32b41be-316c-b62d-0871-ff2655b5172a: failed to make requestVote RPC: target="{Voter server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774}" error="failed to connect to peer: 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774" term=4 +2025-06-15T03:18:03.786Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:03.681466944 +0000 UTC m=+99.393145105 (0 nodes), 60.618866ms from start of poll, 385.966591ms from cluster start. Timeout at 2025-06-15 03:18:03.786749984 +0000 UTC m=+99.498428145, 105.28304ms after stability +2025-06-15T03:18:03.807Z [WARN] server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774: Election timeout reached, restarting election +2025-06-15T03:18:03.807Z [INFO] server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774: entering candidate state: node="Node at 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 [Candidate]" term=5 +2025-06-15T03:18:03.807Z [ERROR] server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774: failed to make requestVote RPC: target="{Voter server-f32b41be-316c-b62d-0871-ff2655b5172a f32b41be-316c-b62d-0871-ff2655b5172a}" error="failed to connect to peer: f32b41be-316c-b62d-0871-ff2655b5172a" term=5 +2025-06-15T03:18:03.823Z [WARN] server-f32b41be-316c-b62d-0871-ff2655b5172a: Election timeout reached, restarting election +2025-06-15T03:18:03.823Z [INFO] server-f32b41be-316c-b62d-0871-ff2655b5172a: entering candidate state: node="Node at f32b41be-316c-b62d-0871-ff2655b5172a [Candidate]" term=5 +2025-06-15T03:18:03.823Z [ERROR] server-f32b41be-316c-b62d-0871-ff2655b5172a: failed to make requestVote RPC: target="{Voter server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774}" error="failed to connect to peer: 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774" term=5 +2025-06-15T03:18:03.885Z [WARN] server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774: Election timeout reached, restarting election +2025-06-15T03:18:03.885Z [INFO] server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774: entering candidate state: node="Node at 4f14a926-0cc5-8ca6-c56b-74dfb6a5d774 [Candidate]" term=6 +2025-06-15T03:18:03.885Z [ERROR] server-4f14a926-0cc5-8ca6-c56b-74dfb6a5d774: failed to make requestVote RPC: target="{Voter server-f32b41be-316c-b62d-0871-ff2655b5172a f32b41be-316c-b62d-0871-ff2655b5172a}" error="failed to connect to peer: f32b41be-316c-b62d-0871-ff2655b5172a" term=6 +--- PASS: TestRaft_LeaderLeaseExpire (0.60s) === RUN TestRaft_Barrier -2024-05-12T20:51:30.148Z [INFO] server-b6a9fd4c-78a9-56fe-7936-212636021c7a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6a9fd4c-78a9-56fe-7936-212636021c7a Address:b6a9fd4c-78a9-56fe-7936-212636021c7a} {Suffrage:Voter ID:server-4816ce1f-c371-93a6-a9f1-e5d3cf21e259 Address:4816ce1f-c371-93a6-a9f1-e5d3cf21e259} {Suffrage:Voter ID:server-184177d5-c5c4-a6cc-e894-5346be52768c Address:184177d5-c5c4-a6cc-e894-5346be52768c}]" -2024-05-12T20:51:30.148Z [INFO] server-4816ce1f-c371-93a6-a9f1-e5d3cf21e259: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6a9fd4c-78a9-56fe-7936-212636021c7a Address:b6a9fd4c-78a9-56fe-7936-212636021c7a} {Suffrage:Voter ID:server-4816ce1f-c371-93a6-a9f1-e5d3cf21e259 Address:4816ce1f-c371-93a6-a9f1-e5d3cf21e259} {Suffrage:Voter ID:server-184177d5-c5c4-a6cc-e894-5346be52768c Address:184177d5-c5c4-a6cc-e894-5346be52768c}]" -2024-05-12T20:51:30.148Z [INFO] server-184177d5-c5c4-a6cc-e894-5346be52768c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6a9fd4c-78a9-56fe-7936-212636021c7a Address:b6a9fd4c-78a9-56fe-7936-212636021c7a} {Suffrage:Voter ID:server-4816ce1f-c371-93a6-a9f1-e5d3cf21e259 Address:4816ce1f-c371-93a6-a9f1-e5d3cf21e259} {Suffrage:Voter ID:server-184177d5-c5c4-a6cc-e894-5346be52768c Address:184177d5-c5c4-a6cc-e894-5346be52768c}]" -2024-05-12T20:51:30.148Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:30.148Z [INFO] server-b6a9fd4c-78a9-56fe-7936-212636021c7a: entering follower state: follower="Node at b6a9fd4c-78a9-56fe-7936-212636021c7a [Follower]" leader-address= leader-id= -2024-05-12T20:51:30.148Z [INFO] server-4816ce1f-c371-93a6-a9f1-e5d3cf21e259: entering follower state: follower="Node at 4816ce1f-c371-93a6-a9f1-e5d3cf21e259 [Follower]" leader-address= leader-id= -2024-05-12T20:51:30.149Z [INFO] server-184177d5-c5c4-a6cc-e894-5346be52768c: entering follower state: follower="Node at 184177d5-c5c4-a6cc-e894-5346be52768c [Follower]" leader-address= leader-id= -2024-05-12T20:51:30.211Z [WARN] server-b6a9fd4c-78a9-56fe-7936-212636021c7a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:30.211Z [INFO] server-b6a9fd4c-78a9-56fe-7936-212636021c7a: entering candidate state: node="Node at b6a9fd4c-78a9-56fe-7936-212636021c7a [Candidate]" term=2 -2024-05-12T20:51:30.211Z [WARN] server-184177d5-c5c4-a6cc-e894-5346be52768c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:30.211Z [INFO] server-184177d5-c5c4-a6cc-e894-5346be52768c: entering candidate state: node="Node at 184177d5-c5c4-a6cc-e894-5346be52768c [Candidate]" term=2 -2024-05-12T20:51:30.211Z [INFO] server-184177d5-c5c4-a6cc-e894-5346be52768c: duplicate requestVote for same term: term=2 -2024-05-12T20:51:30.211Z [INFO] server-b6a9fd4c-78a9-56fe-7936-212636021c7a: duplicate requestVote for same term: term=2 -2024-05-12T20:51:30.211Z [INFO] server-b6a9fd4c-78a9-56fe-7936-212636021c7a: election won: term=2 tally=2 -2024-05-12T20:51:30.211Z [INFO] server-b6a9fd4c-78a9-56fe-7936-212636021c7a: entering leader state: leader="Node at b6a9fd4c-78a9-56fe-7936-212636021c7a [Leader]" -2024-05-12T20:51:30.211Z [INFO] server-b6a9fd4c-78a9-56fe-7936-212636021c7a: added peer, starting replication: peer=server-4816ce1f-c371-93a6-a9f1-e5d3cf21e259 -2024-05-12T20:51:30.211Z [INFO] server-4816ce1f-c371-93a6-a9f1-e5d3cf21e259: duplicate requestVote for same term: term=2 -2024-05-12T20:51:30.211Z [INFO] server-b6a9fd4c-78a9-56fe-7936-212636021c7a: added peer, starting replication: peer=server-184177d5-c5c4-a6cc-e894-5346be52768c -2024-05-12T20:51:30.212Z [INFO] server-184177d5-c5c4-a6cc-e894-5346be52768c: entering follower state: follower="Node at 184177d5-c5c4-a6cc-e894-5346be52768c [Follower]" leader-address=b6a9fd4c-78a9-56fe-7936-212636021c7a leader-id=server-b6a9fd4c-78a9-56fe-7936-212636021c7a -2024-05-12T20:51:30.212Z [INFO] server-b6a9fd4c-78a9-56fe-7936-212636021c7a: pipelining replication: peer="{Voter server-4816ce1f-c371-93a6-a9f1-e5d3cf21e259 4816ce1f-c371-93a6-a9f1-e5d3cf21e259}" -2024-05-12T20:51:30.212Z [INFO] server-b6a9fd4c-78a9-56fe-7936-212636021c7a: pipelining replication: peer="{Voter server-184177d5-c5c4-a6cc-e894-5346be52768c 184177d5-c5c4-a6cc-e894-5346be52768c}" -2024-05-12T20:51:30.318Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:30.212263621 +0000 UTC m=+101.492964338 (1 nodes), 63.420965ms from start of poll, 63.881634ms from cluster start. Timeout at 2024-05-12 20:51:30.318757794 +0000 UTC m=+101.599458453, 106.494115ms after stability -2024-05-12T20:51:30.343Z [INFO] server-b6a9fd4c-78a9-56fe-7936-212636021c7a: aborting pipeline replication: peer="{Voter server-184177d5-c5c4-a6cc-e894-5346be52768c 184177d5-c5c4-a6cc-e894-5346be52768c}" -2024-05-12T20:51:30.343Z [INFO] server-b6a9fd4c-78a9-56fe-7936-212636021c7a: aborting pipeline replication: peer="{Voter server-4816ce1f-c371-93a6-a9f1-e5d3cf21e259 4816ce1f-c371-93a6-a9f1-e5d3cf21e259}" +2025-06-15T03:18:03.892Z [INFO] server-2337f367-8d89-e258-15f1-779d9584e326: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2337f367-8d89-e258-15f1-779d9584e326 Address:2337f367-8d89-e258-15f1-779d9584e326} {Suffrage:Voter ID:server-0a0a9ed6-cb90-977a-d8d4-2e2033b30e7b Address:0a0a9ed6-cb90-977a-d8d4-2e2033b30e7b} {Suffrage:Voter ID:server-d814a6a0-62b7-004d-aa29-233d57e02a1b Address:d814a6a0-62b7-004d-aa29-233d57e02a1b}]" +2025-06-15T03:18:03.892Z [INFO] server-0a0a9ed6-cb90-977a-d8d4-2e2033b30e7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2337f367-8d89-e258-15f1-779d9584e326 Address:2337f367-8d89-e258-15f1-779d9584e326} {Suffrage:Voter ID:server-0a0a9ed6-cb90-977a-d8d4-2e2033b30e7b Address:0a0a9ed6-cb90-977a-d8d4-2e2033b30e7b} {Suffrage:Voter ID:server-d814a6a0-62b7-004d-aa29-233d57e02a1b Address:d814a6a0-62b7-004d-aa29-233d57e02a1b}]" +2025-06-15T03:18:03.892Z [INFO] server-2337f367-8d89-e258-15f1-779d9584e326: entering follower state: follower="Node at 2337f367-8d89-e258-15f1-779d9584e326 [Follower]" leader-address= leader-id= +2025-06-15T03:18:03.893Z [INFO] server-0a0a9ed6-cb90-977a-d8d4-2e2033b30e7b: entering follower state: follower="Node at 0a0a9ed6-cb90-977a-d8d4-2e2033b30e7b [Follower]" leader-address= leader-id= +2025-06-15T03:18:03.893Z [INFO] server-d814a6a0-62b7-004d-aa29-233d57e02a1b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2337f367-8d89-e258-15f1-779d9584e326 Address:2337f367-8d89-e258-15f1-779d9584e326} {Suffrage:Voter ID:server-0a0a9ed6-cb90-977a-d8d4-2e2033b30e7b Address:0a0a9ed6-cb90-977a-d8d4-2e2033b30e7b} {Suffrage:Voter ID:server-d814a6a0-62b7-004d-aa29-233d57e02a1b Address:d814a6a0-62b7-004d-aa29-233d57e02a1b}]" +2025-06-15T03:18:03.893Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:03.893Z [INFO] server-d814a6a0-62b7-004d-aa29-233d57e02a1b: entering follower state: follower="Node at d814a6a0-62b7-004d-aa29-233d57e02a1b [Follower]" leader-address= leader-id= +2025-06-15T03:18:03.965Z [WARN] server-2337f367-8d89-e258-15f1-779d9584e326: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:03.965Z [INFO] server-2337f367-8d89-e258-15f1-779d9584e326: entering candidate state: node="Node at 2337f367-8d89-e258-15f1-779d9584e326 [Candidate]" term=2 +2025-06-15T03:18:03.965Z [INFO] server-2337f367-8d89-e258-15f1-779d9584e326: election won: term=2 tally=2 +2025-06-15T03:18:03.965Z [INFO] server-2337f367-8d89-e258-15f1-779d9584e326: entering leader state: leader="Node at 2337f367-8d89-e258-15f1-779d9584e326 [Leader]" +2025-06-15T03:18:03.965Z [INFO] server-2337f367-8d89-e258-15f1-779d9584e326: added peer, starting replication: peer=server-0a0a9ed6-cb90-977a-d8d4-2e2033b30e7b +2025-06-15T03:18:03.965Z [INFO] server-2337f367-8d89-e258-15f1-779d9584e326: added peer, starting replication: peer=server-d814a6a0-62b7-004d-aa29-233d57e02a1b +2025-06-15T03:18:03.965Z [INFO] server-2337f367-8d89-e258-15f1-779d9584e326: pipelining replication: peer="{Voter server-0a0a9ed6-cb90-977a-d8d4-2e2033b30e7b 0a0a9ed6-cb90-977a-d8d4-2e2033b30e7b}" +2025-06-15T03:18:03.965Z [INFO] server-2337f367-8d89-e258-15f1-779d9584e326: pipelining replication: peer="{Voter server-d814a6a0-62b7-004d-aa29-233d57e02a1b d814a6a0-62b7-004d-aa29-233d57e02a1b}" +2025-06-15T03:18:04.071Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:03.965916711 +0000 UTC m=+99.677594877 (1 nodes), 72.742568ms from start of poll, 73.135346ms from cluster start. Timeout at 2025-06-15 03:18:04.07185805 +0000 UTC m=+99.783536215, 105.941338ms after stability +2025-06-15T03:18:04.088Z [INFO] server-2337f367-8d89-e258-15f1-779d9584e326: aborting pipeline replication: peer="{Voter server-d814a6a0-62b7-004d-aa29-233d57e02a1b d814a6a0-62b7-004d-aa29-233d57e02a1b}" +2025-06-15T03:18:04.088Z [INFO] server-2337f367-8d89-e258-15f1-779d9584e326: aborting pipeline replication: peer="{Voter server-0a0a9ed6-cb90-977a-d8d4-2e2033b30e7b 0a0a9ed6-cb90-977a-d8d4-2e2033b30e7b}" --- PASS: TestRaft_Barrier (0.20s) === RUN TestRaft_VerifyLeader -2024-05-12T20:51:30.344Z [INFO] server-dcffbbd2-3c4f-4170-056f-1b4dbab71530: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dcffbbd2-3c4f-4170-056f-1b4dbab71530 Address:dcffbbd2-3c4f-4170-056f-1b4dbab71530} {Suffrage:Voter ID:server-1d9dc648-49a4-86a5-9380-dd2d416bac4b Address:1d9dc648-49a4-86a5-9380-dd2d416bac4b} {Suffrage:Voter ID:server-e2a7faee-ee25-c533-93b7-b307efad31ea Address:e2a7faee-ee25-c533-93b7-b307efad31ea}]" -2024-05-12T20:51:30.345Z [INFO] server-1d9dc648-49a4-86a5-9380-dd2d416bac4b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dcffbbd2-3c4f-4170-056f-1b4dbab71530 Address:dcffbbd2-3c4f-4170-056f-1b4dbab71530} {Suffrage:Voter ID:server-1d9dc648-49a4-86a5-9380-dd2d416bac4b Address:1d9dc648-49a4-86a5-9380-dd2d416bac4b} {Suffrage:Voter ID:server-e2a7faee-ee25-c533-93b7-b307efad31ea Address:e2a7faee-ee25-c533-93b7-b307efad31ea}]" -2024-05-12T20:51:30.345Z [INFO] server-e2a7faee-ee25-c533-93b7-b307efad31ea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dcffbbd2-3c4f-4170-056f-1b4dbab71530 Address:dcffbbd2-3c4f-4170-056f-1b4dbab71530} {Suffrage:Voter ID:server-1d9dc648-49a4-86a5-9380-dd2d416bac4b Address:1d9dc648-49a4-86a5-9380-dd2d416bac4b} {Suffrage:Voter ID:server-e2a7faee-ee25-c533-93b7-b307efad31ea Address:e2a7faee-ee25-c533-93b7-b307efad31ea}]" -2024-05-12T20:51:30.345Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:30.345Z [INFO] server-dcffbbd2-3c4f-4170-056f-1b4dbab71530: entering follower state: follower="Node at dcffbbd2-3c4f-4170-056f-1b4dbab71530 [Follower]" leader-address= leader-id= -2024-05-12T20:51:30.345Z [INFO] server-1d9dc648-49a4-86a5-9380-dd2d416bac4b: entering follower state: follower="Node at 1d9dc648-49a4-86a5-9380-dd2d416bac4b [Follower]" leader-address= leader-id= -2024-05-12T20:51:30.345Z [INFO] server-e2a7faee-ee25-c533-93b7-b307efad31ea: entering follower state: follower="Node at e2a7faee-ee25-c533-93b7-b307efad31ea [Follower]" leader-address= leader-id= -2024-05-12T20:51:30.418Z [WARN] server-e2a7faee-ee25-c533-93b7-b307efad31ea: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:30.418Z [INFO] server-e2a7faee-ee25-c533-93b7-b307efad31ea: entering candidate state: node="Node at e2a7faee-ee25-c533-93b7-b307efad31ea [Candidate]" term=2 -2024-05-12T20:51:30.419Z [INFO] server-e2a7faee-ee25-c533-93b7-b307efad31ea: election won: term=2 tally=2 -2024-05-12T20:51:30.419Z [INFO] server-e2a7faee-ee25-c533-93b7-b307efad31ea: entering leader state: leader="Node at e2a7faee-ee25-c533-93b7-b307efad31ea [Leader]" -2024-05-12T20:51:30.419Z [INFO] server-e2a7faee-ee25-c533-93b7-b307efad31ea: added peer, starting replication: peer=server-dcffbbd2-3c4f-4170-056f-1b4dbab71530 -2024-05-12T20:51:30.419Z [INFO] server-e2a7faee-ee25-c533-93b7-b307efad31ea: added peer, starting replication: peer=server-1d9dc648-49a4-86a5-9380-dd2d416bac4b -2024-05-12T20:51:30.419Z [INFO] server-e2a7faee-ee25-c533-93b7-b307efad31ea: pipelining replication: peer="{Voter server-1d9dc648-49a4-86a5-9380-dd2d416bac4b 1d9dc648-49a4-86a5-9380-dd2d416bac4b}" -2024-05-12T20:51:30.419Z [INFO] server-e2a7faee-ee25-c533-93b7-b307efad31ea: pipelining replication: peer="{Voter server-dcffbbd2-3c4f-4170-056f-1b4dbab71530 dcffbbd2-3c4f-4170-056f-1b4dbab71530}" -2024-05-12T20:51:30.528Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:30.419467118 +0000 UTC m=+101.700167013 (1 nodes), 74.256733ms from start of poll, 74.686482ms from cluster start. Timeout at 2024-05-12 20:51:30.528106835 +0000 UTC m=+101.808807167, 108.640154ms after stability -2024-05-12T20:51:30.528Z [INFO] server-e2a7faee-ee25-c533-93b7-b307efad31ea: aborting pipeline replication: peer="{Voter server-1d9dc648-49a4-86a5-9380-dd2d416bac4b 1d9dc648-49a4-86a5-9380-dd2d416bac4b}" -2024-05-12T20:51:30.528Z [INFO] server-e2a7faee-ee25-c533-93b7-b307efad31ea: aborting pipeline replication: peer="{Voter server-dcffbbd2-3c4f-4170-056f-1b4dbab71530 dcffbbd2-3c4f-4170-056f-1b4dbab71530}" -2024-05-12T20:51:31.031Z [ERROR] server-e2a7faee-ee25-c533-93b7-b307efad31ea: failed to heartbeat to: peer=dcffbbd2-3c4f-4170-056f-1b4dbab71530 backoff time=10ms error="command timed out" ---- PASS: TestRaft_VerifyLeader (0.69s) +2025-06-15T03:18:04.089Z [INFO] server-b19e8460-8717-d188-aed2-4432c5d83525: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b19e8460-8717-d188-aed2-4432c5d83525 Address:b19e8460-8717-d188-aed2-4432c5d83525} {Suffrage:Voter ID:server-8dcf0522-b4a7-175c-689f-efd2b889c15b Address:8dcf0522-b4a7-175c-689f-efd2b889c15b} {Suffrage:Voter ID:server-68e420a9-bf54-4e0f-7a93-8f60c3136d7a Address:68e420a9-bf54-4e0f-7a93-8f60c3136d7a}]" +2025-06-15T03:18:04.089Z [INFO] server-b19e8460-8717-d188-aed2-4432c5d83525: entering follower state: follower="Node at b19e8460-8717-d188-aed2-4432c5d83525 [Follower]" leader-address= leader-id= +2025-06-15T03:18:04.089Z [INFO] server-8dcf0522-b4a7-175c-689f-efd2b889c15b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b19e8460-8717-d188-aed2-4432c5d83525 Address:b19e8460-8717-d188-aed2-4432c5d83525} {Suffrage:Voter ID:server-8dcf0522-b4a7-175c-689f-efd2b889c15b Address:8dcf0522-b4a7-175c-689f-efd2b889c15b} {Suffrage:Voter ID:server-68e420a9-bf54-4e0f-7a93-8f60c3136d7a Address:68e420a9-bf54-4e0f-7a93-8f60c3136d7a}]" +2025-06-15T03:18:04.089Z [INFO] server-8dcf0522-b4a7-175c-689f-efd2b889c15b: entering follower state: follower="Node at 8dcf0522-b4a7-175c-689f-efd2b889c15b [Follower]" leader-address= leader-id= +2025-06-15T03:18:04.089Z [INFO] server-68e420a9-bf54-4e0f-7a93-8f60c3136d7a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b19e8460-8717-d188-aed2-4432c5d83525 Address:b19e8460-8717-d188-aed2-4432c5d83525} {Suffrage:Voter ID:server-8dcf0522-b4a7-175c-689f-efd2b889c15b Address:8dcf0522-b4a7-175c-689f-efd2b889c15b} {Suffrage:Voter ID:server-68e420a9-bf54-4e0f-7a93-8f60c3136d7a Address:68e420a9-bf54-4e0f-7a93-8f60c3136d7a}]" +2025-06-15T03:18:04.089Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:04.089Z [INFO] server-68e420a9-bf54-4e0f-7a93-8f60c3136d7a: entering follower state: follower="Node at 68e420a9-bf54-4e0f-7a93-8f60c3136d7a [Follower]" leader-address= leader-id= +2025-06-15T03:18:04.156Z [WARN] server-68e420a9-bf54-4e0f-7a93-8f60c3136d7a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:04.156Z [INFO] server-68e420a9-bf54-4e0f-7a93-8f60c3136d7a: entering candidate state: node="Node at 68e420a9-bf54-4e0f-7a93-8f60c3136d7a [Candidate]" term=2 +2025-06-15T03:18:04.156Z [INFO] server-68e420a9-bf54-4e0f-7a93-8f60c3136d7a: election won: term=2 tally=2 +2025-06-15T03:18:04.156Z [INFO] server-68e420a9-bf54-4e0f-7a93-8f60c3136d7a: entering leader state: leader="Node at 68e420a9-bf54-4e0f-7a93-8f60c3136d7a [Leader]" +2025-06-15T03:18:04.156Z [INFO] server-68e420a9-bf54-4e0f-7a93-8f60c3136d7a: added peer, starting replication: peer=server-b19e8460-8717-d188-aed2-4432c5d83525 +2025-06-15T03:18:04.156Z [INFO] server-68e420a9-bf54-4e0f-7a93-8f60c3136d7a: added peer, starting replication: peer=server-8dcf0522-b4a7-175c-689f-efd2b889c15b +2025-06-15T03:18:04.156Z [INFO] server-68e420a9-bf54-4e0f-7a93-8f60c3136d7a: pipelining replication: peer="{Voter server-8dcf0522-b4a7-175c-689f-efd2b889c15b 8dcf0522-b4a7-175c-689f-efd2b889c15b}" +2025-06-15T03:18:04.156Z [INFO] server-68e420a9-bf54-4e0f-7a93-8f60c3136d7a: pipelining replication: peer="{Voter server-b19e8460-8717-d188-aed2-4432c5d83525 b19e8460-8717-d188-aed2-4432c5d83525}" +2025-06-15T03:18:04.262Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:04.156696011 +0000 UTC m=+99.868374251 (1 nodes), 67.026864ms from start of poll, 67.492555ms from cluster start. Timeout at 2025-06-15 03:18:04.262079908 +0000 UTC m=+99.973758073, 105.383822ms after stability +2025-06-15T03:18:04.262Z [INFO] server-68e420a9-bf54-4e0f-7a93-8f60c3136d7a: aborting pipeline replication: peer="{Voter server-8dcf0522-b4a7-175c-689f-efd2b889c15b 8dcf0522-b4a7-175c-689f-efd2b889c15b}" +2025-06-15T03:18:04.262Z [INFO] server-68e420a9-bf54-4e0f-7a93-8f60c3136d7a: aborting pipeline replication: peer="{Voter server-b19e8460-8717-d188-aed2-4432c5d83525 b19e8460-8717-d188-aed2-4432c5d83525}" +2025-06-15T03:18:04.762Z [ERROR] server-68e420a9-bf54-4e0f-7a93-8f60c3136d7a: failed to heartbeat to: peer=b19e8460-8717-d188-aed2-4432c5d83525 backoff time=10ms error="command timed out" +--- PASS: TestRaft_VerifyLeader (0.68s) === RUN TestRaft_VerifyLeader_Single -2024-05-12T20:51:31.032Z [INFO] server-bc734c3c-c17a-1c86-fe18-c1a2815045e8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc734c3c-c17a-1c86-fe18-c1a2815045e8 Address:bc734c3c-c17a-1c86-fe18-c1a2815045e8}]" -2024-05-12T20:51:31.032Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:31.032Z [INFO] server-bc734c3c-c17a-1c86-fe18-c1a2815045e8: entering follower state: follower="Node at bc734c3c-c17a-1c86-fe18-c1a2815045e8 [Follower]" leader-address= leader-id= -2024-05-12T20:51:31.086Z [WARN] server-bc734c3c-c17a-1c86-fe18-c1a2815045e8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:31.087Z [INFO] server-bc734c3c-c17a-1c86-fe18-c1a2815045e8: entering candidate state: node="Node at bc734c3c-c17a-1c86-fe18-c1a2815045e8 [Candidate]" term=2 -2024-05-12T20:51:31.087Z [INFO] server-bc734c3c-c17a-1c86-fe18-c1a2815045e8: election won: term=2 tally=1 -2024-05-12T20:51:31.087Z [INFO] server-bc734c3c-c17a-1c86-fe18-c1a2815045e8: entering leader state: leader="Node at bc734c3c-c17a-1c86-fe18-c1a2815045e8 [Leader]" -2024-05-12T20:51:31.199Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:31.087124886 +0000 UTC m=+102.367825259 (1 nodes), 54.354016ms from start of poll, 54.551547ms from cluster start. Timeout at 2024-05-12 20:51:31.199196943 +0000 UTC m=+102.479897584, 112.072325ms after stability ---- PASS: TestRaft_VerifyLeader_Single (0.17s) +2025-06-15T03:18:04.765Z [INFO] server-084cac33-a546-8e52-ba51-601a1c452729: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-084cac33-a546-8e52-ba51-601a1c452729 Address:084cac33-a546-8e52-ba51-601a1c452729}]" +2025-06-15T03:18:04.765Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:04.765Z [INFO] server-084cac33-a546-8e52-ba51-601a1c452729: entering follower state: follower="Node at 084cac33-a546-8e52-ba51-601a1c452729 [Follower]" leader-address= leader-id= +2025-06-15T03:18:04.845Z [WARN] server-084cac33-a546-8e52-ba51-601a1c452729: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:04.845Z [INFO] server-084cac33-a546-8e52-ba51-601a1c452729: entering candidate state: node="Node at 084cac33-a546-8e52-ba51-601a1c452729 [Candidate]" term=2 +2025-06-15T03:18:04.845Z [INFO] server-084cac33-a546-8e52-ba51-601a1c452729: election won: term=2 tally=1 +2025-06-15T03:18:04.845Z [INFO] server-084cac33-a546-8e52-ba51-601a1c452729: entering leader state: leader="Node at 084cac33-a546-8e52-ba51-601a1c452729 [Leader]" +2025-06-15T03:18:04.951Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:04.845982561 +0000 UTC m=+100.557660739 (1 nodes), 80.538303ms from start of poll, 80.702554ms from cluster start. Timeout at 2025-06-15 03:18:04.951509562 +0000 UTC m=+100.663187761, 105.527022ms after stability +--- PASS: TestRaft_VerifyLeader_Single (0.19s) === RUN TestRaft_VerifyLeader_Fail -2024-05-12T20:51:31.200Z [INFO] server-7dcef25b-45eb-895c-9727-87d4846851f4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7dcef25b-45eb-895c-9727-87d4846851f4 Address:7dcef25b-45eb-895c-9727-87d4846851f4} {Suffrage:Voter ID:server-5179e600-6b1e-4e32-2713-1cf6fd2ef7c6 Address:5179e600-6b1e-4e32-2713-1cf6fd2ef7c6}]" -2024-05-12T20:51:31.200Z [INFO] server-5179e600-6b1e-4e32-2713-1cf6fd2ef7c6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7dcef25b-45eb-895c-9727-87d4846851f4 Address:7dcef25b-45eb-895c-9727-87d4846851f4} {Suffrage:Voter ID:server-5179e600-6b1e-4e32-2713-1cf6fd2ef7c6 Address:5179e600-6b1e-4e32-2713-1cf6fd2ef7c6}]" -2024-05-12T20:51:31.200Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:31.200Z [INFO] server-7dcef25b-45eb-895c-9727-87d4846851f4: entering follower state: follower="Node at 7dcef25b-45eb-895c-9727-87d4846851f4 [Follower]" leader-address= leader-id= -2024-05-12T20:51:31.200Z [INFO] server-5179e600-6b1e-4e32-2713-1cf6fd2ef7c6: entering follower state: follower="Node at 5179e600-6b1e-4e32-2713-1cf6fd2ef7c6 [Follower]" leader-address= leader-id= -2024-05-12T20:51:31.254Z [WARN] server-5179e600-6b1e-4e32-2713-1cf6fd2ef7c6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:31.254Z [INFO] server-5179e600-6b1e-4e32-2713-1cf6fd2ef7c6: entering candidate state: node="Node at 5179e600-6b1e-4e32-2713-1cf6fd2ef7c6 [Candidate]" term=2 -2024-05-12T20:51:31.254Z [INFO] server-5179e600-6b1e-4e32-2713-1cf6fd2ef7c6: election won: term=2 tally=2 -2024-05-12T20:51:31.254Z [INFO] server-5179e600-6b1e-4e32-2713-1cf6fd2ef7c6: entering leader state: leader="Node at 5179e600-6b1e-4e32-2713-1cf6fd2ef7c6 [Leader]" -2024-05-12T20:51:31.255Z [INFO] server-5179e600-6b1e-4e32-2713-1cf6fd2ef7c6: added peer, starting replication: peer=server-7dcef25b-45eb-895c-9727-87d4846851f4 -2024-05-12T20:51:31.255Z [INFO] server-5179e600-6b1e-4e32-2713-1cf6fd2ef7c6: pipelining replication: peer="{Voter server-7dcef25b-45eb-895c-9727-87d4846851f4 7dcef25b-45eb-895c-9727-87d4846851f4}" -2024-05-12T20:51:31.362Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:31.255074652 +0000 UTC m=+102.535774547 (1 nodes), 54.319095ms from start of poll, 54.635698ms from cluster start. Timeout at 2024-05-12 20:51:31.362776891 +0000 UTC m=+102.643476786, 107.702239ms after stability -2024-05-12T20:51:31.362Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:31.470Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:31.362905206 +0000 UTC m=+102.643605101 (1 nodes), 0s from start of poll, 162.466252ms from cluster start. Timeout at 2024-05-12 20:51:31.470773966 +0000 UTC m=+102.751473861, 107.86876ms after stability -2024-05-12T20:51:31.471Z [ERROR] server-5179e600-6b1e-4e32-2713-1cf6fd2ef7c6: peer has newer term, stopping replication: peer="{Voter server-7dcef25b-45eb-895c-9727-87d4846851f4 7dcef25b-45eb-895c-9727-87d4846851f4}" -2024-05-12T20:51:31.471Z [INFO] server-5179e600-6b1e-4e32-2713-1cf6fd2ef7c6: aborting pipeline replication: peer="{Voter server-7dcef25b-45eb-895c-9727-87d4846851f4 7dcef25b-45eb-895c-9727-87d4846851f4}" -2024-05-12T20:51:31.471Z [INFO] server-5179e600-6b1e-4e32-2713-1cf6fd2ef7c6: entering follower state: follower="Node at 5179e600-6b1e-4e32-2713-1cf6fd2ef7c6 [Follower]" leader-address= leader-id= +2025-06-15T03:18:04.952Z [INFO] server-27abbe71-b5b0-aedf-a076-c6e97493a8a3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27abbe71-b5b0-aedf-a076-c6e97493a8a3 Address:27abbe71-b5b0-aedf-a076-c6e97493a8a3} {Suffrage:Voter ID:server-4b76a548-b296-a871-d454-2a36c352ab42 Address:4b76a548-b296-a871-d454-2a36c352ab42}]" +2025-06-15T03:18:04.952Z [INFO] server-27abbe71-b5b0-aedf-a076-c6e97493a8a3: entering follower state: follower="Node at 27abbe71-b5b0-aedf-a076-c6e97493a8a3 [Follower]" leader-address= leader-id= +2025-06-15T03:18:04.952Z [INFO] server-4b76a548-b296-a871-d454-2a36c352ab42: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27abbe71-b5b0-aedf-a076-c6e97493a8a3 Address:27abbe71-b5b0-aedf-a076-c6e97493a8a3} {Suffrage:Voter ID:server-4b76a548-b296-a871-d454-2a36c352ab42 Address:4b76a548-b296-a871-d454-2a36c352ab42}]" +2025-06-15T03:18:04.952Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:04.952Z [INFO] server-4b76a548-b296-a871-d454-2a36c352ab42: entering follower state: follower="Node at 4b76a548-b296-a871-d454-2a36c352ab42 [Follower]" leader-address= leader-id= +2025-06-15T03:18:05.008Z [WARN] server-4b76a548-b296-a871-d454-2a36c352ab42: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:05.008Z [INFO] server-4b76a548-b296-a871-d454-2a36c352ab42: entering candidate state: node="Node at 4b76a548-b296-a871-d454-2a36c352ab42 [Candidate]" term=2 +2025-06-15T03:18:05.008Z [INFO] server-4b76a548-b296-a871-d454-2a36c352ab42: election won: term=2 tally=2 +2025-06-15T03:18:05.008Z [INFO] server-4b76a548-b296-a871-d454-2a36c352ab42: entering leader state: leader="Node at 4b76a548-b296-a871-d454-2a36c352ab42 [Leader]" +2025-06-15T03:18:05.008Z [INFO] server-4b76a548-b296-a871-d454-2a36c352ab42: added peer, starting replication: peer=server-27abbe71-b5b0-aedf-a076-c6e97493a8a3 +2025-06-15T03:18:05.009Z [INFO] server-4b76a548-b296-a871-d454-2a36c352ab42: pipelining replication: peer="{Voter server-27abbe71-b5b0-aedf-a076-c6e97493a8a3 27abbe71-b5b0-aedf-a076-c6e97493a8a3}" +2025-06-15T03:18:05.113Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:05.008952362 +0000 UTC m=+100.720630561 (1 nodes), 56.616647ms from start of poll, 56.809326ms from cluster start. Timeout at 2025-06-15 03:18:05.113967395 +0000 UTC m=+100.825645560, 105.014999ms after stability +2025-06-15T03:18:05.113Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:05.219Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:05.114001858 +0000 UTC m=+100.825680020 (1 nodes), 499ns from start of poll, 161.858785ms from cluster start. Timeout at 2025-06-15 03:18:05.219367465 +0000 UTC m=+100.931045664, 105.365644ms after stability +2025-06-15T03:18:05.223Z [ERROR] server-4b76a548-b296-a871-d454-2a36c352ab42: peer has newer term, stopping replication: peer="{Voter server-27abbe71-b5b0-aedf-a076-c6e97493a8a3 27abbe71-b5b0-aedf-a076-c6e97493a8a3}" +2025-06-15T03:18:05.223Z [INFO] server-4b76a548-b296-a871-d454-2a36c352ab42: aborting pipeline replication: peer="{Voter server-27abbe71-b5b0-aedf-a076-c6e97493a8a3 27abbe71-b5b0-aedf-a076-c6e97493a8a3}" +2025-06-15T03:18:05.223Z [INFO] server-4b76a548-b296-a871-d454-2a36c352ab42: entering follower state: follower="Node at 4b76a548-b296-a871-d454-2a36c352ab42 [Follower]" leader-address= leader-id= --- PASS: TestRaft_VerifyLeader_Fail (0.27s) === RUN TestRaft_VerifyLeader_PartialConnect -2024-05-12T20:51:31.472Z [INFO] server-cd1ffa9d-6408-111d-3941-cccbddd03e53: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd1ffa9d-6408-111d-3941-cccbddd03e53 Address:cd1ffa9d-6408-111d-3941-cccbddd03e53} {Suffrage:Voter ID:server-7a2bf347-ece8-1061-f643-ce6b5acb578a Address:7a2bf347-ece8-1061-f643-ce6b5acb578a} {Suffrage:Voter ID:server-571be5f4-57c6-49ea-4d73-17c6e17acffe Address:571be5f4-57c6-49ea-4d73-17c6e17acffe}]" -2024-05-12T20:51:31.472Z [INFO] server-7a2bf347-ece8-1061-f643-ce6b5acb578a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd1ffa9d-6408-111d-3941-cccbddd03e53 Address:cd1ffa9d-6408-111d-3941-cccbddd03e53} {Suffrage:Voter ID:server-7a2bf347-ece8-1061-f643-ce6b5acb578a Address:7a2bf347-ece8-1061-f643-ce6b5acb578a} {Suffrage:Voter ID:server-571be5f4-57c6-49ea-4d73-17c6e17acffe Address:571be5f4-57c6-49ea-4d73-17c6e17acffe}]" -2024-05-12T20:51:31.473Z [INFO] server-571be5f4-57c6-49ea-4d73-17c6e17acffe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd1ffa9d-6408-111d-3941-cccbddd03e53 Address:cd1ffa9d-6408-111d-3941-cccbddd03e53} {Suffrage:Voter ID:server-7a2bf347-ece8-1061-f643-ce6b5acb578a Address:7a2bf347-ece8-1061-f643-ce6b5acb578a} {Suffrage:Voter ID:server-571be5f4-57c6-49ea-4d73-17c6e17acffe Address:571be5f4-57c6-49ea-4d73-17c6e17acffe}]" -2024-05-12T20:51:31.473Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:31.473Z [INFO] server-cd1ffa9d-6408-111d-3941-cccbddd03e53: entering follower state: follower="Node at cd1ffa9d-6408-111d-3941-cccbddd03e53 [Follower]" leader-address= leader-id= -2024-05-12T20:51:31.473Z [INFO] server-7a2bf347-ece8-1061-f643-ce6b5acb578a: entering follower state: follower="Node at 7a2bf347-ece8-1061-f643-ce6b5acb578a [Follower]" leader-address= leader-id= -2024-05-12T20:51:31.473Z [INFO] server-571be5f4-57c6-49ea-4d73-17c6e17acffe: entering follower state: follower="Node at 571be5f4-57c6-49ea-4d73-17c6e17acffe [Follower]" leader-address= leader-id= -2024-05-12T20:51:31.526Z [WARN] server-571be5f4-57c6-49ea-4d73-17c6e17acffe: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:31.526Z [INFO] server-571be5f4-57c6-49ea-4d73-17c6e17acffe: entering candidate state: node="Node at 571be5f4-57c6-49ea-4d73-17c6e17acffe [Candidate]" term=2 -2024-05-12T20:51:31.527Z [INFO] server-571be5f4-57c6-49ea-4d73-17c6e17acffe: election won: term=2 tally=2 -2024-05-12T20:51:31.527Z [INFO] server-571be5f4-57c6-49ea-4d73-17c6e17acffe: entering leader state: leader="Node at 571be5f4-57c6-49ea-4d73-17c6e17acffe [Leader]" -2024-05-12T20:51:31.527Z [INFO] server-571be5f4-57c6-49ea-4d73-17c6e17acffe: added peer, starting replication: peer=server-cd1ffa9d-6408-111d-3941-cccbddd03e53 -2024-05-12T20:51:31.527Z [INFO] server-571be5f4-57c6-49ea-4d73-17c6e17acffe: added peer, starting replication: peer=server-7a2bf347-ece8-1061-f643-ce6b5acb578a -2024-05-12T20:51:31.527Z [INFO] server-571be5f4-57c6-49ea-4d73-17c6e17acffe: pipelining replication: peer="{Voter server-cd1ffa9d-6408-111d-3941-cccbddd03e53 cd1ffa9d-6408-111d-3941-cccbddd03e53}" -2024-05-12T20:51:31.527Z [INFO] server-571be5f4-57c6-49ea-4d73-17c6e17acffe: pipelining replication: peer="{Voter server-7a2bf347-ece8-1061-f643-ce6b5acb578a 7a2bf347-ece8-1061-f643-ce6b5acb578a}" -2024-05-12T20:51:31.634Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:31.527262149 +0000 UTC m=+102.807962044 (1 nodes), 54.180238ms from start of poll, 54.612705ms from cluster start. Timeout at 2024-05-12 20:51:31.634789229 +0000 UTC m=+102.915492775, 107.530731ms after stability -2024-05-12T20:51:31.643Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:31.750Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:31.643265841 +0000 UTC m=+102.923966157 (2 nodes), 2.854µs from start of poll, 170.616818ms from cluster start. Timeout at 2024-05-12 20:51:31.750764724 +0000 UTC m=+103.031464619, 107.498462ms after stability - raft_test.go:1968: [INFO] Disconnecting Node at cd1ffa9d-6408-111d-3941-cccbddd03e53 [Follower] -2024-05-12T20:51:31.751Z [ERROR] server-571be5f4-57c6-49ea-4d73-17c6e17acffe: failed to heartbeat to: peer=cd1ffa9d-6408-111d-3941-cccbddd03e53 backoff time=10ms error="failed to connect to peer: cd1ffa9d-6408-111d-3941-cccbddd03e53" -2024-05-12T20:51:31.751Z [INFO] server-571be5f4-57c6-49ea-4d73-17c6e17acffe: aborting pipeline replication: peer="{Voter server-7a2bf347-ece8-1061-f643-ce6b5acb578a 7a2bf347-ece8-1061-f643-ce6b5acb578a}" -2024-05-12T20:51:31.751Z [INFO] server-571be5f4-57c6-49ea-4d73-17c6e17acffe: aborting pipeline replication: peer="{Voter server-cd1ffa9d-6408-111d-3941-cccbddd03e53 cd1ffa9d-6408-111d-3941-cccbddd03e53}" +2025-06-15T03:18:05.224Z [INFO] server-8eb4c1bc-c830-de77-ab2d-096586fadf10: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8eb4c1bc-c830-de77-ab2d-096586fadf10 Address:8eb4c1bc-c830-de77-ab2d-096586fadf10} {Suffrage:Voter ID:server-776faae7-5b54-8723-4588-fbc834faeac5 Address:776faae7-5b54-8723-4588-fbc834faeac5} {Suffrage:Voter ID:server-e21aea6e-16fd-95e7-de6a-4d3233e7e811 Address:e21aea6e-16fd-95e7-de6a-4d3233e7e811}]" +2025-06-15T03:18:05.224Z [INFO] server-8eb4c1bc-c830-de77-ab2d-096586fadf10: entering follower state: follower="Node at 8eb4c1bc-c830-de77-ab2d-096586fadf10 [Follower]" leader-address= leader-id= +2025-06-15T03:18:05.224Z [INFO] server-776faae7-5b54-8723-4588-fbc834faeac5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8eb4c1bc-c830-de77-ab2d-096586fadf10 Address:8eb4c1bc-c830-de77-ab2d-096586fadf10} {Suffrage:Voter ID:server-776faae7-5b54-8723-4588-fbc834faeac5 Address:776faae7-5b54-8723-4588-fbc834faeac5} {Suffrage:Voter ID:server-e21aea6e-16fd-95e7-de6a-4d3233e7e811 Address:e21aea6e-16fd-95e7-de6a-4d3233e7e811}]" +2025-06-15T03:18:05.224Z [INFO] server-e21aea6e-16fd-95e7-de6a-4d3233e7e811: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8eb4c1bc-c830-de77-ab2d-096586fadf10 Address:8eb4c1bc-c830-de77-ab2d-096586fadf10} {Suffrage:Voter ID:server-776faae7-5b54-8723-4588-fbc834faeac5 Address:776faae7-5b54-8723-4588-fbc834faeac5} {Suffrage:Voter ID:server-e21aea6e-16fd-95e7-de6a-4d3233e7e811 Address:e21aea6e-16fd-95e7-de6a-4d3233e7e811}]" +2025-06-15T03:18:05.224Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:05.224Z [INFO] server-776faae7-5b54-8723-4588-fbc834faeac5: entering follower state: follower="Node at 776faae7-5b54-8723-4588-fbc834faeac5 [Follower]" leader-address= leader-id= +2025-06-15T03:18:05.224Z [INFO] server-e21aea6e-16fd-95e7-de6a-4d3233e7e811: entering follower state: follower="Node at e21aea6e-16fd-95e7-de6a-4d3233e7e811 [Follower]" leader-address= leader-id= +2025-06-15T03:18:05.286Z [WARN] server-e21aea6e-16fd-95e7-de6a-4d3233e7e811: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:05.286Z [INFO] server-e21aea6e-16fd-95e7-de6a-4d3233e7e811: entering candidate state: node="Node at e21aea6e-16fd-95e7-de6a-4d3233e7e811 [Candidate]" term=2 +2025-06-15T03:18:05.286Z [INFO] server-e21aea6e-16fd-95e7-de6a-4d3233e7e811: election won: term=2 tally=2 +2025-06-15T03:18:05.286Z [INFO] server-e21aea6e-16fd-95e7-de6a-4d3233e7e811: entering leader state: leader="Node at e21aea6e-16fd-95e7-de6a-4d3233e7e811 [Leader]" +2025-06-15T03:18:05.286Z [INFO] server-e21aea6e-16fd-95e7-de6a-4d3233e7e811: added peer, starting replication: peer=server-8eb4c1bc-c830-de77-ab2d-096586fadf10 +2025-06-15T03:18:05.286Z [INFO] server-e21aea6e-16fd-95e7-de6a-4d3233e7e811: added peer, starting replication: peer=server-776faae7-5b54-8723-4588-fbc834faeac5 +2025-06-15T03:18:05.286Z [INFO] server-e21aea6e-16fd-95e7-de6a-4d3233e7e811: pipelining replication: peer="{Voter server-776faae7-5b54-8723-4588-fbc834faeac5 776faae7-5b54-8723-4588-fbc834faeac5}" +2025-06-15T03:18:05.286Z [INFO] server-e21aea6e-16fd-95e7-de6a-4d3233e7e811: pipelining replication: peer="{Voter server-8eb4c1bc-c830-de77-ab2d-096586fadf10 8eb4c1bc-c830-de77-ab2d-096586fadf10}" +2025-06-15T03:18:05.392Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:05.286416462 +0000 UTC m=+100.998094634 (1 nodes), 61.630906ms from start of poll, 61.944362ms from cluster start. Timeout at 2025-06-15 03:18:05.392477144 +0000 UTC m=+101.104155343, 106.060709ms after stability +2025-06-15T03:18:05.397Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:05.503Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:05.397610925 +0000 UTC m=+101.109289128 (2 nodes), 1.49µs from start of poll, 173.138856ms from cluster start. Timeout at 2025-06-15 03:18:05.503580131 +0000 UTC m=+101.215258331, 105.969203ms after stability + raft_test.go:1968: [INFO] Disconnecting Node at 8eb4c1bc-c830-de77-ab2d-096586fadf10 [Follower] +2025-06-15T03:18:05.503Z [ERROR] server-e21aea6e-16fd-95e7-de6a-4d3233e7e811: failed to heartbeat to: peer=8eb4c1bc-c830-de77-ab2d-096586fadf10 backoff time=10ms error="failed to connect to peer: 8eb4c1bc-c830-de77-ab2d-096586fadf10" +2025-06-15T03:18:05.503Z [INFO] server-e21aea6e-16fd-95e7-de6a-4d3233e7e811: aborting pipeline replication: peer="{Voter server-776faae7-5b54-8723-4588-fbc834faeac5 776faae7-5b54-8723-4588-fbc834faeac5}" +2025-06-15T03:18:05.503Z [INFO] server-e21aea6e-16fd-95e7-de6a-4d3233e7e811: aborting pipeline replication: peer="{Voter server-8eb4c1bc-c830-de77-ab2d-096586fadf10 8eb4c1bc-c830-de77-ab2d-096586fadf10}" --- PASS: TestRaft_VerifyLeader_PartialConnect (0.28s) === RUN TestRaft_NotifyCh -2024-05-12T20:51:31.752Z [INFO] server-56085374-1a4d-508a-f5be-de5f3b86c383: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-56085374-1a4d-508a-f5be-de5f3b86c383 Address:56085374-1a4d-508a-f5be-de5f3b86c383}]" -2024-05-12T20:51:31.752Z [INFO] server-56085374-1a4d-508a-f5be-de5f3b86c383: entering follower state: follower="Node at 56085374-1a4d-508a-f5be-de5f3b86c383 [Follower]" leader-address= leader-id= -2024-05-12T20:51:31.824Z [WARN] server-56085374-1a4d-508a-f5be-de5f3b86c383: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:31.824Z [INFO] server-56085374-1a4d-508a-f5be-de5f3b86c383: entering candidate state: node="Node at 56085374-1a4d-508a-f5be-de5f3b86c383 [Candidate]" term=2 -2024-05-12T20:51:31.824Z [INFO] server-56085374-1a4d-508a-f5be-de5f3b86c383: election won: term=2 tally=1 -2024-05-12T20:51:31.824Z [INFO] server-56085374-1a4d-508a-f5be-de5f3b86c383: entering leader state: leader="Node at 56085374-1a4d-508a-f5be-de5f3b86c383 [Leader]" ---- PASS: TestRaft_NotifyCh (0.07s) +2025-06-15T03:18:05.504Z [INFO] server-8c381c71-fa27-2412-f937-47cdc4e0fc65: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8c381c71-fa27-2412-f937-47cdc4e0fc65 Address:8c381c71-fa27-2412-f937-47cdc4e0fc65}]" +2025-06-15T03:18:05.504Z [INFO] server-8c381c71-fa27-2412-f937-47cdc4e0fc65: entering follower state: follower="Node at 8c381c71-fa27-2412-f937-47cdc4e0fc65 [Follower]" leader-address= leader-id= +2025-06-15T03:18:05.564Z [WARN] server-8c381c71-fa27-2412-f937-47cdc4e0fc65: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:05.564Z [INFO] server-8c381c71-fa27-2412-f937-47cdc4e0fc65: entering candidate state: node="Node at 8c381c71-fa27-2412-f937-47cdc4e0fc65 [Candidate]" term=2 +2025-06-15T03:18:05.564Z [INFO] server-8c381c71-fa27-2412-f937-47cdc4e0fc65: election won: term=2 tally=1 +2025-06-15T03:18:05.564Z [INFO] server-8c381c71-fa27-2412-f937-47cdc4e0fc65: entering leader state: leader="Node at 8c381c71-fa27-2412-f937-47cdc4e0fc65 [Leader]" +--- PASS: TestRaft_NotifyCh (0.06s) === RUN TestRaft_AppendEntry -2024-05-12T20:51:31.826Z [INFO] server-e6cdb4b6-d593-3ceb-0182-3253c20f2673: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e6cdb4b6-d593-3ceb-0182-3253c20f2673 Address:e6cdb4b6-d593-3ceb-0182-3253c20f2673} {Suffrage:Voter ID:server-732cf404-af67-1c23-048d-8e3ccb9c74bd Address:732cf404-af67-1c23-048d-8e3ccb9c74bd} {Suffrage:Voter ID:server-499adb9c-15a7-d390-cc72-2c5a4407906d Address:499adb9c-15a7-d390-cc72-2c5a4407906d}]" -2024-05-12T20:51:31.826Z [INFO] server-732cf404-af67-1c23-048d-8e3ccb9c74bd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e6cdb4b6-d593-3ceb-0182-3253c20f2673 Address:e6cdb4b6-d593-3ceb-0182-3253c20f2673} {Suffrage:Voter ID:server-732cf404-af67-1c23-048d-8e3ccb9c74bd Address:732cf404-af67-1c23-048d-8e3ccb9c74bd} {Suffrage:Voter ID:server-499adb9c-15a7-d390-cc72-2c5a4407906d Address:499adb9c-15a7-d390-cc72-2c5a4407906d}]" -2024-05-12T20:51:31.826Z [INFO] server-499adb9c-15a7-d390-cc72-2c5a4407906d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e6cdb4b6-d593-3ceb-0182-3253c20f2673 Address:e6cdb4b6-d593-3ceb-0182-3253c20f2673} {Suffrage:Voter ID:server-732cf404-af67-1c23-048d-8e3ccb9c74bd Address:732cf404-af67-1c23-048d-8e3ccb9c74bd} {Suffrage:Voter ID:server-499adb9c-15a7-d390-cc72-2c5a4407906d Address:499adb9c-15a7-d390-cc72-2c5a4407906d}]" -2024-05-12T20:51:31.826Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:31.826Z [INFO] server-e6cdb4b6-d593-3ceb-0182-3253c20f2673: entering follower state: follower="Node at e6cdb4b6-d593-3ceb-0182-3253c20f2673 [Follower]" leader-address= leader-id= -2024-05-12T20:51:31.826Z [INFO] server-732cf404-af67-1c23-048d-8e3ccb9c74bd: entering follower state: follower="Node at 732cf404-af67-1c23-048d-8e3ccb9c74bd [Follower]" leader-address= leader-id= -2024-05-12T20:51:31.826Z [INFO] server-499adb9c-15a7-d390-cc72-2c5a4407906d: entering follower state: follower="Node at 499adb9c-15a7-d390-cc72-2c5a4407906d [Follower]" leader-address= leader-id= -2024-05-12T20:51:31.930Z [WARN] server-732cf404-af67-1c23-048d-8e3ccb9c74bd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:31.930Z [INFO] server-732cf404-af67-1c23-048d-8e3ccb9c74bd: entering candidate state: node="Node at 732cf404-af67-1c23-048d-8e3ccb9c74bd [Candidate]" term=2 -2024-05-12T20:51:31.931Z [WARN] server-e6cdb4b6-d593-3ceb-0182-3253c20f2673: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:31.931Z [INFO] server-e6cdb4b6-d593-3ceb-0182-3253c20f2673: entering candidate state: node="Node at e6cdb4b6-d593-3ceb-0182-3253c20f2673 [Candidate]" term=2 -2024-05-12T20:51:31.931Z [WARN] server-499adb9c-15a7-d390-cc72-2c5a4407906d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:31.931Z [INFO] server-499adb9c-15a7-d390-cc72-2c5a4407906d: entering candidate state: node="Node at 499adb9c-15a7-d390-cc72-2c5a4407906d [Candidate]" term=2 -2024-05-12T20:51:31.931Z [INFO] server-499adb9c-15a7-d390-cc72-2c5a4407906d: duplicate requestVote for same term: term=2 -2024-05-12T20:51:31.931Z [INFO] server-e6cdb4b6-d593-3ceb-0182-3253c20f2673: duplicate requestVote for same term: term=2 -2024-05-12T20:51:31.931Z [INFO] server-499adb9c-15a7-d390-cc72-2c5a4407906d: duplicate requestVote for same term: term=2 -2024-05-12T20:51:31.931Z [INFO] server-732cf404-af67-1c23-048d-8e3ccb9c74bd: duplicate requestVote for same term: term=2 -2024-05-12T20:51:31.932Z [INFO] server-e6cdb4b6-d593-3ceb-0182-3253c20f2673: duplicate requestVote for same term: term=2 -2024-05-12T20:51:31.932Z [INFO] server-732cf404-af67-1c23-048d-8e3ccb9c74bd: duplicate requestVote for same term: term=2 -2024-05-12T20:51:31.987Z [WARN] server-732cf404-af67-1c23-048d-8e3ccb9c74bd: Election timeout reached, restarting election -2024-05-12T20:51:31.987Z [INFO] server-732cf404-af67-1c23-048d-8e3ccb9c74bd: entering candidate state: node="Node at 732cf404-af67-1c23-048d-8e3ccb9c74bd [Candidate]" term=3 -2024-05-12T20:51:31.987Z [INFO] server-499adb9c-15a7-d390-cc72-2c5a4407906d: entering follower state: follower="Node at 499adb9c-15a7-d390-cc72-2c5a4407906d [Follower]" leader-address= leader-id= -2024-05-12T20:51:31.987Z [INFO] server-732cf404-af67-1c23-048d-8e3ccb9c74bd: election won: term=3 tally=2 -2024-05-12T20:51:31.987Z [INFO] server-732cf404-af67-1c23-048d-8e3ccb9c74bd: entering leader state: leader="Node at 732cf404-af67-1c23-048d-8e3ccb9c74bd [Leader]" -2024-05-12T20:51:31.987Z [INFO] server-732cf404-af67-1c23-048d-8e3ccb9c74bd: added peer, starting replication: peer=server-e6cdb4b6-d593-3ceb-0182-3253c20f2673 -2024-05-12T20:51:31.987Z [INFO] server-732cf404-af67-1c23-048d-8e3ccb9c74bd: added peer, starting replication: peer=server-499adb9c-15a7-d390-cc72-2c5a4407906d -2024-05-12T20:51:31.987Z [INFO] server-732cf404-af67-1c23-048d-8e3ccb9c74bd: pipelining replication: peer="{Voter server-499adb9c-15a7-d390-cc72-2c5a4407906d 499adb9c-15a7-d390-cc72-2c5a4407906d}" -2024-05-12T20:51:31.987Z [INFO] server-e6cdb4b6-d593-3ceb-0182-3253c20f2673: entering follower state: follower="Node at e6cdb4b6-d593-3ceb-0182-3253c20f2673 [Follower]" leader-address= leader-id= -2024-05-12T20:51:31.988Z [INFO] server-732cf404-af67-1c23-048d-8e3ccb9c74bd: pipelining replication: peer="{Voter server-e6cdb4b6-d593-3ceb-0182-3253c20f2673 e6cdb4b6-d593-3ceb-0182-3253c20f2673}" -2024-05-12T20:51:32.101Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:31.988084576 +0000 UTC m=+103.268785036 (2 nodes), 161.645465ms from start of poll, 162.112172ms from cluster start. Timeout at 2024-05-12 20:51:32.101478898 +0000 UTC m=+103.382179408, 113.394372ms after stability -2024-05-12T20:51:32.101Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:32.210Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:32.101611608 +0000 UTC m=+103.382312050 (1 nodes), 3.762µs from start of poll, 275.639186ms from cluster start. Timeout at 2024-05-12 20:51:32.210770071 +0000 UTC m=+103.491470401, 109.158351ms after stability -2024-05-12T20:51:32.210Z [WARN] server-e6cdb4b6-d593-3ceb-0182-3253c20f2673: clearing log suffix: from=1 to=2 -2024-05-12T20:51:32.210Z [WARN] server-e6cdb4b6-d593-3ceb-0182-3253c20f2673: skipping application of old log: index=1 -2024-05-12T20:51:32.211Z [WARN] server-e6cdb4b6-d593-3ceb-0182-3253c20f2673: skipping application of old log: index=1 -2024-05-12T20:51:32.211Z [INFO] server-732cf404-af67-1c23-048d-8e3ccb9c74bd: aborting pipeline replication: peer="{Voter server-499adb9c-15a7-d390-cc72-2c5a4407906d 499adb9c-15a7-d390-cc72-2c5a4407906d}" -2024-05-12T20:51:32.211Z [INFO] server-732cf404-af67-1c23-048d-8e3ccb9c74bd: aborting pipeline replication: peer="{Voter server-e6cdb4b6-d593-3ceb-0182-3253c20f2673 e6cdb4b6-d593-3ceb-0182-3253c20f2673}" -2024-05-12T20:51:32.712Z [ERROR] server-732cf404-af67-1c23-048d-8e3ccb9c74bd: failed to heartbeat to: peer=499adb9c-15a7-d390-cc72-2c5a4407906d backoff time=10ms error="command timed out" ---- PASS: TestRaft_AppendEntry (0.89s) +2025-06-15T03:18:05.564Z [INFO] server-2d16ee3e-6266-0e06-6ea4-0f80df62573f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d16ee3e-6266-0e06-6ea4-0f80df62573f Address:2d16ee3e-6266-0e06-6ea4-0f80df62573f} {Suffrage:Voter ID:server-d559f8f2-c418-8563-5b4c-7c7a83ce90c2 Address:d559f8f2-c418-8563-5b4c-7c7a83ce90c2} {Suffrage:Voter ID:server-8a356b49-715a-23b6-2980-9f2b54412fb3 Address:8a356b49-715a-23b6-2980-9f2b54412fb3}]" +2025-06-15T03:18:05.565Z [INFO] server-d559f8f2-c418-8563-5b4c-7c7a83ce90c2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d16ee3e-6266-0e06-6ea4-0f80df62573f Address:2d16ee3e-6266-0e06-6ea4-0f80df62573f} {Suffrage:Voter ID:server-d559f8f2-c418-8563-5b4c-7c7a83ce90c2 Address:d559f8f2-c418-8563-5b4c-7c7a83ce90c2} {Suffrage:Voter ID:server-8a356b49-715a-23b6-2980-9f2b54412fb3 Address:8a356b49-715a-23b6-2980-9f2b54412fb3}]" +2025-06-15T03:18:05.565Z [INFO] server-2d16ee3e-6266-0e06-6ea4-0f80df62573f: entering follower state: follower="Node at 2d16ee3e-6266-0e06-6ea4-0f80df62573f [Follower]" leader-address= leader-id= +2025-06-15T03:18:05.565Z [INFO] server-d559f8f2-c418-8563-5b4c-7c7a83ce90c2: entering follower state: follower="Node at d559f8f2-c418-8563-5b4c-7c7a83ce90c2 [Follower]" leader-address= leader-id= +2025-06-15T03:18:05.565Z [INFO] server-8a356b49-715a-23b6-2980-9f2b54412fb3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d16ee3e-6266-0e06-6ea4-0f80df62573f Address:2d16ee3e-6266-0e06-6ea4-0f80df62573f} {Suffrage:Voter ID:server-d559f8f2-c418-8563-5b4c-7c7a83ce90c2 Address:d559f8f2-c418-8563-5b4c-7c7a83ce90c2} {Suffrage:Voter ID:server-8a356b49-715a-23b6-2980-9f2b54412fb3 Address:8a356b49-715a-23b6-2980-9f2b54412fb3}]" +2025-06-15T03:18:05.565Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:05.565Z [INFO] server-8a356b49-715a-23b6-2980-9f2b54412fb3: entering follower state: follower="Node at 8a356b49-715a-23b6-2980-9f2b54412fb3 [Follower]" leader-address= leader-id= +2025-06-15T03:18:05.620Z [WARN] server-d559f8f2-c418-8563-5b4c-7c7a83ce90c2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:05.620Z [INFO] server-d559f8f2-c418-8563-5b4c-7c7a83ce90c2: entering candidate state: node="Node at d559f8f2-c418-8563-5b4c-7c7a83ce90c2 [Candidate]" term=2 +2025-06-15T03:18:05.620Z [INFO] server-d559f8f2-c418-8563-5b4c-7c7a83ce90c2: election won: term=2 tally=2 +2025-06-15T03:18:05.620Z [INFO] server-d559f8f2-c418-8563-5b4c-7c7a83ce90c2: entering leader state: leader="Node at d559f8f2-c418-8563-5b4c-7c7a83ce90c2 [Leader]" +2025-06-15T03:18:05.620Z [INFO] server-d559f8f2-c418-8563-5b4c-7c7a83ce90c2: added peer, starting replication: peer=server-2d16ee3e-6266-0e06-6ea4-0f80df62573f +2025-06-15T03:18:05.620Z [INFO] server-d559f8f2-c418-8563-5b4c-7c7a83ce90c2: added peer, starting replication: peer=server-8a356b49-715a-23b6-2980-9f2b54412fb3 +2025-06-15T03:18:05.620Z [INFO] server-d559f8f2-c418-8563-5b4c-7c7a83ce90c2: pipelining replication: peer="{Voter server-8a356b49-715a-23b6-2980-9f2b54412fb3 8a356b49-715a-23b6-2980-9f2b54412fb3}" +2025-06-15T03:18:05.620Z [INFO] server-d559f8f2-c418-8563-5b4c-7c7a83ce90c2: pipelining replication: peer="{Voter server-2d16ee3e-6266-0e06-6ea4-0f80df62573f 2d16ee3e-6266-0e06-6ea4-0f80df62573f}" +2025-06-15T03:18:05.725Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:05.620543673 +0000 UTC m=+101.332221852 (2 nodes), 55.347566ms from start of poll, 55.64463ms from cluster start. Timeout at 2025-06-15 03:18:05.725546099 +0000 UTC m=+101.437224298, 105.002446ms after stability +2025-06-15T03:18:05.725Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:05.831Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:05.725635809 +0000 UTC m=+101.437314008 (1 nodes), 2.335µs from start of poll, 160.736786ms from cluster start. Timeout at 2025-06-15 03:18:05.8317604 +0000 UTC m=+101.543438565, 106.124557ms after stability +2025-06-15T03:18:05.831Z [WARN] server-2d16ee3e-6266-0e06-6ea4-0f80df62573f: clearing log suffix: from=1 to=2 +2025-06-15T03:18:05.831Z [WARN] server-2d16ee3e-6266-0e06-6ea4-0f80df62573f: skipping application of old log: index=1 +2025-06-15T03:18:05.831Z [WARN] server-2d16ee3e-6266-0e06-6ea4-0f80df62573f: skipping application of old log: index=1 +2025-06-15T03:18:05.831Z [INFO] server-d559f8f2-c418-8563-5b4c-7c7a83ce90c2: aborting pipeline replication: peer="{Voter server-8a356b49-715a-23b6-2980-9f2b54412fb3 8a356b49-715a-23b6-2980-9f2b54412fb3}" +2025-06-15T03:18:05.831Z [INFO] server-d559f8f2-c418-8563-5b4c-7c7a83ce90c2: aborting pipeline replication: peer="{Voter server-2d16ee3e-6266-0e06-6ea4-0f80df62573f 2d16ee3e-6266-0e06-6ea4-0f80df62573f}" +--- PASS: TestRaft_AppendEntry (0.27s) === RUN TestRaft_VotingGrant_WhenLeaderAvailable -2024-05-12T20:51:32.714Z [INFO] server-55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6 Address:55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6} {Suffrage:Voter ID:server-e947e6a9-fbfb-8d30-c15f-376ede7cb778 Address:e947e6a9-fbfb-8d30-c15f-376ede7cb778} {Suffrage:Voter ID:server-c9aa4986-a9da-88c7-2a74-161cb0e073c2 Address:c9aa4986-a9da-88c7-2a74-161cb0e073c2}]" -2024-05-12T20:51:32.714Z [INFO] server-e947e6a9-fbfb-8d30-c15f-376ede7cb778: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6 Address:55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6} {Suffrage:Voter ID:server-e947e6a9-fbfb-8d30-c15f-376ede7cb778 Address:e947e6a9-fbfb-8d30-c15f-376ede7cb778} {Suffrage:Voter ID:server-c9aa4986-a9da-88c7-2a74-161cb0e073c2 Address:c9aa4986-a9da-88c7-2a74-161cb0e073c2}]" -2024-05-12T20:51:32.714Z [INFO] server-c9aa4986-a9da-88c7-2a74-161cb0e073c2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6 Address:55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6} {Suffrage:Voter ID:server-e947e6a9-fbfb-8d30-c15f-376ede7cb778 Address:e947e6a9-fbfb-8d30-c15f-376ede7cb778} {Suffrage:Voter ID:server-c9aa4986-a9da-88c7-2a74-161cb0e073c2 Address:c9aa4986-a9da-88c7-2a74-161cb0e073c2}]" -2024-05-12T20:51:32.714Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:32.714Z [INFO] server-55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6: entering follower state: follower="Node at 55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6 [Follower]" leader-address= leader-id= -2024-05-12T20:51:32.714Z [INFO] server-c9aa4986-a9da-88c7-2a74-161cb0e073c2: entering follower state: follower="Node at c9aa4986-a9da-88c7-2a74-161cb0e073c2 [Follower]" leader-address= leader-id= -2024-05-12T20:51:32.714Z [INFO] server-e947e6a9-fbfb-8d30-c15f-376ede7cb778: entering follower state: follower="Node at e947e6a9-fbfb-8d30-c15f-376ede7cb778 [Follower]" leader-address= leader-id= -2024-05-12T20:51:32.782Z [WARN] server-55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:32.783Z [INFO] server-55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6: entering candidate state: node="Node at 55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6 [Candidate]" term=2 -2024-05-12T20:51:32.783Z [INFO] server-55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6: election won: term=2 tally=2 -2024-05-12T20:51:32.783Z [INFO] server-55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6: entering leader state: leader="Node at 55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6 [Leader]" -2024-05-12T20:51:32.783Z [INFO] server-55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6: added peer, starting replication: peer=server-e947e6a9-fbfb-8d30-c15f-376ede7cb778 -2024-05-12T20:51:32.783Z [INFO] server-55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6: added peer, starting replication: peer=server-c9aa4986-a9da-88c7-2a74-161cb0e073c2 -2024-05-12T20:51:32.783Z [INFO] server-55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6: pipelining replication: peer="{Voter server-c9aa4986-a9da-88c7-2a74-161cb0e073c2 c9aa4986-a9da-88c7-2a74-161cb0e073c2}" -2024-05-12T20:51:32.783Z [WARN] server-e947e6a9-fbfb-8d30-c15f-376ede7cb778: rejecting vote request since our last term is greater: candidate=55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6 last-term=2 last-candidate-term=1 -2024-05-12T20:51:32.783Z [INFO] server-55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6: pipelining replication: peer="{Voter server-e947e6a9-fbfb-8d30-c15f-376ede7cb778 e947e6a9-fbfb-8d30-c15f-376ede7cb778}" -2024-05-12T20:51:32.890Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:32.783824866 +0000 UTC m=+104.064525501 (2 nodes), 68.960997ms from start of poll, 69.417908ms from cluster start. Timeout at 2024-05-12 20:51:32.890773024 +0000 UTC m=+104.171473547, 106.948046ms after stability -2024-05-12T20:51:32.890Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:32.998Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:32.891009746 +0000 UTC m=+104.171710124 (1 nodes), 4.639µs from start of poll, 176.602531ms from cluster start. Timeout at 2024-05-12 20:51:32.998799524 +0000 UTC m=+104.279500012, 107.789888ms after stability -2024-05-12T20:51:32.999Z [ERROR] server-55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6: peer has newer term, stopping replication: peer="{Voter server-e947e6a9-fbfb-8d30-c15f-376ede7cb778 e947e6a9-fbfb-8d30-c15f-376ede7cb778}" -2024-05-12T20:51:32.999Z [INFO] server-55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6: aborting pipeline replication: peer="{Voter server-e947e6a9-fbfb-8d30-c15f-376ede7cb778 e947e6a9-fbfb-8d30-c15f-376ede7cb778}" -2024-05-12T20:51:32.999Z [WARN] server-c9aa4986-a9da-88c7-2a74-161cb0e073c2: rejecting vote request since we have a leader: from=e947e6a9-fbfb-8d30-c15f-376ede7cb778 leader=55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6 leader-id=server-55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6 -2024-05-12T20:51:32.999Z [INFO] server-55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6: entering follower state: follower="Node at 55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6 [Follower]" leader-address= leader-id= -2024-05-12T20:51:32.999Z [INFO] server-55cfbf33-e5a7-4f69-b4f0-cb84607b2dc6: aborting pipeline replication: peer="{Voter server-c9aa4986-a9da-88c7-2a74-161cb0e073c2 c9aa4986-a9da-88c7-2a74-161cb0e073c2}" ---- PASS: TestRaft_VotingGrant_WhenLeaderAvailable (0.29s) +2025-06-15T03:18:05.833Z [INFO] server-cccae170-5a75-1474-576b-761f2678e03c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cccae170-5a75-1474-576b-761f2678e03c Address:cccae170-5a75-1474-576b-761f2678e03c} {Suffrage:Voter ID:server-30d3a387-7338-1f9a-2968-e55b42cc358c Address:30d3a387-7338-1f9a-2968-e55b42cc358c} {Suffrage:Voter ID:server-814c73e3-f5c7-705a-3a44-962bcf98422f Address:814c73e3-f5c7-705a-3a44-962bcf98422f}]" +2025-06-15T03:18:05.833Z [INFO] server-cccae170-5a75-1474-576b-761f2678e03c: entering follower state: follower="Node at cccae170-5a75-1474-576b-761f2678e03c [Follower]" leader-address= leader-id= +2025-06-15T03:18:05.833Z [INFO] server-30d3a387-7338-1f9a-2968-e55b42cc358c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cccae170-5a75-1474-576b-761f2678e03c Address:cccae170-5a75-1474-576b-761f2678e03c} {Suffrage:Voter ID:server-30d3a387-7338-1f9a-2968-e55b42cc358c Address:30d3a387-7338-1f9a-2968-e55b42cc358c} {Suffrage:Voter ID:server-814c73e3-f5c7-705a-3a44-962bcf98422f Address:814c73e3-f5c7-705a-3a44-962bcf98422f}]" +2025-06-15T03:18:05.833Z [INFO] server-30d3a387-7338-1f9a-2968-e55b42cc358c: entering follower state: follower="Node at 30d3a387-7338-1f9a-2968-e55b42cc358c [Follower]" leader-address= leader-id= +2025-06-15T03:18:05.834Z [INFO] server-814c73e3-f5c7-705a-3a44-962bcf98422f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cccae170-5a75-1474-576b-761f2678e03c Address:cccae170-5a75-1474-576b-761f2678e03c} {Suffrage:Voter ID:server-30d3a387-7338-1f9a-2968-e55b42cc358c Address:30d3a387-7338-1f9a-2968-e55b42cc358c} {Suffrage:Voter ID:server-814c73e3-f5c7-705a-3a44-962bcf98422f Address:814c73e3-f5c7-705a-3a44-962bcf98422f}]" +2025-06-15T03:18:05.834Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:05.834Z [INFO] server-814c73e3-f5c7-705a-3a44-962bcf98422f: entering follower state: follower="Node at 814c73e3-f5c7-705a-3a44-962bcf98422f [Follower]" leader-address= leader-id= +2025-06-15T03:18:05.895Z [WARN] server-30d3a387-7338-1f9a-2968-e55b42cc358c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:05.895Z [INFO] server-30d3a387-7338-1f9a-2968-e55b42cc358c: entering candidate state: node="Node at 30d3a387-7338-1f9a-2968-e55b42cc358c [Candidate]" term=2 +2025-06-15T03:18:05.895Z [INFO] server-30d3a387-7338-1f9a-2968-e55b42cc358c: election won: term=2 tally=2 +2025-06-15T03:18:05.895Z [INFO] server-30d3a387-7338-1f9a-2968-e55b42cc358c: entering leader state: leader="Node at 30d3a387-7338-1f9a-2968-e55b42cc358c [Leader]" +2025-06-15T03:18:05.895Z [INFO] server-30d3a387-7338-1f9a-2968-e55b42cc358c: added peer, starting replication: peer=server-cccae170-5a75-1474-576b-761f2678e03c +2025-06-15T03:18:05.895Z [INFO] server-30d3a387-7338-1f9a-2968-e55b42cc358c: added peer, starting replication: peer=server-814c73e3-f5c7-705a-3a44-962bcf98422f +2025-06-15T03:18:05.895Z [WARN] server-cccae170-5a75-1474-576b-761f2678e03c: rejecting vote request since our last term is greater: candidate=30d3a387-7338-1f9a-2968-e55b42cc358c last-term=2 last-candidate-term=1 +2025-06-15T03:18:05.895Z [INFO] server-30d3a387-7338-1f9a-2968-e55b42cc358c: pipelining replication: peer="{Voter server-cccae170-5a75-1474-576b-761f2678e03c cccae170-5a75-1474-576b-761f2678e03c}" +2025-06-15T03:18:05.895Z [INFO] server-30d3a387-7338-1f9a-2968-e55b42cc358c: pipelining replication: peer="{Voter server-814c73e3-f5c7-705a-3a44-962bcf98422f 814c73e3-f5c7-705a-3a44-962bcf98422f}" +2025-06-15T03:18:06.001Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:05.895937616 +0000 UTC m=+101.607615777 (2 nodes), 61.845438ms from start of poll, 62.467183ms from cluster start. Timeout at 2025-06-15 03:18:06.001937986 +0000 UTC m=+101.713616185, 106.000408ms after stability +2025-06-15T03:18:06.002Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:06.108Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:06.002038501 +0000 UTC m=+101.713716747 (1 nodes), 1.814µs from start of poll, 168.568153ms from cluster start. Timeout at 2025-06-15 03:18:06.108122115 +0000 UTC m=+101.819800281, 106.083534ms after stability +2025-06-15T03:18:06.108Z [WARN] server-814c73e3-f5c7-705a-3a44-962bcf98422f: rejecting vote request since we have a leader: from=cccae170-5a75-1474-576b-761f2678e03c leader=30d3a387-7338-1f9a-2968-e55b42cc358c leader-id=server-30d3a387-7338-1f9a-2968-e55b42cc358c +2025-06-15T03:18:06.108Z [INFO] server-30d3a387-7338-1f9a-2968-e55b42cc358c: aborting pipeline replication: peer="{Voter server-cccae170-5a75-1474-576b-761f2678e03c cccae170-5a75-1474-576b-761f2678e03c}" +2025-06-15T03:18:06.108Z [INFO] server-30d3a387-7338-1f9a-2968-e55b42cc358c: aborting pipeline replication: peer="{Voter server-814c73e3-f5c7-705a-3a44-962bcf98422f 814c73e3-f5c7-705a-3a44-962bcf98422f}" +--- PASS: TestRaft_VotingGrant_WhenLeaderAvailable (0.28s) === RUN TestRaft_ProtocolVersion_RejectRPC -2024-05-12T20:51:33.001Z [INFO] server-ce4b42df-153f-999d-b16f-1c05ca7f7a4a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ce4b42df-153f-999d-b16f-1c05ca7f7a4a Address:ce4b42df-153f-999d-b16f-1c05ca7f7a4a} {Suffrage:Voter ID:server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c Address:e7ba25e3-a6b7-67b0-6ce7-40de50bde83c} {Suffrage:Voter ID:server-33ac6bf5-a30d-d5de-dfa7-f82f3c93de9a Address:33ac6bf5-a30d-d5de-dfa7-f82f3c93de9a}]" -2024-05-12T20:51:33.001Z [INFO] server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ce4b42df-153f-999d-b16f-1c05ca7f7a4a Address:ce4b42df-153f-999d-b16f-1c05ca7f7a4a} {Suffrage:Voter ID:server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c Address:e7ba25e3-a6b7-67b0-6ce7-40de50bde83c} {Suffrage:Voter ID:server-33ac6bf5-a30d-d5de-dfa7-f82f3c93de9a Address:33ac6bf5-a30d-d5de-dfa7-f82f3c93de9a}]" -2024-05-12T20:51:33.001Z [INFO] server-33ac6bf5-a30d-d5de-dfa7-f82f3c93de9a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ce4b42df-153f-999d-b16f-1c05ca7f7a4a Address:ce4b42df-153f-999d-b16f-1c05ca7f7a4a} {Suffrage:Voter ID:server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c Address:e7ba25e3-a6b7-67b0-6ce7-40de50bde83c} {Suffrage:Voter ID:server-33ac6bf5-a30d-d5de-dfa7-f82f3c93de9a Address:33ac6bf5-a30d-d5de-dfa7-f82f3c93de9a}]" -2024-05-12T20:51:33.001Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:33.001Z [INFO] server-ce4b42df-153f-999d-b16f-1c05ca7f7a4a: entering follower state: follower="Node at ce4b42df-153f-999d-b16f-1c05ca7f7a4a [Follower]" leader-address= leader-id= -2024-05-12T20:51:33.001Z [INFO] server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c: entering follower state: follower="Node at e7ba25e3-a6b7-67b0-6ce7-40de50bde83c [Follower]" leader-address= leader-id= -2024-05-12T20:51:33.001Z [INFO] server-33ac6bf5-a30d-d5de-dfa7-f82f3c93de9a: entering follower state: follower="Node at 33ac6bf5-a30d-d5de-dfa7-f82f3c93de9a [Follower]" leader-address= leader-id= -2024-05-12T20:51:33.075Z [WARN] server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:33.075Z [INFO] server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c: entering candidate state: node="Node at e7ba25e3-a6b7-67b0-6ce7-40de50bde83c [Candidate]" term=2 -2024-05-12T20:51:33.075Z [INFO] server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c: election won: term=2 tally=2 -2024-05-12T20:51:33.075Z [INFO] server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c: entering leader state: leader="Node at e7ba25e3-a6b7-67b0-6ce7-40de50bde83c [Leader]" -2024-05-12T20:51:33.075Z [INFO] server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c: added peer, starting replication: peer=server-ce4b42df-153f-999d-b16f-1c05ca7f7a4a -2024-05-12T20:51:33.075Z [INFO] server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c: added peer, starting replication: peer=server-33ac6bf5-a30d-d5de-dfa7-f82f3c93de9a -2024-05-12T20:51:33.076Z [INFO] server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c: pipelining replication: peer="{Voter server-33ac6bf5-a30d-d5de-dfa7-f82f3c93de9a 33ac6bf5-a30d-d5de-dfa7-f82f3c93de9a}" -2024-05-12T20:51:33.076Z [WARN] server-ce4b42df-153f-999d-b16f-1c05ca7f7a4a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:33.076Z [INFO] server-ce4b42df-153f-999d-b16f-1c05ca7f7a4a: entering candidate state: node="Node at ce4b42df-153f-999d-b16f-1c05ca7f7a4a [Candidate]" term=2 -2024-05-12T20:51:33.076Z [WARN] server-33ac6bf5-a30d-d5de-dfa7-f82f3c93de9a: rejecting vote request since we have a leader: from=ce4b42df-153f-999d-b16f-1c05ca7f7a4a leader=e7ba25e3-a6b7-67b0-6ce7-40de50bde83c leader-id=server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c -2024-05-12T20:51:33.076Z [INFO] server-ce4b42df-153f-999d-b16f-1c05ca7f7a4a: duplicate requestVote for same term: term=2 -2024-05-12T20:51:33.076Z [INFO] server-ce4b42df-153f-999d-b16f-1c05ca7f7a4a: entering follower state: follower="Node at ce4b42df-153f-999d-b16f-1c05ca7f7a4a [Follower]" leader-address=e7ba25e3-a6b7-67b0-6ce7-40de50bde83c leader-id=server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c -2024-05-12T20:51:33.076Z [INFO] server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c: pipelining replication: peer="{Voter server-ce4b42df-153f-999d-b16f-1c05ca7f7a4a ce4b42df-153f-999d-b16f-1c05ca7f7a4a}" -2024-05-12T20:51:33.076Z [WARN] server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c: rejecting vote request since we have a leader: from=ce4b42df-153f-999d-b16f-1c05ca7f7a4a leader=e7ba25e3-a6b7-67b0-6ce7-40de50bde83c leader-id=server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c -2024-05-12T20:51:33.187Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:33.077105288 +0000 UTC m=+104.357805730 (2 nodes), 75.685812ms from start of poll, 76.066935ms from cluster start. Timeout at 2024-05-12 20:51:33.187246669 +0000 UTC m=+104.467946993, 110.141263ms after stability -2024-05-12T20:51:33.187Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:33.294Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:33.187639695 +0000 UTC m=+104.468340013 (1 nodes), 3.724µs from start of poll, 186.601218ms from cluster start. Timeout at 2024-05-12 20:51:33.294778643 +0000 UTC m=+104.575478538, 107.138525ms after stability -2024-05-12T20:51:33.295Z [INFO] server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c: aborting pipeline replication: peer="{Voter server-33ac6bf5-a30d-d5de-dfa7-f82f3c93de9a 33ac6bf5-a30d-d5de-dfa7-f82f3c93de9a}" -2024-05-12T20:51:33.295Z [INFO] server-e7ba25e3-a6b7-67b0-6ce7-40de50bde83c: aborting pipeline replication: peer="{Voter server-ce4b42df-153f-999d-b16f-1c05ca7f7a4a ce4b42df-153f-999d-b16f-1c05ca7f7a4a}" ---- PASS: TestRaft_ProtocolVersion_RejectRPC (0.30s) +2025-06-15T03:18:06.109Z [INFO] server-c3c371ef-4789-b400-d00e-867eee5739b3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3c371ef-4789-b400-d00e-867eee5739b3 Address:c3c371ef-4789-b400-d00e-867eee5739b3} {Suffrage:Voter ID:server-2f7b136f-0324-ee0a-ae61-fcfe7765fd34 Address:2f7b136f-0324-ee0a-ae61-fcfe7765fd34} {Suffrage:Voter ID:server-8f60c748-f5b7-dcc4-488f-c4809e178364 Address:8f60c748-f5b7-dcc4-488f-c4809e178364}]" +2025-06-15T03:18:06.109Z [INFO] server-2f7b136f-0324-ee0a-ae61-fcfe7765fd34: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3c371ef-4789-b400-d00e-867eee5739b3 Address:c3c371ef-4789-b400-d00e-867eee5739b3} {Suffrage:Voter ID:server-2f7b136f-0324-ee0a-ae61-fcfe7765fd34 Address:2f7b136f-0324-ee0a-ae61-fcfe7765fd34} {Suffrage:Voter ID:server-8f60c748-f5b7-dcc4-488f-c4809e178364 Address:8f60c748-f5b7-dcc4-488f-c4809e178364}]" +2025-06-15T03:18:06.109Z [INFO] server-c3c371ef-4789-b400-d00e-867eee5739b3: entering follower state: follower="Node at c3c371ef-4789-b400-d00e-867eee5739b3 [Follower]" leader-address= leader-id= +2025-06-15T03:18:06.109Z [INFO] server-2f7b136f-0324-ee0a-ae61-fcfe7765fd34: entering follower state: follower="Node at 2f7b136f-0324-ee0a-ae61-fcfe7765fd34 [Follower]" leader-address= leader-id= +2025-06-15T03:18:06.109Z [INFO] server-8f60c748-f5b7-dcc4-488f-c4809e178364: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3c371ef-4789-b400-d00e-867eee5739b3 Address:c3c371ef-4789-b400-d00e-867eee5739b3} {Suffrage:Voter ID:server-2f7b136f-0324-ee0a-ae61-fcfe7765fd34 Address:2f7b136f-0324-ee0a-ae61-fcfe7765fd34} {Suffrage:Voter ID:server-8f60c748-f5b7-dcc4-488f-c4809e178364 Address:8f60c748-f5b7-dcc4-488f-c4809e178364}]" +2025-06-15T03:18:06.109Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:06.109Z [INFO] server-8f60c748-f5b7-dcc4-488f-c4809e178364: entering follower state: follower="Node at 8f60c748-f5b7-dcc4-488f-c4809e178364 [Follower]" leader-address= leader-id= +2025-06-15T03:18:06.164Z [WARN] server-8f60c748-f5b7-dcc4-488f-c4809e178364: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:06.164Z [INFO] server-8f60c748-f5b7-dcc4-488f-c4809e178364: entering candidate state: node="Node at 8f60c748-f5b7-dcc4-488f-c4809e178364 [Candidate]" term=2 +2025-06-15T03:18:06.164Z [INFO] server-8f60c748-f5b7-dcc4-488f-c4809e178364: election won: term=2 tally=2 +2025-06-15T03:18:06.164Z [INFO] server-8f60c748-f5b7-dcc4-488f-c4809e178364: entering leader state: leader="Node at 8f60c748-f5b7-dcc4-488f-c4809e178364 [Leader]" +2025-06-15T03:18:06.164Z [INFO] server-8f60c748-f5b7-dcc4-488f-c4809e178364: added peer, starting replication: peer=server-c3c371ef-4789-b400-d00e-867eee5739b3 +2025-06-15T03:18:06.164Z [INFO] server-8f60c748-f5b7-dcc4-488f-c4809e178364: added peer, starting replication: peer=server-2f7b136f-0324-ee0a-ae61-fcfe7765fd34 +2025-06-15T03:18:06.165Z [INFO] server-8f60c748-f5b7-dcc4-488f-c4809e178364: pipelining replication: peer="{Voter server-2f7b136f-0324-ee0a-ae61-fcfe7765fd34 2f7b136f-0324-ee0a-ae61-fcfe7765fd34}" +2025-06-15T03:18:06.165Z [INFO] server-8f60c748-f5b7-dcc4-488f-c4809e178364: pipelining replication: peer="{Voter server-c3c371ef-4789-b400-d00e-867eee5739b3 c3c371ef-4789-b400-d00e-867eee5739b3}" +2025-06-15T03:18:06.270Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:06.164930573 +0000 UTC m=+101.876608735 (2 nodes), 55.643206ms from start of poll, 55.889117ms from cluster start. Timeout at 2025-06-15 03:18:06.270214847 +0000 UTC m=+101.981893046, 105.284311ms after stability +2025-06-15T03:18:06.270Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:06.375Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:06.270287169 +0000 UTC m=+101.981965340 (1 nodes), 1.063µs from start of poll, 161.245722ms from cluster start. Timeout at 2025-06-15 03:18:06.375459232 +0000 UTC m=+102.087137431, 105.172091ms after stability +2025-06-15T03:18:06.375Z [INFO] server-8f60c748-f5b7-dcc4-488f-c4809e178364: aborting pipeline replication: peer="{Voter server-c3c371ef-4789-b400-d00e-867eee5739b3 c3c371ef-4789-b400-d00e-867eee5739b3}" +2025-06-15T03:18:06.375Z [INFO] server-8f60c748-f5b7-dcc4-488f-c4809e178364: aborting pipeline replication: peer="{Voter server-2f7b136f-0324-ee0a-ae61-fcfe7765fd34 2f7b136f-0324-ee0a-ae61-fcfe7765fd34}" +--- PASS: TestRaft_ProtocolVersion_RejectRPC (0.27s) === RUN TestRaft_ProtocolVersion_Upgrade_1_2 raft_test.go:2151: Skipping unreliable test --- SKIP: TestRaft_ProtocolVersion_Upgrade_1_2 (0.00s) @@ -3578,275 +3568,267 @@ raft_test.go:2196: Skipping unreliable test --- SKIP: TestRaft_ProtocolVersion_Upgrade_2_3 (0.00s) === RUN TestRaft_LeaderID_Propagated -2024-05-12T20:51:33.297Z [INFO] server-ce9a02ce-f337-5b7c-6906-105b577bd396: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ce9a02ce-f337-5b7c-6906-105b577bd396 Address:ce9a02ce-f337-5b7c-6906-105b577bd396} {Suffrage:Voter ID:server-e7f3d9fc-8284-da1e-d90d-be407283e6c0 Address:e7f3d9fc-8284-da1e-d90d-be407283e6c0} {Suffrage:Voter ID:server-d3e04564-fe0c-2baf-e14c-6c903f2b8262 Address:d3e04564-fe0c-2baf-e14c-6c903f2b8262}]" -2024-05-12T20:51:33.297Z [INFO] server-e7f3d9fc-8284-da1e-d90d-be407283e6c0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ce9a02ce-f337-5b7c-6906-105b577bd396 Address:ce9a02ce-f337-5b7c-6906-105b577bd396} {Suffrage:Voter ID:server-e7f3d9fc-8284-da1e-d90d-be407283e6c0 Address:e7f3d9fc-8284-da1e-d90d-be407283e6c0} {Suffrage:Voter ID:server-d3e04564-fe0c-2baf-e14c-6c903f2b8262 Address:d3e04564-fe0c-2baf-e14c-6c903f2b8262}]" -2024-05-12T20:51:33.297Z [INFO] server-d3e04564-fe0c-2baf-e14c-6c903f2b8262: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ce9a02ce-f337-5b7c-6906-105b577bd396 Address:ce9a02ce-f337-5b7c-6906-105b577bd396} {Suffrage:Voter ID:server-e7f3d9fc-8284-da1e-d90d-be407283e6c0 Address:e7f3d9fc-8284-da1e-d90d-be407283e6c0} {Suffrage:Voter ID:server-d3e04564-fe0c-2baf-e14c-6c903f2b8262 Address:d3e04564-fe0c-2baf-e14c-6c903f2b8262}]" -2024-05-12T20:51:33.297Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:33.297Z [INFO] server-ce9a02ce-f337-5b7c-6906-105b577bd396: entering follower state: follower="Node at ce9a02ce-f337-5b7c-6906-105b577bd396 [Follower]" leader-address= leader-id= -2024-05-12T20:51:33.297Z [INFO] server-e7f3d9fc-8284-da1e-d90d-be407283e6c0: entering follower state: follower="Node at e7f3d9fc-8284-da1e-d90d-be407283e6c0 [Follower]" leader-address= leader-id= -2024-05-12T20:51:33.297Z [INFO] server-d3e04564-fe0c-2baf-e14c-6c903f2b8262: entering follower state: follower="Node at d3e04564-fe0c-2baf-e14c-6c903f2b8262 [Follower]" leader-address= leader-id= -2024-05-12T20:51:33.367Z [WARN] server-ce9a02ce-f337-5b7c-6906-105b577bd396: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:33.367Z [INFO] server-ce9a02ce-f337-5b7c-6906-105b577bd396: entering candidate state: node="Node at ce9a02ce-f337-5b7c-6906-105b577bd396 [Candidate]" term=2 -2024-05-12T20:51:33.367Z [INFO] server-ce9a02ce-f337-5b7c-6906-105b577bd396: election won: term=2 tally=2 -2024-05-12T20:51:33.367Z [INFO] server-ce9a02ce-f337-5b7c-6906-105b577bd396: entering leader state: leader="Node at ce9a02ce-f337-5b7c-6906-105b577bd396 [Leader]" -2024-05-12T20:51:33.367Z [INFO] server-ce9a02ce-f337-5b7c-6906-105b577bd396: added peer, starting replication: peer=server-e7f3d9fc-8284-da1e-d90d-be407283e6c0 -2024-05-12T20:51:33.367Z [INFO] server-ce9a02ce-f337-5b7c-6906-105b577bd396: added peer, starting replication: peer=server-d3e04564-fe0c-2baf-e14c-6c903f2b8262 -2024-05-12T20:51:33.368Z [INFO] server-ce9a02ce-f337-5b7c-6906-105b577bd396: pipelining replication: peer="{Voter server-d3e04564-fe0c-2baf-e14c-6c903f2b8262 d3e04564-fe0c-2baf-e14c-6c903f2b8262}" -2024-05-12T20:51:33.368Z [INFO] server-ce9a02ce-f337-5b7c-6906-105b577bd396: pipelining replication: peer="{Voter server-e7f3d9fc-8284-da1e-d90d-be407283e6c0 e7f3d9fc-8284-da1e-d90d-be407283e6c0}" -2024-05-12T20:51:33.494Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:33.368467516 +0000 UTC m=+104.649167928 (1 nodes), 71.117795ms from start of poll, 71.525979ms from cluster start. Timeout at 2024-05-12 20:51:33.494808413 +0000 UTC m=+104.775509432, 126.341504ms after stability -2024-05-12T20:51:33.495Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:33.603Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:33.495087192 +0000 UTC m=+104.775787830 (1 nodes), 2.212µs from start of poll, 198.145881ms from cluster start. Timeout at 2024-05-12 20:51:33.603168692 +0000 UTC m=+104.883868587, 108.080757ms after stability -2024-05-12T20:51:33.603Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:33.711Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:33.603574785 +0000 UTC m=+104.884274680 (1 nodes), 6.911µs from start of poll, 306.632731ms from cluster start. Timeout at 2024-05-12 20:51:33.711196253 +0000 UTC m=+104.991896791, 107.622111ms after stability -2024-05-12T20:51:33.711Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:33.818Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:33.711662442 +0000 UTC m=+104.992362974 (1 nodes), 2.253µs from start of poll, 414.721025ms from cluster start. Timeout at 2024-05-12 20:51:33.818825156 +0000 UTC m=+105.099525539, 107.162565ms after stability -2024-05-12T20:51:33.819Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:33.926Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:33.819363989 +0000 UTC m=+105.100064359 (1 nodes), 2.025µs from start of poll, 522.42241ms from cluster start. Timeout at 2024-05-12 20:51:33.926781641 +0000 UTC m=+105.207482094, 107.417735ms after stability -2024-05-12T20:51:33.927Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:34.034Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:33.927581905 +0000 UTC m=+105.208282266 (1 nodes), 2.26µs from start of poll, 630.640317ms from cluster start. Timeout at 2024-05-12 20:51:34.03478265 +0000 UTC m=+105.315483011, 107.200745ms after stability -2024-05-12T20:51:34.143Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:34.250Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:34.143056141 +0000 UTC m=+105.423756485 (1 nodes), 3.361µs from start of poll, 846.114536ms from cluster start. Timeout at 2024-05-12 20:51:34.250817608 +0000 UTC m=+105.531518335, 107.76185ms after stability -2024-05-12T20:51:34.251Z [INFO] server-ce9a02ce-f337-5b7c-6906-105b577bd396: aborting pipeline replication: peer="{Voter server-d3e04564-fe0c-2baf-e14c-6c903f2b8262 d3e04564-fe0c-2baf-e14c-6c903f2b8262}" -2024-05-12T20:51:34.251Z [INFO] server-ce9a02ce-f337-5b7c-6906-105b577bd396: aborting pipeline replication: peer="{Voter server-e7f3d9fc-8284-da1e-d90d-be407283e6c0 e7f3d9fc-8284-da1e-d90d-be407283e6c0}" ---- PASS: TestRaft_LeaderID_Propagated (0.96s) +2025-06-15T03:18:06.376Z [INFO] server-027d022a-4405-e1dc-bea2-8c60a3f389ce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-027d022a-4405-e1dc-bea2-8c60a3f389ce Address:027d022a-4405-e1dc-bea2-8c60a3f389ce} {Suffrage:Voter ID:server-04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72 Address:04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72} {Suffrage:Voter ID:server-d169c0c4-0ed0-c023-b589-5cfff0f55077 Address:d169c0c4-0ed0-c023-b589-5cfff0f55077}]" +2025-06-15T03:18:06.376Z [INFO] server-04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-027d022a-4405-e1dc-bea2-8c60a3f389ce Address:027d022a-4405-e1dc-bea2-8c60a3f389ce} {Suffrage:Voter ID:server-04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72 Address:04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72} {Suffrage:Voter ID:server-d169c0c4-0ed0-c023-b589-5cfff0f55077 Address:d169c0c4-0ed0-c023-b589-5cfff0f55077}]" +2025-06-15T03:18:06.376Z [INFO] server-027d022a-4405-e1dc-bea2-8c60a3f389ce: entering follower state: follower="Node at 027d022a-4405-e1dc-bea2-8c60a3f389ce [Follower]" leader-address= leader-id= +2025-06-15T03:18:06.376Z [INFO] server-04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72: entering follower state: follower="Node at 04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72 [Follower]" leader-address= leader-id= +2025-06-15T03:18:06.376Z [INFO] server-d169c0c4-0ed0-c023-b589-5cfff0f55077: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-027d022a-4405-e1dc-bea2-8c60a3f389ce Address:027d022a-4405-e1dc-bea2-8c60a3f389ce} {Suffrage:Voter ID:server-04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72 Address:04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72} {Suffrage:Voter ID:server-d169c0c4-0ed0-c023-b589-5cfff0f55077 Address:d169c0c4-0ed0-c023-b589-5cfff0f55077}]" +2025-06-15T03:18:06.376Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:06.376Z [INFO] server-d169c0c4-0ed0-c023-b589-5cfff0f55077: entering follower state: follower="Node at d169c0c4-0ed0-c023-b589-5cfff0f55077 [Follower]" leader-address= leader-id= +2025-06-15T03:18:06.431Z [WARN] server-04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:06.431Z [INFO] server-04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72: entering candidate state: node="Node at 04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72 [Candidate]" term=2 +2025-06-15T03:18:06.431Z [INFO] server-04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72: election won: term=2 tally=2 +2025-06-15T03:18:06.431Z [INFO] server-04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72: entering leader state: leader="Node at 04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72 [Leader]" +2025-06-15T03:18:06.432Z [INFO] server-04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72: added peer, starting replication: peer=server-027d022a-4405-e1dc-bea2-8c60a3f389ce +2025-06-15T03:18:06.432Z [INFO] server-04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72: added peer, starting replication: peer=server-d169c0c4-0ed0-c023-b589-5cfff0f55077 +2025-06-15T03:18:06.432Z [INFO] server-04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72: pipelining replication: peer="{Voter server-027d022a-4405-e1dc-bea2-8c60a3f389ce 027d022a-4405-e1dc-bea2-8c60a3f389ce}" +2025-06-15T03:18:06.432Z [INFO] server-04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72: pipelining replication: peer="{Voter server-d169c0c4-0ed0-c023-b589-5cfff0f55077 d169c0c4-0ed0-c023-b589-5cfff0f55077}" +2025-06-15T03:18:06.537Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:06.432315088 +0000 UTC m=+102.143993265 (1 nodes), 55.464539ms from start of poll, 55.759571ms from cluster start. Timeout at 2025-06-15 03:18:06.537832201 +0000 UTC m=+102.249510400, 105.517135ms after stability +2025-06-15T03:18:06.537Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:06.644Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:06.537956939 +0000 UTC m=+102.249635142 (1 nodes), 1.951µs from start of poll, 161.401448ms from cluster start. Timeout at 2025-06-15 03:18:06.644111554 +0000 UTC m=+102.355789720, 106.154578ms after stability +2025-06-15T03:18:06.644Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:06.750Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:06.644255564 +0000 UTC m=+102.355933733 (1 nodes), 684ns from start of poll, 267.700039ms from cluster start. Timeout at 2025-06-15 03:18:06.750160598 +0000 UTC m=+102.461838797, 105.905064ms after stability +2025-06-15T03:18:06.750Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:06.855Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:06.750369631 +0000 UTC m=+102.462047834 (1 nodes), 963ns from start of poll, 373.81414ms from cluster start. Timeout at 2025-06-15 03:18:06.855517517 +0000 UTC m=+102.567195716, 105.147882ms after stability +2025-06-15T03:18:06.856Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:06.962Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:06.856297358 +0000 UTC m=+102.567975521 (1 nodes), 402ns from start of poll, 479.741827ms from cluster start. Timeout at 2025-06-15 03:18:06.96199756 +0000 UTC m=+102.673675759, 105.700238ms after stability +2025-06-15T03:18:06.962Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:07.067Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:06.962259198 +0000 UTC m=+102.673937397 (1 nodes), 762ns from start of poll, 585.703703ms from cluster start. Timeout at 2025-06-15 03:18:07.067507412 +0000 UTC m=+102.779185611, 105.248214ms after stability +2025-06-15T03:18:07.172Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:07.278Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:07.172674951 +0000 UTC m=+102.884353116 (1 nodes), 911ns from start of poll, 796.119422ms from cluster start. Timeout at 2025-06-15 03:18:07.278112247 +0000 UTC m=+102.989790409, 105.437293ms after stability +2025-06-15T03:18:07.278Z [INFO] server-04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72: aborting pipeline replication: peer="{Voter server-d169c0c4-0ed0-c023-b589-5cfff0f55077 d169c0c4-0ed0-c023-b589-5cfff0f55077}" +2025-06-15T03:18:07.278Z [INFO] server-04783bdd-fdb5-3a59-d9e7-1b4e66e2bb72: aborting pipeline replication: peer="{Voter server-027d022a-4405-e1dc-bea2-8c60a3f389ce 027d022a-4405-e1dc-bea2-8c60a3f389ce}" +--- PASS: TestRaft_LeaderID_Propagated (0.90s) === RUN TestRaft_LeadershipTransferInProgress --- PASS: TestRaft_LeadershipTransferInProgress (0.00s) === RUN TestRaft_LeadershipTransferPickServer --- PASS: TestRaft_LeadershipTransferPickServer (0.00s) === RUN TestRaft_LeadershipTransfer -2024-05-12T20:51:34.253Z [INFO] server-aaa13b09-0acb-3da1-d179-77921e84f400: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aaa13b09-0acb-3da1-d179-77921e84f400 Address:aaa13b09-0acb-3da1-d179-77921e84f400} {Suffrage:Voter ID:server-8f34a308-84b7-802a-0eb8-6a23e604dc5a Address:8f34a308-84b7-802a-0eb8-6a23e604dc5a} {Suffrage:Voter ID:server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53 Address:da8fabbf-fd0b-1f6e-be36-a0e5524b3d53}]" -2024-05-12T20:51:34.253Z [INFO] server-8f34a308-84b7-802a-0eb8-6a23e604dc5a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aaa13b09-0acb-3da1-d179-77921e84f400 Address:aaa13b09-0acb-3da1-d179-77921e84f400} {Suffrage:Voter ID:server-8f34a308-84b7-802a-0eb8-6a23e604dc5a Address:8f34a308-84b7-802a-0eb8-6a23e604dc5a} {Suffrage:Voter ID:server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53 Address:da8fabbf-fd0b-1f6e-be36-a0e5524b3d53}]" -2024-05-12T20:51:34.253Z [INFO] server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aaa13b09-0acb-3da1-d179-77921e84f400 Address:aaa13b09-0acb-3da1-d179-77921e84f400} {Suffrage:Voter ID:server-8f34a308-84b7-802a-0eb8-6a23e604dc5a Address:8f34a308-84b7-802a-0eb8-6a23e604dc5a} {Suffrage:Voter ID:server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53 Address:da8fabbf-fd0b-1f6e-be36-a0e5524b3d53}]" -2024-05-12T20:51:34.253Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:34.253Z [INFO] server-aaa13b09-0acb-3da1-d179-77921e84f400: entering follower state: follower="Node at aaa13b09-0acb-3da1-d179-77921e84f400 [Follower]" leader-address= leader-id= -2024-05-12T20:51:34.253Z [INFO] server-8f34a308-84b7-802a-0eb8-6a23e604dc5a: entering follower state: follower="Node at 8f34a308-84b7-802a-0eb8-6a23e604dc5a [Follower]" leader-address= leader-id= -2024-05-12T20:51:34.253Z [INFO] server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53: entering follower state: follower="Node at da8fabbf-fd0b-1f6e-be36-a0e5524b3d53 [Follower]" leader-address= leader-id= -2024-05-12T20:51:34.326Z [WARN] server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:34.326Z [INFO] server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53: entering candidate state: node="Node at da8fabbf-fd0b-1f6e-be36-a0e5524b3d53 [Candidate]" term=2 -2024-05-12T20:51:34.326Z [INFO] server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53: election won: term=2 tally=2 -2024-05-12T20:51:34.326Z [INFO] server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53: entering leader state: leader="Node at da8fabbf-fd0b-1f6e-be36-a0e5524b3d53 [Leader]" -2024-05-12T20:51:34.327Z [INFO] server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53: added peer, starting replication: peer=server-aaa13b09-0acb-3da1-d179-77921e84f400 -2024-05-12T20:51:34.327Z [INFO] server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53: added peer, starting replication: peer=server-8f34a308-84b7-802a-0eb8-6a23e604dc5a -2024-05-12T20:51:34.327Z [INFO] server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53: pipelining replication: peer="{Voter server-8f34a308-84b7-802a-0eb8-6a23e604dc5a 8f34a308-84b7-802a-0eb8-6a23e604dc5a}" -2024-05-12T20:51:34.327Z [INFO] server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53: pipelining replication: peer="{Voter server-aaa13b09-0acb-3da1-d179-77921e84f400 aaa13b09-0acb-3da1-d179-77921e84f400}" -2024-05-12T20:51:34.434Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:34.327340253 +0000 UTC m=+105.608040148 (1 nodes), 73.820644ms from start of poll, 74.244187ms from cluster start. Timeout at 2024-05-12 20:51:34.434946524 +0000 UTC m=+105.715646419, 107.606271ms after stability -2024-05-12T20:51:34.435Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:34.551Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:34.435043491 +0000 UTC m=+105.715743386 (1 nodes), 0s from start of poll, 181.947425ms from cluster start. Timeout at 2024-05-12 20:51:34.551075176 +0000 UTC m=+105.831775071, 116.031685ms after stability -2024-05-12T20:51:34.551Z [INFO] server-aaa13b09-0acb-3da1-d179-77921e84f400: entering candidate state: node="Node at aaa13b09-0acb-3da1-d179-77921e84f400 [Candidate]" term=3 -2024-05-12T20:51:34.551Z [INFO] server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53: entering follower state: follower="Node at da8fabbf-fd0b-1f6e-be36-a0e5524b3d53 [Follower]" leader-address= leader-id= -2024-05-12T20:51:34.551Z [INFO] server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53: aborting pipeline replication: peer="{Voter server-8f34a308-84b7-802a-0eb8-6a23e604dc5a 8f34a308-84b7-802a-0eb8-6a23e604dc5a}" -2024-05-12T20:51:34.551Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:34.551Z [INFO] server-aaa13b09-0acb-3da1-d179-77921e84f400: election won: term=3 tally=2 -2024-05-12T20:51:34.551Z [INFO] server-aaa13b09-0acb-3da1-d179-77921e84f400: entering leader state: leader="Node at aaa13b09-0acb-3da1-d179-77921e84f400 [Leader]" -2024-05-12T20:51:34.551Z [INFO] server-aaa13b09-0acb-3da1-d179-77921e84f400: added peer, starting replication: peer=server-8f34a308-84b7-802a-0eb8-6a23e604dc5a -2024-05-12T20:51:34.551Z [INFO] server-aaa13b09-0acb-3da1-d179-77921e84f400: added peer, starting replication: peer=server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53 -2024-05-12T20:51:34.552Z [INFO] server-aaa13b09-0acb-3da1-d179-77921e84f400: pipelining replication: peer="{Voter server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53 da8fabbf-fd0b-1f6e-be36-a0e5524b3d53}" -2024-05-12T20:51:34.552Z [INFO] server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53: aborting pipeline replication: peer="{Voter server-aaa13b09-0acb-3da1-d179-77921e84f400 aaa13b09-0acb-3da1-d179-77921e84f400}" -2024-05-12T20:51:34.552Z [INFO] server-aaa13b09-0acb-3da1-d179-77921e84f400: pipelining replication: peer="{Voter server-8f34a308-84b7-802a-0eb8-6a23e604dc5a 8f34a308-84b7-802a-0eb8-6a23e604dc5a}" -2024-05-12T20:51:34.658Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:34.552513367 +0000 UTC m=+105.833213804 (1 nodes), 722.105µs from start of poll, 299.417843ms from cluster start. Timeout at 2024-05-12 20:51:34.658792154 +0000 UTC m=+105.939492049, 106.278245ms after stability -2024-05-12T20:51:34.659Z [INFO] server-aaa13b09-0acb-3da1-d179-77921e84f400: aborting pipeline replication: peer="{Voter server-da8fabbf-fd0b-1f6e-be36-a0e5524b3d53 da8fabbf-fd0b-1f6e-be36-a0e5524b3d53}" -2024-05-12T20:51:34.659Z [INFO] server-aaa13b09-0acb-3da1-d179-77921e84f400: aborting pipeline replication: peer="{Voter server-8f34a308-84b7-802a-0eb8-6a23e604dc5a 8f34a308-84b7-802a-0eb8-6a23e604dc5a}" -2024-05-12T20:51:35.164Z [ERROR] server-aaa13b09-0acb-3da1-d179-77921e84f400: failed to heartbeat to: peer=da8fabbf-fd0b-1f6e-be36-a0e5524b3d53 backoff time=10ms error="command timed out" -2024-05-12T20:51:35.164Z [ERROR] server-aaa13b09-0acb-3da1-d179-77921e84f400: failed to heartbeat to: peer=8f34a308-84b7-802a-0eb8-6a23e604dc5a backoff time=10ms error="command timed out" ---- PASS: TestRaft_LeadershipTransfer (0.91s) +2025-06-15T03:18:07.279Z [INFO] server-d3853d10-0d52-752f-ba49-105ea576bac1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3853d10-0d52-752f-ba49-105ea576bac1 Address:d3853d10-0d52-752f-ba49-105ea576bac1} {Suffrage:Voter ID:server-4caef398-cae3-ed4a-0ce0-590100cea3ef Address:4caef398-cae3-ed4a-0ce0-590100cea3ef} {Suffrage:Voter ID:server-e2ca9c62-79ec-af84-9c12-16f3aaa4097e Address:e2ca9c62-79ec-af84-9c12-16f3aaa4097e}]" +2025-06-15T03:18:07.279Z [INFO] server-d3853d10-0d52-752f-ba49-105ea576bac1: entering follower state: follower="Node at d3853d10-0d52-752f-ba49-105ea576bac1 [Follower]" leader-address= leader-id= +2025-06-15T03:18:07.279Z [INFO] server-4caef398-cae3-ed4a-0ce0-590100cea3ef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3853d10-0d52-752f-ba49-105ea576bac1 Address:d3853d10-0d52-752f-ba49-105ea576bac1} {Suffrage:Voter ID:server-4caef398-cae3-ed4a-0ce0-590100cea3ef Address:4caef398-cae3-ed4a-0ce0-590100cea3ef} {Suffrage:Voter ID:server-e2ca9c62-79ec-af84-9c12-16f3aaa4097e Address:e2ca9c62-79ec-af84-9c12-16f3aaa4097e}]" +2025-06-15T03:18:07.279Z [INFO] server-4caef398-cae3-ed4a-0ce0-590100cea3ef: entering follower state: follower="Node at 4caef398-cae3-ed4a-0ce0-590100cea3ef [Follower]" leader-address= leader-id= +2025-06-15T03:18:07.279Z [INFO] server-e2ca9c62-79ec-af84-9c12-16f3aaa4097e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3853d10-0d52-752f-ba49-105ea576bac1 Address:d3853d10-0d52-752f-ba49-105ea576bac1} {Suffrage:Voter ID:server-4caef398-cae3-ed4a-0ce0-590100cea3ef Address:4caef398-cae3-ed4a-0ce0-590100cea3ef} {Suffrage:Voter ID:server-e2ca9c62-79ec-af84-9c12-16f3aaa4097e Address:e2ca9c62-79ec-af84-9c12-16f3aaa4097e}]" +2025-06-15T03:18:07.279Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:07.279Z [INFO] server-e2ca9c62-79ec-af84-9c12-16f3aaa4097e: entering follower state: follower="Node at e2ca9c62-79ec-af84-9c12-16f3aaa4097e [Follower]" leader-address= leader-id= +2025-06-15T03:18:07.335Z [WARN] server-4caef398-cae3-ed4a-0ce0-590100cea3ef: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:07.335Z [INFO] server-4caef398-cae3-ed4a-0ce0-590100cea3ef: entering candidate state: node="Node at 4caef398-cae3-ed4a-0ce0-590100cea3ef [Candidate]" term=2 +2025-06-15T03:18:07.335Z [INFO] server-4caef398-cae3-ed4a-0ce0-590100cea3ef: election won: term=2 tally=2 +2025-06-15T03:18:07.335Z [INFO] server-4caef398-cae3-ed4a-0ce0-590100cea3ef: entering leader state: leader="Node at 4caef398-cae3-ed4a-0ce0-590100cea3ef [Leader]" +2025-06-15T03:18:07.335Z [INFO] server-4caef398-cae3-ed4a-0ce0-590100cea3ef: added peer, starting replication: peer=server-d3853d10-0d52-752f-ba49-105ea576bac1 +2025-06-15T03:18:07.335Z [INFO] server-4caef398-cae3-ed4a-0ce0-590100cea3ef: added peer, starting replication: peer=server-e2ca9c62-79ec-af84-9c12-16f3aaa4097e +2025-06-15T03:18:07.336Z [INFO] server-4caef398-cae3-ed4a-0ce0-590100cea3ef: pipelining replication: peer="{Voter server-d3853d10-0d52-752f-ba49-105ea576bac1 d3853d10-0d52-752f-ba49-105ea576bac1}" +2025-06-15T03:18:07.336Z [INFO] server-4caef398-cae3-ed4a-0ce0-590100cea3ef: pipelining replication: peer="{Voter server-e2ca9c62-79ec-af84-9c12-16f3aaa4097e e2ca9c62-79ec-af84-9c12-16f3aaa4097e}" +2025-06-15T03:18:07.441Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:07.335971638 +0000 UTC m=+103.047649803 (1 nodes), 56.606226ms from start of poll, 56.879458ms from cluster start. Timeout at 2025-06-15 03:18:07.441345567 +0000 UTC m=+103.153023769, 105.373966ms after stability +2025-06-15T03:18:07.441Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:07.547Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:07.441447719 +0000 UTC m=+103.153125919 (1 nodes), 852ns from start of poll, 162.355574ms from cluster start. Timeout at 2025-06-15 03:18:07.547232449 +0000 UTC m=+103.258910614, 105.784695ms after stability +2025-06-15T03:18:07.547Z [INFO] server-d3853d10-0d52-752f-ba49-105ea576bac1: entering candidate state: node="Node at d3853d10-0d52-752f-ba49-105ea576bac1 [Candidate]" term=3 +2025-06-15T03:18:07.547Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:07.547Z [INFO] server-d3853d10-0d52-752f-ba49-105ea576bac1: election won: term=3 tally=2 +2025-06-15T03:18:07.547Z [INFO] server-d3853d10-0d52-752f-ba49-105ea576bac1: entering leader state: leader="Node at d3853d10-0d52-752f-ba49-105ea576bac1 [Leader]" +2025-06-15T03:18:07.547Z [INFO] server-4caef398-cae3-ed4a-0ce0-590100cea3ef: entering follower state: follower="Node at 4caef398-cae3-ed4a-0ce0-590100cea3ef [Follower]" leader-address= leader-id= +2025-06-15T03:18:07.547Z [INFO] server-d3853d10-0d52-752f-ba49-105ea576bac1: added peer, starting replication: peer=server-4caef398-cae3-ed4a-0ce0-590100cea3ef +2025-06-15T03:18:07.547Z [INFO] server-d3853d10-0d52-752f-ba49-105ea576bac1: added peer, starting replication: peer=server-e2ca9c62-79ec-af84-9c12-16f3aaa4097e +2025-06-15T03:18:07.547Z [INFO] server-4caef398-cae3-ed4a-0ce0-590100cea3ef: aborting pipeline replication: peer="{Voter server-e2ca9c62-79ec-af84-9c12-16f3aaa4097e e2ca9c62-79ec-af84-9c12-16f3aaa4097e}" +2025-06-15T03:18:07.547Z [INFO] server-4caef398-cae3-ed4a-0ce0-590100cea3ef: aborting pipeline replication: peer="{Voter server-d3853d10-0d52-752f-ba49-105ea576bac1 d3853d10-0d52-752f-ba49-105ea576bac1}" +2025-06-15T03:18:07.547Z [INFO] server-d3853d10-0d52-752f-ba49-105ea576bac1: pipelining replication: peer="{Voter server-e2ca9c62-79ec-af84-9c12-16f3aaa4097e e2ca9c62-79ec-af84-9c12-16f3aaa4097e}" +2025-06-15T03:18:07.547Z [INFO] server-d3853d10-0d52-752f-ba49-105ea576bac1: pipelining replication: peer="{Voter server-4caef398-cae3-ed4a-0ce0-590100cea3ef 4caef398-cae3-ed4a-0ce0-590100cea3ef}" +2025-06-15T03:18:07.652Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:07.547480365 +0000 UTC m=+103.259158526 (1 nodes), 65.66µs from start of poll, 268.388181ms from cluster start. Timeout at 2025-06-15 03:18:07.652487112 +0000 UTC m=+103.364165277, 105.006751ms after stability +2025-06-15T03:18:07.652Z [INFO] server-d3853d10-0d52-752f-ba49-105ea576bac1: aborting pipeline replication: peer="{Voter server-e2ca9c62-79ec-af84-9c12-16f3aaa4097e e2ca9c62-79ec-af84-9c12-16f3aaa4097e}" +2025-06-15T03:18:07.652Z [INFO] server-d3853d10-0d52-752f-ba49-105ea576bac1: aborting pipeline replication: peer="{Voter server-4caef398-cae3-ed4a-0ce0-590100cea3ef 4caef398-cae3-ed4a-0ce0-590100cea3ef}" +--- PASS: TestRaft_LeadershipTransfer (0.37s) === RUN TestRaft_LeadershipTransferWithOneNode -2024-05-12T20:51:35.165Z [INFO] server-4e11b3c8-4b96-8938-166d-5624a5cb015f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4e11b3c8-4b96-8938-166d-5624a5cb015f Address:4e11b3c8-4b96-8938-166d-5624a5cb015f}]" -2024-05-12T20:51:35.165Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:35.165Z [INFO] server-4e11b3c8-4b96-8938-166d-5624a5cb015f: entering follower state: follower="Node at 4e11b3c8-4b96-8938-166d-5624a5cb015f [Follower]" leader-address= leader-id= -2024-05-12T20:51:35.262Z [WARN] server-4e11b3c8-4b96-8938-166d-5624a5cb015f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:35.262Z [INFO] server-4e11b3c8-4b96-8938-166d-5624a5cb015f: entering candidate state: node="Node at 4e11b3c8-4b96-8938-166d-5624a5cb015f [Candidate]" term=2 -2024-05-12T20:51:35.263Z [INFO] server-4e11b3c8-4b96-8938-166d-5624a5cb015f: election won: term=2 tally=1 -2024-05-12T20:51:35.263Z [INFO] server-4e11b3c8-4b96-8938-166d-5624a5cb015f: entering leader state: leader="Node at 4e11b3c8-4b96-8938-166d-5624a5cb015f [Leader]" -2024-05-12T20:51:35.371Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:35.263155223 +0000 UTC m=+106.543855627 (1 nodes), 97.227906ms from start of poll, 97.411414ms from cluster start. Timeout at 2024-05-12 20:51:35.371333207 +0000 UTC m=+106.652034441, 108.178814ms after stability ---- PASS: TestRaft_LeadershipTransferWithOneNode (0.21s) +2025-06-15T03:18:07.653Z [INFO] server-f2c7880f-c416-b4f1-b169-1e9f09ad81f2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f2c7880f-c416-b4f1-b169-1e9f09ad81f2 Address:f2c7880f-c416-b4f1-b169-1e9f09ad81f2}]" +2025-06-15T03:18:07.653Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:07.653Z [INFO] server-f2c7880f-c416-b4f1-b169-1e9f09ad81f2: entering follower state: follower="Node at f2c7880f-c416-b4f1-b169-1e9f09ad81f2 [Follower]" leader-address= leader-id= +2025-06-15T03:18:07.743Z [WARN] server-f2c7880f-c416-b4f1-b169-1e9f09ad81f2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:07.743Z [INFO] server-f2c7880f-c416-b4f1-b169-1e9f09ad81f2: entering candidate state: node="Node at f2c7880f-c416-b4f1-b169-1e9f09ad81f2 [Candidate]" term=2 +2025-06-15T03:18:07.743Z [INFO] server-f2c7880f-c416-b4f1-b169-1e9f09ad81f2: election won: term=2 tally=1 +2025-06-15T03:18:07.743Z [INFO] server-f2c7880f-c416-b4f1-b169-1e9f09ad81f2: entering leader state: leader="Node at f2c7880f-c416-b4f1-b169-1e9f09ad81f2 [Leader]" +2025-06-15T03:18:07.848Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:07.74340272 +0000 UTC m=+103.455080886 (1 nodes), 90.08109ms from start of poll, 90.189272ms from cluster start. Timeout at 2025-06-15 03:18:07.848978014 +0000 UTC m=+103.560656188, 105.575302ms after stability +--- PASS: TestRaft_LeadershipTransferWithOneNode (0.20s) === RUN TestRaft_LeadershipTransferWithSevenNodes -2024-05-12T20:51:35.374Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d72b54e-dee0-fa11-399a-eb691d48bbf4 Address:9d72b54e-dee0-fa11-399a-eb691d48bbf4} {Suffrage:Voter ID:server-47d80055-051e-e947-3dae-f95b6a91acdb Address:47d80055-051e-e947-3dae-f95b6a91acdb} {Suffrage:Voter ID:server-05d10b34-0d01-bbc9-fea0-bf954127d47e Address:05d10b34-0d01-bbc9-fea0-bf954127d47e} {Suffrage:Voter ID:server-c3b70dd3-5585-2eca-840f-f956af941f62 Address:c3b70dd3-5585-2eca-840f-f956af941f62} {Suffrage:Voter ID:server-c918c3d3-c14a-9607-30fb-032d5cede985 Address:c918c3d3-c14a-9607-30fb-032d5cede985} {Suffrage:Voter ID:server-385583dc-53a0-87cd-3fde-1af6ed56fd66 Address:385583dc-53a0-87cd-3fde-1af6ed56fd66} {Suffrage:Voter ID:server-94608625-50ad-7759-2940-56256c084185 Address:94608625-50ad-7759-2940-56256c084185}]" -2024-05-12T20:51:35.374Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d72b54e-dee0-fa11-399a-eb691d48bbf4 Address:9d72b54e-dee0-fa11-399a-eb691d48bbf4} {Suffrage:Voter ID:server-47d80055-051e-e947-3dae-f95b6a91acdb Address:47d80055-051e-e947-3dae-f95b6a91acdb} {Suffrage:Voter ID:server-05d10b34-0d01-bbc9-fea0-bf954127d47e Address:05d10b34-0d01-bbc9-fea0-bf954127d47e} {Suffrage:Voter ID:server-c3b70dd3-5585-2eca-840f-f956af941f62 Address:c3b70dd3-5585-2eca-840f-f956af941f62} {Suffrage:Voter ID:server-c918c3d3-c14a-9607-30fb-032d5cede985 Address:c918c3d3-c14a-9607-30fb-032d5cede985} {Suffrage:Voter ID:server-385583dc-53a0-87cd-3fde-1af6ed56fd66 Address:385583dc-53a0-87cd-3fde-1af6ed56fd66} {Suffrage:Voter ID:server-94608625-50ad-7759-2940-56256c084185 Address:94608625-50ad-7759-2940-56256c084185}]" -2024-05-12T20:51:35.374Z [INFO] server-05d10b34-0d01-bbc9-fea0-bf954127d47e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d72b54e-dee0-fa11-399a-eb691d48bbf4 Address:9d72b54e-dee0-fa11-399a-eb691d48bbf4} {Suffrage:Voter ID:server-47d80055-051e-e947-3dae-f95b6a91acdb Address:47d80055-051e-e947-3dae-f95b6a91acdb} {Suffrage:Voter ID:server-05d10b34-0d01-bbc9-fea0-bf954127d47e Address:05d10b34-0d01-bbc9-fea0-bf954127d47e} {Suffrage:Voter ID:server-c3b70dd3-5585-2eca-840f-f956af941f62 Address:c3b70dd3-5585-2eca-840f-f956af941f62} {Suffrage:Voter ID:server-c918c3d3-c14a-9607-30fb-032d5cede985 Address:c918c3d3-c14a-9607-30fb-032d5cede985} {Suffrage:Voter ID:server-385583dc-53a0-87cd-3fde-1af6ed56fd66 Address:385583dc-53a0-87cd-3fde-1af6ed56fd66} {Suffrage:Voter ID:server-94608625-50ad-7759-2940-56256c084185 Address:94608625-50ad-7759-2940-56256c084185}]" -2024-05-12T20:51:35.374Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: entering follower state: follower="Node at 47d80055-051e-e947-3dae-f95b6a91acdb [Follower]" leader-address= leader-id= -2024-05-12T20:51:35.374Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: entering follower state: follower="Node at 9d72b54e-dee0-fa11-399a-eb691d48bbf4 [Follower]" leader-address= leader-id= -2024-05-12T20:51:35.375Z [INFO] server-05d10b34-0d01-bbc9-fea0-bf954127d47e: entering follower state: follower="Node at 05d10b34-0d01-bbc9-fea0-bf954127d47e [Follower]" leader-address= leader-id= -2024-05-12T20:51:35.378Z [INFO] server-c3b70dd3-5585-2eca-840f-f956af941f62: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d72b54e-dee0-fa11-399a-eb691d48bbf4 Address:9d72b54e-dee0-fa11-399a-eb691d48bbf4} {Suffrage:Voter ID:server-47d80055-051e-e947-3dae-f95b6a91acdb Address:47d80055-051e-e947-3dae-f95b6a91acdb} {Suffrage:Voter ID:server-05d10b34-0d01-bbc9-fea0-bf954127d47e Address:05d10b34-0d01-bbc9-fea0-bf954127d47e} {Suffrage:Voter ID:server-c3b70dd3-5585-2eca-840f-f956af941f62 Address:c3b70dd3-5585-2eca-840f-f956af941f62} {Suffrage:Voter ID:server-c918c3d3-c14a-9607-30fb-032d5cede985 Address:c918c3d3-c14a-9607-30fb-032d5cede985} {Suffrage:Voter ID:server-385583dc-53a0-87cd-3fde-1af6ed56fd66 Address:385583dc-53a0-87cd-3fde-1af6ed56fd66} {Suffrage:Voter ID:server-94608625-50ad-7759-2940-56256c084185 Address:94608625-50ad-7759-2940-56256c084185}]" -2024-05-12T20:51:35.379Z [INFO] server-c3b70dd3-5585-2eca-840f-f956af941f62: entering follower state: follower="Node at c3b70dd3-5585-2eca-840f-f956af941f62 [Follower]" leader-address= leader-id= -2024-05-12T20:51:35.379Z [INFO] server-c918c3d3-c14a-9607-30fb-032d5cede985: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d72b54e-dee0-fa11-399a-eb691d48bbf4 Address:9d72b54e-dee0-fa11-399a-eb691d48bbf4} {Suffrage:Voter ID:server-47d80055-051e-e947-3dae-f95b6a91acdb Address:47d80055-051e-e947-3dae-f95b6a91acdb} {Suffrage:Voter ID:server-05d10b34-0d01-bbc9-fea0-bf954127d47e Address:05d10b34-0d01-bbc9-fea0-bf954127d47e} {Suffrage:Voter ID:server-c3b70dd3-5585-2eca-840f-f956af941f62 Address:c3b70dd3-5585-2eca-840f-f956af941f62} {Suffrage:Voter ID:server-c918c3d3-c14a-9607-30fb-032d5cede985 Address:c918c3d3-c14a-9607-30fb-032d5cede985} {Suffrage:Voter ID:server-385583dc-53a0-87cd-3fde-1af6ed56fd66 Address:385583dc-53a0-87cd-3fde-1af6ed56fd66} {Suffrage:Voter ID:server-94608625-50ad-7759-2940-56256c084185 Address:94608625-50ad-7759-2940-56256c084185}]" -2024-05-12T20:51:35.379Z [INFO] server-385583dc-53a0-87cd-3fde-1af6ed56fd66: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d72b54e-dee0-fa11-399a-eb691d48bbf4 Address:9d72b54e-dee0-fa11-399a-eb691d48bbf4} {Suffrage:Voter ID:server-47d80055-051e-e947-3dae-f95b6a91acdb Address:47d80055-051e-e947-3dae-f95b6a91acdb} {Suffrage:Voter ID:server-05d10b34-0d01-bbc9-fea0-bf954127d47e Address:05d10b34-0d01-bbc9-fea0-bf954127d47e} {Suffrage:Voter ID:server-c3b70dd3-5585-2eca-840f-f956af941f62 Address:c3b70dd3-5585-2eca-840f-f956af941f62} {Suffrage:Voter ID:server-c918c3d3-c14a-9607-30fb-032d5cede985 Address:c918c3d3-c14a-9607-30fb-032d5cede985} {Suffrage:Voter ID:server-385583dc-53a0-87cd-3fde-1af6ed56fd66 Address:385583dc-53a0-87cd-3fde-1af6ed56fd66} {Suffrage:Voter ID:server-94608625-50ad-7759-2940-56256c084185 Address:94608625-50ad-7759-2940-56256c084185}]" -2024-05-12T20:51:35.379Z [INFO] server-c918c3d3-c14a-9607-30fb-032d5cede985: entering follower state: follower="Node at c918c3d3-c14a-9607-30fb-032d5cede985 [Follower]" leader-address= leader-id= -2024-05-12T20:51:35.379Z [INFO] server-94608625-50ad-7759-2940-56256c084185: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d72b54e-dee0-fa11-399a-eb691d48bbf4 Address:9d72b54e-dee0-fa11-399a-eb691d48bbf4} {Suffrage:Voter ID:server-47d80055-051e-e947-3dae-f95b6a91acdb Address:47d80055-051e-e947-3dae-f95b6a91acdb} {Suffrage:Voter ID:server-05d10b34-0d01-bbc9-fea0-bf954127d47e Address:05d10b34-0d01-bbc9-fea0-bf954127d47e} {Suffrage:Voter ID:server-c3b70dd3-5585-2eca-840f-f956af941f62 Address:c3b70dd3-5585-2eca-840f-f956af941f62} {Suffrage:Voter ID:server-c918c3d3-c14a-9607-30fb-032d5cede985 Address:c918c3d3-c14a-9607-30fb-032d5cede985} {Suffrage:Voter ID:server-385583dc-53a0-87cd-3fde-1af6ed56fd66 Address:385583dc-53a0-87cd-3fde-1af6ed56fd66} {Suffrage:Voter ID:server-94608625-50ad-7759-2940-56256c084185 Address:94608625-50ad-7759-2940-56256c084185}]" -2024-05-12T20:51:35.379Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:35.379Z [INFO] server-385583dc-53a0-87cd-3fde-1af6ed56fd66: entering follower state: follower="Node at 385583dc-53a0-87cd-3fde-1af6ed56fd66 [Follower]" leader-address= leader-id= -2024-05-12T20:51:35.379Z [INFO] server-94608625-50ad-7759-2940-56256c084185: entering follower state: follower="Node at 94608625-50ad-7759-2940-56256c084185 [Follower]" leader-address= leader-id= -2024-05-12T20:51:35.427Z [WARN] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:35.427Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: entering candidate state: node="Node at 9d72b54e-dee0-fa11-399a-eb691d48bbf4 [Candidate]" term=2 -2024-05-12T20:51:35.427Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: election won: term=2 tally=4 -2024-05-12T20:51:35.427Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: entering leader state: leader="Node at 9d72b54e-dee0-fa11-399a-eb691d48bbf4 [Leader]" -2024-05-12T20:51:35.427Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: added peer, starting replication: peer=server-47d80055-051e-e947-3dae-f95b6a91acdb -2024-05-12T20:51:35.427Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: added peer, starting replication: peer=server-05d10b34-0d01-bbc9-fea0-bf954127d47e -2024-05-12T20:51:35.427Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: added peer, starting replication: peer=server-c3b70dd3-5585-2eca-840f-f956af941f62 -2024-05-12T20:51:35.427Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: added peer, starting replication: peer=server-c918c3d3-c14a-9607-30fb-032d5cede985 -2024-05-12T20:51:35.427Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: added peer, starting replication: peer=server-385583dc-53a0-87cd-3fde-1af6ed56fd66 -2024-05-12T20:51:35.427Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: added peer, starting replication: peer=server-94608625-50ad-7759-2940-56256c084185 -2024-05-12T20:51:35.427Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: pipelining replication: peer="{Voter server-94608625-50ad-7759-2940-56256c084185 94608625-50ad-7759-2940-56256c084185}" -2024-05-12T20:51:35.428Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: pipelining replication: peer="{Voter server-05d10b34-0d01-bbc9-fea0-bf954127d47e 05d10b34-0d01-bbc9-fea0-bf954127d47e}" -2024-05-12T20:51:35.428Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: pipelining replication: peer="{Voter server-c918c3d3-c14a-9607-30fb-032d5cede985 c918c3d3-c14a-9607-30fb-032d5cede985}" -2024-05-12T20:51:35.428Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: pipelining replication: peer="{Voter server-385583dc-53a0-87cd-3fde-1af6ed56fd66 385583dc-53a0-87cd-3fde-1af6ed56fd66}" -2024-05-12T20:51:35.428Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: pipelining replication: peer="{Voter server-c3b70dd3-5585-2eca-840f-f956af941f62 c3b70dd3-5585-2eca-840f-f956af941f62}" -2024-05-12T20:51:35.428Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: pipelining replication: peer="{Voter server-47d80055-051e-e947-3dae-f95b6a91acdb 47d80055-051e-e947-3dae-f95b6a91acdb}" -2024-05-12T20:51:35.534Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:35.428780896 +0000 UTC m=+106.709481228 (1 nodes), 49.34182ms from start of poll, 54.678892ms from cluster start. Timeout at 2024-05-12 20:51:35.534779029 +0000 UTC m=+106.815479452, 105.998224ms after stability -2024-05-12T20:51:35.534Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:35.642Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:35.534892906 +0000 UTC m=+106.815593494 (6 nodes), 5.856µs from start of poll, 160.791158ms from cluster start. Timeout at 2024-05-12 20:51:35.64277826 +0000 UTC m=+106.923478816, 107.885322ms after stability -2024-05-12T20:51:35.642Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:35.747Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:35.642882796 +0000 UTC m=+106.923583311 (1 nodes), 2.384µs from start of poll, 268.780975ms from cluster start. Timeout at 2024-05-12 20:51:35.747908225 +0000 UTC m=+107.028608605, 105.025294ms after stability -2024-05-12T20:51:35.748Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: entering candidate state: node="Node at 47d80055-051e-e947-3dae-f95b6a91acdb [Candidate]" term=3 -2024-05-12T20:51:35.748Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:35.748Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: entering follower state: follower="Node at 9d72b54e-dee0-fa11-399a-eb691d48bbf4 [Follower]" leader-address= leader-id= -2024-05-12T20:51:35.748Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: aborting pipeline replication: peer="{Voter server-94608625-50ad-7759-2940-56256c084185 94608625-50ad-7759-2940-56256c084185}" -2024-05-12T20:51:35.748Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: election won: term=3 tally=4 -2024-05-12T20:51:35.748Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: entering leader state: leader="Node at 47d80055-051e-e947-3dae-f95b6a91acdb [Leader]" -2024-05-12T20:51:35.748Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: added peer, starting replication: peer=server-9d72b54e-dee0-fa11-399a-eb691d48bbf4 -2024-05-12T20:51:35.748Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: added peer, starting replication: peer=server-05d10b34-0d01-bbc9-fea0-bf954127d47e -2024-05-12T20:51:35.748Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: added peer, starting replication: peer=server-c3b70dd3-5585-2eca-840f-f956af941f62 -2024-05-12T20:51:35.748Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: added peer, starting replication: peer=server-c918c3d3-c14a-9607-30fb-032d5cede985 -2024-05-12T20:51:35.748Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: added peer, starting replication: peer=server-385583dc-53a0-87cd-3fde-1af6ed56fd66 -2024-05-12T20:51:35.748Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: added peer, starting replication: peer=server-94608625-50ad-7759-2940-56256c084185 -2024-05-12T20:51:35.748Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: pipelining replication: peer="{Voter server-94608625-50ad-7759-2940-56256c084185 94608625-50ad-7759-2940-56256c084185}" -2024-05-12T20:51:35.748Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: aborting pipeline replication: peer="{Voter server-47d80055-051e-e947-3dae-f95b6a91acdb 47d80055-051e-e947-3dae-f95b6a91acdb}" -2024-05-12T20:51:35.748Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: aborting pipeline replication: peer="{Voter server-05d10b34-0d01-bbc9-fea0-bf954127d47e 05d10b34-0d01-bbc9-fea0-bf954127d47e}" -2024-05-12T20:51:35.748Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: aborting pipeline replication: peer="{Voter server-c3b70dd3-5585-2eca-840f-f956af941f62 c3b70dd3-5585-2eca-840f-f956af941f62}" -2024-05-12T20:51:35.748Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: aborting pipeline replication: peer="{Voter server-c918c3d3-c14a-9607-30fb-032d5cede985 c918c3d3-c14a-9607-30fb-032d5cede985}" -2024-05-12T20:51:35.748Z [INFO] server-9d72b54e-dee0-fa11-399a-eb691d48bbf4: aborting pipeline replication: peer="{Voter server-385583dc-53a0-87cd-3fde-1af6ed56fd66 385583dc-53a0-87cd-3fde-1af6ed56fd66}" -2024-05-12T20:51:35.748Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: pipelining replication: peer="{Voter server-9d72b54e-dee0-fa11-399a-eb691d48bbf4 9d72b54e-dee0-fa11-399a-eb691d48bbf4}" -2024-05-12T20:51:35.749Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: pipelining replication: peer="{Voter server-05d10b34-0d01-bbc9-fea0-bf954127d47e 05d10b34-0d01-bbc9-fea0-bf954127d47e}" -2024-05-12T20:51:35.749Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: pipelining replication: peer="{Voter server-c3b70dd3-5585-2eca-840f-f956af941f62 c3b70dd3-5585-2eca-840f-f956af941f62}" -2024-05-12T20:51:35.749Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: pipelining replication: peer="{Voter server-c918c3d3-c14a-9607-30fb-032d5cede985 c918c3d3-c14a-9607-30fb-032d5cede985}" -2024-05-12T20:51:35.750Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: pipelining replication: peer="{Voter server-385583dc-53a0-87cd-3fde-1af6ed56fd66 385583dc-53a0-87cd-3fde-1af6ed56fd66}" -2024-05-12T20:51:35.859Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:35.750930603 +0000 UTC m=+107.031631717 (1 nodes), 2.743728ms from start of poll, 376.829381ms from cluster start. Timeout at 2024-05-12 20:51:35.859330912 +0000 UTC m=+107.140031145, 108.399428ms after stability -2024-05-12T20:51:35.859Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: aborting pipeline replication: peer="{Voter server-94608625-50ad-7759-2940-56256c084185 94608625-50ad-7759-2940-56256c084185}" -2024-05-12T20:51:35.859Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: aborting pipeline replication: peer="{Voter server-9d72b54e-dee0-fa11-399a-eb691d48bbf4 9d72b54e-dee0-fa11-399a-eb691d48bbf4}" -2024-05-12T20:51:35.859Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: aborting pipeline replication: peer="{Voter server-05d10b34-0d01-bbc9-fea0-bf954127d47e 05d10b34-0d01-bbc9-fea0-bf954127d47e}" -2024-05-12T20:51:35.860Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: aborting pipeline replication: peer="{Voter server-c3b70dd3-5585-2eca-840f-f956af941f62 c3b70dd3-5585-2eca-840f-f956af941f62}" -2024-05-12T20:51:35.860Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: aborting pipeline replication: peer="{Voter server-c918c3d3-c14a-9607-30fb-032d5cede985 c918c3d3-c14a-9607-30fb-032d5cede985}" -2024-05-12T20:51:35.860Z [INFO] server-47d80055-051e-e947-3dae-f95b6a91acdb: aborting pipeline replication: peer="{Voter server-385583dc-53a0-87cd-3fde-1af6ed56fd66 385583dc-53a0-87cd-3fde-1af6ed56fd66}" ---- PASS: TestRaft_LeadershipTransferWithSevenNodes (0.49s) +2025-06-15T03:18:07.850Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e713fc35-0872-be65-d2de-f6f4e0576968 Address:e713fc35-0872-be65-d2de-f6f4e0576968} {Suffrage:Voter ID:server-d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182 Address:d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182} {Suffrage:Voter ID:server-7a304fd2-4664-18f1-5df9-feb1ca5c1333 Address:7a304fd2-4664-18f1-5df9-feb1ca5c1333} {Suffrage:Voter ID:server-c48a6026-9ae8-2aae-e9cf-3f894994d958 Address:c48a6026-9ae8-2aae-e9cf-3f894994d958} {Suffrage:Voter ID:server-6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b Address:6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b} {Suffrage:Voter ID:server-181a838f-5af4-0663-4e77-be7fc7d7ed9d Address:181a838f-5af4-0663-4e77-be7fc7d7ed9d} {Suffrage:Voter ID:server-8f58445d-69f7-a385-5aec-ab49c2ccc008 Address:8f58445d-69f7-a385-5aec-ab49c2ccc008}]" +2025-06-15T03:18:07.850Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: entering follower state: follower="Node at e713fc35-0872-be65-d2de-f6f4e0576968 [Follower]" leader-address= leader-id= +2025-06-15T03:18:07.850Z [INFO] server-d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e713fc35-0872-be65-d2de-f6f4e0576968 Address:e713fc35-0872-be65-d2de-f6f4e0576968} {Suffrage:Voter ID:server-d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182 Address:d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182} {Suffrage:Voter ID:server-7a304fd2-4664-18f1-5df9-feb1ca5c1333 Address:7a304fd2-4664-18f1-5df9-feb1ca5c1333} {Suffrage:Voter ID:server-c48a6026-9ae8-2aae-e9cf-3f894994d958 Address:c48a6026-9ae8-2aae-e9cf-3f894994d958} {Suffrage:Voter ID:server-6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b Address:6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b} {Suffrage:Voter ID:server-181a838f-5af4-0663-4e77-be7fc7d7ed9d Address:181a838f-5af4-0663-4e77-be7fc7d7ed9d} {Suffrage:Voter ID:server-8f58445d-69f7-a385-5aec-ab49c2ccc008 Address:8f58445d-69f7-a385-5aec-ab49c2ccc008}]" +2025-06-15T03:18:07.850Z [INFO] server-d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182: entering follower state: follower="Node at d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182 [Follower]" leader-address= leader-id= +2025-06-15T03:18:07.850Z [INFO] server-7a304fd2-4664-18f1-5df9-feb1ca5c1333: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e713fc35-0872-be65-d2de-f6f4e0576968 Address:e713fc35-0872-be65-d2de-f6f4e0576968} {Suffrage:Voter ID:server-d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182 Address:d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182} {Suffrage:Voter ID:server-7a304fd2-4664-18f1-5df9-feb1ca5c1333 Address:7a304fd2-4664-18f1-5df9-feb1ca5c1333} {Suffrage:Voter ID:server-c48a6026-9ae8-2aae-e9cf-3f894994d958 Address:c48a6026-9ae8-2aae-e9cf-3f894994d958} {Suffrage:Voter ID:server-6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b Address:6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b} {Suffrage:Voter ID:server-181a838f-5af4-0663-4e77-be7fc7d7ed9d Address:181a838f-5af4-0663-4e77-be7fc7d7ed9d} {Suffrage:Voter ID:server-8f58445d-69f7-a385-5aec-ab49c2ccc008 Address:8f58445d-69f7-a385-5aec-ab49c2ccc008}]" +2025-06-15T03:18:07.850Z [INFO] server-7a304fd2-4664-18f1-5df9-feb1ca5c1333: entering follower state: follower="Node at 7a304fd2-4664-18f1-5df9-feb1ca5c1333 [Follower]" leader-address= leader-id= +2025-06-15T03:18:07.850Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e713fc35-0872-be65-d2de-f6f4e0576968 Address:e713fc35-0872-be65-d2de-f6f4e0576968} {Suffrage:Voter ID:server-d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182 Address:d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182} {Suffrage:Voter ID:server-7a304fd2-4664-18f1-5df9-feb1ca5c1333 Address:7a304fd2-4664-18f1-5df9-feb1ca5c1333} {Suffrage:Voter ID:server-c48a6026-9ae8-2aae-e9cf-3f894994d958 Address:c48a6026-9ae8-2aae-e9cf-3f894994d958} {Suffrage:Voter ID:server-6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b Address:6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b} {Suffrage:Voter ID:server-181a838f-5af4-0663-4e77-be7fc7d7ed9d Address:181a838f-5af4-0663-4e77-be7fc7d7ed9d} {Suffrage:Voter ID:server-8f58445d-69f7-a385-5aec-ab49c2ccc008 Address:8f58445d-69f7-a385-5aec-ab49c2ccc008}]" +2025-06-15T03:18:07.850Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: entering follower state: follower="Node at c48a6026-9ae8-2aae-e9cf-3f894994d958 [Follower]" leader-address= leader-id= +2025-06-15T03:18:07.850Z [INFO] server-6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e713fc35-0872-be65-d2de-f6f4e0576968 Address:e713fc35-0872-be65-d2de-f6f4e0576968} {Suffrage:Voter ID:server-d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182 Address:d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182} {Suffrage:Voter ID:server-7a304fd2-4664-18f1-5df9-feb1ca5c1333 Address:7a304fd2-4664-18f1-5df9-feb1ca5c1333} {Suffrage:Voter ID:server-c48a6026-9ae8-2aae-e9cf-3f894994d958 Address:c48a6026-9ae8-2aae-e9cf-3f894994d958} {Suffrage:Voter ID:server-6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b Address:6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b} {Suffrage:Voter ID:server-181a838f-5af4-0663-4e77-be7fc7d7ed9d Address:181a838f-5af4-0663-4e77-be7fc7d7ed9d} {Suffrage:Voter ID:server-8f58445d-69f7-a385-5aec-ab49c2ccc008 Address:8f58445d-69f7-a385-5aec-ab49c2ccc008}]" +2025-06-15T03:18:07.850Z [INFO] server-6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b: entering follower state: follower="Node at 6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b [Follower]" leader-address= leader-id= +2025-06-15T03:18:07.850Z [INFO] server-181a838f-5af4-0663-4e77-be7fc7d7ed9d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e713fc35-0872-be65-d2de-f6f4e0576968 Address:e713fc35-0872-be65-d2de-f6f4e0576968} {Suffrage:Voter ID:server-d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182 Address:d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182} {Suffrage:Voter ID:server-7a304fd2-4664-18f1-5df9-feb1ca5c1333 Address:7a304fd2-4664-18f1-5df9-feb1ca5c1333} {Suffrage:Voter ID:server-c48a6026-9ae8-2aae-e9cf-3f894994d958 Address:c48a6026-9ae8-2aae-e9cf-3f894994d958} {Suffrage:Voter ID:server-6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b Address:6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b} {Suffrage:Voter ID:server-181a838f-5af4-0663-4e77-be7fc7d7ed9d Address:181a838f-5af4-0663-4e77-be7fc7d7ed9d} {Suffrage:Voter ID:server-8f58445d-69f7-a385-5aec-ab49c2ccc008 Address:8f58445d-69f7-a385-5aec-ab49c2ccc008}]" +2025-06-15T03:18:07.850Z [INFO] server-181a838f-5af4-0663-4e77-be7fc7d7ed9d: entering follower state: follower="Node at 181a838f-5af4-0663-4e77-be7fc7d7ed9d [Follower]" leader-address= leader-id= +2025-06-15T03:18:07.850Z [INFO] server-8f58445d-69f7-a385-5aec-ab49c2ccc008: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e713fc35-0872-be65-d2de-f6f4e0576968 Address:e713fc35-0872-be65-d2de-f6f4e0576968} {Suffrage:Voter ID:server-d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182 Address:d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182} {Suffrage:Voter ID:server-7a304fd2-4664-18f1-5df9-feb1ca5c1333 Address:7a304fd2-4664-18f1-5df9-feb1ca5c1333} {Suffrage:Voter ID:server-c48a6026-9ae8-2aae-e9cf-3f894994d958 Address:c48a6026-9ae8-2aae-e9cf-3f894994d958} {Suffrage:Voter ID:server-6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b Address:6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b} {Suffrage:Voter ID:server-181a838f-5af4-0663-4e77-be7fc7d7ed9d Address:181a838f-5af4-0663-4e77-be7fc7d7ed9d} {Suffrage:Voter ID:server-8f58445d-69f7-a385-5aec-ab49c2ccc008 Address:8f58445d-69f7-a385-5aec-ab49c2ccc008}]" +2025-06-15T03:18:07.850Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:07.850Z [INFO] server-8f58445d-69f7-a385-5aec-ab49c2ccc008: entering follower state: follower="Node at 8f58445d-69f7-a385-5aec-ab49c2ccc008 [Follower]" leader-address= leader-id= +2025-06-15T03:18:07.901Z [WARN] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:07.901Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: entering candidate state: node="Node at c48a6026-9ae8-2aae-e9cf-3f894994d958 [Candidate]" term=2 +2025-06-15T03:18:07.901Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: election won: term=2 tally=4 +2025-06-15T03:18:07.901Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: entering leader state: leader="Node at c48a6026-9ae8-2aae-e9cf-3f894994d958 [Leader]" +2025-06-15T03:18:07.901Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: added peer, starting replication: peer=server-e713fc35-0872-be65-d2de-f6f4e0576968 +2025-06-15T03:18:07.901Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: added peer, starting replication: peer=server-d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182 +2025-06-15T03:18:07.901Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: added peer, starting replication: peer=server-7a304fd2-4664-18f1-5df9-feb1ca5c1333 +2025-06-15T03:18:07.901Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: added peer, starting replication: peer=server-6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b +2025-06-15T03:18:07.901Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: added peer, starting replication: peer=server-181a838f-5af4-0663-4e77-be7fc7d7ed9d +2025-06-15T03:18:07.901Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: added peer, starting replication: peer=server-8f58445d-69f7-a385-5aec-ab49c2ccc008 +2025-06-15T03:18:07.901Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: pipelining replication: peer="{Voter server-6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b 6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b}" +2025-06-15T03:18:07.901Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: pipelining replication: peer="{Voter server-d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182 d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182}" +2025-06-15T03:18:07.901Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: pipelining replication: peer="{Voter server-e713fc35-0872-be65-d2de-f6f4e0576968 e713fc35-0872-be65-d2de-f6f4e0576968}" +2025-06-15T03:18:07.901Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: pipelining replication: peer="{Voter server-181a838f-5af4-0663-4e77-be7fc7d7ed9d 181a838f-5af4-0663-4e77-be7fc7d7ed9d}" +2025-06-15T03:18:07.901Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: pipelining replication: peer="{Voter server-7a304fd2-4664-18f1-5df9-feb1ca5c1333 7a304fd2-4664-18f1-5df9-feb1ca5c1333}" +2025-06-15T03:18:07.901Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: pipelining replication: peer="{Voter server-8f58445d-69f7-a385-5aec-ab49c2ccc008 8f58445d-69f7-a385-5aec-ab49c2ccc008}" +2025-06-15T03:18:08.007Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:07.901541546 +0000 UTC m=+103.613219708 (1 nodes), 50.760075ms from start of poll, 51.392268ms from cluster start. Timeout at 2025-06-15 03:18:08.007492749 +0000 UTC m=+103.719170948, 105.95124ms after stability +2025-06-15T03:18:08.007Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:08.114Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:08.007584138 +0000 UTC m=+103.719262303 (6 nodes), 1.464µs from start of poll, 157.434863ms from cluster start. Timeout at 2025-06-15 03:18:08.112969284 +0000 UTC m=+103.824647510, 105.385207ms after stability +2025-06-15T03:18:08.114Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:08.220Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:08.114860869 +0000 UTC m=+103.826539034 (1 nodes), 755ns from start of poll, 264.711594ms from cluster start. Timeout at 2025-06-15 03:18:08.220094538 +0000 UTC m=+103.931772737, 105.233703ms after stability +2025-06-15T03:18:08.220Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: entering candidate state: node="Node at e713fc35-0872-be65-d2de-f6f4e0576968 [Candidate]" term=3 +2025-06-15T03:18:08.220Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:08.220Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: entering follower state: follower="Node at c48a6026-9ae8-2aae-e9cf-3f894994d958 [Follower]" leader-address= leader-id= +2025-06-15T03:18:08.220Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: election won: term=3 tally=4 +2025-06-15T03:18:08.220Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: entering leader state: leader="Node at e713fc35-0872-be65-d2de-f6f4e0576968 [Leader]" +2025-06-15T03:18:08.220Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: added peer, starting replication: peer=server-d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182 +2025-06-15T03:18:08.220Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: added peer, starting replication: peer=server-7a304fd2-4664-18f1-5df9-feb1ca5c1333 +2025-06-15T03:18:08.220Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: added peer, starting replication: peer=server-c48a6026-9ae8-2aae-e9cf-3f894994d958 +2025-06-15T03:18:08.220Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: added peer, starting replication: peer=server-6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b +2025-06-15T03:18:08.220Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: added peer, starting replication: peer=server-181a838f-5af4-0663-4e77-be7fc7d7ed9d +2025-06-15T03:18:08.220Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: added peer, starting replication: peer=server-8f58445d-69f7-a385-5aec-ab49c2ccc008 +2025-06-15T03:18:08.220Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: aborting pipeline replication: peer="{Voter server-6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b 6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b}" +2025-06-15T03:18:08.220Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: pipelining replication: peer="{Voter server-d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182 d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182}" +2025-06-15T03:18:08.220Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: aborting pipeline replication: peer="{Voter server-181a838f-5af4-0663-4e77-be7fc7d7ed9d 181a838f-5af4-0663-4e77-be7fc7d7ed9d}" +2025-06-15T03:18:08.220Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: pipelining replication: peer="{Voter server-7a304fd2-4664-18f1-5df9-feb1ca5c1333 7a304fd2-4664-18f1-5df9-feb1ca5c1333}" +2025-06-15T03:18:08.220Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: aborting pipeline replication: peer="{Voter server-8f58445d-69f7-a385-5aec-ab49c2ccc008 8f58445d-69f7-a385-5aec-ab49c2ccc008}" +2025-06-15T03:18:08.220Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: pipelining replication: peer="{Voter server-181a838f-5af4-0663-4e77-be7fc7d7ed9d 181a838f-5af4-0663-4e77-be7fc7d7ed9d}" +2025-06-15T03:18:08.220Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: aborting pipeline replication: peer="{Voter server-7a304fd2-4664-18f1-5df9-feb1ca5c1333 7a304fd2-4664-18f1-5df9-feb1ca5c1333}" +2025-06-15T03:18:08.220Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: pipelining replication: peer="{Voter server-c48a6026-9ae8-2aae-e9cf-3f894994d958 c48a6026-9ae8-2aae-e9cf-3f894994d958}" +2025-06-15T03:18:08.220Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: aborting pipeline replication: peer="{Voter server-e713fc35-0872-be65-d2de-f6f4e0576968 e713fc35-0872-be65-d2de-f6f4e0576968}" +2025-06-15T03:18:08.220Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: pipelining replication: peer="{Voter server-6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b 6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b}" +2025-06-15T03:18:08.220Z [INFO] server-c48a6026-9ae8-2aae-e9cf-3f894994d958: aborting pipeline replication: peer="{Voter server-d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182 d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182}" +2025-06-15T03:18:08.220Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: pipelining replication: peer="{Voter server-8f58445d-69f7-a385-5aec-ab49c2ccc008 8f58445d-69f7-a385-5aec-ab49c2ccc008}" +2025-06-15T03:18:08.325Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:08.220460336 +0000 UTC m=+103.932138502 (1 nodes), 207.892µs from start of poll, 370.311062ms from cluster start. Timeout at 2025-06-15 03:18:08.325784532 +0000 UTC m=+104.037462693, 105.324191ms after stability +2025-06-15T03:18:08.325Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: aborting pipeline replication: peer="{Voter server-d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182 d47b35e0-2d5c-f5ca-a0a0-ee79cd7a5182}" +2025-06-15T03:18:08.325Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: aborting pipeline replication: peer="{Voter server-181a838f-5af4-0663-4e77-be7fc7d7ed9d 181a838f-5af4-0663-4e77-be7fc7d7ed9d}" +2025-06-15T03:18:08.325Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: aborting pipeline replication: peer="{Voter server-6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b 6d02fcd7-9a2b-443e-1bb0-7ee5a9badf6b}" +2025-06-15T03:18:08.325Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: aborting pipeline replication: peer="{Voter server-7a304fd2-4664-18f1-5df9-feb1ca5c1333 7a304fd2-4664-18f1-5df9-feb1ca5c1333}" +2025-06-15T03:18:08.325Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: aborting pipeline replication: peer="{Voter server-c48a6026-9ae8-2aae-e9cf-3f894994d958 c48a6026-9ae8-2aae-e9cf-3f894994d958}" +2025-06-15T03:18:08.325Z [INFO] server-e713fc35-0872-be65-d2de-f6f4e0576968: aborting pipeline replication: peer="{Voter server-8f58445d-69f7-a385-5aec-ab49c2ccc008 8f58445d-69f7-a385-5aec-ab49c2ccc008}" +--- PASS: TestRaft_LeadershipTransferWithSevenNodes (0.48s) === RUN TestRaft_LeadershipTransferToInvalidID -2024-05-12T20:51:35.863Z [INFO] server-c711195e-c75c-18d8-6634-62f6eb6089e1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c711195e-c75c-18d8-6634-62f6eb6089e1 Address:c711195e-c75c-18d8-6634-62f6eb6089e1} {Suffrage:Voter ID:server-9e8deb58-ed99-c188-a46e-ba8ce0e224a5 Address:9e8deb58-ed99-c188-a46e-ba8ce0e224a5} {Suffrage:Voter ID:server-01659135-8585-95a6-cd74-bd0225b08c58 Address:01659135-8585-95a6-cd74-bd0225b08c58}]" -2024-05-12T20:51:35.863Z [INFO] server-9e8deb58-ed99-c188-a46e-ba8ce0e224a5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c711195e-c75c-18d8-6634-62f6eb6089e1 Address:c711195e-c75c-18d8-6634-62f6eb6089e1} {Suffrage:Voter ID:server-9e8deb58-ed99-c188-a46e-ba8ce0e224a5 Address:9e8deb58-ed99-c188-a46e-ba8ce0e224a5} {Suffrage:Voter ID:server-01659135-8585-95a6-cd74-bd0225b08c58 Address:01659135-8585-95a6-cd74-bd0225b08c58}]" -2024-05-12T20:51:35.863Z [INFO] server-c711195e-c75c-18d8-6634-62f6eb6089e1: entering follower state: follower="Node at c711195e-c75c-18d8-6634-62f6eb6089e1 [Follower]" leader-address= leader-id= -2024-05-12T20:51:35.863Z [INFO] server-9e8deb58-ed99-c188-a46e-ba8ce0e224a5: entering follower state: follower="Node at 9e8deb58-ed99-c188-a46e-ba8ce0e224a5 [Follower]" leader-address= leader-id= -2024-05-12T20:51:35.863Z [INFO] server-01659135-8585-95a6-cd74-bd0225b08c58: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c711195e-c75c-18d8-6634-62f6eb6089e1 Address:c711195e-c75c-18d8-6634-62f6eb6089e1} {Suffrage:Voter ID:server-9e8deb58-ed99-c188-a46e-ba8ce0e224a5 Address:9e8deb58-ed99-c188-a46e-ba8ce0e224a5} {Suffrage:Voter ID:server-01659135-8585-95a6-cd74-bd0225b08c58 Address:01659135-8585-95a6-cd74-bd0225b08c58}]" -2024-05-12T20:51:35.863Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:35.863Z [INFO] server-01659135-8585-95a6-cd74-bd0225b08c58: entering follower state: follower="Node at 01659135-8585-95a6-cd74-bd0225b08c58 [Follower]" leader-address= leader-id= -2024-05-12T20:51:35.930Z [WARN] server-9e8deb58-ed99-c188-a46e-ba8ce0e224a5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:35.930Z [INFO] server-9e8deb58-ed99-c188-a46e-ba8ce0e224a5: entering candidate state: node="Node at 9e8deb58-ed99-c188-a46e-ba8ce0e224a5 [Candidate]" term=2 -2024-05-12T20:51:35.931Z [INFO] server-9e8deb58-ed99-c188-a46e-ba8ce0e224a5: election won: term=2 tally=2 -2024-05-12T20:51:35.931Z [INFO] server-9e8deb58-ed99-c188-a46e-ba8ce0e224a5: entering leader state: leader="Node at 9e8deb58-ed99-c188-a46e-ba8ce0e224a5 [Leader]" -2024-05-12T20:51:35.931Z [INFO] server-9e8deb58-ed99-c188-a46e-ba8ce0e224a5: added peer, starting replication: peer=server-c711195e-c75c-18d8-6634-62f6eb6089e1 -2024-05-12T20:51:35.931Z [INFO] server-9e8deb58-ed99-c188-a46e-ba8ce0e224a5: added peer, starting replication: peer=server-01659135-8585-95a6-cd74-bd0225b08c58 -2024-05-12T20:51:35.931Z [INFO] server-9e8deb58-ed99-c188-a46e-ba8ce0e224a5: pipelining replication: peer="{Voter server-01659135-8585-95a6-cd74-bd0225b08c58 01659135-8585-95a6-cd74-bd0225b08c58}" -2024-05-12T20:51:35.931Z [INFO] server-9e8deb58-ed99-c188-a46e-ba8ce0e224a5: pipelining replication: peer="{Voter server-c711195e-c75c-18d8-6634-62f6eb6089e1 c711195e-c75c-18d8-6634-62f6eb6089e1}" -2024-05-12T20:51:36.038Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:35.931636237 +0000 UTC m=+107.212336592 (1 nodes), 68.25199ms from start of poll, 68.770098ms from cluster start. Timeout at 2024-05-12 20:51:36.038779254 +0000 UTC m=+107.319479911, 107.143319ms after stability -2024-05-12T20:51:36.038Z [INFO] server-9e8deb58-ed99-c188-a46e-ba8ce0e224a5: aborting pipeline replication: peer="{Voter server-01659135-8585-95a6-cd74-bd0225b08c58 01659135-8585-95a6-cd74-bd0225b08c58}" -2024-05-12T20:51:36.039Z [INFO] server-9e8deb58-ed99-c188-a46e-ba8ce0e224a5: aborting pipeline replication: peer="{Voter server-c711195e-c75c-18d8-6634-62f6eb6089e1 c711195e-c75c-18d8-6634-62f6eb6089e1}" ---- PASS: TestRaft_LeadershipTransferToInvalidID (0.18s) +2025-06-15T03:18:08.327Z [INFO] server-6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d Address:6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d} {Suffrage:Voter ID:server-7e637aeb-9841-e8d1-b3b3-b2cbfe72b8f5 Address:7e637aeb-9841-e8d1-b3b3-b2cbfe72b8f5} {Suffrage:Voter ID:server-68de0623-18fd-5f9a-6fcc-84edd77d1d75 Address:68de0623-18fd-5f9a-6fcc-84edd77d1d75}]" +2025-06-15T03:18:08.327Z [INFO] server-7e637aeb-9841-e8d1-b3b3-b2cbfe72b8f5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d Address:6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d} {Suffrage:Voter ID:server-7e637aeb-9841-e8d1-b3b3-b2cbfe72b8f5 Address:7e637aeb-9841-e8d1-b3b3-b2cbfe72b8f5} {Suffrage:Voter ID:server-68de0623-18fd-5f9a-6fcc-84edd77d1d75 Address:68de0623-18fd-5f9a-6fcc-84edd77d1d75}]" +2025-06-15T03:18:08.327Z [INFO] server-6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d: entering follower state: follower="Node at 6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d [Follower]" leader-address= leader-id= +2025-06-15T03:18:08.327Z [INFO] server-68de0623-18fd-5f9a-6fcc-84edd77d1d75: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d Address:6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d} {Suffrage:Voter ID:server-7e637aeb-9841-e8d1-b3b3-b2cbfe72b8f5 Address:7e637aeb-9841-e8d1-b3b3-b2cbfe72b8f5} {Suffrage:Voter ID:server-68de0623-18fd-5f9a-6fcc-84edd77d1d75 Address:68de0623-18fd-5f9a-6fcc-84edd77d1d75}]" +2025-06-15T03:18:08.327Z [INFO] server-7e637aeb-9841-e8d1-b3b3-b2cbfe72b8f5: entering follower state: follower="Node at 7e637aeb-9841-e8d1-b3b3-b2cbfe72b8f5 [Follower]" leader-address= leader-id= +2025-06-15T03:18:08.327Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:08.327Z [INFO] server-68de0623-18fd-5f9a-6fcc-84edd77d1d75: entering follower state: follower="Node at 68de0623-18fd-5f9a-6fcc-84edd77d1d75 [Follower]" leader-address= leader-id= +2025-06-15T03:18:08.385Z [WARN] server-6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:08.385Z [INFO] server-6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d: entering candidate state: node="Node at 6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d [Candidate]" term=2 +2025-06-15T03:18:08.385Z [INFO] server-6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d: election won: term=2 tally=2 +2025-06-15T03:18:08.385Z [INFO] server-6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d: entering leader state: leader="Node at 6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d [Leader]" +2025-06-15T03:18:08.385Z [INFO] server-6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d: added peer, starting replication: peer=server-7e637aeb-9841-e8d1-b3b3-b2cbfe72b8f5 +2025-06-15T03:18:08.385Z [INFO] server-6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d: added peer, starting replication: peer=server-68de0623-18fd-5f9a-6fcc-84edd77d1d75 +2025-06-15T03:18:08.385Z [INFO] server-6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d: pipelining replication: peer="{Voter server-7e637aeb-9841-e8d1-b3b3-b2cbfe72b8f5 7e637aeb-9841-e8d1-b3b3-b2cbfe72b8f5}" +2025-06-15T03:18:08.385Z [INFO] server-6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d: pipelining replication: peer="{Voter server-68de0623-18fd-5f9a-6fcc-84edd77d1d75 68de0623-18fd-5f9a-6fcc-84edd77d1d75}" +2025-06-15T03:18:08.491Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:08.385211403 +0000 UTC m=+104.096889606 (1 nodes), 57.62869ms from start of poll, 57.918473ms from cluster start. Timeout at 2025-06-15 03:18:08.491238978 +0000 UTC m=+104.202917180, 106.027574ms after stability +2025-06-15T03:18:08.491Z [INFO] server-6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d: aborting pipeline replication: peer="{Voter server-68de0623-18fd-5f9a-6fcc-84edd77d1d75 68de0623-18fd-5f9a-6fcc-84edd77d1d75}" +2025-06-15T03:18:08.491Z [INFO] server-6e6a68e8-63c0-a69e-3cbd-2fe559f5cf6d: aborting pipeline replication: peer="{Voter server-7e637aeb-9841-e8d1-b3b3-b2cbfe72b8f5 7e637aeb-9841-e8d1-b3b3-b2cbfe72b8f5}" +--- PASS: TestRaft_LeadershipTransferToInvalidID (0.16s) === RUN TestRaft_LeadershipTransferToInvalidAddress -2024-05-12T20:51:36.041Z [INFO] server-9257bcb9-02ac-153a-7542-a5dc03b02104: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9257bcb9-02ac-153a-7542-a5dc03b02104 Address:9257bcb9-02ac-153a-7542-a5dc03b02104} {Suffrage:Voter ID:server-bbe6d849-79b9-820e-259b-ccb3f852f9be Address:bbe6d849-79b9-820e-259b-ccb3f852f9be} {Suffrage:Voter ID:server-abb6123a-6d6d-64d4-eaf2-8d4aaf247a57 Address:abb6123a-6d6d-64d4-eaf2-8d4aaf247a57}]" -2024-05-12T20:51:36.041Z [INFO] server-bbe6d849-79b9-820e-259b-ccb3f852f9be: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9257bcb9-02ac-153a-7542-a5dc03b02104 Address:9257bcb9-02ac-153a-7542-a5dc03b02104} {Suffrage:Voter ID:server-bbe6d849-79b9-820e-259b-ccb3f852f9be Address:bbe6d849-79b9-820e-259b-ccb3f852f9be} {Suffrage:Voter ID:server-abb6123a-6d6d-64d4-eaf2-8d4aaf247a57 Address:abb6123a-6d6d-64d4-eaf2-8d4aaf247a57}]" -2024-05-12T20:51:36.041Z [INFO] server-abb6123a-6d6d-64d4-eaf2-8d4aaf247a57: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9257bcb9-02ac-153a-7542-a5dc03b02104 Address:9257bcb9-02ac-153a-7542-a5dc03b02104} {Suffrage:Voter ID:server-bbe6d849-79b9-820e-259b-ccb3f852f9be Address:bbe6d849-79b9-820e-259b-ccb3f852f9be} {Suffrage:Voter ID:server-abb6123a-6d6d-64d4-eaf2-8d4aaf247a57 Address:abb6123a-6d6d-64d4-eaf2-8d4aaf247a57}]" -2024-05-12T20:51:36.041Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:36.041Z [INFO] server-9257bcb9-02ac-153a-7542-a5dc03b02104: entering follower state: follower="Node at 9257bcb9-02ac-153a-7542-a5dc03b02104 [Follower]" leader-address= leader-id= -2024-05-12T20:51:36.041Z [INFO] server-bbe6d849-79b9-820e-259b-ccb3f852f9be: entering follower state: follower="Node at bbe6d849-79b9-820e-259b-ccb3f852f9be [Follower]" leader-address= leader-id= -2024-05-12T20:51:36.041Z [INFO] server-abb6123a-6d6d-64d4-eaf2-8d4aaf247a57: entering follower state: follower="Node at abb6123a-6d6d-64d4-eaf2-8d4aaf247a57 [Follower]" leader-address= leader-id= -2024-05-12T20:51:36.106Z [WARN] server-bbe6d849-79b9-820e-259b-ccb3f852f9be: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:36.106Z [INFO] server-bbe6d849-79b9-820e-259b-ccb3f852f9be: entering candidate state: node="Node at bbe6d849-79b9-820e-259b-ccb3f852f9be [Candidate]" term=2 -2024-05-12T20:51:36.107Z [INFO] server-bbe6d849-79b9-820e-259b-ccb3f852f9be: election won: term=2 tally=2 -2024-05-12T20:51:36.107Z [INFO] server-bbe6d849-79b9-820e-259b-ccb3f852f9be: entering leader state: leader="Node at bbe6d849-79b9-820e-259b-ccb3f852f9be [Leader]" -2024-05-12T20:51:36.107Z [INFO] server-bbe6d849-79b9-820e-259b-ccb3f852f9be: added peer, starting replication: peer=server-9257bcb9-02ac-153a-7542-a5dc03b02104 -2024-05-12T20:51:36.107Z [INFO] server-bbe6d849-79b9-820e-259b-ccb3f852f9be: added peer, starting replication: peer=server-abb6123a-6d6d-64d4-eaf2-8d4aaf247a57 -2024-05-12T20:51:36.107Z [INFO] server-bbe6d849-79b9-820e-259b-ccb3f852f9be: pipelining replication: peer="{Voter server-abb6123a-6d6d-64d4-eaf2-8d4aaf247a57 abb6123a-6d6d-64d4-eaf2-8d4aaf247a57}" -2024-05-12T20:51:36.107Z [INFO] server-bbe6d849-79b9-820e-259b-ccb3f852f9be: pipelining replication: peer="{Voter server-9257bcb9-02ac-153a-7542-a5dc03b02104 9257bcb9-02ac-153a-7542-a5dc03b02104}" -2024-05-12T20:51:36.214Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:36.107451498 +0000 UTC m=+107.388151943 (2 nodes), 66.006468ms from start of poll, 66.451582ms from cluster start. Timeout at 2024-05-12 20:51:36.214784855 +0000 UTC m=+107.495485425, 107.333482ms after stability -2024-05-12T20:51:36.214Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:36.331Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:36.214954963 +0000 UTC m=+107.495655430 (1 nodes), 2.102µs from start of poll, 173.955069ms from cluster start. Timeout at 2024-05-12 20:51:36.322810478 +0000 UTC m=+107.603511193, 107.855763ms after stability -2024-05-12T20:51:36.333Z [INFO] server-bbe6d849-79b9-820e-259b-ccb3f852f9be: aborting pipeline replication: peer="{Voter server-abb6123a-6d6d-64d4-eaf2-8d4aaf247a57 abb6123a-6d6d-64d4-eaf2-8d4aaf247a57}" -2024-05-12T20:51:36.333Z [INFO] server-bbe6d849-79b9-820e-259b-ccb3f852f9be: aborting pipeline replication: peer="{Voter server-9257bcb9-02ac-153a-7542-a5dc03b02104 9257bcb9-02ac-153a-7542-a5dc03b02104}" ---- PASS: TestRaft_LeadershipTransferToInvalidAddress (0.29s) +2025-06-15T03:18:08.492Z [INFO] server-62755811-963d-0915-8c0f-0a737fc67b0f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-62755811-963d-0915-8c0f-0a737fc67b0f Address:62755811-963d-0915-8c0f-0a737fc67b0f} {Suffrage:Voter ID:server-0602d639-173e-d5f3-a2ba-c8983beadf78 Address:0602d639-173e-d5f3-a2ba-c8983beadf78} {Suffrage:Voter ID:server-f4bbc1e5-d226-34d4-b46e-a0592c053e57 Address:f4bbc1e5-d226-34d4-b46e-a0592c053e57}]" +2025-06-15T03:18:08.492Z [INFO] server-0602d639-173e-d5f3-a2ba-c8983beadf78: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-62755811-963d-0915-8c0f-0a737fc67b0f Address:62755811-963d-0915-8c0f-0a737fc67b0f} {Suffrage:Voter ID:server-0602d639-173e-d5f3-a2ba-c8983beadf78 Address:0602d639-173e-d5f3-a2ba-c8983beadf78} {Suffrage:Voter ID:server-f4bbc1e5-d226-34d4-b46e-a0592c053e57 Address:f4bbc1e5-d226-34d4-b46e-a0592c053e57}]" +2025-06-15T03:18:08.492Z [INFO] server-62755811-963d-0915-8c0f-0a737fc67b0f: entering follower state: follower="Node at 62755811-963d-0915-8c0f-0a737fc67b0f [Follower]" leader-address= leader-id= +2025-06-15T03:18:08.492Z [INFO] server-0602d639-173e-d5f3-a2ba-c8983beadf78: entering follower state: follower="Node at 0602d639-173e-d5f3-a2ba-c8983beadf78 [Follower]" leader-address= leader-id= +2025-06-15T03:18:08.492Z [INFO] server-f4bbc1e5-d226-34d4-b46e-a0592c053e57: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-62755811-963d-0915-8c0f-0a737fc67b0f Address:62755811-963d-0915-8c0f-0a737fc67b0f} {Suffrage:Voter ID:server-0602d639-173e-d5f3-a2ba-c8983beadf78 Address:0602d639-173e-d5f3-a2ba-c8983beadf78} {Suffrage:Voter ID:server-f4bbc1e5-d226-34d4-b46e-a0592c053e57 Address:f4bbc1e5-d226-34d4-b46e-a0592c053e57}]" +2025-06-15T03:18:08.492Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:08.492Z [INFO] server-f4bbc1e5-d226-34d4-b46e-a0592c053e57: entering follower state: follower="Node at f4bbc1e5-d226-34d4-b46e-a0592c053e57 [Follower]" leader-address= leader-id= +2025-06-15T03:18:08.553Z [WARN] server-62755811-963d-0915-8c0f-0a737fc67b0f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:08.553Z [INFO] server-62755811-963d-0915-8c0f-0a737fc67b0f: entering candidate state: node="Node at 62755811-963d-0915-8c0f-0a737fc67b0f [Candidate]" term=2 +2025-06-15T03:18:08.553Z [INFO] server-62755811-963d-0915-8c0f-0a737fc67b0f: election won: term=2 tally=2 +2025-06-15T03:18:08.553Z [INFO] server-62755811-963d-0915-8c0f-0a737fc67b0f: entering leader state: leader="Node at 62755811-963d-0915-8c0f-0a737fc67b0f [Leader]" +2025-06-15T03:18:08.553Z [INFO] server-62755811-963d-0915-8c0f-0a737fc67b0f: added peer, starting replication: peer=server-0602d639-173e-d5f3-a2ba-c8983beadf78 +2025-06-15T03:18:08.553Z [INFO] server-62755811-963d-0915-8c0f-0a737fc67b0f: added peer, starting replication: peer=server-f4bbc1e5-d226-34d4-b46e-a0592c053e57 +2025-06-15T03:18:08.553Z [INFO] server-62755811-963d-0915-8c0f-0a737fc67b0f: pipelining replication: peer="{Voter server-f4bbc1e5-d226-34d4-b46e-a0592c053e57 f4bbc1e5-d226-34d4-b46e-a0592c053e57}" +2025-06-15T03:18:08.553Z [INFO] server-62755811-963d-0915-8c0f-0a737fc67b0f: pipelining replication: peer="{Voter server-0602d639-173e-d5f3-a2ba-c8983beadf78 0602d639-173e-d5f3-a2ba-c8983beadf78}" +2025-06-15T03:18:08.659Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:08.553781412 +0000 UTC m=+104.265459574 (2 nodes), 61.280457ms from start of poll, 61.53913ms from cluster start. Timeout at 2025-06-15 03:18:08.659390989 +0000 UTC m=+104.371069156, 105.609582ms after stability +2025-06-15T03:18:08.659Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:08.765Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:08.659434261 +0000 UTC m=+104.371112426 (1 nodes), 441ns from start of poll, 167.191982ms from cluster start. Timeout at 2025-06-15 03:18:08.765528117 +0000 UTC m=+104.477206316, 106.09389ms after stability +2025-06-15T03:18:08.765Z [INFO] server-62755811-963d-0915-8c0f-0a737fc67b0f: aborting pipeline replication: peer="{Voter server-f4bbc1e5-d226-34d4-b46e-a0592c053e57 f4bbc1e5-d226-34d4-b46e-a0592c053e57}" +2025-06-15T03:18:08.765Z [INFO] server-62755811-963d-0915-8c0f-0a737fc67b0f: aborting pipeline replication: peer="{Voter server-0602d639-173e-d5f3-a2ba-c8983beadf78 0602d639-173e-d5f3-a2ba-c8983beadf78}" +--- PASS: TestRaft_LeadershipTransferToInvalidAddress (0.27s) === RUN TestRaft_LeadershipTransferToBehindServer -2024-05-12T20:51:36.335Z [INFO] server-ed137cfb-28e7-963b-d858-5068b6379982: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ed137cfb-28e7-963b-d858-5068b6379982 Address:ed137cfb-28e7-963b-d858-5068b6379982} {Suffrage:Voter ID:server-707c4513-7eac-3744-e069-c996f3df7ef5 Address:707c4513-7eac-3744-e069-c996f3df7ef5} {Suffrage:Voter ID:server-299d3c2b-d507-3c3f-ee17-9c72098b3e99 Address:299d3c2b-d507-3c3f-ee17-9c72098b3e99}]" -2024-05-12T20:51:36.335Z [INFO] server-707c4513-7eac-3744-e069-c996f3df7ef5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ed137cfb-28e7-963b-d858-5068b6379982 Address:ed137cfb-28e7-963b-d858-5068b6379982} {Suffrage:Voter ID:server-707c4513-7eac-3744-e069-c996f3df7ef5 Address:707c4513-7eac-3744-e069-c996f3df7ef5} {Suffrage:Voter ID:server-299d3c2b-d507-3c3f-ee17-9c72098b3e99 Address:299d3c2b-d507-3c3f-ee17-9c72098b3e99}]" -2024-05-12T20:51:36.335Z [INFO] server-299d3c2b-d507-3c3f-ee17-9c72098b3e99: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ed137cfb-28e7-963b-d858-5068b6379982 Address:ed137cfb-28e7-963b-d858-5068b6379982} {Suffrage:Voter ID:server-707c4513-7eac-3744-e069-c996f3df7ef5 Address:707c4513-7eac-3744-e069-c996f3df7ef5} {Suffrage:Voter ID:server-299d3c2b-d507-3c3f-ee17-9c72098b3e99 Address:299d3c2b-d507-3c3f-ee17-9c72098b3e99}]" -2024-05-12T20:51:36.335Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:36.335Z [INFO] server-ed137cfb-28e7-963b-d858-5068b6379982: entering follower state: follower="Node at ed137cfb-28e7-963b-d858-5068b6379982 [Follower]" leader-address= leader-id= -2024-05-12T20:51:36.335Z [INFO] server-707c4513-7eac-3744-e069-c996f3df7ef5: entering follower state: follower="Node at 707c4513-7eac-3744-e069-c996f3df7ef5 [Follower]" leader-address= leader-id= -2024-05-12T20:51:36.335Z [INFO] server-299d3c2b-d507-3c3f-ee17-9c72098b3e99: entering follower state: follower="Node at 299d3c2b-d507-3c3f-ee17-9c72098b3e99 [Follower]" leader-address= leader-id= -2024-05-12T20:51:36.403Z [WARN] server-707c4513-7eac-3744-e069-c996f3df7ef5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:36.403Z [INFO] server-707c4513-7eac-3744-e069-c996f3df7ef5: entering candidate state: node="Node at 707c4513-7eac-3744-e069-c996f3df7ef5 [Candidate]" term=2 -2024-05-12T20:51:36.403Z [WARN] server-ed137cfb-28e7-963b-d858-5068b6379982: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:36.403Z [INFO] server-ed137cfb-28e7-963b-d858-5068b6379982: entering candidate state: node="Node at ed137cfb-28e7-963b-d858-5068b6379982 [Candidate]" term=2 -2024-05-12T20:51:36.403Z [INFO] server-ed137cfb-28e7-963b-d858-5068b6379982: election won: term=2 tally=2 -2024-05-12T20:51:36.403Z [INFO] server-ed137cfb-28e7-963b-d858-5068b6379982: entering leader state: leader="Node at ed137cfb-28e7-963b-d858-5068b6379982 [Leader]" -2024-05-12T20:51:36.403Z [INFO] server-ed137cfb-28e7-963b-d858-5068b6379982: added peer, starting replication: peer=server-707c4513-7eac-3744-e069-c996f3df7ef5 -2024-05-12T20:51:36.403Z [INFO] server-ed137cfb-28e7-963b-d858-5068b6379982: added peer, starting replication: peer=server-299d3c2b-d507-3c3f-ee17-9c72098b3e99 -2024-05-12T20:51:36.404Z [INFO] server-ed137cfb-28e7-963b-d858-5068b6379982: pipelining replication: peer="{Voter server-299d3c2b-d507-3c3f-ee17-9c72098b3e99 299d3c2b-d507-3c3f-ee17-9c72098b3e99}" -2024-05-12T20:51:36.404Z [WARN] server-299d3c2b-d507-3c3f-ee17-9c72098b3e99: rejecting vote request since we have a leader: from=707c4513-7eac-3744-e069-c996f3df7ef5 leader=ed137cfb-28e7-963b-d858-5068b6379982 leader-id=server-ed137cfb-28e7-963b-d858-5068b6379982 -2024-05-12T20:51:36.404Z [INFO] server-707c4513-7eac-3744-e069-c996f3df7ef5: duplicate requestVote for same term: term=2 -2024-05-12T20:51:36.404Z [INFO] server-707c4513-7eac-3744-e069-c996f3df7ef5: entering follower state: follower="Node at 707c4513-7eac-3744-e069-c996f3df7ef5 [Follower]" leader-address=ed137cfb-28e7-963b-d858-5068b6379982 leader-id=server-ed137cfb-28e7-963b-d858-5068b6379982 -2024-05-12T20:51:36.404Z [INFO] server-ed137cfb-28e7-963b-d858-5068b6379982: pipelining replication: peer="{Voter server-707c4513-7eac-3744-e069-c996f3df7ef5 707c4513-7eac-3744-e069-c996f3df7ef5}" -2024-05-12T20:51:36.404Z [WARN] server-ed137cfb-28e7-963b-d858-5068b6379982: rejecting vote request since we have a leader: from=707c4513-7eac-3744-e069-c996f3df7ef5 leader=ed137cfb-28e7-963b-d858-5068b6379982 leader-id=server-ed137cfb-28e7-963b-d858-5068b6379982 -2024-05-12T20:51:36.510Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:36.404910324 +0000 UTC m=+107.685610677 (1 nodes), 69.232778ms from start of poll, 69.731626ms from cluster start. Timeout at 2024-05-12 20:51:36.510570267 +0000 UTC m=+107.791270655, 105.659978ms after stability -2024-05-12T20:51:36.510Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:36.618Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:36.510664824 +0000 UTC m=+107.791365199 (2 nodes), 2.535µs from start of poll, 175.486148ms from cluster start. Timeout at 2024-05-12 20:51:36.618807647 +0000 UTC m=+107.899508460, 108.143261ms after stability -2024-05-12T20:51:36.875Z [INFO] server-707c4513-7eac-3744-e069-c996f3df7ef5: entering candidate state: node="Node at 707c4513-7eac-3744-e069-c996f3df7ef5 [Candidate]" term=3 -2024-05-12T20:51:36.875Z [INFO] server-707c4513-7eac-3744-e069-c996f3df7ef5: election won: term=3 tally=2 -2024-05-12T20:51:36.875Z [INFO] server-707c4513-7eac-3744-e069-c996f3df7ef5: entering leader state: leader="Node at 707c4513-7eac-3744-e069-c996f3df7ef5 [Leader]" -2024-05-12T20:51:36.875Z [INFO] server-707c4513-7eac-3744-e069-c996f3df7ef5: added peer, starting replication: peer=server-ed137cfb-28e7-963b-d858-5068b6379982 -2024-05-12T20:51:36.875Z [INFO] server-707c4513-7eac-3744-e069-c996f3df7ef5: added peer, starting replication: peer=server-299d3c2b-d507-3c3f-ee17-9c72098b3e99 -2024-05-12T20:51:36.875Z [INFO] server-707c4513-7eac-3744-e069-c996f3df7ef5: pipelining replication: peer="{Voter server-299d3c2b-d507-3c3f-ee17-9c72098b3e99 299d3c2b-d507-3c3f-ee17-9c72098b3e99}" -2024-05-12T20:51:36.876Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:36.876Z [INFO] server-ed137cfb-28e7-963b-d858-5068b6379982: entering follower state: follower="Node at ed137cfb-28e7-963b-d858-5068b6379982 [Follower]" leader-address= leader-id= -2024-05-12T20:51:36.876Z [INFO] server-ed137cfb-28e7-963b-d858-5068b6379982: aborting pipeline replication: peer="{Voter server-299d3c2b-d507-3c3f-ee17-9c72098b3e99 299d3c2b-d507-3c3f-ee17-9c72098b3e99}" -2024-05-12T20:51:36.876Z [INFO] server-707c4513-7eac-3744-e069-c996f3df7ef5: pipelining replication: peer="{Voter server-ed137cfb-28e7-963b-d858-5068b6379982 ed137cfb-28e7-963b-d858-5068b6379982}" -2024-05-12T20:51:36.876Z [INFO] server-ed137cfb-28e7-963b-d858-5068b6379982: aborting pipeline replication: peer="{Voter server-707c4513-7eac-3744-e069-c996f3df7ef5 707c4513-7eac-3744-e069-c996f3df7ef5}" -2024-05-12T20:51:36.992Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:36.876618122 +0000 UTC m=+108.157318608 (1 nodes), 430.439µs from start of poll, 541.439557ms from cluster start. Timeout at 2024-05-12 20:51:36.99192741 +0000 UTC m=+108.272628134, 115.309526ms after stability -2024-05-12T20:51:36.992Z [INFO] server-707c4513-7eac-3744-e069-c996f3df7ef5: aborting pipeline replication: peer="{Voter server-299d3c2b-d507-3c3f-ee17-9c72098b3e99 299d3c2b-d507-3c3f-ee17-9c72098b3e99}" -2024-05-12T20:51:36.992Z [INFO] server-707c4513-7eac-3744-e069-c996f3df7ef5: aborting pipeline replication: peer="{Voter server-ed137cfb-28e7-963b-d858-5068b6379982 ed137cfb-28e7-963b-d858-5068b6379982}" ---- PASS: TestRaft_LeadershipTransferToBehindServer (0.67s) +2025-06-15T03:18:08.767Z [INFO] server-de0e664b-253c-ef46-6292-95c882ae338f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-de0e664b-253c-ef46-6292-95c882ae338f Address:de0e664b-253c-ef46-6292-95c882ae338f} {Suffrage:Voter ID:server-004d57d6-aaec-c680-de34-5bad839d0a4a Address:004d57d6-aaec-c680-de34-5bad839d0a4a} {Suffrage:Voter ID:server-4ff5c366-6bcd-e771-2790-8b7a27621a22 Address:4ff5c366-6bcd-e771-2790-8b7a27621a22}]" +2025-06-15T03:18:08.767Z [INFO] server-de0e664b-253c-ef46-6292-95c882ae338f: entering follower state: follower="Node at de0e664b-253c-ef46-6292-95c882ae338f [Follower]" leader-address= leader-id= +2025-06-15T03:18:08.767Z [INFO] server-004d57d6-aaec-c680-de34-5bad839d0a4a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-de0e664b-253c-ef46-6292-95c882ae338f Address:de0e664b-253c-ef46-6292-95c882ae338f} {Suffrage:Voter ID:server-004d57d6-aaec-c680-de34-5bad839d0a4a Address:004d57d6-aaec-c680-de34-5bad839d0a4a} {Suffrage:Voter ID:server-4ff5c366-6bcd-e771-2790-8b7a27621a22 Address:4ff5c366-6bcd-e771-2790-8b7a27621a22}]" +2025-06-15T03:18:08.767Z [INFO] server-004d57d6-aaec-c680-de34-5bad839d0a4a: entering follower state: follower="Node at 004d57d6-aaec-c680-de34-5bad839d0a4a [Follower]" leader-address= leader-id= +2025-06-15T03:18:08.767Z [INFO] server-4ff5c366-6bcd-e771-2790-8b7a27621a22: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-de0e664b-253c-ef46-6292-95c882ae338f Address:de0e664b-253c-ef46-6292-95c882ae338f} {Suffrage:Voter ID:server-004d57d6-aaec-c680-de34-5bad839d0a4a Address:004d57d6-aaec-c680-de34-5bad839d0a4a} {Suffrage:Voter ID:server-4ff5c366-6bcd-e771-2790-8b7a27621a22 Address:4ff5c366-6bcd-e771-2790-8b7a27621a22}]" +2025-06-15T03:18:08.767Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:08.767Z [INFO] server-4ff5c366-6bcd-e771-2790-8b7a27621a22: entering follower state: follower="Node at 4ff5c366-6bcd-e771-2790-8b7a27621a22 [Follower]" leader-address= leader-id= +2025-06-15T03:18:08.846Z [WARN] server-004d57d6-aaec-c680-de34-5bad839d0a4a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:08.846Z [INFO] server-004d57d6-aaec-c680-de34-5bad839d0a4a: entering candidate state: node="Node at 004d57d6-aaec-c680-de34-5bad839d0a4a [Candidate]" term=2 +2025-06-15T03:18:08.846Z [INFO] server-004d57d6-aaec-c680-de34-5bad839d0a4a: election won: term=2 tally=2 +2025-06-15T03:18:08.846Z [INFO] server-004d57d6-aaec-c680-de34-5bad839d0a4a: entering leader state: leader="Node at 004d57d6-aaec-c680-de34-5bad839d0a4a [Leader]" +2025-06-15T03:18:08.846Z [INFO] server-004d57d6-aaec-c680-de34-5bad839d0a4a: added peer, starting replication: peer=server-de0e664b-253c-ef46-6292-95c882ae338f +2025-06-15T03:18:08.846Z [INFO] server-004d57d6-aaec-c680-de34-5bad839d0a4a: added peer, starting replication: peer=server-4ff5c366-6bcd-e771-2790-8b7a27621a22 +2025-06-15T03:18:08.846Z [INFO] server-004d57d6-aaec-c680-de34-5bad839d0a4a: pipelining replication: peer="{Voter server-4ff5c366-6bcd-e771-2790-8b7a27621a22 4ff5c366-6bcd-e771-2790-8b7a27621a22}" +2025-06-15T03:18:08.846Z [INFO] server-004d57d6-aaec-c680-de34-5bad839d0a4a: pipelining replication: peer="{Voter server-de0e664b-253c-ef46-6292-95c882ae338f de0e664b-253c-ef46-6292-95c882ae338f}" +2025-06-15T03:18:08.952Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:08.846792495 +0000 UTC m=+104.558470694 (1 nodes), 79.039718ms from start of poll, 79.83632ms from cluster start. Timeout at 2025-06-15 03:18:08.952088886 +0000 UTC m=+104.663767086, 105.296392ms after stability +2025-06-15T03:18:08.952Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:09.058Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:08.952163235 +0000 UTC m=+104.663841434 (2 nodes), 1.37µs from start of poll, 185.20706ms from cluster start. Timeout at 2025-06-15 03:18:09.058030901 +0000 UTC m=+104.769709067, 105.867633ms after stability +2025-06-15T03:18:09.083Z [INFO] server-de0e664b-253c-ef46-6292-95c882ae338f: entering candidate state: node="Node at de0e664b-253c-ef46-6292-95c882ae338f [Candidate]" term=3 +2025-06-15T03:18:09.083Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:09.083Z [INFO] server-de0e664b-253c-ef46-6292-95c882ae338f: election won: term=3 tally=2 +2025-06-15T03:18:09.083Z [INFO] server-de0e664b-253c-ef46-6292-95c882ae338f: entering leader state: leader="Node at de0e664b-253c-ef46-6292-95c882ae338f [Leader]" +2025-06-15T03:18:09.083Z [INFO] server-004d57d6-aaec-c680-de34-5bad839d0a4a: entering follower state: follower="Node at 004d57d6-aaec-c680-de34-5bad839d0a4a [Follower]" leader-address= leader-id= +2025-06-15T03:18:09.083Z [INFO] server-de0e664b-253c-ef46-6292-95c882ae338f: added peer, starting replication: peer=server-004d57d6-aaec-c680-de34-5bad839d0a4a +2025-06-15T03:18:09.083Z [INFO] server-de0e664b-253c-ef46-6292-95c882ae338f: added peer, starting replication: peer=server-4ff5c366-6bcd-e771-2790-8b7a27621a22 +2025-06-15T03:18:09.083Z [INFO] server-004d57d6-aaec-c680-de34-5bad839d0a4a: aborting pipeline replication: peer="{Voter server-de0e664b-253c-ef46-6292-95c882ae338f de0e664b-253c-ef46-6292-95c882ae338f}" +2025-06-15T03:18:09.083Z [INFO] server-004d57d6-aaec-c680-de34-5bad839d0a4a: aborting pipeline replication: peer="{Voter server-4ff5c366-6bcd-e771-2790-8b7a27621a22 4ff5c366-6bcd-e771-2790-8b7a27621a22}" +2025-06-15T03:18:09.084Z [INFO] server-de0e664b-253c-ef46-6292-95c882ae338f: pipelining replication: peer="{Voter server-4ff5c366-6bcd-e771-2790-8b7a27621a22 4ff5c366-6bcd-e771-2790-8b7a27621a22}" +2025-06-15T03:18:09.084Z [INFO] server-de0e664b-253c-ef46-6292-95c882ae338f: pipelining replication: peer="{Voter server-004d57d6-aaec-c680-de34-5bad839d0a4a 004d57d6-aaec-c680-de34-5bad839d0a4a}" +2025-06-15T03:18:09.189Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:09.08390389 +0000 UTC m=+104.795582060 (1 nodes), 54.261µs from start of poll, 316.947686ms from cluster start. Timeout at 2025-06-15 03:18:09.188934372 +0000 UTC m=+104.900612571, 105.030511ms after stability +2025-06-15T03:18:09.189Z [INFO] server-de0e664b-253c-ef46-6292-95c882ae338f: aborting pipeline replication: peer="{Voter server-4ff5c366-6bcd-e771-2790-8b7a27621a22 4ff5c366-6bcd-e771-2790-8b7a27621a22}" +2025-06-15T03:18:09.189Z [INFO] server-de0e664b-253c-ef46-6292-95c882ae338f: aborting pipeline replication: peer="{Voter server-004d57d6-aaec-c680-de34-5bad839d0a4a 004d57d6-aaec-c680-de34-5bad839d0a4a}" +--- PASS: TestRaft_LeadershipTransferToBehindServer (0.42s) === RUN TestRaft_LeadershipTransferToItself -2024-05-12T20:51:37.008Z [INFO] server-cd44e7ae-9cf7-47d5-4cbc-534b98079c7e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd44e7ae-9cf7-47d5-4cbc-534b98079c7e Address:cd44e7ae-9cf7-47d5-4cbc-534b98079c7e} {Suffrage:Voter ID:server-2c000467-0a8a-3aed-4d83-b5bf51887d32 Address:2c000467-0a8a-3aed-4d83-b5bf51887d32} {Suffrage:Voter ID:server-982db2a9-b0b1-f7bf-7389-53dc46fed7f9 Address:982db2a9-b0b1-f7bf-7389-53dc46fed7f9}]" -2024-05-12T20:51:37.008Z [INFO] server-2c000467-0a8a-3aed-4d83-b5bf51887d32: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd44e7ae-9cf7-47d5-4cbc-534b98079c7e Address:cd44e7ae-9cf7-47d5-4cbc-534b98079c7e} {Suffrage:Voter ID:server-2c000467-0a8a-3aed-4d83-b5bf51887d32 Address:2c000467-0a8a-3aed-4d83-b5bf51887d32} {Suffrage:Voter ID:server-982db2a9-b0b1-f7bf-7389-53dc46fed7f9 Address:982db2a9-b0b1-f7bf-7389-53dc46fed7f9}]" -2024-05-12T20:51:37.008Z [INFO] server-982db2a9-b0b1-f7bf-7389-53dc46fed7f9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd44e7ae-9cf7-47d5-4cbc-534b98079c7e Address:cd44e7ae-9cf7-47d5-4cbc-534b98079c7e} {Suffrage:Voter ID:server-2c000467-0a8a-3aed-4d83-b5bf51887d32 Address:2c000467-0a8a-3aed-4d83-b5bf51887d32} {Suffrage:Voter ID:server-982db2a9-b0b1-f7bf-7389-53dc46fed7f9 Address:982db2a9-b0b1-f7bf-7389-53dc46fed7f9}]" -2024-05-12T20:51:37.008Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:37.008Z [INFO] server-cd44e7ae-9cf7-47d5-4cbc-534b98079c7e: entering follower state: follower="Node at cd44e7ae-9cf7-47d5-4cbc-534b98079c7e [Follower]" leader-address= leader-id= -2024-05-12T20:51:37.008Z [INFO] server-2c000467-0a8a-3aed-4d83-b5bf51887d32: entering follower state: follower="Node at 2c000467-0a8a-3aed-4d83-b5bf51887d32 [Follower]" leader-address= leader-id= -2024-05-12T20:51:37.008Z [INFO] server-982db2a9-b0b1-f7bf-7389-53dc46fed7f9: entering follower state: follower="Node at 982db2a9-b0b1-f7bf-7389-53dc46fed7f9 [Follower]" leader-address= leader-id= -2024-05-12T20:51:37.070Z [WARN] server-982db2a9-b0b1-f7bf-7389-53dc46fed7f9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:37.071Z [INFO] server-982db2a9-b0b1-f7bf-7389-53dc46fed7f9: entering candidate state: node="Node at 982db2a9-b0b1-f7bf-7389-53dc46fed7f9 [Candidate]" term=2 -2024-05-12T20:51:37.071Z [INFO] server-982db2a9-b0b1-f7bf-7389-53dc46fed7f9: election won: term=2 tally=2 -2024-05-12T20:51:37.071Z [INFO] server-982db2a9-b0b1-f7bf-7389-53dc46fed7f9: entering leader state: leader="Node at 982db2a9-b0b1-f7bf-7389-53dc46fed7f9 [Leader]" -2024-05-12T20:51:37.071Z [INFO] server-982db2a9-b0b1-f7bf-7389-53dc46fed7f9: added peer, starting replication: peer=server-cd44e7ae-9cf7-47d5-4cbc-534b98079c7e -2024-05-12T20:51:37.071Z [INFO] server-982db2a9-b0b1-f7bf-7389-53dc46fed7f9: added peer, starting replication: peer=server-2c000467-0a8a-3aed-4d83-b5bf51887d32 -2024-05-12T20:51:37.071Z [INFO] server-982db2a9-b0b1-f7bf-7389-53dc46fed7f9: pipelining replication: peer="{Voter server-2c000467-0a8a-3aed-4d83-b5bf51887d32 2c000467-0a8a-3aed-4d83-b5bf51887d32}" -2024-05-12T20:51:37.071Z [INFO] server-982db2a9-b0b1-f7bf-7389-53dc46fed7f9: pipelining replication: peer="{Voter server-cd44e7ae-9cf7-47d5-4cbc-534b98079c7e cd44e7ae-9cf7-47d5-4cbc-534b98079c7e}" -2024-05-12T20:51:37.178Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:37.071686361 +0000 UTC m=+108.352386901 (1 nodes), 63.313345ms from start of poll, 63.855739ms from cluster start. Timeout at 2024-05-12 20:51:37.1787741 +0000 UTC m=+108.459474518, 107.087617ms after stability -2024-05-12T20:51:37.178Z [INFO] server-982db2a9-b0b1-f7bf-7389-53dc46fed7f9: cannot transfer leadership to itself -2024-05-12T20:51:37.179Z [INFO] server-982db2a9-b0b1-f7bf-7389-53dc46fed7f9: aborting pipeline replication: peer="{Voter server-2c000467-0a8a-3aed-4d83-b5bf51887d32 2c000467-0a8a-3aed-4d83-b5bf51887d32}" -2024-05-12T20:51:37.179Z [INFO] server-982db2a9-b0b1-f7bf-7389-53dc46fed7f9: aborting pipeline replication: peer="{Voter server-cd44e7ae-9cf7-47d5-4cbc-534b98079c7e cd44e7ae-9cf7-47d5-4cbc-534b98079c7e}" ---- PASS: TestRaft_LeadershipTransferToItself (0.17s) +2025-06-15T03:18:09.190Z [INFO] server-7be80eea-0d5b-e2b0-17c1-7fbbf2e34707: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7be80eea-0d5b-e2b0-17c1-7fbbf2e34707 Address:7be80eea-0d5b-e2b0-17c1-7fbbf2e34707} {Suffrage:Voter ID:server-1a46214b-9894-5599-32ad-72c0b81ed1c5 Address:1a46214b-9894-5599-32ad-72c0b81ed1c5} {Suffrage:Voter ID:server-4b9fa2c9-b655-1bb6-729b-07288e3ef244 Address:4b9fa2c9-b655-1bb6-729b-07288e3ef244}]" +2025-06-15T03:18:09.190Z [INFO] server-7be80eea-0d5b-e2b0-17c1-7fbbf2e34707: entering follower state: follower="Node at 7be80eea-0d5b-e2b0-17c1-7fbbf2e34707 [Follower]" leader-address= leader-id= +2025-06-15T03:18:09.190Z [INFO] server-1a46214b-9894-5599-32ad-72c0b81ed1c5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7be80eea-0d5b-e2b0-17c1-7fbbf2e34707 Address:7be80eea-0d5b-e2b0-17c1-7fbbf2e34707} {Suffrage:Voter ID:server-1a46214b-9894-5599-32ad-72c0b81ed1c5 Address:1a46214b-9894-5599-32ad-72c0b81ed1c5} {Suffrage:Voter ID:server-4b9fa2c9-b655-1bb6-729b-07288e3ef244 Address:4b9fa2c9-b655-1bb6-729b-07288e3ef244}]" +2025-06-15T03:18:09.190Z [INFO] server-1a46214b-9894-5599-32ad-72c0b81ed1c5: entering follower state: follower="Node at 1a46214b-9894-5599-32ad-72c0b81ed1c5 [Follower]" leader-address= leader-id= +2025-06-15T03:18:09.191Z [INFO] server-4b9fa2c9-b655-1bb6-729b-07288e3ef244: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7be80eea-0d5b-e2b0-17c1-7fbbf2e34707 Address:7be80eea-0d5b-e2b0-17c1-7fbbf2e34707} {Suffrage:Voter ID:server-1a46214b-9894-5599-32ad-72c0b81ed1c5 Address:1a46214b-9894-5599-32ad-72c0b81ed1c5} {Suffrage:Voter ID:server-4b9fa2c9-b655-1bb6-729b-07288e3ef244 Address:4b9fa2c9-b655-1bb6-729b-07288e3ef244}]" +2025-06-15T03:18:09.191Z [INFO] server-4b9fa2c9-b655-1bb6-729b-07288e3ef244: entering follower state: follower="Node at 4b9fa2c9-b655-1bb6-729b-07288e3ef244 [Follower]" leader-address= leader-id= +2025-06-15T03:18:09.191Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:09.274Z [WARN] server-4b9fa2c9-b655-1bb6-729b-07288e3ef244: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:09.274Z [INFO] server-4b9fa2c9-b655-1bb6-729b-07288e3ef244: entering candidate state: node="Node at 4b9fa2c9-b655-1bb6-729b-07288e3ef244 [Candidate]" term=2 +2025-06-15T03:18:09.274Z [INFO] server-4b9fa2c9-b655-1bb6-729b-07288e3ef244: election won: term=2 tally=2 +2025-06-15T03:18:09.274Z [INFO] server-4b9fa2c9-b655-1bb6-729b-07288e3ef244: entering leader state: leader="Node at 4b9fa2c9-b655-1bb6-729b-07288e3ef244 [Leader]" +2025-06-15T03:18:09.274Z [INFO] server-4b9fa2c9-b655-1bb6-729b-07288e3ef244: added peer, starting replication: peer=server-7be80eea-0d5b-e2b0-17c1-7fbbf2e34707 +2025-06-15T03:18:09.274Z [INFO] server-4b9fa2c9-b655-1bb6-729b-07288e3ef244: added peer, starting replication: peer=server-1a46214b-9894-5599-32ad-72c0b81ed1c5 +2025-06-15T03:18:09.274Z [INFO] server-4b9fa2c9-b655-1bb6-729b-07288e3ef244: pipelining replication: peer="{Voter server-7be80eea-0d5b-e2b0-17c1-7fbbf2e34707 7be80eea-0d5b-e2b0-17c1-7fbbf2e34707}" +2025-06-15T03:18:09.274Z [INFO] server-4b9fa2c9-b655-1bb6-729b-07288e3ef244: pipelining replication: peer="{Voter server-1a46214b-9894-5599-32ad-72c0b81ed1c5 1a46214b-9894-5599-32ad-72c0b81ed1c5}" +2025-06-15T03:18:09.379Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:09.274241137 +0000 UTC m=+104.985919298 (1 nodes), 82.789737ms from start of poll, 83.713207ms from cluster start. Timeout at 2025-06-15 03:18:09.379594727 +0000 UTC m=+105.091272893, 105.353595ms after stability +2025-06-15T03:18:09.379Z [INFO] server-4b9fa2c9-b655-1bb6-729b-07288e3ef244: cannot transfer leadership to itself +2025-06-15T03:18:09.379Z [INFO] server-4b9fa2c9-b655-1bb6-729b-07288e3ef244: aborting pipeline replication: peer="{Voter server-1a46214b-9894-5599-32ad-72c0b81ed1c5 1a46214b-9894-5599-32ad-72c0b81ed1c5}" +2025-06-15T03:18:09.379Z [INFO] server-4b9fa2c9-b655-1bb6-729b-07288e3ef244: aborting pipeline replication: peer="{Voter server-7be80eea-0d5b-e2b0-17c1-7fbbf2e34707 7be80eea-0d5b-e2b0-17c1-7fbbf2e34707}" +--- PASS: TestRaft_LeadershipTransferToItself (0.19s) === RUN TestRaft_LeadershipTransferLeaderRejectsClientRequests -2024-05-12T20:51:37.180Z [INFO] server-d3ef3388-b663-1617-0192-e274ac159d57: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3ef3388-b663-1617-0192-e274ac159d57 Address:d3ef3388-b663-1617-0192-e274ac159d57} {Suffrage:Voter ID:server-39598ff1-3a2f-6005-506e-e7860edb3bf8 Address:39598ff1-3a2f-6005-506e-e7860edb3bf8} {Suffrage:Voter ID:server-c2381d3b-f49c-09cf-4f9e-fba7ed3b29a0 Address:c2381d3b-f49c-09cf-4f9e-fba7ed3b29a0}]" -2024-05-12T20:51:37.180Z [INFO] server-39598ff1-3a2f-6005-506e-e7860edb3bf8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3ef3388-b663-1617-0192-e274ac159d57 Address:d3ef3388-b663-1617-0192-e274ac159d57} {Suffrage:Voter ID:server-39598ff1-3a2f-6005-506e-e7860edb3bf8 Address:39598ff1-3a2f-6005-506e-e7860edb3bf8} {Suffrage:Voter ID:server-c2381d3b-f49c-09cf-4f9e-fba7ed3b29a0 Address:c2381d3b-f49c-09cf-4f9e-fba7ed3b29a0}]" -2024-05-12T20:51:37.181Z [INFO] server-c2381d3b-f49c-09cf-4f9e-fba7ed3b29a0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3ef3388-b663-1617-0192-e274ac159d57 Address:d3ef3388-b663-1617-0192-e274ac159d57} {Suffrage:Voter ID:server-39598ff1-3a2f-6005-506e-e7860edb3bf8 Address:39598ff1-3a2f-6005-506e-e7860edb3bf8} {Suffrage:Voter ID:server-c2381d3b-f49c-09cf-4f9e-fba7ed3b29a0 Address:c2381d3b-f49c-09cf-4f9e-fba7ed3b29a0}]" -2024-05-12T20:51:37.181Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:37.181Z [INFO] server-d3ef3388-b663-1617-0192-e274ac159d57: entering follower state: follower="Node at d3ef3388-b663-1617-0192-e274ac159d57 [Follower]" leader-address= leader-id= -2024-05-12T20:51:37.181Z [INFO] server-39598ff1-3a2f-6005-506e-e7860edb3bf8: entering follower state: follower="Node at 39598ff1-3a2f-6005-506e-e7860edb3bf8 [Follower]" leader-address= leader-id= -2024-05-12T20:51:37.181Z [INFO] server-c2381d3b-f49c-09cf-4f9e-fba7ed3b29a0: entering follower state: follower="Node at c2381d3b-f49c-09cf-4f9e-fba7ed3b29a0 [Follower]" leader-address= leader-id= -2024-05-12T20:51:37.254Z [WARN] server-39598ff1-3a2f-6005-506e-e7860edb3bf8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:37.254Z [INFO] server-39598ff1-3a2f-6005-506e-e7860edb3bf8: entering candidate state: node="Node at 39598ff1-3a2f-6005-506e-e7860edb3bf8 [Candidate]" term=2 -2024-05-12T20:51:37.255Z [INFO] server-39598ff1-3a2f-6005-506e-e7860edb3bf8: election won: term=2 tally=2 -2024-05-12T20:51:37.255Z [INFO] server-39598ff1-3a2f-6005-506e-e7860edb3bf8: entering leader state: leader="Node at 39598ff1-3a2f-6005-506e-e7860edb3bf8 [Leader]" -2024-05-12T20:51:37.255Z [INFO] server-39598ff1-3a2f-6005-506e-e7860edb3bf8: added peer, starting replication: peer=server-d3ef3388-b663-1617-0192-e274ac159d57 -2024-05-12T20:51:37.255Z [INFO] server-39598ff1-3a2f-6005-506e-e7860edb3bf8: added peer, starting replication: peer=server-c2381d3b-f49c-09cf-4f9e-fba7ed3b29a0 -2024-05-12T20:51:37.255Z [INFO] server-39598ff1-3a2f-6005-506e-e7860edb3bf8: pipelining replication: peer="{Voter server-c2381d3b-f49c-09cf-4f9e-fba7ed3b29a0 c2381d3b-f49c-09cf-4f9e-fba7ed3b29a0}" -2024-05-12T20:51:37.255Z [INFO] server-39598ff1-3a2f-6005-506e-e7860edb3bf8: pipelining replication: peer="{Voter server-d3ef3388-b663-1617-0192-e274ac159d57 d3ef3388-b663-1617-0192-e274ac159d57}" -2024-05-12T20:51:37.363Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:37.255269771 +0000 UTC m=+108.535970065 (1 nodes), 74.160295ms from start of poll, 74.579881ms from cluster start. Timeout at 2024-05-12 20:51:37.363183456 +0000 UTC m=+108.643883351, 107.913286ms after stability +2025-06-15T03:18:09.380Z [INFO] server-1c88d3b9-9121-05ea-b7ee-59b7047f4c49: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1c88d3b9-9121-05ea-b7ee-59b7047f4c49 Address:1c88d3b9-9121-05ea-b7ee-59b7047f4c49} {Suffrage:Voter ID:server-691ace8c-b2ec-05c5-f23c-05355ed4f072 Address:691ace8c-b2ec-05c5-f23c-05355ed4f072} {Suffrage:Voter ID:server-8a0fc0a5-2cf7-e09b-e90a-1a454f3bdd4a Address:8a0fc0a5-2cf7-e09b-e90a-1a454f3bdd4a}]" +2025-06-15T03:18:09.380Z [INFO] server-691ace8c-b2ec-05c5-f23c-05355ed4f072: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1c88d3b9-9121-05ea-b7ee-59b7047f4c49 Address:1c88d3b9-9121-05ea-b7ee-59b7047f4c49} {Suffrage:Voter ID:server-691ace8c-b2ec-05c5-f23c-05355ed4f072 Address:691ace8c-b2ec-05c5-f23c-05355ed4f072} {Suffrage:Voter ID:server-8a0fc0a5-2cf7-e09b-e90a-1a454f3bdd4a Address:8a0fc0a5-2cf7-e09b-e90a-1a454f3bdd4a}]" +2025-06-15T03:18:09.380Z [INFO] server-1c88d3b9-9121-05ea-b7ee-59b7047f4c49: entering follower state: follower="Node at 1c88d3b9-9121-05ea-b7ee-59b7047f4c49 [Follower]" leader-address= leader-id= +2025-06-15T03:18:09.380Z [INFO] server-691ace8c-b2ec-05c5-f23c-05355ed4f072: entering follower state: follower="Node at 691ace8c-b2ec-05c5-f23c-05355ed4f072 [Follower]" leader-address= leader-id= +2025-06-15T03:18:09.380Z [INFO] server-8a0fc0a5-2cf7-e09b-e90a-1a454f3bdd4a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1c88d3b9-9121-05ea-b7ee-59b7047f4c49 Address:1c88d3b9-9121-05ea-b7ee-59b7047f4c49} {Suffrage:Voter ID:server-691ace8c-b2ec-05c5-f23c-05355ed4f072 Address:691ace8c-b2ec-05c5-f23c-05355ed4f072} {Suffrage:Voter ID:server-8a0fc0a5-2cf7-e09b-e90a-1a454f3bdd4a Address:8a0fc0a5-2cf7-e09b-e90a-1a454f3bdd4a}]" +2025-06-15T03:18:09.380Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:09.380Z [INFO] server-8a0fc0a5-2cf7-e09b-e90a-1a454f3bdd4a: entering follower state: follower="Node at 8a0fc0a5-2cf7-e09b-e90a-1a454f3bdd4a [Follower]" leader-address= leader-id= +2025-06-15T03:18:09.436Z [WARN] server-1c88d3b9-9121-05ea-b7ee-59b7047f4c49: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:09.436Z [INFO] server-1c88d3b9-9121-05ea-b7ee-59b7047f4c49: entering candidate state: node="Node at 1c88d3b9-9121-05ea-b7ee-59b7047f4c49 [Candidate]" term=2 +2025-06-15T03:18:09.436Z [INFO] server-1c88d3b9-9121-05ea-b7ee-59b7047f4c49: election won: term=2 tally=2 +2025-06-15T03:18:09.436Z [INFO] server-1c88d3b9-9121-05ea-b7ee-59b7047f4c49: entering leader state: leader="Node at 1c88d3b9-9121-05ea-b7ee-59b7047f4c49 [Leader]" +2025-06-15T03:18:09.436Z [INFO] server-1c88d3b9-9121-05ea-b7ee-59b7047f4c49: added peer, starting replication: peer=server-691ace8c-b2ec-05c5-f23c-05355ed4f072 +2025-06-15T03:18:09.436Z [INFO] server-1c88d3b9-9121-05ea-b7ee-59b7047f4c49: added peer, starting replication: peer=server-8a0fc0a5-2cf7-e09b-e90a-1a454f3bdd4a +2025-06-15T03:18:09.436Z [INFO] server-1c88d3b9-9121-05ea-b7ee-59b7047f4c49: pipelining replication: peer="{Voter server-691ace8c-b2ec-05c5-f23c-05355ed4f072 691ace8c-b2ec-05c5-f23c-05355ed4f072}" +2025-06-15T03:18:09.436Z [INFO] server-1c88d3b9-9121-05ea-b7ee-59b7047f4c49: pipelining replication: peer="{Voter server-8a0fc0a5-2cf7-e09b-e90a-1a454f3bdd4a 8a0fc0a5-2cf7-e09b-e90a-1a454f3bdd4a}" +2025-06-15T03:18:09.542Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:09.436712208 +0000 UTC m=+105.148390407 (1 nodes), 55.965926ms from start of poll, 56.262193ms from cluster start. Timeout at 2025-06-15 03:18:09.542196169 +0000 UTC m=+105.253874335, 105.483928ms after stability raft_test.go:2457: waiting on future 0 raft_test.go:2457: waiting on future 1 raft_test.go:2457: waiting on future 2 @@ -3854,343 +3836,338 @@ raft_test.go:2457: waiting on future 4 raft_test.go:2457: waiting on future 5 raft_test.go:2457: waiting on future 6 -2024-05-12T20:51:37.363Z [INFO] server-39598ff1-3a2f-6005-506e-e7860edb3bf8: aborting pipeline replication: peer="{Voter server-c2381d3b-f49c-09cf-4f9e-fba7ed3b29a0 c2381d3b-f49c-09cf-4f9e-fba7ed3b29a0}" -2024-05-12T20:51:37.363Z [INFO] server-39598ff1-3a2f-6005-506e-e7860edb3bf8: aborting pipeline replication: peer="{Voter server-d3ef3388-b663-1617-0192-e274ac159d57 d3ef3388-b663-1617-0192-e274ac159d57}" ---- PASS: TestRaft_LeadershipTransferLeaderRejectsClientRequests (0.18s) +2025-06-15T03:18:09.542Z [INFO] server-1c88d3b9-9121-05ea-b7ee-59b7047f4c49: aborting pipeline replication: peer="{Voter server-8a0fc0a5-2cf7-e09b-e90a-1a454f3bdd4a 8a0fc0a5-2cf7-e09b-e90a-1a454f3bdd4a}" +2025-06-15T03:18:09.542Z [INFO] server-1c88d3b9-9121-05ea-b7ee-59b7047f4c49: aborting pipeline replication: peer="{Voter server-691ace8c-b2ec-05c5-f23c-05355ed4f072 691ace8c-b2ec-05c5-f23c-05355ed4f072}" +--- PASS: TestRaft_LeadershipTransferLeaderRejectsClientRequests (0.16s) === RUN TestRaft_LeadershipTransferLeaderReplicationTimeout -2024-05-12T20:51:37.365Z [INFO] server-0968bd36-18ff-4f1d-3c54-d6923d8b22a2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0968bd36-18ff-4f1d-3c54-d6923d8b22a2 Address:0968bd36-18ff-4f1d-3c54-d6923d8b22a2} {Suffrage:Voter ID:server-4b06fd6f-0d95-8198-7991-2ed92f3db404 Address:4b06fd6f-0d95-8198-7991-2ed92f3db404} {Suffrage:Voter ID:server-2e161650-1512-f69a-b6fa-092f6238c66a Address:2e161650-1512-f69a-b6fa-092f6238c66a}]" -2024-05-12T20:51:37.365Z [INFO] server-4b06fd6f-0d95-8198-7991-2ed92f3db404: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0968bd36-18ff-4f1d-3c54-d6923d8b22a2 Address:0968bd36-18ff-4f1d-3c54-d6923d8b22a2} {Suffrage:Voter ID:server-4b06fd6f-0d95-8198-7991-2ed92f3db404 Address:4b06fd6f-0d95-8198-7991-2ed92f3db404} {Suffrage:Voter ID:server-2e161650-1512-f69a-b6fa-092f6238c66a Address:2e161650-1512-f69a-b6fa-092f6238c66a}]" -2024-05-12T20:51:37.365Z [INFO] server-2e161650-1512-f69a-b6fa-092f6238c66a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0968bd36-18ff-4f1d-3c54-d6923d8b22a2 Address:0968bd36-18ff-4f1d-3c54-d6923d8b22a2} {Suffrage:Voter ID:server-4b06fd6f-0d95-8198-7991-2ed92f3db404 Address:4b06fd6f-0d95-8198-7991-2ed92f3db404} {Suffrage:Voter ID:server-2e161650-1512-f69a-b6fa-092f6238c66a Address:2e161650-1512-f69a-b6fa-092f6238c66a}]" -2024-05-12T20:51:37.365Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:37.365Z [INFO] server-0968bd36-18ff-4f1d-3c54-d6923d8b22a2: entering follower state: follower="Node at 0968bd36-18ff-4f1d-3c54-d6923d8b22a2 [Follower]" leader-address= leader-id= -2024-05-12T20:51:37.365Z [INFO] server-4b06fd6f-0d95-8198-7991-2ed92f3db404: entering follower state: follower="Node at 4b06fd6f-0d95-8198-7991-2ed92f3db404 [Follower]" leader-address= leader-id= -2024-05-12T20:51:37.365Z [INFO] server-2e161650-1512-f69a-b6fa-092f6238c66a: entering follower state: follower="Node at 2e161650-1512-f69a-b6fa-092f6238c66a [Follower]" leader-address= leader-id= -2024-05-12T20:51:37.427Z [WARN] server-2e161650-1512-f69a-b6fa-092f6238c66a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:37.427Z [INFO] server-2e161650-1512-f69a-b6fa-092f6238c66a: entering candidate state: node="Node at 2e161650-1512-f69a-b6fa-092f6238c66a [Candidate]" term=2 -2024-05-12T20:51:37.427Z [INFO] server-2e161650-1512-f69a-b6fa-092f6238c66a: election won: term=2 tally=2 -2024-05-12T20:51:37.427Z [INFO] server-2e161650-1512-f69a-b6fa-092f6238c66a: entering leader state: leader="Node at 2e161650-1512-f69a-b6fa-092f6238c66a [Leader]" -2024-05-12T20:51:37.427Z [INFO] server-2e161650-1512-f69a-b6fa-092f6238c66a: added peer, starting replication: peer=server-0968bd36-18ff-4f1d-3c54-d6923d8b22a2 -2024-05-12T20:51:37.427Z [INFO] server-2e161650-1512-f69a-b6fa-092f6238c66a: added peer, starting replication: peer=server-4b06fd6f-0d95-8198-7991-2ed92f3db404 -2024-05-12T20:51:37.427Z [INFO] server-2e161650-1512-f69a-b6fa-092f6238c66a: pipelining replication: peer="{Voter server-0968bd36-18ff-4f1d-3c54-d6923d8b22a2 0968bd36-18ff-4f1d-3c54-d6923d8b22a2}" -2024-05-12T20:51:37.427Z [INFO] server-2e161650-1512-f69a-b6fa-092f6238c66a: pipelining replication: peer="{Voter server-4b06fd6f-0d95-8198-7991-2ed92f3db404 4b06fd6f-0d95-8198-7991-2ed92f3db404}" -2024-05-12T20:51:37.535Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:37.42783752 +0000 UTC m=+108.708537857 (1 nodes), 62.024945ms from start of poll, 62.452916ms from cluster start. Timeout at 2024-05-12 20:51:37.535000769 +0000 UTC m=+108.815701108, 107.163251ms after stability -2024-05-12T20:51:37.535Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:37.643Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:37.535106674 +0000 UTC m=+108.815806569 (2 nodes), 0s from start of poll, 169.721628ms from cluster start. Timeout at 2024-05-12 20:51:37.643049556 +0000 UTC m=+108.923750636, 107.944067ms after stability -2024-05-12T20:51:39.144Z [INFO] server-2e161650-1512-f69a-b6fa-092f6238c66a: aborting pipeline replication: peer="{Voter server-4b06fd6f-0d95-8198-7991-2ed92f3db404 4b06fd6f-0d95-8198-7991-2ed92f3db404}" -2024-05-12T20:51:39.145Z [INFO] server-2e161650-1512-f69a-b6fa-092f6238c66a: aborting pipeline replication: peer="{Voter server-0968bd36-18ff-4f1d-3c54-d6923d8b22a2 0968bd36-18ff-4f1d-3c54-d6923d8b22a2}" -2024-05-12T20:51:39.667Z [ERROR] server-2e161650-1512-f69a-b6fa-092f6238c66a: failed to heartbeat to: peer=4b06fd6f-0d95-8198-7991-2ed92f3db404 backoff time=10ms error="command timed out" ---- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (2.30s) +2025-06-15T03:18:09.543Z [INFO] server-f4f5342e-69d8-2423-7390-55dbd58d60a6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f4f5342e-69d8-2423-7390-55dbd58d60a6 Address:f4f5342e-69d8-2423-7390-55dbd58d60a6} {Suffrage:Voter ID:server-f366234e-0c64-a935-7463-94dc3457b495 Address:f366234e-0c64-a935-7463-94dc3457b495} {Suffrage:Voter ID:server-43df4f8f-dfa7-962b-cc56-4f762e53e29a Address:43df4f8f-dfa7-962b-cc56-4f762e53e29a}]" +2025-06-15T03:18:09.543Z [INFO] server-f366234e-0c64-a935-7463-94dc3457b495: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f4f5342e-69d8-2423-7390-55dbd58d60a6 Address:f4f5342e-69d8-2423-7390-55dbd58d60a6} {Suffrage:Voter ID:server-f366234e-0c64-a935-7463-94dc3457b495 Address:f366234e-0c64-a935-7463-94dc3457b495} {Suffrage:Voter ID:server-43df4f8f-dfa7-962b-cc56-4f762e53e29a Address:43df4f8f-dfa7-962b-cc56-4f762e53e29a}]" +2025-06-15T03:18:09.543Z [INFO] server-f4f5342e-69d8-2423-7390-55dbd58d60a6: entering follower state: follower="Node at f4f5342e-69d8-2423-7390-55dbd58d60a6 [Follower]" leader-address= leader-id= +2025-06-15T03:18:09.543Z [INFO] server-f366234e-0c64-a935-7463-94dc3457b495: entering follower state: follower="Node at f366234e-0c64-a935-7463-94dc3457b495 [Follower]" leader-address= leader-id= +2025-06-15T03:18:09.543Z [INFO] server-43df4f8f-dfa7-962b-cc56-4f762e53e29a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f4f5342e-69d8-2423-7390-55dbd58d60a6 Address:f4f5342e-69d8-2423-7390-55dbd58d60a6} {Suffrage:Voter ID:server-f366234e-0c64-a935-7463-94dc3457b495 Address:f366234e-0c64-a935-7463-94dc3457b495} {Suffrage:Voter ID:server-43df4f8f-dfa7-962b-cc56-4f762e53e29a Address:43df4f8f-dfa7-962b-cc56-4f762e53e29a}]" +2025-06-15T03:18:09.543Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:09.543Z [INFO] server-43df4f8f-dfa7-962b-cc56-4f762e53e29a: entering follower state: follower="Node at 43df4f8f-dfa7-962b-cc56-4f762e53e29a [Follower]" leader-address= leader-id= +2025-06-15T03:18:09.595Z [WARN] server-43df4f8f-dfa7-962b-cc56-4f762e53e29a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:09.595Z [INFO] server-43df4f8f-dfa7-962b-cc56-4f762e53e29a: entering candidate state: node="Node at 43df4f8f-dfa7-962b-cc56-4f762e53e29a [Candidate]" term=2 +2025-06-15T03:18:09.595Z [INFO] server-43df4f8f-dfa7-962b-cc56-4f762e53e29a: election won: term=2 tally=2 +2025-06-15T03:18:09.595Z [INFO] server-43df4f8f-dfa7-962b-cc56-4f762e53e29a: entering leader state: leader="Node at 43df4f8f-dfa7-962b-cc56-4f762e53e29a [Leader]" +2025-06-15T03:18:09.595Z [INFO] server-43df4f8f-dfa7-962b-cc56-4f762e53e29a: added peer, starting replication: peer=server-f4f5342e-69d8-2423-7390-55dbd58d60a6 +2025-06-15T03:18:09.595Z [INFO] server-43df4f8f-dfa7-962b-cc56-4f762e53e29a: added peer, starting replication: peer=server-f366234e-0c64-a935-7463-94dc3457b495 +2025-06-15T03:18:09.595Z [INFO] server-43df4f8f-dfa7-962b-cc56-4f762e53e29a: pipelining replication: peer="{Voter server-f366234e-0c64-a935-7463-94dc3457b495 f366234e-0c64-a935-7463-94dc3457b495}" +2025-06-15T03:18:09.595Z [INFO] server-43df4f8f-dfa7-962b-cc56-4f762e53e29a: pipelining replication: peer="{Voter server-f4f5342e-69d8-2423-7390-55dbd58d60a6 f4f5342e-69d8-2423-7390-55dbd58d60a6}" +2025-06-15T03:18:09.701Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:09.595843011 +0000 UTC m=+105.307521172 (1 nodes), 52.366986ms from start of poll, 52.664206ms from cluster start. Timeout at 2025-06-15 03:18:09.701670195 +0000 UTC m=+105.413348356, 105.827184ms after stability +2025-06-15T03:18:09.701Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:09.807Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:09.701701418 +0000 UTC m=+105.413379584 (2 nodes), 747ns from start of poll, 158.522618ms from cluster start. Timeout at 2025-06-15 03:18:09.80782788 +0000 UTC m=+105.519506082, 106.126498ms after stability +2025-06-15T03:18:10.047Z [INFO] server-43df4f8f-dfa7-962b-cc56-4f762e53e29a: aborting pipeline replication: peer="{Voter server-f4f5342e-69d8-2423-7390-55dbd58d60a6 f4f5342e-69d8-2423-7390-55dbd58d60a6}" +2025-06-15T03:18:10.047Z [INFO] server-43df4f8f-dfa7-962b-cc56-4f762e53e29a: aborting pipeline replication: peer="{Voter server-f366234e-0c64-a935-7463-94dc3457b495 f366234e-0c64-a935-7463-94dc3457b495}" +--- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (0.51s) === RUN TestRaft_LeadershipTransferIgnoresNonvoters -2024-05-12T20:51:39.668Z [INFO] server-85a2ff96-2d17-2958-11ac-70399204383c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-85a2ff96-2d17-2958-11ac-70399204383c Address:85a2ff96-2d17-2958-11ac-70399204383c} {Suffrage:Voter ID:server-a24d893b-8ada-88c5-2f6c-09426d4a5f35 Address:a24d893b-8ada-88c5-2f6c-09426d4a5f35}]" -2024-05-12T20:51:39.669Z [INFO] server-a24d893b-8ada-88c5-2f6c-09426d4a5f35: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-85a2ff96-2d17-2958-11ac-70399204383c Address:85a2ff96-2d17-2958-11ac-70399204383c} {Suffrage:Voter ID:server-a24d893b-8ada-88c5-2f6c-09426d4a5f35 Address:a24d893b-8ada-88c5-2f6c-09426d4a5f35}]" -2024-05-12T20:51:39.669Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:39.669Z [INFO] server-85a2ff96-2d17-2958-11ac-70399204383c: entering follower state: follower="Node at 85a2ff96-2d17-2958-11ac-70399204383c [Follower]" leader-address= leader-id= -2024-05-12T20:51:39.669Z [INFO] server-a24d893b-8ada-88c5-2f6c-09426d4a5f35: entering follower state: follower="Node at a24d893b-8ada-88c5-2f6c-09426d4a5f35 [Follower]" leader-address= leader-id= -2024-05-12T20:51:39.750Z [WARN] server-a24d893b-8ada-88c5-2f6c-09426d4a5f35: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:39.750Z [INFO] server-a24d893b-8ada-88c5-2f6c-09426d4a5f35: entering candidate state: node="Node at a24d893b-8ada-88c5-2f6c-09426d4a5f35 [Candidate]" term=2 -2024-05-12T20:51:39.751Z [WARN] server-85a2ff96-2d17-2958-11ac-70399204383c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:39.751Z [INFO] server-85a2ff96-2d17-2958-11ac-70399204383c: entering candidate state: node="Node at 85a2ff96-2d17-2958-11ac-70399204383c [Candidate]" term=2 -2024-05-12T20:51:39.751Z [INFO] server-85a2ff96-2d17-2958-11ac-70399204383c: duplicate requestVote for same term: term=2 -2024-05-12T20:51:39.751Z [INFO] server-a24d893b-8ada-88c5-2f6c-09426d4a5f35: duplicate requestVote for same term: term=2 -2024-05-12T20:51:39.815Z [WARN] server-85a2ff96-2d17-2958-11ac-70399204383c: Election timeout reached, restarting election -2024-05-12T20:51:39.815Z [INFO] server-85a2ff96-2d17-2958-11ac-70399204383c: entering candidate state: node="Node at 85a2ff96-2d17-2958-11ac-70399204383c [Candidate]" term=3 -2024-05-12T20:51:39.815Z [INFO] server-a24d893b-8ada-88c5-2f6c-09426d4a5f35: entering follower state: follower="Node at a24d893b-8ada-88c5-2f6c-09426d4a5f35 [Follower]" leader-address= leader-id= -2024-05-12T20:51:39.815Z [INFO] server-85a2ff96-2d17-2958-11ac-70399204383c: election won: term=3 tally=2 -2024-05-12T20:51:39.815Z [INFO] server-85a2ff96-2d17-2958-11ac-70399204383c: entering leader state: leader="Node at 85a2ff96-2d17-2958-11ac-70399204383c [Leader]" -2024-05-12T20:51:39.815Z [INFO] server-85a2ff96-2d17-2958-11ac-70399204383c: added peer, starting replication: peer=server-a24d893b-8ada-88c5-2f6c-09426d4a5f35 -2024-05-12T20:51:39.815Z [INFO] server-85a2ff96-2d17-2958-11ac-70399204383c: pipelining replication: peer="{Voter server-a24d893b-8ada-88c5-2f6c-09426d4a5f35 a24d893b-8ada-88c5-2f6c-09426d4a5f35}" -2024-05-12T20:51:39.922Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:39.815759934 +0000 UTC m=+111.096460024 (1 nodes), 146.63287ms from start of poll, 146.922933ms from cluster start. Timeout at 2024-05-12 20:51:39.922772858 +0000 UTC m=+111.203473350, 107.013326ms after stability -2024-05-12T20:51:39.922Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:40.031Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:39.922875635 +0000 UTC m=+111.203575956 (1 nodes), 2.33µs from start of poll, 254.038865ms from cluster start. Timeout at 2024-05-12 20:51:40.031423844 +0000 UTC m=+111.312123739, 108.547783ms after stability -2024-05-12T20:51:40.031Z [INFO] server-85a2ff96-2d17-2958-11ac-70399204383c: updating configuration: command=DemoteVoter server-id=server-a24d893b-8ada-88c5-2f6c-09426d4a5f35 server-addr= servers="[{Suffrage:Voter ID:server-85a2ff96-2d17-2958-11ac-70399204383c Address:85a2ff96-2d17-2958-11ac-70399204383c} {Suffrage:Nonvoter ID:server-a24d893b-8ada-88c5-2f6c-09426d4a5f35 Address:a24d893b-8ada-88c5-2f6c-09426d4a5f35}]" -2024-05-12T20:51:40.031Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:40.140Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:40.031723442 +0000 UTC m=+111.312423753 (1 nodes), 1.785µs from start of poll, 362.886662ms from cluster start. Timeout at 2024-05-12 20:51:40.140859825 +0000 UTC m=+111.421560115, 109.136362ms after stability -2024-05-12T20:51:40.141Z [INFO] server-85a2ff96-2d17-2958-11ac-70399204383c: aborting pipeline replication: peer="{Voter server-a24d893b-8ada-88c5-2f6c-09426d4a5f35 a24d893b-8ada-88c5-2f6c-09426d4a5f35}" ---- PASS: TestRaft_LeadershipTransferIgnoresNonvoters (0.47s) +2025-06-15T03:18:10.049Z [INFO] server-4dfeeeac-2de1-95bb-1318-4dd582cca7b1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4dfeeeac-2de1-95bb-1318-4dd582cca7b1 Address:4dfeeeac-2de1-95bb-1318-4dd582cca7b1} {Suffrage:Voter ID:server-16bb8994-9ab6-19dc-4ec4-5fb55b592a81 Address:16bb8994-9ab6-19dc-4ec4-5fb55b592a81}]" +2025-06-15T03:18:10.049Z [INFO] server-4dfeeeac-2de1-95bb-1318-4dd582cca7b1: entering follower state: follower="Node at 4dfeeeac-2de1-95bb-1318-4dd582cca7b1 [Follower]" leader-address= leader-id= +2025-06-15T03:18:10.049Z [INFO] server-16bb8994-9ab6-19dc-4ec4-5fb55b592a81: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4dfeeeac-2de1-95bb-1318-4dd582cca7b1 Address:4dfeeeac-2de1-95bb-1318-4dd582cca7b1} {Suffrage:Voter ID:server-16bb8994-9ab6-19dc-4ec4-5fb55b592a81 Address:16bb8994-9ab6-19dc-4ec4-5fb55b592a81}]" +2025-06-15T03:18:10.049Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:10.049Z [INFO] server-16bb8994-9ab6-19dc-4ec4-5fb55b592a81: entering follower state: follower="Node at 16bb8994-9ab6-19dc-4ec4-5fb55b592a81 [Follower]" leader-address= leader-id= +2025-06-15T03:18:10.112Z [WARN] server-4dfeeeac-2de1-95bb-1318-4dd582cca7b1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:10.112Z [INFO] server-4dfeeeac-2de1-95bb-1318-4dd582cca7b1: entering candidate state: node="Node at 4dfeeeac-2de1-95bb-1318-4dd582cca7b1 [Candidate]" term=2 +2025-06-15T03:18:10.112Z [INFO] server-4dfeeeac-2de1-95bb-1318-4dd582cca7b1: election won: term=2 tally=2 +2025-06-15T03:18:10.112Z [INFO] server-4dfeeeac-2de1-95bb-1318-4dd582cca7b1: entering leader state: leader="Node at 4dfeeeac-2de1-95bb-1318-4dd582cca7b1 [Leader]" +2025-06-15T03:18:10.112Z [INFO] server-4dfeeeac-2de1-95bb-1318-4dd582cca7b1: added peer, starting replication: peer=server-16bb8994-9ab6-19dc-4ec4-5fb55b592a81 +2025-06-15T03:18:10.112Z [INFO] server-4dfeeeac-2de1-95bb-1318-4dd582cca7b1: pipelining replication: peer="{Voter server-16bb8994-9ab6-19dc-4ec4-5fb55b592a81 16bb8994-9ab6-19dc-4ec4-5fb55b592a81}" +2025-06-15T03:18:10.217Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:10.112747009 +0000 UTC m=+105.824425190 (1 nodes), 63.630251ms from start of poll, 63.851749ms from cluster start. Timeout at 2025-06-15 03:18:10.217928345 +0000 UTC m=+105.929606510, 105.18132ms after stability +2025-06-15T03:18:10.217Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:10.323Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:10.21797081 +0000 UTC m=+105.929648976 (1 nodes), 513ns from start of poll, 169.075535ms from cluster start. Timeout at 2025-06-15 03:18:10.323193884 +0000 UTC m=+106.034872050, 105.223074ms after stability +2025-06-15T03:18:10.323Z [INFO] server-4dfeeeac-2de1-95bb-1318-4dd582cca7b1: updating configuration: command=DemoteVoter server-id=server-16bb8994-9ab6-19dc-4ec4-5fb55b592a81 server-addr= servers="[{Suffrage:Voter ID:server-4dfeeeac-2de1-95bb-1318-4dd582cca7b1 Address:4dfeeeac-2de1-95bb-1318-4dd582cca7b1} {Suffrage:Nonvoter ID:server-16bb8994-9ab6-19dc-4ec4-5fb55b592a81 Address:16bb8994-9ab6-19dc-4ec4-5fb55b592a81}]" +2025-06-15T03:18:10.323Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:10.428Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:10.323315454 +0000 UTC m=+106.034993616 (1 nodes), 551ns from start of poll, 274.420175ms from cluster start. Timeout at 2025-06-15 03:18:10.428791318 +0000 UTC m=+106.140469479, 105.475863ms after stability +2025-06-15T03:18:10.428Z [INFO] server-4dfeeeac-2de1-95bb-1318-4dd582cca7b1: aborting pipeline replication: peer="{Voter server-16bb8994-9ab6-19dc-4ec4-5fb55b592a81 16bb8994-9ab6-19dc-4ec4-5fb55b592a81}" +--- PASS: TestRaft_LeadershipTransferIgnoresNonvoters (0.38s) === RUN TestRaft_LeadershipTransferStopRightAway --- PASS: TestRaft_LeadershipTransferStopRightAway (0.00s) === RUN TestRaft_GetConfigurationNoBootstrap -2024-05-12T20:51:40.142Z [INFO] server-eca4dbf6-5d2c-1174-dd87-9ed66194ff14: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eca4dbf6-5d2c-1174-dd87-9ed66194ff14 Address:eca4dbf6-5d2c-1174-dd87-9ed66194ff14} {Suffrage:Voter ID:server-a18c13ab-bf56-5f98-5305-066c327fac50 Address:a18c13ab-bf56-5f98-5305-066c327fac50}]" -2024-05-12T20:51:40.142Z [INFO] server-a18c13ab-bf56-5f98-5305-066c327fac50: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eca4dbf6-5d2c-1174-dd87-9ed66194ff14 Address:eca4dbf6-5d2c-1174-dd87-9ed66194ff14} {Suffrage:Voter ID:server-a18c13ab-bf56-5f98-5305-066c327fac50 Address:a18c13ab-bf56-5f98-5305-066c327fac50}]" -2024-05-12T20:51:40.142Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:40.142Z [INFO] server-eca4dbf6-5d2c-1174-dd87-9ed66194ff14: entering follower state: follower="Node at eca4dbf6-5d2c-1174-dd87-9ed66194ff14 [Follower]" leader-address= leader-id= -2024-05-12T20:51:40.142Z [INFO] server-a18c13ab-bf56-5f98-5305-066c327fac50: entering follower state: follower="Node at a18c13ab-bf56-5f98-5305-066c327fac50 [Follower]" leader-address= leader-id= -2024-05-12T20:51:40.194Z [WARN] server-eca4dbf6-5d2c-1174-dd87-9ed66194ff14: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:40.194Z [INFO] server-eca4dbf6-5d2c-1174-dd87-9ed66194ff14: entering candidate state: node="Node at eca4dbf6-5d2c-1174-dd87-9ed66194ff14 [Candidate]" term=2 -2024-05-12T20:51:40.195Z [INFO] server-eca4dbf6-5d2c-1174-dd87-9ed66194ff14: election won: term=2 tally=2 -2024-05-12T20:51:40.195Z [INFO] server-eca4dbf6-5d2c-1174-dd87-9ed66194ff14: entering leader state: leader="Node at eca4dbf6-5d2c-1174-dd87-9ed66194ff14 [Leader]" -2024-05-12T20:51:40.195Z [INFO] server-eca4dbf6-5d2c-1174-dd87-9ed66194ff14: added peer, starting replication: peer=server-a18c13ab-bf56-5f98-5305-066c327fac50 -2024-05-12T20:51:40.195Z [INFO] server-eca4dbf6-5d2c-1174-dd87-9ed66194ff14: pipelining replication: peer="{Voter server-a18c13ab-bf56-5f98-5305-066c327fac50 a18c13ab-bf56-5f98-5305-066c327fac50}" -2024-05-12T20:51:40.307Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:40.195260874 +0000 UTC m=+111.475961547 (1 nodes), 52.67041ms from start of poll, 52.932356ms from cluster start. Timeout at 2024-05-12 20:51:40.307514525 +0000 UTC m=+111.588214821, 112.253274ms after stability -2024-05-12T20:51:40.307Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:40.414Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:40.307638411 +0000 UTC m=+111.588338702 (1 nodes), 3.29µs from start of poll, 165.309511ms from cluster start. Timeout at 2024-05-12 20:51:40.414768987 +0000 UTC m=+111.695469484, 107.130782ms after stability -2024-05-12T20:51:40.431Z [INFO] server-a18c13ab-bf56-5f98-5305-066c327fac50: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eca4dbf6-5d2c-1174-dd87-9ed66194ff14 Address:eca4dbf6-5d2c-1174-dd87-9ed66194ff14} {Suffrage:Voter ID:server-a18c13ab-bf56-5f98-5305-066c327fac50 Address:a18c13ab-bf56-5f98-5305-066c327fac50}]" -2024-05-12T20:51:40.431Z [INFO] server-eca4dbf6-5d2c-1174-dd87-9ed66194ff14: aborting pipeline replication: peer="{Voter server-a18c13ab-bf56-5f98-5305-066c327fac50 a18c13ab-bf56-5f98-5305-066c327fac50}" ---- PASS: TestRaft_GetConfigurationNoBootstrap (0.29s) +2025-06-15T03:18:10.429Z [INFO] server-5de6d917-9e6e-bd04-ec71-5423f8c110a4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5de6d917-9e6e-bd04-ec71-5423f8c110a4 Address:5de6d917-9e6e-bd04-ec71-5423f8c110a4} {Suffrage:Voter ID:server-b4132569-6c22-b950-e0fa-990072904394 Address:b4132569-6c22-b950-e0fa-990072904394}]" +2025-06-15T03:18:10.429Z [INFO] server-b4132569-6c22-b950-e0fa-990072904394: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5de6d917-9e6e-bd04-ec71-5423f8c110a4 Address:5de6d917-9e6e-bd04-ec71-5423f8c110a4} {Suffrage:Voter ID:server-b4132569-6c22-b950-e0fa-990072904394 Address:b4132569-6c22-b950-e0fa-990072904394}]" +2025-06-15T03:18:10.429Z [INFO] server-5de6d917-9e6e-bd04-ec71-5423f8c110a4: entering follower state: follower="Node at 5de6d917-9e6e-bd04-ec71-5423f8c110a4 [Follower]" leader-address= leader-id= +2025-06-15T03:18:10.429Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:10.429Z [INFO] server-b4132569-6c22-b950-e0fa-990072904394: entering follower state: follower="Node at b4132569-6c22-b950-e0fa-990072904394 [Follower]" leader-address= leader-id= +2025-06-15T03:18:10.522Z [WARN] server-5de6d917-9e6e-bd04-ec71-5423f8c110a4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:10.522Z [INFO] server-5de6d917-9e6e-bd04-ec71-5423f8c110a4: entering candidate state: node="Node at 5de6d917-9e6e-bd04-ec71-5423f8c110a4 [Candidate]" term=2 +2025-06-15T03:18:10.523Z [INFO] server-5de6d917-9e6e-bd04-ec71-5423f8c110a4: election won: term=2 tally=2 +2025-06-15T03:18:10.523Z [INFO] server-5de6d917-9e6e-bd04-ec71-5423f8c110a4: entering leader state: leader="Node at 5de6d917-9e6e-bd04-ec71-5423f8c110a4 [Leader]" +2025-06-15T03:18:10.523Z [INFO] server-5de6d917-9e6e-bd04-ec71-5423f8c110a4: added peer, starting replication: peer=server-b4132569-6c22-b950-e0fa-990072904394 +2025-06-15T03:18:10.523Z [INFO] server-5de6d917-9e6e-bd04-ec71-5423f8c110a4: pipelining replication: peer="{Voter server-b4132569-6c22-b950-e0fa-990072904394 b4132569-6c22-b950-e0fa-990072904394}" +2025-06-15T03:18:10.628Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:10.523148179 +0000 UTC m=+106.234826345 (1 nodes), 93.335872ms from start of poll, 93.54422ms from cluster start. Timeout at 2025-06-15 03:18:10.628509472 +0000 UTC m=+106.340187651, 105.361306ms after stability +2025-06-15T03:18:10.628Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:10.733Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:10.628553808 +0000 UTC m=+106.340231991 (1 nodes), 1.032µs from start of poll, 198.949866ms from cluster start. Timeout at 2025-06-15 03:18:10.733598482 +0000 UTC m=+106.445276664, 105.044673ms after stability +2025-06-15T03:18:10.744Z [INFO] server-b4132569-6c22-b950-e0fa-990072904394: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5de6d917-9e6e-bd04-ec71-5423f8c110a4 Address:5de6d917-9e6e-bd04-ec71-5423f8c110a4} {Suffrage:Voter ID:server-b4132569-6c22-b950-e0fa-990072904394 Address:b4132569-6c22-b950-e0fa-990072904394}]" +2025-06-15T03:18:10.744Z [INFO] server-5de6d917-9e6e-bd04-ec71-5423f8c110a4: aborting pipeline replication: peer="{Voter server-b4132569-6c22-b950-e0fa-990072904394 b4132569-6c22-b950-e0fa-990072904394}" +--- PASS: TestRaft_GetConfigurationNoBootstrap (0.32s) === RUN TestRaft_LogStoreIsMonotonic -2024-05-12T20:51:40.432Z [INFO] server-a91c21b6-05ba-e9a7-632d-3dd7f9bdc4d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a91c21b6-05ba-e9a7-632d-3dd7f9bdc4d0 Address:a91c21b6-05ba-e9a7-632d-3dd7f9bdc4d0}]" -2024-05-12T20:51:40.432Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:40.432Z [INFO] server-a91c21b6-05ba-e9a7-632d-3dd7f9bdc4d0: entering follower state: follower="Node at a91c21b6-05ba-e9a7-632d-3dd7f9bdc4d0 [Follower]" leader-address= leader-id= -2024-05-12T20:51:40.495Z [WARN] server-a91c21b6-05ba-e9a7-632d-3dd7f9bdc4d0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:40.495Z [INFO] server-a91c21b6-05ba-e9a7-632d-3dd7f9bdc4d0: entering candidate state: node="Node at a91c21b6-05ba-e9a7-632d-3dd7f9bdc4d0 [Candidate]" term=2 -2024-05-12T20:51:40.495Z [INFO] server-a91c21b6-05ba-e9a7-632d-3dd7f9bdc4d0: election won: term=2 tally=1 -2024-05-12T20:51:40.495Z [INFO] server-a91c21b6-05ba-e9a7-632d-3dd7f9bdc4d0: entering leader state: leader="Node at a91c21b6-05ba-e9a7-632d-3dd7f9bdc4d0 [Leader]" -2024-05-12T20:51:40.603Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:40.49543987 +0000 UTC m=+111.776140215 (1 nodes), 63.087438ms from start of poll, 63.245659ms from cluster start. Timeout at 2024-05-12 20:51:40.603078693 +0000 UTC m=+111.883779398, 107.639183ms after stability +2025-06-15T03:18:10.745Z [INFO] server-7480be26-9fd0-614b-d46d-4b098fe76fd7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7480be26-9fd0-614b-d46d-4b098fe76fd7 Address:7480be26-9fd0-614b-d46d-4b098fe76fd7}]" +2025-06-15T03:18:10.745Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:10.745Z [INFO] server-7480be26-9fd0-614b-d46d-4b098fe76fd7: entering follower state: follower="Node at 7480be26-9fd0-614b-d46d-4b098fe76fd7 [Follower]" leader-address= leader-id= +2025-06-15T03:18:10.807Z [WARN] server-7480be26-9fd0-614b-d46d-4b098fe76fd7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:10.807Z [INFO] server-7480be26-9fd0-614b-d46d-4b098fe76fd7: entering candidate state: node="Node at 7480be26-9fd0-614b-d46d-4b098fe76fd7 [Candidate]" term=2 +2025-06-15T03:18:10.807Z [INFO] server-7480be26-9fd0-614b-d46d-4b098fe76fd7: election won: term=2 tally=1 +2025-06-15T03:18:10.807Z [INFO] server-7480be26-9fd0-614b-d46d-4b098fe76fd7: entering leader state: leader="Node at 7480be26-9fd0-614b-d46d-4b098fe76fd7 [Leader]" +2025-06-15T03:18:10.912Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:10.807882245 +0000 UTC m=+106.519560445 (1 nodes), 62.766689ms from start of poll, 62.873621ms from cluster start. Timeout at 2025-06-15 03:18:10.912891738 +0000 UTC m=+106.624569937, 105.009492ms after stability --- PASS: TestRaft_LogStoreIsMonotonic (0.17s) === RUN TestRaft_CacheLogWithStoreError -2024-05-12T20:51:40.604Z [INFO] server-4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62 Address:4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62} {Suffrage:Voter ID:server-5f475248-45c0-b3d6-f8b5-8336baf2669b Address:5f475248-45c0-b3d6-f8b5-8336baf2669b}]" -2024-05-12T20:51:40.604Z [INFO] server-5f475248-45c0-b3d6-f8b5-8336baf2669b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62 Address:4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62} {Suffrage:Voter ID:server-5f475248-45c0-b3d6-f8b5-8336baf2669b Address:5f475248-45c0-b3d6-f8b5-8336baf2669b}]" -2024-05-12T20:51:40.604Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:40.604Z [INFO] server-4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62: entering follower state: follower="Node at 4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62 [Follower]" leader-address= leader-id= -2024-05-12T20:51:40.604Z [INFO] server-5f475248-45c0-b3d6-f8b5-8336baf2669b: entering follower state: follower="Node at 5f475248-45c0-b3d6-f8b5-8336baf2669b [Follower]" leader-address= leader-id= -2024-05-12T20:51:40.663Z [WARN] server-5f475248-45c0-b3d6-f8b5-8336baf2669b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:40.663Z [INFO] server-5f475248-45c0-b3d6-f8b5-8336baf2669b: entering candidate state: node="Node at 5f475248-45c0-b3d6-f8b5-8336baf2669b [Candidate]" term=2 -2024-05-12T20:51:40.663Z [INFO] server-5f475248-45c0-b3d6-f8b5-8336baf2669b: election won: term=2 tally=2 -2024-05-12T20:51:40.663Z [INFO] server-5f475248-45c0-b3d6-f8b5-8336baf2669b: entering leader state: leader="Node at 5f475248-45c0-b3d6-f8b5-8336baf2669b [Leader]" -2024-05-12T20:51:40.663Z [INFO] server-5f475248-45c0-b3d6-f8b5-8336baf2669b: added peer, starting replication: peer=server-4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62 -2024-05-12T20:51:40.663Z [INFO] server-5f475248-45c0-b3d6-f8b5-8336baf2669b: pipelining replication: peer="{Voter server-4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62 4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62}" -2024-05-12T20:51:40.770Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:40.663694655 +0000 UTC m=+111.944395052 (1 nodes), 59.413271ms from start of poll, 59.648121ms from cluster start. Timeout at 2024-05-12 20:51:40.770788121 +0000 UTC m=+112.051488762, 107.09371ms after stability -2024-05-12T20:51:40.770Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:40.878Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:40.770918673 +0000 UTC m=+112.051619297 (1 nodes), 3.733µs from start of poll, 166.872366ms from cluster start. Timeout at 2024-05-12 20:51:40.87878809 +0000 UTC m=+112.159488528, 107.869231ms after stability -2024-05-12T20:51:40.880Z [ERROR] server-4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62: failed to append to logs: error="unable to store logs within log store, err: "some error"" -2024-05-12T20:51:40.880Z [WARN] server-4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62: failed to get previous log: previous-index=9 last-index=7 error="log not found" -2024-05-12T20:51:40.880Z [INFO] server-5f475248-45c0-b3d6-f8b5-8336baf2669b: aborting pipeline replication: peer="{Voter server-4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62 4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62}" -2024-05-12T20:51:40.890Z [INFO] server-5f475248-45c0-b3d6-f8b5-8336baf2669b: pipelining replication: peer="{Voter server-4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62 4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62}" -2024-05-12T20:51:40.891Z [INFO] server-4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62 Address:4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62} {Suffrage:Voter ID:server-5f475248-45c0-b3d6-f8b5-8336baf2669b Address:5f475248-45c0-b3d6-f8b5-8336baf2669b}]" -2024-05-12T20:51:40.891Z [INFO] server-5f475248-45c0-b3d6-f8b5-8336baf2669b: aborting pipeline replication: peer="{Voter server-4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62 4bff0a5e-04fd-6fd1-438d-ba8e15fe1d62}" ---- PASS: TestRaft_CacheLogWithStoreError (0.29s) +2025-06-15T03:18:10.913Z [INFO] server-3a289a42-95b3-392d-9197-4f237770f1d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3a289a42-95b3-392d-9197-4f237770f1d0 Address:3a289a42-95b3-392d-9197-4f237770f1d0} {Suffrage:Voter ID:server-8310ab92-4ddd-e15e-864e-6a3ec73a9746 Address:8310ab92-4ddd-e15e-864e-6a3ec73a9746}]" +2025-06-15T03:18:10.913Z [INFO] server-8310ab92-4ddd-e15e-864e-6a3ec73a9746: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3a289a42-95b3-392d-9197-4f237770f1d0 Address:3a289a42-95b3-392d-9197-4f237770f1d0} {Suffrage:Voter ID:server-8310ab92-4ddd-e15e-864e-6a3ec73a9746 Address:8310ab92-4ddd-e15e-864e-6a3ec73a9746}]" +2025-06-15T03:18:10.913Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:10.913Z [INFO] server-3a289a42-95b3-392d-9197-4f237770f1d0: entering follower state: follower="Node at 3a289a42-95b3-392d-9197-4f237770f1d0 [Follower]" leader-address= leader-id= +2025-06-15T03:18:10.913Z [INFO] server-8310ab92-4ddd-e15e-864e-6a3ec73a9746: entering follower state: follower="Node at 8310ab92-4ddd-e15e-864e-6a3ec73a9746 [Follower]" leader-address= leader-id= +2025-06-15T03:18:10.976Z [WARN] server-8310ab92-4ddd-e15e-864e-6a3ec73a9746: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:10.977Z [INFO] server-8310ab92-4ddd-e15e-864e-6a3ec73a9746: entering candidate state: node="Node at 8310ab92-4ddd-e15e-864e-6a3ec73a9746 [Candidate]" term=2 +2025-06-15T03:18:10.977Z [INFO] server-8310ab92-4ddd-e15e-864e-6a3ec73a9746: election won: term=2 tally=2 +2025-06-15T03:18:10.977Z [INFO] server-8310ab92-4ddd-e15e-864e-6a3ec73a9746: entering leader state: leader="Node at 8310ab92-4ddd-e15e-864e-6a3ec73a9746 [Leader]" +2025-06-15T03:18:10.977Z [INFO] server-8310ab92-4ddd-e15e-864e-6a3ec73a9746: added peer, starting replication: peer=server-3a289a42-95b3-392d-9197-4f237770f1d0 +2025-06-15T03:18:10.977Z [INFO] server-8310ab92-4ddd-e15e-864e-6a3ec73a9746: pipelining replication: peer="{Voter server-3a289a42-95b3-392d-9197-4f237770f1d0 3a289a42-95b3-392d-9197-4f237770f1d0}" +2025-06-15T03:18:11.082Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:10.977152149 +0000 UTC m=+106.688830349 (1 nodes), 63.466577ms from start of poll, 63.643538ms from cluster start. Timeout at 2025-06-15 03:18:11.08215952 +0000 UTC m=+106.793837753, 105.007404ms after stability +2025-06-15T03:18:11.082Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:11.187Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:11.082233 +0000 UTC m=+106.793911202 (1 nodes), 1.713µs from start of poll, 168.724391ms from cluster start. Timeout at 2025-06-15 03:18:11.18731381 +0000 UTC m=+106.898992010, 105.080808ms after stability +2025-06-15T03:18:11.188Z [ERROR] server-3a289a42-95b3-392d-9197-4f237770f1d0: failed to append to logs: error="unable to store logs within log store, err: "some error"" +2025-06-15T03:18:11.188Z [WARN] server-3a289a42-95b3-392d-9197-4f237770f1d0: failed to get previous log: previous-index=9 last-index=7 error="log not found" +2025-06-15T03:18:11.188Z [INFO] server-8310ab92-4ddd-e15e-864e-6a3ec73a9746: aborting pipeline replication: peer="{Voter server-3a289a42-95b3-392d-9197-4f237770f1d0 3a289a42-95b3-392d-9197-4f237770f1d0}" +2025-06-15T03:18:11.194Z [INFO] server-8310ab92-4ddd-e15e-864e-6a3ec73a9746: pipelining replication: peer="{Voter server-3a289a42-95b3-392d-9197-4f237770f1d0 3a289a42-95b3-392d-9197-4f237770f1d0}" +2025-06-15T03:18:11.195Z [INFO] server-3a289a42-95b3-392d-9197-4f237770f1d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3a289a42-95b3-392d-9197-4f237770f1d0 Address:3a289a42-95b3-392d-9197-4f237770f1d0} {Suffrage:Voter ID:server-8310ab92-4ddd-e15e-864e-6a3ec73a9746 Address:8310ab92-4ddd-e15e-864e-6a3ec73a9746}]" +2025-06-15T03:18:11.195Z [INFO] server-8310ab92-4ddd-e15e-864e-6a3ec73a9746: aborting pipeline replication: peer="{Voter server-3a289a42-95b3-392d-9197-4f237770f1d0 3a289a42-95b3-392d-9197-4f237770f1d0}" +--- PASS: TestRaft_CacheLogWithStoreError (0.28s) === RUN TestRaft_ReloadConfig -2024-05-12T20:51:40.892Z [INFO] server-ed7bb695-9eef-ed2c-bf80-e2da145905fa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ed7bb695-9eef-ed2c-bf80-e2da145905fa Address:ed7bb695-9eef-ed2c-bf80-e2da145905fa}]" +2025-06-15T03:18:11.195Z [INFO] server-59f39839-35f6-41ac-6b1a-8acfc6b838ae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-59f39839-35f6-41ac-6b1a-8acfc6b838ae Address:59f39839-35f6-41ac-6b1a-8acfc6b838ae}]" +2025-06-15T03:18:11.195Z [INFO] server-59f39839-35f6-41ac-6b1a-8acfc6b838ae: entering follower state: follower="Node at 59f39839-35f6-41ac-6b1a-8acfc6b838ae [Shutdown]" leader-address= leader-id= --- PASS: TestRaft_ReloadConfig (0.00s) === RUN TestRaft_ReloadConfigValidates -2024-05-12T20:51:40.892Z [INFO] server-2cd4404b-3d24-12e1-d98e-998c98fb8890: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2cd4404b-3d24-12e1-d98e-998c98fb8890 Address:2cd4404b-3d24-12e1-d98e-998c98fb8890}]" +2025-06-15T03:18:11.196Z [INFO] server-8950caa9-4ea8-0fdc-20bf-0df8bbfc50a3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8950caa9-4ea8-0fdc-20bf-0df8bbfc50a3 Address:8950caa9-4ea8-0fdc-20bf-0df8bbfc50a3}]" +2025-06-15T03:18:11.196Z [INFO] server-8950caa9-4ea8-0fdc-20bf-0df8bbfc50a3: entering follower state: follower="Node at 8950caa9-4ea8-0fdc-20bf-0df8bbfc50a3 [Shutdown]" leader-address= leader-id= --- PASS: TestRaft_ReloadConfigValidates (0.00s) === RUN TestRaft_InstallSnapshot_InvalidPeers -2024-05-12T20:51:40.893Z [ERROR] failed to install snapshot: error="failed to decode peers: codec.decoder: Only encoded map or array can be decoded into a slice. (valueType: 2)" +2025-06-15T03:18:11.196Z [ERROR] failed to install snapshot: error="failed to decode peers: codec.decoder: Only encoded map or array can be decoded into a slice. (valueType: 2)" --- PASS: TestRaft_InstallSnapshot_InvalidPeers (0.00s) === RUN TestRaft_VoteNotGranted_WhenNodeNotInCluster -2024-05-12T20:51:40.893Z [INFO] server-8a1fcf96-bf4d-e102-680c-4cc4bfd15d3a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a1fcf96-bf4d-e102-680c-4cc4bfd15d3a Address:8a1fcf96-bf4d-e102-680c-4cc4bfd15d3a} {Suffrage:Voter ID:server-1367bbc6-7a8f-a429-028e-680c125cdb0a Address:1367bbc6-7a8f-a429-028e-680c125cdb0a} {Suffrage:Voter ID:server-ade32191-6b7b-5dad-98fa-28c7950c6a81 Address:ade32191-6b7b-5dad-98fa-28c7950c6a81}]" -2024-05-12T20:51:40.894Z [INFO] server-1367bbc6-7a8f-a429-028e-680c125cdb0a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a1fcf96-bf4d-e102-680c-4cc4bfd15d3a Address:8a1fcf96-bf4d-e102-680c-4cc4bfd15d3a} {Suffrage:Voter ID:server-1367bbc6-7a8f-a429-028e-680c125cdb0a Address:1367bbc6-7a8f-a429-028e-680c125cdb0a} {Suffrage:Voter ID:server-ade32191-6b7b-5dad-98fa-28c7950c6a81 Address:ade32191-6b7b-5dad-98fa-28c7950c6a81}]" -2024-05-12T20:51:40.894Z [INFO] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a1fcf96-bf4d-e102-680c-4cc4bfd15d3a Address:8a1fcf96-bf4d-e102-680c-4cc4bfd15d3a} {Suffrage:Voter ID:server-1367bbc6-7a8f-a429-028e-680c125cdb0a Address:1367bbc6-7a8f-a429-028e-680c125cdb0a} {Suffrage:Voter ID:server-ade32191-6b7b-5dad-98fa-28c7950c6a81 Address:ade32191-6b7b-5dad-98fa-28c7950c6a81}]" -2024-05-12T20:51:40.894Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:40.894Z [INFO] server-8a1fcf96-bf4d-e102-680c-4cc4bfd15d3a: entering follower state: follower="Node at 8a1fcf96-bf4d-e102-680c-4cc4bfd15d3a [Follower]" leader-address= leader-id= -2024-05-12T20:51:40.894Z [INFO] server-1367bbc6-7a8f-a429-028e-680c125cdb0a: entering follower state: follower="Node at 1367bbc6-7a8f-a429-028e-680c125cdb0a [Follower]" leader-address= leader-id= -2024-05-12T20:51:40.894Z [INFO] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: entering follower state: follower="Node at ade32191-6b7b-5dad-98fa-28c7950c6a81 [Follower]" leader-address= leader-id= -2024-05-12T20:51:40.955Z [WARN] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:40.955Z [INFO] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: entering candidate state: node="Node at ade32191-6b7b-5dad-98fa-28c7950c6a81 [Candidate]" term=2 -2024-05-12T20:51:40.955Z [INFO] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: election won: term=2 tally=2 -2024-05-12T20:51:40.955Z [INFO] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: entering leader state: leader="Node at ade32191-6b7b-5dad-98fa-28c7950c6a81 [Leader]" -2024-05-12T20:51:40.955Z [INFO] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: added peer, starting replication: peer=server-8a1fcf96-bf4d-e102-680c-4cc4bfd15d3a -2024-05-12T20:51:40.955Z [INFO] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: added peer, starting replication: peer=server-1367bbc6-7a8f-a429-028e-680c125cdb0a -2024-05-12T20:51:40.955Z [INFO] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: pipelining replication: peer="{Voter server-1367bbc6-7a8f-a429-028e-680c125cdb0a 1367bbc6-7a8f-a429-028e-680c125cdb0a}" -2024-05-12T20:51:40.956Z [INFO] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: pipelining replication: peer="{Voter server-8a1fcf96-bf4d-e102-680c-4cc4bfd15d3a 8a1fcf96-bf4d-e102-680c-4cc4bfd15d3a}" -2024-05-12T20:51:41.062Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:40.956569204 +0000 UTC m=+112.237269555 (1 nodes), 62.410532ms from start of poll, 62.721958ms from cluster start. Timeout at 2024-05-12 20:51:41.062823222 +0000 UTC m=+112.343523668, 106.254113ms after stability -2024-05-12T20:51:41.070Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:41.179Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:41.070858227 +0000 UTC m=+112.351558556 (2 nodes), 3.603µs from start of poll, 177.010959ms from cluster start. Timeout at 2024-05-12 20:51:41.179263054 +0000 UTC m=+112.459963612, 108.405056ms after stability -2024-05-12T20:51:41.179Z [INFO] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: updating configuration: command=RemoveServer server-id=server-8a1fcf96-bf4d-e102-680c-4cc4bfd15d3a server-addr= servers="[{Suffrage:Voter ID:server-1367bbc6-7a8f-a429-028e-680c125cdb0a Address:1367bbc6-7a8f-a429-028e-680c125cdb0a} {Suffrage:Voter ID:server-ade32191-6b7b-5dad-98fa-28c7950c6a81 Address:ade32191-6b7b-5dad-98fa-28c7950c6a81}]" -2024-05-12T20:51:41.179Z [INFO] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: removed peer, stopping replication: peer=server-8a1fcf96-bf4d-e102-680c-4cc4bfd15d3a last-index=3 -2024-05-12T20:51:41.179Z [INFO] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: aborting pipeline replication: peer="{Voter server-8a1fcf96-bf4d-e102-680c-4cc4bfd15d3a 8a1fcf96-bf4d-e102-680c-4cc4bfd15d3a}" -2024-05-12T20:51:41.262Z [WARN] server-8a1fcf96-bf4d-e102-680c-4cc4bfd15d3a: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2024-05-12T20:51:41.318Z [ERROR] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: failed to heartbeat to: peer=1367bbc6-7a8f-a429-028e-680c125cdb0a backoff time=10ms error="failed to connect to peer: 1367bbc6-7a8f-a429-028e-680c125cdb0a" -2024-05-12T20:51:41.319Z [ERROR] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: failed to pipeline appendEntries: peer="{Voter server-1367bbc6-7a8f-a429-028e-680c125cdb0a 1367bbc6-7a8f-a429-028e-680c125cdb0a}" error="append pipeline closed" -2024-05-12T20:51:41.319Z [INFO] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: aborting pipeline replication: peer="{Voter server-1367bbc6-7a8f-a429-028e-680c125cdb0a 1367bbc6-7a8f-a429-028e-680c125cdb0a}" -2024-05-12T20:51:41.326Z [ERROR] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: failed to appendEntries to: peer="{Voter server-1367bbc6-7a8f-a429-028e-680c125cdb0a 1367bbc6-7a8f-a429-028e-680c125cdb0a}" error="failed to connect to peer: 1367bbc6-7a8f-a429-028e-680c125cdb0a" -2024-05-12T20:51:41.342Z [ERROR] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: failed to heartbeat to: peer=1367bbc6-7a8f-a429-028e-680c125cdb0a backoff time=10ms error="failed to connect to peer: 1367bbc6-7a8f-a429-028e-680c125cdb0a" -2024-05-12T20:51:41.347Z [ERROR] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: failed to appendEntries to: peer="{Voter server-1367bbc6-7a8f-a429-028e-680c125cdb0a 1367bbc6-7a8f-a429-028e-680c125cdb0a}" error="failed to connect to peer: 1367bbc6-7a8f-a429-028e-680c125cdb0a" -2024-05-12T20:51:41.350Z [WARN] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: failed to contact: server-id=server-1367bbc6-7a8f-a429-028e-680c125cdb0a time=71.340325ms -2024-05-12T20:51:41.350Z [WARN] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: failed to contact quorum of nodes, stepping down -2024-05-12T20:51:41.350Z [INFO] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: entering follower state: follower="Node at ade32191-6b7b-5dad-98fa-28c7950c6a81 [Follower]" leader-address= leader-id= -2024-05-12T20:51:41.411Z [WARN] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:41.411Z [INFO] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: entering candidate state: node="Node at ade32191-6b7b-5dad-98fa-28c7950c6a81 [Candidate]" term=3 -2024-05-12T20:51:41.411Z [ERROR] server-ade32191-6b7b-5dad-98fa-28c7950c6a81: failed to make requestVote RPC: target="{Voter server-1367bbc6-7a8f-a429-028e-680c125cdb0a 1367bbc6-7a8f-a429-028e-680c125cdb0a}" error="failed to connect to peer: 1367bbc6-7a8f-a429-028e-680c125cdb0a" term=3 -2024-05-12T20:51:41.411Z [WARN] server-1367bbc6-7a8f-a429-028e-680c125cdb0a: heartbeat timeout reached, starting election: last-leader-addr=ade32191-6b7b-5dad-98fa-28c7950c6a81 last-leader-id=server-ade32191-6b7b-5dad-98fa-28c7950c6a81 -2024-05-12T20:51:41.411Z [INFO] server-1367bbc6-7a8f-a429-028e-680c125cdb0a: entering candidate state: node="Node at 1367bbc6-7a8f-a429-028e-680c125cdb0a [Candidate]" term=3 -2024-05-12T20:51:41.411Z [ERROR] server-1367bbc6-7a8f-a429-028e-680c125cdb0a: failed to make requestVote RPC: target="{Voter server-ade32191-6b7b-5dad-98fa-28c7950c6a81 ade32191-6b7b-5dad-98fa-28c7950c6a81}" error="failed to connect to peer: ade32191-6b7b-5dad-98fa-28c7950c6a81" term=3 -2024-05-12T20:51:41.414Z [WARN] server-1367bbc6-7a8f-a429-028e-680c125cdb0a: rejecting vote request since node is not in configuration: from=8a1fcf96-bf4d-e102-680c-4cc4bfd15d3a +2025-06-15T03:18:11.197Z [INFO] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4 Address:1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4} {Suffrage:Voter ID:server-58163e0e-8256-218f-7d48-7a4e56474e63 Address:58163e0e-8256-218f-7d48-7a4e56474e63} {Suffrage:Voter ID:server-b8a5793f-27c7-3722-76bf-85c54bde82c7 Address:b8a5793f-27c7-3722-76bf-85c54bde82c7}]" +2025-06-15T03:18:11.197Z [INFO] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: entering follower state: follower="Node at 1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4 [Follower]" leader-address= leader-id= +2025-06-15T03:18:11.197Z [INFO] server-58163e0e-8256-218f-7d48-7a4e56474e63: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4 Address:1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4} {Suffrage:Voter ID:server-58163e0e-8256-218f-7d48-7a4e56474e63 Address:58163e0e-8256-218f-7d48-7a4e56474e63} {Suffrage:Voter ID:server-b8a5793f-27c7-3722-76bf-85c54bde82c7 Address:b8a5793f-27c7-3722-76bf-85c54bde82c7}]" +2025-06-15T03:18:11.197Z [INFO] server-58163e0e-8256-218f-7d48-7a4e56474e63: entering follower state: follower="Node at 58163e0e-8256-218f-7d48-7a4e56474e63 [Follower]" leader-address= leader-id= +2025-06-15T03:18:11.197Z [INFO] server-b8a5793f-27c7-3722-76bf-85c54bde82c7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4 Address:1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4} {Suffrage:Voter ID:server-58163e0e-8256-218f-7d48-7a4e56474e63 Address:58163e0e-8256-218f-7d48-7a4e56474e63} {Suffrage:Voter ID:server-b8a5793f-27c7-3722-76bf-85c54bde82c7 Address:b8a5793f-27c7-3722-76bf-85c54bde82c7}]" +2025-06-15T03:18:11.197Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:11.197Z [INFO] server-b8a5793f-27c7-3722-76bf-85c54bde82c7: entering follower state: follower="Node at b8a5793f-27c7-3722-76bf-85c54bde82c7 [Follower]" leader-address= leader-id= +2025-06-15T03:18:11.250Z [WARN] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:11.250Z [INFO] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: entering candidate state: node="Node at 1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4 [Candidate]" term=2 +2025-06-15T03:18:11.250Z [INFO] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: election won: term=2 tally=2 +2025-06-15T03:18:11.251Z [INFO] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: entering leader state: leader="Node at 1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4 [Leader]" +2025-06-15T03:18:11.251Z [INFO] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: added peer, starting replication: peer=server-58163e0e-8256-218f-7d48-7a4e56474e63 +2025-06-15T03:18:11.251Z [INFO] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: added peer, starting replication: peer=server-b8a5793f-27c7-3722-76bf-85c54bde82c7 +2025-06-15T03:18:11.251Z [INFO] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: pipelining replication: peer="{Voter server-b8a5793f-27c7-3722-76bf-85c54bde82c7 b8a5793f-27c7-3722-76bf-85c54bde82c7}" +2025-06-15T03:18:11.251Z [INFO] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: pipelining replication: peer="{Voter server-58163e0e-8256-218f-7d48-7a4e56474e63 58163e0e-8256-218f-7d48-7a4e56474e63}" +2025-06-15T03:18:11.356Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:11.251038865 +0000 UTC m=+106.962717064 (1 nodes), 53.805538ms from start of poll, 54.08255ms from cluster start. Timeout at 2025-06-15 03:18:11.356355267 +0000 UTC m=+107.068033499, 105.316435ms after stability +2025-06-15T03:18:11.362Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:11.467Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:11.362515713 +0000 UTC m=+107.074193956 (2 nodes), 2.125µs from start of poll, 165.559442ms from cluster start. Timeout at 2025-06-15 03:18:11.467721933 +0000 UTC m=+107.179400132, 105.206176ms after stability +2025-06-15T03:18:11.467Z [INFO] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: updating configuration: command=RemoveServer server-id=server-58163e0e-8256-218f-7d48-7a4e56474e63 server-addr= servers="[{Suffrage:Voter ID:server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4 Address:1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4} {Suffrage:Voter ID:server-b8a5793f-27c7-3722-76bf-85c54bde82c7 Address:b8a5793f-27c7-3722-76bf-85c54bde82c7}]" +2025-06-15T03:18:11.467Z [INFO] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: removed peer, stopping replication: peer=server-58163e0e-8256-218f-7d48-7a4e56474e63 last-index=3 +2025-06-15T03:18:11.467Z [INFO] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: aborting pipeline replication: peer="{Voter server-58163e0e-8256-218f-7d48-7a4e56474e63 58163e0e-8256-218f-7d48-7a4e56474e63}" +2025-06-15T03:18:11.517Z [WARN] server-58163e0e-8256-218f-7d48-7a4e56474e63: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-06-15T03:18:11.579Z [ERROR] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: failed to heartbeat to: peer=b8a5793f-27c7-3722-76bf-85c54bde82c7 backoff time=10ms error="failed to connect to peer: b8a5793f-27c7-3722-76bf-85c54bde82c7" +2025-06-15T03:18:11.582Z [ERROR] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: failed to pipeline appendEntries: peer="{Voter server-b8a5793f-27c7-3722-76bf-85c54bde82c7 b8a5793f-27c7-3722-76bf-85c54bde82c7}" error="append pipeline closed" +2025-06-15T03:18:11.582Z [INFO] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: aborting pipeline replication: peer="{Voter server-b8a5793f-27c7-3722-76bf-85c54bde82c7 b8a5793f-27c7-3722-76bf-85c54bde82c7}" +2025-06-15T03:18:11.591Z [ERROR] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: failed to appendEntries to: peer="{Voter server-b8a5793f-27c7-3722-76bf-85c54bde82c7 b8a5793f-27c7-3722-76bf-85c54bde82c7}" error="failed to connect to peer: b8a5793f-27c7-3722-76bf-85c54bde82c7" +2025-06-15T03:18:11.598Z [ERROR] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: failed to heartbeat to: peer=b8a5793f-27c7-3722-76bf-85c54bde82c7 backoff time=10ms error="failed to connect to peer: b8a5793f-27c7-3722-76bf-85c54bde82c7" +2025-06-15T03:18:11.608Z [ERROR] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: failed to appendEntries to: peer="{Voter server-b8a5793f-27c7-3722-76bf-85c54bde82c7 b8a5793f-27c7-3722-76bf-85c54bde82c7}" error="failed to connect to peer: b8a5793f-27c7-3722-76bf-85c54bde82c7" +2025-06-15T03:18:11.618Z [ERROR] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: failed to heartbeat to: peer=b8a5793f-27c7-3722-76bf-85c54bde82c7 backoff time=10ms error="failed to connect to peer: b8a5793f-27c7-3722-76bf-85c54bde82c7" +2025-06-15T03:18:11.620Z [WARN] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: failed to contact: server-id=server-b8a5793f-27c7-3722-76bf-85c54bde82c7 time=51.124584ms +2025-06-15T03:18:11.620Z [WARN] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: failed to contact quorum of nodes, stepping down +2025-06-15T03:18:11.620Z [INFO] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: entering follower state: follower="Node at 1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4 [Follower]" leader-address= leader-id= +2025-06-15T03:18:11.626Z [ERROR] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: failed to appendEntries to: peer="{Voter server-b8a5793f-27c7-3722-76bf-85c54bde82c7 b8a5793f-27c7-3722-76bf-85c54bde82c7}" error="failed to connect to peer: b8a5793f-27c7-3722-76bf-85c54bde82c7" +2025-06-15T03:18:11.698Z [WARN] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:11.698Z [INFO] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: entering candidate state: node="Node at 1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4 [Candidate]" term=3 +2025-06-15T03:18:11.699Z [ERROR] server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4: failed to make requestVote RPC: target="{Voter server-b8a5793f-27c7-3722-76bf-85c54bde82c7 b8a5793f-27c7-3722-76bf-85c54bde82c7}" error="failed to connect to peer: b8a5793f-27c7-3722-76bf-85c54bde82c7" term=3 +2025-06-15T03:18:11.711Z [WARN] server-b8a5793f-27c7-3722-76bf-85c54bde82c7: heartbeat timeout reached, starting election: last-leader-addr=1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4 last-leader-id=server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4 +2025-06-15T03:18:11.711Z [INFO] server-b8a5793f-27c7-3722-76bf-85c54bde82c7: entering candidate state: node="Node at b8a5793f-27c7-3722-76bf-85c54bde82c7 [Candidate]" term=3 +2025-06-15T03:18:11.711Z [ERROR] server-b8a5793f-27c7-3722-76bf-85c54bde82c7: failed to make requestVote RPC: target="{Voter server-1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4 1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4}" error="failed to connect to peer: 1b6fe47e-8a45-fb15-5c23-2fa3b47a04a4" term=3 +2025-06-15T03:18:11.712Z [WARN] server-b8a5793f-27c7-3722-76bf-85c54bde82c7: rejecting vote request since node is not in configuration: from=58163e0e-8256-218f-7d48-7a4e56474e63 --- PASS: TestRaft_VoteNotGranted_WhenNodeNotInCluster (0.52s) === RUN TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin -2024-05-12T20:51:41.417Z [INFO] server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f Address:86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f} {Suffrage:Voter ID:server-c42efae7-7d14-eaca-8c65-6367108a7cdc Address:c42efae7-7d14-eaca-8c65-6367108a7cdc} {Suffrage:Voter ID:server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c Address:bde19fcf-9bbd-5bd5-3c5c-7a600f26861c}]" -2024-05-12T20:51:41.417Z [INFO] server-c42efae7-7d14-eaca-8c65-6367108a7cdc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f Address:86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f} {Suffrage:Voter ID:server-c42efae7-7d14-eaca-8c65-6367108a7cdc Address:c42efae7-7d14-eaca-8c65-6367108a7cdc} {Suffrage:Voter ID:server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c Address:bde19fcf-9bbd-5bd5-3c5c-7a600f26861c}]" -2024-05-12T20:51:41.417Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f Address:86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f} {Suffrage:Voter ID:server-c42efae7-7d14-eaca-8c65-6367108a7cdc Address:c42efae7-7d14-eaca-8c65-6367108a7cdc} {Suffrage:Voter ID:server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c Address:bde19fcf-9bbd-5bd5-3c5c-7a600f26861c}]" -2024-05-12T20:51:41.417Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:41.417Z [INFO] server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f: entering follower state: follower="Node at 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f [Follower]" leader-address= leader-id= -2024-05-12T20:51:41.417Z [INFO] server-c42efae7-7d14-eaca-8c65-6367108a7cdc: entering follower state: follower="Node at c42efae7-7d14-eaca-8c65-6367108a7cdc [Follower]" leader-address= leader-id= -2024-05-12T20:51:41.417Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: entering follower state: follower="Node at bde19fcf-9bbd-5bd5-3c5c-7a600f26861c [Follower]" leader-address= leader-id= -2024-05-12T20:51:41.474Z [WARN] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:41.475Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: entering candidate state: node="Node at bde19fcf-9bbd-5bd5-3c5c-7a600f26861c [Candidate]" term=2 -2024-05-12T20:51:41.475Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: election won: term=2 tally=2 -2024-05-12T20:51:41.475Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: entering leader state: leader="Node at bde19fcf-9bbd-5bd5-3c5c-7a600f26861c [Leader]" -2024-05-12T20:51:41.475Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: added peer, starting replication: peer=server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f -2024-05-12T20:51:41.475Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: added peer, starting replication: peer=server-c42efae7-7d14-eaca-8c65-6367108a7cdc -2024-05-12T20:51:41.475Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: pipelining replication: peer="{Voter server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f}" -2024-05-12T20:51:41.475Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: pipelining replication: peer="{Voter server-c42efae7-7d14-eaca-8c65-6367108a7cdc c42efae7-7d14-eaca-8c65-6367108a7cdc}" -2024-05-12T20:51:41.583Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:41.475385065 +0000 UTC m=+112.756085523 (1 nodes), 57.695189ms from start of poll, 58.10699ms from cluster start. Timeout at 2024-05-12 20:51:41.583198169 +0000 UTC m=+112.863898729, 107.813206ms after stability -2024-05-12T20:51:41.590Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:41.698Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:41.590969649 +0000 UTC m=+112.871670317 (2 nodes), 3.433µs from start of poll, 173.691784ms from cluster start. Timeout at 2024-05-12 20:51:41.698795823 +0000 UTC m=+112.979496273, 107.825956ms after stability -2024-05-12T20:51:41.699Z [ERROR] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: failed to pipeline appendEntries: peer="{Voter server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f}" error="append pipeline closed" -2024-05-12T20:51:41.699Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: aborting pipeline replication: peer="{Voter server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f}" -2024-05-12T20:51:41.699Z [ERROR] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: failed to heartbeat to: peer=86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f backoff time=10ms error="failed to connect to peer: 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f" -2024-05-12T20:51:41.718Z [ERROR] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: failed to appendEntries to: peer="{Voter server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f}" error="failed to connect to peer: 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f" -2024-05-12T20:51:41.730Z [ERROR] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: failed to heartbeat to: peer=86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f backoff time=10ms error="failed to connect to peer: 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f" -2024-05-12T20:51:41.739Z [ERROR] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: failed to appendEntries to: peer="{Voter server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f}" error="failed to connect to peer: 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f" -2024-05-12T20:51:41.742Z [WARN] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: failed to contact: server-id=server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f time=52.021888ms -2024-05-12T20:51:41.743Z [WARN] server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f: heartbeat timeout reached, starting election: last-leader-addr=bde19fcf-9bbd-5bd5-3c5c-7a600f26861c last-leader-id=server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c -2024-05-12T20:51:41.743Z [INFO] server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f: entering candidate state: node="Node at 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f [Candidate]" term=3 -2024-05-12T20:51:41.743Z [ERROR] server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f: failed to make requestVote RPC: target="{Voter server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c bde19fcf-9bbd-5bd5-3c5c-7a600f26861c}" error="failed to connect to peer: bde19fcf-9bbd-5bd5-3c5c-7a600f26861c" term=3 -2024-05-12T20:51:41.743Z [ERROR] server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f: failed to make requestVote RPC: target="{Voter server-c42efae7-7d14-eaca-8c65-6367108a7cdc c42efae7-7d14-eaca-8c65-6367108a7cdc}" error="failed to connect to peer: c42efae7-7d14-eaca-8c65-6367108a7cdc" term=3 -2024-05-12T20:51:41.758Z [ERROR] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: failed to heartbeat to: peer=86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f backoff time=10ms error="failed to connect to peer: 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f" -2024-05-12T20:51:41.762Z [ERROR] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: failed to appendEntries to: peer="{Voter server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f}" error="failed to connect to peer: 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f" -2024-05-12T20:51:41.778Z [ERROR] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: failed to heartbeat to: peer=86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f backoff time=20ms error="failed to connect to peer: 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f" -2024-05-12T20:51:41.790Z [WARN] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: failed to contact: server-id=server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f time=99.877392ms -2024-05-12T20:51:41.794Z [ERROR] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: failed to appendEntries to: peer="{Voter server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f}" error="failed to connect to peer: 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f" -2024-05-12T20:51:41.810Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: updating configuration: command=RemoveServer server-id=server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f server-addr= servers="[{Suffrage:Voter ID:server-c42efae7-7d14-eaca-8c65-6367108a7cdc Address:c42efae7-7d14-eaca-8c65-6367108a7cdc} {Suffrage:Voter ID:server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c Address:bde19fcf-9bbd-5bd5-3c5c-7a600f26861c}]" -2024-05-12T20:51:41.811Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: removed peer, stopping replication: peer=server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f last-index=3 -2024-05-12T20:51:41.811Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: updating configuration: command=AddNonvoter server-id=server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f server-addr=86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f servers="[{Suffrage:Voter ID:server-c42efae7-7d14-eaca-8c65-6367108a7cdc Address:c42efae7-7d14-eaca-8c65-6367108a7cdc} {Suffrage:Voter ID:server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c Address:bde19fcf-9bbd-5bd5-3c5c-7a600f26861c} {Suffrage:Nonvoter ID:server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f Address:86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f}]" -2024-05-12T20:51:41.811Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: added peer, starting replication: peer=server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f -2024-05-12T20:51:41.811Z [ERROR] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: failed to appendEntries to: peer="{Nonvoter server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f}" error="failed to connect to peer: 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f" -2024-05-12T20:51:41.839Z [ERROR] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: peer has newer term, stopping replication: peer="{Nonvoter server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f}" -2024-05-12T20:51:41.839Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: entering follower state: follower="Node at bde19fcf-9bbd-5bd5-3c5c-7a600f26861c [Follower]" leader-address= leader-id= -2024-05-12T20:51:41.839Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: aborting pipeline replication: peer="{Voter server-c42efae7-7d14-eaca-8c65-6367108a7cdc c42efae7-7d14-eaca-8c65-6367108a7cdc}" -2024-05-12T20:51:41.839Z [WARN] server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f: Election timeout reached, restarting election -2024-05-12T20:51:41.839Z [INFO] server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f: entering candidate state: node="Node at 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f [Candidate]" term=23 -2024-05-12T20:51:41.839Z [WARN] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: rejecting vote request since node is not a voter: from=86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f -2024-05-12T20:51:41.839Z [WARN] server-c42efae7-7d14-eaca-8c65-6367108a7cdc: rejecting vote request since we have a leader: from=86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f leader=bde19fcf-9bbd-5bd5-3c5c-7a600f26861c leader-id=server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c -2024-05-12T20:51:41.850Z [ERROR] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: peer has newer term, stopping replication: peer="{Voter server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f}" -2024-05-12T20:51:41.910Z [WARN] server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f: Election timeout reached, restarting election -2024-05-12T20:51:41.910Z [INFO] server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f: entering candidate state: node="Node at 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f [Candidate]" term=24 -2024-05-12T20:51:41.910Z [WARN] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: rejecting vote request since node is not a voter: from=86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f -2024-05-12T20:51:41.911Z [WARN] server-c42efae7-7d14-eaca-8c65-6367108a7cdc: rejecting vote request since we have a leader: from=86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f leader=bde19fcf-9bbd-5bd5-3c5c-7a600f26861c leader-id=server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c -2024-05-12T20:51:41.919Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:41.938Z [WARN] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:41.938Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: entering candidate state: node="Node at bde19fcf-9bbd-5bd5-3c5c-7a600f26861c [Candidate]" term=25 -2024-05-12T20:51:41.939Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: election won: term=25 tally=2 -2024-05-12T20:51:41.939Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: entering leader state: leader="Node at bde19fcf-9bbd-5bd5-3c5c-7a600f26861c [Leader]" -2024-05-12T20:51:41.939Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: added peer, starting replication: peer=server-c42efae7-7d14-eaca-8c65-6367108a7cdc -2024-05-12T20:51:41.939Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: added peer, starting replication: peer=server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f -2024-05-12T20:51:41.939Z [WARN] server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f: failed to get previous log: previous-index=4 last-index=2 error="log not found" -2024-05-12T20:51:41.939Z [INFO] server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f: entering follower state: follower="Node at 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f [Follower]" leader-address=bde19fcf-9bbd-5bd5-3c5c-7a600f26861c leader-id=server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c -2024-05-12T20:51:41.939Z [WARN] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: appendEntries rejected, sending older logs: peer="{Nonvoter server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f}" next=3 -2024-05-12T20:51:41.939Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: pipelining replication: peer="{Nonvoter server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f}" -2024-05-12T20:51:41.939Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: pipelining replication: peer="{Voter server-c42efae7-7d14-eaca-8c65-6367108a7cdc c42efae7-7d14-eaca-8c65-6367108a7cdc}" -2024-05-12T20:51:42.062Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:41.939697165 +0000 UTC m=+113.220397493 (1 nodes), 20.606709ms from start of poll, 522.41896ms from cluster start. Timeout at 2024-05-12 20:51:42.062768608 +0000 UTC m=+113.343469105, 123.071612ms after stability -2024-05-12T20:51:42.114Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: updating configuration: command=RemoveServer server-id=server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f server-addr= servers="[{Suffrage:Voter ID:server-c42efae7-7d14-eaca-8c65-6367108a7cdc Address:c42efae7-7d14-eaca-8c65-6367108a7cdc} {Suffrage:Voter ID:server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c Address:bde19fcf-9bbd-5bd5-3c5c-7a600f26861c}]" -2024-05-12T20:51:42.115Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: removed peer, stopping replication: peer=server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f last-index=106 -2024-05-12T20:51:42.115Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: aborting pipeline replication: peer="{Nonvoter server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f}" -2024-05-12T20:51:42.115Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: updating configuration: command=AddVoter server-id=server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f server-addr=86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f servers="[{Suffrage:Voter ID:server-c42efae7-7d14-eaca-8c65-6367108a7cdc Address:c42efae7-7d14-eaca-8c65-6367108a7cdc} {Suffrage:Voter ID:server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c Address:bde19fcf-9bbd-5bd5-3c5c-7a600f26861c} {Suffrage:Voter ID:server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f Address:86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f}]" -2024-05-12T20:51:42.115Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: added peer, starting replication: peer=server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f -2024-05-12T20:51:42.115Z [WARN] server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f: failed to get previous log: previous-index=107 last-index=106 error="log not found" -2024-05-12T20:51:42.115Z [WARN] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: appendEntries rejected, sending older logs: peer="{Voter server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f}" next=107 -2024-05-12T20:51:42.115Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: pipelining replication: peer="{Voter server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f}" -2024-05-12T20:51:42.861Z [WARN] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: failed to contact: server-id=server-c42efae7-7d14-eaca-8c65-6367108a7cdc time=51.999632ms -2024-05-12T20:51:43.198Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:43.305Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:43.199052785 +0000 UTC m=+114.479753145 (1 nodes), 2.336µs from start of poll, 1.781774612s from cluster start. Timeout at 2024-05-12 20:51:43.305195518 +0000 UTC m=+114.585895856, 106.142711ms after stability -2024-05-12T20:51:43.305Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: aborting pipeline replication: peer="{Voter server-86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f 86b5396e-9e36-e4f9-c2aa-26cfae3b0a1f}" -2024-05-12T20:51:43.305Z [INFO] server-bde19fcf-9bbd-5bd5-3c5c-7a600f26861c: aborting pipeline replication: peer="{Voter server-c42efae7-7d14-eaca-8c65-6367108a7cdc c42efae7-7d14-eaca-8c65-6367108a7cdc}" ---- PASS: TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin (1.89s) +2025-06-15T03:18:11.713Z [INFO] server-cc70b568-7754-eb8b-c819-44d1617f4f0a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cc70b568-7754-eb8b-c819-44d1617f4f0a Address:cc70b568-7754-eb8b-c819-44d1617f4f0a} {Suffrage:Voter ID:server-7ca724a0-e06d-0c46-1921-d560eda28c5d Address:7ca724a0-e06d-0c46-1921-d560eda28c5d} {Suffrage:Voter ID:server-c3d72669-3c63-4724-b0af-9ed607222c89 Address:c3d72669-3c63-4724-b0af-9ed607222c89}]" +2025-06-15T03:18:11.713Z [INFO] server-cc70b568-7754-eb8b-c819-44d1617f4f0a: entering follower state: follower="Node at cc70b568-7754-eb8b-c819-44d1617f4f0a [Follower]" leader-address= leader-id= +2025-06-15T03:18:11.713Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cc70b568-7754-eb8b-c819-44d1617f4f0a Address:cc70b568-7754-eb8b-c819-44d1617f4f0a} {Suffrage:Voter ID:server-7ca724a0-e06d-0c46-1921-d560eda28c5d Address:7ca724a0-e06d-0c46-1921-d560eda28c5d} {Suffrage:Voter ID:server-c3d72669-3c63-4724-b0af-9ed607222c89 Address:c3d72669-3c63-4724-b0af-9ed607222c89}]" +2025-06-15T03:18:11.713Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: entering follower state: follower="Node at 7ca724a0-e06d-0c46-1921-d560eda28c5d [Follower]" leader-address= leader-id= +2025-06-15T03:18:11.713Z [INFO] server-c3d72669-3c63-4724-b0af-9ed607222c89: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cc70b568-7754-eb8b-c819-44d1617f4f0a Address:cc70b568-7754-eb8b-c819-44d1617f4f0a} {Suffrage:Voter ID:server-7ca724a0-e06d-0c46-1921-d560eda28c5d Address:7ca724a0-e06d-0c46-1921-d560eda28c5d} {Suffrage:Voter ID:server-c3d72669-3c63-4724-b0af-9ed607222c89 Address:c3d72669-3c63-4724-b0af-9ed607222c89}]" +2025-06-15T03:18:11.713Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:11.713Z [INFO] server-c3d72669-3c63-4724-b0af-9ed607222c89: entering follower state: follower="Node at c3d72669-3c63-4724-b0af-9ed607222c89 [Follower]" leader-address= leader-id= +2025-06-15T03:18:11.774Z [WARN] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:11.774Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: entering candidate state: node="Node at 7ca724a0-e06d-0c46-1921-d560eda28c5d [Candidate]" term=2 +2025-06-15T03:18:11.774Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: election won: term=2 tally=2 +2025-06-15T03:18:11.774Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: entering leader state: leader="Node at 7ca724a0-e06d-0c46-1921-d560eda28c5d [Leader]" +2025-06-15T03:18:11.774Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: added peer, starting replication: peer=server-cc70b568-7754-eb8b-c819-44d1617f4f0a +2025-06-15T03:18:11.774Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: added peer, starting replication: peer=server-c3d72669-3c63-4724-b0af-9ed607222c89 +2025-06-15T03:18:11.774Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: pipelining replication: peer="{Voter server-c3d72669-3c63-4724-b0af-9ed607222c89 c3d72669-3c63-4724-b0af-9ed607222c89}" +2025-06-15T03:18:11.774Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: pipelining replication: peer="{Voter server-cc70b568-7754-eb8b-c819-44d1617f4f0a cc70b568-7754-eb8b-c819-44d1617f4f0a}" +2025-06-15T03:18:11.880Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:11.774885466 +0000 UTC m=+107.486563627 (1 nodes), 61.332057ms from start of poll, 61.594589ms from cluster start. Timeout at 2025-06-15 03:18:11.880284906 +0000 UTC m=+107.591963108, 105.399481ms after stability +2025-06-15T03:18:11.886Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:11.991Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:11.886423225 +0000 UTC m=+107.598101391 (2 nodes), 1.435µs from start of poll, 173.132353ms from cluster start. Timeout at 2025-06-15 03:18:11.991433368 +0000 UTC m=+107.703111567, 105.010176ms after stability +2025-06-15T03:18:11.992Z [ERROR] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: failed to pipeline appendEntries: peer="{Voter server-cc70b568-7754-eb8b-c819-44d1617f4f0a cc70b568-7754-eb8b-c819-44d1617f4f0a}" error="append pipeline closed" +2025-06-15T03:18:11.992Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: aborting pipeline replication: peer="{Voter server-cc70b568-7754-eb8b-c819-44d1617f4f0a cc70b568-7754-eb8b-c819-44d1617f4f0a}" +2025-06-15T03:18:11.998Z [ERROR] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: failed to heartbeat to: peer=cc70b568-7754-eb8b-c819-44d1617f4f0a backoff time=10ms error="failed to connect to peer: cc70b568-7754-eb8b-c819-44d1617f4f0a" +2025-06-15T03:18:12.001Z [ERROR] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: failed to appendEntries to: peer="{Voter server-cc70b568-7754-eb8b-c819-44d1617f4f0a cc70b568-7754-eb8b-c819-44d1617f4f0a}" error="failed to connect to peer: cc70b568-7754-eb8b-c819-44d1617f4f0a" +2025-06-15T03:18:12.015Z [ERROR] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: failed to heartbeat to: peer=cc70b568-7754-eb8b-c819-44d1617f4f0a backoff time=10ms error="failed to connect to peer: cc70b568-7754-eb8b-c819-44d1617f4f0a" +2025-06-15T03:18:12.018Z [ERROR] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: failed to appendEntries to: peer="{Voter server-cc70b568-7754-eb8b-c819-44d1617f4f0a cc70b568-7754-eb8b-c819-44d1617f4f0a}" error="failed to connect to peer: cc70b568-7754-eb8b-c819-44d1617f4f0a" +2025-06-15T03:18:12.032Z [ERROR] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: failed to heartbeat to: peer=cc70b568-7754-eb8b-c819-44d1617f4f0a backoff time=10ms error="failed to connect to peer: cc70b568-7754-eb8b-c819-44d1617f4f0a" +2025-06-15T03:18:12.038Z [ERROR] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: failed to appendEntries to: peer="{Voter server-cc70b568-7754-eb8b-c819-44d1617f4f0a cc70b568-7754-eb8b-c819-44d1617f4f0a}" error="failed to connect to peer: cc70b568-7754-eb8b-c819-44d1617f4f0a" +2025-06-15T03:18:12.039Z [WARN] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: failed to contact: server-id=server-cc70b568-7754-eb8b-c819-44d1617f4f0a time=50.849263ms +2025-06-15T03:18:12.049Z [ERROR] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: failed to heartbeat to: peer=cc70b568-7754-eb8b-c819-44d1617f4f0a backoff time=20ms error="failed to connect to peer: cc70b568-7754-eb8b-c819-44d1617f4f0a" +2025-06-15T03:18:12.069Z [ERROR] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: failed to appendEntries to: peer="{Voter server-cc70b568-7754-eb8b-c819-44d1617f4f0a cc70b568-7754-eb8b-c819-44d1617f4f0a}" error="failed to connect to peer: cc70b568-7754-eb8b-c819-44d1617f4f0a" +2025-06-15T03:18:12.078Z [ERROR] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: failed to heartbeat to: peer=cc70b568-7754-eb8b-c819-44d1617f4f0a backoff time=25ms error="failed to connect to peer: cc70b568-7754-eb8b-c819-44d1617f4f0a" +2025-06-15T03:18:12.085Z [WARN] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: failed to contact: server-id=server-cc70b568-7754-eb8b-c819-44d1617f4f0a time=96.384131ms +2025-06-15T03:18:12.097Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: updating configuration: command=RemoveServer server-id=server-cc70b568-7754-eb8b-c819-44d1617f4f0a server-addr= servers="[{Suffrage:Voter ID:server-7ca724a0-e06d-0c46-1921-d560eda28c5d Address:7ca724a0-e06d-0c46-1921-d560eda28c5d} {Suffrage:Voter ID:server-c3d72669-3c63-4724-b0af-9ed607222c89 Address:c3d72669-3c63-4724-b0af-9ed607222c89}]" +2025-06-15T03:18:12.097Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: removed peer, stopping replication: peer=server-cc70b568-7754-eb8b-c819-44d1617f4f0a last-index=3 +2025-06-15T03:18:12.097Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: updating configuration: command=AddNonvoter server-id=server-cc70b568-7754-eb8b-c819-44d1617f4f0a server-addr=cc70b568-7754-eb8b-c819-44d1617f4f0a servers="[{Suffrage:Voter ID:server-7ca724a0-e06d-0c46-1921-d560eda28c5d Address:7ca724a0-e06d-0c46-1921-d560eda28c5d} {Suffrage:Voter ID:server-c3d72669-3c63-4724-b0af-9ed607222c89 Address:c3d72669-3c63-4724-b0af-9ed607222c89} {Suffrage:Nonvoter ID:server-cc70b568-7754-eb8b-c819-44d1617f4f0a Address:cc70b568-7754-eb8b-c819-44d1617f4f0a}]" +2025-06-15T03:18:12.097Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: added peer, starting replication: peer=server-cc70b568-7754-eb8b-c819-44d1617f4f0a +2025-06-15T03:18:12.097Z [ERROR] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: failed to appendEntries to: peer="{Nonvoter server-cc70b568-7754-eb8b-c819-44d1617f4f0a cc70b568-7754-eb8b-c819-44d1617f4f0a}" error="failed to connect to peer: cc70b568-7754-eb8b-c819-44d1617f4f0a" +2025-06-15T03:18:12.113Z [ERROR] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: peer has newer term, stopping replication: peer="{Nonvoter server-cc70b568-7754-eb8b-c819-44d1617f4f0a cc70b568-7754-eb8b-c819-44d1617f4f0a}" +2025-06-15T03:18:12.113Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: entering follower state: follower="Node at 7ca724a0-e06d-0c46-1921-d560eda28c5d [Follower]" leader-address= leader-id= +2025-06-15T03:18:12.113Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: aborting pipeline replication: peer="{Voter server-c3d72669-3c63-4724-b0af-9ed607222c89 c3d72669-3c63-4724-b0af-9ed607222c89}" +2025-06-15T03:18:12.118Z [WARN] server-cc70b568-7754-eb8b-c819-44d1617f4f0a: heartbeat timeout reached, starting election: last-leader-addr=7ca724a0-e06d-0c46-1921-d560eda28c5d last-leader-id=server-7ca724a0-e06d-0c46-1921-d560eda28c5d +2025-06-15T03:18:12.118Z [INFO] server-cc70b568-7754-eb8b-c819-44d1617f4f0a: entering candidate state: node="Node at cc70b568-7754-eb8b-c819-44d1617f4f0a [Candidate]" term=23 +2025-06-15T03:18:12.118Z [ERROR] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: peer has newer term, stopping replication: peer="{Voter server-cc70b568-7754-eb8b-c819-44d1617f4f0a cc70b568-7754-eb8b-c819-44d1617f4f0a}" +2025-06-15T03:18:12.118Z [WARN] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: rejecting vote request since node is not a voter: from=cc70b568-7754-eb8b-c819-44d1617f4f0a +2025-06-15T03:18:12.118Z [WARN] server-c3d72669-3c63-4724-b0af-9ed607222c89: rejecting vote request since we have a leader: from=cc70b568-7754-eb8b-c819-44d1617f4f0a leader=7ca724a0-e06d-0c46-1921-d560eda28c5d leader-id=server-7ca724a0-e06d-0c46-1921-d560eda28c5d +2025-06-15T03:18:12.195Z [WARN] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:12.195Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: entering candidate state: node="Node at 7ca724a0-e06d-0c46-1921-d560eda28c5d [Candidate]" term=24 +2025-06-15T03:18:12.195Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: election won: term=24 tally=2 +2025-06-15T03:18:12.195Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: entering leader state: leader="Node at 7ca724a0-e06d-0c46-1921-d560eda28c5d [Leader]" +2025-06-15T03:18:12.195Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: added peer, starting replication: peer=server-c3d72669-3c63-4724-b0af-9ed607222c89 +2025-06-15T03:18:12.195Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: added peer, starting replication: peer=server-cc70b568-7754-eb8b-c819-44d1617f4f0a +2025-06-15T03:18:12.195Z [WARN] server-cc70b568-7754-eb8b-c819-44d1617f4f0a: failed to get previous log: previous-index=4 last-index=2 error="log not found" +2025-06-15T03:18:12.195Z [INFO] server-cc70b568-7754-eb8b-c819-44d1617f4f0a: entering follower state: follower="Node at cc70b568-7754-eb8b-c819-44d1617f4f0a [Follower]" leader-address=7ca724a0-e06d-0c46-1921-d560eda28c5d leader-id=server-7ca724a0-e06d-0c46-1921-d560eda28c5d +2025-06-15T03:18:12.195Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: pipelining replication: peer="{Voter server-c3d72669-3c63-4724-b0af-9ed607222c89 c3d72669-3c63-4724-b0af-9ed607222c89}" +2025-06-15T03:18:12.195Z [WARN] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: appendEntries rejected, sending older logs: peer="{Nonvoter server-cc70b568-7754-eb8b-c819-44d1617f4f0a cc70b568-7754-eb8b-c819-44d1617f4f0a}" next=3 +2025-06-15T03:18:12.195Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: pipelining replication: peer="{Nonvoter server-cc70b568-7754-eb8b-c819-44d1617f4f0a cc70b568-7754-eb8b-c819-44d1617f4f0a}" +2025-06-15T03:18:12.202Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:12.308Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:12.202741246 +0000 UTC m=+107.914419411 (1 nodes), 23.036µs from start of poll, 489.450373ms from cluster start. Timeout at 2025-06-15 03:18:12.308191685 +0000 UTC m=+108.019869850, 105.450439ms after stability +2025-06-15T03:18:12.327Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: updating configuration: command=RemoveServer server-id=server-cc70b568-7754-eb8b-c819-44d1617f4f0a server-addr= servers="[{Suffrage:Voter ID:server-7ca724a0-e06d-0c46-1921-d560eda28c5d Address:7ca724a0-e06d-0c46-1921-d560eda28c5d} {Suffrage:Voter ID:server-c3d72669-3c63-4724-b0af-9ed607222c89 Address:c3d72669-3c63-4724-b0af-9ed607222c89}]" +2025-06-15T03:18:12.327Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: removed peer, stopping replication: peer=server-cc70b568-7754-eb8b-c819-44d1617f4f0a last-index=106 +2025-06-15T03:18:12.327Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: aborting pipeline replication: peer="{Nonvoter server-cc70b568-7754-eb8b-c819-44d1617f4f0a cc70b568-7754-eb8b-c819-44d1617f4f0a}" +2025-06-15T03:18:12.327Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: updating configuration: command=AddVoter server-id=server-cc70b568-7754-eb8b-c819-44d1617f4f0a server-addr=cc70b568-7754-eb8b-c819-44d1617f4f0a servers="[{Suffrage:Voter ID:server-7ca724a0-e06d-0c46-1921-d560eda28c5d Address:7ca724a0-e06d-0c46-1921-d560eda28c5d} {Suffrage:Voter ID:server-c3d72669-3c63-4724-b0af-9ed607222c89 Address:c3d72669-3c63-4724-b0af-9ed607222c89} {Suffrage:Voter ID:server-cc70b568-7754-eb8b-c819-44d1617f4f0a Address:cc70b568-7754-eb8b-c819-44d1617f4f0a}]" +2025-06-15T03:18:12.327Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: added peer, starting replication: peer=server-cc70b568-7754-eb8b-c819-44d1617f4f0a +2025-06-15T03:18:12.328Z [WARN] server-cc70b568-7754-eb8b-c819-44d1617f4f0a: failed to get previous log: previous-index=107 last-index=106 error="log not found" +2025-06-15T03:18:12.328Z [WARN] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: appendEntries rejected, sending older logs: peer="{Voter server-cc70b568-7754-eb8b-c819-44d1617f4f0a cc70b568-7754-eb8b-c819-44d1617f4f0a}" next=107 +2025-06-15T03:18:12.328Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: pipelining replication: peer="{Voter server-cc70b568-7754-eb8b-c819-44d1617f4f0a cc70b568-7754-eb8b-c819-44d1617f4f0a}" +2025-06-15T03:18:13.393Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:13.498Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:13.393756754 +0000 UTC m=+109.105434919 (1 nodes), 857ns from start of poll, 1.680465881s from cluster start. Timeout at 2025-06-15 03:18:13.498819505 +0000 UTC m=+109.210497737, 105.062818ms after stability +2025-06-15T03:18:13.498Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: aborting pipeline replication: peer="{Voter server-cc70b568-7754-eb8b-c819-44d1617f4f0a cc70b568-7754-eb8b-c819-44d1617f4f0a}" +2025-06-15T03:18:13.498Z [INFO] server-7ca724a0-e06d-0c46-1921-d560eda28c5d: aborting pipeline replication: peer="{Voter server-c3d72669-3c63-4724-b0af-9ed607222c89 c3d72669-3c63-4724-b0af-9ed607222c89}" +--- PASS: TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin (1.79s) === RUN TestRaft_FollowerRemovalNoElection -2024-05-12T20:51:43.313Z [INFO] server-57a787be-49b0-4985-eb02-b37ec49099cb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-57a787be-49b0-4985-eb02-b37ec49099cb Address:57a787be-49b0-4985-eb02-b37ec49099cb} {Suffrage:Voter ID:server-e2d29791-bed1-a9ae-b2e0-7af5430fe46c Address:e2d29791-bed1-a9ae-b2e0-7af5430fe46c} {Suffrage:Voter ID:server-b059b4f6-923e-2a93-f6b2-8aa4a1e7fabd Address:b059b4f6-923e-2a93-f6b2-8aa4a1e7fabd}]" -2024-05-12T20:51:43.318Z [INFO] server-e2d29791-bed1-a9ae-b2e0-7af5430fe46c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-57a787be-49b0-4985-eb02-b37ec49099cb Address:57a787be-49b0-4985-eb02-b37ec49099cb} {Suffrage:Voter ID:server-e2d29791-bed1-a9ae-b2e0-7af5430fe46c Address:e2d29791-bed1-a9ae-b2e0-7af5430fe46c} {Suffrage:Voter ID:server-b059b4f6-923e-2a93-f6b2-8aa4a1e7fabd Address:b059b4f6-923e-2a93-f6b2-8aa4a1e7fabd}]" -2024-05-12T20:51:43.318Z [INFO] server-57a787be-49b0-4985-eb02-b37ec49099cb: entering follower state: follower="Node at 57a787be-49b0-4985-eb02-b37ec49099cb [Follower]" leader-address= leader-id= -2024-05-12T20:51:43.318Z [INFO] server-b059b4f6-923e-2a93-f6b2-8aa4a1e7fabd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-57a787be-49b0-4985-eb02-b37ec49099cb Address:57a787be-49b0-4985-eb02-b37ec49099cb} {Suffrage:Voter ID:server-e2d29791-bed1-a9ae-b2e0-7af5430fe46c Address:e2d29791-bed1-a9ae-b2e0-7af5430fe46c} {Suffrage:Voter ID:server-b059b4f6-923e-2a93-f6b2-8aa4a1e7fabd Address:b059b4f6-923e-2a93-f6b2-8aa4a1e7fabd}]" -2024-05-12T20:51:43.318Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:43.318Z [INFO] server-e2d29791-bed1-a9ae-b2e0-7af5430fe46c: entering follower state: follower="Node at e2d29791-bed1-a9ae-b2e0-7af5430fe46c [Follower]" leader-address= leader-id= -2024-05-12T20:51:43.318Z [INFO] server-b059b4f6-923e-2a93-f6b2-8aa4a1e7fabd: entering follower state: follower="Node at b059b4f6-923e-2a93-f6b2-8aa4a1e7fabd [Follower]" leader-address= leader-id= -2024-05-12T20:51:43.457Z [WARN] server-e2d29791-bed1-a9ae-b2e0-7af5430fe46c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:43.457Z [INFO] server-e2d29791-bed1-a9ae-b2e0-7af5430fe46c: entering candidate state: node="Node at e2d29791-bed1-a9ae-b2e0-7af5430fe46c [Candidate]" term=2 -2024-05-12T20:51:43.458Z [INFO] server-e2d29791-bed1-a9ae-b2e0-7af5430fe46c: election won: term=2 tally=2 -2024-05-12T20:51:43.458Z [INFO] server-e2d29791-bed1-a9ae-b2e0-7af5430fe46c: entering leader state: leader="Node at e2d29791-bed1-a9ae-b2e0-7af5430fe46c [Leader]" -2024-05-12T20:51:43.458Z [INFO] server-e2d29791-bed1-a9ae-b2e0-7af5430fe46c: added peer, starting replication: peer=server-57a787be-49b0-4985-eb02-b37ec49099cb -2024-05-12T20:51:43.458Z [INFO] server-e2d29791-bed1-a9ae-b2e0-7af5430fe46c: added peer, starting replication: peer=server-b059b4f6-923e-2a93-f6b2-8aa4a1e7fabd -2024-05-12T20:51:43.458Z [INFO] server-e2d29791-bed1-a9ae-b2e0-7af5430fe46c: pipelining replication: peer="{Voter server-b059b4f6-923e-2a93-f6b2-8aa4a1e7fabd b059b4f6-923e-2a93-f6b2-8aa4a1e7fabd}" -2024-05-12T20:51:43.458Z [INFO] server-e2d29791-bed1-a9ae-b2e0-7af5430fe46c: pipelining replication: peer="{Voter server-57a787be-49b0-4985-eb02-b37ec49099cb 57a787be-49b0-4985-eb02-b37ec49099cb}" -2024-05-12T20:51:43.679Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:43.459038865 +0000 UTC m=+114.739739266 (1 nodes), 140.26347ms from start of poll, 146.231647ms from cluster start. Timeout at 2024-05-12 20:51:43.679128542 +0000 UTC m=+114.959828437, 220.089171ms after stability -2024-05-12T20:51:43.679Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:43.885Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:43.679549723 +0000 UTC m=+114.960250014 (1 nodes), 3.758µs from start of poll, 366.742395ms from cluster start. Timeout at 2024-05-12 20:51:43.885328046 +0000 UTC m=+115.166028323, 205.778309ms after stability -2024-05-12T20:51:43.890Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:44.099Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:43.891014857 +0000 UTC m=+115.171715149 (2 nodes), 3.713µs from start of poll, 578.20753ms from cluster start. Timeout at 2024-05-12 20:51:44.099065767 +0000 UTC m=+115.379766401, 208.051252ms after stability -2024-05-12T20:51:44.099Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:44.307Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:44.099351146 +0000 UTC m=+115.380051041 (1 nodes), 4.935µs from start of poll, 786.543422ms from cluster start. Timeout at 2024-05-12 20:51:44.307244133 +0000 UTC m=+115.587944871, 207.89383ms after stability - raft_test.go:2961: [INFO] restarting Node at 57a787be-49b0-4985-eb02-b37ec49099cb [Follower] -2024-05-12T20:51:44.307Z [INFO] server-57a787be-49b0-4985-eb02-b37ec49099cb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-57a787be-49b0-4985-eb02-b37ec49099cb Address:57a787be-49b0-4985-eb02-b37ec49099cb} {Suffrage:Voter ID:server-e2d29791-bed1-a9ae-b2e0-7af5430fe46c Address:e2d29791-bed1-a9ae-b2e0-7af5430fe46c} {Suffrage:Voter ID:server-b059b4f6-923e-2a93-f6b2-8aa4a1e7fabd Address:b059b4f6-923e-2a93-f6b2-8aa4a1e7fabd}]" -2024-05-12T20:51:44.307Z [INFO] server-57a787be-49b0-4985-eb02-b37ec49099cb: entering follower state: follower="Node at 57a787be-49b0-4985-eb02-b37ec49099cb [Follower]" leader-address= leader-id= -2024-05-12T20:51:44.573Z [INFO] server-e2d29791-bed1-a9ae-b2e0-7af5430fe46c: aborting pipeline replication: peer="{Voter server-b059b4f6-923e-2a93-f6b2-8aa4a1e7fabd b059b4f6-923e-2a93-f6b2-8aa4a1e7fabd}" -2024-05-12T20:51:44.980Z [ERROR] server-e2d29791-bed1-a9ae-b2e0-7af5430fe46c: failed to pipeline appendEntries: peer="{Voter server-57a787be-49b0-4985-eb02-b37ec49099cb 57a787be-49b0-4985-eb02-b37ec49099cb}" error="command enqueue timeout" -2024-05-12T20:51:44.980Z [INFO] server-e2d29791-bed1-a9ae-b2e0-7af5430fe46c: aborting pipeline replication: peer="{Voter server-57a787be-49b0-4985-eb02-b37ec49099cb 57a787be-49b0-4985-eb02-b37ec49099cb}" -2024-05-12T20:51:45.091Z [ERROR] server-e2d29791-bed1-a9ae-b2e0-7af5430fe46c: failed to heartbeat to: peer=57a787be-49b0-4985-eb02-b37ec49099cb backoff time=10ms error="command timed out" ---- PASS: TestRaft_FollowerRemovalNoElection (1.79s) +2025-06-15T03:18:13.499Z [INFO] server-e10a7fde-9e87-b7d9-1df5-6434a210d642: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e10a7fde-9e87-b7d9-1df5-6434a210d642 Address:e10a7fde-9e87-b7d9-1df5-6434a210d642} {Suffrage:Voter ID:server-83510e06-8ba7-d2be-b637-44eff5478c80 Address:83510e06-8ba7-d2be-b637-44eff5478c80} {Suffrage:Voter ID:server-ec853504-3fcb-60ee-066d-dbdbb931cfa7 Address:ec853504-3fcb-60ee-066d-dbdbb931cfa7}]" +2025-06-15T03:18:13.500Z [INFO] server-e10a7fde-9e87-b7d9-1df5-6434a210d642: entering follower state: follower="Node at e10a7fde-9e87-b7d9-1df5-6434a210d642 [Follower]" leader-address= leader-id= +2025-06-15T03:18:13.500Z [INFO] server-83510e06-8ba7-d2be-b637-44eff5478c80: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e10a7fde-9e87-b7d9-1df5-6434a210d642 Address:e10a7fde-9e87-b7d9-1df5-6434a210d642} {Suffrage:Voter ID:server-83510e06-8ba7-d2be-b637-44eff5478c80 Address:83510e06-8ba7-d2be-b637-44eff5478c80} {Suffrage:Voter ID:server-ec853504-3fcb-60ee-066d-dbdbb931cfa7 Address:ec853504-3fcb-60ee-066d-dbdbb931cfa7}]" +2025-06-15T03:18:13.500Z [INFO] server-83510e06-8ba7-d2be-b637-44eff5478c80: entering follower state: follower="Node at 83510e06-8ba7-d2be-b637-44eff5478c80 [Follower]" leader-address= leader-id= +2025-06-15T03:18:13.500Z [INFO] server-ec853504-3fcb-60ee-066d-dbdbb931cfa7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e10a7fde-9e87-b7d9-1df5-6434a210d642 Address:e10a7fde-9e87-b7d9-1df5-6434a210d642} {Suffrage:Voter ID:server-83510e06-8ba7-d2be-b637-44eff5478c80 Address:83510e06-8ba7-d2be-b637-44eff5478c80} {Suffrage:Voter ID:server-ec853504-3fcb-60ee-066d-dbdbb931cfa7 Address:ec853504-3fcb-60ee-066d-dbdbb931cfa7}]" +2025-06-15T03:18:13.500Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:13.500Z [INFO] server-ec853504-3fcb-60ee-066d-dbdbb931cfa7: entering follower state: follower="Node at ec853504-3fcb-60ee-066d-dbdbb931cfa7 [Follower]" leader-address= leader-id= +2025-06-15T03:18:13.636Z [WARN] server-e10a7fde-9e87-b7d9-1df5-6434a210d642: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:13.636Z [INFO] server-e10a7fde-9e87-b7d9-1df5-6434a210d642: entering candidate state: node="Node at e10a7fde-9e87-b7d9-1df5-6434a210d642 [Candidate]" term=2 +2025-06-15T03:18:13.636Z [INFO] server-e10a7fde-9e87-b7d9-1df5-6434a210d642: election won: term=2 tally=2 +2025-06-15T03:18:13.636Z [INFO] server-e10a7fde-9e87-b7d9-1df5-6434a210d642: entering leader state: leader="Node at e10a7fde-9e87-b7d9-1df5-6434a210d642 [Leader]" +2025-06-15T03:18:13.636Z [INFO] server-e10a7fde-9e87-b7d9-1df5-6434a210d642: added peer, starting replication: peer=server-83510e06-8ba7-d2be-b637-44eff5478c80 +2025-06-15T03:18:13.636Z [INFO] server-e10a7fde-9e87-b7d9-1df5-6434a210d642: added peer, starting replication: peer=server-ec853504-3fcb-60ee-066d-dbdbb931cfa7 +2025-06-15T03:18:13.636Z [INFO] server-e10a7fde-9e87-b7d9-1df5-6434a210d642: pipelining replication: peer="{Voter server-ec853504-3fcb-60ee-066d-dbdbb931cfa7 ec853504-3fcb-60ee-066d-dbdbb931cfa7}" +2025-06-15T03:18:13.636Z [INFO] server-e10a7fde-9e87-b7d9-1df5-6434a210d642: pipelining replication: peer="{Voter server-83510e06-8ba7-d2be-b637-44eff5478c80 83510e06-8ba7-d2be-b637-44eff5478c80}" +2025-06-15T03:18:13.841Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:13.636554757 +0000 UTC m=+109.348232956 (1 nodes), 136.391754ms from start of poll, 136.679703ms from cluster start. Timeout at 2025-06-15 03:18:13.841715324 +0000 UTC m=+109.553393526, 205.16057ms after stability +2025-06-15T03:18:13.841Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:14.047Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:13.84182338 +0000 UTC m=+109.553501579 (1 nodes), 2.155µs from start of poll, 341.948326ms from cluster start. Timeout at 2025-06-15 03:18:14.047684304 +0000 UTC m=+109.759362534, 205.860955ms after stability +2025-06-15T03:18:14.053Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:14.259Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:14.05390506 +0000 UTC m=+109.765583240 (2 nodes), 2.474µs from start of poll, 554.029987ms from cluster start. Timeout at 2025-06-15 03:18:14.25902383 +0000 UTC m=+109.970701992, 205.118752ms after stability +2025-06-15T03:18:14.259Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:14.464Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:14.259067226 +0000 UTC m=+109.970745393 (1 nodes), 1.03µs from start of poll, 759.19214ms from cluster start. Timeout at 2025-06-15 03:18:14.464140904 +0000 UTC m=+110.175819083, 205.07369ms after stability + raft_test.go:2961: [INFO] restarting Node at 83510e06-8ba7-d2be-b637-44eff5478c80 [Follower] +2025-06-15T03:18:14.464Z [INFO] server-83510e06-8ba7-d2be-b637-44eff5478c80: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e10a7fde-9e87-b7d9-1df5-6434a210d642 Address:e10a7fde-9e87-b7d9-1df5-6434a210d642} {Suffrage:Voter ID:server-83510e06-8ba7-d2be-b637-44eff5478c80 Address:83510e06-8ba7-d2be-b637-44eff5478c80} {Suffrage:Voter ID:server-ec853504-3fcb-60ee-066d-dbdbb931cfa7 Address:ec853504-3fcb-60ee-066d-dbdbb931cfa7}]" +2025-06-15T03:18:14.464Z [INFO] server-83510e06-8ba7-d2be-b637-44eff5478c80: entering follower state: follower="Node at 83510e06-8ba7-d2be-b637-44eff5478c80 [Follower]" leader-address= leader-id= +2025-06-15T03:18:14.715Z [INFO] server-e10a7fde-9e87-b7d9-1df5-6434a210d642: aborting pipeline replication: peer="{Voter server-ec853504-3fcb-60ee-066d-dbdbb931cfa7 ec853504-3fcb-60ee-066d-dbdbb931cfa7}" +2025-06-15T03:18:15.091Z [ERROR] server-e10a7fde-9e87-b7d9-1df5-6434a210d642: failed to pipeline appendEntries: peer="{Voter server-83510e06-8ba7-d2be-b637-44eff5478c80 83510e06-8ba7-d2be-b637-44eff5478c80}" error="command enqueue timeout" +2025-06-15T03:18:15.091Z [INFO] server-e10a7fde-9e87-b7d9-1df5-6434a210d642: aborting pipeline replication: peer="{Voter server-83510e06-8ba7-d2be-b637-44eff5478c80 83510e06-8ba7-d2be-b637-44eff5478c80}" +2025-06-15T03:18:15.223Z [ERROR] server-e10a7fde-9e87-b7d9-1df5-6434a210d642: failed to heartbeat to: peer=83510e06-8ba7-d2be-b637-44eff5478c80 backoff time=10ms error="command timed out" +--- PASS: TestRaft_FollowerRemovalNoElection (1.73s) === RUN TestRaft_VoteWithNoIDNoAddr -2024-05-12T20:51:45.092Z [INFO] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-79da7aaa-3d5a-7516-6537-daf9b9bf121e Address:79da7aaa-3d5a-7516-6537-daf9b9bf121e} {Suffrage:Voter ID:server-088a1985-8369-2b88-c9c0-1a6f8e6e7723 Address:088a1985-8369-2b88-c9c0-1a6f8e6e7723} {Suffrage:Voter ID:server-97c03471-54b0-3396-0b87-9ac0c39685f5 Address:97c03471-54b0-3396-0b87-9ac0c39685f5}]" -2024-05-12T20:51:45.093Z [INFO] server-088a1985-8369-2b88-c9c0-1a6f8e6e7723: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-79da7aaa-3d5a-7516-6537-daf9b9bf121e Address:79da7aaa-3d5a-7516-6537-daf9b9bf121e} {Suffrage:Voter ID:server-088a1985-8369-2b88-c9c0-1a6f8e6e7723 Address:088a1985-8369-2b88-c9c0-1a6f8e6e7723} {Suffrage:Voter ID:server-97c03471-54b0-3396-0b87-9ac0c39685f5 Address:97c03471-54b0-3396-0b87-9ac0c39685f5}]" -2024-05-12T20:51:45.093Z [INFO] server-97c03471-54b0-3396-0b87-9ac0c39685f5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-79da7aaa-3d5a-7516-6537-daf9b9bf121e Address:79da7aaa-3d5a-7516-6537-daf9b9bf121e} {Suffrage:Voter ID:server-088a1985-8369-2b88-c9c0-1a6f8e6e7723 Address:088a1985-8369-2b88-c9c0-1a6f8e6e7723} {Suffrage:Voter ID:server-97c03471-54b0-3396-0b87-9ac0c39685f5 Address:97c03471-54b0-3396-0b87-9ac0c39685f5}]" -2024-05-12T20:51:45.093Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:45.093Z [INFO] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: entering follower state: follower="Node at 79da7aaa-3d5a-7516-6537-daf9b9bf121e [Follower]" leader-address= leader-id= -2024-05-12T20:51:45.093Z [INFO] server-088a1985-8369-2b88-c9c0-1a6f8e6e7723: entering follower state: follower="Node at 088a1985-8369-2b88-c9c0-1a6f8e6e7723 [Follower]" leader-address= leader-id= -2024-05-12T20:51:45.093Z [INFO] server-97c03471-54b0-3396-0b87-9ac0c39685f5: entering follower state: follower="Node at 97c03471-54b0-3396-0b87-9ac0c39685f5 [Follower]" leader-address= leader-id= -2024-05-12T20:51:45.158Z [WARN] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2024-05-12T20:51:45.158Z [INFO] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: entering candidate state: node="Node at 79da7aaa-3d5a-7516-6537-daf9b9bf121e [Candidate]" term=2 -2024-05-12T20:51:45.159Z [INFO] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: election won: term=2 tally=2 -2024-05-12T20:51:45.159Z [INFO] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: entering leader state: leader="Node at 79da7aaa-3d5a-7516-6537-daf9b9bf121e [Leader]" -2024-05-12T20:51:45.159Z [INFO] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: added peer, starting replication: peer=server-088a1985-8369-2b88-c9c0-1a6f8e6e7723 -2024-05-12T20:51:45.159Z [INFO] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: added peer, starting replication: peer=server-97c03471-54b0-3396-0b87-9ac0c39685f5 -2024-05-12T20:51:45.159Z [INFO] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: pipelining replication: peer="{Voter server-088a1985-8369-2b88-c9c0-1a6f8e6e7723 088a1985-8369-2b88-c9c0-1a6f8e6e7723}" -2024-05-12T20:51:45.159Z [INFO] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: pipelining replication: peer="{Voter server-97c03471-54b0-3396-0b87-9ac0c39685f5 97c03471-54b0-3396-0b87-9ac0c39685f5}" -2024-05-12T20:51:45.267Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:45.159553467 +0000 UTC m=+116.440253362 (1 nodes), 66.347231ms from start of poll, 66.769781ms from cluster start. Timeout at 2024-05-12 20:51:45.266838623 +0000 UTC m=+116.547539180, 107.285818ms after stability -2024-05-12T20:51:45.267Z [INFO] cluster: starting stability test: raft-state=Leader -2024-05-12T20:51:45.375Z [INFO] cluster: stable state for Leader reached at 2024-05-12 20:51:45.267181663 +0000 UTC m=+116.547882136 (1 nodes), 3.637µs from start of poll, 174.398555ms from cluster start. Timeout at 2024-05-12 20:51:45.375566228 +0000 UTC m=+116.656266123, 108.383987ms after stability -2024-05-12T20:51:45.382Z [INFO] cluster: starting stability test: raft-state=Follower -2024-05-12T20:51:45.489Z [INFO] cluster: stable state for Follower reached at 2024-05-12 20:51:45.382946637 +0000 UTC m=+116.663646532 (2 nodes), 0s from start of poll, 290.162951ms from cluster start. Timeout at 2024-05-12 20:51:45.488875055 +0000 UTC m=+116.769575341, 105.928809ms after stability -2024-05-12T20:51:45.495Z [ERROR] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: failed to heartbeat to: peer=088a1985-8369-2b88-c9c0-1a6f8e6e7723 backoff time=10ms error="failed to connect to peer: 088a1985-8369-2b88-c9c0-1a6f8e6e7723" -2024-05-12T20:51:45.495Z [ERROR] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: failed to pipeline appendEntries: peer="{Voter server-97c03471-54b0-3396-0b87-9ac0c39685f5 97c03471-54b0-3396-0b87-9ac0c39685f5}" error="append pipeline closed" -2024-05-12T20:51:45.495Z [INFO] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: aborting pipeline replication: peer="{Voter server-97c03471-54b0-3396-0b87-9ac0c39685f5 97c03471-54b0-3396-0b87-9ac0c39685f5}" -2024-05-12T20:51:45.495Z [ERROR] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: failed to pipeline appendEntries: peer="{Voter server-088a1985-8369-2b88-c9c0-1a6f8e6e7723 088a1985-8369-2b88-c9c0-1a6f8e6e7723}" error="append pipeline closed" -2024-05-12T20:51:45.495Z [INFO] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: aborting pipeline replication: peer="{Voter server-088a1985-8369-2b88-c9c0-1a6f8e6e7723 088a1985-8369-2b88-c9c0-1a6f8e6e7723}" -2024-05-12T20:51:45.495Z [ERROR] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: failed to heartbeat to: peer=97c03471-54b0-3396-0b87-9ac0c39685f5 backoff time=10ms error="failed to connect to peer: 97c03471-54b0-3396-0b87-9ac0c39685f5" -2024-05-12T20:51:45.507Z [ERROR] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: failed to appendEntries to: peer="{Voter server-088a1985-8369-2b88-c9c0-1a6f8e6e7723 088a1985-8369-2b88-c9c0-1a6f8e6e7723}" error="failed to connect to peer: 088a1985-8369-2b88-c9c0-1a6f8e6e7723" -2024-05-12T20:51:45.507Z [ERROR] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: failed to appendEntries to: peer="{Voter server-97c03471-54b0-3396-0b87-9ac0c39685f5 97c03471-54b0-3396-0b87-9ac0c39685f5}" error="failed to connect to peer: 97c03471-54b0-3396-0b87-9ac0c39685f5" -2024-05-12T20:51:45.519Z [ERROR] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: failed to heartbeat to: peer=97c03471-54b0-3396-0b87-9ac0c39685f5 backoff time=10ms error="failed to connect to peer: 97c03471-54b0-3396-0b87-9ac0c39685f5" -2024-05-12T20:51:45.519Z [ERROR] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: failed to heartbeat to: peer=088a1985-8369-2b88-c9c0-1a6f8e6e7723 backoff time=10ms error="failed to connect to peer: 088a1985-8369-2b88-c9c0-1a6f8e6e7723" -2024-05-12T20:51:45.530Z [ERROR] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: failed to appendEntries to: peer="{Voter server-088a1985-8369-2b88-c9c0-1a6f8e6e7723 088a1985-8369-2b88-c9c0-1a6f8e6e7723}" error="failed to connect to peer: 088a1985-8369-2b88-c9c0-1a6f8e6e7723" -2024-05-12T20:51:45.531Z [ERROR] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: failed to appendEntries to: peer="{Voter server-97c03471-54b0-3396-0b87-9ac0c39685f5 97c03471-54b0-3396-0b87-9ac0c39685f5}" error="failed to connect to peer: 97c03471-54b0-3396-0b87-9ac0c39685f5" -2024-05-12T20:51:45.534Z [WARN] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: failed to contact: server-id=server-088a1985-8369-2b88-c9c0-1a6f8e6e7723 time=51.045712ms -2024-05-12T20:51:45.534Z [WARN] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: failed to contact: server-id=server-97c03471-54b0-3396-0b87-9ac0c39685f5 time=51.505265ms -2024-05-12T20:51:45.534Z [WARN] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: failed to contact quorum of nodes, stepping down -2024-05-12T20:51:45.534Z [INFO] server-79da7aaa-3d5a-7516-6537-daf9b9bf121e: entering follower state: follower="Node at 79da7aaa-3d5a-7516-6537-daf9b9bf121e [Follower]" leader-address= leader-id= -2024-05-12T20:51:45.555Z [WARN] server-088a1985-8369-2b88-c9c0-1a6f8e6e7723: heartbeat timeout reached, starting election: last-leader-addr=79da7aaa-3d5a-7516-6537-daf9b9bf121e last-leader-id=server-79da7aaa-3d5a-7516-6537-daf9b9bf121e -2024-05-12T20:51:45.555Z [INFO] server-088a1985-8369-2b88-c9c0-1a6f8e6e7723: entering candidate state: node="Node at 088a1985-8369-2b88-c9c0-1a6f8e6e7723 [Candidate]" term=3 -2024-05-12T20:51:45.555Z [ERROR] server-088a1985-8369-2b88-c9c0-1a6f8e6e7723: failed to make requestVote RPC: target="{Voter server-79da7aaa-3d5a-7516-6537-daf9b9bf121e 79da7aaa-3d5a-7516-6537-daf9b9bf121e}" error="failed to connect to peer: 79da7aaa-3d5a-7516-6537-daf9b9bf121e" term=3 -2024-05-12T20:51:45.555Z [WARN] server-97c03471-54b0-3396-0b87-9ac0c39685f5: rejecting vote request since we have a leader: from=088a1985-8369-2b88-c9c0-1a6f8e6e7723 leader=79da7aaa-3d5a-7516-6537-daf9b9bf121e leader-id=server-79da7aaa-3d5a-7516-6537-daf9b9bf121e -2024-05-12T20:51:45.595Z [WARN] server-97c03471-54b0-3396-0b87-9ac0c39685f5: heartbeat timeout reached, starting election: last-leader-addr=79da7aaa-3d5a-7516-6537-daf9b9bf121e last-leader-id=server-79da7aaa-3d5a-7516-6537-daf9b9bf121e -2024-05-12T20:51:45.595Z [INFO] server-97c03471-54b0-3396-0b87-9ac0c39685f5: entering candidate state: node="Node at 97c03471-54b0-3396-0b87-9ac0c39685f5 [Candidate]" term=3 -2024-05-12T20:51:45.595Z [INFO] server-088a1985-8369-2b88-c9c0-1a6f8e6e7723: duplicate requestVote for same term: term=3 -2024-05-12T20:51:45.595Z [ERROR] server-97c03471-54b0-3396-0b87-9ac0c39685f5: failed to make requestVote RPC: target="{Voter server-79da7aaa-3d5a-7516-6537-daf9b9bf121e 79da7aaa-3d5a-7516-6537-daf9b9bf121e}" error="failed to connect to peer: 79da7aaa-3d5a-7516-6537-daf9b9bf121e" term=3 -2024-05-12T20:51:45.598Z [INFO] server-088a1985-8369-2b88-c9c0-1a6f8e6e7723: entering follower state: follower="Node at 088a1985-8369-2b88-c9c0-1a6f8e6e7723 [Follower]" leader-address= leader-id= ---- PASS: TestRaft_VoteWithNoIDNoAddr (0.51s) +2025-06-15T03:18:15.225Z [INFO] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a Address:5cf9d3ab-975b-57f6-fafe-6440a3a8d94a} {Suffrage:Voter ID:server-b337b630-6dc1-1d60-edcb-3be6f5ac68da Address:b337b630-6dc1-1d60-edcb-3be6f5ac68da} {Suffrage:Voter ID:server-21c3c95d-472f-70b7-f342-15ab972f07ec Address:21c3c95d-472f-70b7-f342-15ab972f07ec}]" +2025-06-15T03:18:15.225Z [INFO] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: entering follower state: follower="Node at 5cf9d3ab-975b-57f6-fafe-6440a3a8d94a [Follower]" leader-address= leader-id= +2025-06-15T03:18:15.225Z [INFO] server-b337b630-6dc1-1d60-edcb-3be6f5ac68da: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a Address:5cf9d3ab-975b-57f6-fafe-6440a3a8d94a} {Suffrage:Voter ID:server-b337b630-6dc1-1d60-edcb-3be6f5ac68da Address:b337b630-6dc1-1d60-edcb-3be6f5ac68da} {Suffrage:Voter ID:server-21c3c95d-472f-70b7-f342-15ab972f07ec Address:21c3c95d-472f-70b7-f342-15ab972f07ec}]" +2025-06-15T03:18:15.225Z [INFO] server-b337b630-6dc1-1d60-edcb-3be6f5ac68da: entering follower state: follower="Node at b337b630-6dc1-1d60-edcb-3be6f5ac68da [Follower]" leader-address= leader-id= +2025-06-15T03:18:15.225Z [INFO] server-21c3c95d-472f-70b7-f342-15ab972f07ec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a Address:5cf9d3ab-975b-57f6-fafe-6440a3a8d94a} {Suffrage:Voter ID:server-b337b630-6dc1-1d60-edcb-3be6f5ac68da Address:b337b630-6dc1-1d60-edcb-3be6f5ac68da} {Suffrage:Voter ID:server-21c3c95d-472f-70b7-f342-15ab972f07ec Address:21c3c95d-472f-70b7-f342-15ab972f07ec}]" +2025-06-15T03:18:15.225Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:15.225Z [INFO] server-21c3c95d-472f-70b7-f342-15ab972f07ec: entering follower state: follower="Node at 21c3c95d-472f-70b7-f342-15ab972f07ec [Follower]" leader-address= leader-id= +2025-06-15T03:18:15.293Z [WARN] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:15.293Z [INFO] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: entering candidate state: node="Node at 5cf9d3ab-975b-57f6-fafe-6440a3a8d94a [Candidate]" term=2 +2025-06-15T03:18:15.293Z [INFO] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: election won: term=2 tally=2 +2025-06-15T03:18:15.293Z [INFO] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: entering leader state: leader="Node at 5cf9d3ab-975b-57f6-fafe-6440a3a8d94a [Leader]" +2025-06-15T03:18:15.293Z [INFO] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: added peer, starting replication: peer=server-b337b630-6dc1-1d60-edcb-3be6f5ac68da +2025-06-15T03:18:15.293Z [INFO] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: added peer, starting replication: peer=server-21c3c95d-472f-70b7-f342-15ab972f07ec +2025-06-15T03:18:15.293Z [INFO] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: pipelining replication: peer="{Voter server-21c3c95d-472f-70b7-f342-15ab972f07ec 21c3c95d-472f-70b7-f342-15ab972f07ec}" +2025-06-15T03:18:15.293Z [INFO] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: pipelining replication: peer="{Voter server-b337b630-6dc1-1d60-edcb-3be6f5ac68da b337b630-6dc1-1d60-edcb-3be6f5ac68da}" +2025-06-15T03:18:15.398Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:15.293619868 +0000 UTC m=+111.005298117 (1 nodes), 67.83729ms from start of poll, 68.143041ms from cluster start. Timeout at 2025-06-15 03:18:15.398809513 +0000 UTC m=+111.110487746, 105.189629ms after stability +2025-06-15T03:18:15.398Z [INFO] cluster: starting stability test: raft-state=Leader +2025-06-15T03:18:15.504Z [INFO] cluster: stable state for Leader reached at 2025-06-15 03:18:15.398901776 +0000 UTC m=+111.110580009 (1 nodes), 2.042µs from start of poll, 173.424933ms from cluster start. Timeout at 2025-06-15 03:18:15.504238087 +0000 UTC m=+111.215916286, 105.336277ms after stability +2025-06-15T03:18:15.509Z [INFO] cluster: starting stability test: raft-state=Follower +2025-06-15T03:18:15.615Z [INFO] cluster: stable state for Follower reached at 2025-06-15 03:18:15.509574356 +0000 UTC m=+111.221252555 (2 nodes), 3.144µs from start of poll, 284.097479ms from cluster start. Timeout at 2025-06-15 03:18:15.615573568 +0000 UTC m=+111.327251734, 105.999179ms after stability +2025-06-15T03:18:15.617Z [ERROR] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to heartbeat to: peer=b337b630-6dc1-1d60-edcb-3be6f5ac68da backoff time=10ms error="failed to connect to peer: b337b630-6dc1-1d60-edcb-3be6f5ac68da" +2025-06-15T03:18:15.617Z [ERROR] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to heartbeat to: peer=21c3c95d-472f-70b7-f342-15ab972f07ec backoff time=10ms error="failed to connect to peer: 21c3c95d-472f-70b7-f342-15ab972f07ec" +2025-06-15T03:18:15.623Z [ERROR] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to pipeline appendEntries: peer="{Voter server-21c3c95d-472f-70b7-f342-15ab972f07ec 21c3c95d-472f-70b7-f342-15ab972f07ec}" error="append pipeline closed" +2025-06-15T03:18:15.623Z [INFO] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: aborting pipeline replication: peer="{Voter server-21c3c95d-472f-70b7-f342-15ab972f07ec 21c3c95d-472f-70b7-f342-15ab972f07ec}" +2025-06-15T03:18:15.624Z [ERROR] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to pipeline appendEntries: peer="{Voter server-b337b630-6dc1-1d60-edcb-3be6f5ac68da b337b630-6dc1-1d60-edcb-3be6f5ac68da}" error="append pipeline closed" +2025-06-15T03:18:15.624Z [INFO] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: aborting pipeline replication: peer="{Voter server-b337b630-6dc1-1d60-edcb-3be6f5ac68da b337b630-6dc1-1d60-edcb-3be6f5ac68da}" +2025-06-15T03:18:15.632Z [ERROR] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to appendEntries to: peer="{Voter server-b337b630-6dc1-1d60-edcb-3be6f5ac68da b337b630-6dc1-1d60-edcb-3be6f5ac68da}" error="failed to connect to peer: b337b630-6dc1-1d60-edcb-3be6f5ac68da" +2025-06-15T03:18:15.632Z [ERROR] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to appendEntries to: peer="{Voter server-21c3c95d-472f-70b7-f342-15ab972f07ec 21c3c95d-472f-70b7-f342-15ab972f07ec}" error="failed to connect to peer: 21c3c95d-472f-70b7-f342-15ab972f07ec" +2025-06-15T03:18:15.635Z [ERROR] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to heartbeat to: peer=b337b630-6dc1-1d60-edcb-3be6f5ac68da backoff time=10ms error="failed to connect to peer: b337b630-6dc1-1d60-edcb-3be6f5ac68da" +2025-06-15T03:18:15.638Z [ERROR] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to heartbeat to: peer=21c3c95d-472f-70b7-f342-15ab972f07ec backoff time=10ms error="failed to connect to peer: 21c3c95d-472f-70b7-f342-15ab972f07ec" +2025-06-15T03:18:15.648Z [ERROR] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to appendEntries to: peer="{Voter server-21c3c95d-472f-70b7-f342-15ab972f07ec 21c3c95d-472f-70b7-f342-15ab972f07ec}" error="failed to connect to peer: 21c3c95d-472f-70b7-f342-15ab972f07ec" +2025-06-15T03:18:15.653Z [ERROR] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to appendEntries to: peer="{Voter server-b337b630-6dc1-1d60-edcb-3be6f5ac68da b337b630-6dc1-1d60-edcb-3be6f5ac68da}" error="failed to connect to peer: b337b630-6dc1-1d60-edcb-3be6f5ac68da" +2025-06-15T03:18:15.653Z [ERROR] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to heartbeat to: peer=b337b630-6dc1-1d60-edcb-3be6f5ac68da backoff time=10ms error="failed to connect to peer: b337b630-6dc1-1d60-edcb-3be6f5ac68da" +2025-06-15T03:18:15.657Z [ERROR] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to heartbeat to: peer=21c3c95d-472f-70b7-f342-15ab972f07ec backoff time=10ms error="failed to connect to peer: 21c3c95d-472f-70b7-f342-15ab972f07ec" +2025-06-15T03:18:15.664Z [WARN] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to contact: server-id=server-21c3c95d-472f-70b7-f342-15ab972f07ec time=50.444715ms +2025-06-15T03:18:15.668Z [ERROR] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to appendEntries to: peer="{Voter server-21c3c95d-472f-70b7-f342-15ab972f07ec 21c3c95d-472f-70b7-f342-15ab972f07ec}" error="failed to connect to peer: 21c3c95d-472f-70b7-f342-15ab972f07ec" +2025-06-15T03:18:15.672Z [ERROR] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to appendEntries to: peer="{Voter server-b337b630-6dc1-1d60-edcb-3be6f5ac68da b337b630-6dc1-1d60-edcb-3be6f5ac68da}" error="failed to connect to peer: b337b630-6dc1-1d60-edcb-3be6f5ac68da" +2025-06-15T03:18:15.672Z [ERROR] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to heartbeat to: peer=b337b630-6dc1-1d60-edcb-3be6f5ac68da backoff time=20ms error="failed to connect to peer: b337b630-6dc1-1d60-edcb-3be6f5ac68da" +2025-06-15T03:18:15.675Z [ERROR] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to heartbeat to: peer=21c3c95d-472f-70b7-f342-15ab972f07ec backoff time=20ms error="failed to connect to peer: 21c3c95d-472f-70b7-f342-15ab972f07ec" +2025-06-15T03:18:15.675Z [WARN] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to contact: server-id=server-b337b630-6dc1-1d60-edcb-3be6f5ac68da time=60.209966ms +2025-06-15T03:18:15.675Z [WARN] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to contact: server-id=server-21c3c95d-472f-70b7-f342-15ab972f07ec time=61.344214ms +2025-06-15T03:18:15.675Z [WARN] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: failed to contact quorum of nodes, stepping down +2025-06-15T03:18:15.675Z [INFO] server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a: entering follower state: follower="Node at 5cf9d3ab-975b-57f6-fafe-6440a3a8d94a [Follower]" leader-address= leader-id= +2025-06-15T03:18:15.680Z [WARN] server-b337b630-6dc1-1d60-edcb-3be6f5ac68da: heartbeat timeout reached, starting election: last-leader-addr=5cf9d3ab-975b-57f6-fafe-6440a3a8d94a last-leader-id=server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a +2025-06-15T03:18:15.680Z [INFO] server-b337b630-6dc1-1d60-edcb-3be6f5ac68da: entering candidate state: node="Node at b337b630-6dc1-1d60-edcb-3be6f5ac68da [Candidate]" term=3 +2025-06-15T03:18:15.680Z [ERROR] server-b337b630-6dc1-1d60-edcb-3be6f5ac68da: failed to make requestVote RPC: target="{Voter server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a 5cf9d3ab-975b-57f6-fafe-6440a3a8d94a}" error="failed to connect to peer: 5cf9d3ab-975b-57f6-fafe-6440a3a8d94a" term=3 +2025-06-15T03:18:15.680Z [WARN] server-21c3c95d-472f-70b7-f342-15ab972f07ec: rejecting vote request since we have a leader: from=b337b630-6dc1-1d60-edcb-3be6f5ac68da leader=5cf9d3ab-975b-57f6-fafe-6440a3a8d94a leader-id=server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a +2025-06-15T03:18:15.684Z [WARN] server-21c3c95d-472f-70b7-f342-15ab972f07ec: heartbeat timeout reached, starting election: last-leader-addr=5cf9d3ab-975b-57f6-fafe-6440a3a8d94a last-leader-id=server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a +2025-06-15T03:18:15.684Z [INFO] server-21c3c95d-472f-70b7-f342-15ab972f07ec: entering candidate state: node="Node at 21c3c95d-472f-70b7-f342-15ab972f07ec [Candidate]" term=3 +2025-06-15T03:18:15.684Z [INFO] server-b337b630-6dc1-1d60-edcb-3be6f5ac68da: duplicate requestVote for same term: term=3 +2025-06-15T03:18:15.684Z [ERROR] server-21c3c95d-472f-70b7-f342-15ab972f07ec: failed to make requestVote RPC: target="{Voter server-5cf9d3ab-975b-57f6-fafe-6440a3a8d94a 5cf9d3ab-975b-57f6-fafe-6440a3a8d94a}" error="failed to connect to peer: 5cf9d3ab-975b-57f6-fafe-6440a3a8d94a" term=3 +2025-06-15T03:18:15.720Z [INFO] server-b337b630-6dc1-1d60-edcb-3be6f5ac68da: entering follower state: follower="Node at b337b630-6dc1-1d60-edcb-3be6f5ac68da [Follower]" leader-address= leader-id= +--- PASS: TestRaft_VoteWithNoIDNoAddr (0.50s) === RUN TestRaft_runFollower_State_Transition === RUN TestRaft_runFollower_State_Transition/NonVoter -2024-05-12T20:51:45.599Z [INFO] 127.0.0.1:35553 :: starting node: addr=127.0.0.1:35553 -2024-05-12T20:51:45.599Z [INFO] 127.0.0.1:35553 :: initial configuration: index=0 servers=[] -2024-05-12T20:51:45.599Z [INFO] 127.0.0.1:35553 :: entering follower state: follower="Node at 127.0.0.1:35553 [Follower]" leader-address= leader-id= -2024-05-12T20:51:45.658Z [WARN] 127.0.0.1:35553 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-06-15T03:18:15.721Z [INFO] 127.0.0.1:39937 :: starting node: addr=127.0.0.1:39937 +2025-06-15T03:18:15.721Z [INFO] 127.0.0.1:39937 :: initial configuration: index=0 servers=[] +2025-06-15T03:18:15.721Z [INFO] 127.0.0.1:39937 :: entering follower state: follower="Node at 127.0.0.1:39937 [Follower]" leader-address= leader-id= +2025-06-15T03:18:15.773Z [WARN] 127.0.0.1:39937 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election === RUN TestRaft_runFollower_State_Transition/Voter -2024-05-12T20:51:45.751Z [INFO] 127.0.0.1:43251 :: starting node: addr=127.0.0.1:43251 -2024-05-12T20:51:45.751Z [INFO] 127.0.0.1:43251 :: initial configuration: index=0 servers=[] -2024-05-12T20:51:45.751Z [INFO] 127.0.0.1:43251 :: entering follower state: follower="Node at 127.0.0.1:43251 [Follower]" leader-address= leader-id= -2024-05-12T20:51:45.811Z [WARN] 127.0.0.1:43251 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-06-15T03:18:15.872Z [INFO] 127.0.0.1:37017 :: starting node: addr=127.0.0.1:37017 +2025-06-15T03:18:15.872Z [INFO] 127.0.0.1:37017 :: initial configuration: index=0 servers=[] +2025-06-15T03:18:15.872Z [INFO] 127.0.0.1:37017 :: entering follower state: follower="Node at 127.0.0.1:37017 [Follower]" leader-address= leader-id= +2025-06-15T03:18:15.972Z [WARN] 127.0.0.1:37017 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= === RUN TestRaft_runFollower_State_Transition/Not_in_Config -2024-05-12T20:51:45.903Z [INFO] 127.0.0.1:33375 :: starting node: addr=127.0.0.1:33375 -2024-05-12T20:51:45.903Z [INFO] 127.0.0.1:33375 :: initial configuration: index=0 servers=[] -2024-05-12T20:51:45.903Z [INFO] 127.0.0.1:33375 :: entering follower state: follower="Node at 127.0.0.1:33375 [Follower]" leader-address= leader-id= -2024-05-12T20:51:45.975Z [WARN] 127.0.0.1:33375 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election ---- PASS: TestRaft_runFollower_State_Transition (0.46s) +2025-06-15T03:18:16.024Z [INFO] 127.0.0.1:42597 :: starting node: addr=127.0.0.1:42597 +2025-06-15T03:18:16.024Z [INFO] 127.0.0.1:42597 :: initial configuration: index=0 servers=[] +2025-06-15T03:18:16.024Z [INFO] 127.0.0.1:42597 :: entering follower state: follower="Node at 127.0.0.1:42597 [Follower]" leader-address= leader-id= +2025-06-15T03:18:16.093Z [WARN] 127.0.0.1:42597 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +--- PASS: TestRaft_runFollower_State_Transition (0.45s) --- PASS: TestRaft_runFollower_State_Transition/NonVoter (0.15s) --- PASS: TestRaft_runFollower_State_Transition/Voter (0.15s) --- PASS: TestRaft_runFollower_State_Transition/Not_in_Config (0.15s) === RUN TestRaft_runFollower_ReloadTimeoutConfigs -2024-05-12T20:51:46.055Z [INFO] 127.0.0.1:36857 :: starting node: addr=127.0.0.1:36857 -2024-05-12T20:51:46.055Z [INFO] 127.0.0.1:36857 :: initial configuration: index=0 servers=[] -2024-05-12T20:51:46.055Z [INFO] 127.0.0.1:36857 :: entering follower state: follower="Node at 127.0.0.1:36857 [Follower]" leader-address= leader-id= -2024-05-12T20:51:46.055Z [WARN] 127.0.0.1:36857 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= ---- PASS: TestRaft_runFollower_ReloadTimeoutConfigs (0.16s) +2025-06-15T03:18:16.175Z [INFO] 127.0.0.1:34857 :: starting node: addr=127.0.0.1:34857 +2025-06-15T03:18:16.175Z [INFO] 127.0.0.1:34857 :: initial configuration: index=0 servers=[] +2025-06-15T03:18:16.175Z [INFO] 127.0.0.1:34857 :: entering follower state: follower="Node at 127.0.0.1:34857 [Follower]" leader-address= leader-id= +2025-06-15T03:18:16.175Z [WARN] 127.0.0.1:34857 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +--- PASS: TestRaft_runFollower_ReloadTimeoutConfigs (0.15s) === RUN TestSaturationMetric === RUN TestSaturationMetric/without_smoothing --- PASS: TestSaturationMetric (0.00s) @@ -4222,7 +4199,7 @@ === RUN TestTransport_EncodeDecode --- PASS: TestTransport_EncodeDecode (0.00s) === RUN TestRandomTimeout ---- PASS: TestRandomTimeout (0.01s) +--- PASS: TestRandomTimeout (0.00s) === RUN TestNewSeed --- PASS: TestNewSeed (0.00s) === RUN TestRandomTimeout_NoTime @@ -4232,13 +4209,13 @@ === RUN TestMax --- PASS: TestMax (0.00s) === RUN TestGenerateUUID ---- PASS: TestGenerateUUID (0.06s) +--- PASS: TestGenerateUUID (0.00s) === RUN TestBackoff --- PASS: TestBackoff (0.00s) === RUN TestOverrideNotifyBool --- PASS: TestOverrideNotifyBool (0.00s) PASS -ok github.com/hashicorp/raft 117.592s +ok github.com/hashicorp/raft 112.049s make[1]: Leaving directory '/build/reproducible-path/golang-github-hashicorp-raft-1.5.0' create-stamp debian/debhelper-build-stamp dh_testroot -O--buildsystem=golang @@ -4266,12 +4243,14 @@ dpkg-buildpackage: info: binary-only upload (no source included) dpkg-genchanges: info: including full source code in upload I: copying local configuration +I: user script /srv/workspace/pbuilder/2083635/tmp/hooks/B01_cleanup starting +I: user script /srv/workspace/pbuilder/2083635/tmp/hooks/B01_cleanup finished I: unmounting dev/ptmx filesystem I: unmounting dev/pts filesystem I: unmounting dev/shm filesystem I: unmounting proc filesystem I: unmounting sys filesystem I: cleaning the build env -I: removing directory /srv/workspace/pbuilder/1949903 and its subdirectories -I: Current time: Sun May 12 08:52:04 -12 2024 -I: pbuilder-time-stamp: 1715547124 +I: removing directory /srv/workspace/pbuilder/2083635 and its subdirectories +I: Current time: Sun Jun 15 17:18:21 +14 2025 +I: pbuilder-time-stamp: 1749957501